From b83144fdcce5993f91ff710b1fbeff8ab65e26a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 13 Aug 2025 11:17:55 +0200 Subject: [PATCH 01/55] Update version number for dev branch --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dbdd465..f363f5b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ eu.sse-labs marin - 1.0.0 + 1.1.0-SNAPSHOT jar From 9af60cae36fe99c1aa10eeaf796b903770b44d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Tue, 26 Aug 2025 12:42:23 +0200 Subject: [PATCH 02/55] Fix broken examples in README file --- README.md | 83 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 38adea9..83c03d0 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,13 @@ public class AnalysisRunner { public static void main(String[] args){ MavenCentralAnalysis myAnalysis = new MyAnalysisImplementation(); - myAnalysis.runAnalysis(args); + + try { + myAnalysis.runAnalysis(args); + } catch (Exception e) { + System.err.println("Error while running Analysis: " + e.getMessage()); + } + } } @@ -68,84 +74,87 @@ You can run each example on the first 1000 Maven artifacts by invoking `java -ja ### Counting all classFiles from jar artifacts: ``` java -public class ExampleImplementation extends MavenCentralAnalysis { +public class ClassFileCountImplementation extends MavenCentralAnalysis { + private long numberOfClassfiles; - - public ExampleImplementation() { - super(); - numberOfClassfiles = 0; + + public ClassFileCountImplementation() { + super(); + this.resolveJar = true; + this.numberOfClassfiles = 0; } - + @Override public void analyzeArtifact(Artifact toAnalyze) { if(toAnalyze.getJarInformation() != null) { - numberOfClassfiles += toAnalyze.getJarInformation().getNumClassFiles(); + numberOfClassfiles += toAnalyze.getJarInformation().getNumClassFiles(); } } public long getNumberOfClassfiles() { return numberOfClassfiles; } -} +} ``` ### Find all Unique Licenses from pom artifacts ``` java -public class ExampleImplementation extends MavenCentralAnalysis { - private Set uniqueLicenses; +public class LicenseImplementation extends MavenCentralAnalysis { + private final Set uniqueLicenses; - public ExampleImplementation() { - super(); - uniqueLicenses = new HashSet<>(); - } + public LicenseImplementation() { + super(); + this.resolvePom = true; + this.uniqueLicenses = new HashSet<>(); + } @Override public void analyzeArtifact(Artifact toAnalyze) { if(toAnalyze.getPomInformation() != null) { - PomInformation info = toAnalyze.getPomInformation(); - if(!info.getRawPomFeatures().getLicenses().isEmpty()) { - for(License license : info.getRawPomFeatures().getLicenses()) { - if(!uniqueLicenses.contains(license)) { - uniqueLicenses.add(license); - } - } - } + PomInformation info = toAnalyze.getPomInformation(); + if(!info.getRawPomFeatures().getLicenses().isEmpty()) { + for(License license : info.getRawPomFeatures().getLicenses()) { + uniqueLicenses.add(license); + } + } } } public Set getUniqueLicenses() { return uniqueLicenses; } -} +} ``` ### Collect all artifacts that have javadocs ``` java -public class ExampleImplementation extends MavenCentralAnalysis { - private Set hasJavadocs; +public class JavaDocImplementation extends MavenCentralAnalysis { - public ExampleImplementation() { - super(); - hasJavadocs = new HashSet<>(); - } + private final Set hasJavadocs; + + public JavaDocImplementation() { + super(); + this.resolveIndex = true; + this.hasJavadocs = new HashSet<>(); + } @Override public void analyzeArtifact(Artifact toAnalyze) { if(toAnalyze.getIndexInformation() != null) { - List packages = toAnalyze.getIndexInformation().getPackages(); - for(Package current : packages) { - if(current.getJavadocExists() > 0) { - hasJavadocs.add(toAnalyze); - break; + List packages = toAnalyze.getIndexInformation().getPackages(); + for(Package current : packages) { + if(current.getJavadocExists() > 0) { + hasJavadocs.add(toAnalyze); + break; + } } - } } } public Set getHasJavadocs() { return hasJavadocs; } -} +} ``` ## IndexWalker From 4482950c36b4b635738962d2cf1ce645915fad8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 27 Aug 2025 10:35:05 +0200 Subject: [PATCH 03/55] Make it so that all clients must specify their data requirements --- .../org/tudo/sse/MavenCentralAnalysis.java | 37 ++++++++++---- .../tudo/sse/MavenCentralAnalysisTest.java | 49 ++++++------------- .../testutils/DummyEvaluationAnalysis.java | 12 +---- .../utils/MavenCentralAnalysisFactory.java | 29 +++++++++++ 4 files changed, 73 insertions(+), 54 deletions(-) create mode 100644 src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java diff --git a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/MavenCentralAnalysis.java index 8622c38..bcf1caf 100644 --- a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/MavenCentralAnalysis.java @@ -36,35 +36,52 @@ public abstract class MavenCentralAnalysis { /** * Defines whether this analysis requires artifacts to have index information annotated. */ - protected boolean resolveIndex; + protected final boolean resolveIndex; /** * Defines whether this analysis requires artifacts to have pom information annotated. */ - protected boolean resolvePom; + protected final boolean resolvePom; /** * Defines whether this analysis requires artifacts to have resolved transitive pom information. */ - protected boolean processTransitives; + protected final boolean processTransitives; /** * Defines whether this analysis requires artifacts to have jar information annotated. */ - protected boolean resolveJar; + protected final boolean resolveJar; private static final Logger log = LogManager.getLogger(MavenCentralAnalysis.class); /** - * Creates a new Maven Central Analysis with default configuration options. + * Creates a new Maven Central Analysis with the given configuration options. + * + * @param requiresIndex Whether this analysis requires index information + * @param requiresPom Whether this analysis requires POM information + * @param requiresTransitives Whether this analysis requires transitive POM information. If true, "requiresPOM" will + * be set to true no matter its given parameter value. + * @param requiresJar Whether this analysis requires JAR information */ - public MavenCentralAnalysis() { + protected MavenCentralAnalysis(boolean requiresIndex, + boolean requiresPom, + boolean requiresTransitives, + boolean requiresJar) { + + if(!requiresIndex && !requiresPom && !requiresTransitives && !requiresJar){ + log.warn("Potential misconfiguration - no data sources (index, POM, JAR) are required by this analysis"); + } else if(requiresTransitives && !requiresPom){ + log.warn("Potential misconfiguration - analysis requires transitive information but no POM information. " + + "POM information will also be collected to provide transitive information."); + } + setupInfo = new CliInformation(); - resolveIndex = false; - resolvePom = false; - processTransitives = false; - resolveJar = false; + resolveIndex = requiresIndex; + resolvePom = requiresPom || requiresTransitives; // Cannot have transitive information without POM information + processTransitives = requiresTransitives; + resolveJar = requiresJar; } diff --git a/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java b/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java index efd2613..1fe6ef4 100644 --- a/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java +++ b/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java @@ -10,6 +10,7 @@ import org.tudo.sse.model.pom.PomInformation; import org.tudo.sse.testutils.DummyEvaluationAnalysis; import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MavenCentralAnalysisFactory; import scala.Tuple2; import java.io.*; @@ -22,7 +23,7 @@ import static org.junit.jupiter.api.Assertions.*; class MavenCentralAnalysisTest { - DummyEvaluationAnalysis tester = new DummyEvaluationAnalysis(); + final MavenCentralAnalysis analysisUnderTest = MavenCentralAnalysisFactory.buildEmptyAnalysisWithIndexRequirement(); final String base = "https://repo1.maven.org/maven2/"; final Map json; final Gson gson = new Gson(); @@ -58,12 +59,7 @@ void parseCmdLinePositive() { int i = 0; for(String[] input : cliInputs) { - MavenCentralAnalysis tester = new MavenCentralAnalysis() { - @Override - public void analyzeArtifact(Artifact current) { - - } - }; + MavenCentralAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); tester.parseCmdLine(input); CliInformation result = tester.getSetupInfo(); List currentExp = expected.get(i); @@ -113,7 +109,7 @@ void parseCmdLineNegative() { cliInputs.add(args); for(String[] input : cliInputs) { - assertThrows(RuntimeException.class, () -> tester.parseCmdLine(input)); + assertThrows(RuntimeException.class, () -> analysisUnderTest.parseCmdLine(input)); } } @@ -133,7 +129,7 @@ void walkPaginated() { try { IndexIterator iterator = new IndexIterator(new URI(base), start1); - List collected1 = new ArrayList<>(tester.walkPaginated(take, iterator)); + List collected1 = new ArrayList<>(analysisUnderTest.walkPaginated(take, iterator)); Artifact lastOne = collected1.get(collected1.size() - 1); int i = 2; @@ -144,7 +140,7 @@ void walkPaginated() { iterator = new IndexIterator(new URI(base), start2); - List collected2 = new ArrayList<>(tester.walkPaginated(1, iterator)); + List collected2 = new ArrayList<>(analysisUnderTest.walkPaginated(1, iterator)); assertEquals(lastOne.getIndexInformation().getIndex(), collected2.get(0).getIndexInformation().getIndex()); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); @@ -154,6 +150,7 @@ void walkPaginated() { @Test void indexProcessor() { + List cliInputs = new ArrayList<>(); String[] args = {"-st", "200:70"}; cliInputs.add(args); @@ -169,12 +166,9 @@ void indexProcessor() { int i = 0; for(String[] arg : cliInputs) { try { - if(i == 2) { - tester.setIndex(true); - } - tester.parseCmdLine(arg); - CliInformation current = tester.getSetupInfo(); - tester.indexProcessor(); + analysisUnderTest.parseCmdLine(arg); + CliInformation current = analysisUnderTest.getSetupInfo(); + analysisUnderTest.indexProcessor(); long ending = getEndingIndex(current.getName()); assertEquals(expectedEndings[i], ending); @@ -208,10 +202,7 @@ void readIdentsIn() { int i = 0; for(String[] arg : cliInputs) { List curExpected = expected.get(i); - MavenCentralAnalysis tester = new MavenCentralAnalysis() { - @Override - public void analyzeArtifact(Artifact current) {} - }; + MavenCentralAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); tester.parseCmdLine(arg); CliInformation current = tester.getSetupInfo(); List idents = tester.readIdentsIn(); @@ -254,13 +245,7 @@ void checkMultiThreading() { for(int i = 0; i < singleArgs.size(); i++) { try { - MavenCentralAnalysis tester = new MavenCentralAnalysis() { - @Override - public void analyzeArtifact(Artifact current) { - - } - }; - tester.resolvePom = true; + MavenCentralAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithPomRequirement(); Map singleResult = tester.runAnalysis(singleArgs.get(i)); Map multiResult = tester.runAnalysis(multiArgs.get(i)); assertEquals(singleResult.size(), multiResult.size()); @@ -275,7 +260,7 @@ public void analyzeArtifact(Artifact current) { } @Test void checkUseCases() { - MavenCentralAnalysis jarUseCase = new MavenCentralAnalysis() { + MavenCentralAnalysis jarUseCase = new MavenCentralAnalysis(false, false, false, true) { public long numberOfClassfiles = 0; @Override public void analyzeArtifact(Artifact current) { @@ -285,11 +270,9 @@ public void analyzeArtifact(Artifact current) { } }; - - jarUseCase.resolveJar = true; assertDoesNotThrow( () -> jarUseCase.runAnalysis(new String[]{"-st", "0:300"})); - MavenCentralAnalysis pomUseCase = new MavenCentralAnalysis() { + MavenCentralAnalysis pomUseCase = new MavenCentralAnalysis(false, true, false, false) { public final Set uniqueLicenses = new HashSet<>(); @Override public void analyzeArtifact(Artifact toAnalyze) { @@ -306,10 +289,9 @@ public void analyzeArtifact(Artifact toAnalyze) { } }; - pomUseCase.resolvePom = true; assertDoesNotThrow( () -> pomUseCase.runAnalysis(new String[]{"-st", "0:300"})); - MavenCentralAnalysis indexUseCase = new MavenCentralAnalysis() { + MavenCentralAnalysis indexUseCase = new MavenCentralAnalysis(true, false, false, false) { public final Set hasJavadocs = new HashSet<>(); @Override public void analyzeArtifact(Artifact toAnalyze) { @@ -325,7 +307,6 @@ public void analyzeArtifact(Artifact toAnalyze) { } }; - indexUseCase.resolveIndex = true; assertDoesNotThrow( () -> indexUseCase.runAnalysis(new String[]{"-st", "0:300"})); } diff --git a/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java b/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java index 97070a1..b31252e 100644 --- a/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java +++ b/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java @@ -10,15 +10,11 @@ public class DummyEvaluationAnalysis extends MavenCentralAnalysis { private final AtomicInteger artifactCnt = new AtomicInteger(0); public DummyEvaluationAnalysis() { - super(); + this(false, false, false, false); } public DummyEvaluationAnalysis(boolean resolveIndex, boolean resolvePom, boolean processTransitives, boolean resolveJar) { - super(); - this.resolveIndex = resolveIndex; - this.resolvePom = resolvePom; - this.processTransitives = processTransitives; - this.resolveJar = resolveJar; + super(resolveIndex, resolvePom, processTransitives, resolveJar); } @Override @@ -27,8 +23,4 @@ public void analyzeArtifact(Artifact toAnalyze) { System.out.println("Client analysis implementation got " + artifactCnt.get() + " artifacts so far"); } } - - public void setIndex(boolean resolveIndex) { - this.resolveIndex = resolveIndex; - } } diff --git a/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java b/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java new file mode 100644 index 0000000..34b76ef --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java @@ -0,0 +1,29 @@ +package org.tudo.sse.utils; + +import org.tudo.sse.MavenCentralAnalysis; +import org.tudo.sse.model.Artifact; + +public class MavenCentralAnalysisFactory { + + public static MavenCentralAnalysis buildEmptyAnalysisWithNoRequirements() { + return buildAnalysisWithRequirements(false, false, false, false); + } + + public static MavenCentralAnalysis buildEmptyAnalysisWithPomRequirement() { + return buildAnalysisWithRequirements(false, true, false, false); + } + + public static MavenCentralAnalysis buildEmptyAnalysisWithIndexRequirement() { + return buildAnalysisWithRequirements(true, false, false, false); + } + + private static MavenCentralAnalysis buildAnalysisWithRequirements(boolean requiresIndex, boolean requiresPom, + boolean requiresTransitives, boolean requiresJar) { + return new MavenCentralAnalysis(requiresIndex, requiresPom, requiresTransitives, requiresJar) { + @Override + public void analyzeArtifact(Artifact current) { + + } + }; + } +} From b5b7de6cfd196b876c380a5442ed750623d236ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 27 Aug 2025 10:45:02 +0200 Subject: [PATCH 04/55] Update examples in README. Bump develop version to 2.0.0-SNAPSHOT, as this is a breaking change --- README.md | 12 ++++++------ pom.xml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 83c03d0..291c7a4 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,7 @@ public class ClassFileCountImplementation extends MavenCentralAnalysis { private long numberOfClassfiles; public ClassFileCountImplementation() { - super(); - this.resolveJar = true; + super(false, false, false, true); this.numberOfClassfiles = 0; } @@ -89,6 +88,8 @@ public class ClassFileCountImplementation extends MavenCentralAnalysis { if(toAnalyze.getJarInformation() != null) { numberOfClassfiles += toAnalyze.getJarInformation().getNumClassFiles(); } + URI ArtifactJarURI = toAnalyze.getIdent().getMavenCentralJarUri(); + System.out.println(ArtifactJarURI); } public long getNumberOfClassfiles() { @@ -103,8 +104,7 @@ public class LicenseImplementation extends MavenCentralAnalysis { private final Set uniqueLicenses; public LicenseImplementation() { - super(); - this.resolvePom = true; + super(false, true, false, false); this.uniqueLicenses = new HashSet<>(); } @@ -133,8 +133,7 @@ public class JavaDocImplementation extends MavenCentralAnalysis { private final Set hasJavadocs; public JavaDocImplementation() { - super(); - this.resolveIndex = true; + super(true, false, false, false); this.hasJavadocs = new HashSet<>(); } @@ -154,6 +153,7 @@ public class JavaDocImplementation extends MavenCentralAnalysis { public Set getHasJavadocs() { return hasJavadocs; } + } ``` diff --git a/pom.xml b/pom.xml index f363f5b..fa9bce7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ eu.sse-labs marin - 1.1.0-SNAPSHOT + 2.0.0-SNAPSHOT jar From a4de31a4caa2ded019ac71e8304b7db0522428a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 27 Aug 2025 10:58:05 +0200 Subject: [PATCH 05/55] Remove now unused test utility class --- .../tudo/sse/MavenCentralAnalysisTest.java | 1 - .../testutils/DummyEvaluationAnalysis.java | 26 ------------------- 2 files changed, 27 deletions(-) delete mode 100644 src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java diff --git a/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java b/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java index 1fe6ef4..2afacc8 100644 --- a/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java +++ b/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java @@ -8,7 +8,6 @@ import org.tudo.sse.model.index.Package; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.testutils.DummyEvaluationAnalysis; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralAnalysisFactory; import scala.Tuple2; diff --git a/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java b/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java deleted file mode 100644 index b31252e..0000000 --- a/src/test/java/org/tudo/sse/testutils/DummyEvaluationAnalysis.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.tudo.sse.testutils; - -import org.tudo.sse.MavenCentralAnalysis; -import org.tudo.sse.model.Artifact; - -import java.util.concurrent.atomic.AtomicInteger; - -public class DummyEvaluationAnalysis extends MavenCentralAnalysis { - - private final AtomicInteger artifactCnt = new AtomicInteger(0); - - public DummyEvaluationAnalysis() { - this(false, false, false, false); - } - - public DummyEvaluationAnalysis(boolean resolveIndex, boolean resolvePom, boolean processTransitives, boolean resolveJar) { - super(resolveIndex, resolvePom, processTransitives, resolveJar); - } - - @Override - public void analyzeArtifact(Artifact toAnalyze) { - if(artifactCnt.incrementAndGet() % 100 == 0){ - System.out.println("Client analysis implementation got " + artifactCnt.get() + " artifacts so far"); - } - } -} From 643ef04bb51110df2e9046213c0991778eb92851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 27 Aug 2025 15:48:00 +0200 Subject: [PATCH 06/55] Remove unnecessary code from example in README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 291c7a4..d479b18 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,6 @@ public class ClassFileCountImplementation extends MavenCentralAnalysis { if(toAnalyze.getJarInformation() != null) { numberOfClassfiles += toAnalyze.getJarInformation().getNumClassFiles(); } - URI ArtifactJarURI = toAnalyze.getIdent().getMavenCentralJarUri(); - System.out.println(ArtifactJarURI); } public long getNumberOfClassfiles() { From 17697588523b4e1ec7c79c644ee0860be4608b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 5 Sep 2025 12:56:49 +0200 Subject: [PATCH 07/55] Add JAR file utilitiy methods to JarInformation, including access to OPAL representations --- .../tudo/sse/model/jar/JarInformation.java | 176 ++++++++++++++++++ .../sse/model/jar/JarInformationTest.java | 140 ++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 src/test/java/org/tudo/sse/model/jar/JarInformationTest.java diff --git a/src/main/java/org/tudo/sse/model/jar/JarInformation.java b/src/main/java/org/tudo/sse/model/jar/JarInformation.java index 8c139d3..17bc7c8 100644 --- a/src/main/java/org/tudo/sse/model/jar/JarInformation.java +++ b/src/main/java/org/tudo/sse/model/jar/JarInformation.java @@ -1,16 +1,38 @@ package org.tudo.sse.model.jar; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opalj.br.analyses.Project; +import org.opalj.br.reader.Java17Framework$; +import org.opalj.br.reader.Java17LibraryFramework; +import org.opalj.br.reader.Java17LibraryFramework$; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.ArtifactInformation; +import org.tudo.sse.resolution.FileNotFoundException; +import org.tudo.sse.utils.MavenCentralRepository; +import scala.Tuple2; +import scala.jdk.CollectionConverters; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.jar.JarInputStream; /** * This class contains all the information that is parsed during jar resolution. * This information contains some statistics like codesize and number of class files, as well as a map of packages, contains lists of classfiles. */ public class JarInformation extends ArtifactInformation { + + private static final Logger log = LogManager.getLogger(JarInformation.class); + private long codesize; private long numClassFiles; private long numMethods; @@ -121,4 +143,158 @@ public Map> getPackages() { public void setPackages(Map> packages) { this.packages = packages; } + + /** + * Downloads the JAR file associated with this ArtifactInformation into a temporary directory and returns the file + * object representing it. Note that all temporary files will be deleted once the JVM shuts down. + * @return The File object representing the JAR file + * @throws IOException If any IO errors occur + * @throws FileNotFoundException If no JAR file eixsts for this ArtifactInformation + */ + public File getJarFile() throws IOException, FileNotFoundException { + String prefix = ident.getGA().replace(".","_").replace(":","__"); + Path theFile = Files.createTempFile(prefix,".jar"); + + try(InputStream is = getJarFileInputStream()){ + Files.copy(is, theFile, StandardCopyOption.REPLACE_EXISTING); + } + + return theFile.toFile(); + } + + /** + * Creates and opens an InputStream to the JAR file referenced by this ArtifactInformation. + * @return The JAR file InputStream + * @throws IOException If IO errors occur while opening the stream + * @throws FileNotFoundException If no JAR file exists for this ArtifactInformation + */ + public InputStream getJarFileInputStream() throws IOException, FileNotFoundException { + return MavenCentralRepository.getInstance().openJarFileInputStream(this.ident); + } + + /** + * Creates and opens a JarInputStream to the JAR file referenced by this ArtifactInformation. + * @return The JarInputStream + * @throws IOException If IO errors occur while opening the stream + * @throws FileNotFoundException If no JAR file exists for this ArtifactInformation + */ + public JarInputStream getJarInputStream() throws IOException, FileNotFoundException { + return getJarInputStream(this.ident); + } + + /** + * Retrieves a list of all class files contained in JAR file referenced by this ArtifactInformation. The class + * files are returned as represented by the OPAL framework. + * @return List of OPAL class file representations, or null if no JAR file was found + * @throws IOException If reading the JAR file fails + * @implNote OPAL classes are not designed to be used in large scale analyses. Over time, OPAL's internal caches + * will continue to claim heap space that is never freed, so eventually the program will crash with an + * OutOfMemory error. There is currently no good way to avoid this, so use OPAL instances only if you + * really need them. + */ + public List getOpalClassFileRepresentations() throws IOException { + return getOpalClassFileRepresentations(this.ident); + } + + /** + * Retrieves all class files for the JAR referenced by this ArtifactInformation, and uses them to initialize an + * OPAL project instance. This instance can be used to conduct complex static program analyses. + * @return The OPAL project instance, or null if no JAR file was found + * @throws IOException If reading the JAR file fails + * @implNote OPAL projects are not designed to be used in large scale analyses. Over time, OPAL's internal caches + * will continue to claim heap space that is never freed, so eventually the program will crash with an + * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you + * really need them. + */ + public Project getOpalProject() throws IOException { + try(JarInputStream jarStream = getJarInputStream()){ + return Project.apply(Java17Framework$.MODULE$.ClassFiles(() -> jarStream).map( t -> new Tuple2<>((org.opalj.br.ClassFile)t._1, t._2))); + } catch(FileNotFoundException fnfx){ + log.error("No JAR file found for {}", ident.getCoordinates(), fnfx); + return null; + } + } + + /** + * Initializes an OPAL project instance for the current JAR and the given set of dependencies. This project instance + * can be used to conduct complex, static whole-program analyses. Dependencies will be loaded as interfaces only, + * meaning no actual method implementations (i.e. bytecode) will be present for analysis. + * + * @param dependencies The transitive set of dependencies to use. Can be obtained using the PomInformation instance + * for this artifact + * @return The OPAL project instance, or null if no main project JAR was found + * @throws IOException If an IO error occurs while loading the main JAR or the dependencies + */ + public Project getOpalProject(Set dependencies) throws IOException { + return getOpalProject(dependencies, false, true); + } + + /** + * Initializes an OPAL project instance for the current JAR and the given set of dependencies. This project instance + * can be used to conduct complex, static whole-program analyses. + * @param dependencies The transitive set of dependencies to use. Can be obtained using the PomInformation instance + * for this artifact + * @param fullyLoadLibraries Whether to fully load the dependency's class file contents. If set to false, only their + * interface definitions (class names, method signatures) are loaded, not their actual + * content. + * @param breakOnFailure If set to true, any IO exception when loading dependencies interrupts the method execution + * and throws the exception up to the calling method. If set to false, dependencies will be + * loaded in a best-effort fashion, and errors will only be logged to the console. + * @return The OPAL project instance, or null if no main project JAR was found + * @throws IOException If an IO error occurs while loading the main JAR, or the dependencies (when breakOnFailure is + * enabled). + */ + public Project getOpalProject(Set dependencies, boolean fullyLoadLibraries, boolean breakOnFailure) throws IOException { + List> allLibraryClasses = new ArrayList<>(); + + for(ArtifactIdent dependency : dependencies){ + try(JarInputStream jarStream = getJarInputStream(dependency)){ + addClassFilesToList(allLibraryClasses, jarStream, fullyLoadLibraries); + } catch (IOException iox){ + log.error("Failed to obtain dependency JAR {} for project {}", dependency, ident, iox); + if(breakOnFailure) throw iox; + } catch (FileNotFoundException fnfx){ + // Empty, there might be dependencies without a JAR + } + } + + List> allProjectClasses = new ArrayList<>(); + try(JarInputStream jarStream = getJarInputStream()){ + addClassFilesToList(allProjectClasses, jarStream, true); + } catch (FileNotFoundException fnfx){ + log.error("No JAR file found for {}", ident.getCoordinates(), fnfx); + return null; + } + + return Project.apply(CollectionConverters.ListHasAsScala(allProjectClasses).asScala().toSeq(), + CollectionConverters.ListHasAsScala(allLibraryClasses).asScala().toSeq(), !fullyLoadLibraries); + } + + private List getOpalClassFileRepresentations(ArtifactIdent ident) throws IOException { + List opalCFs = new ArrayList<>(); + + try(JarInputStream jarStream = getJarInputStream(ident)){ + Java17LibraryFramework$.MODULE$.ClassFiles(() -> jarStream) + .foreach(cf -> opalCFs.add((org.opalj.br.ClassFile)cf._1)); + } catch(FileNotFoundException fnfx){ + return null; + } + + return opalCFs; + } + + private JarInputStream getJarInputStream(ArtifactIdent ident) throws IOException, FileNotFoundException { + return new JarInputStream(MavenCentralRepository.getInstance().openJarFileInputStream(ident)); + } + + private void addClassFilesToList(List> cfList, JarInputStream jarStream, boolean loadFully){ + Java17LibraryFramework reader = loadFully ? Java17Framework$.MODULE$ : Java17LibraryFramework$.MODULE$; + CollectionConverters + .SeqHasAsJava(reader.ClassFiles(() -> jarStream)) + .asJava() + .forEach( tuple -> + cfList.add(new Tuple2<>((org.opalj.br.ClassFile)tuple._1, tuple._2)) + ); + + } } diff --git a/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java b/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java new file mode 100644 index 0000000..a5c421b --- /dev/null +++ b/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java @@ -0,0 +1,140 @@ +package org.tudo.sse.model.jar; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.br.ClassFile; +import org.opalj.br.ObjectType; +import org.opalj.br.analyses.Project; +import org.tudo.sse.model.ArtifactIdent; + +import java.io.File; +import java.io.InputStream; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarInputStream; + +import static org.junit.jupiter.api.Assertions.*; + +public class JarInformationTest { + + private final long expectedFileSizeBytes = 79347L; + private final ArtifactIdent ident = new ArtifactIdent("eu.sse-labs", "marin", "1.0.0"); + private final JarInformation jarInfo = new JarInformation(ident); + + @Test + @DisplayName("JAR information should be able to download file to temp directory") + void testLocalFile(){ + try { + File tempFile = jarInfo.getJarFile(); + assertTrue(tempFile.exists()); + assertTrue(tempFile.canRead()); + assertEquals(tempFile.length(), expectedFileSizeBytes); + } catch(Exception x) { + fail(x); + } + } + + @Test + @DisplayName("JAR information should be able to open an InputStream to the actual file") + void testInputStream(){ + try { + InputStream is = jarInfo.getJarFileInputStream(); + byte[] jarFileBytes = is.readAllBytes(); + is.close(); + assertEquals(expectedFileSizeBytes, jarFileBytes.length); + } catch(Exception x){ + fail(x); + } + } + + @Test + @DisplayName("JAR information should be able to open a JarInputStream to the underlying JAR file") + void testJarInputStream(){ + final String className = "org/tudo/sse/ArtifactFactory.class"; + try { + JarInputStream jis = jarInfo.getJarInputStream(); + JarEntry entry = jis.getNextJarEntry(); + boolean classFound = false; + + while(entry != null){ + if(entry.getName().equals(className)){ + classFound = true; + break; + } + entry = jis.getNextJarEntry(); + } + + assertTrue(classFound); + assertNotNull(entry); + assertEquals(className, entry.getName()); + jis.close(); + } catch (Exception x) { + fail(x); + } + } + + @Test + @DisplayName("JAR information should be able to build OPAL class file representations for the underlying JAR") + void testOpalClassFileRepresentations(){ + try { + List cfs = jarInfo.getOpalClassFileRepresentations(); + assertNotNull(cfs); + assertFalse(cfs.isEmpty()); + + boolean foundClass = false; + + for(ClassFile cf : cfs){ + if(cf.thisType().simpleName().equals("ArtifactFactory")){ + foundClass = true; + break; + } + } + + assertTrue(foundClass); + } catch(Exception x){ + fail(x); + } + } + + @Test + @DisplayName("JAR information should be able to initialize an OPAL project instance for the underlying JAR") + void testOpalProjectSimple(){ + try { + Project project = jarInfo.getOpalProject(); + ObjectType afType = project.allProjectClassFiles().find( cf -> cf.thisType().simpleName().equals("ArtifactFactory")).get().thisType(); + assertNotNull(afType); + assertTrue(project.classHierarchy().isKnown(afType)); + assertTrue(project.allLibraryClassFiles().isEmpty()); + } catch (Exception x) { + fail(x); + } + } + + @Test + @DisplayName("JAR information should be able to initialize a complex OPAL project instance with dependencies") + void testOpalProjectComplex(){ + final ArtifactIdent actualDependency = new ArtifactIdent("org.apache.maven.indexer", "indexer-reader", "7.1.3"); + final Set dependencies = new HashSet<>(); + dependencies.add(actualDependency); + + try { + Project project = jarInfo.getOpalProject(dependencies, true, true); + + // Check that main project is loaded + ObjectType afType = project.allProjectClassFiles().find( cf -> cf.thisType().simpleName().equals("ArtifactFactory")).get().thisType(); + assertNotNull(afType); + assertTrue(project.classHierarchy().isKnown(afType)); + + // Check that libraries are in fact loaded + assertFalse(project.allLibraryClassFiles().isEmpty()); + ObjectType irType = project.allLibraryClassFiles().find(cf -> cf.thisType().simpleName().equals("IndexReader")).get().thisType(); + assertNotNull(irType); + assertTrue(project.classHierarchy().isKnown(irType)); + } catch (Exception x){ + fail(x); + } + } + +} From a7e02e9c1d0d26e8f73eba6f0bc2c4aa4b425eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 17 Oct 2025 15:56:14 +0200 Subject: [PATCH 08/55] Update OPAL to 6.0.0 --- .gitignore | 3 ++- pom.xml | 2 +- src/main/java/org/tudo/sse/resolution/JarResolver.java | 4 ++-- .../java/org/tudo/sse/model/jar/JarInformationTest.java | 8 ++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index f8c4a8d..1be7d23 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ settings.xml #maven resolution files maven-resolution-files -.DS_Store \ No newline at end of file +.DS_Store +.mvn \ No newline at end of file diff --git a/pom.xml b/pom.xml index fa9bce7..6d6788d 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ de.opal-project framework_2.13 - 5.0.0 + 6.0.0 diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index 6b0dea6..936e75d 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -5,7 +5,7 @@ import org.apache.logging.log4j.Logger; import org.opalj.br.ClassFile; import org.opalj.br.Method; -import org.opalj.br.ObjectType; +import org.opalj.br.ClassType; import org.opalj.br.analyses.Project$; import org.opalj.br.package$; import org.opalj.br.reader.Java16LibraryFramework; @@ -212,7 +212,7 @@ public org.tudo.sse.model.jar.ClassFile processClassFile(ClassFile classFile) { } if(!classFile.interfaceTypes().isEmpty()) { - for(ObjectType type : JavaConverters.asJava(classFile.interfaceTypes())) { + for(ClassType type : JavaConverters.asJava(classFile.interfaceTypes())) { interfaces.add(new ObjType(type.id(), type.fqn(), type.packageName())); } } diff --git a/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java b/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java index a5c421b..f22c6fc 100644 --- a/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java +++ b/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java @@ -3,7 +3,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.opalj.br.ClassFile; -import org.opalj.br.ObjectType; +import org.opalj.br.ClassType; import org.opalj.br.analyses.Project; import org.tudo.sse.model.ArtifactIdent; @@ -103,7 +103,7 @@ void testOpalClassFileRepresentations(){ void testOpalProjectSimple(){ try { Project project = jarInfo.getOpalProject(); - ObjectType afType = project.allProjectClassFiles().find( cf -> cf.thisType().simpleName().equals("ArtifactFactory")).get().thisType(); + ClassType afType = project.allProjectClassFiles().find( cf -> cf.thisType().simpleName().equals("ArtifactFactory")).get().thisType(); assertNotNull(afType); assertTrue(project.classHierarchy().isKnown(afType)); assertTrue(project.allLibraryClassFiles().isEmpty()); @@ -123,13 +123,13 @@ void testOpalProjectComplex(){ Project project = jarInfo.getOpalProject(dependencies, true, true); // Check that main project is loaded - ObjectType afType = project.allProjectClassFiles().find( cf -> cf.thisType().simpleName().equals("ArtifactFactory")).get().thisType(); + ClassType afType = project.allProjectClassFiles().find( cf -> cf.thisType().simpleName().equals("ArtifactFactory")).get().thisType(); assertNotNull(afType); assertTrue(project.classHierarchy().isKnown(afType)); // Check that libraries are in fact loaded assertFalse(project.allLibraryClassFiles().isEmpty()); - ObjectType irType = project.allLibraryClassFiles().find(cf -> cf.thisType().simpleName().equals("IndexReader")).get().thisType(); + ClassType irType = project.allLibraryClassFiles().find(cf -> cf.thisType().simpleName().equals("IndexReader")).get().thisType(); assertNotNull(irType); assertTrue(project.classHierarchy().isKnown(irType)); } catch (Exception x){ From 36643a3900c0e43294287253fd7eb05245eeadb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 17 Oct 2025 16:03:13 +0200 Subject: [PATCH 09/55] Update CI build to update dependabot alerts in the future --- .github/workflows/maven.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 09d8667..d427c4d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -32,3 +32,5 @@ jobs: run: mvn -B compile --file pom.xml - name: Run unit tests run: mvn -B test --file pom.xml + - name: Update dependency graph + uses: advanced-security/maven-dependency-submission-action@v5 \ No newline at end of file From fff6b655286c0e0454b6cbd56726c26f067e9334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 17 Oct 2025 17:52:19 +0200 Subject: [PATCH 10/55] Introduce an interator on unique library IDs --- .../tudo/sse/utils/LibraryIndexIterator.java | 126 ++++++++++++++++++ .../sse/utils/LibraryIndexIteratorTest.java | 88 ++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java create mode 100644 src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java new file mode 100644 index 0000000..4594029 --- /dev/null +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -0,0 +1,126 @@ +package org.tudo.sse.utils; + +import org.apache.maven.index.reader.ChunkReader; +import org.apache.maven.index.reader.IndexReader; +import org.tudo.sse.IndexWalker; + +import java.io.IOException; +import java.net.URI; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +public class LibraryIndexIterator implements Iterator, AutoCloseable { + + private final Set libraryHashesSeen; + private final URI baseUri; + private final Iterator> entryIterator; + + private final IndexReader mavenIndexReader; + private final ChunkReader mavenChunkReader; + + private long currentLibraryIndexPosition; + private long currentPosition; + + private String currentLibraryGA; + private String nextLibraryGA; + + public LibraryIndexIterator(URI baseUri) throws IOException { + this.libraryHashesSeen = new HashSet<>(); + this.baseUri = baseUri; + + // Build intermediate readers that must be kept open until iterator is done and resources can be closed + this.mavenIndexReader = new IndexReader(null, new HttpResourceHandler(baseUri.resolve(".index/"))); + this.mavenChunkReader = this.mavenIndexReader.iterator().next(); + + // Build the actual iterator for entries in the Maven Central index + this.entryIterator = this.mavenChunkReader.iterator(); + + this.currentLibraryIndexPosition = -1; + this.currentPosition = -1; + this.currentLibraryGA = null; + } + + public boolean hasNext() { + // If currentLibraryGA is null, we have the first hasNext() call after a call to next(). We must find our new + // currentLibraryGA first. + if(currentLibraryGA == null){ + + // If nextLibraryGA is null, we have no information stored from a previous iteration about the next artifact. + // We thus must determine the current artifact ourselves and, while we are there, detect the new next artifact + // as well. + if(this.nextLibraryGA == null){ + // Walk to the first actual entry + while(currentLibraryGA == null && entryIterator.hasNext()){ + currentLibraryGA = getGAFromEntry(nextEntry()); + currentLibraryIndexPosition = currentPosition; + } + } else { + // If we have information about the next artifact, we just copy it + this.currentLibraryGA = this.nextLibraryGA; + this.currentLibraryIndexPosition = this.currentPosition; + this.nextLibraryGA = null; + } + + // Walk up to the next entry that has a different GA (is a different library) + String nextNewGA = null; + + while(entryIterator.hasNext()){ + final Map nextEntry = nextEntry(); + final String nextGA = getGAFromEntry(nextEntry); + + // Silently ignore malformed entries + if(nextGA == null) continue; + + if(!currentLibraryGA.equals(nextGA) && isNewLibrary(nextGA)){ + nextNewGA = nextGA; + break; + } + } + + this.nextLibraryGA = nextNewGA; + } + + return this.nextLibraryGA != null; + } + + public String next() { + if(hasNext()){ + final String valueToReturn = this.currentLibraryGA; + this.libraryHashesSeen.add(this.currentLibraryGA.hashCode()); + this.currentLibraryGA = null; + return valueToReturn; + } else throw new IllegalStateException("No libraries left on iterator (position " + currentPosition + ")"); + } + + + + private boolean isNewLibrary(String libraryGA){ + final int hash = libraryGA.hashCode(); + return !libraryHashesSeen.contains(hash); + } + + private String getGAFromEntry(Map entry){ + final String uVal = entry.get("u"); + if(uVal != null){ + String[] parts = uVal.split(IndexWalker.splitPattern); + return parts[0] + ":" + parts[1]; + } else return null; + } + + private Map nextEntry(){ + currentPosition += 1; + return entryIterator.next(); + } + + @Override + public void close() throws Exception { + if(this.mavenChunkReader != null){ + this.mavenChunkReader.close(); + } + if(this.mavenIndexReader != null) { + this.mavenIndexReader.close(); + } + } +} diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java new file mode 100644 index 0000000..c067d54 --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -0,0 +1,88 @@ +package org.tudo.sse.utils; + +import org.junit.jupiter.api.*; + +import java.net.URI; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.fail; + +class LibraryIndexIteratorTest { + + private LibraryIndexIterator iteratorUnderTest; + + @BeforeEach + void setIndexIterator() { + try { + iteratorUnderTest = new LibraryIndexIterator(new URI("https://repo1.maven.org/maven2/")); + } catch (Exception x) { fail(x); } + } + + @Test + @DisplayName("The Library Index Iterator must produce valid GAs") + void validGAFormat() { + int cutoff = 10000; + int idx = 0; + + while(iteratorUnderTest.hasNext() && idx < cutoff) { + final String ga = iteratorUnderTest.next(); + final String[] parts = ga.split(":"); + + assert(parts.length == 2); + assert(!parts[0].isBlank() && !parts[1].isBlank()); + + idx += 1; + + if(idx % 100 == 0){ + System.out.println("Got " + idx + " unique library names so far"); + } + } + } + + @Test + @DisplayName("The Library Index Iterator must produce unique GAs") + void uniqueGAs() { + int cutoff = 100000; + Set gasSeen = new HashSet<>(); + + while(iteratorUnderTest.hasNext() && gasSeen.size() < cutoff) { + final String ga = iteratorUnderTest.next(); + + assert(!gasSeen.contains(ga)); + + gasSeen.add(ga); + + if(gasSeen.size() % 1000 == 0){ + System.out.println("Got " + gasSeen.size() + " unique library names so far"); + } + } + } + + @Test + @Disabled + @DisplayName("The Library Index Iterator must terminate") + void terminate() { + int idx = 0; + while(iteratorUnderTest.hasNext()){ + iteratorUnderTest.next(); + idx += 1; + + if(idx % 10000 == 0){ + System.out.println("Got " + idx + " unique library names so far"); + } + } + // There are more than 600.000 libraries on maven central + assert(idx > 600000); + } + + + @AfterEach + void resetIndexIterator() { + if(iteratorUnderTest != null) { + try { iteratorUnderTest.close(); } catch (Exception x) { fail (x); } + } + iteratorUnderTest = null; + } + +} From b442adc0d96de4445da7fa181eea0d391aa382bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 20 Oct 2025 12:01:01 +0200 Subject: [PATCH 11/55] Add capabilitis to perform per-library analysis --- .../java/org/tudo/sse/ArtifactFactory.java | 16 + .../tudo/sse/analyses/ArtifactAnalysis.java | 62 ++++ .../analyses/MavenCentralLibraryAnalysis.java | 313 ++++++++++++++++++ .../java/org/tudo/sse/model/Artifact.java | 8 + .../ProcessIdentifierMessage.java | 2 +- .../multithreading/ProcessLibraryMessage.java | 15 + .../tudo/sse/multithreading/QueueActor.java | 30 +- .../sse/multithreading/ResolverActor.java | 4 + .../org/tudo/sse/multithreading/WorkItem.java | 4 + .../DefaultMavenReleaseListProvider.java | 13 +- .../releases/IReleaseListProvider.java | 18 +- .../java/org/tudo/sse/utils/CLIParser.java | 69 ++++ .../tudo/sse/utils/LibraryIndexIterator.java | 27 +- .../sse/utils/MavenCentralRepository.java | 43 ++- .../sse/model/LocalPomInformationTest.java | 10 +- .../tudo/sse/resolution/PomResolverTest.java | 11 +- .../sse/utils/LibraryIndexIteratorTest.java | 16 + 17 files changed, 623 insertions(+), 38 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java create mode 100644 src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java create mode 100644 src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java create mode 100644 src/main/java/org/tudo/sse/multithreading/WorkItem.java create mode 100644 src/main/java/org/tudo/sse/utils/CLIParser.java diff --git a/src/main/java/org/tudo/sse/ArtifactFactory.java b/src/main/java/org/tudo/sse/ArtifactFactory.java index f3e4dde..d4c2998 100644 --- a/src/main/java/org/tudo/sse/ArtifactFactory.java +++ b/src/main/java/org/tudo/sse/ArtifactFactory.java @@ -20,6 +20,22 @@ private ArtifactFactory() {} */ public static final Map artifacts = new HashMap<>(); + /** + * Creates a new artifact or retrieves an existing one. If a new artifact is created, no information is attached. + * @param ident The artifact identifier + * @return The artifact with the given identifier + */ + public static Artifact createArtifact(ArtifactIdent ident) { + assert(ident != null); + if(artifacts.containsKey(ident)) { + return artifacts.get(ident); + } else { + Artifact newArtifact = new Artifact(ident); + artifacts.put(ident, newArtifact); + return newArtifact; + } + } + /** * This method creates a new Artifact or retrieves it from the map. * @param artifactInformation an identifier used to keep track of artifacts diff --git a/src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java new file mode 100644 index 0000000..523f366 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java @@ -0,0 +1,62 @@ +package org.tudo.sse.analyses; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +abstract class ArtifactAnalysis { + + /** + * Defines whether this analysis requires artifacts to have index information annotated. + */ + protected final boolean resolveIndex; + + /** + * Defines whether this analysis requires artifacts to have pom information annotated. + */ + protected final boolean resolvePom; + + /** + * Defines whether this analysis requires artifacts to have resolved transitive pom information. + */ + protected final boolean processTransitives; + + /** + * Defines whether this analysis requires artifacts to have jar information annotated. + */ + protected final boolean resolveJar; + + /** + * Logger instance for subclasses + */ + protected final Logger log = LogManager.getLogger(getClass()); + + + /** + * Creates a new Maven Central Analysis with the given configuration options. + * + * @param requiresIndex Whether this analysis requires index information + * @param requiresPom Whether this analysis requires POM information + * @param requiresTransitives Whether this analysis requires transitive POM information. If true, "requiresPOM" will + * be set to true no matter its given parameter value. + * @param requiresJar Whether this analysis requires JAR information + */ + protected ArtifactAnalysis(boolean requiresIndex, + boolean requiresPom, + boolean requiresTransitives, + boolean requiresJar){ + if(!requiresIndex && !requiresPom && !requiresTransitives && !requiresJar){ + log.warn("Potential misconfiguration - no data sources (index, POM, JAR) are required by this analysis"); + } else if(requiresTransitives && !requiresPom){ + log.warn("Potential misconfiguration - analysis requires transitive information but no POM information. " + + "POM information will also be collected to provide transitive information."); + } + + resolveIndex = requiresIndex; + resolvePom = requiresPom || requiresTransitives; // Cannot have transitive information without POM information + processTransitives = requiresTransitives; + resolveJar = requiresJar; + } + + public abstract void runAnalysis(String[] args); + +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java new file mode 100644 index 0000000..6933f18 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -0,0 +1,313 @@ +package org.tudo.sse.analyses; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import org.tudo.sse.ArtifactFactory; +import org.tudo.sse.CLIException; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.multithreading.ProcessLibraryMessage; +import org.tudo.sse.multithreading.QueueActor; +import org.tudo.sse.resolution.ResolverFactory; +import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; +import org.tudo.sse.resolution.releases.IReleaseListProvider; +import org.tudo.sse.utils.CLIParser; +import org.tudo.sse.utils.LibraryIndexIterator; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public abstract class MavenCentralLibraryAnalysis extends ArtifactAnalysis { + + private final IReleaseListProvider releaseListProvider; + private final ResolverFactory resolverFactory; + + protected final LibraryAnalysisConfiguration config; + + private ActorRef queueActorRef; + + /** + * Creates a new Maven Central Analysis with the given configuration options. + * + * @param requiresPom Whether this analysis requires POM information + * @param requiresTransitives Whether this analysis requires transitive POM information. If true, "requiresPOM" will + * be set to true no matter its given parameter value. + * @param requiresJar Whether this analysis requires JAR information + */ + protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTransitives, boolean requiresJar) { + super(false, requiresPom, requiresTransitives, requiresJar); + + this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); + this.resolverFactory = new ResolverFactory(processTransitives); + this.config = new LibraryAnalysisConfiguration(); + } + + @Override + public final void runAnalysis(String[] args){ + this.config.parseArguments(args); + printRunInfo(); + + ActorSystem system = null; + + if(this.config.multipleThreads){ + system = ActorSystem.create("marin-actors"); + this.queueActorRef = system.actorOf(QueueActor.props(config.threadCount, system), "marin-queue-actor"); + } + + Iterator gaIterator = getGaIterator(); + + if(config.skip > 0){ + log.info("Skipping {} library names", config.skip); + for(int i = 0; i < config.skip; i++){ + if(!gaIterator.hasNext()){ + log.warn("No more GA inputs available while skipping to desired position"); + break; + } else gaIterator.next(); + } + } + + long entriesTaken = 0; + + if(config.take >= 0) + log.info("Taking {} library names", config.take); + + while(entriesTaken < config.take && gaIterator.hasNext()){ + processEntry(gaIterator.next()); + + entriesTaken += 1; + } + + log.info("Processed a total of {} library names", entriesTaken); + + // Close index iterator if it was used + if(gaIterator instanceof LibraryIndexIterator){ + LibraryIndexIterator libraryIndexIterator = (LibraryIndexIterator)gaIterator; + try { + libraryIndexIterator.close(); + } catch (Exception x) { log.warn("Error closing index iterator: {}", x.getMessage());} + } + + // Close actor system if it was used + if(system != null){ + try{ + system.getWhenTerminated().toCompletableFuture().get(); + } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} + } + } + + /** + * Main analysis implementation. Defines how a single library shall be analyzed. All artifacts for a given + * library will have information annotated corresponding to the protected attributes' values. + * @param libraryGA The library GA tuple (i.e., the library name) + * @param releases A list of artifacts as ordered by Maven Central's metadata.xml file + */ + protected abstract void analyzeLibrary(String libraryGA, List releases); + + private void processEntry(String ga){ + final String[] parts = ga.split(":"); + + if(parts.length != 2){ + log.warn("Not a valid GA tuple (need :): {}", ga); + return; + } + + final String groupId = parts[0]; + final String artifactId = parts[1]; + + if(!config.multipleThreads){ + process(ga, groupId, artifactId); + } else { + final ProcessLibraryMessage msg = new ProcessLibraryMessage(() -> process(ga, groupId, artifactId)); + queueActorRef.tell(msg, ActorRef.noSender()); + } + } + + private Void process(String ga, String groupId, String artifactId){ + List identifiers = getReleaseIdentifiers(groupId, artifactId); + + // Abort if we find no releases, error has already been logged at this point + if(identifiers == null) return null; + + // Resolve all information for all library releases as defined by this analysis + List libraryArtifacts = new ArrayList<>(); + for(ArtifactIdent ident : identifiers){ + Artifact a = ArtifactFactory.createArtifact(ident); + resolveDataAsNeeded(ident); + libraryArtifacts.add(a); + } + + // Call the custom analysis implementation + analyzeLibrary(ga, libraryArtifacts); + return null; + } + + private List getReleaseIdentifiers(String groupId, String artifactId){ + try { + final List libraryVersions = releaseListProvider.getReleases(groupId, artifactId); + List identifiers = new ArrayList<>(); + for(String libraryVersion : libraryVersions){ + identifiers.add(new ArtifactIdent(groupId, artifactId, libraryVersion)); + } + return identifiers; + } catch (Exception x) { + log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + return null; + } + } + + private Iterator getGaIterator(){ + if(config.gaInputListFile != null){ + try { + final List inputs = Files.readAllLines(config.gaInputListFile); + log.info("Read {} GAs from input file {}", inputs.size(), config.gaInputListFile); + return inputs.iterator(); + } catch(IOException iox){ + log.error("Failed to read GA tuples from input file at {}", config.gaInputListFile, iox); + throw new RuntimeException(iox); + } + } else { + try { + final LibraryIndexIterator indexIterator = new LibraryIndexIterator(new URI(MavenCentralRepository.RepoBasePath)); + + if(config.progressRestoreFile != null){ + final long startingPosition = config.getProgressFromRestoreFile(); + log.info("Restoring previous progress (index position {})", startingPosition); + indexIterator.setIndexPosition(startingPosition); + } + + return indexIterator; + } catch (URISyntaxException | IOException iox){ + log.error("Failed to create Maven Central library iterator", iox); + throw new RuntimeException(iox); + } + + } + } + + private void resolveDataAsNeeded(ArtifactIdent identifier){ + if(resolvePom && resolveJar) { + resolverFactory.runBoth(identifier); + } else if(resolvePom) { + resolverFactory.runPom(identifier); + } else if(resolveJar) { + resolverFactory.runJar(identifier); + } + } + + private void printRunInfo(){ + log.info("Running a Maven Central Library Analysis Implementation:"); + if(resolveIndex) log.info ("\t - The analysis requires index information"); + if(resolvePom) log.info ("\t - The analysis requires pom information"); + if(processTransitives) log.info("\t - The analysis requires transitive pom dependencies"); + if(resolveJar)log.info ("\t - The analysis requires jar information"); + + log.info("The current run has been configured as follows:"); + + if(config.multipleThreads){ + log.info("\t - Using " + config.threadCount + " threads"); + } else { + log.info("\t - Using one thread"); + } + + if(config.gaInputListFile == null){ + log.info("\t - Reading libraries from Maven Central index"); + if(config.progressRestoreFile != null) log.info("\t - Restoring last index position from " + config.progressRestoreFile); + if(config.progressOutputFile != null) log.info("\t - Writing last index position to " + config.progressOutputFile); + if(config.skip >= 0) log.info("\t - Skipping " + config.skip + " artifacts"); + if(config.take >= 0) log.info("\t - Taking " + config.take + " artifacts"); + } else { + log.info("\t - Reading libraries from GA-list at " + config.gaInputListFile); + } + } + + protected static class LibraryAnalysisConfiguration extends CLIParser { + + public long skip; + public long take; + + public Path gaInputListFile; + public Path progressOutputFile; + public Path progressRestoreFile; + + public boolean multipleThreads; + public int threadCount; + + public int progressWriteInterval; + + public LibraryAnalysisConfiguration() { + skip = -1L; + take = -1L; + gaInputListFile = null; + progressOutputFile = Paths.get("marin-progress"); + progressRestoreFile = null; + multipleThreads = false; + threadCount = 1; + progressWriteInterval = 1000; + } + + @Override + public void parseArguments(String[] args) { + try { + for(int i = 0; i < args.length; i += 2){ + switch (args[i]){ + case "-st": + if(skip != -1L) + throw new CLIException("Values for skip and take cannot be set twice!"); + + final long[] skipTake = nextArgAsLongPair(args, i); + skip = skipTake[0]; + take = skipTake[1]; + break; + case "-ip": + if(gaInputListFile != null) + throw new CLIException("Cannot restore index position when a custom input list is used!"); + if(progressRestoreFile != null) + throw new CLIException("Progress restore file cannot be set twice!"); + + progressRestoreFile = nextArgAsRegularFileReference(args, i); + break; + case "--coordinates": + if(gaInputListFile != null) + throw new CLIException("Input file cannot be set twice!"); + gaInputListFile = nextArgAsRegularFileReference(args, i); + break; + case "--name": + progressOutputFile = nextArgAsPath(args, i); + break; + case "--multi": + final int threads = nextArgAsInt(args, i); + multipleThreads = threads > 1; + threadCount = threads; + break; + default: + throw new CLIException(args[i]); + } + } + } catch(CLIException clix){ + throw new RuntimeException(clix); + } + } + + long getProgressFromRestoreFile() { + BufferedReader indexReader; + try { + indexReader = new BufferedReader(new FileReader(progressRestoreFile.toFile())); + String line = indexReader.readLine(); + return Integer.parseInt(line); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + +} diff --git a/src/main/java/org/tudo/sse/model/Artifact.java b/src/main/java/org/tudo/sse/model/Artifact.java index d81d000..a66033e 100644 --- a/src/main/java/org/tudo/sse/model/Artifact.java +++ b/src/main/java/org/tudo/sse/model/Artifact.java @@ -31,6 +31,14 @@ public class Artifact { private PomInformation pomInformation; private JarInformation jarInformation; + /** + * Create a new artifact with no information attached. + * @param ident The artifact identifier + */ + public Artifact(ArtifactIdent ident){ + this.ident = ident; + } + /** * Creates a new artifact based on given IndexInformation. This artifact will have no POM or JAR information * associated. diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index 470e731..19295a7 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -7,7 +7,7 @@ * A message passed to the processing queue to indicate that a given analysis instance must process the given * artifact identifier. */ -public class ProcessIdentifierMessage { +public class ProcessIdentifierMessage implements WorkItem { private final ArtifactIdent identifier; private final MavenCentralAnalysis instance; diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java new file mode 100644 index 0000000..7868766 --- /dev/null +++ b/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java @@ -0,0 +1,15 @@ +package org.tudo.sse.multithreading; + +import java.util.function.Supplier; + +public class ProcessLibraryMessage implements WorkItem { + + private final Supplier processEntryCallback; + + public ProcessLibraryMessage(Supplier callback) { + this.processEntryCallback = callback; + } + + public Supplier getProcessEntryCallback() { return processEntryCallback; } + +} diff --git a/src/main/java/org/tudo/sse/multithreading/QueueActor.java b/src/main/java/org/tudo/sse/multithreading/QueueActor.java index d33850c..7a1b383 100644 --- a/src/main/java/org/tudo/sse/multithreading/QueueActor.java +++ b/src/main/java/org/tudo/sse/multithreading/QueueActor.java @@ -18,7 +18,7 @@ public class QueueActor extends AbstractActor { private final int numResolverActors; private final AtomicInteger curNumResolvers; - private final Queue jobQueue; + private final Queue jobQueue; private boolean indexFinished = false; private final ActorSystem system; @@ -49,19 +49,8 @@ public static Props props(int numResolverActors, ActorSystem system) { @Override public Receive createReceive() { return ReceiveBuilder.create() - .match(ProcessIdentifierMessage.class, message -> { - synchronized (curNumResolvers){ - if(curNumResolvers.get() < numResolverActors) { - ActorRef processor = getContext().actorOf(ResolverActor.props()); - processor.tell(message, getSelf()); - log.info("New resolver created"); - curNumResolvers.incrementAndGet(); - } else { - jobQueue.add(message); - } - } - - }) + .match(ProcessIdentifierMessage.class, this::forwardToResolvers) + .match(ProcessLibraryMessage.class, this::forwardToResolvers) .match(String.class, message -> { synchronized (jobQueue){ if(!jobQueue.isEmpty()) { @@ -92,4 +81,17 @@ public Receive createReceive() { .build(); } + private void forwardToResolvers(WorkItem message){ + synchronized (curNumResolvers){ + if(curNumResolvers.get() < numResolverActors) { + ActorRef processor = getContext().actorOf(ResolverActor.props()); + processor.tell(message, getSelf()); + log.info("New resolver created"); + curNumResolvers.incrementAndGet(); + } else { + jobQueue.add(message); + } + } + } + } diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index e658c30..2316083 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -31,6 +31,10 @@ public Receive createReceive() { message.getInstance().callResolver(message.getIdentifier()); message.getInstance().analyzeArtifact(ArtifactFactory.getArtifact(message.getIdentifier())); getSender().tell("Finished", getSelf()); + }) + .match(ProcessLibraryMessage.class, message -> { + message.getProcessEntryCallback().get(); + getSender().tell("Finished", getSelf()); }).build(); } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkItem.java b/src/main/java/org/tudo/sse/multithreading/WorkItem.java new file mode 100644 index 0000000..ee16765 --- /dev/null +++ b/src/main/java/org/tudo/sse/multithreading/WorkItem.java @@ -0,0 +1,4 @@ +package org.tudo.sse.multithreading; + +public interface WorkItem { +} diff --git a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java index 74677ae..99b0690 100644 --- a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java +++ b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -23,7 +24,7 @@ * * @author Johannes Düsing */ -public class DefaultMavenReleaseListProvider implements IReleaseListProvider{ +public class DefaultMavenReleaseListProvider extends IReleaseListProvider{ private final MetadataXpp3Reader reader = new MetadataXpp3Reader(); private final MavenCentralRepository mavenRepo = MavenCentralRepository.getInstance(); @@ -43,10 +44,8 @@ public static DefaultMavenReleaseListProvider getInstance() { private DefaultMavenReleaseListProvider(){} @Override - public List getReleases(ArtifactIdent identifier) throws IOException { - Objects.requireNonNull(identifier); - - try(InputStream xmlInputStream = mavenRepo.openXMLFileInputStream(identifier)) { + public List getReleases(String groupId, String artifactId) throws IOException { + try(InputStream xmlInputStream = mavenRepo.openXMLFileInputStream(groupId, artifactId)) { Metadata versionListData = reader.read(new BufferedReader(new InputStreamReader(xmlInputStream))); Versioning versioning = versionListData.getVersioning(); @@ -61,8 +60,8 @@ public List getReleases(ArtifactIdent identifier) throws IOException { return List.of(); } - } catch (XmlPullParserException | IOException | FileNotFoundException x) { - throw new IOException("Failed to obtain version list for " + identifier, x); + } catch (XmlPullParserException | IOException | FileNotFoundException | URISyntaxException x) { + throw new IOException("Failed to obtain version list for " + groupId + ":" + artifactId, x); } } diff --git a/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java index 01f56e8..819b5d6 100644 --- a/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java +++ b/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java @@ -4,12 +4,13 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; /** * Interface defining functionality to obtain a list of Maven Central releases (i.e. version numbers) for a given * library (i.e. GA-Tuple). */ -public interface IReleaseListProvider { +public abstract class IReleaseListProvider { /** * Gets the ordered list of releases (i.e. version numbers) for the given identifier. The identifier's version is @@ -19,6 +20,19 @@ public interface IReleaseListProvider { * @return List of version numbers as ordered by the underlying source * @throws IOException If a connection error occurs */ - List getReleases(ArtifactIdent identifier) throws IOException; + public List getReleases(ArtifactIdent identifier) throws IOException{ + Objects.requireNonNull(identifier); + return getReleases(identifier.getGroupID(), identifier.getArtifactID()); + } + + /** + * Gets the ordered list of releases (i.e. version numbers) for the given library. + * + * @param groupId The library groupId + * @param artifactId The library artifactId + * @return List of version numbers as ordered by the underlying source + * @throws IOException If a connection error occurs + */ + public abstract List getReleases(String groupId, String artifactId) throws IOException; } diff --git a/src/main/java/org/tudo/sse/utils/CLIParser.java b/src/main/java/org/tudo/sse/utils/CLIParser.java new file mode 100644 index 0000000..c8bc0f6 --- /dev/null +++ b/src/main/java/org/tudo/sse/utils/CLIParser.java @@ -0,0 +1,69 @@ +package org.tudo.sse.utils; + +import org.tudo.sse.CLIException; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public abstract class CLIParser { + + public abstract void parseArguments(String[] args); + + protected int nextArgAsInt(String[] args, int i) throws CLIException { + if(i + 1 < args.length) { + try{ + return Integer.parseInt(args[i + 1]); + } catch(NumberFormatException e) { + throw new CLIException(args[i], e.getMessage()); + } + } else { + throw new CLIException(args[i]); + } + } + + protected long[] nextArgAsLongPair(String[] args, int i) throws CLIException { + long[] toReturn = new long[2]; + if(i + 1 < args.length) { + String[] ints = args[i + 1].split(":"); + if(ints.length == 2) { + try { + toReturn[0] = Long.parseLong(ints[0]); + toReturn[1] = Long.parseLong(ints[1]); + } catch(NumberFormatException e) { + throw(new CLIException(args[i], e.getMessage())); + } + } else { + throw(new CLIException(args[i], "Correct format: first:second")); + } + } else { + throw(new CLIException(args[i], "Missing argument: first:second")); + } + return toReturn; + } + + protected Path nextArgAsPath(String[] args, int i) throws CLIException { + if(i + 1 < args.length) { + return Paths.get(args[i + 1]); + } else { + throw(new CLIException(args[i], "Missing argument: path/to/file")); + } + } + + protected Path nextArgAsRegularFileReference(String[] args, int i) throws CLIException { + final Path path = nextArgAsPath(args, i); + if(Files.isRegularFile(path)) + return path; + else + throw new CLIException(args[i], "Expected a regular file but got:" + path); + } + + protected Path nextArgAsDirectoryReference(String[] args, int i) throws CLIException { + final Path path = nextArgAsPath(args, i); + if(Files.isDirectory(path)) + return path; + else + throw new CLIException(args[i], "Expected a directory but got:" + path); + } + +} diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 4594029..5b25d09 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -42,6 +42,17 @@ public LibraryIndexIterator(URI baseUri) throws IOException { this.currentLibraryGA = null; } + public void setIndexPosition(long startIdx){ + Map entry; + + do { + entry = nextEntry(); + } while(this.currentPosition < startIdx); + + this.currentLibraryGA = getGAFromEntry(entry); + this.currentLibraryIndexPosition = this.currentPosition; + } + public boolean hasNext() { // If currentLibraryGA is null, we have the first hasNext() call after a call to next(). We must find our new // currentLibraryGA first. @@ -104,8 +115,20 @@ private boolean isNewLibrary(String libraryGA){ private String getGAFromEntry(Map entry){ final String uVal = entry.get("u"); if(uVal != null){ - String[] parts = uVal.split(IndexWalker.splitPattern); - return parts[0] + ":" + parts[1]; + final StringBuilder gaBuilder = new StringBuilder(); + boolean patternSeen = false; + for(char current: uVal.toCharArray()){ + if(current == '|'){ + if(patternSeen) break; + + patternSeen = true; + gaBuilder.append(':'); + } else { + gaBuilder.append(current); + } + } + + return gaBuilder.toString(); } else return null; } diff --git a/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java b/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java index 5016310..4b60df8 100644 --- a/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java +++ b/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java @@ -16,7 +16,7 @@ */ public final class MavenCentralRepository { - private static final String RepoBasePath = "https://repo1.maven.org/maven2/"; + public static final String RepoBasePath = "https://repo1.maven.org/maven2/"; private static MavenCentralRepository theInstance = null; @@ -35,6 +35,19 @@ public InputStream openXMLFileInputStream(ArtifactIdent ident) throws IOExceptio return ResourceConnections.openInputStream(ident.getMavenCentralXMLUri()); } + /** + * Opens an input stream to the version list of the given library. + * @param groupId The library's groupId + * @param artifactId The library's artifactId + * @return An input stream for the library's version list XML file + * @throws IOException If accessing the resource fails + * @throws FileNotFoundException If the library does not exist / does not have a version list file + */ + public InputStream openXMLFileInputStream(String groupId, String artifactId) + throws IOException, FileNotFoundException, URISyntaxException{ + return ResourceConnections.openInputStream(MavenCentralRepository.buildVersionsFileURI(groupId, artifactId)); + } + /** * Opens an input stream to the POM file of the given artifact. * @param ident Artifact identifier for which to open the POM file input stream @@ -136,6 +149,19 @@ public static URI buildVersionsFileURI(ArtifactIdent artifact) .resolve(encode("maven-metadata") + ".xml"); } + /** + * Builds the URI that references a library's version list (maven-metadata.xml) file. + * @param groupId The library's groupId + * @param artifactId The library's artifactId + * @return Fully encoded URI referencing the library's version list + * @throws URISyntaxException If the URI is invalid + */ + public static URI buildVersionsFileURI(String groupId, String artifactId) + throws URISyntaxException { + return buildLibraryBaseURI(groupId, artifactId) + .resolve(encode("maven-metadata") + ".xml"); + } + /** * Builds the URI that references an artifact base directory on the Central repository * @param artifact The artifact identifier @@ -173,9 +199,20 @@ public static URI buildSecondaryArtifactBaseURI(ArtifactIdent artifact, String s */ public static URI buildLibraryBaseURI(ArtifactIdent artifact) throws URISyntaxException { + return buildLibraryBaseURI(artifact.getGroupID(), artifact.getArtifactID()); + } + + /** + * Builds the URI that references a library's base directory on the Central repository + * @param groupId The library's groupId + * @param artifactId The library's artifactId + * @return Fully encoded URI referencing the library base directory + * @throws URISyntaxException If the URI is invalid + */ + public static URI buildLibraryBaseURI(String groupId, String artifactId) throws URISyntaxException { return new URI(RepoBasePath) - .resolve(encode(artifact.getGroupID()).replace(".", "/") + "/") - .resolve(encode(artifact.getArtifactID()) + "/"); + .resolve(encode(groupId).replace(".", "/") + "/") + .resolve(encode(artifactId) + "/"); } private static String encode(String path) { diff --git a/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java b/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java index d370227..c806392 100644 --- a/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java +++ b/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java @@ -44,15 +44,17 @@ private void buildMap(){ } @Override - public List getReleases(ArtifactIdent identifier) throws IOException { + public List getReleases(String groupId, String artifactId) throws IOException { if(releaseListData.isEmpty()){ buildMap(); } - if(releaseListData.containsKey(identifier.getGA())){ - return releaseListData.get(identifier.getGA()); + final String ga = groupId + ":" + artifactId; + + if(releaseListData.containsKey(ga)){ + return releaseListData.get(ga); } else { - fail("No mock release data available for " + identifier.getGA()); + fail("No mock release data available for " + ga); return null; } } diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index b923497..70d254f 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -119,11 +119,12 @@ void resolveDependencies() { IReleaseListProvider mockProvider = new IReleaseListProvider() { @Override - public List getReleases(ArtifactIdent identifier) throws IOException { - if(releaseListData.containsKey(identifier.getGA())){ - return releaseListData.get(identifier.getGA()); + public List getReleases(String groupId, String artifactId) throws IOException { + final String ga = groupId + ":" + artifactId; + if(releaseListData.containsKey(ga)){ + return releaseListData.get(ga); } else { - fail("No mock release data available for " + identifier.getGA()); + fail("No mock release data available for " + ga); return null; } } @@ -251,7 +252,7 @@ void resolveVersionRanges() { IReleaseListProvider mockProvider = new IReleaseListProvider() { @Override - public List getReleases(ArtifactIdent identifier) throws IOException { + public List getReleases(String groupId, String artifactId) throws IOException { return versionsAvailable; } }; diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java index c067d54..e5fb53b 100644 --- a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -59,6 +59,22 @@ void uniqueGAs() { } } + @Test + @DisplayName("The Library Index Iterator must apply custom starting positions") + void customStartingPositions() { + iteratorUnderTest.setIndexPosition(42000); + + int cutoff = 1000; + int idx = 0; + + while(iteratorUnderTest.hasNext() && idx < cutoff){ + final String ga = iteratorUnderTest.next(); + assert(!ga.equals("yom:yom")); // Make sure we skipped the first library in index! + idx += 1; + } + + } + @Test @Disabled @DisplayName("The Library Index Iterator must terminate") From 1e9a5382eec75d5592afd161ab7ca7e19a31ba9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 20 Oct 2025 17:11:30 +0200 Subject: [PATCH 12/55] Begin test implementation for new analysis --- .../analyses/MavenCentralLibraryAnalysis.java | 9 +- .../java/org/tudo/sse/utils/CLIParser.java | 10 +- .../tudo/sse/utils/LibraryIndexIterator.java | 31 +++-- .../MavenCentralLibraryAnalysisTest.java | 106 ++++++++++++++++++ .../sse/utils/LibraryIndexIteratorTest.java | 4 +- .../utils/MavenCentralAnalysisFactory.java | 18 +++ 6 files changed, 159 insertions(+), 19 deletions(-) create mode 100644 src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 6933f18..84088c2 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -15,9 +15,7 @@ import org.tudo.sse.utils.LibraryIndexIterator; import org.tudo.sse.utils.MavenCentralRepository; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; @@ -177,7 +175,8 @@ private Iterator getGaIterator(){ } } else { try { - final LibraryIndexIterator indexIterator = new LibraryIndexIterator(new URI(MavenCentralRepository.RepoBasePath)); + final LibraryIndexIterator indexIterator = new LibraryIndexIterator( + new URI(MavenCentralRepository.RepoBasePath), config.progressOutputFile, config.progressWriteInterval); if(config.progressRestoreFile != null){ final long startingPosition = config.getProgressFromRestoreFile(); @@ -279,6 +278,8 @@ public void parseArguments(String[] args) { case "--coordinates": if(gaInputListFile != null) throw new CLIException("Input file cannot be set twice!"); + if(progressRestoreFile != null) + throw new CLIException("Cannot use custom input list when progress restore file is set!"); gaInputListFile = nextArgAsRegularFileReference(args, i); break; case "--name": diff --git a/src/main/java/org/tudo/sse/utils/CLIParser.java b/src/main/java/org/tudo/sse/utils/CLIParser.java index c8bc0f6..285fbb1 100644 --- a/src/main/java/org/tudo/sse/utils/CLIParser.java +++ b/src/main/java/org/tudo/sse/utils/CLIParser.java @@ -34,10 +34,10 @@ protected long[] nextArgAsLongPair(String[] args, int i) throws CLIException { throw(new CLIException(args[i], e.getMessage())); } } else { - throw(new CLIException(args[i], "Correct format: first:second")); + throw(new CLIException("Correct format: first:second", args[i])); } } else { - throw(new CLIException(args[i], "Missing argument: first:second")); + throw(new CLIException("Missing argument: first:second", args[i])); } return toReturn; } @@ -46,7 +46,7 @@ protected Path nextArgAsPath(String[] args, int i) throws CLIException { if(i + 1 < args.length) { return Paths.get(args[i + 1]); } else { - throw(new CLIException(args[i], "Missing argument: path/to/file")); + throw(new CLIException("Missing argument: path/to/file", args[i])); } } @@ -55,7 +55,7 @@ protected Path nextArgAsRegularFileReference(String[] args, int i) throws CLIExc if(Files.isRegularFile(path)) return path; else - throw new CLIException(args[i], "Expected a regular file but got:" + path); + throw new CLIException("Expected an existing file but got: " + path, args[i]); } protected Path nextArgAsDirectoryReference(String[] args, int i) throws CLIException { @@ -63,7 +63,7 @@ protected Path nextArgAsDirectoryReference(String[] args, int i) throws CLIExcep if(Files.isDirectory(path)) return path; else - throw new CLIException(args[i], "Expected a directory but got:" + path); + throw new CLIException("Expected an existing directory but got: " + path, args[i]); } } diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 5b25d09..206eee4 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -2,10 +2,12 @@ import org.apache.maven.index.reader.ChunkReader; import org.apache.maven.index.reader.IndexReader; -import org.tudo.sse.IndexWalker; +import java.io.BufferedWriter; +import java.io.FileWriter; import java.io.IOException; import java.net.URI; +import java.nio.file.Path; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -14,21 +16,25 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { private final Set libraryHashesSeen; - private final URI baseUri; private final Iterator> entryIterator; private final IndexReader mavenIndexReader; private final ChunkReader mavenChunkReader; - private long currentLibraryIndexPosition; + private final Path progressOutputFilePath; + + final long progressSaveInteral; + private long lastPositionSaved; private long currentPosition; private String currentLibraryGA; private String nextLibraryGA; - public LibraryIndexIterator(URI baseUri) throws IOException { + public LibraryIndexIterator(URI baseUri, Path progressOutputFile, long progressSaveInterval) throws IOException { this.libraryHashesSeen = new HashSet<>(); - this.baseUri = baseUri; + + this.progressOutputFilePath = progressOutputFile; + this.progressSaveInteral = progressSaveInterval; // Build intermediate readers that must be kept open until iterator is done and resources can be closed this.mavenIndexReader = new IndexReader(null, new HttpResourceHandler(baseUri.resolve(".index/"))); @@ -37,7 +43,7 @@ public LibraryIndexIterator(URI baseUri) throws IOException { // Build the actual iterator for entries in the Maven Central index this.entryIterator = this.mavenChunkReader.iterator(); - this.currentLibraryIndexPosition = -1; + this.lastPositionSaved = -1; this.currentPosition = -1; this.currentLibraryGA = null; } @@ -50,7 +56,6 @@ public void setIndexPosition(long startIdx){ } while(this.currentPosition < startIdx); this.currentLibraryGA = getGAFromEntry(entry); - this.currentLibraryIndexPosition = this.currentPosition; } public boolean hasNext() { @@ -65,12 +70,10 @@ public boolean hasNext() { // Walk to the first actual entry while(currentLibraryGA == null && entryIterator.hasNext()){ currentLibraryGA = getGAFromEntry(nextEntry()); - currentLibraryIndexPosition = currentPosition; } } else { // If we have information about the next artifact, we just copy it this.currentLibraryGA = this.nextLibraryGA; - this.currentLibraryIndexPosition = this.currentPosition; this.nextLibraryGA = null; } @@ -133,10 +136,20 @@ private String getGAFromEntry(Map entry){ } private Map nextEntry(){ + writeProgressIfNeeded(); currentPosition += 1; return entryIterator.next(); } + private void writeProgressIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved >= this.progressSaveInteral){ + try(BufferedWriter writer = new BufferedWriter(new FileWriter(this.progressOutputFilePath.toFile()))){ + writer.write(String.valueOf(currentPosition)); + } catch(IOException ignored){} + this.lastPositionSaved = currentPosition; + } + } + @Override public void close() throws Exception { if(this.mavenChunkReader != null){ diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java new file mode 100644 index 0000000..35c8d42 --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -0,0 +1,106 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.utils.MavenCentralAnalysisFactory; + +import java.nio.file.Paths; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class MavenCentralLibraryAnalysisTest { + + final MavenCentralLibraryAnalysis emptyAnalysis = + MavenCentralAnalysisFactory.buildEmptyLibraryAnalysisWithNoRequirements(); + + @Test + @DisplayName("The CLI parser must parse common argument values correctly") + void parseCLIRegular(){ + final String validArgs = "-st 20:10000 -ip pom.xml --name marin-progress --multi 12"; + final String[] argsArray = validArgs.split(" "); + + emptyAnalysis.config.parseArguments(argsArray); + + assert(emptyAnalysis.config.multipleThreads); + assertEquals(12, emptyAnalysis.config.threadCount); + assertEquals(Paths.get("pom.xml"), emptyAnalysis.config.progressRestoreFile); + assertEquals(Paths.get("marin-progress"), emptyAnalysis.config.progressOutputFile); + assertEquals(20, emptyAnalysis.config.skip); + assertEquals(10000, emptyAnalysis.config.take); + assertNull(emptyAnalysis.config.gaInputListFile); + } + + @Test + @DisplayName("The CLI parser must set the correct defaults for empty arguments") + void parseCLIDefaults(){ + emptyAnalysis.config.parseArguments(new String[]{}); + + assertFalse(emptyAnalysis.config.multipleThreads); + assertEquals(1, emptyAnalysis.config.threadCount); + assertNull(emptyAnalysis.config.progressRestoreFile); + assertEquals(Paths.get("marin-progress"), emptyAnalysis.config.progressOutputFile); + assertEquals(-1, emptyAnalysis.config.skip); + assertEquals(-1, emptyAnalysis.config.take); + assertNull(emptyAnalysis.config.gaInputListFile); + } + + @Test + @DisplayName("The CLI parser must fail if input files do not exist") + void parseNonExistingFile(){ + final String validArgs = "-ip foo.input"; + final String[] argsArray = validArgs.split(" "); + + assertThrows(RuntimeException.class, () -> emptyAnalysis.config.parseArguments(argsArray)); + } + + @Test + @DisplayName("The CLI parser must accept pagination on custom GA input lists") + void parseCustomGA(){ + final String validArgs = "-st 20:10000 --coordinates pom.xml"; + final String[] argsArray = validArgs.split(" "); + + emptyAnalysis.config.parseArguments(argsArray); + + assertEquals(Paths.get("pom.xml"), emptyAnalysis.config.gaInputListFile); + assertEquals(20, emptyAnalysis.config.skip); + } + + @Test + @DisplayName("The CLI parser must not accept setting an index position on a custom GA input list") + void parseNonExistingIndex(){ + final String validArgs = "-ip pom.xml --coordinates pom.xml"; + final String[] argsArray = validArgs.split(" "); + + assertThrows(RuntimeException.class, () -> emptyAnalysis.config.parseArguments(argsArray)); + } + + @Test + void simpleIndexAnalysis(){ + + final List librariesHit = new java.util.ArrayList<>(List.of()); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + assertFalse(releases.isEmpty()); + assertNull(releases.get(0).getIndexInformation()); + assertNull(releases.get(0).getPomInformation()); + assertNull(releases.get(0).getJarInformation()); + System.out.println(libraryGA); + librariesHit.add(libraryGA); + } + }; + + final String args = "-st 0:3"; + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + assertEquals(3, librariesHit.size()); + assert(librariesHit.contains("yom:yom")); + + } + +} diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java index e5fb53b..12daf43 100644 --- a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.*; import java.net.URI; +import java.nio.file.Paths; import java.util.HashSet; import java.util.Set; @@ -15,7 +16,8 @@ class LibraryIndexIteratorTest { @BeforeEach void setIndexIterator() { try { - iteratorUnderTest = new LibraryIndexIterator(new URI("https://repo1.maven.org/maven2/")); + iteratorUnderTest = new LibraryIndexIterator(new URI("https://repo1.maven.org/maven2/"), + Paths.get("lastIndexProcessed"), 1000); } catch (Exception x) { fail(x); } } diff --git a/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java b/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java index 34b76ef..bcd0484 100644 --- a/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java +++ b/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java @@ -1,8 +1,11 @@ package org.tudo.sse.utils; import org.tudo.sse.MavenCentralAnalysis; +import org.tudo.sse.analyses.MavenCentralLibraryAnalysis; import org.tudo.sse.model.Artifact; +import java.util.List; + public class MavenCentralAnalysisFactory { public static MavenCentralAnalysis buildEmptyAnalysisWithNoRequirements() { @@ -26,4 +29,19 @@ public void analyzeArtifact(Artifact current) { } }; } + + public static MavenCentralLibraryAnalysis buildEmptyLibraryAnalysisWithNoRequirements(){ + return buildLibraryAnalysisWithRequirements(false, false, false); + } + + private static MavenCentralLibraryAnalysis buildLibraryAnalysisWithRequirements(boolean requiresPom, + boolean requiresTransitives, + boolean requiresJar) { + return new MavenCentralLibraryAnalysis(requiresPom, requiresTransitives, requiresJar) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + + } + }; + } } From c4357508fe0f249926887ea09b57b0b81bc8d5eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Tue, 21 Oct 2025 10:33:50 +0200 Subject: [PATCH 13/55] Proper testing for library analysis, bugfixes --- .../org/tudo/sse/MavenCentralAnalysis.java | 16 +- .../analyses/MavenCentralLibraryAnalysis.java | 7 +- .../IndexProcessingMessage.java | 26 --- .../tudo/sse/multithreading/QueueActor.java | 2 +- .../WorkloadIsFinalMessage.java | 18 ++ .../tudo/sse/utils/LibraryIndexIterator.java | 45 ++-- .../MavenCentralLibraryAnalysisTest.java | 218 +++++++++++++++++- src/test/resources/library-names-invalid.txt | 2 + src/test/resources/library-names-valid.txt | 5 + 9 files changed, 280 insertions(+), 59 deletions(-) delete mode 100644 src/main/java/org/tudo/sse/multithreading/IndexProcessingMessage.java create mode 100644 src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java create mode 100644 src/test/resources/library-names-invalid.txt create mode 100644 src/test/resources/library-names-valid.txt diff --git a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/MavenCentralAnalysis.java index bcf1caf..996946b 100644 --- a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/MavenCentralAnalysis.java @@ -8,7 +8,7 @@ import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.multithreading.ProcessIdentifierMessage; -import org.tudo.sse.multithreading.IndexProcessingMessage; +import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; @@ -369,7 +369,7 @@ public List walkAllIndexes(IndexIterator indexIterator) throws IOExcep } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); @@ -405,7 +405,7 @@ public List walkPaginated(long take, IndexIterator indexIterator) thro } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); @@ -447,7 +447,7 @@ public List walkDates(long since, long until, IndexIterator indexItera } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); @@ -489,7 +489,7 @@ public List lazyWalkAllIndexes(IndexIterator indexIterator) throw } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); @@ -523,7 +523,7 @@ public List lazyWalkPaginated(long take, IndexIterator indexItera } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); return idents; @@ -563,7 +563,7 @@ public List lazyWalkDates(long since, long until, IndexIterator i } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); @@ -645,7 +645,7 @@ public List readIdentsIn() { } if(setupInfo.isMulti()) { - queueActorRef.tell(new IndexProcessingMessage("Finished"), ActorRef.noSender()); + queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } writeLastProcessed(i, setupInfo.getName()); diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 84088c2..a331aaa 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -6,6 +6,7 @@ import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.multithreading.ProcessLibraryMessage; import org.tudo.sse.multithreading.QueueActor; import org.tudo.sse.resolution.ResolverFactory; @@ -79,7 +80,7 @@ public final void runAnalysis(String[] args){ if(config.take >= 0) log.info("Taking {} library names", config.take); - while(entriesTaken < config.take && gaIterator.hasNext()){ + while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ processEntry(gaIterator.next()); entriesTaken += 1; @@ -97,7 +98,9 @@ public final void runAnalysis(String[] args){ // Close actor system if it was used if(system != null){ - try{ + try { + // Tell the queue that no more work items will follow + this.queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); system.getWhenTerminated().toCompletableFuture().get(); } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} } diff --git a/src/main/java/org/tudo/sse/multithreading/IndexProcessingMessage.java b/src/main/java/org/tudo/sse/multithreading/IndexProcessingMessage.java deleted file mode 100644 index b68b450..0000000 --- a/src/main/java/org/tudo/sse/multithreading/IndexProcessingMessage.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.tudo.sse.multithreading; - -/** - * This class contains a plain message sent from the Index to the Queue class to signal that it finished processing indexes. - */ -public class IndexProcessingMessage { - - private final String message; - - /** - * Creates a new instance of this class with the given message - * - * @param message The message to pass to the processing queue - */ - public IndexProcessingMessage(String message) { - this.message = message; - } - - /** - * Retrieves the message string contained in this class - * @return Message as plain string - */ - public String getMessage() { - return message; - } -} diff --git a/src/main/java/org/tudo/sse/multithreading/QueueActor.java b/src/main/java/org/tudo/sse/multithreading/QueueActor.java index 7a1b383..75ef339 100644 --- a/src/main/java/org/tudo/sse/multithreading/QueueActor.java +++ b/src/main/java/org/tudo/sse/multithreading/QueueActor.java @@ -72,7 +72,7 @@ public Receive createReceive() { } }) - .match(IndexProcessingMessage.class, indexProcessingMessage -> { + .match(WorkloadIsFinalMessage.class, workloadIsFinalMessage -> { indexFinished = true; if(curNumResolvers.get() == 0) { system.terminate(); diff --git a/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java b/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java new file mode 100644 index 0000000..9e04131 --- /dev/null +++ b/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java @@ -0,0 +1,18 @@ +package org.tudo.sse.multithreading; + +/** + * This class is a marker message sent to the queue actor. It signals that no more work items will be scheduled in the + * current analysis run. + */ +public class WorkloadIsFinalMessage { + + private static WorkloadIsFinalMessage _instance; + + private WorkloadIsFinalMessage() {} + + + public static WorkloadIsFinalMessage getInstance(){ + if(_instance == null) _instance = new WorkloadIsFinalMessage(); + return _instance; + } +} diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 206eee4..1cafbb2 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -23,7 +23,7 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { private final Path progressOutputFilePath; - final long progressSaveInteral; + final long progressSaveInternal; private long lastPositionSaved; private long currentPosition; @@ -34,7 +34,7 @@ public LibraryIndexIterator(URI baseUri, Path progressOutputFile, long progressS this.libraryHashesSeen = new HashSet<>(); this.progressOutputFilePath = progressOutputFile; - this.progressSaveInteral = progressSaveInterval; + this.progressSaveInternal = progressSaveInterval; // Build intermediate readers that must be kept open until iterator is done and resources can be closed this.mavenIndexReader = new IndexReader(null, new HttpResourceHandler(baseUri.resolve(".index/"))); @@ -56,6 +56,7 @@ public void setIndexPosition(long startIdx){ } while(this.currentPosition < startIdx); this.currentLibraryGA = getGAFromEntry(entry); + findNextGA(); } public boolean hasNext() { @@ -77,23 +78,7 @@ public boolean hasNext() { this.nextLibraryGA = null; } - // Walk up to the next entry that has a different GA (is a different library) - String nextNewGA = null; - - while(entryIterator.hasNext()){ - final Map nextEntry = nextEntry(); - final String nextGA = getGAFromEntry(nextEntry); - - // Silently ignore malformed entries - if(nextGA == null) continue; - - if(!currentLibraryGA.equals(nextGA) && isNewLibrary(nextGA)){ - nextNewGA = nextGA; - break; - } - } - - this.nextLibraryGA = nextNewGA; + findNextGA(); } return this.nextLibraryGA != null; @@ -135,6 +120,26 @@ private String getGAFromEntry(Map entry){ } else return null; } + private void findNextGA(){ + // Walk up to the next entry that has a different GA (is a different library) + String nextNewGA = null; + + while(entryIterator.hasNext()){ + final Map nextEntry = nextEntry(); + final String nextGA = getGAFromEntry(nextEntry); + + // Silently ignore malformed entries + if(nextGA == null) continue; + + if(!currentLibraryGA.equals(nextGA) && isNewLibrary(nextGA)){ + nextNewGA = nextGA; + break; + } + } + + this.nextLibraryGA = nextNewGA; + } + private Map nextEntry(){ writeProgressIfNeeded(); currentPosition += 1; @@ -142,7 +147,7 @@ private Map nextEntry(){ } private void writeProgressIfNeeded(){ - if(this.currentPosition - this.lastPositionSaved >= this.progressSaveInteral){ + if(this.currentPosition - this.lastPositionSaved >= this.progressSaveInternal){ try(BufferedWriter writer = new BufferedWriter(new FileWriter(this.progressOutputFilePath.toFile()))){ writer.write(String.valueOf(currentPosition)); } catch(IOException ignored){} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index 35c8d42..f555ef8 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -1,12 +1,18 @@ package org.tudo.sse.analyses; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.tudo.sse.ArtifactFactory; import org.tudo.sse.model.Artifact; import org.tudo.sse.utils.MavenCentralAnalysisFactory; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; @@ -15,6 +21,11 @@ public class MavenCentralLibraryAnalysisTest { final MavenCentralLibraryAnalysis emptyAnalysis = MavenCentralAnalysisFactory.buildEmptyLibraryAnalysisWithNoRequirements(); + @AfterEach + public void cleanup(){ + ArtifactFactory.artifacts.clear(); + } + @Test @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ @@ -77,9 +88,10 @@ void parseNonExistingIndex(){ } @Test + @DisplayName("An analysis with no requirements must not produce information instances") void simpleIndexAnalysis(){ - final List librariesHit = new java.util.ArrayList<>(List.of()); + final List librariesHit = new java.util.ArrayList<>(); final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { @Override @@ -88,7 +100,6 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertNull(releases.get(0).getIndexInformation()); assertNull(releases.get(0).getPomInformation()); assertNull(releases.get(0).getJarInformation()); - System.out.println(libraryGA); librariesHit.add(libraryGA); } }; @@ -100,7 +111,210 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertEquals(3, librariesHit.size()); assert(librariesHit.contains("yom:yom")); + } + + @Test + @DisplayName("An analysis with JAR requirements must produce JAR information instances") + void simpleIndexWithJarAnalysis(){ + + final List librariesHit = new java.util.ArrayList<>(); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, true) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + assertFalse(releases.isEmpty()); + assertNull(releases.get(0).getIndexInformation()); + assertNull(releases.get(0).getPomInformation()); + assertNotNull(releases.get(0).getJarInformation()); + librariesHit.add(libraryGA); + } + }; + + final String args = "-st 3:3"; + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + assertEquals(3, librariesHit.size()); + assertFalse(librariesHit.contains("yom:yom")); + assert(librariesHit.contains("yan:yan")); + } + + @Test + @DisplayName("An analysis with POM requirements must produce POM information instances") + void simpleIndexWithPomAnalysis(){ + + final List librariesHit = new java.util.ArrayList<>(); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(true, true, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + assertFalse(releases.isEmpty()); + assertNull(releases.get(0).getIndexInformation()); + assertNotNull(releases.get(0).getPomInformation()); + assertNull(releases.get(0).getJarInformation()); + librariesHit.add(libraryGA); + } + }; + + final String args = "-st 5000:10"; + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + assertEquals(10, librariesHit.size()); + assertFalse(librariesHit.contains("yom:yom")); + assertFalse(librariesHit.contains("yan:yan")); + } + + @Test + @DisplayName("An analysis with multithreading must not miss any libraries") + void parallelIndexAnalysis(){ + + final AtomicInteger count = new AtomicInteger(0); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + assertFalse(releases.isEmpty()); + assertNull(releases.get(0).getIndexInformation()); + assertNull(releases.get(0).getPomInformation()); + assertNull(releases.get(0).getJarInformation()); + count.incrementAndGet(); + } + }; + + final String args = "-st 5000:100 --multi 8"; + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + assertEquals(100, count.get()); + } + + @Test + @DisplayName("An analysis with input file must not miss any libraries") + void analysisFromFile() throws IOException { + + final Path validLibraryInput = testResource("library-names-valid.txt"); + + assertNotNull(validLibraryInput); + assert(Files.exists(validLibraryInput)); + + final List expectedLibraries = Files.readAllLines(validLibraryInput); + + final List librariesHit = new java.util.ArrayList<>(); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + assertFalse(releases.isEmpty()); + assertNull(releases.get(0).getIndexInformation()); + assertNull(releases.get(0).getPomInformation()); + assertNull(releases.get(0).getJarInformation()); + librariesHit.add(libraryGA); + } + }; + + final String args = "--coordinates " + validLibraryInput.toAbsolutePath(); + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + assertEquals(expectedLibraries, librariesHit); + } + + @Test + @DisplayName("An analysis with input file must allow pagination") + void analysisFromFileWithPagination() throws IOException { + + final Path validLibraryInput = testResource("library-names-valid.txt"); + + assertNotNull(validLibraryInput); + assert(Files.exists(validLibraryInput)); + + final List expectedLibraries = Files.readAllLines(validLibraryInput); + + final List librariesHit = new java.util.ArrayList<>(); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + assertFalse(releases.isEmpty()); + assertNull(releases.get(0).getIndexInformation()); + assertNull(releases.get(0).getPomInformation()); + assertNull(releases.get(0).getJarInformation()); + librariesHit.add(libraryGA); + } + }; + + final String args = "-st 1:2 --coordinates " + validLibraryInput.toAbsolutePath(); + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + assertEquals(2, librariesHit.size()); + assert(librariesHit.contains(expectedLibraries.get(1))); + assert(librariesHit.contains(expectedLibraries.get(2))); + } + + @Test + @DisplayName("An analysis with input file must not fail on invalid inputs") + void analysisFromInvalidFile() throws IOException { + + final Path validLibraryInput = testResource("library-names-invalid.txt"); + + assertNotNull(validLibraryInput); + assert (Files.exists(validLibraryInput)); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + fail("Analysis should not be executed on invalid input: " + libraryGA); + } + }; + + final String args = "--coordinates " + validLibraryInput.toAbsolutePath(); + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + } + + @Test + @DisplayName("An analysis must apply pagination on top of progress restore values") + void analysisWithRestoreAndPagination() throws IOException { + + final Path restoreFile = testResource("testingIndexPosition.txt"); + + assertNotNull(restoreFile); + assert (Files.exists(restoreFile)); + + final List librariesHit = new java.util.ArrayList<>(); + + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { + @Override + protected void analyzeLibrary(String libraryGA, List releases) { + librariesHit.add(libraryGA); + } + }; + + final String args = "-st 0:5 -ip " + restoreFile.toAbsolutePath(); + final String[] argsArray = args.split(" "); + + analysis.runAnalysis(argsArray); + + // When restoring progress - even though we do not skip anything - we do not want to see the first index entry! + assertEquals(5, librariesHit.size()); + assertFalse(librariesHit.contains("yom:yom")); + } + private Path testResource(String pathToResource){ + try { + return Path.of(getClass().getClassLoader().getResource(pathToResource).toURI()); + } catch (Exception x){ + fail("Test setup: Failed to load resource file " + pathToResource, x); + } + return null; } } diff --git a/src/test/resources/library-names-invalid.txt b/src/test/resources/library-names-invalid.txt new file mode 100644 index 0000000..1bcd3f2 --- /dev/null +++ b/src/test/resources/library-names-invalid.txt @@ -0,0 +1,2 @@ +not-valid-format +not:existing \ No newline at end of file diff --git a/src/test/resources/library-names-valid.txt b/src/test/resources/library-names-valid.txt new file mode 100644 index 0000000..34d9fc7 --- /dev/null +++ b/src/test/resources/library-names-valid.txt @@ -0,0 +1,5 @@ +com.google.code.gson:gson +org.scala-lang:scala-library +junit:junit +org.slf4j:slf4j-api +commons-logging:commons-logging \ No newline at end of file From 8424e40f859a8a127742f592f506f0b631682616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Tue, 21 Oct 2025 11:45:28 +0200 Subject: [PATCH 14/55] Ensure progress is properly saved in multithreaded settings --- .gitignore | 3 +- .../org/tudo/sse/MavenCentralAnalysis.java | 2 +- .../analyses/MavenCentralLibraryAnalysis.java | 58 ++++++++++++---- .../tudo/sse/multithreading/QueueActor.java | 68 ++++++++++++++++++- .../sse/multithreading/ResolverActor.java | 2 +- .../WorkItemFinishedMessage.java | 12 ++++ .../tudo/sse/utils/LibraryIndexIterator.java | 30 +++++--- .../MavenCentralLibraryAnalysisTest.java | 29 +++++--- .../sse/utils/LibraryIndexIteratorTest.java | 2 +- 9 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java diff --git a/.gitignore b/.gitignore index 1be7d23..34440dd 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ settings.xml #maven resolution files maven-resolution-files .DS_Store -.mvn \ No newline at end of file +.mvn +marin-progress \ No newline at end of file diff --git a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/MavenCentralAnalysis.java index 996946b..4fe130e 100644 --- a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/MavenCentralAnalysis.java @@ -270,7 +270,7 @@ public Map runAnalysis(String[] args) throws URISyntaxE if(setupInfo.isMulti()) { ActorSystem system = ActorSystem.create("my-system"); - queueActorRef = system.actorOf(QueueActor.props(setupInfo.getThreads(), system), "queueActor"); + queueActorRef = system.actorOf(QueueActor.props(setupInfo.getThreads(), system, 0,0, null), "queueActor"); if(setupInfo.getToCoordinates() == null) { indexProcessor(); diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index a331aaa..6ff5395 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -53,18 +53,18 @@ protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTrans @Override public final void runAnalysis(String[] args){ + // Obtain CLI arguments this.config.parseArguments(args); printRunInfo(); ActorSystem system = null; - if(this.config.multipleThreads){ - system = ActorSystem.create("marin-actors"); - this.queueActorRef = system.actorOf(QueueActor.props(config.threadCount, system), "marin-queue-actor"); - } - + // Get input iterator - this will either be from a configured input file or from the Maven Central Index. The + // iterator will produce strings of form :. If a progress restore file is specified, the + // progress will be applied to the index iterator automatically. Iterator gaIterator = getGaIterator(); + // If there are values to skip, we skip them manually if(config.skip > 0){ log.info("Skipping {} library names", config.skip); for(int i = 0; i < config.skip; i++){ @@ -75,18 +75,34 @@ public final void runAnalysis(String[] args){ } } + // If we want to use multiple threads, we initialize the queue actor that managers workers + if(this.config.multipleThreads){ + // Get the initial progress so the progress output file is up-to-date + long initialProgress = gaIterator instanceof LibraryIndexIterator ? + ((LibraryIndexIterator)gaIterator).getPosition() : config.skip; + system = ActorSystem.create("marin-actors"); + this.queueActorRef = system.actorOf(QueueActor.props(config.threadCount, system, initialProgress, + config.progressWriteInterval, config.progressOutputFile), "marin-queue-actor"); + } + long entriesTaken = 0; if(config.take >= 0) log.info("Taking {} library names", config.take); + // If specified, we only take the configured amount of entries. If not, we process as long as the iterator + // provides new entries while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ processEntry(gaIterator.next()); entriesTaken += 1; } - log.info("Processed a total of {} library names", entriesTaken); + // At this point all libraries are queued (multithreaded) or processed (single-threaded) + if(config.multipleThreads) + log.info("Queued a total of {} library names for processing", entriesTaken); + else + log.info("Processed a total of {} library names", entriesTaken); // Close index iterator if it was used if(gaIterator instanceof LibraryIndexIterator){ @@ -147,8 +163,14 @@ private Void process(String ga, String groupId, String artifactId){ libraryArtifacts.add(a); } - // Call the custom analysis implementation - analyzeLibrary(ga, libraryArtifacts); + try { + // Call the custom analysis implementation + analyzeLibrary(ga, libraryArtifacts); + } catch(Exception x){ + // Whatever the user-defined code does, we do not want to abort! + log.error("Unknown error in analysis implementation", x); + } + return null; } @@ -181,10 +203,15 @@ private Iterator getGaIterator(){ final LibraryIndexIterator indexIterator = new LibraryIndexIterator( new URI(MavenCentralRepository.RepoBasePath), config.progressOutputFile, config.progressWriteInterval); + // If we use multiple threads, the queue actor will store progress. If we use a single thread, the + // iterator must store the progress. This is because we want to store progress after the analysis is + // actually done, which the iterator cannot know precisely in the multithreaded setting. + indexIterator.setSaveProgress(!config.multipleThreads); + if(config.progressRestoreFile != null){ final long startingPosition = config.getProgressFromRestoreFile(); log.info("Restoring previous progress (index position {})", startingPosition); - indexIterator.setIndexPosition(startingPosition); + indexIterator.setPosition(startingPosition); } return indexIterator; @@ -254,7 +281,7 @@ public LibraryAnalysisConfiguration() { progressRestoreFile = null; multipleThreads = false; threadCount = 1; - progressWriteInterval = 1000; + progressWriteInterval = 100; } @Override @@ -270,7 +297,7 @@ public void parseArguments(String[] args) { skip = skipTake[0]; take = skipTake[1]; break; - case "-ip": + case "--progress-restore-file": if(gaInputListFile != null) throw new CLIException("Cannot restore index position when a custom input list is used!"); if(progressRestoreFile != null) @@ -278,6 +305,12 @@ public void parseArguments(String[] args) { progressRestoreFile = nextArgAsRegularFileReference(args, i); break; + case "--save-progress-interval": + progressWriteInterval = nextArgAsInt(args, i); + break; + case "--progress-output-file": + progressOutputFile = nextArgAsPath(args, i); + break; case "--coordinates": if(gaInputListFile != null) throw new CLIException("Input file cannot be set twice!"); @@ -285,9 +318,6 @@ public void parseArguments(String[] args) { throw new CLIException("Cannot use custom input list when progress restore file is set!"); gaInputListFile = nextArgAsRegularFileReference(args, i); break; - case "--name": - progressOutputFile = nextArgAsPath(args, i); - break; case "--multi": final int threads = nextArgAsInt(args, i); multipleThreads = threads > 1; diff --git a/src/main/java/org/tudo/sse/multithreading/QueueActor.java b/src/main/java/org/tudo/sse/multithreading/QueueActor.java index 75ef339..a276e8b 100644 --- a/src/main/java/org/tudo/sse/multithreading/QueueActor.java +++ b/src/main/java/org/tudo/sse/multithreading/QueueActor.java @@ -5,9 +5,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; import java.util.LinkedList; import java.util.Queue; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** * This class represents the processing queue that resolves jobs and sends them to different threads. @@ -22,28 +27,43 @@ public class QueueActor extends AbstractActor { private boolean indexFinished = false; private final ActorSystem system; + private final AtomicLong workItemsCompleted = new AtomicLong(0L); + private final long progressSaveInterval; + private long lastProgressSaved = 0L; + private final Path progressOutputFilePath; + private static final Logger log = LogManager.getLogger(QueueActor.class); /** * Creates a new processing queue with the given number of actors and the given actor system. * @param numResolverActors Number of ResolverActor instances that will process jobs * @param system The underlying ActorSystem + * @param initialWorkItemPosition The number of work items that have been skipped due to progress restore or skip + * @param progressSaveInterval The number of completed work items after which to save progress + * @param progressOutputFilePath The file path to which to write progress to */ - public QueueActor(int numResolverActors, ActorSystem system) { + public QueueActor(int numResolverActors, ActorSystem system, long initialWorkItemPosition, long progressSaveInterval, Path progressOutputFilePath) { this.numResolverActors = numResolverActors; this.system = system; this.curNumResolvers = new AtomicInteger(0); this.jobQueue = new LinkedList<>(); + + this.progressSaveInterval = progressSaveInterval; + this.progressOutputFilePath = progressOutputFilePath; + this.workItemsCompleted.set(initialWorkItemPosition); } /** * Creates the properties needed to initialize an actor instance of this queue * @param numResolverActors The number of ResolverActor instances that shall be used to process jobs * @param system The underlying ActorSystem + * @param initialWorkItemPosition The number of work items that have been skipped due to progress restore or skip + * @param progressSaveInterval The number of completed work items after which to save progress + * @param progressOutputFilePath The file path to which to write progress to * @return The AKKA actor properties */ - public static Props props(int numResolverActors, ActorSystem system) { - return Props.create(QueueActor.class, () -> new QueueActor(numResolverActors, system)); + public static Props props(int numResolverActors, ActorSystem system, long initialWorkItemPosition, long progressSaveInterval, Path progressOutputFilePath) { + return Props.create(QueueActor.class, () -> new QueueActor(numResolverActors, system, initialWorkItemPosition, progressSaveInterval, progressOutputFilePath)); } @Override @@ -72,6 +92,31 @@ public Receive createReceive() { } }) + .match(WorkItemFinishedMessage.class, workItemFinishedMessage -> { + // Track completion of work items + workItemsCompleted.incrementAndGet(); + // Write progress file if needed + writeProgressIfNeeded(); + + // Distribute next job to worker that is now free, or kill worker if no jobs are left + synchronized (jobQueue){ + if(!jobQueue.isEmpty()) { + getSender().tell(jobQueue.remove(), getSelf()); + if(jobQueue.size() % 10 == 0) log.trace("Distributed a job, queue size " + jobQueue.size()); + } else { + synchronized(curNumResolvers) { + if(indexFinished && curNumResolvers.get() == 1) { + log.trace("Shutting down system"); + system.terminate(); + } else { + log.trace("Killing a worker thread"); + getSender().tell(PoisonPill.getInstance(), getSelf()); + curNumResolvers.decrementAndGet(); + } + } + } + } + }) .match(WorkloadIsFinalMessage.class, workloadIsFinalMessage -> { indexFinished = true; if(curNumResolvers.get() == 0) { @@ -94,4 +139,21 @@ private void forwardToResolvers(WorkItem message){ } } + private void writeProgressIfNeeded(){ + final boolean needsWrite = (this.workItemsCompleted.get() - this.lastProgressSaved) > this.progressSaveInterval; + + if(needsWrite){ + synchronized (this.progressOutputFilePath){ + // Double-Check for efficient synchronization + final boolean doesNeedWrite = (this.workItemsCompleted.get() - this.lastProgressSaved) > this.progressSaveInterval; + if(doesNeedWrite){ + try(BufferedWriter writer = new BufferedWriter(new FileWriter(this.progressOutputFilePath.toFile()))){ + writer.write(String.valueOf(this.workItemsCompleted)); + } catch(IOException ignored){} + this.lastProgressSaved = this.workItemsCompleted.get(); + } + } + } + } + } diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index 2316083..c5f5704 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -34,7 +34,7 @@ public Receive createReceive() { }) .match(ProcessLibraryMessage.class, message -> { message.getProcessEntryCallback().get(); - getSender().tell("Finished", getSelf()); + getSender().tell(WorkItemFinishedMessage.getInstance(), getSelf()); }).build(); } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java b/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java new file mode 100644 index 0000000..7707157 --- /dev/null +++ b/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java @@ -0,0 +1,12 @@ +package org.tudo.sse.multithreading; + +public class WorkItemFinishedMessage { + + private static final WorkItemFinishedMessage _instance = new WorkItemFinishedMessage(); + + private WorkItemFinishedMessage() {} + + public static WorkItemFinishedMessage getInstance() { + return _instance; + } +} diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 1cafbb2..5d59b25 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -26,6 +26,7 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { final long progressSaveInternal; private long lastPositionSaved; private long currentPosition; + private boolean saveProgress; private String currentLibraryGA; private String nextLibraryGA; @@ -33,6 +34,7 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { public LibraryIndexIterator(URI baseUri, Path progressOutputFile, long progressSaveInterval) throws IOException { this.libraryHashesSeen = new HashSet<>(); + this.saveProgress = true; this.progressOutputFilePath = progressOutputFile; this.progressSaveInternal = progressSaveInterval; @@ -48,15 +50,18 @@ public LibraryIndexIterator(URI baseUri, Path progressOutputFile, long progressS this.currentLibraryGA = null; } - public void setIndexPosition(long startIdx){ - Map entry; + public void setSaveProgress(boolean saveProgress){ + this.saveProgress = saveProgress; + } - do { - entry = nextEntry(); - } while(this.currentPosition < startIdx); + public void setPosition(long startIdx){ + while(this.currentPosition < startIdx && this.hasNext()){ + this.next(); + } + } - this.currentLibraryGA = getGAFromEntry(entry); - findNextGA(); + public long getPosition() { + return this.currentPosition; } public boolean hasNext() { @@ -89,6 +94,10 @@ public String next() { final String valueToReturn = this.currentLibraryGA; this.libraryHashesSeen.add(this.currentLibraryGA.hashCode()); this.currentLibraryGA = null; + + writeProgressIfNeeded(); + currentPosition += 1; + return valueToReturn; } else throw new IllegalStateException("No libraries left on iterator (position " + currentPosition + ")"); } @@ -141,15 +150,14 @@ private void findNextGA(){ } private Map nextEntry(){ - writeProgressIfNeeded(); - currentPosition += 1; return entryIterator.next(); } private void writeProgressIfNeeded(){ - if(this.currentPosition - this.lastPositionSaved >= this.progressSaveInternal){ + if(this.saveProgress && this.currentPosition - this.lastPositionSaved >= this.progressSaveInternal){ try(BufferedWriter writer = new BufferedWriter(new FileWriter(this.progressOutputFilePath.toFile()))){ - writer.write(String.valueOf(currentPosition)); + // Write position of last GA, as it has now been completed! + writer.write(String.valueOf(currentPosition - 1)); } catch(IOException ignored){} this.lastPositionSaved = currentPosition; } diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index f555ef8..aaed42c 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -29,7 +29,7 @@ public void cleanup(){ @Test @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ - final String validArgs = "-st 20:10000 -ip pom.xml --name marin-progress --multi 12"; + final String validArgs = "-st 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --multi 12 --save-progress-interval 20"; final String[] argsArray = validArgs.split(" "); emptyAnalysis.config.parseArguments(argsArray); @@ -40,6 +40,7 @@ void parseCLIRegular(){ assertEquals(Paths.get("marin-progress"), emptyAnalysis.config.progressOutputFile); assertEquals(20, emptyAnalysis.config.skip); assertEquals(10000, emptyAnalysis.config.take); + assertEquals(20, emptyAnalysis.config.progressWriteInterval); assertNull(emptyAnalysis.config.gaInputListFile); } @@ -54,13 +55,14 @@ void parseCLIDefaults(){ assertEquals(Paths.get("marin-progress"), emptyAnalysis.config.progressOutputFile); assertEquals(-1, emptyAnalysis.config.skip); assertEquals(-1, emptyAnalysis.config.take); + assertEquals(100, emptyAnalysis.config.progressWriteInterval); assertNull(emptyAnalysis.config.gaInputListFile); } @Test @DisplayName("The CLI parser must fail if input files do not exist") void parseNonExistingFile(){ - final String validArgs = "-ip foo.input"; + final String validArgs = "--progress-restore-file foo.input"; final String[] argsArray = validArgs.split(" "); assertThrows(RuntimeException.class, () -> emptyAnalysis.config.parseArguments(argsArray)); @@ -81,7 +83,7 @@ void parseCustomGA(){ @Test @DisplayName("The CLI parser must not accept setting an index position on a custom GA input list") void parseNonExistingIndex(){ - final String validArgs = "-ip pom.xml --coordinates pom.xml"; + final String validArgs = "--progress-restore-file pom.xml --coordinates pom.xml"; final String[] argsArray = validArgs.split(" "); assertThrows(RuntimeException.class, () -> emptyAnalysis.config.parseArguments(argsArray)); @@ -142,7 +144,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { @Test @DisplayName("An analysis with POM requirements must produce POM information instances") - void simpleIndexWithPomAnalysis(){ + void simpleIndexWithPomAnalysis() throws IOException{ final List librariesHit = new java.util.ArrayList<>(); @@ -157,7 +159,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 5000:10"; + final String args = "-st 5000:10 --save-progress-interval 5"; final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); @@ -165,11 +167,15 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertEquals(10, librariesHit.size()); assertFalse(librariesHit.contains("yom:yom")); assertFalse(librariesHit.contains("yan:yan")); + + long progress = Long.parseLong(Files.readString(analysis.config.progressOutputFile)); + + assert((progress - 5000) < analysis.config.progressWriteInterval); } @Test @DisplayName("An analysis with multithreading must not miss any libraries") - void parallelIndexAnalysis(){ + void parallelIndexAnalysis() throws IOException{ final AtomicInteger count = new AtomicInteger(0); @@ -184,12 +190,17 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 5000:100 --multi 8"; + final String args = "-st 5000:500 --multi 8"; final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); - assertEquals(100, count.get()); + assertEquals(500, count.get()); + + long progress = Long.parseLong(Files.readString(analysis.config.progressOutputFile)); + + // Skip values must be part of the progress! Last progress write must have been somewhere after 5400 (interval 100) + assert(progress >= 5400); } @Test @@ -298,7 +309,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 0:5 -ip " + restoreFile.toAbsolutePath(); + final String args = "-st 0:5 --progress-restore-file " + restoreFile.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java index 12daf43..2d910a4 100644 --- a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -64,7 +64,7 @@ void uniqueGAs() { @Test @DisplayName("The Library Index Iterator must apply custom starting positions") void customStartingPositions() { - iteratorUnderTest.setIndexPosition(42000); + iteratorUnderTest.setPosition(42000); int cutoff = 1000; int idx = 0; From fa183a3d18550278b9fc199bdb56bd1a5edc5044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 22 Oct 2025 11:20:41 +0200 Subject: [PATCH 15/55] Use new parsing infrastructure and common configuration class in both analysis types. Move analysis to common package. Update README. Renamed CLI parameters. --- README.md | 72 ++-- .../java/org/tudo/sse/CliInformation.java | 228 ----------- ...nalysis.java => MavenCentralAnalysis.java} | 10 +- .../MavenCentralArtifactAnalysis.java} | 296 +++++--------- .../analyses/MavenCentralLibraryAnalysis.java | 177 +++------ .../ProcessIdentifierMessage.java | 8 +- .../tudo/sse/utils/ArtifactConfigParser.java | 82 ++++ ...LIParser.java => CLIParsingUtilities.java} | 14 +- .../tudo/sse/utils/CommonConfigParser.java | 88 ++++ .../tudo/sse/utils/LibraryIndexIterator.java | 32 +- .../tudo/sse/MavenCentralAnalysisTest.java | 312 --------------- .../MavenCentralArtifactAnalysisTest.java | 375 ++++++++++++++++++ .../MavenCentralLibraryAnalysisTest.java | 116 +++--- .../sse/utils/LibraryIndexIteratorTest.java | 3 +- .../utils/MavenCentralAnalysisFactory.java | 14 +- src/test/resources/stop.txt | 2 +- 16 files changed, 825 insertions(+), 1004 deletions(-) delete mode 100644 src/main/java/org/tudo/sse/CliInformation.java rename src/main/java/org/tudo/sse/analyses/{ArtifactAnalysis.java => MavenCentralAnalysis.java} (88%) rename src/main/java/org/tudo/sse/{MavenCentralAnalysis.java => analyses/MavenCentralArtifactAnalysis.java} (62%) create mode 100644 src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java rename src/main/java/org/tudo/sse/utils/{CLIParser.java => CLIParsingUtilities.java} (79%) create mode 100644 src/main/java/org/tudo/sse/utils/CommonConfigParser.java delete mode 100644 src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java create mode 100644 src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java diff --git a/README.md b/README.md index d479b18..7f17eb7 100644 --- a/README.md +++ b/README.md @@ -11,43 +11,39 @@ Add the following dependency to your `pom.xml` to add MARIN to your project: ``` ## Required Java Version -The Maven Research Interface requires Java 11. - -## MavenCentralAnalysis -An abstract class that can be extended to easily run a multitude of analyses on artifacts of the Maven Central repository. Boolean values are used to control which type of information to collect (index, pom, jar) and a cli is in place to configure other aspects of the run. - -The boolean values to be set include: -- index: sets if metadata from the Maven Central Index should be collected -- pom: sets if pom artifacts are to be resolved -- transitive: sets if transitive dependencies should be resolved if pom artifacts are also being resolved -- jar: sets if jar artifacts are to be resolved - -The CLI includes the following: -- skip/take - - description: Set how many artifacts to skip from the beginning of the index, and how many indexes to attempt to resolve. - - usage: ```-st skip:take``` -- since/until - - description: Filter the artifact identifiers collected by a given lastModified range. - - usage: ```-su since:until ``` -- coordinates - - description: Specify a path to a file containing artifact identifiers to resolve. - - usage: ```--coordinates path/to/file``` -- lastIndexProcessed - - description: Specify a file path containing which index was last processed, in order to skip already processed indexes - - usage: ```-ip path/to/file ``` -- name - - description: Specify a file path / file name to write the lastIndexProcessed information out to. - - usage: ```--name path/to/file ``` -- output - - description: Specify whether to write files that resolution is being performed on out to a directory. - - usage: ```--output path/to/dir ``` -- multi - - description: Specify to run the multithreaded implementation, and how many threads should be used - - usage: ```--multi threads``` +You will need Java 11 or higher to build MARIN yourself. + +## Writing an Analysis +MARIN provides two main classes that can be extended to write custom large-scale analyses: + +* `MavenCentralArtifactAnalysis` This class can be extended to write an analysis that processes individual *artifacts* (meaning GAV-triple) hosted on Maven Central. Override the abstract method `void analyzeArtifact(Artifact current)` to define how a single artifact shall be analyzed. +* `MavenCentralLibraryAnalysis` This class can be extended to write analyses that process entire libraries (meaning GA-tuple) hosted on Maven Central at a time. Override the abstract method `void analyzeLibrary(String libraryGA, List releases)` to define how the library shall be analyzed. + +Extending both classes requires you to set three boolean values (when invoking the `super` constructor): + +* `boolean resolvePom` If set to true, the analysis will automatically download and parse `pom.xml` files, build a `PomInformation` object and annotate `Artifact` objects with it. +* `boolean resolveTransitives` If set to true, the analysis will automatically download and parse all `pom.xml` files that are transitively referenced by the artifact's root `pom.xml` file. These references mainly include `` relations and import-scope dependencies. The information will be added to the artifact's `PomInformation`. +* `boolean resolveJar` If set to true, the analysis will automatically download and parse the artifact's JAR file, build a `JarInformation` object and annotate `Artifact` objects with it. + +Additionally, the `MavenCentralArtifactAnalysis` provides the `resolveIndex` option. If set to true, an `IndexInformation` object will be created based on the information available in Maven Centrals Lucene index. + +Both analysis types provide a method called `void runAnalysis(String[] args)`. Invoking this method will start the analysis. By default, MARIN analyses will support the following CLI parameters: + +| **Argument** | **Artifact Analysis** | **Library Analysis** | **Description** | **Example** | +|------------------------------------------------------------|-----------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| +| `-st :`
`--skip-take :` | Yes | Yes | Skips the first `n` inputs and then only
processes the next `t` ones. | `-st 200:10` | +| `-su :`
`--since-until :` | Yes | No | Skips artifacts released before the
timestamp `s` or after timestamp `t`. | | +| `-i `
`--inputs ` | Yes | Yes | Processes inputs from a file instead of the
Maven Central index. Expects on input
per line. Inputs are either G:A:V triple (artifacts)
or G:A tuple (libraries). | `-i libraries.txt` | +| `-pof `
`--progress-output-file ` | Yes | Yes | File to periodically write number of processed
artifacts to. Defaults to `./marin-progress`. | `-pof marin-progress` | +| `-prf `
`--progress-restore-file ` | Yes | Yes | File to load a previous run's progress from.
Inputs will be skipped until progress is restored.
Not used by default. | `-prf marin-progress` | +| `-spi `
`--save-progress-interval ` | Yes | Yes | Number of inputs after which to store progress.
Defaults to 100. | `-spi 10` | +| `-t `
`--threads ` | Yes | Yes | Number of threads to use. Defaults to 1. | `-t 8` | +| `-o `
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | + ## Usage To use MARIN, you will need to implement two components: -1. You need an implementation of the abstract class `MavenCentralAnalysis` that defines the `void analyzeArtifact(Artifact toAnalyze)` method. This is your actual analysis implementation that defines how a single artifact shall be processed. +1. You need an implementation of either the abstract class `MavenCentralArtifactAnalysis` or `MavenCentralLibraryAnalysis`. This is your actual analysis implementation that defines how a single artifact or library shall be processed. 2. You need a runner class that passes command line arguments to your analysis implementation. Usually, this will look like this: ```java public class AnalysisRunner { @@ -69,12 +65,12 @@ Once this is implemented, you can run your analysis using the following command. ```java -jar executableName *INSERT CLI HERE* ``` ## Example Use Cases: -In the following, there are some example implementations of `MavenCentralAnalaysis`. All of them can be used with the same `AnalysisRunner` implementation seen above, just replace `MyAnalysisImplementation` with the actual implementation name. +In the following, there are some example implementations of `MavenCentralArtifactAnalysis`. All of them can be used with the same `AnalysisRunner` implementation seen above, just replace `MyAnalysisImplementation` with the actual implementation name. You can run each example on the first 1000 Maven artifacts by invoking `java -jar executableName -st 0:1000` for the respective project executable JAR. ### Counting all classFiles from jar artifacts: ``` java -public class ClassFileCountImplementation extends MavenCentralAnalysis { +public class ClassFileCountImplementation extends MavenCentralArtifactAnalysis { private long numberOfClassfiles; @@ -98,7 +94,7 @@ public class ClassFileCountImplementation extends MavenCentralAnalysis { ### Find all Unique Licenses from pom artifacts ``` java -public class LicenseImplementation extends MavenCentralAnalysis { +public class LicenseImplementation extends MavenCentralArtifactAnalysis { private final Set uniqueLicenses; public LicenseImplementation() { @@ -126,7 +122,7 @@ public class LicenseImplementation extends MavenCentralAnalysis { ### Collect all artifacts that have javadocs ``` java -public class JavaDocImplementation extends MavenCentralAnalysis { +public class JavaDocImplementation extends MavenCentralArtifactAnalysis { private final Set hasJavadocs; diff --git a/src/main/java/org/tudo/sse/CliInformation.java b/src/main/java/org/tudo/sse/CliInformation.java deleted file mode 100644 index 7fbf233..0000000 --- a/src/main/java/org/tudo/sse/CliInformation.java +++ /dev/null @@ -1,228 +0,0 @@ -package org.tudo.sse; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * This class holds the configuration information for the MavenCentralAnalysis class. - * This allows for easier setup for different run configurations. - */ -public class CliInformation { - private long skip; - private long take; - private long since; - private long until; - private Path name; - private Path toCoordinates; - private Path toIndexPos; - private boolean output; - private Path toOutputDirectory; - private boolean multi; - private int threads; - private int writeProcessedIndexes; - - /** - * Initializes a new CliInformation object with all parameters set to default values (where appropriate) - */ - public CliInformation() { - skip = -1; - take = -1; - since = -1; - until = -1; - name = Paths.get("lastIndexProcessed"); - writeProcessedIndexes = 1000; - toCoordinates = null; - toIndexPos = null; - toOutputDirectory = null; - output = false; - multi = false; - } - - /** - * Retrieves the number of artifacts to skip from the index (pagination). - * @return The number of artifacts to skip - */ - public long getSkip() { - return skip; - } - - /** - * Sets the number of artifacts to skip from the index (pagination). - * @param skip The number of artifacts to skip - */ - public void setSkip(long skip) { - this.skip = skip; - } - - /** - * Retrieves the number of artifacts to process from the index (pagination). - * @return The number of artifacts to process from the index. A value of -1 indicates no limit. - */ - public long getTake() { - return take; - } - - /** - * Sets the number of artifacts to process from the index (pagination). - * @param take The number of artifacts to process from the index. A value of -1 indicates no limit. - */ - public void setTake(long take) { - this.take = take; - } - - /** - * Retrieves the timestamp to use as a lower bound in terms of release time when processing artifacts. - * @return The timestamp to use as a lower bound - */ - public long getSince() { - return since; - } - - /** - * Sets the timestamp to use as a lower bound in terms of release time when processing artifacts. - * @param since The timestamp to use as a lower bound - */ - public void setSince(long since) { - this.since = since; - } - - /** - * Retrieves the timestamp to use as an upper bound in terms of release time when processing artifacts. - * @return The timestamp to use as an upper bound - */ - public long getUntil() { - return until; - } - - /** - * Sets the timestamp to use as an upper bound in terms of release time when processing artifacts. - * @param until The timestamp to use as an upper bound - */ - public void setUntil(long until) { - this.until = until; - } - - /** - * Get the path to the text file containing a list of GAV triples to process. - * @return Path to input list file - */ - public Path getToCoordinates() { - return toCoordinates; - } - - /** - * Sets the path to the text file containing a list of GAV triples to process. - * @param toCoordinates Path to input list file - */ - public void setToCoordinates(Path toCoordinates) { - this.toCoordinates = toCoordinates; - } - - /** - * Retrieves the path from which to restore progress - * @return The progress restore path - */ - public Path getToIndexPos() { - return toIndexPos; - } - - /** - * Sets the path from which to restore progress - * @param toIndexPos The progress restore path - */ - public void setToIndexPos(Path toIndexPos) { - this.toIndexPos = toIndexPos; - } - - /** - * Retrieves the file path to write progress information to. - * @return The progress file path - */ - public Path getName() { - return name; - } - - /** - * Sets the file path to write progress information to. - * @param name The progress file path - */ - public void setName(Path name) { - this.name = name; - } - - /** - * Retrieves whether the application is to be run in multithreaded mode. - * @return True if multithreading is enabled, false otherwise - */ - public boolean isMulti() { - return multi; - } - - /** - * Sets whether the application is to be run in multithreaded mode. - * @param multi True for multithreading, false otherwise - */ - public void setMulti(boolean multi) { - this.multi = multi; - } - - /** - * Retrieves the number of threads configured for multithreading. - * @return The number of threads configured - */ - public int getThreads() { - return threads; - } - - /** - * Sets the number of threads for multithreading - * @param threads The number of threads to use - */ - public void setThreads(int threads) { - this.threads = threads; - } - - /** - * Retrieves whether all artifacts processed should be persisted to an output directory. - * @return True if artifacts are persisted, false otherwise - */ - public boolean isOutput() { - return output; - } - - /** - * Sets whether all artifacts processed should be persisted to an output directory. - * @param output True for persisting artifacts, false otherwise - */ - public void setOutput(boolean output) { - this.output = output; - } - - /** - * Retrieves the path under which to persist artifacts that have been processed. - * @return The output path - */ - public Path getToOutputDirectory() { - return toOutputDirectory; - } - - /** - * Sets the path under which to persist artifacts that have been processed. - * @param toOutputDirectory Path to use for artifact output - */ - public void setToOutputDirectory(Path toOutputDirectory) { - this.toOutputDirectory = toOutputDirectory; - } - - /** - * Gets the number of artifacts after which to write progress (to the progress file). - * @return The number of artifacts after which progress is saved - */ - public int getWriteProcessedIndexes() { return writeProcessedIndexes; } - - /** - * Sets the number of artifacts after which to write progress to the progress file. - * @param writeProcessedIndexes The number of artifacts after which progress is saved - */ - public void setWriteProcessedIndexes(int writeProcessedIndexes) {this.writeProcessedIndexes = writeProcessedIndexes;} -} diff --git a/src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java similarity index 88% rename from src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java rename to src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java index 523f366..4f28fea 100644 --- a/src/main/java/org/tudo/sse/analyses/ArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java @@ -3,7 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -abstract class ArtifactAnalysis { +abstract class MavenCentralAnalysis { /** * Defines whether this analysis requires artifacts to have index information annotated. @@ -40,10 +40,10 @@ abstract class ArtifactAnalysis { * be set to true no matter its given parameter value. * @param requiresJar Whether this analysis requires JAR information */ - protected ArtifactAnalysis(boolean requiresIndex, - boolean requiresPom, - boolean requiresTransitives, - boolean requiresJar){ + protected MavenCentralAnalysis(boolean requiresIndex, + boolean requiresPom, + boolean requiresTransitives, + boolean requiresJar){ if(!requiresIndex && !requiresPom && !requiresTransitives && !requiresJar){ log.warn("Potential misconfiguration - no data sources (index, POM, JAR) are required by this analysis"); } else if(requiresTransitives && !requiresPom){ diff --git a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java similarity index 62% rename from src/main/java/org/tudo/sse/MavenCentralAnalysis.java rename to src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 4fe130e..c6231ea 100644 --- a/src/main/java/org/tudo/sse/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -1,15 +1,18 @@ -package org.tudo.sse; +package org.tudo.sse.analyses; import akka.actor.ActorRef; import akka.actor.ActorSystem; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tudo.sse.ArtifactFactory; +import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.multithreading.ProcessIdentifierMessage; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; +import org.tudo.sse.utils.ArtifactConfigParser; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; @@ -18,7 +21,6 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -27,9 +29,9 @@ * The MavenCentralAnalysis enables analysis of artifacts on the maven central repository for jobs of any size. * Takes in cli to be configured to the specific job desired. */ -public abstract class MavenCentralAnalysis { +public abstract class MavenCentralArtifactAnalysis { - private final CliInformation setupInfo; + private ArtifactConfigParser.ArtifactConfig artifactConfig; private ActorRef queueActorRef; private ResolverFactory resolverFactory; @@ -54,7 +56,7 @@ public abstract class MavenCentralAnalysis { protected final boolean resolveJar; - private static final Logger log = LogManager.getLogger(MavenCentralAnalysis.class); + private static final Logger log = LogManager.getLogger(MavenCentralArtifactAnalysis.class); /** * Creates a new Maven Central Analysis with the given configuration options. @@ -65,10 +67,10 @@ public abstract class MavenCentralAnalysis { * be set to true no matter its given parameter value. * @param requiresJar Whether this analysis requires JAR information */ - protected MavenCentralAnalysis(boolean requiresIndex, - boolean requiresPom, - boolean requiresTransitives, - boolean requiresJar) { + protected MavenCentralArtifactAnalysis(boolean requiresIndex, + boolean requiresPom, + boolean requiresTransitives, + boolean requiresJar) { if(!requiresIndex && !requiresPom && !requiresTransitives && !requiresJar){ log.warn("Potential misconfiguration - no data sources (index, POM, JAR) are required by this analysis"); @@ -76,12 +78,12 @@ protected MavenCentralAnalysis(boolean requiresIndex, log.warn("Potential misconfiguration - analysis requires transitive information but no POM information. " + "POM information will also be collected to provide transitive information."); } - - setupInfo = new CliInformation(); resolveIndex = requiresIndex; resolvePom = requiresPom || requiresTransitives; // Cannot have transitive information without POM information processTransitives = requiresTransitives; resolveJar = requiresJar; + // Initialize with default config, update later + artifactConfig = new ArtifactConfigParser.ArtifactConfig(); } @@ -92,74 +94,12 @@ protected MavenCentralAnalysis(boolean requiresIndex, */ public abstract void analyzeArtifact(Artifact current); - /** - * This method handles parsing the command line arguments and stores it into a CliInformation object. - * - * @param args - cli to be parsed - * @see CliInformation when there is an issue processing the cli passed to the program - */ - public void parseCmdLine(String[] args) { - boolean flagSet1 = false; - boolean flagSet2 = false; - boolean flagSet3 = false; - try { - for (int i = 0; i < args.length; i += 2) { - switch (args[i]) { - case "-st": - checkConflict(flagSet1, args[i]); - long[] parsed = parseLong(args, i); - setupInfo.setSkip(parsed[0]); - setupInfo.setTake(parsed[1]); - flagSet1 = true; - break; - case "-su": - checkTwoConflicts(flagSet1, flagSet2, args[i]); - long[] parsed1 = parseLong(args, i); - setupInfo.setSince(parsed1[0]); - setupInfo.setUntil(parsed1[1]); - flagSet1 = true; - flagSet2 = true; - break; - case "-ip": - checkConflict(flagSet1, args[i]); - setupInfo.setToIndexPos(parsePathName(args, i)); - flagSet1 = true; - break; - case "--coordinates": - checkConflict(flagSet2, args[i]); - checkConflict(flagSet3, args[i]); - setupInfo.setToCoordinates(parsePathName(args, i)); - flagSet2 = true; - flagSet3 = true; - break; - case "--name": - if(i + 1 < args.length) { - setupInfo.setName(parsePathName(args, i)); - } - break; - case "--multi": - setupInfo.setMulti(true); - setupInfo.setThreads(parseInt(args, i)); - break; - case "--output": - setupInfo.setOutput(true); - setupInfo.setToOutputDirectory(parsePathName(args, i)); - break; - default: - throw new CLIException(args[i]); - } - } - } catch(CLIException e) { - throw new RuntimeException(e); - } - } - /** * Returns the CLI configuration for this analysis. * @return CLI information for this analysis */ - public CliInformation getSetupInfo() { - return setupInfo; + public ArtifactConfigParser.ArtifactConfig getSetupInfo() { + return this.artifactConfig; } private void printRunInfo(){ @@ -171,81 +111,23 @@ private void printRunInfo(){ log.info("The current run has been configured as follows:"); - if(setupInfo.isMulti()){ - log.info("\t - Using " + setupInfo.getThreads() + " threads"); + if(this.artifactConfig.multipleThreads){ + log.info("\t - Using " + this.artifactConfig.threadCount + " threads"); } else { log.info("\t - Using one thread"); } - if(setupInfo.getToCoordinates() == null){ + if(this.artifactConfig.inputListFile == null){ log.info("\t - Reading artifacts from Maven Central index"); - if(setupInfo.getToIndexPos() != null) log.info("\t - Restoring last index position from " + setupInfo.getToIndexPos()); - if(setupInfo.getName() != null) log.info("\t - Writing last index position to " + setupInfo.getName()); - if(setupInfo.getSkip() >= 0) log.info("\t - Skipping " + setupInfo.getSkip() + " artifacts"); - if(setupInfo.getTake() >= 0) log.info("\t - Taking " + setupInfo.getTake() + " artifacts"); - if(setupInfo.getSince() >= 0) log.info("\t - Skipping artifacts before " + setupInfo.getSince()); - if(setupInfo.getUntil() >= 0) log.info("\t - Taking artifacts until " + setupInfo.getUntil()); - - } else { - log.info("\t - Reading artifacts from GAV-list at " + setupInfo.getToCoordinates()); - } - } - - private void checkTwoConflicts(boolean checkConflict1, boolean checkConflict2, String flag) throws CLIException { - if(checkConflict1 || checkConflict2) { - throw new CLIException("Other flag already set.", flag); - } - } + if(this.artifactConfig.progressRestoreFile != null) log.info("\t - Restoring last index position from " + this.artifactConfig.progressRestoreFile); + if(this.artifactConfig.progressOutputFile != null) log.info("\t - Writing last index position to " + this.artifactConfig.progressOutputFile); + if(this.artifactConfig.skip >= 0) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); + if(this.artifactConfig.take >= 0) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); + if(this.artifactConfig.since >= 0) log.info("\t - Skipping artifacts before " + this.artifactConfig.since); + if(this.artifactConfig.until >= 0) log.info("\t - Taking artifacts until " + this.artifactConfig.until); - private void checkConflict(boolean checkConflict, String flag) throws CLIException { - if(checkConflict) { - throw new CLIException("Other flag already set.", flag); - } - } - - private long[] parseLong(String[] args, int i) throws CLIException { - long[] toReturn = new long[2]; - if(i + 1 < args.length) { - String[] ints = args[i + 1].split(":"); - if(ints.length == 2) { - try { - toReturn[0] = Long.parseLong(ints[0]); - toReturn[1] = Long.parseLong(ints[1]); - } catch(NumberFormatException e) { - throw(new CLIException(args[i], e.getMessage())); - } - } else { - throw(new CLIException(args[i], "Correct format: first:second")); - } } else { - throw(new CLIException(args[i], "Missing argument: first:second")); - } - return toReturn; - } - - private int parseInt(String[] args, int i) throws CLIException { - if(i + 1 < args.length) { - try{ - return Integer.parseInt(args[i + 1]); - } catch(NumberFormatException e) { - throw new CLIException(args[i], e.getMessage()); - } - } else { - throw new CLIException(args[i]); - } - } - - private Path parsePathName(String[] args, int i) throws CLIException { - if(i + 1 < args.length) { - if(Files.isRegularFile(Paths.get(args[i + 1])) || args[i].equals("--name")) { - return Paths.get(args[i + 1]); - } else if(args[i].equals("--output") && Files.isDirectory(Paths.get(args[i + 1]))) { - return Paths.get(args[i + 1]); - } else { - throw new CLIException(args[i], "Invalid path"); - } - } else { - throw(new CLIException(args[i], "Missing argument: path/to/file")); + log.info("\t - Reading artifacts from GAV-list at " + this.artifactConfig.inputListFile); } } @@ -259,20 +141,26 @@ private Path parsePathName(String[] args, int i) throws CLIException { * @throws URISyntaxException when there is an issue with the url built * @throws IOException when there is an issue opening a file */ - public Map runAnalysis(String[] args) throws URISyntaxException, IOException { - parseCmdLine(args); - printRunInfo(); - if(setupInfo.isOutput()) { - resolverFactory = new ResolverFactory(setupInfo.isOutput(), setupInfo.getToOutputDirectory(), processTransitives); + public final Map runAnalysis(String[] args) throws URISyntaxException, IOException { + // Obtain CLI arguments + try { + this.artifactConfig = new ArtifactConfigParser().parseArtifactConfig(args); + printRunInfo(); + } catch(CLIException clix){ + throw new RuntimeException(clix); + } + + if(this.artifactConfig.outputEnabled) { + resolverFactory = new ResolverFactory(this.artifactConfig.outputEnabled, this.artifactConfig.outputDirectory, processTransitives); } else { resolverFactory = new ResolverFactory(processTransitives); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { ActorSystem system = ActorSystem.create("my-system"); - queueActorRef = system.actorOf(QueueActor.props(setupInfo.getThreads(), system, 0,0, null), "queueActor"); + queueActorRef = system.actorOf(QueueActor.props(this.artifactConfig.threadCount, system, 0,0, null), "queueActor"); - if(setupInfo.getToCoordinates() == null) { + if(this.artifactConfig.inputListFile == null) { indexProcessor(); } else { readIdentsIn(); @@ -284,7 +172,7 @@ public Map runAnalysis(String[] args) throws URISyntaxE e.printStackTrace(); } } else { - if(setupInfo.getToCoordinates() == null) { + if(this.artifactConfig.inputListFile == null) { indexProcessor(); } else { readIdentsIn(); @@ -306,35 +194,35 @@ public void indexProcessor() throws URISyntaxException, IOException { IndexIterator indexIterator; //set up indexIterator here (skip to a position or start from the start) - if (setupInfo.getToIndexPos() != null) { + if (this.artifactConfig.progressRestoreFile != null) { indexIterator = new IndexIterator(new URI(base), getStartingPos()); - } else if(setupInfo.getSkip() != -1) { - indexIterator = new IndexIterator(new URI(base), setupInfo.getSkip()); + } else if(this.artifactConfig.skip != -1) { + indexIterator = new IndexIterator(new URI(base), this.artifactConfig.skip); } else { indexIterator = new IndexIterator(new URI(base)); } if (resolveIndex) { - if (setupInfo.getSkip() != -1 && setupInfo.getTake() != -1) { - walkPaginated(setupInfo.getTake(), indexIterator); - } else if (setupInfo.getSince() != -1 && setupInfo.getUntil() != -1) { - walkDates(setupInfo.getSince(), setupInfo.getUntil(), indexIterator); + if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { + walkPaginated(this.artifactConfig.take, indexIterator); + } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { + walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); } else { walkAllIndexes(indexIterator); } - } else if (setupInfo.getSkip() != -1 && setupInfo.getTake() != -1) { - lazyWalkPaginated(setupInfo.getTake(), indexIterator); - } else if (setupInfo.getSince() != -1 && setupInfo.getUntil() != -1) { - lazyWalkDates(setupInfo.getSince(), setupInfo.getUntil(), indexIterator); + } else if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { + lazyWalkPaginated(this.artifactConfig.take, indexIterator); + } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { + lazyWalkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); } else { lazyWalkAllIndexes(indexIterator); } - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } private void processIndex(Artifact current) { - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(new ProcessIdentifierMessage(current.getIdent(), this), ActorRef.noSender()); } else { callResolver(current.getIdent()); @@ -357,18 +245,18 @@ public List walkAllIndexes(IndexIterator indexIterator) throws IOExcep while(indexIterator.hasNext()) { Artifact current = ArtifactFactory.createArtifact(indexIterator.next()); artifacts.add(current); - if(setupInfo.isOutput() && !resolvePom && !resolveJar) { - Path filePath = setupInfo.getToOutputDirectory().resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { + Path filePath = this.artifactConfig.outputDirectory.resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } processIndex(current); - if(indexIterator.getIndex() % setupInfo.getWriteProcessedIndexes() == 0) - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } @@ -392,19 +280,19 @@ public List walkPaginated(long take, IndexIterator indexIterator) thro take += indexIterator.getIndex(); while(indexIterator.hasNext() && indexIterator.getIndex() < take) { Artifact current = ArtifactFactory.createArtifact(indexIterator.next()); - if(setupInfo.isOutput() && !resolvePom && !resolveJar) { - Path filePath = setupInfo.getToOutputDirectory().resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { + Path filePath = this.artifactConfig.outputDirectory.resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } artifacts.add(current); processIndex(current); - if(indexIterator.getIndex() % setupInfo.getWriteProcessedIndexes() == 0) - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } @@ -433,8 +321,8 @@ public List walkDates(long since, long until, IndexIterator indexItera currentToSince = temp.getLastModified(); if(currentToSince >= since && currentToSince < until) { Artifact current = ArtifactFactory.createArtifact(indexIterator.next()); - if(setupInfo.isOutput() && !resolvePom && !resolveJar) { - Path filePath = setupInfo.getToOutputDirectory().resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { + Path filePath = this.artifactConfig.outputDirectory.resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } @@ -442,11 +330,11 @@ public List walkDates(long since, long until, IndexIterator indexItera artifacts.add(current); processIndex(current); } - if(indexIterator.getIndex() % setupInfo.getWriteProcessedIndexes() == 0) - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } @@ -455,7 +343,7 @@ public List walkDates(long since, long until, IndexIterator indexItera } private void processIndexIdentifier(ArtifactIdent ident) { - if(setupInfo.isMulti()){ + if(this.artifactConfig.multipleThreads){ queueActorRef.tell(new ProcessIdentifierMessage(ident, this), ActorRef.noSender()); } else { callResolver(ident); @@ -477,18 +365,18 @@ public List lazyWalkAllIndexes(IndexIterator indexIterator) throw while(indexIterator.hasNext()) { ArtifactIdent ident = indexIterator.next().getIdent(); idents.add(ident); - if(setupInfo.isOutput() && !resolvePom && !resolveJar) { - Path filePath = setupInfo.getToOutputDirectory().resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { + Path filePath = this.artifactConfig.outputDirectory.resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } processIndexIdentifier(ident); - if(indexIterator.getIndex() % setupInfo.getWriteProcessedIndexes() == 0) - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } @@ -511,18 +399,18 @@ public List lazyWalkPaginated(long take, IndexIterator indexItera while(indexIterator.hasNext() && indexIterator.getIndex() < take) { ArtifactIdent ident = indexIterator.next().getIdent(); idents.add(ident); - if(setupInfo.isOutput() && !resolvePom && !resolveJar) { - Path filePath = setupInfo.getToOutputDirectory().resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { + Path filePath = this.artifactConfig.outputDirectory.resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } processIndexIdentifier(ident); - if(indexIterator.getIndex() % setupInfo.getWriteProcessedIndexes() == 0) - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } indexIterator.closeReader(); @@ -550,19 +438,19 @@ public List lazyWalkDates(long since, long until, IndexIterator i if(currentToSince >= since && currentToSince < until) { ArtifactIdent ident = temp.getIdent(); idents.add(ident); - if(setupInfo.isOutput() && !resolvePom && !resolveJar) { - Path filePath = setupInfo.getToOutputDirectory().resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { + Path filePath = this.artifactConfig.outputDirectory.resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } processIndexIdentifier(ident); } - if(indexIterator.getIndex() % setupInfo.getWriteProcessedIndexes() == 0) - writeLastProcessed(indexIterator.getIndex(), setupInfo.getName()); + if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } @@ -583,7 +471,7 @@ private void writeLastProcessed(long lastIndexProcessed, Path name) throws IOExc * @return a list of identifiers collected from the file */ public List readIdentsIn() { - Path toCoordinates = setupInfo.getToCoordinates(); + Path toCoordinates = this.artifactConfig.inputListFile; BufferedReader coordinatesReader; List identifiers = new ArrayList<>(); @@ -592,11 +480,11 @@ public List readIdentsIn() { String line = coordinatesReader.readLine(); int i = -1; - if(setupInfo.getSkip() != -1 && setupInfo.getTake() != -1) { + if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { int toSkip = 0; int curTake = 0; - while(line != null && curTake < setupInfo.getTake()) { - if(toSkip >= setupInfo.getSkip()) { + while(line != null && curTake < this.artifactConfig.take) { + if(toSkip >= this.artifactConfig.skip) { String[] parts = line.split(":"); if(parts.length == 3) { ArtifactIdent current = new ArtifactIdent(parts[0], parts[1], parts[2]); @@ -611,7 +499,7 @@ public List readIdentsIn() { toSkip++; i++; } - } else if(setupInfo.getToIndexPos() != null) { + } else if(this.artifactConfig.progressRestoreFile != null) { long start = getStartingPos(); int curPos = 0; while(line != null) { @@ -644,11 +532,11 @@ public List readIdentsIn() { } } - if(setupInfo.isMulti()) { + if(this.artifactConfig.multipleThreads) { queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); } - writeLastProcessed(i, setupInfo.getName()); + writeLastProcessed(i, this.artifactConfig.progressOutputFile); coordinatesReader.close(); } catch(IOException e) { throw new RuntimeException(e); @@ -659,7 +547,7 @@ public List readIdentsIn() { private long getStartingPos() { BufferedReader indexReader; try { - indexReader = new BufferedReader(new FileReader(setupInfo.getToIndexPos().toFile())); + indexReader = new BufferedReader(new FileReader(this.artifactConfig.progressRestoreFile.toFile())); String line = indexReader.readLine(); return Integer.parseInt(line); } catch (IOException e) { diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 6ff5395..6cc48a8 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -12,7 +12,7 @@ import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; -import org.tudo.sse.utils.CLIParser; +import org.tudo.sse.utils.CommonConfigParser; import org.tudo.sse.utils.LibraryIndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -20,20 +20,19 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -public abstract class MavenCentralLibraryAnalysis extends ArtifactAnalysis { +public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis { private final IReleaseListProvider releaseListProvider; private final ResolverFactory resolverFactory; - protected final LibraryAnalysisConfiguration config; + protected CommonConfigParser.CommonConfig config; private ActorRef queueActorRef; + private long lastPositionSaved; /** * Creates a new Maven Central Analysis with the given configuration options. @@ -48,44 +47,59 @@ protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTrans this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); this.resolverFactory = new ResolverFactory(processTransitives); - this.config = new LibraryAnalysisConfiguration(); + this.lastPositionSaved = -1; } @Override public final void runAnalysis(String[] args){ // Obtain CLI arguments - this.config.parseArguments(args); - printRunInfo(); + try { + this.config = new CommonConfigParser().parseCommonConfig(args); + printRunInfo(); + } catch(CLIException clix){ + throw new RuntimeException(clix); + } ActorSystem system = null; + long currentPosition = 0L; // Get input iterator - this will either be from a configured input file or from the Maven Central Index. The - // iterator will produce strings of form :. If a progress restore file is specified, the - // progress will be applied to the index iterator automatically. + // iterator will produce strings of form :. Iterator gaIterator = getGaIterator(); + // Restore previous progress + if(config.progressRestoreFile != null){ + final long startingPosition = getProgressFromRestoreFile(); + log.info("Restoring previous progress (position {})", startingPosition); + for(int i = 0; i < startingPosition; i++){ + if(!gaIterator.hasNext()){ + log.warn("No more GA values available while restoring previous progress"); + break; + } else gaIterator.next(); + currentPosition += 1L; + } + } + // If there are values to skip, we skip them manually if(config.skip > 0){ log.info("Skipping {} library names", config.skip); for(int i = 0; i < config.skip; i++){ if(!gaIterator.hasNext()){ - log.warn("No more GA inputs available while skipping to desired position"); + log.warn("No more GA values available while skipping to desired position"); break; } else gaIterator.next(); + currentPosition += 1L; } } // If we want to use multiple threads, we initialize the queue actor that managers workers if(this.config.multipleThreads){ - // Get the initial progress so the progress output file is up-to-date - long initialProgress = gaIterator instanceof LibraryIndexIterator ? - ((LibraryIndexIterator)gaIterator).getPosition() : config.skip; system = ActorSystem.create("marin-actors"); - this.queueActorRef = system.actorOf(QueueActor.props(config.threadCount, system, initialProgress, + this.queueActorRef = system.actorOf(QueueActor.props(config.threadCount, system, currentPosition, config.progressWriteInterval, config.progressOutputFile), "marin-queue-actor"); } - long entriesTaken = 0; + long entriesTaken = 0L; if(config.take >= 0) log.info("Taking {} library names", config.take); @@ -95,7 +109,10 @@ public final void runAnalysis(String[] args){ while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ processEntry(gaIterator.next()); - entriesTaken += 1; + entriesTaken += 1L; + currentPosition += 1L; + + writeProgressIfNeeded(currentPosition); } // At this point all libraries are queued (multithreaded) or processed (single-threaded) @@ -189,32 +206,18 @@ private List getReleaseIdentifiers(String groupId, String artifac } private Iterator getGaIterator(){ - if(config.gaInputListFile != null){ + if(config.inputListFile != null){ try { - final List inputs = Files.readAllLines(config.gaInputListFile); - log.info("Read {} GAs from input file {}", inputs.size(), config.gaInputListFile); + final List inputs = Files.readAllLines(config.inputListFile); + log.info("Read {} GAs from input file {}", inputs.size(), config.inputListFile); return inputs.iterator(); } catch(IOException iox){ - log.error("Failed to read GA tuples from input file at {}", config.gaInputListFile, iox); + log.error("Failed to read GA tuples from input file at {}", config.inputListFile, iox); throw new RuntimeException(iox); } } else { try { - final LibraryIndexIterator indexIterator = new LibraryIndexIterator( - new URI(MavenCentralRepository.RepoBasePath), config.progressOutputFile, config.progressWriteInterval); - - // If we use multiple threads, the queue actor will store progress. If we use a single thread, the - // iterator must store the progress. This is because we want to store progress after the analysis is - // actually done, which the iterator cannot know precisely in the multithreaded setting. - indexIterator.setSaveProgress(!config.multipleThreads); - - if(config.progressRestoreFile != null){ - final long startingPosition = config.getProgressFromRestoreFile(); - log.info("Restoring previous progress (index position {})", startingPosition); - indexIterator.setPosition(startingPosition); - } - - return indexIterator; + return new LibraryIndexIterator(new URI(MavenCentralRepository.RepoBasePath)); } catch (URISyntaxException | IOException iox){ log.error("Failed to create Maven Central library iterator", iox); throw new RuntimeException(iox); @@ -238,7 +241,7 @@ private void printRunInfo(){ if(resolveIndex) log.info ("\t - The analysis requires index information"); if(resolvePom) log.info ("\t - The analysis requires pom information"); if(processTransitives) log.info("\t - The analysis requires transitive pom dependencies"); - if(resolveJar)log.info ("\t - The analysis requires jar information"); + if(resolveJar)log.info ("\t - The analysis requires jar information"); log.info("The current run has been configured as follows:"); @@ -248,98 +251,36 @@ private void printRunInfo(){ log.info("\t - Using one thread"); } - if(config.gaInputListFile == null){ + if(config.inputListFile == null){ log.info("\t - Reading libraries from Maven Central index"); if(config.progressRestoreFile != null) log.info("\t - Restoring last index position from " + config.progressRestoreFile); if(config.progressOutputFile != null) log.info("\t - Writing last index position to " + config.progressOutputFile); if(config.skip >= 0) log.info("\t - Skipping " + config.skip + " artifacts"); if(config.take >= 0) log.info("\t - Taking " + config.take + " artifacts"); } else { - log.info("\t - Reading libraries from GA-list at " + config.gaInputListFile); + log.info("\t - Reading libraries from GA-list at " + config.inputListFile); } } - protected static class LibraryAnalysisConfiguration extends CLIParser { - - public long skip; - public long take; - - public Path gaInputListFile; - public Path progressOutputFile; - public Path progressRestoreFile; - - public boolean multipleThreads; - public int threadCount; - - public int progressWriteInterval; - - public LibraryAnalysisConfiguration() { - skip = -1L; - take = -1L; - gaInputListFile = null; - progressOutputFile = Paths.get("marin-progress"); - progressRestoreFile = null; - multipleThreads = false; - threadCount = 1; - progressWriteInterval = 100; - } - - @Override - public void parseArguments(String[] args) { - try { - for(int i = 0; i < args.length; i += 2){ - switch (args[i]){ - case "-st": - if(skip != -1L) - throw new CLIException("Values for skip and take cannot be set twice!"); - - final long[] skipTake = nextArgAsLongPair(args, i); - skip = skipTake[0]; - take = skipTake[1]; - break; - case "--progress-restore-file": - if(gaInputListFile != null) - throw new CLIException("Cannot restore index position when a custom input list is used!"); - if(progressRestoreFile != null) - throw new CLIException("Progress restore file cannot be set twice!"); - - progressRestoreFile = nextArgAsRegularFileReference(args, i); - break; - case "--save-progress-interval": - progressWriteInterval = nextArgAsInt(args, i); - break; - case "--progress-output-file": - progressOutputFile = nextArgAsPath(args, i); - break; - case "--coordinates": - if(gaInputListFile != null) - throw new CLIException("Input file cannot be set twice!"); - if(progressRestoreFile != null) - throw new CLIException("Cannot use custom input list when progress restore file is set!"); - gaInputListFile = nextArgAsRegularFileReference(args, i); - break; - case "--multi": - final int threads = nextArgAsInt(args, i); - multipleThreads = threads > 1; - threadCount = threads; - break; - default: - throw new CLIException(args[i]); - } - } - } catch(CLIException clix){ - throw new RuntimeException(clix); - } + private long getProgressFromRestoreFile() { + BufferedReader indexReader; + try { + indexReader = new BufferedReader(new FileReader(config.progressRestoreFile.toFile())); + String line = indexReader.readLine(); + return Integer.parseInt(line); + } catch (IOException e) { + throw new RuntimeException(e); } + } - long getProgressFromRestoreFile() { - BufferedReader indexReader; - try { - indexReader = new BufferedReader(new FileReader(progressRestoreFile.toFile())); - String line = indexReader.readLine(); - return Integer.parseInt(line); - } catch (IOException e) { - throw new RuntimeException(e); + private void writeProgressIfNeeded(long currentPosition){ + if(!this.config.multipleThreads){ + if(currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ + try(BufferedWriter writer = new BufferedWriter(new FileWriter(this.config.progressOutputFile.toFile()))){ + // Write position of last GA, as it has now been completed! + writer.write(String.valueOf(currentPosition)); + } catch(IOException ignored){} + this.lastPositionSaved = currentPosition; } } } diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index 19295a7..9770ee3 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -1,6 +1,6 @@ package org.tudo.sse.multithreading; -import org.tudo.sse.MavenCentralAnalysis; +import org.tudo.sse.analyses.MavenCentralArtifactAnalysis; import org.tudo.sse.model.ArtifactIdent; /** @@ -10,14 +10,14 @@ public class ProcessIdentifierMessage implements WorkItem { private final ArtifactIdent identifier; - private final MavenCentralAnalysis instance; + private final MavenCentralArtifactAnalysis instance; /** * Creates a new message with the given artifact identifier and analysis instance. * @param identifier The artifact identifier that must be processed * @param instance The analysis instance that must process the identifier */ - public ProcessIdentifierMessage(ArtifactIdent identifier, MavenCentralAnalysis instance) { + public ProcessIdentifierMessage(ArtifactIdent identifier, MavenCentralArtifactAnalysis instance) { this.identifier = identifier; this.instance = instance; } @@ -34,7 +34,7 @@ public ArtifactIdent getIdentifier() { * Retrieves the analysis instance that must process the identifier * @return The analysis instance */ - public MavenCentralAnalysis getInstance() { + public MavenCentralArtifactAnalysis getInstance() { return instance; } } diff --git a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java new file mode 100644 index 0000000..34a1520 --- /dev/null +++ b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java @@ -0,0 +1,82 @@ +package org.tudo.sse.utils; + +import org.tudo.sse.CLIException; + +import java.nio.file.Path; + +public class ArtifactConfigParser extends CommonConfigParser implements CLIParsingUtilities { + + public ArtifactConfig parseArtifactConfig(String[] args) throws CLIException { + final ArtifactConfig artifactConfig = new ArtifactConfig(); + + for(int i = 0; i < args.length; i += 2) { + handleArtifactParameter(args, i, artifactConfig); + } + + return artifactConfig; + } + + + private void handleArtifactParameter(String[] args, int i, ArtifactConfig config) throws CLIException { + switch(args[i]) { + case "-su": + case "--since-until": + if(config.skip != -1L) + throw new CLIException("Cannot apply time-based filtering when pagination (-st) is used", args[i]); + if(config.since != -1L) + throw new CLIException("Values for since and until cannot be set twice!", args[i]); + if(config.inputListFile != null) + throw new CLIException("Cannot apply time-based filtering when input list (-i) is used", args[i]); + + final long[] sinceUntil = nextArgAsLongPair(args, i); + config.since = sinceUntil[0]; + config.until = sinceUntil[1]; + break; + + case "-o": + case "--output": + config.outputEnabled = true; + config.outputDirectory = nextArgAsDirectoryReference(args, i); + break; + + case "-st": + case "--skip-take": + if(config.since != -1L) + throw new CLIException("Cannot apply pagination when time-based filtering (-su) is used", args[i]); + super.handleParameter(args, i, config); + break; + + case "-i": + case "--inputs": + if(config.since != -1L) + throw new CLIException("Cannot use input list when time-based filtering (-su) is applied", args[i]); + super.handleParameter(args, i, config); + break; + + default: + super.handleParameter(args, i , config); + + } + } + + public static class ArtifactConfig extends CommonConfig { + + public long since; + public long until; + + public Path outputDirectory; + public boolean outputEnabled; + + public ArtifactConfig(){ + super(); + + since = -1L; + until = -1L; + outputEnabled = false; + outputDirectory = null; + } + + + } + +} diff --git a/src/main/java/org/tudo/sse/utils/CLIParser.java b/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java similarity index 79% rename from src/main/java/org/tudo/sse/utils/CLIParser.java rename to src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java index 285fbb1..80fffba 100644 --- a/src/main/java/org/tudo/sse/utils/CLIParser.java +++ b/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java @@ -6,11 +6,9 @@ import java.nio.file.Path; import java.nio.file.Paths; -public abstract class CLIParser { +interface CLIParsingUtilities { - public abstract void parseArguments(String[] args); - - protected int nextArgAsInt(String[] args, int i) throws CLIException { + default int nextArgAsInt(String[] args, int i) throws CLIException { if(i + 1 < args.length) { try{ return Integer.parseInt(args[i + 1]); @@ -22,7 +20,7 @@ protected int nextArgAsInt(String[] args, int i) throws CLIException { } } - protected long[] nextArgAsLongPair(String[] args, int i) throws CLIException { + default long[] nextArgAsLongPair(String[] args, int i) throws CLIException { long[] toReturn = new long[2]; if(i + 1 < args.length) { String[] ints = args[i + 1].split(":"); @@ -42,7 +40,7 @@ protected long[] nextArgAsLongPair(String[] args, int i) throws CLIException { return toReturn; } - protected Path nextArgAsPath(String[] args, int i) throws CLIException { + default Path nextArgAsPath(String[] args, int i) throws CLIException { if(i + 1 < args.length) { return Paths.get(args[i + 1]); } else { @@ -50,7 +48,7 @@ protected Path nextArgAsPath(String[] args, int i) throws CLIException { } } - protected Path nextArgAsRegularFileReference(String[] args, int i) throws CLIException { + default Path nextArgAsRegularFileReference(String[] args, int i) throws CLIException { final Path path = nextArgAsPath(args, i); if(Files.isRegularFile(path)) return path; @@ -58,7 +56,7 @@ protected Path nextArgAsRegularFileReference(String[] args, int i) throws CLIExc throw new CLIException("Expected an existing file but got: " + path, args[i]); } - protected Path nextArgAsDirectoryReference(String[] args, int i) throws CLIException { + default Path nextArgAsDirectoryReference(String[] args, int i) throws CLIException { final Path path = nextArgAsPath(args, i); if(Files.isDirectory(path)) return path; diff --git a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java new file mode 100644 index 0000000..4d1941d --- /dev/null +++ b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java @@ -0,0 +1,88 @@ +package org.tudo.sse.utils; + +import org.tudo.sse.CLIException; + +import java.nio.file.Path; +import java.nio.file.Paths; + +public class CommonConfigParser implements CLIParsingUtilities { + + public CommonConfig parseCommonConfig(String[] args) throws CLIException { + final CommonConfig config = new CommonConfig(); + + + for(int i = 0; i < args.length; i += 2){ + handleParameter(args, i, config); + } + + return config; + } + + protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException{ + switch (args[i]){ + case "-st": + case "--skip-take": + if(config.skip != -1L) + throw new CLIException("Values for skip and take cannot be set twice!", args[i]); + + final long[] skipTake = nextArgAsLongPair(args, i); + config.skip = skipTake[0]; + config.take = skipTake[1]; + break; + case "-prf": + case "--progress-restore-file": + if(config.progressRestoreFile != null) + throw new CLIException("Progress restore file cannot be set twice!", args[i]); + + config.progressRestoreFile = nextArgAsRegularFileReference(args, i); + break; + case "-spi": + case "--save-progress-interval": + config.progressWriteInterval = nextArgAsInt(args, i); + break; + case "-pof": + case "--progress-output-file": + config.progressOutputFile = nextArgAsPath(args, i); + break; + case "-i": + case "--inputs": + if(config.inputListFile != null) + throw new CLIException("Input file cannot be set twice!", args[i]); + config.inputListFile = nextArgAsRegularFileReference(args, i); + break; + case "-t": + case "--threads": + final int threads = nextArgAsInt(args, i); + config.multipleThreads = threads > 1; + config.threadCount = threads; + break; + default: + throw new CLIException(args[i]); + } + } + + public static class CommonConfig { + public long skip; + public long take; + + public Path inputListFile; + public Path progressOutputFile; + public Path progressRestoreFile; + + public boolean multipleThreads; + public int threadCount; + + public int progressWriteInterval; + + public CommonConfig() { + skip = -1L; + take = -1L; + inputListFile = null; + progressOutputFile = Paths.get("marin-progress"); + progressRestoreFile = null; + multipleThreads = false; + threadCount = 1; + progressWriteInterval = 100; + } + } +} diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 5d59b25..746435c 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -3,11 +3,8 @@ import org.apache.maven.index.reader.ChunkReader; import org.apache.maven.index.reader.IndexReader; -import java.io.BufferedWriter; -import java.io.FileWriter; import java.io.IOException; import java.net.URI; -import java.nio.file.Path; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -21,23 +18,14 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { private final IndexReader mavenIndexReader; private final ChunkReader mavenChunkReader; - private final Path progressOutputFilePath; - - final long progressSaveInternal; - private long lastPositionSaved; private long currentPosition; - private boolean saveProgress; private String currentLibraryGA; private String nextLibraryGA; - public LibraryIndexIterator(URI baseUri, Path progressOutputFile, long progressSaveInterval) throws IOException { + public LibraryIndexIterator(URI baseUri) throws IOException { this.libraryHashesSeen = new HashSet<>(); - this.saveProgress = true; - this.progressOutputFilePath = progressOutputFile; - this.progressSaveInternal = progressSaveInterval; - // Build intermediate readers that must be kept open until iterator is done and resources can be closed this.mavenIndexReader = new IndexReader(null, new HttpResourceHandler(baseUri.resolve(".index/"))); this.mavenChunkReader = this.mavenIndexReader.iterator().next(); @@ -45,14 +33,10 @@ public LibraryIndexIterator(URI baseUri, Path progressOutputFile, long progressS // Build the actual iterator for entries in the Maven Central index this.entryIterator = this.mavenChunkReader.iterator(); - this.lastPositionSaved = -1; this.currentPosition = -1; this.currentLibraryGA = null; } - public void setSaveProgress(boolean saveProgress){ - this.saveProgress = saveProgress; - } public void setPosition(long startIdx){ while(this.currentPosition < startIdx && this.hasNext()){ @@ -60,10 +44,6 @@ public void setPosition(long startIdx){ } } - public long getPosition() { - return this.currentPosition; - } - public boolean hasNext() { // If currentLibraryGA is null, we have the first hasNext() call after a call to next(). We must find our new // currentLibraryGA first. @@ -95,7 +75,6 @@ public String next() { this.libraryHashesSeen.add(this.currentLibraryGA.hashCode()); this.currentLibraryGA = null; - writeProgressIfNeeded(); currentPosition += 1; return valueToReturn; @@ -153,15 +132,6 @@ private Map nextEntry(){ return entryIterator.next(); } - private void writeProgressIfNeeded(){ - if(this.saveProgress && this.currentPosition - this.lastPositionSaved >= this.progressSaveInternal){ - try(BufferedWriter writer = new BufferedWriter(new FileWriter(this.progressOutputFilePath.toFile()))){ - // Write position of last GA, as it has now been completed! - writer.write(String.valueOf(currentPosition - 1)); - } catch(IOException ignored){} - this.lastPositionSaved = currentPosition; - } - } @Override public void close() throws Exception { diff --git a/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java b/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java deleted file mode 100644 index 2afacc8..0000000 --- a/src/test/java/org/tudo/sse/MavenCentralAnalysisTest.java +++ /dev/null @@ -1,312 +0,0 @@ -package org.tudo.sse; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.junit.jupiter.api.Test; -import org.tudo.sse.model.Artifact; -import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.index.Package; -import org.tudo.sse.model.pom.License; -import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.utils.IndexIterator; -import org.tudo.sse.utils.MavenCentralAnalysisFactory; -import scala.Tuple2; - -import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; - -class MavenCentralAnalysisTest { - final MavenCentralAnalysis analysisUnderTest = MavenCentralAnalysisFactory.buildEmptyAnalysisWithIndexRequirement(); - final String base = "https://repo1.maven.org/maven2/"; - final Map json; - final Gson gson = new Gson(); - - { - InputStream resource = this.getClass().getClassLoader().getResourceAsStream("MavenAnalysis.json"); - assert resource != null; - Reader targetReader = new InputStreamReader(resource); - json = gson.fromJson(targetReader, new TypeToken>() {}.getType()); - } - - @Test - void parseCmdLinePositive() { - List cliInputs = new ArrayList<>(); - String[] args = {"-st", "500:223"}; - cliInputs.add(args); - args = new String[]{"--coordinates", "src/test/resources/localPom.xml"}; - cliInputs.add(args); - args = new String[]{"--coordinates", "src/test/resources/localPom.xml", "-ip", "src/test/resources/localPom.xml"}; - cliInputs.add(args); - args = new String[]{"-su", "53245:13243"}; - cliInputs.add(args); - args = new String[]{"-ip", "src/test/resources/localPom.xml", "--coordinates", "src/test/resources/localPom.xml"}; - cliInputs.add(args); - args = new String[]{"-ip", "src/test/resources/localPom.xml"}; - cliInputs.add(args); - - try { - Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - args = new String[]{"--output", tmpDir.toString()}; - cliInputs.add(args); - List> expected = (List>) json.get("cliParsePos"); - - int i = 0; - for(String[] input : cliInputs) { - MavenCentralAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); - tester.parseCmdLine(input); - CliInformation result = tester.getSetupInfo(); - List currentExp = expected.get(i); - //run asserts here - assertEquals(Integer.parseInt(currentExp.get(0)), result.getSkip()); - assertEquals(Integer.parseInt(currentExp.get(1)), result.getTake()); - assertEquals(Integer.parseInt(currentExp.get(2)), result.getSince()); - assertEquals(Integer.parseInt(currentExp.get(3)), result.getUntil()); - - if(result.getToCoordinates() != null) { - assertEquals(currentExp.get(4), result.getToCoordinates().toString().replace("\\","/")); - } else { - assertEquals(currentExp.get(4), "null"); - } - - if(result.getToIndexPos() != null) { - assertEquals(currentExp.get(5), result.getToIndexPos().toString().replace("\\","/")); - } else { - assertEquals(currentExp.get(5), "null"); - } - - assertEquals(Boolean.parseBoolean(currentExp.get(6)), result.isOutput()); - i++; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - - } - - @Test - void parseCmdLineNegative() { - List cliInputs = new ArrayList<>(); - String[] args = {"-st", "500:22:3", "--index", "--pom"}; - cliInputs.add(args); - args = new String[]{"--jar", "--coordinates", "-/xcd/"}; - cliInputs.add(args); - args = new String[]{"-st", "88880000", "-su", "--coordinates", "path/to/file", "-ip", "path/to/file", "--index", "--pom", "--jar"}; - cliInputs.add(args); - args = new String[]{"--pom", "53245:13243", "--index"}; - cliInputs.add(args); - args = new String[]{"--jr", "--index"}; - cliInputs.add(args); - args = new String[]{"-ip", "--coordinates", "file/to/path"}; - cliInputs.add(args); - args = new String[]{"--coordinates"}; - cliInputs.add(args); - - for(String[] input : cliInputs) { - assertThrows(RuntimeException.class, () -> analysisUnderTest.parseCmdLine(input)); - } - } - - @Test - void walkPaginated() { - List> inputs = new ArrayList<>(); - inputs.add(new Tuple2<>(500, 10)); - inputs.add(new Tuple2<>(0, 10)); - inputs.add(new Tuple2<>(50000, 100)); - inputs.add(new Tuple2<>(763, 20)); - - for(Tuple2 input : inputs) { - int start1 = (int) input._1; - int take = (int) input._2; - - int start2 = (start1 + take) - 1; - - try { - IndexIterator iterator = new IndexIterator(new URI(base), start1); - List collected1 = new ArrayList<>(analysisUnderTest.walkPaginated(take, iterator)); - - Artifact lastOne = collected1.get(collected1.size() - 1); - int i = 2; - while(lastOne.getIndexInformation().getIndex() > start2) { - lastOne = collected1.get(collected1.size() - i); - i++; - } - - iterator = new IndexIterator(new URI(base), start2); - - List collected2 = new ArrayList<>(analysisUnderTest.walkPaginated(1, iterator)); - assertEquals(lastOne.getIndexInformation().getIndex(), collected2.get(0).getIndexInformation().getIndex()); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - } - - @Test - void indexProcessor() { - - List cliInputs = new ArrayList<>(); - String[] args = {"-st", "200:70"}; - cliInputs.add(args); - args = new String[] {"-st", "499:11", "--name", "src/test/resources/stop.txt"}; - cliInputs.add(args); - args = new String[]{"-st", "500:10"}; - cliInputs.add(args); - args = new String[]{"-st", "275:70", "--name", "src/test/resources/stop.txt"}; - cliInputs.add(args); - - int[] expectedEndings = {270, 510, 510, 345}; - - int i = 0; - for(String[] arg : cliInputs) { - try { - analysisUnderTest.parseCmdLine(arg); - CliInformation current = analysisUnderTest.getSetupInfo(); - analysisUnderTest.indexProcessor(); - - long ending = getEndingIndex(current.getName()); - assertEquals(expectedEndings[i], ending); - - } catch (URISyntaxException | IOException e) { - throw new RuntimeException(e); - } - i++; - } - } - - @Test - void readIdentsIn() { - List cliInputs = new ArrayList<>(); - String[] args = {"--coordinates", "src/main/resources/coordinates.txt"}; - cliInputs.add(args); - args = new String[] {"--coordinates", "src/main/resources/coordinates.txt", "--name", "src/test/resources/stop.txt"}; - cliInputs.add(args); - args = new String[]{"--coordinates", "src/main/resources/coordinates.txt", "-st", "4:5"}; - cliInputs.add(args); - args = new String[]{"--coordinates", "src/main/resources/coordinates.txt", "-st", "4:5", "--name", "src/test/resources/stop.txt"}; - cliInputs.add(args); - args = new String[]{"--coordinates", "src/main/resources/coordinates.txt", "-ip", "src/test/resources/testingIndexPosition.txt"}; - cliInputs.add(args); - args = new String[]{"--coordinates", "src/main/resources/coordinates.txt", "-ip", "src/test/resources/testingIndexPosition.txt", "--name", "src/test/resources/stop.txt"}; - cliInputs.add(args); - - List> expected = (List>) json.get("readIdentsIn"); - int[] expectedEndings = {9, 9, 8, 8, 9, 9}; - - int i = 0; - for(String[] arg : cliInputs) { - List curExpected = expected.get(i); - MavenCentralAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); - tester.parseCmdLine(arg); - CliInformation current = tester.getSetupInfo(); - List idents = tester.readIdentsIn(); - - assertEquals(curExpected.size(), idents.size()); - - long ending = getEndingIndex(current.getName()); - - assertEquals(expectedEndings[i], ending); - - for(int j = 0; j < curExpected.size(); j++) { - assertEquals(curExpected.get(j), idents.get(j).getCoordinates()); - } - i++; - } - - } - - public long getEndingIndex(Path fileName) { - BufferedReader indexReader; - try { - indexReader = new BufferedReader(new FileReader(fileName.toFile())); - String line = indexReader.readLine(); - return Integer.parseInt(line); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Test - void checkMultiThreading() { - List singleArgs = new ArrayList<>(); - List multiArgs = new ArrayList<>(); - - singleArgs.add(new String[]{"-st", "10:1000"}); - multiArgs.add(new String[]{"--multi", "5", "-st", "10:1000"}); - - singleArgs.add(new String[]{"--coordinates", "src/main/resources/coordinates.txt"}); - multiArgs.add(new String[]{"--multi", "5", "--coordinates", "src/main/resources/coordinates.txt"}); - - for(int i = 0; i < singleArgs.size(); i++) { - try { - MavenCentralAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithPomRequirement(); - Map singleResult = tester.runAnalysis(singleArgs.get(i)); - Map multiResult = tester.runAnalysis(multiArgs.get(i)); - assertEquals(singleResult.size(), multiResult.size()); - for(Map.Entry entry : singleResult.entrySet()) { - assert(multiResult.containsKey(entry.getKey())); - } - } catch(URISyntaxException | IOException e) { - fail("Threw an exception"); - } - - } - } - @Test - void checkUseCases() { - MavenCentralAnalysis jarUseCase = new MavenCentralAnalysis(false, false, false, true) { - public long numberOfClassfiles = 0; - @Override - public void analyzeArtifact(Artifact current) { - if(current.getJarInformation() != null) { - numberOfClassfiles += current.getJarInformation().getNumClassFiles(); - } - } - - }; - assertDoesNotThrow( () -> jarUseCase.runAnalysis(new String[]{"-st", "0:300"})); - - MavenCentralAnalysis pomUseCase = new MavenCentralAnalysis(false, true, false, false) { - public final Set uniqueLicenses = new HashSet<>(); - @Override - public void analyzeArtifact(Artifact toAnalyze) { - if(toAnalyze.getPomInformation() != null) { - PomInformation info = toAnalyze.getPomInformation(); - if(!info.getRawPomFeatures().getLicenses().isEmpty()) { - for(License license : info.getRawPomFeatures().getLicenses()) { - if(!uniqueLicenses.contains(license)) { - uniqueLicenses.add(license); - } - } - } - } - } - }; - - assertDoesNotThrow( () -> pomUseCase.runAnalysis(new String[]{"-st", "0:300"})); - - MavenCentralAnalysis indexUseCase = new MavenCentralAnalysis(true, false, false, false) { - public final Set hasJavadocs = new HashSet<>(); - @Override - public void analyzeArtifact(Artifact toAnalyze) { - if(toAnalyze.getIndexInformation() != null) { - List packages = toAnalyze.getIndexInformation().getPackages(); - for(Package current : packages) { - if(current.getJavadocExists() > 0) { - hasJavadocs.add(toAnalyze); - break; - } - } - } - } - }; - - assertDoesNotThrow( () -> indexUseCase.runAnalysis(new String[]{"-st", "0:300"})); - } - -} \ No newline at end of file diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java new file mode 100644 index 0000000..700431e --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -0,0 +1,375 @@ +package org.tudo.sse.analyses; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.tudo.sse.ArtifactFactory; +import org.tudo.sse.CLIException; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.model.index.Package; +import org.tudo.sse.model.pom.License; +import org.tudo.sse.model.pom.PomInformation; +import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MavenCentralAnalysisFactory; +import scala.Tuple2; + +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +class MavenCentralArtifactAnalysisTest { + final MavenCentralArtifactAnalysis analysisUnderTest = MavenCentralAnalysisFactory.buildEmptyAnalysisWithIndexRequirement(); + final String base = "https://repo1.maven.org/maven2/"; + final Map json; + final Gson gson = new Gson(); + + { + InputStream resource = this.getClass().getClassLoader().getResourceAsStream("MavenAnalysis.json"); + assert resource != null; + Reader targetReader = new InputStreamReader(resource); + json = gson.fromJson(targetReader, new TypeToken>() {}.getType()); + } + + @AfterEach + public void cleanup(){ + ArtifactFactory.artifacts.clear(); + } + + @Test + @DisplayName("The CLI parser must parse common argument values correctly") + void parseCLIRegular() throws IOException{ + Path tmpDir = Files.createTempDirectory("maven-resolution-files"); + + ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + parseCLI("--skip-take 500:223"), + parseCLI("--inputs src/test/resources/localPom.xml"), + parseCLI("--progress-restore-file src/test/resources/localPom.xml --inputs src/test/resources/localPom.xml"), + parseCLI("--since-until 53245:13243"), + parseCLI("--inputs src/test/resources/localPom.xml --progress-restore-file src/test/resources/localPom.xml"), + parseCLI("--progress-restore-file src/test/resources/localPom.xml"), + parseCLI("--output " + tmpDir.toString()) + }; + + List> expected = (List>) json.get("cliParsePos"); + + for(int i = 0; i < configs.length; i++){ + final List currExpected = expected.get(i); + final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + + assertEquals(asInt(currExpected.get(0)), currConfig.skip); + assertEquals(asInt(currExpected.get(1)), currConfig.take); + assertEquals(asInt(currExpected.get(2)), currConfig.since); + assertEquals(asInt(currExpected.get(3)), currConfig.until); + + final String expectedInputFile = currExpected.get(4); + + if(expectedInputFile.equalsIgnoreCase("null")){ + assertNull(currConfig.inputListFile); + } else { + assertEquals(expectedInputFile, currConfig.inputListFile.toString().replace("\\","/")); + } + + final String expectedIndexFile = currExpected.get(5); + + if(expectedIndexFile.equalsIgnoreCase("null")){ + assertNull(currConfig.progressRestoreFile); + } else { + assertEquals(expectedIndexFile, currConfig.progressRestoreFile.toString().replace("\\","/")); + } + + assertEquals(Boolean.parseBoolean(currExpected.get(6)), currConfig.outputEnabled); + } + } + + @Test + @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") + void parseCLIShorthands() throws IOException{ + Path tmpDir = Files.createTempDirectory("maven-resolution-files"); + + ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + parseCLI("-st 500:223"), + parseCLI("-i src/test/resources/localPom.xml"), + parseCLI("-prf src/test/resources/localPom.xml -i src/test/resources/localPom.xml"), + parseCLI("-su 53245:13243"), + parseCLI("-i src/test/resources/localPom.xml -prf src/test/resources/localPom.xml"), + parseCLI("-prf src/test/resources/localPom.xml"), + parseCLI("-o " + tmpDir.toString()) + }; + + List> expected = (List>) json.get("cliParsePos"); + + for(int i = 0; i < configs.length; i++){ + final List currExpected = expected.get(i); + final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + + assertEquals(asInt(currExpected.get(0)), currConfig.skip); + assertEquals(asInt(currExpected.get(1)), currConfig.take); + assertEquals(asInt(currExpected.get(2)), currConfig.since); + assertEquals(asInt(currExpected.get(3)), currConfig.until); + + final String expectedInputFile = currExpected.get(4); + + if(expectedInputFile.equalsIgnoreCase("null")){ + assertNull(currConfig.inputListFile); + } else { + assertEquals(expectedInputFile, currConfig.inputListFile.toString().replace("\\","/")); + } + + final String expectedIndexFile = currExpected.get(5); + + if(expectedIndexFile.equalsIgnoreCase("null")){ + assertNull(currConfig.progressRestoreFile); + } else { + assertEquals(expectedIndexFile, currConfig.progressRestoreFile.toString().replace("\\","/")); + } + + assertEquals(Boolean.parseBoolean(currExpected.get(6)), currConfig.outputEnabled); + } + } + + @Test + @DisplayName("The CLI parser must reject invalid inputs") + void parseCmdLineNegative() { + List cliInputs = new ArrayList<>(); + cliInputs.add("--skip-take 500:223:3"); + cliInputs.add("--inputs -/xcd/"); + cliInputs.add("-st 88880000 -su --inputs path/to/file -ip path/to/file"); + cliInputs.add("--inputs"); + + + for(String input : cliInputs) { + assertThrows(RuntimeException.class, () -> parseCLI(input)); + } + } + + @Test + @DisplayName("An analysis must adhere to pagination configurations") + void walkPaginated() { + List> inputs = new ArrayList<>(); + inputs.add(new Tuple2<>(500, 10)); + inputs.add(new Tuple2<>(0, 10)); + inputs.add(new Tuple2<>(50000, 100)); + inputs.add(new Tuple2<>(763, 20)); + + for(Tuple2 input : inputs) { + int start1 = (int) input._1; + int take = (int) input._2; + + int start2 = (start1 + take) - 1; + + try { + IndexIterator iterator = new IndexIterator(new URI(base), start1); + List collected1 = new ArrayList<>(analysisUnderTest.walkPaginated(take, iterator)); + + Artifact lastOne = collected1.get(collected1.size() - 1); + int i = 2; + while(lastOne.getIndexInformation().getIndex() > start2) { + lastOne = collected1.get(collected1.size() - i); + i++; + } + + iterator = new IndexIterator(new URI(base), start2); + + List collected2 = new ArrayList<>(analysisUnderTest.walkPaginated(1, iterator)); + assertEquals(lastOne.getIndexInformation().getIndex(), collected2.get(0).getIndexInformation().getIndex()); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + @Test + @DisplayName("An analysis must store its last processed index regardless of the store interval") + void indexProcessorProgress() { + + List cliInputs = new ArrayList<>(); + String[] args = {"-st", "200:70"}; + cliInputs.add(args); + args = new String[] {"-st", "499:11", "-pof", "src/test/resources/stop.txt"}; + cliInputs.add(args); + args = new String[]{"-st", "500:10"}; + cliInputs.add(args); + args = new String[]{"-st", "275:70", "-pof", "src/test/resources/stop.txt", "-spi", "1000000"}; + cliInputs.add(args); + + int[] expectedEndings = {270, 510, 510, 345}; + + int i = 0; + for(String[] arg : cliInputs) { + try { + analysisUnderTest.runAnalysis(arg); + + Path indexPath = analysisUnderTest.getSetupInfo().progressOutputFile; + assert(indexPath != null); + long ending = getEndingIndex(indexPath); + + assertEquals(expectedEndings[i], ending); + + } catch (URISyntaxException | IOException e) { + throw new RuntimeException(e); + } + i++; + } + } + + @Test + @DisplayName("An analysis must correctly apply CLI options when reading custom input lists") + void readIdentsIn() throws URISyntaxException, IOException { + List cliInputs = new ArrayList<>(); + String[] args = {"--inputs", "src/main/resources/coordinates.txt"}; + cliInputs.add(args); + args = new String[] {"--inputs", "src/main/resources/coordinates.txt", "-pof", "src/test/resources/stop.txt"}; + cliInputs.add(args); + args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5"}; + cliInputs.add(args); + args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; + cliInputs.add(args); + args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; + cliInputs.add(args); + args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; + cliInputs.add(args); + + List> expected = (List>) json.get("readIdentsIn"); + int[] expectedEndings = {9, 9, 8, 8, 9, 9}; + + int i = 0; + for(String[] arg : cliInputs) { + List curExpected = expected.get(i); + MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); + // Apply arguments once + tester.runAnalysis(arg); + List idents = tester.readIdentsIn(); + Path progressFile = tester.getSetupInfo().progressOutputFile; + + assertEquals(curExpected.size(), idents.size()); + + long ending = getEndingIndex(progressFile); + + assertEquals(expectedEndings[i], ending); + + for(ArtifactIdent actual: idents) { + final String actualCoordinates = actual.getCoordinates(); + assert(curExpected.contains(actualCoordinates)); + } + i++; + } + } + + @Test + @DisplayName("An analysis must produce the same results in multithreaded mode") + void checkMultiThreading() { + List singleArgs = new ArrayList<>(); + List multiArgs = new ArrayList<>(); + + singleArgs.add(new String[]{"-st", "10:1000"}); + multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"}); + + singleArgs.add(new String[]{"--inputs", "src/main/resources/coordinates.txt"}); + multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/main/resources/coordinates.txt"}); + + for(int i = 0; i < singleArgs.size(); i++) { + try { + MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithPomRequirement(); + Map singleResult = tester.runAnalysis(singleArgs.get(i)); + Map multiResult = tester.runAnalysis(multiArgs.get(i)); + assertEquals(singleResult.size(), multiResult.size()); + for(Map.Entry entry : singleResult.entrySet()) { + assert(multiResult.containsKey(entry.getKey())); + } + } catch(URISyntaxException | IOException e) { + fail("Threw an exception"); + } + + } + } + + @Test + @DisplayName("An analysis must execute without exception for basic use cases") + void checkUseCases() { + MavenCentralArtifactAnalysis jarUseCase = new MavenCentralArtifactAnalysis(false, false, false, true) { + public long numberOfClassfiles = 0; + @Override + public void analyzeArtifact(Artifact current) { + if(current.getJarInformation() != null) { + numberOfClassfiles += current.getJarInformation().getNumClassFiles(); + } + } + + }; + assertDoesNotThrow( () -> jarUseCase.runAnalysis(new String[]{"-st", "0:300"})); + + MavenCentralArtifactAnalysis pomUseCase = new MavenCentralArtifactAnalysis(false, true, false, false) { + public final Set uniqueLicenses = new HashSet<>(); + @Override + public void analyzeArtifact(Artifact toAnalyze) { + if(toAnalyze.getPomInformation() != null) { + PomInformation info = toAnalyze.getPomInformation(); + if(!info.getRawPomFeatures().getLicenses().isEmpty()) { + for(License license : info.getRawPomFeatures().getLicenses()) { + if(!uniqueLicenses.contains(license)) { + uniqueLicenses.add(license); + } + } + } + } + } + }; + + assertDoesNotThrow( () -> pomUseCase.runAnalysis(new String[]{"-st", "0:300"})); + + MavenCentralArtifactAnalysis indexUseCase = new MavenCentralArtifactAnalysis(true, false, false, false) { + public final Set hasJavadocs = new HashSet<>(); + @Override + public void analyzeArtifact(Artifact toAnalyze) { + if(toAnalyze.getIndexInformation() != null) { + List packages = toAnalyze.getIndexInformation().getPackages(); + for(Package current : packages) { + if(current.getJavadocExists() > 0) { + hasJavadocs.add(toAnalyze); + break; + } + } + } + } + }; + + assertDoesNotThrow( () -> indexUseCase.runAnalysis(new String[]{"-st", "0:300"})); + } + + private long getEndingIndex(Path fileName) { + BufferedReader indexReader; + try { + indexReader = new BufferedReader(new FileReader(fileName.toFile())); + String line = indexReader.readLine(); + return Integer.parseInt(line); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private ArtifactConfigParser.ArtifactConfig parseCLI(String cli) { + try { + final ArtifactConfigParser parser = new ArtifactConfigParser(); + if(cli.isBlank()) return parser.parseArtifactConfig(new String[] {}); + else return parser.parseArtifactConfig(cli.split(" ")); + } catch(CLIException clix){ + throw new RuntimeException(clix); + } + + } + + private int asInt(String s){ + return Integer.parseInt(s); + } + + +} \ No newline at end of file diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index aaed42c..de853fa 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -4,23 +4,22 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.ArtifactFactory; +import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; -import org.tudo.sse.utils.MavenCentralAnalysisFactory; +import org.tudo.sse.utils.CommonConfigParser; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; public class MavenCentralLibraryAnalysisTest { - final MavenCentralLibraryAnalysis emptyAnalysis = - MavenCentralAnalysisFactory.buildEmptyLibraryAnalysisWithNoRequirements(); - @AfterEach public void cleanup(){ ArtifactFactory.artifacts.clear(); @@ -29,64 +28,80 @@ public void cleanup(){ @Test @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ - final String validArgs = "-st 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --multi 12 --save-progress-interval 20"; - final String[] argsArray = validArgs.split(" "); - - emptyAnalysis.config.parseArguments(argsArray); - - assert(emptyAnalysis.config.multipleThreads); - assertEquals(12, emptyAnalysis.config.threadCount); - assertEquals(Paths.get("pom.xml"), emptyAnalysis.config.progressRestoreFile); - assertEquals(Paths.get("marin-progress"), emptyAnalysis.config.progressOutputFile); - assertEquals(20, emptyAnalysis.config.skip); - assertEquals(10000, emptyAnalysis.config.take); - assertEquals(20, emptyAnalysis.config.progressWriteInterval); - assertNull(emptyAnalysis.config.gaInputListFile); + final String validArgs = "--skip-take 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --threads 12 --save-progress-interval 20"; + final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + + assert(config.multipleThreads); + assertEquals(12, config.threadCount); + assertEquals(Paths.get("pom.xml"), config.progressRestoreFile); + assertEquals(Paths.get("marin-progress"), config.progressOutputFile); + assertEquals(20, config.skip); + assertEquals(10000, config.take); + assertEquals(20, config.progressWriteInterval); + assertNull(config.inputListFile); + } + + @Test + @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") + void parseCLIShorthands(){ + final String validArgs = "-st 20:10000 -prf pom.xml -pof marin-progress -t 12 -spi 20"; + final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + + assert(config.multipleThreads); + assertEquals(12, config.threadCount); + assertEquals(Paths.get("pom.xml"), config.progressRestoreFile); + assertEquals(Paths.get("marin-progress"), config.progressOutputFile); + assertEquals(20, config.skip); + assertEquals(10000, config.take); + assertEquals(20, config.progressWriteInterval); + assertNull(config.inputListFile); } + + @Test @DisplayName("The CLI parser must set the correct defaults for empty arguments") void parseCLIDefaults(){ - emptyAnalysis.config.parseArguments(new String[]{}); - - assertFalse(emptyAnalysis.config.multipleThreads); - assertEquals(1, emptyAnalysis.config.threadCount); - assertNull(emptyAnalysis.config.progressRestoreFile); - assertEquals(Paths.get("marin-progress"), emptyAnalysis.config.progressOutputFile); - assertEquals(-1, emptyAnalysis.config.skip); - assertEquals(-1, emptyAnalysis.config.take); - assertEquals(100, emptyAnalysis.config.progressWriteInterval); - assertNull(emptyAnalysis.config.gaInputListFile); + final CommonConfigParser.CommonConfig config = parseCLI(""); + + assertFalse(config.multipleThreads); + assertEquals(1, config.threadCount); + assertNull(config.progressRestoreFile); + assertEquals(Paths.get("marin-progress"), config.progressOutputFile); + assertEquals(-1, config.skip); + assertEquals(-1, config.take); + assertEquals(100, config.progressWriteInterval); + assertNull(config.inputListFile); } @Test @DisplayName("The CLI parser must fail if input files do not exist") void parseNonExistingFile(){ final String validArgs = "--progress-restore-file foo.input"; - final String[] argsArray = validArgs.split(" "); - assertThrows(RuntimeException.class, () -> emptyAnalysis.config.parseArguments(argsArray)); + assertThrows(RuntimeException.class, () -> parseCLI(validArgs)); } @Test @DisplayName("The CLI parser must accept pagination on custom GA input lists") void parseCustomGA(){ - final String validArgs = "-st 20:10000 --coordinates pom.xml"; - final String[] argsArray = validArgs.split(" "); + final String validArgs = "-st 20:10000 --inputs pom.xml"; - emptyAnalysis.config.parseArguments(argsArray); + final CommonConfigParser.CommonConfig config = parseCLI(validArgs); - assertEquals(Paths.get("pom.xml"), emptyAnalysis.config.gaInputListFile); - assertEquals(20, emptyAnalysis.config.skip); + assertEquals(Paths.get("pom.xml"), config.inputListFile); + assertEquals(20, config.skip); } @Test - @DisplayName("The CLI parser must not accept setting an index position on a custom GA input list") + @DisplayName("The CLI parser must accept setting an index position on a custom GA input list") void parseNonExistingIndex(){ - final String validArgs = "--progress-restore-file pom.xml --coordinates pom.xml"; - final String[] argsArray = validArgs.split(" "); + final String validArgs = "--progress-restore-file pom.xml --inputs pom.xml"; - assertThrows(RuntimeException.class, () -> emptyAnalysis.config.parseArguments(argsArray)); + final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + + assertEquals(Paths.get("pom.xml"), config.inputListFile); + assertEquals(Paths.get("pom.xml"), config.progressRestoreFile); } @Test @@ -170,7 +185,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { long progress = Long.parseLong(Files.readString(analysis.config.progressOutputFile)); - assert((progress - 5000) < analysis.config.progressWriteInterval); + assert(progress >= 5005); } @Test @@ -190,7 +205,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 5000:500 --multi 8"; + final String args = "-st 5000:500 --threads 8"; final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); @@ -227,7 +242,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "--coordinates " + validLibraryInput.toAbsolutePath(); + final String args = "--inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); @@ -259,7 +274,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 1:2 --coordinates " + validLibraryInput.toAbsolutePath(); + final String args = "-st 1:2 --inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); @@ -271,7 +286,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { @Test @DisplayName("An analysis with input file must not fail on invalid inputs") - void analysisFromInvalidFile() throws IOException { + void analysisFromInvalidFile() { final Path validLibraryInput = testResource("library-names-invalid.txt"); @@ -285,7 +300,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "--coordinates " + validLibraryInput.toAbsolutePath(); + final String args = "--inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); @@ -293,7 +308,7 @@ protected void analyzeLibrary(String libraryGA, List releases) { @Test @DisplayName("An analysis must apply pagination on top of progress restore values") - void analysisWithRestoreAndPagination() throws IOException { + void analysisWithRestoreAndPagination() { final Path restoreFile = testResource("testingIndexPosition.txt"); @@ -321,11 +336,20 @@ protected void analyzeLibrary(String libraryGA, List releases) { private Path testResource(String pathToResource){ try { - return Path.of(getClass().getClassLoader().getResource(pathToResource).toURI()); + return Path.of(Objects.requireNonNull(getClass().getClassLoader().getResource(pathToResource)).toURI()); } catch (Exception x){ fail("Test setup: Failed to load resource file " + pathToResource, x); } return null; } + private CommonConfigParser.CommonConfig parseCLI(String cli) { + try { + if(cli.isBlank()) return new CommonConfigParser().parseCommonConfig(new String[]{}); + else return new CommonConfigParser().parseCommonConfig(cli.split(" ")); + } catch(CLIException clix) { + throw new RuntimeException(clix); + } + } + } diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java index 2d910a4..8d3b53e 100644 --- a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -16,8 +16,7 @@ class LibraryIndexIteratorTest { @BeforeEach void setIndexIterator() { try { - iteratorUnderTest = new LibraryIndexIterator(new URI("https://repo1.maven.org/maven2/"), - Paths.get("lastIndexProcessed"), 1000); + iteratorUnderTest = new LibraryIndexIterator(new URI("https://repo1.maven.org/maven2/")); } catch (Exception x) { fail(x); } } diff --git a/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java b/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java index bcd0484..a39771e 100644 --- a/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java +++ b/src/test/java/org/tudo/sse/utils/MavenCentralAnalysisFactory.java @@ -1,6 +1,6 @@ package org.tudo.sse.utils; -import org.tudo.sse.MavenCentralAnalysis; +import org.tudo.sse.analyses.MavenCentralArtifactAnalysis; import org.tudo.sse.analyses.MavenCentralLibraryAnalysis; import org.tudo.sse.model.Artifact; @@ -8,21 +8,21 @@ public class MavenCentralAnalysisFactory { - public static MavenCentralAnalysis buildEmptyAnalysisWithNoRequirements() { + public static MavenCentralArtifactAnalysis buildEmptyAnalysisWithNoRequirements() { return buildAnalysisWithRequirements(false, false, false, false); } - public static MavenCentralAnalysis buildEmptyAnalysisWithPomRequirement() { + public static MavenCentralArtifactAnalysis buildEmptyAnalysisWithPomRequirement() { return buildAnalysisWithRequirements(false, true, false, false); } - public static MavenCentralAnalysis buildEmptyAnalysisWithIndexRequirement() { + public static MavenCentralArtifactAnalysis buildEmptyAnalysisWithIndexRequirement() { return buildAnalysisWithRequirements(true, false, false, false); } - private static MavenCentralAnalysis buildAnalysisWithRequirements(boolean requiresIndex, boolean requiresPom, - boolean requiresTransitives, boolean requiresJar) { - return new MavenCentralAnalysis(requiresIndex, requiresPom, requiresTransitives, requiresJar) { + private static MavenCentralArtifactAnalysis buildAnalysisWithRequirements(boolean requiresIndex, boolean requiresPom, + boolean requiresTransitives, boolean requiresJar) { + return new MavenCentralArtifactAnalysis(requiresIndex, requiresPom, requiresTransitives, requiresJar) { @Override public void analyzeArtifact(Artifact current) { diff --git a/src/test/resources/stop.txt b/src/test/resources/stop.txt index f11c82a..5ca234c 100644 --- a/src/test/resources/stop.txt +++ b/src/test/resources/stop.txt @@ -1 +1 @@ -9 \ No newline at end of file +345 \ No newline at end of file From d7c8ef31cfebece9be512a89e2efec091eae2d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 22 Oct 2025 11:43:51 +0200 Subject: [PATCH 16/55] Use common subclass for both analysis types --- .../MavenCentralArtifactAnalysis.java | 102 ++++++------------ .../sse/utils/MavenCentralRepository.java | 1 + .../MavenCentralArtifactAnalysisTest.java | 40 +++---- 3 files changed, 55 insertions(+), 88 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index c6231ea..f4f9b36 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -15,6 +15,7 @@ import org.tudo.sse.utils.ArtifactConfigParser; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; +import org.tudo.sse.utils.MavenCentralRepository; import java.io.*; import java.net.URI; @@ -29,35 +30,12 @@ * The MavenCentralAnalysis enables analysis of artifacts on the maven central repository for jobs of any size. * Takes in cli to be configured to the specific job desired. */ -public abstract class MavenCentralArtifactAnalysis { +public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis { private ArtifactConfigParser.ArtifactConfig artifactConfig; private ActorRef queueActorRef; private ResolverFactory resolverFactory; - /** - * Defines whether this analysis requires artifacts to have index information annotated. - */ - protected final boolean resolveIndex; - - /** - * Defines whether this analysis requires artifacts to have pom information annotated. - */ - protected final boolean resolvePom; - - /** - * Defines whether this analysis requires artifacts to have resolved transitive pom information. - */ - protected final boolean processTransitives; - - /** - * Defines whether this analysis requires artifacts to have jar information annotated. - */ - protected final boolean resolveJar; - - - private static final Logger log = LogManager.getLogger(MavenCentralArtifactAnalysis.class); - /** * Creates a new Maven Central Analysis with the given configuration options. * @@ -71,17 +49,8 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, boolean requiresPom, boolean requiresTransitives, boolean requiresJar) { + super(requiresIndex, requiresPom, requiresTransitives, requiresJar); - if(!requiresIndex && !requiresPom && !requiresTransitives && !requiresJar){ - log.warn("Potential misconfiguration - no data sources (index, POM, JAR) are required by this analysis"); - } else if(requiresTransitives && !requiresPom){ - log.warn("Potential misconfiguration - analysis requires transitive information but no POM information. " + - "POM information will also be collected to provide transitive information."); - } - resolveIndex = requiresIndex; - resolvePom = requiresPom || requiresTransitives; // Cannot have transitive information without POM information - processTransitives = requiresTransitives; - resolveJar = requiresJar; // Initialize with default config, update later artifactConfig = new ArtifactConfigParser.ArtifactConfig(); } @@ -137,11 +106,9 @@ private void printRunInfo(){ * There's a single-threaded implementation contained in this class, as well as a multithreaded one called here but defined in different actor classes. * * @param args cli passed to the program to configure the run - * @return A map of all artifacts collected during the run - * @throws URISyntaxException when there is an issue with the url built - * @throws IOException when there is an issue opening a file */ - public final Map runAnalysis(String[] args) throws URISyntaxException, IOException { + @Override + public final void runAnalysis(String[] args) { // Obtain CLI arguments try { this.artifactConfig = new ArtifactConfigParser().parseArtifactConfig(args); @@ -157,8 +124,8 @@ public final Map runAnalysis(String[] args) throws URIS } if(this.artifactConfig.multipleThreads) { - ActorSystem system = ActorSystem.create("my-system"); - queueActorRef = system.actorOf(QueueActor.props(this.artifactConfig.threadCount, system, 0,0, null), "queueActor"); + final ActorSystem system = ActorSystem.create("marin-actors"); + queueActorRef = system.actorOf(QueueActor.props(this.artifactConfig.threadCount, system, 0, artifactConfig.progressWriteInterval, artifactConfig.progressOutputFile), "queueActor"); if(this.artifactConfig.inputListFile == null) { indexProcessor(); @@ -169,7 +136,7 @@ public final Map runAnalysis(String[] args) throws URIS try { system.getWhenTerminated().toCompletableFuture().get(); } catch (Exception e) { - e.printStackTrace(); + log.warn("Failed to close actor system", e); } } else { if(this.artifactConfig.inputListFile == null) { @@ -178,47 +145,46 @@ public final Map runAnalysis(String[] args) throws URIS readIdentsIn(); } } - - return ArtifactFactory.artifacts; } /** * Handles walking the maven central index, choosing how to do so based on the configuration. - * - * @throws URISyntaxException when there is an issue with the url built - * @throws IOException when there is an issue opening a file * @see ArtifactIdent */ - public void indexProcessor() throws URISyntaxException, IOException { - String base = "https://repo1.maven.org/maven2/"; + public void indexProcessor() { IndexIterator indexIterator; - //set up indexIterator here (skip to a position or start from the start) - if (this.artifactConfig.progressRestoreFile != null) { - indexIterator = new IndexIterator(new URI(base), getStartingPos()); - } else if(this.artifactConfig.skip != -1) { - indexIterator = new IndexIterator(new URI(base), this.artifactConfig.skip); - } else { - indexIterator = new IndexIterator(new URI(base)); - } + try { + //set up indexIterator here (skip to a position or start from the start) + if (this.artifactConfig.progressRestoreFile != null) { + indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI, getStartingPos()); + } else if(this.artifactConfig.skip != -1) { + indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI, this.artifactConfig.skip); + } else { + indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI); + } - if (resolveIndex) { - if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { - walkPaginated(this.artifactConfig.take, indexIterator); + if (resolveIndex) { + if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { + walkPaginated(this.artifactConfig.take, indexIterator); + } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { + walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); + } else { + walkAllIndexes(indexIterator); + } + } else if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { + lazyWalkPaginated(this.artifactConfig.take, indexIterator); } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { - walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); + lazyWalkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); } else { - walkAllIndexes(indexIterator); + lazyWalkAllIndexes(indexIterator); } - } else if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { - lazyWalkPaginated(this.artifactConfig.take, indexIterator); - } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { - lazyWalkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); - } else { - lazyWalkAllIndexes(indexIterator); + + writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); + } catch (IOException iox){ + throw new RuntimeException(iox); } - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } private void processIndex(Artifact current) { diff --git a/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java b/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java index 4b60df8..4aa682b 100644 --- a/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java +++ b/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java @@ -17,6 +17,7 @@ public final class MavenCentralRepository { public static final String RepoBasePath = "https://repo1.maven.org/maven2/"; + public static final URI RepoBaseURI = URI.create(RepoBasePath); private static MavenCentralRepository theInstance = null; diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 700431e..753a98a 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -205,18 +205,14 @@ void indexProcessorProgress() { int i = 0; for(String[] arg : cliInputs) { - try { - analysisUnderTest.runAnalysis(arg); + analysisUnderTest.runAnalysis(arg); - Path indexPath = analysisUnderTest.getSetupInfo().progressOutputFile; - assert(indexPath != null); - long ending = getEndingIndex(indexPath); + Path indexPath = analysisUnderTest.getSetupInfo().progressOutputFile; + assert(indexPath != null); + long ending = getEndingIndex(indexPath); - assertEquals(expectedEndings[i], ending); + assertEquals(expectedEndings[i], ending); - } catch (URISyntaxException | IOException e) { - throw new RuntimeException(e); - } i++; } } @@ -277,18 +273,22 @@ void checkMultiThreading() { multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/main/resources/coordinates.txt"}); for(int i = 0; i < singleArgs.size(); i++) { - try { - MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithPomRequirement(); - Map singleResult = tester.runAnalysis(singleArgs.get(i)); - Map multiResult = tester.runAnalysis(multiArgs.get(i)); - assertEquals(singleResult.size(), multiResult.size()); - for(Map.Entry entry : singleResult.entrySet()) { - assert(multiResult.containsKey(entry.getKey())); - } - } catch(URISyntaxException | IOException e) { - fail("Threw an exception"); - } + MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithPomRequirement(); + + tester.runAnalysis(singleArgs.get(i)); + Set singleResult = ArtifactFactory.artifacts.keySet(); + cleanup(); + + tester.runAnalysis(multiArgs.get(i)); + Set multiResult = ArtifactFactory.artifacts.keySet(); + cleanup(); + + assertEquals(singleResult.size(), multiResult.size()); + + for(ArtifactIdent single : singleResult) { + assert(multiResult.contains(single)); + } } } From b7e62ad9aa5f359aa922045cef5516a67bbda86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 22 Oct 2025 14:37:16 +0200 Subject: [PATCH 17/55] Streamline handling of progress counter: It is the number of GAs / GAVs that have been fully processed, not just queued. Some bugfixes. Skip will no longer be applied if progress is restored. --- .../MavenCentralArtifactAnalysis.java | 464 ++++++++---------- .../analyses/MavenCentralLibraryAnalysis.java | 11 +- .../tudo/sse/multithreading/QueueActor.java | 24 +- .../sse/multithreading/ResolverActor.java | 2 +- .../sse/utils/FileBasedArtifactIterator.java | 45 ++ .../MavenCentralArtifactAnalysisTest.java | 28 +- 6 files changed, 261 insertions(+), 313 deletions(-) create mode 100644 src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index f4f9b36..fda1fbc 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -2,8 +2,6 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.tudo.sse.ArtifactFactory; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; @@ -13,18 +11,17 @@ import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.utils.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; import org.tudo.sse.utils.MavenCentralRepository; import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; -import java.util.Map; /** * The MavenCentralAnalysis enables analysis of artifacts on the maven central repository for jobs of any size. @@ -36,6 +33,10 @@ public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis private ActorRef queueActorRef; private ResolverFactory resolverFactory; + + private long currentPosition = 0; + private long lastPositionSaved = 0; + /** * Creates a new Maven Central Analysis with the given configuration options. * @@ -116,7 +117,10 @@ public final void runAnalysis(String[] args) { } catch(CLIException clix){ throw new RuntimeException(clix); } - + + ActorSystem system = null; + + // Initialize resolver factory if(this.artifactConfig.outputEnabled) { resolverFactory = new ResolverFactory(this.artifactConfig.outputEnabled, this.artifactConfig.outputDirectory, processTransitives); } else { @@ -124,26 +128,25 @@ public final void runAnalysis(String[] args) { } if(this.artifactConfig.multipleThreads) { - final ActorSystem system = ActorSystem.create("marin-actors"); - queueActorRef = system.actorOf(QueueActor.props(this.artifactConfig.threadCount, system, 0, artifactConfig.progressWriteInterval, artifactConfig.progressOutputFile), "queueActor"); + system = ActorSystem.create("marin-actors"); + // Compute position at which processing will start + final long initialPosition = getInitialPosition(); + this.queueActorRef = system.actorOf(QueueActor.props(this.artifactConfig.threadCount, system, initialPosition, + artifactConfig.progressWriteInterval, artifactConfig.progressOutputFile), "queueActor"); + } - if(this.artifactConfig.inputListFile == null) { - indexProcessor(); - } else { - readIdentsIn(); - } + if(this.artifactConfig.inputListFile == null) { + indexProcessor(); + } else { + processArtifactsFromInputFile(); + } + if(system != null){ try { + // Tell the queue that no more work items will follow + this.queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); system.getWhenTerminated().toCompletableFuture().get(); - } catch (Exception e) { - log.warn("Failed to close actor system", e); - } - } else { - if(this.artifactConfig.inputListFile == null) { - indexProcessor(); - } else { - readIdentsIn(); - } + } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} } } @@ -152,35 +155,24 @@ public final void runAnalysis(String[] args) { * @see ArtifactIdent */ public void indexProcessor() { - IndexIterator indexIterator; try { - //set up indexIterator here (skip to a position or start from the start) - if (this.artifactConfig.progressRestoreFile != null) { - indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI, getStartingPos()); - } else if(this.artifactConfig.skip != -1) { - indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI, this.artifactConfig.skip); - } else { - indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI); - } + final IndexIterator indexIterator = new IndexIterator(MavenCentralRepository.RepoBaseURI); - if (resolveIndex) { - if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { - walkPaginated(this.artifactConfig.take, indexIterator); - } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { - walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); - } else { - walkAllIndexes(indexIterator); - } - } else if (this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { - lazyWalkPaginated(this.artifactConfig.take, indexIterator); - } else if (this.artifactConfig.since != -1 && this.artifactConfig.until != -1) { - lazyWalkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); + // Skip to starting position + final long startingPosition = getInitialPosition(); + skipN(startingPosition, indexIterator); + + if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1){ + walkPaginated(this.artifactConfig.take, indexIterator); + } else if(this.artifactConfig.since != -1 && this.artifactConfig.until != -1){ + walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); } else { - lazyWalkAllIndexes(indexIterator); + walkAllIndexes(indexIterator); } - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); + // Write final position only if not in multithreaded mode - otherwise Queue actor handles progress + if(!artifactConfig.multipleThreads) writePosition(); } catch (IOException iox){ throw new RuntimeException(iox); } @@ -200,34 +192,45 @@ private void processIndex(Artifact current) { * Iterates over all indexes in the maven central index and creates an artifact with the metadata collected. * * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifacts containing the maven central index metadata - * @see Artifact - * @see IndexInformation + * @return a list of artifact identifiers processed by this method + * @see ArtifactIdent * @throws IOException when there is an issue opening a file */ - public List walkAllIndexes(IndexIterator indexIterator) throws IOException { - List artifacts = new ArrayList<>(); + List walkAllIndexes(IndexIterator indexIterator) throws IOException { + final List artifactIdents = new ArrayList<>(); + long entriesTaken = 0L; while(indexIterator.hasNext()) { - Artifact current = ArtifactFactory.createArtifact(indexIterator.next()); - artifacts.add(current); + final IndexInformation current = indexIterator.next(); + entriesTaken += 1L; + this.currentPosition += 1L; + if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { Path filePath = this.artifactConfig.outputDirectory.resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } - processIndex(current); - if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); - } - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); + if(resolveIndex){ + final Artifact artifact = ArtifactFactory.createArtifact(current); + processIndex(artifact); + } else { + processIndexIdentifier(current.getIdent()); + } + + artifactIdents.add(current.getIdent()); + + writePositionIfNeeded(); } + if(artifactConfig.multipleThreads) + log.info("Queued a total of {} entries for processing.", entriesTaken); + else + log.info("Processed a total of {} entries.", entriesTaken); + indexIterator.closeReader(); - return artifacts; + return artifactIdents; } /** @@ -235,77 +238,99 @@ public List walkAllIndexes(IndexIterator indexIterator) throws IOExcep * * @param take number of artifacts from the starting point to capture * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifacts containing the maven central index metadata - * @see Artifact - * @see IndexInformation + * @return a list of artifact identifiers processed by this method + * @see ArtifactIdent * @throws IOException when there is an issue opening a file */ - public List walkPaginated(long take, IndexIterator indexIterator) throws IOException { - List artifacts = new ArrayList<>(); + List walkPaginated(long take, IndexIterator indexIterator) throws IOException { + final List artifactIdents = new ArrayList<>(); + long entriesTaken = 0L; + + while(indexIterator.hasNext() && entriesTaken < take) { + final IndexInformation current = indexIterator.next(); + entriesTaken += 1; + this.currentPosition += 1L; - take += indexIterator.getIndex(); - while(indexIterator.hasNext() && indexIterator.getIndex() < take) { - Artifact current = ArtifactFactory.createArtifact(indexIterator.next()); if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { Path filePath = this.artifactConfig.outputDirectory.resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } - artifacts.add(current); - processIndex(current); - if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); - } - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); + if(resolveIndex){ + final Artifact artifact = ArtifactFactory.createArtifact(current); + processIndex(artifact); + } else { + processIndexIdentifier(current.getIdent()); + } + + artifactIdents.add(current.getIdent()); + + writePositionIfNeeded(); } + if(artifactConfig.multipleThreads) + log.info("Queued a total of {} entries for processing.", entriesTaken); + else + log.info("Processed a total of {} entries.", entriesTaken); + indexIterator.closeReader(); - return artifacts; + return artifactIdents; } /** * Iterates over all indexes in the maven central index. - * It collects a list of artifacts that are within the range of since and until. + * It collects a list of artifact identifiers that are within the range of since and until. * * @param since lower bound of dates of artifacts to collect * @param until upper bound of dates of artifacts to collect * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifacts containing the maven central index metadata - * @see Artifact - * @see IndexInformation + * @return a list of artifact identifiers processed by this method + * @see ArtifactIdent * @throws IOException when there is an issue opening a file */ - public List walkDates(long since, long until, IndexIterator indexIterator) throws IOException { - List artifacts = new ArrayList<>(); + List walkDates(long since, long until, IndexIterator indexIterator) throws IOException { + final List artifactIdents = new ArrayList<>(); + long entriesTaken = 0L; long currentToSince; + while(indexIterator.hasNext()) { - IndexInformation temp = indexIterator.next(); - currentToSince = temp.getLastModified(); + IndexInformation current = indexIterator.next(); + entriesTaken += 1L; + this.currentPosition += 1L; + + currentToSince = current.getLastModified(); + if(currentToSince >= since && currentToSince < until) { - Artifact current = ArtifactFactory.createArtifact(indexIterator.next()); if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { Path filePath = this.artifactConfig.outputDirectory.resolve(current.getIdent().getGroupID() + "-" + current.getIdent().getArtifactID() + "-" + current.getIdent().getVersion() + ".txt"); if(!Files.exists(filePath)) { Files.createFile(filePath); } } - artifacts.add(current); - processIndex(current); + + if(resolveIndex){ + final Artifact artifact = ArtifactFactory.createArtifact(current); + processIndex(artifact); + } else { + processIndexIdentifier(current.getIdent()); + } + + artifactIdents.add(current.getIdent()); } - if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); - } - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); + writePositionIfNeeded(); } + if(artifactConfig.multipleThreads) + log.info("Queued a total of {} entries for processing.", entriesTaken); + else + log.info("Processed a total of {} entries.", entriesTaken); + indexIterator.closeReader(); - return artifacts; + return artifactIdents; } private void processIndexIdentifier(ArtifactIdent ident) { @@ -320,194 +345,81 @@ private void processIndexIdentifier(ArtifactIdent ident) { } /** - * Iterates over all the indexes in the maven central index and just collects the identifiers - * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifact identifiers - * @see ArtifactIdent - * @throws IOException when there is an issue opening a file + * Reads in identifiers from a file, using the configuration passed into it. + * + * @return a list of identifiers collected from the file */ - public List lazyWalkAllIndexes(IndexIterator indexIterator) throws IOException { - List idents = new ArrayList<>(); - while(indexIterator.hasNext()) { - ArtifactIdent ident = indexIterator.next().getIdent(); - idents.add(ident); - if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { - Path filePath = this.artifactConfig.outputDirectory.resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); - if(!Files.exists(filePath)) { - Files.createFile(filePath); - } - } - processIndexIdentifier(ident); - if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); + List processArtifactsFromInputFile() { + final Iterator fileIterator = new FileBasedArtifactIterator(this.artifactConfig.inputListFile); + + // Restore from progress file if available + if(this.artifactConfig.progressRestoreFile != null){ + long lastProgress = getStartingPos(); + log.info("Restoring previous progress from file (position {})", lastProgress); + + this.skipN(lastProgress, fileIterator); + } else if(this.artifactConfig.skip > 0){ + // Skip configured values only if we did not restore from progress file + log.info("Skipping {} entries from file", artifactConfig.skip); + skipN(this.artifactConfig.skip, fileIterator); } - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); - } + // Warn if after skipping there are no entries left + if(!fileIterator.hasNext()) + log.warn("No more contents left to process in input file: {}", this.artifactConfig.inputListFile); - indexIterator.closeReader(); - return idents; - } + final List identifiers = new ArrayList<>(); + final boolean takeLimited = this.artifactConfig.take >= 0; - /** - * Iterates over a given number of indexes from the maven central index, and collects just the identifiers. - * @param take how many indexes from the starting position to traverse - * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifact identifiers - * @see ArtifactIdent - * @throws IOException when there is an issue opening a file - */ - public List lazyWalkPaginated(long take, IndexIterator indexIterator) throws IOException{ - List idents = new ArrayList<>(); + long entriesTaken = 0L; + while ((!takeLimited || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { - take += indexIterator.getIndex(); - while(indexIterator.hasNext() && indexIterator.getIndex() < take) { - ArtifactIdent ident = indexIterator.next().getIdent(); - idents.add(ident); - if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { - Path filePath = this.artifactConfig.outputDirectory.resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); - if(!Files.exists(filePath)) { - Files.createFile(filePath); - } - } - processIndexIdentifier(ident); - if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); - } + ArtifactIdent current = null; - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); - } - indexIterator.closeReader(); - return idents; - } + try { current = fileIterator.next(); } catch (Exception x){ + log.error("Malformed GAV triple in input file {} line {}", this.artifactConfig.inputListFile, + this.currentPosition); + } - /** - * Iterates over all index in the maven central index. - * It collects a list of artifact identifiers that are within the range of since and until. - * - * @param since lower bound of dates of artifacts to collect - * @param until upper bound of dates of artifacts to collect - * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifact identifiers within since and until - * @see ArtifactIdent - * @throws IOException when there is an issue opening a file - */ - public List lazyWalkDates(long since, long until, IndexIterator indexIterator) throws IOException{ - List idents = new ArrayList<>(); + this.currentPosition += 1L; + entriesTaken += 1L; - long currentToSince; - while(indexIterator.hasNext()) { - IndexInformation temp = indexIterator.next(); - currentToSince = temp.getLastModified(); - if(currentToSince >= since && currentToSince < until) { - ArtifactIdent ident = temp.getIdent(); - idents.add(ident); - if(this.artifactConfig.outputEnabled && !resolvePom && !resolveJar) { - Path filePath = this.artifactConfig.outputDirectory.resolve(ident.getGroupID() + "-" + ident.getArtifactID() + "-" + ident.getVersion() + ".txt"); - if(!Files.exists(filePath)) { - Files.createFile(filePath); - } - } - processIndexIdentifier(ident); + if(current != null){ + identifiers.add(current); + processIndexIdentifier(current); } - if(indexIterator.getIndex() % this.artifactConfig.progressWriteInterval == 0) - writeLastProcessed(indexIterator.getIndex(), this.artifactConfig.progressOutputFile); } - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); - } - - indexIterator.closeReader(); + if(artifactConfig.multipleThreads){ + log.info("Queued a total of {} entries for processing from file {}.", entriesTaken, this.artifactConfig.inputListFile); + } else { + log.info("Processed a total of {} entries from file {}.", entriesTaken, this.artifactConfig.inputListFile); - return idents; - } + // Write position one final time - in multithreaded mode the queue worker will take care of this + writePosition(); + } - private void writeLastProcessed(long lastIndexProcessed, Path name) throws IOException { - BufferedWriter writer = new BufferedWriter(new FileWriter(name.toFile())); - writer.write(String.valueOf(lastIndexProcessed)); - writer.close(); + return identifiers; } /** - * Reads in identifiers from a file, using the configuration passed into it. - * - * @return a list of identifiers collected from the file + * Invokes all resolvers as defined by the analysis configuration to enrich the given artifact identifier. + * @param identifier Artifact identifier to enrich */ - public List readIdentsIn() { - Path toCoordinates = this.artifactConfig.inputListFile; - - BufferedReader coordinatesReader; - List identifiers = new ArrayList<>(); - try{ - coordinatesReader = new BufferedReader(new FileReader(toCoordinates.toFile())); - String line = coordinatesReader.readLine(); - - int i = -1; - if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1) { - int toSkip = 0; - int curTake = 0; - while(line != null && curTake < this.artifactConfig.take) { - if(toSkip >= this.artifactConfig.skip) { - String[] parts = line.split(":"); - if(parts.length == 3) { - ArtifactIdent current = new ArtifactIdent(parts[0], parts[1], parts[2]); - identifiers.add(current); - processIndexIdentifier(current); - curTake++; - } else { - log.error("unable to process Artifact Identifier {} at position {}", line, i); - } - } - line = coordinatesReader.readLine(); - toSkip++; - i++; - } - } else if(this.artifactConfig.progressRestoreFile != null) { - long start = getStartingPos(); - int curPos = 0; - while(line != null) { - if(curPos >= start) { - String[] parts = line.split(":"); - if(parts.length == 3) { - ArtifactIdent current = new ArtifactIdent(parts[0], parts[1], parts[2]); - identifiers.add(current); - processIndexIdentifier(current); - } else { - log.error("unable to process Artifact Identifier {} at position {}", line, i); - } - } - line = coordinatesReader.readLine(); - curPos++; - i++; - } - } else { - while(line != null) { - String[] parts = line.split(":"); - if(parts.length == 3) { - ArtifactIdent current = new ArtifactIdent(parts[0], parts[1], parts[2]); - identifiers.add(current); - processIndexIdentifier(current); - } else { - log.error("unable to process Artifact Identifier {} at position {}", line, i); - } - line = coordinatesReader.readLine(); - i++; - } - } - - if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); - } - - writeLastProcessed(i, this.artifactConfig.progressOutputFile); - coordinatesReader.close(); - } catch(IOException e) { - throw new RuntimeException(e); + public void callResolver(ArtifactIdent identifier) { + if(resolvePom && resolveJar) { + resolverFactory.runBoth(identifier); + } else if(resolvePom) { + resolverFactory.runPom(identifier); + } else if(resolveJar) { + resolverFactory.runJar(identifier); } - return identifiers; + } + + private long getInitialPosition() { + if(artifactConfig.progressRestoreFile != null) return getStartingPos(); + else if(artifactConfig.skip > 0) return artifactConfig.skip; + else return 0L; } private long getStartingPos() { @@ -521,17 +433,25 @@ private long getStartingPos() { } } - /** - * Invokes all resolvers as defined by the analysis configuration to enrich the given artifact identifier. - * @param identifier Artifact identifier to enrich - */ - public void callResolver(ArtifactIdent identifier) { - if(resolvePom && resolveJar) { - resolverFactory.runBoth(identifier); - } else if(resolvePom) { - resolverFactory.runPom(identifier); - } else if(resolveJar) { - resolverFactory.runJar(identifier); + private void skipN(long N, Iterator it){ + for(int i = 0; i < N && it.hasNext(); i++){ + it.next(); + this.currentPosition += 1L; } } + + private void writePositionIfNeeded(){ + if(!this.artifactConfig.multipleThreads && + this.currentPosition - this.lastPositionSaved > artifactConfig.progressWriteInterval){ + writePosition(); + } + } + + private void writePosition() { + try(BufferedWriter writer = Files.newBufferedWriter(this.artifactConfig.progressOutputFile)) { + writer.write(Long.toString(this.currentPosition)); + } catch(IOException ignored) {} + + this.lastPositionSaved = currentPosition; + } } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 6cc48a8..b761820 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -67,7 +67,7 @@ public final void runAnalysis(String[] args){ // iterator will produce strings of form :. Iterator gaIterator = getGaIterator(); - // Restore previous progress + // We first start by restoring progress if possible if(config.progressRestoreFile != null){ final long startingPosition = getProgressFromRestoreFile(); log.info("Restoring previous progress (position {})", startingPosition); @@ -78,10 +78,11 @@ public final void runAnalysis(String[] args){ } else gaIterator.next(); currentPosition += 1L; } - } - - // If there are values to skip, we skip them manually - if(config.skip > 0){ + // Notify users that skip will not be applied + if(config.skip > 0) + log.info("Not applying skip value because progress was restored from previous run"); + } else if(config.skip > 0){ + // We only apply a skip if we did not restore previous progress log.info("Skipping {} library names", config.skip); for(int i = 0; i < config.skip; i++){ if(!gaIterator.hasNext()){ diff --git a/src/main/java/org/tudo/sse/multithreading/QueueActor.java b/src/main/java/org/tudo/sse/multithreading/QueueActor.java index a276e8b..664d871 100644 --- a/src/main/java/org/tudo/sse/multithreading/QueueActor.java +++ b/src/main/java/org/tudo/sse/multithreading/QueueActor.java @@ -17,7 +17,6 @@ /** * This class represents the processing queue that resolves jobs and sends them to different threads. * The size of the jobs and threads are determined by the configuration set in CliInformation. - * @see org.tudo.sse.CliInformation */ public class QueueActor extends AbstractActor { @@ -71,27 +70,6 @@ public Receive createReceive() { return ReceiveBuilder.create() .match(ProcessIdentifierMessage.class, this::forwardToResolvers) .match(ProcessLibraryMessage.class, this::forwardToResolvers) - .match(String.class, message -> { - synchronized (jobQueue){ - if(!jobQueue.isEmpty()) { - getSender().tell(jobQueue.peek(), getSelf()); - jobQueue.remove(); - if(jobQueue.size() % 10 == 0) log.trace("Distributed a job, queue size " + jobQueue.size()); - } else { - synchronized(curNumResolvers) { - if(indexFinished && curNumResolvers.get() == 1) { - log.trace("Shutting down system"); - system.terminate(); - } else { - log.trace("Killing a worker thread"); - getSender().tell(PoisonPill.getInstance(), getSelf()); - curNumResolvers.decrementAndGet(); - } - } - } - } - - }) .match(WorkItemFinishedMessage.class, workItemFinishedMessage -> { // Track completion of work items workItemsCompleted.incrementAndGet(); @@ -102,7 +80,7 @@ public Receive createReceive() { synchronized (jobQueue){ if(!jobQueue.isEmpty()) { getSender().tell(jobQueue.remove(), getSelf()); - if(jobQueue.size() % 10 == 0) log.trace("Distributed a job, queue size " + jobQueue.size()); + if(jobQueue.size() % 10 == 0) log.trace("Distributed a job, queue size {}", jobQueue.size()); } else { synchronized(curNumResolvers) { if(indexFinished && curNumResolvers.get() == 1) { diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index c5f5704..828cfd9 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -30,7 +30,7 @@ public Receive createReceive() { .match(ProcessIdentifierMessage.class, message -> { message.getInstance().callResolver(message.getIdentifier()); message.getInstance().analyzeArtifact(ArtifactFactory.getArtifact(message.getIdentifier())); - getSender().tell("Finished", getSelf()); + getSender().tell(WorkItemFinishedMessage.getInstance(), getSelf()); }) .match(ProcessLibraryMessage.class, message -> { message.getProcessEntryCallback().get(); diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java new file mode 100644 index 0000000..7f7aa85 --- /dev/null +++ b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java @@ -0,0 +1,45 @@ +package org.tudo.sse.utils; + +import org.tudo.sse.model.ArtifactIdent; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; + +public class FileBasedArtifactIterator implements Iterator { + + private final Path inputPath; + + private Iterator lineIterator; + + public FileBasedArtifactIterator(Path input){ + this.inputPath = input; + } + + @Override + public boolean hasNext() { + if(this.lineIterator == null){ + try { + var lines = Files.readAllLines(this.inputPath); + this.lineIterator = lines.iterator(); + } catch (IOException iox){ + throw new IllegalArgumentException("Failed to access input file", iox); + } + } + + return this.lineIterator.hasNext(); + } + + @Override + public ArtifactIdent next() { + String line = this.lineIterator.next(); + + String[] parts = line.split(":"); + if(parts.length == 3) { + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } else { + throw new IllegalStateException("Not a valid GAV-Triple: " + line); + } + } +} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 753a98a..e3eee4a 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -168,19 +168,20 @@ void walkPaginated() { try { IndexIterator iterator = new IndexIterator(new URI(base), start1); - List collected1 = new ArrayList<>(analysisUnderTest.walkPaginated(take, iterator)); + List collected1 = new ArrayList<>(analysisUnderTest.walkPaginated(take, iterator)); - Artifact lastOne = collected1.get(collected1.size() - 1); + Artifact lastOne = ArtifactFactory.getArtifact(collected1.get(collected1.size() - 1)); int i = 2; while(lastOne.getIndexInformation().getIndex() > start2) { - lastOne = collected1.get(collected1.size() - i); + lastOne = ArtifactFactory.getArtifact(collected1.get(collected1.size() - i)); i++; } iterator = new IndexIterator(new URI(base), start2); - List collected2 = new ArrayList<>(analysisUnderTest.walkPaginated(1, iterator)); - assertEquals(lastOne.getIndexInformation().getIndex(), collected2.get(0).getIndexInformation().getIndex()); + List collected2 = new ArrayList<>(analysisUnderTest.walkPaginated(1, iterator)); + long lastOne2 = ArtifactFactory.getArtifact(collected2.get(0)).getIndexInformation().getIndex(); + assertEquals(lastOne.getIndexInformation().getIndex(), lastOne2); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); } @@ -205,9 +206,10 @@ void indexProcessorProgress() { int i = 0; for(String[] arg : cliInputs) { - analysisUnderTest.runAnalysis(arg); + MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); + tester.runAnalysis(arg); - Path indexPath = analysisUnderTest.getSetupInfo().progressOutputFile; + Path indexPath = tester.getSetupInfo().progressOutputFile; assert(indexPath != null); long ending = getEndingIndex(indexPath); @@ -235,22 +237,24 @@ void readIdentsIn() throws URISyntaxException, IOException { cliInputs.add(args); List> expected = (List>) json.get("readIdentsIn"); - int[] expectedEndings = {9, 9, 8, 8, 9, 9}; + int[] expectedEndings = {10, 10, 9, 9, 10, 10}; int i = 0; for(String[] arg : cliInputs) { List curExpected = expected.get(i); MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); - // Apply arguments once + + // Apply arguments, check progress is stored correctly tester.runAnalysis(arg); - List idents = tester.readIdentsIn(); Path progressFile = tester.getSetupInfo().progressOutputFile; + long ending = getEndingIndex(progressFile); + assertEquals(expectedEndings[i], ending); + // Process file a second time to obtain return value + List idents = tester.processArtifactsFromInputFile(); assertEquals(curExpected.size(), idents.size()); - long ending = getEndingIndex(progressFile); - assertEquals(expectedEndings[i], ending); for(ArtifactIdent actual: idents) { final String actualCoordinates = actual.getCoordinates(); From 9a85d600155cab579b02163396123463878877a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Thu, 23 Oct 2025 15:58:39 +0200 Subject: [PATCH 18/55] Add custom logging bridge for internal Log4j and OPAL logging. Disable global OPAL logging. Enable custom logging configurations when explicitly using OPAL projects --- .../java/org/tudo/sse/ArtifactFactory.java | 9 ++ .../sse/analyses/MavenCentralAnalysis.java | 6 ++ .../tudo/sse/model/jar/JarInformation.java | 49 ++++++++- .../org/tudo/sse/utils/MarinOpalLogger.java | 101 ++++++++++++++++++ src/main/resources/log4j2.xml | 2 +- .../sse/model/jar/JarInformationTest.java | 10 ++ 6 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/tudo/sse/utils/MarinOpalLogger.java diff --git a/src/main/java/org/tudo/sse/ArtifactFactory.java b/src/main/java/org/tudo/sse/ArtifactFactory.java index d4c2998..45582a7 100644 --- a/src/main/java/org/tudo/sse/ArtifactFactory.java +++ b/src/main/java/org/tudo/sse/ArtifactFactory.java @@ -2,10 +2,14 @@ import java.util.HashMap; import java.util.Map; + +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; import org.tudo.sse.model.*; import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.model.jar.JarInformation; import org.tudo.sse.model.pom.PomInformation; +import org.tudo.sse.utils.MarinOpalLogger; /** * The ArtifactFactory handles the creation and storage of artifacts resolved. @@ -13,6 +17,11 @@ */ public final class ArtifactFactory { + static { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + private ArtifactFactory() {} /** diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java index 4f28fea..a9f33b5 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java @@ -2,6 +2,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.utils.MarinOpalLogger; abstract class MavenCentralAnalysis { @@ -51,6 +54,9 @@ protected MavenCentralAnalysis(boolean requiresIndex, "POM information will also be collected to provide transitive information."); } + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + resolveIndex = requiresIndex; resolvePom = requiresPom || requiresTransitives; // Cannot have transitive information without POM information processTransitives = requiresTransitives; diff --git a/src/main/java/org/tudo/sse/model/jar/JarInformation.java b/src/main/java/org/tudo/sse/model/jar/JarInformation.java index 17bc7c8..08ac190 100644 --- a/src/main/java/org/tudo/sse/model/jar/JarInformation.java +++ b/src/main/java/org/tudo/sse/model/jar/JarInformation.java @@ -3,15 +3,18 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opalj.br.analyses.Project; +import org.opalj.br.package$; import org.opalj.br.reader.Java17Framework$; import org.opalj.br.reader.Java17LibraryFramework; import org.opalj.br.reader.Java17LibraryFramework$; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.ArtifactInformation; import org.tudo.sse.resolution.FileNotFoundException; +import org.tudo.sse.utils.MarinOpalLogger; import org.tudo.sse.utils.MavenCentralRepository; import scala.Tuple2; import scala.jdk.CollectionConverters; +import scala.runtime.BoxedUnit; import java.io.File; import java.io.IOException; @@ -207,8 +210,23 @@ public List getOpalClassFileRepresentations() throws IOE * really need them. */ public Project getOpalProject() throws IOException { + return getOpalProject(MarinOpalLogger.newInfoLogger(LogManager.getLogger("[OPAL-Project]"))); + } + + /** + * Retrieves all class files for the JAR referenced by this ArtifactInformation, and uses them to initialize an + * OPAL project instance. This instance can be used to conduct complex static program analyses. + * @return The OPAL project instance, or null if no JAR file was found + * @param projectLogger A custom logger instance to configure the amount of output that is produced by OPAL. + * @throws IOException If reading the JAR file fails + * @implNote OPAL projects are not designed to be used in large scale analyses. Over time, OPAL's internal caches + * will continue to claim heap space that is never freed, so eventually the program will crash with an + * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you + * really need them. + */ + public Project getOpalProject(MarinOpalLogger projectLogger) throws IOException { try(JarInputStream jarStream = getJarInputStream()){ - return Project.apply(Java17Framework$.MODULE$.ClassFiles(() -> jarStream).map( t -> new Tuple2<>((org.opalj.br.ClassFile)t._1, t._2))); + return Project.apply(Java17Framework$.MODULE$.ClassFiles(() -> jarStream).map( t -> new Tuple2<>((org.opalj.br.ClassFile)t._1, t._2)), projectLogger); } catch(FileNotFoundException fnfx){ log.error("No JAR file found for {}", ident.getCoordinates(), fnfx); return null; @@ -245,6 +263,28 @@ public Project getOpalProject(Set dependencies) throws IO * enabled). */ public Project getOpalProject(Set dependencies, boolean fullyLoadLibraries, boolean breakOnFailure) throws IOException { + return getOpalProject(dependencies, fullyLoadLibraries, breakOnFailure, + MarinOpalLogger.newInfoLogger(LogManager.getLogger("[OPAL-Project]"))); + } + + /** + * Initializes an OPAL project instance for the current JAR and the given set of dependencies. This project instance + * can be used to conduct complex, static whole-program analyses. + * @param dependencies The transitive set of dependencies to use. Can be obtained using the PomInformation instance + * for this artifact + * @param fullyLoadLibraries Whether to fully load the dependency's class file contents. If set to false, only their + * interface definitions (class names, method signatures) are loaded, not their actual + * content. + * @param breakOnFailure If set to true, any IO exception when loading dependencies interrupts the method execution + * and throws the exception up to the calling method. If set to false, dependencies will be + * loaded in a best-effort fashion, and errors will only be logged to the console. + * @param projectLogger A custom logger instance to configure the amount of output that is produced by OPAL. + * @return The OPAL project instance, or null if no main project JAR was found + * @throws IOException If an IO error occurs while loading the main JAR, or the dependencies (when breakOnFailure is + * enabled). + */ + public Project getOpalProject(Set dependencies, boolean fullyLoadLibraries, + boolean breakOnFailure, MarinOpalLogger projectLogger) throws IOException { List> allLibraryClasses = new ArrayList<>(); for(ArtifactIdent dependency : dependencies){ @@ -267,7 +307,12 @@ public Project getOpalProject(Set dependencies, boolean f } return Project.apply(CollectionConverters.ListHasAsScala(allProjectClasses).asScala().toSeq(), - CollectionConverters.ListHasAsScala(allLibraryClasses).asScala().toSeq(), !fullyLoadLibraries); + CollectionConverters.ListHasAsScala(allLibraryClasses).asScala().toSeq(), + !fullyLoadLibraries, + CollectionConverters.ListHasAsScala(new ArrayList()).asScala().toIterable(), + (a, b) -> BoxedUnit.UNIT, + package$.MODULE$.BaseConfig(), + projectLogger); } private List getOpalClassFileRepresentations(ArtifactIdent ident) throws IOException { diff --git a/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java b/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java new file mode 100644 index 0000000..9eb1c8f --- /dev/null +++ b/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java @@ -0,0 +1,101 @@ +package org.tudo.sse.utils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opalj.log.*; + +/** + * Acts as a logging bridge between Log4j and OPAL. A MarinOpalLogger instance can be passed when creating an OPAL + * project, and can be configured to selectively enable certain log levels. You can pass an existing Log4j logger + * to use as the underlying backend, otherwise the bridge will create its own Log4j logger backend. + */ +public class MarinOpalLogger implements OPALLogger { + + private final Logger internalLog; + private boolean infoEnabled = true, warnEnabled = true, errorEnabled = true; + + private static final MarinOpalLogger _globalInstance = newErrorOnlyLogger(); + + /** + * Create a new instance that creates a new underlying Log4j backend. + */ + public MarinOpalLogger(){ + this.internalLog = LogManager.getLogger(MarinOpalLogger.class); + } + + /** + * Creates a new instance that uses the given Log4j backend. + * @param logger The Log4j logger to use as a backend + */ + public MarinOpalLogger(Logger logger) { + this.internalLog = logger; + } + + /** + * Sets whether the info level of OPAL shall be forwarded to the backend. + * @param enabled True if info level shall be forwarded, false otherwise + */ + public void setInfoEnabled(boolean enabled) { this.infoEnabled = enabled; } + /** + * Sets whether the warning level of OPAL shall be forwarded to the backend. + * @param enabled True if warning level shall be forwarded, false otherwise + */ + public void setWarnEnabled(boolean enabled) { this.warnEnabled = enabled; } + /** + * Sets whether the error level of OPAL shall be forwarded to the backend. + * @param enabled True if error level shall be forwarded, false otherwise + */ + public void setErrorEnabled(boolean enabled) { this.errorEnabled = enabled; } + + /** + * Creates a new logger bridge instance that only forwards error messages. Uses a new Log4j backend. + * @return New logger bridge instance + */ + public static MarinOpalLogger newErrorOnlyLogger(){ + MarinOpalLogger logger = new MarinOpalLogger(); + logger.setInfoEnabled(false); + logger.setWarnEnabled(false); + logger.setErrorEnabled(true); + return logger; + } + + /** + * Creates a new logger bridge instance that forwards info, warning and error messages. Uses the given Log4j backend. + * @param log The Log4j backend to use + * @return New logger bridge instance + */ + public static MarinOpalLogger newInfoLogger(Logger log){ + MarinOpalLogger logger = new MarinOpalLogger(log); + logger.setInfoEnabled(true); + logger.setWarnEnabled(true); + logger.setErrorEnabled(true); + return logger; + } + + /** + * Gets the singleton logger bridge that is used as the OPAL global logger for MARIN. + * @return The global logging bridge + */ + public static MarinOpalLogger getGlobalLogger(){ + return _globalInstance; + } + + + @Override + public void log(LogMessage message, LogContext ctx) { + if(message.level() == Error$.MODULE$ && errorEnabled){ + internalLog.error(message.message()); + } else if(message.level() == Warn$.MODULE$ && warnEnabled){ + internalLog.warn(message.message()); + } else if(message.level() == Info$.MODULE$ && infoEnabled){ + internalLog.info(message.message()); + } else if(message.level() == Fatal$.MODULE$){ + internalLog.fatal(message.message()); + } + } + + @Override + public void logOnce(LogMessage message, LogContext ctx) { + OPALLogger.super.logOnce(message, ctx); + } +} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 52b1d79..96b396c 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -2,7 +2,7 @@ - + diff --git a/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java b/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java index f22c6fc..462e617 100644 --- a/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java +++ b/src/test/java/org/tudo/sse/model/jar/JarInformationTest.java @@ -1,11 +1,15 @@ package org.tudo.sse.model.jar; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.opalj.br.ClassFile; import org.opalj.br.ClassType; import org.opalj.br.analyses.Project; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.utils.MarinOpalLogger; import java.io.File; import java.io.InputStream; @@ -23,6 +27,12 @@ public class JarInformationTest { private final ArtifactIdent ident = new ArtifactIdent("eu.sse-labs", "marin", "1.0.0"); private final JarInformation jarInfo = new JarInformation(ident); + @BeforeAll + static void setup(){ + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + @Test @DisplayName("JAR information should be able to download file to temp directory") void testLocalFile(){ From ff1ab42dd89b5545cfbdf6c194305b1739577823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 12 Nov 2025 13:43:23 +0100 Subject: [PATCH 19/55] Introduce additional analysis lifecycle hooks. Fix two deprecation warnings --- .../sse/analyses/MavenCentralAnalysis.java | 23 ++++++ .../MavenCentralArtifactAnalysis.java | 29 ++++---- .../analyses/MavenCentralLibraryAnalysis.java | 70 +++++++++++++++++++ .../tudo/sse/model/jar/JarInformation.java | 2 +- .../org/tudo/sse/resolution/JarResolver.java | 2 +- .../tudo/sse/resolution/PomResolverTest.java | 3 +- 6 files changed, 113 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java index a9f33b5..7d3b546 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java @@ -63,6 +63,29 @@ protected MavenCentralAnalysis(boolean requiresIndex, resolveJar = requiresJar; } + /** + * Analysis lifecycle hook that is executed before the actual analysis run is started, but after command line + * parameters and resolvers have been initialized. This method is called exactly once per analysis run. + */ + protected void beforeRunStart(){ + log.debug("Starting Maven Central analysis run."); + } + + + /** + * Analysis lifecycle hook that is executed after the analysis run has completed, immediately before the end of + * {@link #runAnalysis runAnalysis}. This method is called exactly once per analysis run. + */ + protected void afterRunEnd(){ + log.debug("Finished Maven Central analysis run."); + } + + /** + * Starts a new analysis run. Will parse the given command line arguments and create a matching analysis + * configuration. Depending on the arguments provided, the analysis run will use single- or multithreaded execution. + * + * @param args Command line arguments to parse - usually taken directly from the main method's arguments. + */ public abstract void runAnalysis(String[] args); } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index fda1fbc..ff5c40a 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -101,13 +101,6 @@ private void printRunInfo(){ } } - /** - * This method is the driver code for the analysis being done on Maven Central. - * It can be configured using different command line arguments being passed to it. - * There's a single-threaded implementation contained in this class, as well as a multithreaded one called here but defined in different actor classes. - * - * @param args cli passed to the program to configure the run - */ @Override public final void runAnalysis(String[] args) { // Obtain CLI arguments @@ -135,6 +128,14 @@ public final void runAnalysis(String[] args) { artifactConfig.progressWriteInterval, artifactConfig.progressOutputFile), "queueActor"); } + // Invoke the beforeRunStart lifecycle hook + try { + this.beforeRunStart(); + } catch (Exception x){ + log.error("Unexpected exception in analysis lifecycle hook beforeRunStart", x); + return; + } + if(this.artifactConfig.inputListFile == null) { indexProcessor(); } else { @@ -148,6 +149,13 @@ public final void runAnalysis(String[] args) { system.getWhenTerminated().toCompletableFuture().get(); } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} } + + // Invoke the afterRunEnd lifecycle hook + try { + this.afterRunEnd(); + } catch(Exception x){ + log.error("Unexpected exception in analysis lifecycle hook afterRunEnd", x); + } } /** @@ -192,12 +200,10 @@ private void processIndex(Artifact current) { * Iterates over all indexes in the maven central index and creates an artifact with the metadata collected. * * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifact identifiers processed by this method * @see ArtifactIdent * @throws IOException when there is an issue opening a file */ - List walkAllIndexes(IndexIterator indexIterator) throws IOException { - final List artifactIdents = new ArrayList<>(); + private void walkAllIndexes(IndexIterator indexIterator) throws IOException { long entriesTaken = 0L; while(indexIterator.hasNext()) { @@ -219,8 +225,6 @@ List walkAllIndexes(IndexIterator indexIterator) throws IOExcepti processIndexIdentifier(current.getIdent()); } - artifactIdents.add(current.getIdent()); - writePositionIfNeeded(); } @@ -230,7 +234,6 @@ List walkAllIndexes(IndexIterator indexIterator) throws IOExcepti log.info("Processed a total of {} entries.", entriesTaken); indexIterator.closeReader(); - return artifactIdents; } /** diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index b761820..c1c2e16 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -105,6 +105,14 @@ public final void runAnalysis(String[] args){ if(config.take >= 0) log.info("Taking {} library names", config.take); + // Invoke the beforeRunStart lifecycle hook + try { + this.beforeRunStart(); + } catch (Exception x){ + log.error("Unexpected exception in analysis lifecycle hook beforeRunStart", x); + return; + } + // If specified, we only take the configured amount of entries. If not, we process as long as the iterator // provides new entries while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ @@ -138,8 +146,53 @@ public final void runAnalysis(String[] args){ system.getWhenTerminated().toCompletableFuture().get(); } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} } + + // Invoke the afterRunEnd lifecycle hook + try { + this.afterRunEnd(); + } catch(Exception x){ + log.error("Unexpected exception in analysis lifecycle hook afterRunEnd", x); + } } + /** + * Analysis lifecycle hook that is executed before a library is being processed, i.e., before any releases are + * discovered. + * + * @implNote This method may be called concurrently by multiple threads if the analysis uses parallel execution + * + * @param groupId The library's group ID + * @param artifactId The library's artifact ID + */ + protected void beforeLibraryStart(String groupId, String artifactId) { + log.debug("Start processing library {}:{}", groupId, artifactId); + } + + /** + * Analysis lifecycle hook that is executed after a library has been processed, i.e., after all releases have been + * discovered and the analysis implementation has been executed. + * + * @implNote This method may be called concurrently by multiple threads if the analysis uses parallel execution + * + * @param groupId The library's group ID + * @param artifactId The library's artifact ID + * @param artifacts The set of artifacts for the given library + */ + protected void afterLibraryEnd(String groupId, String artifactId, List artifacts) { + log.debug("Finished processing {} artifacts for library {}:{}", artifacts.size(), groupId, artifactId); + } + + /** + * Analysis lifecycle hook that is executed when the analysis failed to obtain a version list for a given library. + * + * @implNote This method may be called concurrently by multiple threads if the analysis uses parallel execution + * + * @param groupId The library's group ID + * @param artifactId The library's artifact ID + * @param cause The exception that caused the failure + */ + protected void onVersionListError(String groupId, String artifactId, Exception cause){} + /** * Main analysis implementation. Defines how a single library shall be analyzed. All artifacts for a given * library will have information annotated corresponding to the protected attributes' values. @@ -168,6 +221,12 @@ private void processEntry(String ga){ } private Void process(String ga, String groupId, String artifactId){ + + try { this.beforeLibraryStart(groupId, artifactId); } + catch(Exception x){ + log.error("Unexpected exception in analysis lifecycle hook beforeLibraryStart for library {}, {}", groupId, artifactId, x); + } + List identifiers = getReleaseIdentifiers(groupId, artifactId); // Abort if we find no releases, error has already been logged at this point @@ -189,6 +248,11 @@ private Void process(String ga, String groupId, String artifactId){ log.error("Unknown error in analysis implementation", x); } + try { this.afterLibraryEnd(groupId, artifactId, libraryArtifacts); } + catch(Exception x){ + log.error("Unexpected exception in analysis lifecycle hook afterLibraryEnd for library {}, {}", groupId, artifactId, x); + } + return null; } @@ -202,6 +266,12 @@ private List getReleaseIdentifiers(String groupId, String artifac return identifiers; } catch (Exception x) { log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + + try { this.onVersionListError(groupId, artifactId, x); } + catch(Exception inner) { + log.warn("Unexcepted exception in analysis lifecycle hook onVersionListError for library {}:{}, {}", groupId, artifactId, x); + } + return null; } } diff --git a/src/main/java/org/tudo/sse/model/jar/JarInformation.java b/src/main/java/org/tudo/sse/model/jar/JarInformation.java index 08ac190..a0500a1 100644 --- a/src/main/java/org/tudo/sse/model/jar/JarInformation.java +++ b/src/main/java/org/tudo/sse/model/jar/JarInformation.java @@ -309,7 +309,7 @@ public Project getOpalProject(Set dependencies, boolean f return Project.apply(CollectionConverters.ListHasAsScala(allProjectClasses).asScala().toSeq(), CollectionConverters.ListHasAsScala(allLibraryClasses).asScala().toSeq(), !fullyLoadLibraries, - CollectionConverters.ListHasAsScala(new ArrayList()).asScala().toIterable(), + CollectionConverters.ListHasAsScala(new ArrayList()).asScala().toSeq(), (a, b) -> BoxedUnit.UNIT, package$.MODULE$.BaseConfig(), projectLogger); diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index 936e75d..863aa3f 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -168,7 +168,7 @@ public JarInformation parsingClassFiles(List> classList, numMethods += current.methods().size(); numFields += current.fields().size(); - List methods = JavaConverters.seqAsJavaList(classfile._1.methods()); + List methods = scala.jdk.CollectionConverters.SeqHasAsJava(classfile._1.methods()).asJava(); for(Method method : methods) { //tally up bytecode for each method here if(method.body().isDefined()) { diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index 70d254f..275a027 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -9,6 +9,7 @@ import org.tudo.sse.IndexWalker; import org.tudo.sse.model.*; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.io.*; @@ -59,7 +60,7 @@ void processRawPomFeatures() { for(String input : inputs) { RawPomFeatures current = null; try { - current = pomResolver.processRawPomFeatures(IOUtils.toInputStream(input), null); + current = pomResolver.processRawPomFeatures(IOUtils.toInputStream(input, StandardCharsets.UTF_8), null); } catch (PomResolutionException e) { fail(e); } From 8d39a4c08e92bb6c732724c06fc369c07d7bd3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 28 Nov 2025 09:49:15 +0100 Subject: [PATCH 20/55] Introduce liner warnings during compilation, fix issues where approriated, suppress some warnings in tests --- pom.xml | 13 +- .../sse/model/pom/LocalPomInformation.java | 76 ++++++----- .../tudo/sse/model/pom/PomInformation.java | 8 +- .../LocalPomInformationFactory.java | 126 ++++++++++++++++++ .../org/tudo/sse/resolution/PomResolver.java | 2 +- .../tudo/sse/utils/LibraryIndexIterator.java | 23 +++- .../java/org/tudo/sse/IndexWalkerTest.java | 1 + .../MavenCentralArtifactAnalysisTest.java | 7 +- .../sse/model/LocalPomInformationTest.java | 24 ++-- .../org/tudo/sse/model/TypeStructureTest.java | 2 +- .../tudo/sse/resolution/JarResolverTest.java | 1 + .../tudo/sse/resolution/PomResolverTest.java | 2 +- .../org/tudo/sse/utils/IndexIteratorTest.java | 1 + 13 files changed, 226 insertions(+), 60 deletions(-) create mode 100644 src/main/java/org/tudo/sse/resolution/LocalPomInformationFactory.java diff --git a/pom.xml b/pom.xml index 6d6788d..99e7d10 100644 --- a/pom.xml +++ b/pom.xml @@ -37,8 +37,7 @@ - 11 - 11 + 11 UTF-8 @@ -136,6 +135,16 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.1 + + + -Xlint:all,-serial + + + diff --git a/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java b/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java index c6e8f1c..927d55d 100644 --- a/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java +++ b/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java @@ -20,45 +20,55 @@ */ public class LocalPomInformation extends PomInformation { - private Path pomPath; + private final Path pomPath; + private final boolean loadTransitives; + private final InputStream pomStream; + /** * Creates a new LocalPomInformation object for a POM file located at the given path. * - * @param path The Path object identifying a pom file - * @param resolver The PomResolver to use for resolution - * @throws FileNotFoundException If the given Path does not exist + * @param pomPath The Path object identifying a pom file + * @param loadTransitives Whether to load transitive pom information upon initialization */ - public LocalPomInformation(String path, PomResolver resolver) throws FileNotFoundException { - this(new FileInputStream(path), resolver); - this.pomPath = Paths.get(path); + public LocalPomInformation(Path pomPath, boolean loadTransitives){ + this.pomPath = pomPath; + this.loadTransitives = loadTransitives; + this.pomStream = null; } /** - * Creates a new LocalPomInformation object for a given POM file - * @param localPom The file object identifying the local POM file - * @param resolver The PomResolver to use for resolution - * @throws FileNotFoundException If the given File does not exist + * Creates a new LocalPomInformation object for a pom file input stream without a file path on the current drive. + * @param pomStream Input Stream containing pom file contents + * @param loadTransitives Whether to load transitive pom information upon initialization */ - public LocalPomInformation(File localPom, PomResolver resolver) throws FileNotFoundException{ - this(new FileInputStream(localPom), resolver); - this.pomPath = localPom.toPath(); + public LocalPomInformation(InputStream pomStream, boolean loadTransitives){ + this.pomPath = null; + this.loadTransitives = loadTransitives; + this.pomStream = pomStream; } /** - * Creates a new LocalPomInformation object for a given InputStream (representing a text file). - * @param localPom An InputStream containing the POM file contents - * @param resolver The PomResolver to use for resolution + * Resolve the local pom file and compute all information as specified upon object creation. + * + * @param resolver The pom resolver to use for resolution + * @throws IOException If reading the pom file contents fails + * @throws XmlPullParserException If parsing the pom.xml file fails */ - public LocalPomInformation(InputStream localPom, PomResolver resolver) { - super(); - try { - resolveLocalFile(resolver, localPom); + public void resolveLocalPom(PomResolver resolver) throws IOException, XmlPullParserException { + if(pomStream != null) { + resolveLocalFile(resolver, this.pomStream); + } else { + try (FileInputStream is = new FileInputStream(this.pomPath.toFile())) { + resolveLocalFile(resolver, is); + } + } + + if(this.loadTransitives){ resolveParentAndImport(resolver); resolveDependencies(resolver); - } catch (IOException | XmlPullParserException e) { - throw new RuntimeException(e); } + } /** @@ -78,7 +88,7 @@ public Path getPomPath(){ * @throws IOException when there's an issue opening the file * @throws XmlPullParserException when there's an issue parsing the local file using the maven model */ - public void resolveLocalFile(PomResolver resolver, InputStream pomFile) throws IOException, XmlPullParserException{ + private void resolveLocalFile(PomResolver resolver, InputStream pomFile) throws IOException, XmlPullParserException{ MavenXpp3Reader reader = new MavenXpp3Reader(); Model model = reader.read(pomFile, true); @@ -97,8 +107,8 @@ public void resolveLocalFile(PomResolver resolver, InputStream pomFile) throws I throw new NullPointerException(); } - setIdent(new ArtifactIdent(groupId, model.getArtifactId(), version)); - setRawPomFeatures(processRawPomFeatures(model, resolver)); + this.ident = new ArtifactIdent(groupId, model.getArtifactId(), version); + this.rawPomFeatures = processRawPomFeatures(model, resolver); } /** @@ -108,7 +118,7 @@ public void resolveLocalFile(PomResolver resolver, InputStream pomFile) throws I * @param resolver pom resolver to aid in feature collection * @return raw features that are collected during resolution */ - public RawPomFeatures processRawPomFeatures(Model model, PomResolver resolver) { + private RawPomFeatures processRawPomFeatures(Model model, PomResolver resolver) { RawPomFeatures rawPomFeatures = new RawPomFeatures(); //parent information @@ -157,18 +167,18 @@ public RawPomFeatures processRawPomFeatures(Model model, PomResolver resolver) { * @see PomResolver * @param resolver the resolver used to resolve the parents and imports of the local pom */ - public void resolveParentAndImport(PomResolver resolver) { - if(getRawPomFeatures().getParent() != null) { + private void resolveParentAndImport(PomResolver resolver) { + if(this.rawPomFeatures.getParent() != null) { try { - setParent(resolver.processArtifact(getRawPomFeatures().getParent())); + this.parent = resolver.processArtifact(this.rawPomFeatures.getParent()); } catch (PomResolutionException | org.tudo.sse.resolution.FileNotFoundException | IOException e) { throw new RuntimeException(e); } } - if(getRawPomFeatures().getDependencyManagement() != null) { + if(this.rawPomFeatures.getDependencyManagement() != null) { try { - setImports(resolver.resolveImports(getRawPomFeatures().getDependencyManagement(), this)); + this.imports = resolver.resolveImports(this.rawPomFeatures.getDependencyManagement(), this); } catch (PomResolutionException | org.tudo.sse.resolution.FileNotFoundException | IOException e) { throw new RuntimeException(e); } @@ -181,7 +191,7 @@ public void resolveParentAndImport(PomResolver resolver) { * @see PomResolver * @param resolver the pomResolver used to drive this method */ - public void resolveDependencies(PomResolver resolver) { + private void resolveDependencies(PomResolver resolver) { setResolvedDependencies(resolver.resolveDependencies(this)); } diff --git a/src/main/java/org/tudo/sse/model/pom/PomInformation.java b/src/main/java/org/tudo/sse/model/pom/PomInformation.java index db145b1..2d93de3 100644 --- a/src/main/java/org/tudo/sse/model/pom/PomInformation.java +++ b/src/main/java/org/tudo/sse/model/pom/PomInformation.java @@ -9,9 +9,9 @@ * This class is where all the information collected during pom resolution is stored. From the raw features to resolved transitive dependencies. */ public class PomInformation extends ArtifactInformation { - private RawPomFeatures rawPomFeatures; - private List imports; - private Artifact parent; + protected RawPomFeatures rawPomFeatures; + protected List imports; + protected Artifact parent; private List resolvedDependencies; private List allTransitiveDependencies; private List effectiveTransitiveDependencies; @@ -46,7 +46,7 @@ public RawPomFeatures getRawPomFeatures() { * Updates the value of the current rawPomFeatures. * @param rawPomFeatures new rawPomFeatures to set */ - public void setRawPomFeatures(RawPomFeatures rawPomFeatures) { + public final void setRawPomFeatures(RawPomFeatures rawPomFeatures) { this.rawPomFeatures = rawPomFeatures; } diff --git a/src/main/java/org/tudo/sse/resolution/LocalPomInformationFactory.java b/src/main/java/org/tudo/sse/resolution/LocalPomInformationFactory.java new file mode 100644 index 0000000..c90b8f5 --- /dev/null +++ b/src/main/java/org/tudo/sse/resolution/LocalPomInformationFactory.java @@ -0,0 +1,126 @@ +package org.tudo.sse.resolution; + +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; +import org.tudo.sse.model.pom.LocalPomInformation; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +/** + * Class providing static utility methods to build LocalPomFileInformation for pom.xml files not hosted on any repository. + */ +public class LocalPomInformationFactory { + + private static final PomResolver defaultTransitiveResolver = new PomResolver(true); + private static final PomResolver defaultDirectResolver = new PomResolver(false); + + private LocalPomInformationFactory() {} + + /** + * Load a local pom.xml file from the given input stream, using the given resolver and resolution mode. + * + * @param pomInputStream InputStream containing the pom.xml file contents + * @param loadTransitiveDependencies Whether to transitively load all referenced pom files + * @param theResolver PomResolver to used for resolution + * @return LocalPomInformation object representing the pom.xml file + * @throws IOException If accessing the stream fails + */ + public static LocalPomInformation loadLocalPom(InputStream pomInputStream, + boolean loadTransitiveDependencies, + PomResolver theResolver) throws IOException { + final LocalPomInformation pomInfo = new LocalPomInformation(pomInputStream, loadTransitiveDependencies); + + try { + pomInfo.resolveLocalPom(theResolver); + return pomInfo; + } catch (XmlPullParserException xppx){ + throw new RuntimeException(xppx); + } + } + + /** + * Load a local pom.xml file from the given input stream, using the given resolution mode and a default resolver. + * + * @param pomInputStream InputStream containing the pom.xml file contents + * @param loadTransitiveDependencies Whether to transitively load all referenced pom files + * @return LocalPomInformation object representing the pom.xml file + * @throws IOException If accessing the stream fails + */ + public static LocalPomInformation loadLocalPom(InputStream pomInputStream, + boolean loadTransitiveDependencies) throws IOException { + return loadLocalPom(pomInputStream, loadTransitiveDependencies, getDefaultResolver(loadTransitiveDependencies)); + } + + /** + * Load a local pom.xml file from the given path, using the given resolver and resolution mode. + * + * @param pomPath Local path to pom.xml file + * @param loadTransitiveDependencies Whether to transitively load all referenced pom files + * @param theResolver PomResolver to used for resolution + * @return LocalPomInformation object representing the pom.xml file + * @throws IOException If accessing the file contents fails + */ + public static LocalPomInformation loadLocalPom(Path pomPath, + boolean loadTransitiveDependencies, + PomResolver theResolver) throws IOException { + final LocalPomInformation pomInfo = new LocalPomInformation(pomPath, loadTransitiveDependencies); + + try { + pomInfo.resolveLocalPom(theResolver); + return pomInfo; + } catch (XmlPullParserException xppx){ + throw new RuntimeException(xppx); + } + } + + /** + * Load a local pom.xml file from the given path, using the given resolution mode and a default resolver. + * + * @param pomPath Local path to pom.xml file + * @param loadTransitiveDependencies Whether to transitively load all referenced pom files + * @return LocalPomInformation object representing the pom.xml file + * @throws IOException If accessing the file contents fails + */ + public static LocalPomInformation loadLocalPom(Path pomPath, + boolean loadTransitiveDependencies) throws IOException { + return loadLocalPom(pomPath, loadTransitiveDependencies, getDefaultResolver(loadTransitiveDependencies)); + } + + /** + * Load a local pom.xml file from the given File, using the given resolver and resolution mode. + * + * @param pomFile File object representing a pom.xml file + * @param loadTransitiveDependencies Whether to transitively load all referenced pom files + * @param theResolver PomResolver to used for resolution + * @return LocalPomInformation object representing the pom.xml file + * @throws IOException If accessing the file contents fails + */ + public static LocalPomInformation loadLocalPom(File pomFile, + boolean loadTransitiveDependencies, + PomResolver theResolver) throws IOException { + return loadLocalPom(pomFile.toPath(), loadTransitiveDependencies, theResolver); + } + + /** + * Load a local pom.xml file from the given File, using the given resolution mode and a default resolver. + * + * @param pomFile File object representing a pom.xml file + * @param loadTransitiveDependencies Whether to transitively load all referenced pom files + * @return LocalPomInformation object representing the pom.xml file + * @throws IOException If accessing the file contents fails + */ + public static LocalPomInformation loadLocalPom(File pomFile, + boolean loadTransitiveDependencies) throws IOException { + return loadLocalPom(pomFile.toPath(), loadTransitiveDependencies, getDefaultResolver(loadTransitiveDependencies)); + } + + private static PomResolver getDefaultResolver(boolean loadTransitives){ + if(loadTransitives) + return defaultTransitiveResolver; + else + return defaultDirectResolver; + } + +} diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java index 7f9a8d0..02cbdeb 100644 --- a/src/main/java/org/tudo/sse/resolution/PomResolver.java +++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java @@ -671,7 +671,7 @@ private org.tudo.sse.model.pom.Dependency resolveVersion(org.tudo.sse.model.pom. return toReturn; } - private Tuple2 findGA(String missing, List management) { + private Tuple2 findGA(String missing, List management) { for (org.tudo.sse.model.pom.Dependency dependency : management) { if (missing.equals(dependency.getIdent().getGroupID() + ":" + dependency.getIdent().getArtifactID())) { if(dependency.getScope() != null) { diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 746435c..c3b31da 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -1,5 +1,8 @@ package org.tudo.sse.utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import org.apache.maven.index.reader.ChunkReader; import org.apache.maven.index.reader.IndexReader; @@ -12,6 +15,8 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { + private final Logger logger = LogManager.getLogger(getClass()); + private final Set libraryHashesSeen; private final Iterator> entryIterator; @@ -134,12 +139,18 @@ private Map nextEntry(){ @Override - public void close() throws Exception { - if(this.mavenChunkReader != null){ - this.mavenChunkReader.close(); - } - if(this.mavenIndexReader != null) { - this.mavenIndexReader.close(); + public void close(){ + try { + if(this.mavenChunkReader != null){ + this.mavenChunkReader.close(); + } + if(this.mavenIndexReader != null) { + this.mavenIndexReader.close(); + } + } catch(IOException iox){ + logger.warn("Exception while closing maven index reader", iox); } + + } } diff --git a/src/test/java/org/tudo/sse/IndexWalkerTest.java b/src/test/java/org/tudo/sse/IndexWalkerTest.java index c71345d..99347c8 100644 --- a/src/test/java/org/tudo/sse/IndexWalkerTest.java +++ b/src/test/java/org/tudo/sse/IndexWalkerTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; +@SuppressWarnings("unchecked") class IndexWalkerTest { IndexWalker indexWalker; diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index e3eee4a..774991d 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.*; +@SuppressWarnings("unchecked") class MavenCentralArtifactAnalysisTest { final MavenCentralArtifactAnalysis analysisUnderTest = MavenCentralAnalysisFactory.buildEmptyAnalysisWithIndexRequirement(); final String base = "https://repo1.maven.org/maven2/"; @@ -160,9 +161,9 @@ void walkPaginated() { inputs.add(new Tuple2<>(50000, 100)); inputs.add(new Tuple2<>(763, 20)); - for(Tuple2 input : inputs) { - int start1 = (int) input._1; - int take = (int) input._2; + for(Tuple2 input : inputs) { + int start1 = input._1; + int take = input._2; int start2 = (start1 + take) - 1; diff --git a/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java b/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java index c806392..02b828a 100644 --- a/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java +++ b/src/test/java/org/tudo/sse/model/LocalPomInformationTest.java @@ -6,10 +6,12 @@ import org.tudo.sse.model.pom.Dependency; import org.tudo.sse.model.pom.LocalPomInformation; import org.tudo.sse.model.pom.RawPomFeatures; +import org.tudo.sse.resolution.LocalPomInformationFactory; import org.tudo.sse.resolution.PomResolver; import org.tudo.sse.resolution.releases.IReleaseListProvider; import java.io.*; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -17,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.*; +@SuppressWarnings("unchecked") class LocalPomInformationTest { final Gson gson = new Gson(); @@ -30,7 +33,7 @@ class LocalPomInformationTest { final IReleaseListProvider mockProvider = new IReleaseListProvider() { - private Map> releaseListData = new HashMap<>(); + private final Map> releaseListData = new HashMap<>(); private void buildMap(){ for(Map entry : (List>)json.get("versionLists")){ @@ -66,11 +69,11 @@ void localPom() { PomResolver resolver = new PomResolver(true, mockProvider); try { - tests.add(new LocalPomInformation("src/test/resources/localPom.xml", resolver)); - tests.add(new LocalPomInformation(new File("src/test/resources/localPom.xml"), resolver)); - tests.add(new LocalPomInformation(new FileInputStream("src/test/resources/localPom.xml"), resolver)); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); + tests.add(LocalPomInformationFactory.loadLocalPom(Path.of("src/test/resources/localPom.xml"), true, resolver)); + tests.add(LocalPomInformationFactory.loadLocalPom(new File("src/test/resources/localPom.xml"), true, resolver)); + tests.add(LocalPomInformationFactory.loadLocalPom(new FileInputStream("src/test/resources/localPom.xml"), true, resolver)); + } catch (IOException iox) { + throw new RuntimeException(iox); } testFeatureExtraction(tests); @@ -80,9 +83,12 @@ void localPom() { @Test void localPomException() { PomResolver resolver = new PomResolver(true, mockProvider); - assertThrows(RuntimeException.class, () -> new LocalPomInformation("src/test/resources/brokenlocalPom.xml", resolver)); - assertThrows(RuntimeException.class, () -> new LocalPomInformation(new File("src/test/resources/brokenlocalPom.xml"), resolver)); - assertThrows(RuntimeException.class, () -> new LocalPomInformation(new FileInputStream("src/test/resources/brokenlocalPom.xml"), resolver)); + assertThrows(RuntimeException.class, () -> + LocalPomInformationFactory.loadLocalPom(Path.of("src/test/resources/brokenlocalPom.xml"), true, resolver)); + assertThrows(RuntimeException.class, () -> + LocalPomInformationFactory.loadLocalPom(new File("src/test/resources/brokenlocalPom.xml"), true, resolver)); + assertThrows(RuntimeException.class, () -> + LocalPomInformationFactory.loadLocalPom(new FileInputStream("src/test/resources/brokenlocalPom.xml"), true, resolver)); } void testFeatureExtraction(List tests) { diff --git a/src/test/java/org/tudo/sse/model/TypeStructureTest.java b/src/test/java/org/tudo/sse/model/TypeStructureTest.java index 083ea88..e7f768d 100644 --- a/src/test/java/org/tudo/sse/model/TypeStructureTest.java +++ b/src/test/java/org/tudo/sse/model/TypeStructureTest.java @@ -19,7 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - +@SuppressWarnings("unchecked") class TypeStructureTest { JarResolver resolver = new JarResolver(); diff --git a/src/test/java/org/tudo/sse/resolution/JarResolverTest.java b/src/test/java/org/tudo/sse/resolution/JarResolverTest.java index d6f73d6..9d929cc 100644 --- a/src/test/java/org/tudo/sse/resolution/JarResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/JarResolverTest.java @@ -16,6 +16,7 @@ import static org.junit.jupiter.api.Assertions.*; +@SuppressWarnings("unchecked") class JarResolverTest { JarResolver jarResolver; diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index 275a027..f95e72f 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -29,7 +29,7 @@ import static org.junit.jupiter.api.Assertions.*; -@SuppressWarnings("ALL") +@SuppressWarnings("unchecked") class PomResolverTest { PomResolver pomResolver; diff --git a/src/test/java/org/tudo/sse/utils/IndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/IndexIteratorTest.java index 3a1bfa5..817cc05 100644 --- a/src/test/java/org/tudo/sse/utils/IndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/IndexIteratorTest.java @@ -14,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.*; +@SuppressWarnings("unchecked") class IndexIteratorTest { IndexIterator indexIterator; From 7dde5724926602a6bb65e96940ae6c8c5d707e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 3 Dec 2025 09:54:20 +0100 Subject: [PATCH 21/55] Introduce resolution contexts to avoid caching artifacts indefinetely. Start phasing out artifact factory usage --- .../MavenCentralArtifactAnalysis.java | 62 +++++++++------ .../analyses/MavenCentralLibraryAnalysis.java | 28 +++---- .../java/org/tudo/sse/model/Artifact.java | 7 +- .../sse/model/pom/LocalPomInformation.java | 12 +-- .../resolution/ArtifactResolutionContext.java | 31 ++++++++ .../resolution/LibraryResolutionContext.java | 43 ++++++++++ .../model/resolution/ResolutionContext.java | 53 +++++++++++++ .../ProcessIdentifierMessage.java | 19 +++-- .../sse/multithreading/ResolverActor.java | 10 ++- .../org/tudo/sse/resolution/JarResolver.java | 23 ++++-- .../org/tudo/sse/resolution/PomResolver.java | 79 +++++++++++-------- .../tudo/sse/resolution/ResolverFactory.java | 15 ++-- .../MavenCentralArtifactAnalysisTest.java | 25 ++++-- .../org/tudo/sse/model/TypeStructureTest.java | 13 +-- .../tudo/sse/resolution/JarResolverTest.java | 4 +- .../tudo/sse/resolution/PomResolverTest.java | 13 +-- 16 files changed, 323 insertions(+), 114 deletions(-) create mode 100644 src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java create mode 100644 src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java create mode 100644 src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index ff5c40a..c9ba41d 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -7,6 +7,8 @@ import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; +import org.tudo.sse.model.resolution.ArtifactResolutionContext; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.multithreading.ProcessIdentifierMessage; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; @@ -187,11 +189,11 @@ public void indexProcessor() { } - private void processIndex(Artifact current) { + private void processIndex(Artifact current, ArtifactResolutionContext ctx) { if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(new ProcessIdentifierMessage(current.getIdent(), this), ActorRef.noSender()); + queueActorRef.tell(new ProcessIdentifierMessage(ctx, this), ActorRef.noSender()); } else { - callResolver(current.getIdent()); + callResolver(current.getIdent(), ctx); analyzeArtifact(current); } } @@ -218,11 +220,15 @@ private void walkAllIndexes(IndexIterator indexIterator) throws IOException { } } + final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(current.getIdent()); + if(resolveIndex){ - final Artifact artifact = ArtifactFactory.createArtifact(current); - processIndex(artifact); + final Artifact currentArtifact = ctx.createArtifact(current.getIdent()); + currentArtifact.setIndexInformation(current); + + processIndex(currentArtifact, ctx); } else { - processIndexIdentifier(current.getIdent()); + processIndexIdentifier(current.getIdent(), ctx); } writePositionIfNeeded(); @@ -261,11 +267,15 @@ List walkPaginated(long take, IndexIterator indexIterator) throws } } + final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(current.getIdent()); + if(resolveIndex){ - final Artifact artifact = ArtifactFactory.createArtifact(current); - processIndex(artifact); + final Artifact currentArtifact = ctx.createArtifact(current.getIdent()); + currentArtifact.setIndexInformation(current); + + processIndex(currentArtifact, ctx); } else { - processIndexIdentifier(current.getIdent()); + processIndexIdentifier(current.getIdent(), ctx); } artifactIdents.add(current.getIdent()); @@ -314,11 +324,16 @@ List walkDates(long since, long until, IndexIterator indexIterato } } + final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(current.getIdent()); + if(resolveIndex){ - final Artifact artifact = ArtifactFactory.createArtifact(current); - processIndex(artifact); + // If we want to retain index information, build an artifact and attach it + final Artifact currentArtifact = ctx.createArtifact(current.getIdent()); + currentArtifact.setIndexInformation(current); + + processIndex(currentArtifact, ctx); } else { - processIndexIdentifier(current.getIdent()); + processIndexIdentifier(current.getIdent(), ctx); } artifactIdents.add(current.getIdent()); @@ -336,13 +351,14 @@ List walkDates(long since, long until, IndexIterator indexIterato return artifactIdents; } - private void processIndexIdentifier(ArtifactIdent ident) { + private void processIndexIdentifier(ArtifactIdent ident, ArtifactResolutionContext ctx) { if(this.artifactConfig.multipleThreads){ - queueActorRef.tell(new ProcessIdentifierMessage(ident, this), ActorRef.noSender()); + queueActorRef.tell(new ProcessIdentifierMessage(ctx, this), ActorRef.noSender()); } else { - callResolver(ident); - if(ArtifactFactory.getArtifact(ident) != null) { - analyzeArtifact(ArtifactFactory.getArtifact(ident)); + callResolver(ident, ctx); + final Artifact artifact = ctx.getArtifact(ident); + if(artifact != null) { + analyzeArtifact(artifact); } } } @@ -388,8 +404,10 @@ List processArtifactsFromInputFile() { entriesTaken += 1L; if(current != null){ + final ArtifactResolutionContext resolutionCtx = ArtifactResolutionContext.newInstance(current); + identifiers.add(current); - processIndexIdentifier(current); + processIndexIdentifier(current, resolutionCtx); } } @@ -409,13 +427,13 @@ List processArtifactsFromInputFile() { * Invokes all resolvers as defined by the analysis configuration to enrich the given artifact identifier. * @param identifier Artifact identifier to enrich */ - public void callResolver(ArtifactIdent identifier) { + public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) { if(resolvePom && resolveJar) { - resolverFactory.runBoth(identifier); + resolverFactory.runBoth(identifier, ctx); } else if(resolvePom) { - resolverFactory.runPom(identifier); + resolverFactory.runPom(identifier, ctx); } else if(resolveJar) { - resolverFactory.runJar(identifier); + resolverFactory.runJar(identifier, ctx); } } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index c1c2e16..5d96423 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -2,10 +2,10 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.model.resolution.LibraryResolutionContext; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.multithreading.ProcessLibraryMessage; import org.tudo.sse.multithreading.QueueActor; @@ -203,6 +203,7 @@ protected void onVersionListError(String groupId, String artifactId, Exception c private void processEntry(String ga){ final String[] parts = ga.split(":"); + final LibraryResolutionContext resolutionCtx = LibraryResolutionContext.newInstance(ga); if(parts.length != 2){ log.warn("Not a valid GA tuple (need :): {}", ga); @@ -213,14 +214,14 @@ private void processEntry(String ga){ final String artifactId = parts[1]; if(!config.multipleThreads){ - process(ga, groupId, artifactId); + process(groupId, artifactId, resolutionCtx); } else { - final ProcessLibraryMessage msg = new ProcessLibraryMessage(() -> process(ga, groupId, artifactId)); + final ProcessLibraryMessage msg = new ProcessLibraryMessage(() -> process(groupId, artifactId, resolutionCtx)); queueActorRef.tell(msg, ActorRef.noSender()); } } - private Void process(String ga, String groupId, String artifactId){ + private Void process(String groupId, String artifactId, LibraryResolutionContext resolutionCtx){ try { this.beforeLibraryStart(groupId, artifactId); } catch(Exception x){ @@ -233,22 +234,21 @@ private Void process(String ga, String groupId, String artifactId){ if(identifiers == null) return null; // Resolve all information for all library releases as defined by this analysis - List libraryArtifacts = new ArrayList<>(); for(ArtifactIdent ident : identifiers){ - Artifact a = ArtifactFactory.createArtifact(ident); - resolveDataAsNeeded(ident); - libraryArtifacts.add(a); + Artifact a = resolutionCtx.createArtifact(ident); + resolveDataAsNeeded(ident, resolutionCtx); + resolutionCtx.addLibraryArtifact(a); } try { // Call the custom analysis implementation - analyzeLibrary(ga, libraryArtifacts); + analyzeLibrary(resolutionCtx.getLibraryGA(), resolutionCtx.getLibraryArtifacts()); } catch(Exception x){ // Whatever the user-defined code does, we do not want to abort! log.error("Unknown error in analysis implementation", x); } - try { this.afterLibraryEnd(groupId, artifactId, libraryArtifacts); } + try { this.afterLibraryEnd(groupId, artifactId, resolutionCtx.getLibraryArtifacts()); } catch(Exception x){ log.error("Unexpected exception in analysis lifecycle hook afterLibraryEnd for library {}, {}", groupId, artifactId, x); } @@ -297,13 +297,13 @@ private Iterator getGaIterator(){ } } - private void resolveDataAsNeeded(ArtifactIdent identifier){ + private void resolveDataAsNeeded(ArtifactIdent identifier, LibraryResolutionContext resolutionCtx){ if(resolvePom && resolveJar) { - resolverFactory.runBoth(identifier); + resolverFactory.runBoth(identifier, resolutionCtx); } else if(resolvePom) { - resolverFactory.runPom(identifier); + resolverFactory.runPom(identifier, resolutionCtx); } else if(resolveJar) { - resolverFactory.runJar(identifier); + resolverFactory.runJar(identifier, resolutionCtx); } } diff --git a/src/main/java/org/tudo/sse/model/Artifact.java b/src/main/java/org/tudo/sse/model/Artifact.java index a66033e..1f2c569 100644 --- a/src/main/java/org/tudo/sse/model/Artifact.java +++ b/src/main/java/org/tudo/sse/model/Artifact.java @@ -5,6 +5,7 @@ import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.model.jar.*; import org.tudo.sse.model.pom.PomInformation; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.JarResolutionException; import org.tudo.sse.resolution.JarResolver; @@ -186,10 +187,12 @@ public Map buildTypeStructure() { Map depArts = new HashMap<>(); if(pomInformation != null) { - JarResolver resolver = new JarResolver(); + final ResolutionContext resolutionCtx = ResolutionContext.createAnonymousContext(); + final JarResolver resolver = new JarResolver(); + for(Artifact artifact : pomInformation.getEffectiveTransitiveDependencies()) { try { - artifact.setJarInformation(resolver.parseJar(artifact.getIdent()).getJarInformation()); + artifact.setJarInformation(resolver.parseJar(artifact.getIdent(), resolutionCtx).getJarInformation()); depArts.put(artifact.getIdent().getGroupID() + ":" + artifact.getIdent().getArtifactID(), artifact); } catch (JarResolutionException e) { log.error(e); diff --git a/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java b/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java index 927d55d..54e8371 100644 --- a/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java +++ b/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java @@ -4,12 +4,12 @@ import org.apache.maven.model.io.xpp3.MavenXpp3Reader; import org.codehaus.plexus.util.xml.pull.XmlPullParserException; import org.tudo.sse.model.*; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.PomResolutionException; import org.tudo.sse.resolution.PomResolver; import java.io.*; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -58,14 +58,14 @@ public LocalPomInformation(InputStream pomStream, boolean loadTransitives){ public void resolveLocalPom(PomResolver resolver) throws IOException, XmlPullParserException { if(pomStream != null) { resolveLocalFile(resolver, this.pomStream); - } else { + } else if(pomPath != null) { try (FileInputStream is = new FileInputStream(this.pomPath.toFile())) { resolveLocalFile(resolver, is); } } if(this.loadTransitives){ - resolveParentAndImport(resolver); + resolveParentAndImport(resolver, ResolutionContext.createAnonymousContext()); resolveDependencies(resolver); } @@ -167,10 +167,10 @@ private RawPomFeatures processRawPomFeatures(Model model, PomResolver resolver) * @see PomResolver * @param resolver the resolver used to resolve the parents and imports of the local pom */ - private void resolveParentAndImport(PomResolver resolver) { + private void resolveParentAndImport(PomResolver resolver, ResolutionContext ctx) { if(this.rawPomFeatures.getParent() != null) { try { - this.parent = resolver.processArtifact(this.rawPomFeatures.getParent()); + this.parent = resolver.processArtifact(this.rawPomFeatures.getParent(), ctx); } catch (PomResolutionException | org.tudo.sse.resolution.FileNotFoundException | IOException e) { throw new RuntimeException(e); } @@ -178,7 +178,7 @@ private void resolveParentAndImport(PomResolver resolver) { if(this.rawPomFeatures.getDependencyManagement() != null) { try { - this.imports = resolver.resolveImports(this.rawPomFeatures.getDependencyManagement(), this); + this.imports = resolver.resolveImports(this.rawPomFeatures.getDependencyManagement(), this, ctx); } catch (PomResolutionException | org.tudo.sse.resolution.FileNotFoundException | IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java new file mode 100644 index 0000000..59ffab0 --- /dev/null +++ b/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java @@ -0,0 +1,31 @@ +package org.tudo.sse.model.resolution; + +import org.tudo.sse.model.ArtifactIdent; + +public class ArtifactResolutionContext extends ResolutionContext { + + private final ArtifactIdent identifier; + + private ArtifactResolutionContext(ArtifactIdent identifier) { + this.identifier = identifier; + } + + public static ArtifactResolutionContext newInstance(ArtifactIdent identifier) { + return new ArtifactResolutionContext(identifier); + } + + public ArtifactIdent getIdentifier() { + return identifier; + } + + @Override + public boolean isArtifactResolutionContext() { + return true; + } + + @Override + public ArtifactResolutionContext asArtifactResolutionContext() { + return this; + } + +} diff --git a/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java b/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java new file mode 100644 index 0000000..cefea1b --- /dev/null +++ b/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java @@ -0,0 +1,43 @@ +package org.tudo.sse.model.resolution; + +import org.tudo.sse.model.Artifact; + +import java.util.ArrayList; +import java.util.List; + +public final class LibraryResolutionContext extends ResolutionContext { + + private final String libraryGA; + private final List libraryArtifacts; + + private LibraryResolutionContext(String libraryGA) { + this.libraryGA = libraryGA; + this.libraryArtifacts = new ArrayList<>(); + } + + public static LibraryResolutionContext newInstance(String libraryGA) { + return new LibraryResolutionContext(libraryGA); + } + + public void addLibraryArtifact(Artifact a){ + this.libraryArtifacts.add(a); + } + + public List getLibraryArtifacts(){ + return this.libraryArtifacts; + } + + public String getLibraryGA() { + return libraryGA; + } + + @Override + public boolean isLibraryResolutionContext() { + return true; + } + + @Override + public LibraryResolutionContext asLibraryResolutionContext(){ + return this; + } +} diff --git a/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java b/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java new file mode 100644 index 0000000..14437c9 --- /dev/null +++ b/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java @@ -0,0 +1,53 @@ +package org.tudo.sse.model.resolution; + +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public abstract class ResolutionContext { + + public static ResolutionContext createAnonymousContext(){ + return new ResolutionContext() {}; + } + + protected Map artifactsResolved = new HashMap<>(); + + public Artifact createArtifact(ArtifactIdent ident) { + if(!artifactsResolved.containsKey(ident)){ + final Artifact artifact = new Artifact(ident); + artifactsResolved.put(ident, artifact); + } + + return artifactsResolved.get(ident); + } + + public Artifact getArtifact(ArtifactIdent ident) { + return artifactsResolved.getOrDefault(ident, null); + } + + public Set getAllArtifactsResolved(){ + return new HashSet<>(artifactsResolved.values()); + } + + public boolean isLibraryResolutionContext(){ + return false; + } + + public LibraryResolutionContext asLibraryResolutionContext(){ + throw new IllegalStateException("Not a library resolution context"); + } + + public boolean isArtifactResolutionContext(){ + return false; + } + + public ArtifactResolutionContext asArtifactResolutionContext(){ + throw new IllegalStateException("Not an artifact resolution context"); + } + + +} diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index 9770ee3..cda276e 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -2,6 +2,7 @@ import org.tudo.sse.analyses.MavenCentralArtifactAnalysis; import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.model.resolution.ArtifactResolutionContext; /** * A message passed to the processing queue to indicate that a given analysis instance must process the given @@ -9,16 +10,16 @@ */ public class ProcessIdentifierMessage implements WorkItem { - private final ArtifactIdent identifier; + private final ArtifactResolutionContext artifactCtx; private final MavenCentralArtifactAnalysis instance; /** * Creates a new message with the given artifact identifier and analysis instance. - * @param identifier The artifact identifier that must be processed + * @param resolutionContext The artifact resolution context to process * @param instance The analysis instance that must process the identifier */ - public ProcessIdentifierMessage(ArtifactIdent identifier, MavenCentralArtifactAnalysis instance) { - this.identifier = identifier; + public ProcessIdentifierMessage(ArtifactResolutionContext resolutionContext, MavenCentralArtifactAnalysis instance) { + this.artifactCtx = resolutionContext; this.instance = instance; } @@ -27,7 +28,15 @@ public ProcessIdentifierMessage(ArtifactIdent identifier, MavenCentralArtifactAn * @return The artifact identifier */ public ArtifactIdent getIdentifier() { - return identifier; + return this.artifactCtx.getIdentifier(); + } + + /** + * Retrieves the resolution context of this message + * @return ResolutionContext contained + */ + public ArtifactResolutionContext getArtifactResolutionContext() { + return this.artifactCtx; } /** diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index 828cfd9..74a7638 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -4,6 +4,8 @@ import akka.actor.Props; import akka.japi.pf.ReceiveBuilder; import org.tudo.sse.ArtifactFactory; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.model.resolution.ArtifactResolutionContext; /** * This class is spawned in multiple threads @@ -28,8 +30,12 @@ public static Props props() { public Receive createReceive() { return ReceiveBuilder.create() .match(ProcessIdentifierMessage.class, message -> { - message.getInstance().callResolver(message.getIdentifier()); - message.getInstance().analyzeArtifact(ArtifactFactory.getArtifact(message.getIdentifier())); + final ArtifactIdent identifier = message.getIdentifier(); + final ArtifactResolutionContext ctx = message.getArtifactResolutionContext(); + + message.getInstance().callResolver(identifier, ctx); + message.getInstance().analyzeArtifact(ctx.getArtifact(identifier)); + getSender().tell(WorkItemFinishedMessage.getInstance(), getSelf()); }) .match(ProcessLibraryMessage.class, message -> { diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index 863aa3f..bcc22af 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -15,6 +15,7 @@ import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.JarInformation; import org.tudo.sse.model.jar.ObjType; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.utils.MavenCentralRepository; import scala.Tuple2; @@ -77,12 +78,12 @@ public void setOutput(boolean output) { * @return a list of resolved artifacts * @see Artifact */ - public List resolveJars(List identifiers) { + public List resolveJars(List identifiers, ResolutionContext ctx) { List toReturn = new ArrayList<>(); int count = 0; for(ArtifactIdent current : identifiers) { try { - toReturn.add(parseJar(current)); + toReturn.add(parseJar(current, ctx)); } catch (JarResolutionException e) { log.error(e); } @@ -104,10 +105,12 @@ public List resolveJars(List identifiers) { * @return a resolved artifact * @throws JarResolutionException when there is an issue resolving the given jar artifact */ - public Artifact parseJar(ArtifactIdent identifier) throws JarResolutionException { - if(ArtifactFactory.getArtifact(identifier) != null && Objects.requireNonNull(ArtifactFactory.getArtifact(identifier)).getJarInformation() != null) { - return ArtifactFactory.getArtifact(identifier); - } + public Artifact parseJar(ArtifactIdent identifier, ResolutionContext ctx) throws JarResolutionException { + final Artifact currentArtifact = ctx.createArtifact(identifier); + + // Only build jar information if it is not already present for the current resolution context + if(currentArtifact.hasJarInformation()) + return currentArtifact; try { URL jarURL = identifier.getMavenCentralJarUri().toURL(); @@ -137,7 +140,13 @@ public Artifact parseJar(ArtifactIdent identifier) throws JarResolutionException } List> classList = readClassesFromJarStream(jarInput, jarURL); - return ArtifactFactory.createArtifact(parsingClassFiles(classList, identifier)); + + final JarInformation theJarInformation = parsingClassFiles(classList, identifier); + + // Finally set jar information for current artifact, then return + currentArtifact.setJarInformation(theJarInformation); + + return currentArtifact; } catch (IOException e) { log.error(e); diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java index 02cbdeb..5c97f67 100644 --- a/src/main/java/org/tudo/sse/resolution/PomResolver.java +++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java @@ -16,6 +16,7 @@ import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; import org.tudo.sse.model.pom.RawPomFeatures; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; import org.tudo.sse.utils.MavenCentralRepository; @@ -153,14 +154,14 @@ public PomResolver(boolean output, Path pathToDirectory, boolean resolveTransiti * @return list of resolved artifacts * @see Artifact */ - public List resolveArtifacts(List idents) { + public List resolveArtifacts(List idents, ResolutionContext ctx) { List poms = new ArrayList<>(); log.info("There are {} identifiers to resolve", idents.size()); int count =0; for(ArtifactIdent ident : idents) { try { - poms.add(resolveArtifact(ident)); + poms.add(resolveArtifact(ident, ctx)); } catch(PomResolutionException e) { log.error(e); } catch ( IOException e) { @@ -188,7 +189,7 @@ public List resolveArtifacts(List idents) { * @throws IOException If connection errors occur * @throws PomResolutionException If the POM resolution process fails due to invalid file contents */ - public Artifact resolveArtifact(ArtifactIdent identifier) throws FileNotFoundException, IOException, PomResolutionException { + public Artifact resolveArtifact(ArtifactIdent identifier, ResolutionContext ctx) throws FileNotFoundException, IOException, PomResolutionException { Map alrEncountered = new HashMap<>(); if(output) { try(InputStream inputStream = MavenRepo.openPomFileInputStream(identifier)){ @@ -202,12 +203,12 @@ public Artifact resolveArtifact(ArtifactIdent identifier) throws FileNotFoundExc } } - Artifact toReturn = processArtifact(identifier); + Artifact toReturn = processArtifact(identifier, ctx); toReturn.getPomInformation().setResolvedDependencies(resolveDependencies(toReturn.getPomInformation())); if(resolveTransitives) { - resolveAllTransitives(toReturn, alrEncountered, null); - resolveEffectiveTransitives(toReturn); + resolveAllTransitives(toReturn, alrEncountered, null, ctx); + resolveEffectiveTransitives(toReturn, ctx); } return toReturn; } @@ -346,10 +347,12 @@ public List getDependencies(List * @throws FileNotFoundException When the POM file does not exist for the given artifact * @throws IOException If connection errors occur */ - public Artifact processArtifact(ArtifactIdent identifier) throws PomResolutionException, FileNotFoundException, IOException { - if(ArtifactFactory.getArtifact(identifier) != null && Objects.requireNonNull(ArtifactFactory.getArtifact(identifier)).getPomInformation() != null) { - return ArtifactFactory.getArtifact(identifier); - } + public Artifact processArtifact(ArtifactIdent identifier, ResolutionContext ctx) throws PomResolutionException, FileNotFoundException, IOException { + final Artifact currentArtifact = ctx.createArtifact(identifier); + + // Only build pom information if it is not already present for the current resolution context + if(currentArtifact.hasPomInformation()) + return currentArtifact; PomInformation pomInformation = new PomInformation(identifier); @@ -371,16 +374,19 @@ public Artifact processArtifact(ArtifactIdent identifier) throws PomResolutionEx pomInformation.setRawPomFeatures(rawPomFeatures); if(pomInformation.getRawPomFeatures() != null) { - getAllRelevantFiles(pomInformation); + getAllRelevantFiles(pomInformation, ctx); } - return ArtifactFactory.createArtifact(pomInformation); + // Finally set pom information for current artifact, then return + currentArtifact.setPomInformation(pomInformation); + + return currentArtifact; } - private void getAllRelevantFiles(PomInformation pomInformation) throws PomResolutionException, FileNotFoundException, IOException { + private void getAllRelevantFiles(PomInformation pomInformation, ResolutionContext ctx) throws PomResolutionException, FileNotFoundException, IOException { if(pomInformation.getRawPomFeatures().getParent() != null) { try { - pomInformation.setParent(processArtifact(pomInformation.getRawPomFeatures().getParent())); + pomInformation.setParent(processArtifact(pomInformation.getRawPomFeatures().getParent(), ctx)); } catch(PomResolutionException e) { log.error("Failed to resolve parent: {}", e.getMessage()); } catch (FileNotFoundException ignored) {} @@ -392,7 +398,7 @@ private void getAllRelevantFiles(PomInformation pomInformation) throws PomResolu } if(pomInformation.getRawPomFeatures().getDependencyManagement() != null) { - pomInformation.setImports(resolveImports(pomInformation.getRawPomFeatures().getDependencyManagement(), pomInformation)); + pomInformation.setImports(resolveImports(pomInformation.getRawPomFeatures().getDependencyManagement(), pomInformation, ctx)); } } @@ -405,14 +411,14 @@ private void getAllRelevantFiles(PomInformation pomInformation) throws PomResolu * @throws FileNotFoundException handles errors with not finding a file * @throws IOException handles errors with opening and closing files */ - public List resolveImports(List managedDependencies, PomInformation info) throws PomResolutionException, FileNotFoundException, IOException { + public List resolveImports(List managedDependencies, PomInformation info, ResolutionContext ctx) throws PomResolutionException, FileNotFoundException, IOException { ArrayList imports = new ArrayList<>(); for(org.tudo.sse.model.pom.Dependency dependency: managedDependencies) { try { if(dependency.getScope() != null && dependency.getScope().equals("import")) { dependency = resolveVersion(dependency, info); - Artifact temp = processArtifact(dependency.getIdent()); + Artifact temp = processArtifact(dependency.getIdent(), ctx); if(temp != null) { imports.add(temp); } @@ -803,8 +809,8 @@ private static boolean isVersionRange(String version) { * @param toResolve the current artifact transitive dependencies are being resolved for. * @throws IOException thrown when there's an issue opening the pom for an artifact */ - public void resolveAllTransitives(Artifact toResolve) throws IOException { - resolveAllTransitives(toResolve, new HashMap<>(), null); + public void resolveAllTransitives(Artifact toResolve, ResolutionContext ctx) throws IOException { + resolveAllTransitives(toResolve, new HashMap<>(), null, ctx); } /** @@ -815,7 +821,7 @@ public void resolveAllTransitives(Artifact toResolve) throws IOException { * @param exclusions a Set of G:A values to not include in resolution * @throws IOException thrown when there's an issue opening the pom for an artifact */ - public void resolveAllTransitives(Artifact current, Map alrEncountered, Set exclusions) throws IOException { + public void resolveAllTransitives(Artifact current, Map alrEncountered, Set exclusions, ResolutionContext ctx) throws IOException { List transitives = new ArrayList<>(); for (org.tudo.sse.model.pom.Dependency dependency : current.getPomInformation().getResolvedDependencies()) { if(exclusions == null || !exclusions.contains(dependency.getIdent().getGA())) { @@ -825,7 +831,7 @@ public void resolveAllTransitives(Artifact current, Map if (dependencyRelevant && !alrEncountered.containsKey(dependency.getIdent())) { alrEncountered.put(dependency.getIdent(), null); - Artifact transitive = resolveTransitives(current, dependency, alrEncountered, dependency.getExclusions()); + Artifact transitive = resolveTransitives(current, dependency, alrEncountered, dependency.getExclusions(), ctx); if (transitive != null) { if(transitive.getRelocation() != null) { alrEncountered.remove(dependency.getIdent()); @@ -844,32 +850,40 @@ public void resolveAllTransitives(Artifact current, Map current.getPomInformation().setAllTransitiveDependencies(transitives); } - private Artifact resolveTransitives(Artifact current, org.tudo.sse.model.pom.Dependency toResolve, Map alrEncountered, Set exclusions) throws IOException { + private Artifact resolveTransitives(Artifact current, + org.tudo.sse.model.pom.Dependency toResolve, + Map alrEncountered, + Set exclusions, + ResolutionContext ctx) throws IOException { try { - return recursiveResolver(toResolve.getIdent(), alrEncountered, exclusions); + return recursiveResolver(toResolve.getIdent(), alrEncountered, exclusions, ctx); } catch(FileNotFoundException | PomResolutionException e) { if(!current.getPomInformation().getRawPomFeatures().getRepositories().isEmpty()) { - return resolveFromSecondaryRepo(current.getPomInformation().getRawPomFeatures().getRepositories(), toResolve, alrEncountered, exclusions); + return resolveFromSecondaryRepo(current.getPomInformation().getRawPomFeatures().getRepositories(), toResolve, alrEncountered, exclusions, ctx); } if(!(e instanceof FileNotFoundException)) log.error(e); return null; } } - private Artifact recursiveResolver(ArtifactIdent identifier, Map alrEncountered, Set exclusions) throws FileNotFoundException, IOException, PomResolutionException { - Artifact current = processArtifact(identifier); + private Artifact recursiveResolver(ArtifactIdent identifier, Map alrEncountered, Set exclusions, ResolutionContext ctx) throws FileNotFoundException, IOException, PomResolutionException { + Artifact current = processArtifact(identifier, ctx); current.getPomInformation().setResolvedDependencies(resolveDependencies(current.getPomInformation())); - resolveAllTransitives(current, alrEncountered, exclusions); + resolveAllTransitives(current, alrEncountered, exclusions, ctx); return current; } - private Artifact resolveFromSecondaryRepo(List repos, org.tudo.sse.model.pom.Dependency toResolve, Map alrEncountered, Set exclusions) { + private Artifact resolveFromSecondaryRepo(List repos, + org.tudo.sse.model.pom.Dependency toResolve, + Map alrEncountered, + Set exclusions, + ResolutionContext ctx) { int i = 0; Artifact toReturn = null; while(i < repos.size() && toReturn == null) { toResolve.getIdent().setRepository(repos.get(i)); try { - toReturn = recursiveResolver(toResolve.getIdent(), alrEncountered, exclusions); + toReturn = recursiveResolver(toResolve.getIdent(), alrEncountered, exclusions, ctx); } catch (IOException | PomResolutionException e) { log.error(e); } catch (FileNotFoundException ignored) {} @@ -883,7 +897,7 @@ private Artifact resolveFromSecondaryRepo(List repos, org.tudo.sse.model * This method attempts to get all effective transitive dependencies, resolving conflicts, and removing duplicates. * @param toResolve the artifact for which to resolve the effective transitive dependencies */ - public void resolveEffectiveTransitives(Artifact toResolve) { + public void resolveEffectiveTransitives(Artifact toResolve, ResolutionContext ctx) { // Can only resolve dependencies if POM info is present. // IMPROVE: Do we want to load POM info on demand? if(!toResolve.hasPomInformation()){ @@ -895,7 +909,7 @@ public void resolveEffectiveTransitives(Artifact toResolve) { // If "normal" transitive dependencies have not been computed yet, we try to do that on demand if(allTransitive == null){ try { - this.resolveAllTransitives(toResolve, new HashMap<>(), null); + this.resolveAllTransitives(toResolve, new HashMap<>(), null, ctx); allTransitive = toResolve.getPomInformation().getAllTransitiveDependencies(); } catch (IOException iox){ throw new IllegalStateException("Failed to compute transitive dependencies on demand", iox); @@ -937,8 +951,9 @@ public void resolveEffectiveTransitives(Artifact toResolve) { } for(Map.Entry entry : foundGA.entrySet()) { - toReturn.add(ArtifactFactory.getArtifact(entry.getValue())); + toReturn.add(ctx.getArtifact(entry.getValue())); } + toResolve.getPomInformation().setEffectiveTransitiveDependencies(toReturn); toResolve.getPomInformation().setTransitiveConflicts(conflicts); } diff --git a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java index ac0537a..f2f3d0c 100644 --- a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java +++ b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.model.resolution.ResolutionContext; /** * This class manages the pom and jar resolver, giving a way to run one or the other. @@ -45,9 +46,9 @@ public ResolverFactory(boolean output, Path pathToDirectory, boolean pomIncludeT * * @param identifier Artifact identifier to resolve */ - public void runPom(ArtifactIdent identifier) { + public void runPom(ArtifactIdent identifier, ResolutionContext ctx) { try { - pomResolver.resolveArtifact(identifier); + pomResolver.resolveArtifact(identifier, ctx); } catch (IOException | PomResolutionException e) { log.error(e); } catch (FileNotFoundException ignored) {} @@ -58,9 +59,9 @@ public void runPom(ArtifactIdent identifier) { * * @param identifier Artifact identifier to resolve */ - public void runJar(ArtifactIdent identifier) { + public void runJar(ArtifactIdent identifier, ResolutionContext ctx) { try { - jarResolver.parseJar(identifier); + jarResolver.parseJar(identifier, ctx); } catch (JarResolutionException e) { log.error(e); } @@ -71,16 +72,16 @@ public void runJar(ArtifactIdent identifier) { * * @param identifier Artifact identifier to resolve */ - public void runBoth(ArtifactIdent identifier) { + public void runBoth(ArtifactIdent identifier, ResolutionContext ctx) { try { - pomResolver.resolveArtifact(identifier); + pomResolver.resolveArtifact(identifier, ctx); } catch (IOException | PomResolutionException e) { log.error(e); } catch(FileNotFoundException ignored){} try { jarResolver.setOutput(false); - jarResolver.parseJar(identifier); + jarResolver.parseJar(identifier, ctx); } catch (JarResolutionException e) { log.error(e); } diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 774991d..2d0bae4 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -155,6 +155,17 @@ void parseCmdLineNegative() { @Test @DisplayName("An analysis must adhere to pagination configurations") void walkPaginated() { + + final Map artifactsSeen = new HashMap<>(); + final MavenCentralArtifactAnalysis theAnalysis = new MavenCentralArtifactAnalysis(true, false, false, false) { + @Override + public void analyzeArtifact(Artifact current) { + artifactsSeen.put(current.getIdent(), current); + } + }; + + + List> inputs = new ArrayList<>(); inputs.add(new Tuple2<>(500, 10)); inputs.add(new Tuple2<>(0, 10)); @@ -169,19 +180,23 @@ void walkPaginated() { try { IndexIterator iterator = new IndexIterator(new URI(base), start1); - List collected1 = new ArrayList<>(analysisUnderTest.walkPaginated(take, iterator)); + List collected1 = new ArrayList<>(theAnalysis.walkPaginated(take, iterator)); + + Artifact lastOne = artifactsSeen.get(collected1.get(collected1.size() - 1)); + + assertNotNull(lastOne); - Artifact lastOne = ArtifactFactory.getArtifact(collected1.get(collected1.size() - 1)); int i = 2; while(lastOne.getIndexInformation().getIndex() > start2) { - lastOne = ArtifactFactory.getArtifact(collected1.get(collected1.size() - i)); + lastOne = artifactsSeen.get(collected1.get(collected1.size() - i)); + assertNotNull(lastOne); i++; } iterator = new IndexIterator(new URI(base), start2); - List collected2 = new ArrayList<>(analysisUnderTest.walkPaginated(1, iterator)); - long lastOne2 = ArtifactFactory.getArtifact(collected2.get(0)).getIndexInformation().getIndex(); + List collected2 = new ArrayList<>(theAnalysis.walkPaginated(1, iterator)); + long lastOne2 = artifactsSeen.get(collected2.get(0)).getIndexInformation().getIndex(); assertEquals(lastOne.getIndexInformation().getIndex(), lastOne2); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); diff --git a/src/test/java/org/tudo/sse/model/TypeStructureTest.java b/src/test/java/org/tudo/sse/model/TypeStructureTest.java index e7f768d..82d87b4 100644 --- a/src/test/java/org/tudo/sse/model/TypeStructureTest.java +++ b/src/test/java/org/tudo/sse/model/TypeStructureTest.java @@ -5,9 +5,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.model.jar.ClassFileNode; import org.tudo.sse.model.jar.DefinedClassFileNode; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.*; import java.io.IOException; @@ -42,11 +42,13 @@ void buildTypeStructure() { Artifact jarArt1; ArtifactIdent ident = new ArtifactIdent("org.springframework", "spring-web", "6.1.11"); Artifact jarArt2; - jarArt2 = ArtifactFactory.getArtifact(ident); + + final ResolutionContext testCtx = ResolutionContext.createAnonymousContext(); + try { - jarArt1 = resolver.parseJar(new ArtifactIdent("org.bouncycastle", "bcpg-lts8on", "2.73.6")); - jarArt2 = resolver.parseJar(ident); - pomResolver.resolveArtifact(ident); + jarArt1 = resolver.parseJar(new ArtifactIdent("org.bouncycastle", "bcpg-lts8on", "2.73.6"), testCtx); + jarArt2 = resolver.parseJar(ident, testCtx); + pomResolver.resolveArtifact(ident, testCtx); } catch (JarResolutionException | PomResolutionException | FileNotFoundException | IOException e) { throw new RuntimeException(e); } @@ -65,7 +67,6 @@ void buildTypeStructure() { multipleChildren(results2.get("reactor/core/observability/DefaultSignalListener"), (Map) expected2.get("test1")); multiLevelChild(results2.get("java/lang/RuntimeException"), (Map) expected2.get("test2")); interfacesCheck(results2.get("java/lang/Object"), (Map) expected2.get("test3")); - } ClassFileNode findNode(ClassFileNode root, String toFind) { diff --git a/src/test/java/org/tudo/sse/resolution/JarResolverTest.java b/src/test/java/org/tudo/sse/resolution/JarResolverTest.java index 9d929cc..2d7e607 100644 --- a/src/test/java/org/tudo/sse/resolution/JarResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/JarResolverTest.java @@ -7,6 +7,7 @@ import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.ClassFile; import org.tudo.sse.model.jar.JarInformation; +import org.tudo.sse.model.resolution.ResolutionContext; import java.io.InputStream; import java.io.InputStreamReader; @@ -38,8 +39,9 @@ void setUp() { @Test void parseJar() { JarInformation toTest; + final ResolutionContext resolutionContext = ResolutionContext.createAnonymousContext(); try { - toTest = jarResolver.parseJar(new ArtifactIdent("com.google.common.html.types", "types", "1.0.0")).getJarInformation(); + toTest = jarResolver.parseJar(new ArtifactIdent("com.google.common.html.types", "types", "1.0.0"), resolutionContext).getJarInformation(); } catch (JarResolutionException e) { throw new RuntimeException(e); } diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index f95e72f..06992c2 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -23,6 +23,7 @@ import org.tudo.sse.model.pom.Dependency; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.RawPomFeatures; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; @@ -33,6 +34,7 @@ class PomResolverTest { PomResolver pomResolver; + ResolutionContext testContext; Map json; Gson gson = new Gson(); @@ -48,6 +50,7 @@ class PomResolverTest { @BeforeEach void setUp() { pomResolver = new PomResolver(true); + testContext = ResolutionContext.createAnonymousContext(); } @Test @@ -83,7 +86,7 @@ void processArtifacts() { ArrayList> expected = (ArrayList>) json.get("resolveArtifacts"); - List poms = pomResolver.resolveArtifacts(idents); + List poms = pomResolver.resolveArtifacts(idents, testContext); //check to see that the parsed data from the POM for these indexes are correct for actually retrieving the files from url for(int i = 0; i < expected.size(); i++) { @@ -148,7 +151,7 @@ public List getReleases(String groupId, String artifactId) throws IOExce ArrayList> temp = (ArrayList>) allTestData.get("tests"); //check expected values against processed ones? - List results = pomResolver.resolveArtifacts(idents); + List results = pomResolver.resolveArtifacts(idents, testContext); for(int i = 0; i < results.size(); i++) { List current = results.get(i).getPomInformation().getResolvedDependencies(); @@ -276,7 +279,7 @@ void resolveTransitiveDependencies() { ArrayList currentDependencies = (ArrayList) allTransitives.get("dependencies"); List idents = new ArrayList<>(); idents.add(new ArtifactIdent("org.openengsb", "openengsb-maven-plugin", "1.3.1")); - List results = pomResolver.resolveArtifacts(idents); + List results = pomResolver.resolveArtifacts(idents, testContext); //create a recursive driver, that takes in the main map and the dependencies list, so it can recur to each level of the tree for(Artifact current : results) { @@ -310,7 +313,7 @@ void resolveEffectiveTransitiveDependencies() { ArrayList>> conflicts = (ArrayList>>) json.get("conflicts"); //check expected values against processed ones? - List results = pomResolver.resolveArtifacts(idents); + List results = pomResolver.resolveArtifacts(idents, testContext); for(int i = 0; i < results.size(); i++) { @@ -357,7 +360,7 @@ void resolveFromSecondaryRepository() { //set up expected values from mvn dependency tree to the json file ArrayList expectedDeps = (ArrayList) json.get("2ndRepo"); - List results = pomResolver.resolveArtifacts(idents); + List results = pomResolver.resolveArtifacts(idents, testContext); for(int i = 0; i < results.size(); i++) { List current = results.get(i).getPomInformation().getEffectiveTransitiveDependencies(); From a8267471d9fd988f208f2279085d81ac99d744cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 3 Dec 2025 10:11:51 +0100 Subject: [PATCH 22/55] Remove ArtifactFactory --- .../java/org/tudo/sse/ArtifactFactory.java | 97 ------------------- src/main/java/org/tudo/sse/IndexWalker.java | 31 +++++- .../MavenCentralArtifactAnalysis.java | 1 - .../sse/multithreading/ResolverActor.java | 1 - .../org/tudo/sse/resolution/JarResolver.java | 1 - .../org/tudo/sse/resolution/PomResolver.java | 1 - .../MavenCentralArtifactAnalysisTest.java | 23 +++-- .../MavenCentralLibraryAnalysisTest.java | 7 -- 8 files changed, 38 insertions(+), 124 deletions(-) delete mode 100644 src/main/java/org/tudo/sse/ArtifactFactory.java diff --git a/src/main/java/org/tudo/sse/ArtifactFactory.java b/src/main/java/org/tudo/sse/ArtifactFactory.java deleted file mode 100644 index 45582a7..0000000 --- a/src/main/java/org/tudo/sse/ArtifactFactory.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.tudo.sse; - -import java.util.HashMap; -import java.util.Map; - -import org.opalj.log.GlobalLogContext$; -import org.opalj.log.OPALLogger; -import org.tudo.sse.model.*; -import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.model.jar.JarInformation; -import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.utils.MarinOpalLogger; - -/** - * The ArtifactFactory handles the creation and storage of artifacts resolved. - * Using a map double resolutions are avoided and faster retrievals are possible. - */ -public final class ArtifactFactory { - - static { - // Use global OPAL Logger - will only forward OPAL messages with level error or fatal - OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); - } - - private ArtifactFactory() {} - - /** - * A map that stores all artifacts collected during index, pom, and jar resolution. - */ - public static final Map artifacts = new HashMap<>(); - - /** - * Creates a new artifact or retrieves an existing one. If a new artifact is created, no information is attached. - * @param ident The artifact identifier - * @return The artifact with the given identifier - */ - public static Artifact createArtifact(ArtifactIdent ident) { - assert(ident != null); - if(artifacts.containsKey(ident)) { - return artifacts.get(ident); - } else { - Artifact newArtifact = new Artifact(ident); - artifacts.put(ident, newArtifact); - return newArtifact; - } - } - - /** - * This method creates a new Artifact or retrieves it from the map. - * @param artifactInformation an identifier used to keep track of artifacts - * @return a newly created or retrieved artifact - */ - public static Artifact createArtifact(ArtifactInformation artifactInformation) { - - //first check if it exists in artifacts - if(artifacts.containsKey(artifactInformation.getIdent())) { - Artifact current = artifacts.get(artifactInformation.getIdent()); - - //see if it matches an empty field - if(current.getIndexInformation() == null && (artifactInformation instanceof IndexInformation)) { - current.setIndexInformation((IndexInformation) artifactInformation); - } else if(current.getPomInformation() == null && (artifactInformation instanceof PomInformation)) { - current.setPomInformation((PomInformation) artifactInformation); - } else if (current.getJarInformation() == null && (artifactInformation instanceof JarInformation)) { - current.setJarInformation((JarInformation) artifactInformation); - } - - //return it - return current; - } - - Artifact newArtifact; - if(artifactInformation instanceof IndexInformation) { - newArtifact = new Artifact((IndexInformation) artifactInformation); - } else if (artifactInformation instanceof PomInformation) { - newArtifact = new Artifact((PomInformation) artifactInformation); - } else { - newArtifact = new Artifact((JarInformation) artifactInformation); - } - - artifacts.put(artifactInformation.getIdent(), newArtifact); - return newArtifact; - } - - /** - * Searches for an artifact with the matching identifier, returning it if found. - * @param ident The artifact identifier for which to look up an artifact definition - * - * @return The artifact object belonging to this identifier, or null if no such object exists - */ - public static Artifact getArtifact(ArtifactIdent ident) { - if(artifacts.containsKey(ident)) { - return artifacts.get(ident); - } - return null; - } -} diff --git a/src/main/java/org/tudo/sse/IndexWalker.java b/src/main/java/org/tudo/sse/IndexWalker.java index 2c25465..03d3679 100644 --- a/src/main/java/org/tudo/sse/IndexWalker.java +++ b/src/main/java/org/tudo/sse/IndexWalker.java @@ -8,6 +8,7 @@ import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; +import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.utils.IndexIterator; import java.net.URI; @@ -82,9 +83,16 @@ public List walkAllIndexes() throws IOException { indexIterator = new IndexIterator(base); } + final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); + List artifacts = new ArrayList<>(); while(indexIterator.hasNext()) { - artifacts.add(ArtifactFactory.createArtifact(indexIterator.next())); + final IndexInformation indexInformation = indexIterator.next(); + final Artifact theArtifact = ctx.createArtifact(indexInformation.getIdent()); + + theArtifact.setIndexInformation(indexInformation); + + artifacts.add(theArtifact); } if(artifacts.size() % 500000 == 0) { @@ -135,13 +143,21 @@ public List walkPaginated(long skip, long take) throws IOException { if(resetIterator) { indexIterator = new IndexIterator(base); } + + final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); + List artifacts = new ArrayList<>(); int count = 0; int fromFront = 0; while(indexIterator.hasNext() && count < take) { if(fromFront >= skip) { - artifacts.add(ArtifactFactory.createArtifact(indexIterator.next())); + final IndexInformation indexInformation = indexIterator.next(); + final Artifact theArtifact = ctx.createArtifact(indexInformation.getIdent()); + + theArtifact.setIndexInformation(indexInformation); + + artifacts.add(theArtifact); count++; } else { indexIterator.next(); @@ -163,15 +179,22 @@ public List walkDates(long since, long until) throws IOException { if(resetIterator) { indexIterator = new IndexIterator(base); } + + final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); + List artifacts = new ArrayList<>(); long currentToSince = 0; long sinceToUntil = since; while(indexIterator.hasNext() && sinceToUntil < until) { - IndexInformation temp = indexIterator.next(); + final IndexInformation temp = indexIterator.next(); if(currentToSince >= since) { sinceToUntil = temp.getLastModified(); - artifacts.add(ArtifactFactory.createArtifact(temp)); + + final Artifact theArtifact = ctx.createArtifact(temp.getIdent()); + theArtifact.setIndexInformation(temp); + + artifacts.add(theArtifact); } else { currentToSince = temp.getLastModified(); } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index c9ba41d..2f29f8f 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -2,7 +2,6 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index 74a7638..e0cfeaf 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -3,7 +3,6 @@ import akka.actor.AbstractActor; import akka.actor.Props; import akka.japi.pf.ReceiveBuilder; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.resolution.ArtifactResolutionContext; diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index bcc22af..db96aa7 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -10,7 +10,6 @@ import org.opalj.br.package$; import org.opalj.br.reader.Java16LibraryFramework; import org.opalj.log.GlobalLogContext$; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.JarInformation; diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java index 5c97f67..5b45293 100644 --- a/src/main/java/org/tudo/sse/resolution/PomResolver.java +++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java @@ -11,7 +11,6 @@ import org.eclipse.aether.version.InvalidVersionSpecificationException; import org.eclipse.aether.version.Version; import org.eclipse.aether.version.VersionRange; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.model.*; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 2d0bae4..3ee1b22 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -2,10 +2,8 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; @@ -40,11 +38,6 @@ class MavenCentralArtifactAnalysisTest { json = gson.fromJson(targetReader, new TypeToken>() {}.getType()); } - @AfterEach - public void cleanup(){ - ArtifactFactory.artifacts.clear(); - } - @Test @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular() throws IOException{ @@ -294,15 +287,21 @@ void checkMultiThreading() { for(int i = 0; i < singleArgs.size(); i++) { - MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithPomRequirement(); + Set identifiersSeen = new HashSet<>(); + MavenCentralArtifactAnalysis tester = new MavenCentralArtifactAnalysis(false, true, false, false) { + @Override + public void analyzeArtifact(Artifact current) { + identifiersSeen.add(current.ident); + } + }; tester.runAnalysis(singleArgs.get(i)); - Set singleResult = ArtifactFactory.artifacts.keySet(); - cleanup(); + Set singleResult = new HashSet<>(identifiersSeen); + + identifiersSeen.clear(); tester.runAnalysis(multiArgs.get(i)); - Set multiResult = ArtifactFactory.artifacts.keySet(); - cleanup(); + Set multiResult = new HashSet<>(identifiersSeen); assertEquals(singleResult.size(), multiResult.size()); diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index de853fa..e5c120b 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -1,9 +1,7 @@ package org.tudo.sse.analyses; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.tudo.sse.ArtifactFactory; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.utils.CommonConfigParser; @@ -20,11 +18,6 @@ public class MavenCentralLibraryAnalysisTest { - @AfterEach - public void cleanup(){ - ArtifactFactory.artifacts.clear(); - } - @Test @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ From aee1513d28c077c4d06506ec2a4f09675844ef0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 3 Dec 2025 10:16:43 +0100 Subject: [PATCH 23/55] Reduce clutter --- .../resolution/ArtifactResolutionContext.java | 10 ---------- .../resolution/LibraryResolutionContext.java | 9 --------- .../sse/model/resolution/ResolutionContext.java | 17 ----------------- 3 files changed, 36 deletions(-) diff --git a/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java index 59ffab0..44dff54 100644 --- a/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java @@ -18,14 +18,4 @@ public ArtifactIdent getIdentifier() { return identifier; } - @Override - public boolean isArtifactResolutionContext() { - return true; - } - - @Override - public ArtifactResolutionContext asArtifactResolutionContext() { - return this; - } - } diff --git a/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java b/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java index cefea1b..5e2e720 100644 --- a/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java @@ -31,13 +31,4 @@ public String getLibraryGA() { return libraryGA; } - @Override - public boolean isLibraryResolutionContext() { - return true; - } - - @Override - public LibraryResolutionContext asLibraryResolutionContext(){ - return this; - } } diff --git a/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java b/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java index 14437c9..de1a16e 100644 --- a/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java @@ -33,21 +33,4 @@ public Set getAllArtifactsResolved(){ return new HashSet<>(artifactsResolved.values()); } - public boolean isLibraryResolutionContext(){ - return false; - } - - public LibraryResolutionContext asLibraryResolutionContext(){ - throw new IllegalStateException("Not a library resolution context"); - } - - public boolean isArtifactResolutionContext(){ - return false; - } - - public ArtifactResolutionContext asArtifactResolutionContext(){ - throw new IllegalStateException("Not an artifact resolution context"); - } - - } From f3c9e76813bcdec2ec03172b33eaba2a9d5efb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 3 Dec 2025 10:23:04 +0100 Subject: [PATCH 24/55] Reduce visibility of artifact constructor, move contexts into model package --- src/main/java/org/tudo/sse/IndexWalker.java | 2 +- .../MavenCentralArtifactAnalysis.java | 4 +- .../analyses/MavenCentralLibraryAnalysis.java | 2 +- .../java/org/tudo/sse/model/Artifact.java | 44 +------------------ .../ArtifactResolutionContext.java | 6 +-- .../LibraryResolutionContext.java | 4 +- .../{resolution => }/ResolutionContext.java | 5 +-- .../sse/model/pom/LocalPomInformation.java | 2 +- .../ProcessIdentifierMessage.java | 2 +- .../sse/multithreading/ResolverActor.java | 2 +- .../org/tudo/sse/resolution/JarResolver.java | 2 +- .../org/tudo/sse/resolution/PomResolver.java | 2 +- .../tudo/sse/resolution/ResolverFactory.java | 3 +- .../org/tudo/sse/model/TypeStructureTest.java | 1 - .../tudo/sse/resolution/JarResolverTest.java | 2 +- .../tudo/sse/resolution/PomResolverTest.java | 3 +- 16 files changed, 17 insertions(+), 69 deletions(-) rename src/main/java/org/tudo/sse/model/{resolution => }/ArtifactResolutionContext.java (72%) rename src/main/java/org/tudo/sse/model/{resolution => }/LibraryResolutionContext.java (90%) rename src/main/java/org/tudo/sse/model/{resolution => }/ResolutionContext.java (88%) diff --git a/src/main/java/org/tudo/sse/IndexWalker.java b/src/main/java/org/tudo/sse/IndexWalker.java index 03d3679..4a9799a 100644 --- a/src/main/java/org/tudo/sse/IndexWalker.java +++ b/src/main/java/org/tudo/sse/IndexWalker.java @@ -8,7 +8,7 @@ import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.utils.IndexIterator; import java.net.URI; diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 2f29f8f..f0a07b8 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -6,8 +6,8 @@ import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.model.resolution.ArtifactResolutionContext; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ArtifactResolutionContext; +import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.multithreading.ProcessIdentifierMessage; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 5d96423..4f707e2 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -5,7 +5,7 @@ import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.resolution.LibraryResolutionContext; +import org.tudo.sse.model.LibraryResolutionContext; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.multithreading.ProcessLibraryMessage; import org.tudo.sse.multithreading.QueueActor; diff --git a/src/main/java/org/tudo/sse/model/Artifact.java b/src/main/java/org/tudo/sse/model/Artifact.java index 1f2c569..aac48cc 100644 --- a/src/main/java/org/tudo/sse/model/Artifact.java +++ b/src/main/java/org/tudo/sse/model/Artifact.java @@ -5,7 +5,6 @@ import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.model.jar.*; import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.JarResolutionException; import org.tudo.sse.resolution.JarResolver; @@ -36,51 +35,10 @@ public class Artifact { * Create a new artifact with no information attached. * @param ident The artifact identifier */ - public Artifact(ArtifactIdent ident){ + Artifact(ArtifactIdent ident){ this.ident = ident; } - /** - * Creates a new artifact based on given IndexInformation. This artifact will have no POM or JAR information - * associated. - * - * @param indexInformation The IndexInformation for which to create the artifact - */ - public Artifact(IndexInformation indexInformation) { - this.indexInformation = indexInformation; - this.ident = indexInformation.getIdent(); - pomInformation = null; - jarInformation = null; - } - - /** - * Creates a new artifact based on given PomInformation. This artifact will have no Index or JAR information - * associated. - * - * @param pomInformation The PomInformation for which to create the artifact - */ - public Artifact(PomInformation pomInformation) { - this.pomInformation = pomInformation; - this.ident = pomInformation.getIdent(); - this.relocation = pomInformation.getRelocation(); - indexInformation = null; - jarInformation = null; - } - - /** - * Creates a new artifact based on the given JarInformation. This artifact will have no Index or POM information - * associated. - * - * @param jarInformation The JarInformation for which to create the artifact - */ - public Artifact(JarInformation jarInformation) { - this.jarInformation = jarInformation; - this.ident = jarInformation.getIdent(); - indexInformation = null; - pomInformation = null; - } - - /** * Returns the artifact identifier for this artifact. * diff --git a/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java similarity index 72% rename from src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java rename to src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java index 44dff54..15fe56c 100644 --- a/src/main/java/org/tudo/sse/model/resolution/ArtifactResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java @@ -1,8 +1,6 @@ -package org.tudo.sse.model.resolution; +package org.tudo.sse.model; -import org.tudo.sse.model.ArtifactIdent; - -public class ArtifactResolutionContext extends ResolutionContext { +public final class ArtifactResolutionContext extends ResolutionContext { private final ArtifactIdent identifier; diff --git a/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java b/src/main/java/org/tudo/sse/model/LibraryResolutionContext.java similarity index 90% rename from src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java rename to src/main/java/org/tudo/sse/model/LibraryResolutionContext.java index 5e2e720..8681a6e 100644 --- a/src/main/java/org/tudo/sse/model/resolution/LibraryResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/LibraryResolutionContext.java @@ -1,6 +1,4 @@ -package org.tudo.sse.model.resolution; - -import org.tudo.sse.model.Artifact; +package org.tudo.sse.model; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java b/src/main/java/org/tudo/sse/model/ResolutionContext.java similarity index 88% rename from src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java rename to src/main/java/org/tudo/sse/model/ResolutionContext.java index de1a16e..44ef27a 100644 --- a/src/main/java/org/tudo/sse/model/resolution/ResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ResolutionContext.java @@ -1,7 +1,4 @@ -package org.tudo.sse.model.resolution; - -import org.tudo.sse.model.Artifact; -import org.tudo.sse.model.ArtifactIdent; +package org.tudo.sse.model; import java.util.HashMap; import java.util.HashSet; diff --git a/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java b/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java index 54e8371..44aeaa2 100644 --- a/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java +++ b/src/main/java/org/tudo/sse/model/pom/LocalPomInformation.java @@ -4,7 +4,7 @@ import org.apache.maven.model.io.xpp3.MavenXpp3Reader; import org.codehaus.plexus.util.xml.pull.XmlPullParserException; import org.tudo.sse.model.*; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.resolution.PomResolutionException; import org.tudo.sse.resolution.PomResolver; diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index cda276e..5e68bdf 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -2,7 +2,7 @@ import org.tudo.sse.analyses.MavenCentralArtifactAnalysis; import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.resolution.ArtifactResolutionContext; +import org.tudo.sse.model.ArtifactResolutionContext; /** * A message passed to the processing queue to indicate that a given analysis instance must process the given diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index e0cfeaf..3d0e659 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -4,7 +4,7 @@ import akka.actor.Props; import akka.japi.pf.ReceiveBuilder; import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.resolution.ArtifactResolutionContext; +import org.tudo.sse.model.ArtifactResolutionContext; /** * This class is spawned in multiple threads diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index db96aa7..4012478 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -14,7 +14,7 @@ import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.JarInformation; import org.tudo.sse.model.jar.ObjType; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.utils.MavenCentralRepository; import scala.Tuple2; diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java index 5b45293..ceda332 100644 --- a/src/main/java/org/tudo/sse/resolution/PomResolver.java +++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java @@ -15,7 +15,7 @@ import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; import org.tudo.sse.model.pom.RawPomFeatures; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; import org.tudo.sse.utils.MavenCentralRepository; diff --git a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java index f2f3d0c..9b2b381 100644 --- a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java +++ b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java @@ -1,13 +1,12 @@ package org.tudo.sse.resolution; import java.io.IOException; -import java.net.URI; import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ResolutionContext; /** * This class manages the pom and jar resolver, giving a way to run one or the other. diff --git a/src/test/java/org/tudo/sse/model/TypeStructureTest.java b/src/test/java/org/tudo/sse/model/TypeStructureTest.java index 82d87b4..83803c5 100644 --- a/src/test/java/org/tudo/sse/model/TypeStructureTest.java +++ b/src/test/java/org/tudo/sse/model/TypeStructureTest.java @@ -7,7 +7,6 @@ import org.junit.jupiter.api.Test; import org.tudo.sse.model.jar.ClassFileNode; import org.tudo.sse.model.jar.DefinedClassFileNode; -import org.tudo.sse.model.resolution.ResolutionContext; import org.tudo.sse.resolution.*; import java.io.IOException; diff --git a/src/test/java/org/tudo/sse/resolution/JarResolverTest.java b/src/test/java/org/tudo/sse/resolution/JarResolverTest.java index 2d7e607..10554f0 100644 --- a/src/test/java/org/tudo/sse/resolution/JarResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/JarResolverTest.java @@ -7,7 +7,7 @@ import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.ClassFile; import org.tudo.sse.model.jar.JarInformation; -import org.tudo.sse.model.resolution.ResolutionContext; +import org.tudo.sse.model.ResolutionContext; import java.io.InputStream; import java.io.InputStreamReader; diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index 06992c2..a2d48f1 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -23,8 +23,7 @@ import org.tudo.sse.model.pom.Dependency; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.RawPomFeatures; -import org.tudo.sse.model.resolution.ResolutionContext; -import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; +import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.resolution.releases.IReleaseListProvider; From 39b8e17b18ca3536f8d39c06d308ed3e0921e2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 3 Dec 2025 10:49:49 +0100 Subject: [PATCH 25/55] Add JavaDoc to all changes for this branch --- .../MavenCentralArtifactAnalysis.java | 1 + .../sse/model/ArtifactResolutionContext.java | 14 +++++++ .../sse/model/LibraryResolutionContext.java | 22 +++++++++++ .../org/tudo/sse/model/ResolutionContext.java | 39 ++++++++++++++++++- .../org/tudo/sse/resolution/JarResolver.java | 2 + .../org/tudo/sse/resolution/PomResolver.java | 7 ++++ .../tudo/sse/resolution/ResolverFactory.java | 3 ++ 7 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index f0a07b8..24333ab 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -425,6 +425,7 @@ List processArtifactsFromInputFile() { /** * Invokes all resolvers as defined by the analysis configuration to enrich the given artifact identifier. * @param identifier Artifact identifier to enrich + * @param ctx The current resolution context */ public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) { if(resolvePom && resolveJar) { diff --git a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java index 15fe56c..be261f1 100644 --- a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java @@ -1,5 +1,10 @@ package org.tudo.sse.model; +/** + * The specific type of resolution context for resolving individual artifact, i.e. *not libraries*. + * + * @author Johannes Düsing + */ public final class ArtifactResolutionContext extends ResolutionContext { private final ArtifactIdent identifier; @@ -8,10 +13,19 @@ private ArtifactResolutionContext(ArtifactIdent identifier) { this.identifier = identifier; } + /** + * Factory method to create a new ArtifactResolutionContext for the given identifier. + * @param identifier Artifact identifier to create a context for + * @return New ArtifactResolutionContext instance for the given artifact + */ public static ArtifactResolutionContext newInstance(ArtifactIdent identifier) { return new ArtifactResolutionContext(identifier); } + /** + * Returns the identifier of the artifact for which this context was created. + * @return ArtifactIdent for this context + */ public ArtifactIdent getIdentifier() { return identifier; } diff --git a/src/main/java/org/tudo/sse/model/LibraryResolutionContext.java b/src/main/java/org/tudo/sse/model/LibraryResolutionContext.java index 8681a6e..f548e93 100644 --- a/src/main/java/org/tudo/sse/model/LibraryResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/LibraryResolutionContext.java @@ -3,6 +3,11 @@ import java.util.ArrayList; import java.util.List; +/** + * The specific type of resolution context for resolving libraries. Tracks what artifacts belong to the library. + * + * @author Johannes Düsing + */ public final class LibraryResolutionContext extends ResolutionContext { private final String libraryGA; @@ -13,18 +18,35 @@ private LibraryResolutionContext(String libraryGA) { this.libraryArtifacts = new ArrayList<>(); } + /** + * Factory method to create a new LibraryResolutionContext for the given GA. + * @param libraryGA Maven library name (GroupID:ArtifactID) + * @return New LibraryResolutionContext instance for the given library + */ public static LibraryResolutionContext newInstance(String libraryGA) { return new LibraryResolutionContext(libraryGA); } + /** + * Appends a new artifact to the list of library artifacts, i.e., the list of library releases. + * @param a The new library artifact + */ public void addLibraryArtifact(Artifact a){ this.libraryArtifacts.add(a); } + /** + * Returns the list of library artifacts, i.e., the list of library releases. + * @return List of artifacts belonging to the library + */ public List getLibraryArtifacts(){ return this.libraryArtifacts; } + /** + * Returns the library's name for which this context was created. + * @return Library name (GroupID:ArtifactID) + */ public String getLibraryGA() { return libraryGA; } diff --git a/src/main/java/org/tudo/sse/model/ResolutionContext.java b/src/main/java/org/tudo/sse/model/ResolutionContext.java index 44ef27a..7b66f4f 100644 --- a/src/main/java/org/tudo/sse/model/ResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ResolutionContext.java @@ -5,14 +5,42 @@ import java.util.Map; import java.util.Set; +/** + * Class representing the context of resolving software artifacts. It is used to cache artifacts, so they are unique + * per resolution context and previously resolved information (index, pom, jar) can be reused. + * + * @author Johannes Düsing + */ public abstract class ResolutionContext { + /** + * Create an anonymous context that is neither an ArtifactResolutionContext nor a LibraryResolutionContext. + * + * @return new anonymous context instance + */ public static ResolutionContext createAnonymousContext(){ return new ResolutionContext() {}; } - protected Map artifactsResolved = new HashMap<>(); + /** + * Map of artifacts resolved in this context. + */ + protected final Map artifactsResolved; + /** + * Builds a new ResolutionContext instance. + */ + protected ResolutionContext(){ + this.artifactsResolved = new HashMap<>(); + } + + /** + * Creates a new artifact for the given identifier within the current resolution contexts - or returns the existing + * artifact if present. + * + * @param ident Identifier for which to retrieve an Artifact + * @return Artifact object that will be unique to the given identifier within this context + */ public Artifact createArtifact(ArtifactIdent ident) { if(!artifactsResolved.containsKey(ident)){ final Artifact artifact = new Artifact(ident); @@ -22,10 +50,19 @@ public Artifact createArtifact(ArtifactIdent ident) { return artifactsResolved.get(ident); } + /** + * Retrieves an Artifact instance for the given identifier only if it already exists within this context. + * @param ident The identifier for which to retrieve an Artifact + * @return The Artifact for the given identifier, or null if no such artifact exists in this context. + */ public Artifact getArtifact(ArtifactIdent ident) { return artifactsResolved.getOrDefault(ident, null); } + /** + * Returns a set of all artifacts resolved within this context. + * @return Set of resolved artifacts + */ public Set getAllArtifactsResolved(){ return new HashSet<>(artifactsResolved.values()); } diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index 4012478..2f22827 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -74,6 +74,7 @@ public void setOutput(boolean output) { * This method resolves jar artifacts from a given list of artifact identifiers. * * @param identifiers artifact identifiers used to retrieve jar artifacts to process + * @param ctx The resolution context to use for creating artifacts * @return a list of resolved artifacts * @see Artifact */ @@ -101,6 +102,7 @@ public List resolveJars(List identifiers, ResolutionCon * This method resolve a single jar artifact given an artifact identifier * * @param identifier an artifact identifier to retrieve the jar artifact to process + * @param ctx The resolution context to use for creating artifacts * @return a resolved artifact * @throws JarResolutionException when there is an issue resolving the given jar artifact */ diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java index ceda332..d3780c8 100644 --- a/src/main/java/org/tudo/sse/resolution/PomResolver.java +++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java @@ -150,6 +150,7 @@ public PomResolver(boolean output, Path pathToDirectory, boolean resolveTransiti * This method resolves all artifacts given a list of identifiers. * * @param idents list of identifiers to resolve + * @param ctx The resolution context to use for caching artifacts * @return list of resolved artifacts * @see Artifact */ @@ -182,6 +183,7 @@ public List resolveArtifacts(List idents, ResolutionCon * This method given an Artifact Identifier, resolves a single artifact (raw features, parent, imports, dependencies, and transitive dependencies). * * @param identifier id for the pom artifact to resolve + * @param ctx The resolution context to use for caching artifacts * @return an artifact with resolved PomInformation * @see PomInformation * @throws FileNotFoundException When the POM file does not exist for the given artifact @@ -341,6 +343,7 @@ public List getDependencies(List /** * This method handles resolving an artifact via raw Pom information and parent and dependency resolution * @param identifier used to retrieve the pom artifact from the maven central repository + * @param ctx The resolution context to use for caching artifacts * @return an artifact with resolved rawPomFeatures, parent, and import information * @throws PomResolutionException If the POM resolution process fails due to invalid file contents * @throws FileNotFoundException When the POM file does not exist for the given artifact @@ -405,6 +408,7 @@ private void getAllRelevantFiles(PomInformation pomInformation, ResolutionContex * Resolves import scope dependencies and returns their corresponding artifacts. * @param managedDependencies list of managed dependencies from the rawPomFeatures, to be looked through for an import * @param info the current information object + * @param ctx The resolution context to use for caching artifacts * @return a list of imported artifacts * @throws PomResolutionException handles errors that arise with pom resolution * @throws FileNotFoundException handles errors with not finding a file @@ -806,6 +810,7 @@ private static boolean isVersionRange(String version) { * This method resolves all the transitive dependencies of a given artifact. This is done recursively working with resolveTransitives and the recursiveHandler. * * @param toResolve the current artifact transitive dependencies are being resolved for. + * @param ctx The resolution context to use for caching artifacts * @throws IOException thrown when there's an issue opening the pom for an artifact */ public void resolveAllTransitives(Artifact toResolve, ResolutionContext ctx) throws IOException { @@ -818,6 +823,7 @@ public void resolveAllTransitives(Artifact toResolve, ResolutionContext ctx) thr * @param current the current artifact transitive dependencies are being resolved for. * @param alrEncountered a map of dependencies that have already been resolved to avoid double resolutions * @param exclusions a Set of G:A values to not include in resolution + * @param ctx The resolution context to use for caching artifacts * @throws IOException thrown when there's an issue opening the pom for an artifact */ public void resolveAllTransitives(Artifact current, Map alrEncountered, Set exclusions, ResolutionContext ctx) throws IOException { @@ -895,6 +901,7 @@ private Artifact resolveFromSecondaryRepo(List repos, /** * This method attempts to get all effective transitive dependencies, resolving conflicts, and removing duplicates. * @param toResolve the artifact for which to resolve the effective transitive dependencies + * @param ctx The resolution context to use for caching artifacts */ public void resolveEffectiveTransitives(Artifact toResolve, ResolutionContext ctx) { // Can only resolve dependencies if POM info is present. diff --git a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java index 9b2b381..71426a6 100644 --- a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java +++ b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java @@ -44,6 +44,7 @@ public ResolverFactory(boolean output, Path pathToDirectory, boolean pomIncludeT * Resolve the POM file of the given artifact. * * @param identifier Artifact identifier to resolve + * @param ctx The resolution context for this run */ public void runPom(ArtifactIdent identifier, ResolutionContext ctx) { try { @@ -57,6 +58,7 @@ public void runPom(ArtifactIdent identifier, ResolutionContext ctx) { * Resolve the JAR file of the given artifact. * * @param identifier Artifact identifier to resolve + * @param ctx The resolution context for this run */ public void runJar(ArtifactIdent identifier, ResolutionContext ctx) { try { @@ -70,6 +72,7 @@ public void runJar(ArtifactIdent identifier, ResolutionContext ctx) { * Resolve both the POM and JAR file for the given artifact. * * @param identifier Artifact identifier to resolve + * @param ctx The resolution context for this run */ public void runBoth(ArtifactIdent identifier, ResolutionContext ctx) { try { From c92bb0d261dcfd4dc9e574195eda30e7208d5de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 3 Dec 2025 12:59:10 +0100 Subject: [PATCH 26/55] Remove some more reundant global state that was only used in tests --- .../MavenCentralArtifactAnalysis.java | 30 ++++---------- .../MavenCentralArtifactAnalysisTest.java | 40 +++++++++++-------- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 24333ab..7e59237 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -20,9 +20,7 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; /** * The MavenCentralAnalysis enables analysis of artifacts on the maven central repository for jobs of any size. @@ -246,12 +244,10 @@ private void walkAllIndexes(IndexIterator indexIterator) throws IOException { * * @param take number of artifacts from the starting point to capture * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifact identifiers processed by this method * @see ArtifactIdent * @throws IOException when there is an issue opening a file */ - List walkPaginated(long take, IndexIterator indexIterator) throws IOException { - final List artifactIdents = new ArrayList<>(); + void walkPaginated(long take, IndexIterator indexIterator) throws IOException { long entriesTaken = 0L; while(indexIterator.hasNext() && entriesTaken < take) { @@ -277,8 +273,6 @@ List walkPaginated(long take, IndexIterator indexIterator) throws processIndexIdentifier(current.getIdent(), ctx); } - artifactIdents.add(current.getIdent()); - writePositionIfNeeded(); } @@ -288,22 +282,18 @@ List walkPaginated(long take, IndexIterator indexIterator) throws log.info("Processed a total of {} entries.", entriesTaken); indexIterator.closeReader(); - return artifactIdents; } /** - * Iterates over all indexes in the maven central index. - * It collects a list of artifact identifiers that are within the range of since and until. + * Invokes analysis configuration for all artifact identifiers in the date window defined by given boundaries. * * @param since lower bound of dates of artifacts to collect * @param until upper bound of dates of artifacts to collect * @param indexIterator an iterator for traversing the maven central index - * @return a list of artifact identifiers processed by this method * @see ArtifactIdent * @throws IOException when there is an issue opening a file */ - List walkDates(long since, long until, IndexIterator indexIterator) throws IOException { - final List artifactIdents = new ArrayList<>(); + void walkDates(long since, long until, IndexIterator indexIterator) throws IOException { long entriesTaken = 0L; long currentToSince; @@ -334,8 +324,6 @@ List walkDates(long since, long until, IndexIterator indexIterato } else { processIndexIdentifier(current.getIdent(), ctx); } - - artifactIdents.add(current.getIdent()); } writePositionIfNeeded(); @@ -347,7 +335,6 @@ List walkDates(long since, long until, IndexIterator indexIterato log.info("Processed a total of {} entries.", entriesTaken); indexIterator.closeReader(); - return artifactIdents; } private void processIndexIdentifier(ArtifactIdent ident, ArtifactResolutionContext ctx) { @@ -364,10 +351,8 @@ private void processIndexIdentifier(ArtifactIdent ident, ArtifactResolutionConte /** * Reads in identifiers from a file, using the configuration passed into it. - * - * @return a list of identifiers collected from the file */ - List processArtifactsFromInputFile() { + void processArtifactsFromInputFile() { final Iterator fileIterator = new FileBasedArtifactIterator(this.artifactConfig.inputListFile); // Restore from progress file if available @@ -386,7 +371,6 @@ List processArtifactsFromInputFile() { if(!fileIterator.hasNext()) log.warn("No more contents left to process in input file: {}", this.artifactConfig.inputListFile); - final List identifiers = new ArrayList<>(); final boolean takeLimited = this.artifactConfig.take >= 0; long entriesTaken = 0L; @@ -405,7 +389,6 @@ List processArtifactsFromInputFile() { if(current != null){ final ArtifactResolutionContext resolutionCtx = ArtifactResolutionContext.newInstance(current); - identifiers.add(current); processIndexIdentifier(current, resolutionCtx); } } @@ -418,8 +401,6 @@ List processArtifactsFromInputFile() { // Write position one final time - in multithreaded mode the queue worker will take care of this writePosition(); } - - return identifiers; } /** @@ -434,6 +415,9 @@ public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) { resolverFactory.runPom(identifier, ctx); } else if(resolveJar) { resolverFactory.runJar(identifier, ctx); + } else { + // If no sources are configured, we still want to have an "empty" artifact so it can be returned later + ctx.createArtifact(identifier); } } diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 3ee1b22..70baa56 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -149,11 +149,11 @@ void parseCmdLineNegative() { @DisplayName("An analysis must adhere to pagination configurations") void walkPaginated() { - final Map artifactsSeen = new HashMap<>(); + final List artifactsSeen = new ArrayList<>(); final MavenCentralArtifactAnalysis theAnalysis = new MavenCentralArtifactAnalysis(true, false, false, false) { @Override public void analyzeArtifact(Artifact current) { - artifactsSeen.put(current.getIdent(), current); + artifactsSeen.add(current); } }; @@ -173,23 +173,27 @@ public void analyzeArtifact(Artifact current) { try { IndexIterator iterator = new IndexIterator(new URI(base), start1); - List collected1 = new ArrayList<>(theAnalysis.walkPaginated(take, iterator)); + theAnalysis.walkPaginated(take, iterator); - Artifact lastOne = artifactsSeen.get(collected1.get(collected1.size() - 1)); + assertFalse(artifactsSeen.isEmpty()); + + Artifact lastOne = artifactsSeen.get(artifactsSeen.size() - 1); assertNotNull(lastOne); int i = 2; while(lastOne.getIndexInformation().getIndex() > start2) { - lastOne = artifactsSeen.get(collected1.get(collected1.size() - i)); + lastOne = artifactsSeen.get(artifactsSeen.size() - i); assertNotNull(lastOne); i++; } iterator = new IndexIterator(new URI(base), start2); - List collected2 = new ArrayList<>(theAnalysis.walkPaginated(1, iterator)); - long lastOne2 = artifactsSeen.get(collected2.get(0)).getIndexInformation().getIndex(); + artifactsSeen.clear(); + theAnalysis.walkPaginated(1, iterator); + + long lastOne2 = artifactsSeen.get(0).getIndexInformation().getIndex(); assertEquals(lastOne.getIndexInformation().getIndex(), lastOne2); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); @@ -230,7 +234,7 @@ void indexProcessorProgress() { @Test @DisplayName("An analysis must correctly apply CLI options when reading custom input lists") - void readIdentsIn() throws URISyntaxException, IOException { + void readIdentsIn() { List cliInputs = new ArrayList<>(); String[] args = {"--inputs", "src/main/resources/coordinates.txt"}; cliInputs.add(args); @@ -251,22 +255,26 @@ void readIdentsIn() throws URISyntaxException, IOException { int i = 0; for(String[] arg : cliInputs) { List curExpected = expected.get(i); - MavenCentralArtifactAnalysis tester = MavenCentralAnalysisFactory.buildEmptyAnalysisWithNoRequirements(); + List artifactsSeen = new ArrayList<>(); + MavenCentralArtifactAnalysis collectorAnalysis = new MavenCentralArtifactAnalysis(false, false, false, false) { + @Override + public void analyzeArtifact(Artifact current) { + artifactsSeen.add(current); + } + }; // Apply arguments, check progress is stored correctly - tester.runAnalysis(arg); - Path progressFile = tester.getSetupInfo().progressOutputFile; + collectorAnalysis.runAnalysis(arg); + Path progressFile = collectorAnalysis.getSetupInfo().progressOutputFile; long ending = getEndingIndex(progressFile); assertEquals(expectedEndings[i], ending); - // Process file a second time to obtain return value - List idents = tester.processArtifactsFromInputFile(); - assertEquals(curExpected.size(), idents.size()); + assertEquals(curExpected.size(), artifactsSeen.size()); - for(ArtifactIdent actual: idents) { - final String actualCoordinates = actual.getCoordinates(); + for(Artifact actual: artifactsSeen) { + final String actualCoordinates = actual.getIdent().getCoordinates(); assert(curExpected.contains(actualCoordinates)); } i++; From 116eb6b65d2711523fbe109b8a46cbe6e389cef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Thu, 4 Dec 2025 15:25:28 +0100 Subject: [PATCH 27/55] Use a fresh class file reader instance for every JAR, which should reduce memory pileup due to caching --- .../org/tudo/sse/resolution/JarResolver.java | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index 2f22827..a42d4b6 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -6,10 +6,8 @@ import org.opalj.br.ClassFile; import org.opalj.br.Method; import org.opalj.br.ClassType; -import org.opalj.br.analyses.Project$; -import org.opalj.br.package$; -import org.opalj.br.reader.Java16LibraryFramework; -import org.opalj.log.GlobalLogContext$; +import org.opalj.br.reader.BytecodeInstructionsCache; +import org.opalj.br.reader.Java17FrameworkWithCaching; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.JarInformation; @@ -22,6 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; +import java.lang.ref.SoftReference; import java.net.MalformedURLException; import java.net.URL; import java.io.InputStream; @@ -40,7 +39,6 @@ public class JarResolver { private final Path pathToDirectory; private boolean output; - private final Java16LibraryFramework cfReader = Project$.MODULE$.JavaClassFileReader(GlobalLogContext$.MODULE$, package$.MODULE$.BaseConfig()); private static final MavenCentralRepository MavenRepo = MavenCentralRepository.getInstance(); private static final Logger log = LogManager.getLogger(JarResolver.class); @@ -231,7 +229,8 @@ public org.tudo.sse.model.jar.ClassFile processClassFile(ClassFile classFile) { } private List> readClassesFromJarStream(InputStream jarStream, URL source) throws JarResolutionException { - var entries = new ArrayList>(); + final var entries = new ArrayList>(); + final var cfReader = newClassFileReader(); try (JarInputStream jarInputStream = new JarInputStream(jarStream)){ var currentEntry = jarInputStream.getNextJarEntry(); @@ -259,21 +258,19 @@ private List> readClassesFromJarStream(InputStream jarStr return entries; } - private DataInputStream getEntryByteStream(InputStream in) throws IOException { - - var baos = new ByteArrayOutputStream(); - var buffer = new byte[32 * 1024]; - - int bytesRead = in.read(buffer); - - while(bytesRead > 0){ - baos.write(buffer, 0, bytesRead); - baos.close(); - baos.flush(); - bytesRead = in.read(buffer); - } + /** + * Create a new class file reader with a weak reference to a new cache, so that cached instruction objects may be + * released at some point. + * + * @return New class file reader instance without rewriting + */ + private Java17FrameworkWithCaching newClassFileReader(){ + SoftReference cacheRef = new SoftReference<>(new BytecodeInstructionsCache()); + return new Java17FrameworkWithCaching(cacheRef.get()); + } - return new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); + private DataInputStream getEntryByteStream(InputStream in) { + return new DataInputStream(in); } } From b3b36bad873ff2f1350e7fe67b46e81ce228bd42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 5 Dec 2025 10:37:21 +0100 Subject: [PATCH 28/55] Remove errors in JavaDoc, make JavaDoc generation fail on warning, add separate doc generation step to CI --- .github/workflows/maven.yml | 2 ++ pom.xml | 30 +++++++++------- .../analyses/MavenCentralLibraryAnalysis.java | 13 ++++--- .../tudo/sse/model/jar/JarInformation.java | 36 ++++++++++++------- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d427c4d..9c20571 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -32,5 +32,7 @@ jobs: run: mvn -B compile --file pom.xml - name: Run unit tests run: mvn -B test --file pom.xml + - name: Generate JavaDoc + run: mvn -B javadoc:javadoc --file pom.xml - name: Update dependency graph uses: advanced-security/maven-dependency-submission-action@v5 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 99e7d10..777e0e7 100644 --- a/pom.xml +++ b/pom.xml @@ -142,9 +142,26 @@ -Xlint:all,-serial + -Werror + + org.apache.maven.plugins + maven-javadoc-plugin + 3.12.0 + + + attach-sources + + jar + + + + + true + + @@ -167,19 +184,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - 3.11.2 - - - attach-sources - - jar - - - - org.apache.maven.plugins maven-gpg-plugin diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 4f707e2..04fa95d 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -158,8 +158,10 @@ public final void runAnalysis(String[] args){ /** * Analysis lifecycle hook that is executed before a library is being processed, i.e., before any releases are * discovered. + *

+ * Note:This method may be called concurrently by multiple threads if the analysis uses parallel execution + *

* - * @implNote This method may be called concurrently by multiple threads if the analysis uses parallel execution * * @param groupId The library's group ID * @param artifactId The library's artifact ID @@ -171,8 +173,9 @@ protected void beforeLibraryStart(String groupId, String artifactId) { /** * Analysis lifecycle hook that is executed after a library has been processed, i.e., after all releases have been * discovered and the analysis implementation has been executed. - * - * @implNote This method may be called concurrently by multiple threads if the analysis uses parallel execution + *

+ * Note:This method may be called concurrently by multiple threads if the analysis uses parallel execution + *

* * @param groupId The library's group ID * @param artifactId The library's artifact ID @@ -185,7 +188,9 @@ protected void afterLibraryEnd(String groupId, String artifactId, List /** * Analysis lifecycle hook that is executed when the analysis failed to obtain a version list for a given library. * - * @implNote This method may be called concurrently by multiple threads if the analysis uses parallel execution + *

+ * Note:This method may be called concurrently by multiple threads if the analysis uses parallel execution + *

* * @param groupId The library's group ID * @param artifactId The library's artifact ID diff --git a/src/main/java/org/tudo/sse/model/jar/JarInformation.java b/src/main/java/org/tudo/sse/model/jar/JarInformation.java index a0500a1..e7fde16 100644 --- a/src/main/java/org/tudo/sse/model/jar/JarInformation.java +++ b/src/main/java/org/tudo/sse/model/jar/JarInformation.java @@ -186,43 +186,55 @@ public JarInputStream getJarInputStream() throws IOException, FileNotFoundExcept } /** + *

* Retrieves a list of all class files contained in JAR file referenced by this ArtifactInformation. The class * files are returned as represented by the OPAL framework. + *

+ *

+ * Note: OPAL classes are not designed to be used in large scale analyses. Over time, OPAL's internal caches + * will continue to claim heap space that is never freed, so eventually the program will crash with an + * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you + * really need them. + *

* @return List of OPAL class file representations, or null if no JAR file was found * @throws IOException If reading the JAR file fails - * @implNote OPAL classes are not designed to be used in large scale analyses. Over time, OPAL's internal caches - * will continue to claim heap space that is never freed, so eventually the program will crash with an - * OutOfMemory error. There is currently no good way to avoid this, so use OPAL instances only if you - * really need them. */ public List getOpalClassFileRepresentations() throws IOException { return getOpalClassFileRepresentations(this.ident); } /** + *

* Retrieves all class files for the JAR referenced by this ArtifactInformation, and uses them to initialize an * OPAL project instance. This instance can be used to conduct complex static program analyses. + *

+ *

+ * Note: OPAL projects are not designed to be used in large scale analyses. Over time, OPAL's internal caches + * will continue to claim heap space that is never freed, so eventually the program will crash with an + * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you + * really need them. + *

* @return The OPAL project instance, or null if no JAR file was found * @throws IOException If reading the JAR file fails - * @implNote OPAL projects are not designed to be used in large scale analyses. Over time, OPAL's internal caches - * will continue to claim heap space that is never freed, so eventually the program will crash with an - * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you - * really need them. */ public Project getOpalProject() throws IOException { return getOpalProject(MarinOpalLogger.newInfoLogger(LogManager.getLogger("[OPAL-Project]"))); } /** + *

* Retrieves all class files for the JAR referenced by this ArtifactInformation, and uses them to initialize an * OPAL project instance. This instance can be used to conduct complex static program analyses. + *

+ *

+ * Note: OPAL projects are not designed to be used in large scale analyses. Over time, OPAL's internal caches + * will continue to claim heap space that is never freed, so eventually the program will crash with an + * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you + * really need them. + *

* @return The OPAL project instance, or null if no JAR file was found * @param projectLogger A custom logger instance to configure the amount of output that is produced by OPAL. * @throws IOException If reading the JAR file fails - * @implNote OPAL projects are not designed to be used in large scale analyses. Over time, OPAL's internal caches - * will continue to claim heap space that is never freed, so eventually the program will crash with an - * OutOfMemory error. There is currently no good way to avoid this, so use project instances only if you - * really need them. */ public Project getOpalProject(MarinOpalLogger projectLogger) throws IOException { try(JarInputStream jarStream = getJarInputStream()){ From 148e72ae24630518670fa9b1048496a6b4ec91ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 5 Dec 2025 11:38:20 +0100 Subject: [PATCH 29/55] Fix all Javadoc issues --- pom.xml | 2 +- .../analyses/MavenCentralLibraryAnalysis.java | 7 +++ .../tudo/sse/model/pom/PomInformation.java | 13 ++++ .../multithreading/ProcessLibraryMessage.java | 11 ++++ .../org/tudo/sse/multithreading/WorkItem.java | 3 + .../WorkItemFinishedMessage.java | 8 +++ .../WorkloadIsFinalMessage.java | 5 +- .../DefaultMavenReleaseListProvider.java | 2 +- .../releases/IReleaseListProvider.java | 6 +- .../tudo/sse/utils/ArtifactConfigParser.java | 36 +++++++++++ .../tudo/sse/utils/CLIParsingUtilities.java | 43 +++++++++++++ .../tudo/sse/utils/CommonConfigParser.java | 60 ++++++++++++++++++- .../sse/utils/FileBasedArtifactIterator.java | 8 +++ .../tudo/sse/utils/LibraryIndexIterator.java | 17 ++++++ .../sse/utils/MavenCentralRepository.java | 8 +++ 15 files changed, 222 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 777e0e7..115a509 100644 --- a/pom.xml +++ b/pom.xml @@ -141,7 +141,7 @@ 3.14.1 - -Xlint:all,-serial + -Xlint:all,-serial,-processing -Werror diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 04fa95d..fd3656a 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -24,11 +24,18 @@ import java.util.Iterator; import java.util.List; +/** + * Base class for analyses that process libraries hosted on Maven Central. For each library, all artifacts (releases) + * are aggregated and enriched with pom / JAR information as specified by the implementing class. + */ public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis { private final IReleaseListProvider releaseListProvider; private final ResolverFactory resolverFactory; + /** + * The configuration for this analysis instance. Only available after runAnalysis has been called. + */ protected CommonConfigParser.CommonConfig config; private ActorRef queueActorRef; diff --git a/src/main/java/org/tudo/sse/model/pom/PomInformation.java b/src/main/java/org/tudo/sse/model/pom/PomInformation.java index 2d93de3..8aa5fb4 100644 --- a/src/main/java/org/tudo/sse/model/pom/PomInformation.java +++ b/src/main/java/org/tudo/sse/model/pom/PomInformation.java @@ -9,9 +9,22 @@ * This class is where all the information collected during pom resolution is stored. From the raw features to resolved transitive dependencies. */ public class PomInformation extends ArtifactInformation { + + /** + * The raw, unresolved features specified in this pom file. + */ protected RawPomFeatures rawPomFeatures; + + /** + * List of artifacts that are imported via this artifact's pom file. + */ protected List imports; + + /** + * The parent specified by this artifact's pom file, or null if no parent is specified. + */ protected Artifact parent; + private List resolvedDependencies; private List allTransitiveDependencies; private List effectiveTransitiveDependencies; diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java index 7868766..a742453 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessLibraryMessage.java @@ -2,14 +2,25 @@ import java.util.function.Supplier; +/** + * Message used in multithreaded mode to queue processing of a callback method (for analyzing libraries). + */ public class ProcessLibraryMessage implements WorkItem { private final Supplier processEntryCallback; + /** + * Creates a new message with the given callback method + * @param callback The callback that analyzes a library + */ public ProcessLibraryMessage(Supplier callback) { this.processEntryCallback = callback; } + /** + * Returns the callback method stored in this message. + * @return The callback + */ public Supplier getProcessEntryCallback() { return processEntryCallback; } } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkItem.java b/src/main/java/org/tudo/sse/multithreading/WorkItem.java index ee16765..f8d0baa 100644 --- a/src/main/java/org/tudo/sse/multithreading/WorkItem.java +++ b/src/main/java/org/tudo/sse/multithreading/WorkItem.java @@ -1,4 +1,7 @@ package org.tudo.sse.multithreading; +/** + * Marker interface for items of work relevant to the queue manager in multithreaded mode. + */ public interface WorkItem { } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java b/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java index 7707157..2b3eaf1 100644 --- a/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java @@ -1,11 +1,19 @@ package org.tudo.sse.multithreading; +/** + * A singleton message that is used in multithreaded mode to let the queue manager keep track of the number of completed + * work items (libraries or artifacts processed). + */ public class WorkItemFinishedMessage { private static final WorkItemFinishedMessage _instance = new WorkItemFinishedMessage(); private WorkItemFinishedMessage() {} + /** + * Retrieves the singleton instance of this message + * @return The singleton instance + */ public static WorkItemFinishedMessage getInstance() { return _instance; } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java b/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java index 9e04131..09d2f0e 100644 --- a/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java @@ -10,7 +10,10 @@ public class WorkloadIsFinalMessage { private WorkloadIsFinalMessage() {} - + /** + * Retrieves the singleton instance of this message. + * @return The singleton instance. + */ public static WorkloadIsFinalMessage getInstance(){ if(_instance == null) _instance = new WorkloadIsFinalMessage(); return _instance; diff --git a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java index 99b0690..0a83c83 100644 --- a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java +++ b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java @@ -24,7 +24,7 @@ * * @author Johannes Düsing */ -public class DefaultMavenReleaseListProvider extends IReleaseListProvider{ +public class DefaultMavenReleaseListProvider implements IReleaseListProvider{ private final MetadataXpp3Reader reader = new MetadataXpp3Reader(); private final MavenCentralRepository mavenRepo = MavenCentralRepository.getInstance(); diff --git a/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java index 819b5d6..812deb2 100644 --- a/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java +++ b/src/main/java/org/tudo/sse/resolution/releases/IReleaseListProvider.java @@ -10,7 +10,7 @@ * Interface defining functionality to obtain a list of Maven Central releases (i.e. version numbers) for a given * library (i.e. GA-Tuple). */ -public abstract class IReleaseListProvider { +public interface IReleaseListProvider { /** * Gets the ordered list of releases (i.e. version numbers) for the given identifier. The identifier's version is @@ -20,7 +20,7 @@ public abstract class IReleaseListProvider { * @return List of version numbers as ordered by the underlying source * @throws IOException If a connection error occurs */ - public List getReleases(ArtifactIdent identifier) throws IOException{ + default List getReleases(ArtifactIdent identifier) throws IOException{ Objects.requireNonNull(identifier); return getReleases(identifier.getGroupID(), identifier.getArtifactID()); @@ -34,5 +34,5 @@ public List getReleases(ArtifactIdent identifier) throws IOException{ * @return List of version numbers as ordered by the underlying source * @throws IOException If a connection error occurs */ - public abstract List getReleases(String groupId, String artifactId) throws IOException; + List getReleases(String groupId, String artifactId) throws IOException; } diff --git a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java index 34a1520..7fde305 100644 --- a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java +++ b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java @@ -4,8 +4,24 @@ import java.nio.file.Path; +/** + * Parser for ArtifactConfig objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis}. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ public class ArtifactConfigParser extends CommonConfigParser implements CLIParsingUtilities { + /** + * Creates a new artifact config parser instance. + */ + public ArtifactConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link ArtifactConfig} object. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link ArtifactConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ public ArtifactConfig parseArtifactConfig(String[] args) throws CLIException { final ArtifactConfig artifactConfig = new ArtifactConfig(); @@ -59,14 +75,34 @@ private void handleArtifactParameter(String[] args, int i, ArtifactConfig config } } + /** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. + */ public static class ArtifactConfig extends CommonConfig { + /** + * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. + */ public long since; + + /** + * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. + */ public long until; + /** + * Directory to write file outputs to, or null if disabled. + */ public Path outputDirectory; + + /** + * Whether files shall be written to the output directory. + */ public boolean outputEnabled; + /** + * Creates a new configuration object with default values. + */ public ArtifactConfig(){ super(); diff --git a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java index 80fffba..8d67fbc 100644 --- a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java +++ b/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java @@ -6,8 +6,19 @@ import java.nio.file.Path; import java.nio.file.Paths; +/** + * Interface providing default implementations for parsing command line arguments. + */ interface CLIParsingUtilities { + /** + * Parses the next argument from the given array of arguments as an integer value and returns it. + * + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as an integer value + * @throws CLIException If there is no next argument, or it is not a valid integer + */ default int nextArgAsInt(String[] args, int i) throws CLIException { if(i + 1 < args.length) { try{ @@ -20,6 +31,15 @@ default int nextArgAsInt(String[] args, int i) throws CLIException { } } + /** + * Parses the next argument from the given array of arguments as a pair of long values, separated by a colon. The + * pair is returned as a long-array of size 2. + * + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as a pair of long values + * @throws CLIException If there is no next argument, or it is not formatted as two long values separated by a colon. + */ default long[] nextArgAsLongPair(String[] args, int i) throws CLIException { long[] toReturn = new long[2]; if(i + 1 < args.length) { @@ -40,6 +60,14 @@ default long[] nextArgAsLongPair(String[] args, int i) throws CLIException { return toReturn; } + /** + * Parses the next argument from the given array of arguments as a Path value. + * + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as a Path + * @throws CLIException If there is no next argument + */ default Path nextArgAsPath(String[] args, int i) throws CLIException { if(i + 1 < args.length) { return Paths.get(args[i + 1]); @@ -48,6 +76,14 @@ default Path nextArgAsPath(String[] args, int i) throws CLIException { } } + /** + * Parses the next argument from the given array of arguments as a Path and ensures that it references a regular file. + * + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as a Path that is guaranteed to point to a regular file + * @throws CLIException If there is no next argument, or it does not point to a file (i.e. directory), or it does not exist + */ default Path nextArgAsRegularFileReference(String[] args, int i) throws CLIException { final Path path = nextArgAsPath(args, i); if(Files.isRegularFile(path)) @@ -56,6 +92,13 @@ default Path nextArgAsRegularFileReference(String[] args, int i) throws CLIExcep throw new CLIException("Expected an existing file but got: " + path, args[i]); } + /** + * Parses the next argument from the given array of arguments as a Path and ensures that it references a directory. + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as a Path that is guaranteed to point to a directory + * @throws CLIException If there is no next argument, or it does not point to a directory, or it does not exist + */ default Path nextArgAsDirectoryReference(String[] args, int i) throws CLIException { final Path path = nextArgAsPath(args, i); if(Files.isDirectory(path)) diff --git a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java index 4d1941d..474449e 100644 --- a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java +++ b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java @@ -5,8 +5,24 @@ import java.nio.file.Path; import java.nio.file.Paths; +/** + * Parser for {@link CommonConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ public class CommonConfigParser implements CLIParsingUtilities { + /** + * Creates a new artifact config parsers instance. + */ + public CommonConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link CommonConfig} objects. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link CommonConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ public CommonConfig parseCommonConfig(String[] args) throws CLIException { final CommonConfig config = new CommonConfig(); @@ -18,7 +34,14 @@ public CommonConfig parseCommonConfig(String[] args) throws CLIException { return config; } - protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException{ + /** + * Parses a given parameter from the array of CLI arguments and applies it to the config object. + * @param args CLI arguments array + * @param i Current parsing position + * @param config Current configuration object + * @throws CLIException If parsing the current argument fails. + */ + protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException { switch (args[i]){ case "-st": case "--skip-take": @@ -61,19 +84,54 @@ protected void handleParameter(String[] args, int i, CommonConfig config) throws } } + /** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis} instance. + */ public static class CommonConfig { + + /** + * Number of artifacts to skip from the underlying source, or -1 if disabled. + */ public long skip; + + /** + * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. + */ public long take; + /** + * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. + */ public Path inputListFile; + + /** + * File path to output analysis progress to. + */ public Path progressOutputFile; + + /** + * Path to read previous progress from, in order to restore it. Null if disabled. + */ public Path progressRestoreFile; + /** + * True if multiple threads shall be used, false for single-threaded analysis. + */ public boolean multipleThreads; + + /** + * Number of threads to use. + */ public int threadCount; + /** + * Interval of entities after which progress shall be saved. Defaults to 100. + */ public int progressWriteInterval; + /** + * Creates a new configuration with default values. + */ public CommonConfig() { skip = -1L; take = -1L; diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java index 7f7aa85..c769515 100644 --- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java +++ b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java @@ -7,12 +7,20 @@ import java.nio.file.Path; import java.util.Iterator; +/** + * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of + * groupID:artifactID:version per line. + */ public class FileBasedArtifactIterator implements Iterator { private final Path inputPath; private Iterator lineIterator; + /** + * Creates a new iterator instance for the given file path. + * @param input Path to a text file containing GAV triples + */ public FileBasedArtifactIterator(Path input){ this.inputPath = input; } diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index c3b31da..651617e 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -13,6 +13,10 @@ import java.util.Map; import java.util.Set; +/** + * Iterator over all unique library names in the Maven Central index. A library name is a tuple of GroupId:ArtifactId, + * separated by a colon. + */ public class LibraryIndexIterator implements Iterator, AutoCloseable { private final Logger logger = LogManager.getLogger(getClass()); @@ -28,6 +32,11 @@ public class LibraryIndexIterator implements Iterator, AutoCloseable { private String currentLibraryGA; private String nextLibraryGA; + /** + * Creates a new iterator instance for the given Maven repository. + * @param baseUri URI of the Maven-complient repository to iterator over + * @throws IOException If accessing the repository fails. + */ public LibraryIndexIterator(URI baseUri) throws IOException { this.libraryHashesSeen = new HashSet<>(); @@ -43,12 +52,19 @@ public LibraryIndexIterator(URI baseUri) throws IOException { } + /** + * Sets this iterator to a specific position by skipping over the given amount of entries. Aborts early if no more + * entries are available on the iterator. + * + * @param startIdx The iterator position to skip to + */ public void setPosition(long startIdx){ while(this.currentPosition < startIdx && this.hasNext()){ this.next(); } } + @Override public boolean hasNext() { // If currentLibraryGA is null, we have the first hasNext() call after a call to next(). We must find our new // currentLibraryGA first. @@ -74,6 +90,7 @@ public boolean hasNext() { return this.nextLibraryGA != null; } + @Override public String next() { if(hasNext()){ final String valueToReturn = this.currentLibraryGA; diff --git a/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java b/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java index 4aa682b..8f7c2f8 100644 --- a/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java +++ b/src/main/java/org/tudo/sse/utils/MavenCentralRepository.java @@ -16,7 +16,14 @@ */ public final class MavenCentralRepository { + /** + * The Maven Central repository base path. + */ public static final String RepoBasePath = "https://repo1.maven.org/maven2/"; + + /** + * The URI representing this repository's base path. + */ public static final URI RepoBaseURI = URI.create(RepoBasePath); private static MavenCentralRepository theInstance = null; @@ -43,6 +50,7 @@ public InputStream openXMLFileInputStream(ArtifactIdent ident) throws IOExceptio * @return An input stream for the library's version list XML file * @throws IOException If accessing the resource fails * @throws FileNotFoundException If the library does not exist / does not have a version list file + * @throws URISyntaxException If groupId or artifactId contained invalid URI characters */ public InputStream openXMLFileInputStream(String groupId, String artifactId) throws IOException, FileNotFoundException, URISyntaxException{ From 844e2c1632de1f415bab6cfc64a394f735a7b77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 5 Dec 2025 15:14:43 +0100 Subject: [PATCH 30/55] Drop Akka Actors in favor of Apache Pekko (typed) actors. --- pom.xml | 21 ++- .../MavenCentralArtifactAnalysis.java | 24 ++-- .../analyses/MavenCentralLibraryAnalysis.java | 22 ++- .../tudo/sse/multithreading/QueueActor.java | 130 +++++++++++------- .../sse/multithreading/ResolverActor.java | 78 +++++++---- .../WorkItemFinishedMessage.java | 24 ++-- .../WorkloadIsFinalMessage.java | 2 +- 7 files changed, 189 insertions(+), 112 deletions(-) diff --git a/pom.xml b/pom.xml index 115a509..e735b2a 100644 --- a/pom.xml +++ b/pom.xml @@ -39,12 +39,25 @@ 11 UTF-8 + 2.13 + + + + + org.apache.pekko + pekko-bom_${scala.binary.version} + 1.3.0 + pom + import + + + + - com.typesafe.akka - akka-actor_2.13 - 2.9.0-M2 + org.apache.pekko + pekko-actor-typed_${scala.binary.version} org.junit.jupiter @@ -96,7 +109,7 @@ de.opal-project - framework_2.13 + framework_${scala.binary.version} 6.0.0 diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 7e59237..2446f86 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -1,7 +1,7 @@ package org.tudo.sse.analyses; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; +import org.apache.pekko.actor.typed.ActorRef; +import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; @@ -9,6 +9,7 @@ import org.tudo.sse.model.ArtifactResolutionContext; import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.multithreading.ProcessIdentifierMessage; +import org.tudo.sse.multithreading.WorkItem; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.utils.ArtifactConfigParser; @@ -29,7 +30,7 @@ public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis { private ArtifactConfigParser.ArtifactConfig artifactConfig; - private ActorRef queueActorRef; + private ActorSystem system; private ResolverFactory resolverFactory; @@ -110,8 +111,6 @@ public final void runAnalysis(String[] args) { throw new RuntimeException(clix); } - ActorSystem system = null; - // Initialize resolver factory if(this.artifactConfig.outputEnabled) { resolverFactory = new ResolverFactory(this.artifactConfig.outputEnabled, this.artifactConfig.outputDirectory, processTransitives); @@ -120,11 +119,9 @@ public final void runAnalysis(String[] args) { } if(this.artifactConfig.multipleThreads) { - system = ActorSystem.create("marin-actors"); - // Compute position at which processing will start - final long initialPosition = getInitialPosition(); - this.queueActorRef = system.actorOf(QueueActor.props(this.artifactConfig.threadCount, system, initialPosition, - artifactConfig.progressWriteInterval, artifactConfig.progressOutputFile), "queueActor"); + system = ActorSystem.create(QueueActor.create(this.artifactConfig.threadCount, getInitialPosition(), + artifactConfig.progressWriteInterval, artifactConfig.progressOutputFile), "marin-actors"); + } // Invoke the beforeRunStart lifecycle hook @@ -144,8 +141,9 @@ public final void runAnalysis(String[] args) { if(system != null){ try { // Tell the queue that no more work items will follow - this.queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); + system.tell(WorkloadIsFinalMessage.getInstance()); system.getWhenTerminated().toCompletableFuture().get(); + system.close(); } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} } @@ -188,7 +186,7 @@ public void indexProcessor() { private void processIndex(Artifact current, ArtifactResolutionContext ctx) { if(this.artifactConfig.multipleThreads) { - queueActorRef.tell(new ProcessIdentifierMessage(ctx, this), ActorRef.noSender()); + system.tell(new ProcessIdentifierMessage(ctx, this)); } else { callResolver(current.getIdent(), ctx); analyzeArtifact(current); @@ -339,7 +337,7 @@ void walkDates(long since, long until, IndexIterator indexIterator) throws IOExc private void processIndexIdentifier(ArtifactIdent ident, ArtifactResolutionContext ctx) { if(this.artifactConfig.multipleThreads){ - queueActorRef.tell(new ProcessIdentifierMessage(ctx, this), ActorRef.noSender()); + system.tell(new ProcessIdentifierMessage(ctx, this)); } else { callResolver(ident, ctx); final Artifact artifact = ctx.getArtifact(ident); diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index fd3656a..eb648ea 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -1,11 +1,11 @@ package org.tudo.sse.analyses; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; +import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.LibraryResolutionContext; +import org.tudo.sse.multithreading.WorkItem; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.multithreading.ProcessLibraryMessage; import org.tudo.sse.multithreading.QueueActor; @@ -38,7 +38,7 @@ public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis { */ protected CommonConfigParser.CommonConfig config; - private ActorRef queueActorRef; + private ActorSystem system; private long lastPositionSaved; /** @@ -67,8 +67,6 @@ public final void runAnalysis(String[] args){ throw new RuntimeException(clix); } - ActorSystem system = null; - long currentPosition = 0L; // Get input iterator - this will either be from a configured input file or from the Maven Central Index. The // iterator will produce strings of form :. @@ -102,9 +100,8 @@ public final void runAnalysis(String[] args){ // If we want to use multiple threads, we initialize the queue actor that managers workers if(this.config.multipleThreads){ - system = ActorSystem.create("marin-actors"); - this.queueActorRef = system.actorOf(QueueActor.props(config.threadCount, system, currentPosition, - config.progressWriteInterval, config.progressOutputFile), "marin-queue-actor"); + this.system = ActorSystem.create(QueueActor.create(config.threadCount, currentPosition, config.progressWriteInterval, + config.progressOutputFile), "marin-actors"); } long entriesTaken = 0L; @@ -146,11 +143,12 @@ public final void runAnalysis(String[] args){ } // Close actor system if it was used - if(system != null){ + if(this.system != null){ try { // Tell the queue that no more work items will follow - this.queueActorRef.tell(WorkloadIsFinalMessage.getInstance(), ActorRef.noSender()); - system.getWhenTerminated().toCompletableFuture().get(); + this.system.tell(WorkloadIsFinalMessage.getInstance()); + this.system.getWhenTerminated().toCompletableFuture().get(); + this.system.close(); } catch (Exception x) { log.warn("Error closing actor system: {}", x.getMessage());} } @@ -229,7 +227,7 @@ private void processEntry(String ga){ process(groupId, artifactId, resolutionCtx); } else { final ProcessLibraryMessage msg = new ProcessLibraryMessage(() -> process(groupId, artifactId, resolutionCtx)); - queueActorRef.tell(msg, ActorRef.noSender()); + this.system.tell(msg); } } diff --git a/src/main/java/org/tudo/sse/multithreading/QueueActor.java b/src/main/java/org/tudo/sse/multithreading/QueueActor.java index 664d871..f5fe6b3 100644 --- a/src/main/java/org/tudo/sse/multithreading/QueueActor.java +++ b/src/main/java/org/tudo/sse/multithreading/QueueActor.java @@ -1,9 +1,14 @@ package org.tudo.sse.multithreading; -import akka.actor.*; -import akka.japi.pf.ReceiveBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.pekko.actor.typed.ActorRef; +import org.apache.pekko.actor.typed.Behavior; +import org.apache.pekko.actor.typed.PostStop; +import org.apache.pekko.actor.typed.javadsl.AbstractBehavior; +import org.apache.pekko.actor.typed.javadsl.ActorContext; +import org.apache.pekko.actor.typed.javadsl.Behaviors; +import org.apache.pekko.actor.typed.javadsl.Receive; import java.io.BufferedWriter; import java.io.FileWriter; @@ -18,13 +23,13 @@ * This class represents the processing queue that resolves jobs and sends them to different threads. * The size of the jobs and threads are determined by the configuration set in CliInformation. */ -public class QueueActor extends AbstractActor { +public class QueueActor extends AbstractBehavior { - private final int numResolverActors; - private final AtomicInteger curNumResolvers; + private final int maxNumberOfResolvers; + private final AtomicInteger currentNumberOfResolvers; private final Queue jobQueue; + private boolean indexFinished = false; - private final ActorSystem system; private final AtomicLong workItemsCompleted = new AtomicLong(0L); private final long progressSaveInterval; @@ -33,84 +38,107 @@ public class QueueActor extends AbstractActor { private static final Logger log = LogManager.getLogger(QueueActor.class); + + public static Behavior create(int maxNumberOfResolvers, + long initialWorkItemPosition, + long progressSaveInterval, + Path progressOutputFilePath) { + return Behaviors.setup(ctx -> + new QueueActor(ctx, maxNumberOfResolvers, initialWorkItemPosition, progressSaveInterval, progressOutputFilePath) + ); + } + /** * Creates a new processing queue with the given number of actors and the given actor system. - * @param numResolverActors Number of ResolverActor instances that will process jobs - * @param system The underlying ActorSystem + * @param maxNumberOfResolvers Number of ResolverActor instances that will process jobs * @param initialWorkItemPosition The number of work items that have been skipped due to progress restore or skip * @param progressSaveInterval The number of completed work items after which to save progress * @param progressOutputFilePath The file path to which to write progress to */ - public QueueActor(int numResolverActors, ActorSystem system, long initialWorkItemPosition, long progressSaveInterval, Path progressOutputFilePath) { - this.numResolverActors = numResolverActors; - this.system = system; - this.curNumResolvers = new AtomicInteger(0); + public QueueActor(ActorContext ctx, + int maxNumberOfResolvers, + long initialWorkItemPosition, + long progressSaveInterval, + Path progressOutputFilePath) { + super(ctx); + + this.maxNumberOfResolvers = maxNumberOfResolvers; + this.currentNumberOfResolvers = new AtomicInteger(0); this.jobQueue = new LinkedList<>(); this.progressSaveInterval = progressSaveInterval; this.progressOutputFilePath = progressOutputFilePath; this.workItemsCompleted.set(initialWorkItemPosition); + + log.info("Created queue actor"); } - /** - * Creates the properties needed to initialize an actor instance of this queue - * @param numResolverActors The number of ResolverActor instances that shall be used to process jobs - * @param system The underlying ActorSystem - * @param initialWorkItemPosition The number of work items that have been skipped due to progress restore or skip - * @param progressSaveInterval The number of completed work items after which to save progress - * @param progressOutputFilePath The file path to which to write progress to - * @return The AKKA actor properties - */ - public static Props props(int numResolverActors, ActorSystem system, long initialWorkItemPosition, long progressSaveInterval, Path progressOutputFilePath) { - return Props.create(QueueActor.class, () -> new QueueActor(numResolverActors, system, initialWorkItemPosition, progressSaveInterval, progressOutputFilePath)); + private Behavior onPostStop() { + log.info("Stopped QueueActor"); + return this; } @Override - public Receive createReceive() { - return ReceiveBuilder.create() - .match(ProcessIdentifierMessage.class, this::forwardToResolvers) - .match(ProcessLibraryMessage.class, this::forwardToResolvers) - .match(WorkItemFinishedMessage.class, workItemFinishedMessage -> { + public Receive createReceive(){ + return newReceiveBuilder() + .onMessage(ProcessIdentifierMessage.class, msg -> { + forwardToResolvers(msg); + return Behaviors.same(); + }) + .onMessage(ProcessLibraryMessage.class, msg -> { + forwardToResolvers(msg); + return Behaviors.same(); + }) + .onMessage(WorkItemFinishedMessage.class, msg -> { // Track completion of work items workItemsCompleted.incrementAndGet(); // Write progress file if needed writeProgressIfNeeded(); - // Distribute next job to worker that is now free, or kill worker if no jobs are left - synchronized (jobQueue){ - if(!jobQueue.isEmpty()) { - getSender().tell(jobQueue.remove(), getSelf()); - if(jobQueue.size() % 10 == 0) log.trace("Distributed a job, queue size {}", jobQueue.size()); + final ActorRef sender = msg.getSender(); + + synchronized (jobQueue) { + if(!jobQueue.isEmpty()){ + WorkItem workItem = jobQueue.poll(); + sender.tell(workItem); + if(jobQueue.size() % 10 == 0) + log.trace("Distributed a job, queue size {}", jobQueue.size()); } else { - synchronized(curNumResolvers) { - if(indexFinished && curNumResolvers.get() == 1) { - log.trace("Shutting down system"); - system.terminate(); + synchronized (currentNumberOfResolvers){ + if(indexFinished && currentNumberOfResolvers.get() == 1){ + log.trace("Shutting down queue actor..."); + return Behaviors.stopped(); } else { - log.trace("Killing a worker thread"); - getSender().tell(PoisonPill.getInstance(), getSelf()); - curNumResolvers.decrementAndGet(); + log.trace("Stopping a ResolverActor ..."); + sender.tell(WorkloadIsFinalMessage.getInstance()); + currentNumberOfResolvers.decrementAndGet(); } } } } + + return Behaviors.same(); }) - .match(WorkloadIsFinalMessage.class, workloadIsFinalMessage -> { - indexFinished = true; - if(curNumResolvers.get() == 0) { - system.terminate(); - } + .onMessage(WorkloadIsFinalMessage.class, msg -> { + this.indexFinished = true; + if(currentNumberOfResolvers.get() == 0) + return Behaviors.stopped(); + else + return Behaviors.same(); + }) + .onSignal(PostStop.class, s -> onPostStop()) .build(); } private void forwardToResolvers(WorkItem message){ - synchronized (curNumResolvers){ - if(curNumResolvers.get() < numResolverActors) { - ActorRef processor = getContext().actorOf(ResolverActor.props()); - processor.tell(message, getSelf()); - log.info("New resolver created"); - curNumResolvers.incrementAndGet(); + synchronized (currentNumberOfResolvers){ + if(currentNumberOfResolvers.get() < maxNumberOfResolvers) { + final ActorContext ctx = getContext(); + final String name = "resolver-" + currentNumberOfResolvers.get(); + ActorRef newResolver = ctx.spawn(ResolverActor.create(ctx.getSelf()), name); + currentNumberOfResolvers.incrementAndGet(); + newResolver.tell(message); } else { jobQueue.add(message); } diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index 3d0e659..ce06159 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -1,46 +1,78 @@ package org.tudo.sse.multithreading; -import akka.actor.AbstractActor; -import akka.actor.Props; -import akka.japi.pf.ReceiveBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.apache.pekko.actor.typed.ActorRef; +import org.apache.pekko.actor.typed.Behavior; +import org.apache.pekko.actor.typed.PostStop; +import org.apache.pekko.actor.typed.javadsl.AbstractBehavior; +import org.apache.pekko.actor.typed.javadsl.ActorContext; +import org.apache.pekko.actor.typed.javadsl.Behaviors; +import org.apache.pekko.actor.typed.javadsl.Receive; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.ArtifactResolutionContext; +import java.util.concurrent.atomic.AtomicInteger; + /** * This class is spawned in multiple threads * allowing for faster resolution of large quantity of artifacts. */ -public class ResolverActor extends AbstractActor { - - /** - * Creates a new ResolverActor - */ - public ResolverActor() {} - - /** - * Sets up the inherited properties for the actor. - * @return properties created for the ResolverActor class - */ - public static Props props() { - return Props.create(ResolverActor.class, ResolverActor::new); +public class ResolverActor extends AbstractBehavior { + + private static final AtomicInteger idCounter = new AtomicInteger(0); + + private final Logger log = LogManager.getLogger(getClass()); + private final int id = idCounter.getAndIncrement(); + private final ActorRef queueActor; + + static Behavior create(ActorRef queueActor) { + return Behaviors.setup(ctx -> new ResolverActor(ctx, queueActor)); + } + + private ResolverActor(ActorContext ctx, ActorRef queueActor) { + super(ctx); + + this.queueActor = queueActor; + + log.info("Created resolver actor #{}", id); + } + + private Behavior onPostStop(){ + log.info("Stopped resolver actor #{}", id); + return this; } @Override - public Receive createReceive() { - return ReceiveBuilder.create() - .match(ProcessIdentifierMessage.class, message -> { + public Receive createReceive(){ + return newReceiveBuilder() + .onMessage(ProcessIdentifierMessage.class, message -> { final ArtifactIdent identifier = message.getIdentifier(); final ArtifactResolutionContext ctx = message.getArtifactResolutionContext(); message.getInstance().callResolver(identifier, ctx); message.getInstance().analyzeArtifact(ctx.getArtifact(identifier)); - getSender().tell(WorkItemFinishedMessage.getInstance(), getSelf()); + final var queueResponse = new WorkItemFinishedMessage(this.getContext().getSelf()); + + queueActor.tell(queueResponse); + + return Behaviors.same(); }) - .match(ProcessLibraryMessage.class, message -> { + .onMessage(ProcessLibraryMessage.class, message -> { message.getProcessEntryCallback().get(); - getSender().tell(WorkItemFinishedMessage.getInstance(), getSelf()); - }).build(); + + final var queueResponse = new WorkItemFinishedMessage(this.getContext().getSelf()); + + queueActor.tell(queueResponse); + + return Behaviors.same(); + }) + .onMessage(WorkloadIsFinalMessage.class, msg -> Behaviors.stopped()) + .onSignal(PostStop.class, s -> onPostStop()) + .build(); } } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java b/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java index 2b3eaf1..3511612 100644 --- a/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/WorkItemFinishedMessage.java @@ -1,20 +1,28 @@ package org.tudo.sse.multithreading; +import org.apache.pekko.actor.typed.ActorRef; + /** - * A singleton message that is used in multithreaded mode to let the queue manager keep track of the number of completed + * A message that is used in multithreaded mode to let the queue manager keep track of the number of completed * work items (libraries or artifacts processed). */ -public class WorkItemFinishedMessage { +public class WorkItemFinishedMessage implements WorkItem { - private static final WorkItemFinishedMessage _instance = new WorkItemFinishedMessage(); + private final ActorRef sender; - private WorkItemFinishedMessage() {} + /** + * Creates a new WorkItemFinishedMessage for the given sender. + * @param sender Actor that send this message + */ + public WorkItemFinishedMessage(ActorRef sender) { + this.sender = sender; + } /** - * Retrieves the singleton instance of this message - * @return The singleton instance + * Gets the actor ref that sent this message. + * @return Sender reference */ - public static WorkItemFinishedMessage getInstance() { - return _instance; + public ActorRef getSender() { + return this.sender; } } diff --git a/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java b/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java index 09d2f0e..d4d89ef 100644 --- a/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/WorkloadIsFinalMessage.java @@ -4,7 +4,7 @@ * This class is a marker message sent to the queue actor. It signals that no more work items will be scheduled in the * current analysis run. */ -public class WorkloadIsFinalMessage { +public class WorkloadIsFinalMessage implements WorkItem { private static WorkloadIsFinalMessage _instance; From e93c65b40bb3f1c50dff5a244ea0d8e2031a3ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 5 Dec 2025 16:00:17 +0100 Subject: [PATCH 31/55] Drop Log4J in favor of SLF4J - this aligns with the logging framework used by apache pekko. Stopped shipping a logging backend in production, only used for testing now --- README.md | 24 +++++++++++++++++++ pom.xml | 13 +++++----- src/main/java/org/tudo/sse/IndexWalker.java | 7 +++--- .../sse/analyses/MavenCentralAnalysis.java | 6 ++--- .../analyses/MavenCentralLibraryAnalysis.java | 2 +- .../java/org/tudo/sse/model/Artifact.java | 8 +++---- .../org/tudo/sse/model/ArtifactIdent.java | 7 +++--- .../tudo/sse/model/jar/JarInformation.java | 10 ++++---- .../tudo/sse/multithreading/QueueActor.java | 23 +++++++++++------- .../sse/multithreading/ResolverActor.java | 9 ++----- .../org/tudo/sse/resolution/JarResolver.java | 10 ++++---- .../org/tudo/sse/resolution/PomResolver.java | 14 +++++------ .../tudo/sse/resolution/ResolverFactory.java | 14 +++++------ .../org/tudo/sse/utils/IndexIterator.java | 6 ++--- .../tudo/sse/utils/LibraryIndexIterator.java | 7 +++--- .../org/tudo/sse/utils/MarinOpalLogger.java | 8 +++---- src/main/resources/log4j2.xml | 13 ---------- .../org/tudo/sse/model/TypeStructureTest.java | 6 ++--- .../tudo/sse/resolution/PomResolverTest.java | 6 ++--- src/test/resources/logback.xml | 19 +++++++++++++++ 20 files changed, 121 insertions(+), 91 deletions(-) delete mode 100644 src/main/resources/log4j2.xml create mode 100644 src/test/resources/logback.xml diff --git a/README.md b/README.md index 7f17eb7..1344acb 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,30 @@ public class JavaDocImplementation extends MavenCentralArtifactAnalysis { } ``` +## Logging +MARIN relies on `slf4j-api` for logging purposes. You will need an SLF4J backend like `ch.qos.logback:logback-classic` on your path to enable and configure logging. +Note that MARIN uses Apache Pekko Actors for multithreaded processing. If your analysis implementations perform a lot of logging operations, we suggest you [use an async appender](https://pekko.apache.org/docs/pekko/current/typed/logging.html#logback) as to not block multithreaded execution. +The following example for `logback.xml` demonstrates how to set up an asnyc appender that logs to the console: +```xml + + + + + [%date{ISO8601}] [%level] [%logger] - %msg %n + + + + 8192 + true + + + + + + +``` +Depending on your implementation's logging output and logback queue size for, some log messages may be dropped. + ## IndexWalker This part of the interface enables an easy traversal and collection of information from the Maven Central Index. This relies on the IndexIterator which traverses the index storing the values of artifacts with the same identifier in a single artifact objects as a list of packages (representation of each unique artifact under the same identifier). diff --git a/pom.xml b/pom.xml index e735b2a..835dd5c 100644 --- a/pom.xml +++ b/pom.xml @@ -71,14 +71,15 @@ 7.1.3 - org.apache.logging.log4j - log4j-api - 2.17.2 + org.slf4j + slf4j-api + 2.0.17 - org.apache.logging.log4j - log4j-core - 2.17.2 + ch.qos.logback + logback-classic + 1.5.13 + test diff --git a/src/main/java/org/tudo/sse/IndexWalker.java b/src/main/java/org/tudo/sse/IndexWalker.java index 4a9799a..084831c 100644 --- a/src/main/java/org/tudo/sse/IndexWalker.java +++ b/src/main/java/org/tudo/sse/IndexWalker.java @@ -1,10 +1,9 @@ package org.tudo.sse; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; @@ -32,7 +31,7 @@ public class IndexWalker implements Iterable { private boolean resetIterator; private final URI base; - private static final Logger log = LogManager.getLogger(IndexWalker.class); + private static final Logger log = LoggerFactory.getLogger(IndexWalker.class); /** * Creates a new IndexWalker with the given repository base URI. diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java index 7d3b546..435c192 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java @@ -1,9 +1,9 @@ package org.tudo.sse.analyses; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opalj.log.GlobalLogContext$; import org.opalj.log.OPALLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.utils.MarinOpalLogger; abstract class MavenCentralAnalysis { @@ -31,7 +31,7 @@ abstract class MavenCentralAnalysis { /** * Logger instance for subclasses */ - protected final Logger log = LogManager.getLogger(getClass()); + protected final Logger log = LoggerFactory.getLogger(getClass()); /** diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index eb648ea..b03dc61 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -279,7 +279,7 @@ private List getReleaseIdentifiers(String groupId, String artifac try { this.onVersionListError(groupId, artifactId, x); } catch(Exception inner) { - log.warn("Unexcepted exception in analysis lifecycle hook onVersionListError for library {}:{}, {}", groupId, artifactId, x); + log.warn("Unexcepted exception in analysis lifecycle hook onVersionListError for library {}:{}", groupId, artifactId, x); } return null; diff --git a/src/main/java/org/tudo/sse/model/Artifact.java b/src/main/java/org/tudo/sse/model/Artifact.java index aac48cc..6d9ef83 100644 --- a/src/main/java/org/tudo/sse/model/Artifact.java +++ b/src/main/java/org/tudo/sse/model/Artifact.java @@ -1,7 +1,7 @@ package org.tudo.sse.model; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.model.jar.*; import org.tudo.sse.model.pom.PomInformation; @@ -21,7 +21,7 @@ public class Artifact { * The identifier object for the artifact. */ public final ArtifactIdent ident; - private static final Logger log = LogManager.getLogger(Artifact.class); + private static final Logger log = LoggerFactory.getLogger(Artifact.class); /** * A secondary identifier, for if its pom information has been moved on the maven central repository. @@ -153,7 +153,7 @@ public Map buildTypeStructure() { artifact.setJarInformation(resolver.parseJar(artifact.getIdent(), resolutionCtx).getJarInformation()); depArts.put(artifact.getIdent().getGroupID() + ":" + artifact.getIdent().getArtifactID(), artifact); } catch (JarResolutionException e) { - log.error(e); + log.error("Failed to resolve transitive dependency: {}", artifact.getIdent(), e); } } } diff --git a/src/main/java/org/tudo/sse/model/ArtifactIdent.java b/src/main/java/org/tudo/sse/model/ArtifactIdent.java index 8441326..09ee7c3 100644 --- a/src/main/java/org/tudo/sse/model/ArtifactIdent.java +++ b/src/main/java/org/tudo/sse/model/ArtifactIdent.java @@ -2,8 +2,9 @@ import java.net.URI; import java.util.Objects; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.utils.MavenCentralRepository; @@ -46,7 +47,7 @@ public class ArtifactIdent { */ private String customRepository; - private static final Logger log = LogManager.getLogger(ArtifactIdent.class); + private static final Logger log = LoggerFactory.getLogger(ArtifactIdent.class); /** * Creates a new artifact identifier with the given attributes. Artifact identifiers correspond to Maven GAV-Triples. diff --git a/src/main/java/org/tudo/sse/model/jar/JarInformation.java b/src/main/java/org/tudo/sse/model/jar/JarInformation.java index e7fde16..49606f0 100644 --- a/src/main/java/org/tudo/sse/model/jar/JarInformation.java +++ b/src/main/java/org/tudo/sse/model/jar/JarInformation.java @@ -1,12 +1,12 @@ package org.tudo.sse.model.jar; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opalj.br.analyses.Project; import org.opalj.br.package$; import org.opalj.br.reader.Java17Framework$; import org.opalj.br.reader.Java17LibraryFramework; import org.opalj.br.reader.Java17LibraryFramework$; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.ArtifactInformation; import org.tudo.sse.resolution.FileNotFoundException; @@ -34,7 +34,7 @@ */ public class JarInformation extends ArtifactInformation { - private static final Logger log = LogManager.getLogger(JarInformation.class); + private static final Logger log = LoggerFactory.getLogger(JarInformation.class); private long codesize; private long numClassFiles; @@ -218,7 +218,7 @@ public List getOpalClassFileRepresentations() throws IOE * @throws IOException If reading the JAR file fails */ public Project getOpalProject() throws IOException { - return getOpalProject(MarinOpalLogger.newInfoLogger(LogManager.getLogger("[OPAL-Project]"))); + return getOpalProject(MarinOpalLogger.newInfoLogger(LoggerFactory.getLogger("[OPAL-Project]"))); } /** @@ -276,7 +276,7 @@ public Project getOpalProject(Set dependencies) throws IO */ public Project getOpalProject(Set dependencies, boolean fullyLoadLibraries, boolean breakOnFailure) throws IOException { return getOpalProject(dependencies, fullyLoadLibraries, breakOnFailure, - MarinOpalLogger.newInfoLogger(LogManager.getLogger("[OPAL-Project]"))); + MarinOpalLogger.newInfoLogger(LoggerFactory.getLogger("[OPAL-Project]"))); } /** diff --git a/src/main/java/org/tudo/sse/multithreading/QueueActor.java b/src/main/java/org/tudo/sse/multithreading/QueueActor.java index f5fe6b3..610bfeb 100644 --- a/src/main/java/org/tudo/sse/multithreading/QueueActor.java +++ b/src/main/java/org/tudo/sse/multithreading/QueueActor.java @@ -1,7 +1,5 @@ package org.tudo.sse.multithreading; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.pekko.actor.typed.ActorRef; import org.apache.pekko.actor.typed.Behavior; import org.apache.pekko.actor.typed.PostStop; @@ -36,9 +34,15 @@ public class QueueActor extends AbstractBehavior { private long lastProgressSaved = 0L; private final Path progressOutputFilePath; - private static final Logger log = LogManager.getLogger(QueueActor.class); - + /** + * Creates a new queue actor behavior using the given configuration values + * @param maxNumberOfResolvers Number of ResolverActor instances that will process jobs + * @param initialWorkItemPosition The number of work items that have been skipped due to progress restore or skip + * @param progressSaveInterval The number of completed work items after which to save progress + * @param progressOutputFilePath The file path to which to write progress to + * @return A new QueueActor behavior + */ public static Behavior create(int maxNumberOfResolvers, long initialWorkItemPosition, long progressSaveInterval, @@ -50,6 +54,7 @@ public static Behavior create(int maxNumberOfResolvers, /** * Creates a new processing queue with the given number of actors and the given actor system. + * @param ctx This actor's context * @param maxNumberOfResolvers Number of ResolverActor instances that will process jobs * @param initialWorkItemPosition The number of work items that have been skipped due to progress restore or skip * @param progressSaveInterval The number of completed work items after which to save progress @@ -70,11 +75,11 @@ public QueueActor(ActorContext ctx, this.progressOutputFilePath = progressOutputFilePath; this.workItemsCompleted.set(initialWorkItemPosition); - log.info("Created queue actor"); + ctx.getLog().info("Created queue actor"); } private Behavior onPostStop() { - log.info("Stopped QueueActor"); + getContext().getLog().info("Stopped QueueActor"); return this; } @@ -102,14 +107,14 @@ public Receive createReceive(){ WorkItem workItem = jobQueue.poll(); sender.tell(workItem); if(jobQueue.size() % 10 == 0) - log.trace("Distributed a job, queue size {}", jobQueue.size()); + getContext().getLog().trace("Distributed a job, queue size {}", jobQueue.size()); } else { synchronized (currentNumberOfResolvers){ if(indexFinished && currentNumberOfResolvers.get() == 1){ - log.trace("Shutting down queue actor..."); + getContext().getLog().trace("Shutting down queue actor..."); return Behaviors.stopped(); } else { - log.trace("Stopping a ResolverActor ..."); + getContext().getLog().trace("Stopping a ResolverActor ..."); sender.tell(WorkloadIsFinalMessage.getInstance()); currentNumberOfResolvers.decrementAndGet(); } diff --git a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java index ce06159..f908d79 100644 --- a/src/main/java/org/tudo/sse/multithreading/ResolverActor.java +++ b/src/main/java/org/tudo/sse/multithreading/ResolverActor.java @@ -1,9 +1,5 @@ package org.tudo.sse.multithreading; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import org.apache.pekko.actor.typed.ActorRef; import org.apache.pekko.actor.typed.Behavior; import org.apache.pekko.actor.typed.PostStop; @@ -24,7 +20,6 @@ public class ResolverActor extends AbstractBehavior { private static final AtomicInteger idCounter = new AtomicInteger(0); - private final Logger log = LogManager.getLogger(getClass()); private final int id = idCounter.getAndIncrement(); private final ActorRef queueActor; @@ -37,11 +32,11 @@ private ResolverActor(ActorContext ctx, ActorRef queueActor) this.queueActor = queueActor; - log.info("Created resolver actor #{}", id); + ctx.getLog().info("Created resolver actor #{}", id); } private Behavior onPostStop(){ - log.info("Stopped resolver actor #{}", id); + getContext().getLog().info("Stopped resolver actor #{}", id); return this; } diff --git a/src/main/java/org/tudo/sse/resolution/JarResolver.java b/src/main/java/org/tudo/sse/resolution/JarResolver.java index a42d4b6..cf9544e 100644 --- a/src/main/java/org/tudo/sse/resolution/JarResolver.java +++ b/src/main/java/org/tudo/sse/resolution/JarResolver.java @@ -1,13 +1,13 @@ package org.tudo.sse.resolution; import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opalj.br.ClassFile; import org.opalj.br.Method; import org.opalj.br.ClassType; import org.opalj.br.reader.BytecodeInstructionsCache; import org.opalj.br.reader.Java17FrameworkWithCaching; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.jar.JarInformation; @@ -40,7 +40,7 @@ public class JarResolver { private final Path pathToDirectory; private boolean output; private static final MavenCentralRepository MavenRepo = MavenCentralRepository.getInstance(); - private static final Logger log = LogManager.getLogger(JarResolver.class); + private static final Logger log = LoggerFactory.getLogger(JarResolver.class); /** * Creates a new empty JAR resolver instance @@ -83,7 +83,7 @@ public List resolveJars(List identifiers, ResolutionCon try { toReturn.add(parseJar(current, ctx)); } catch (JarResolutionException e) { - log.error(e); + log.error("Failed to resolve JAR for {}", current, e); } count++; @@ -148,7 +148,7 @@ public Artifact parseJar(ArtifactIdent identifier, ResolutionContext ctx) throws return currentArtifact; } catch (IOException e) { - log.error(e); + log.error("IO error while accessing JAR for {}", identifier, e); } catch (FileNotFoundException ignored) {} return null; } diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java index d3780c8..6c8312a 100644 --- a/src/main/java/org/tudo/sse/resolution/PomResolver.java +++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java @@ -1,7 +1,5 @@ package org.tudo.sse.resolution; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.maven.model.*; import org.apache.maven.model.io.xpp3.MavenXpp3Reader; @@ -11,6 +9,8 @@ import org.eclipse.aether.version.InvalidVersionSpecificationException; import org.eclipse.aether.version.Version; import org.eclipse.aether.version.VersionRange; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.*; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; @@ -39,7 +39,7 @@ public class PomResolver { private static final MavenCentralRepository MavenRepo = MavenCentralRepository.getInstance(); private final boolean resolveTransitives; - private static final Logger log = LogManager.getLogger(PomResolver.class); + private static final Logger log = LoggerFactory.getLogger(PomResolver.class); private final Map> predefinedPomValues; private final IReleaseListProvider releaseListProvider; @@ -163,7 +163,7 @@ public List resolveArtifacts(List idents, ResolutionCon try { poms.add(resolveArtifact(ident, ctx)); } catch(PomResolutionException e) { - log.error(e); + log.error("Failed to resolve artifact {}", ident, e); } catch ( IOException e) { throw new RuntimeException(e); } catch(FileNotFoundException ignored) {} @@ -766,7 +766,7 @@ public String resolveVersionRange(org.tudo.sse.model.pom.Dependency toResolve) { } } catch (IOException | InvalidVersionSpecificationException e) { - log.error(e); + log.error("Failed to resolve version range", e); } return highestMatching; @@ -866,7 +866,7 @@ private Artifact resolveTransitives(Artifact current, if(!current.getPomInformation().getRawPomFeatures().getRepositories().isEmpty()) { return resolveFromSecondaryRepo(current.getPomInformation().getRawPomFeatures().getRepositories(), toResolve, alrEncountered, exclusions, ctx); } - if(!(e instanceof FileNotFoundException)) log.error(e); + if(!(e instanceof FileNotFoundException)) log.error("Exception while transitively resolving dependency: {}", toResolve.getIdent(), e); return null; } } @@ -890,7 +890,7 @@ private Artifact resolveFromSecondaryRepo(List repos, try { toReturn = recursiveResolver(toResolve.getIdent(), alrEncountered, exclusions, ctx); } catch (IOException | PomResolutionException e) { - log.error(e); + log.error("Exception while resolving from secondary repository: {}", toResolve.getIdent(), e); } catch (FileNotFoundException ignored) {} i++; } diff --git a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java index 71426a6..a99adba 100644 --- a/src/main/java/org/tudo/sse/resolution/ResolverFactory.java +++ b/src/main/java/org/tudo/sse/resolution/ResolverFactory.java @@ -3,8 +3,8 @@ import java.io.IOException; import java.nio.file.Path; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.ResolutionContext; @@ -16,7 +16,7 @@ public class ResolverFactory { private final PomResolver pomResolver; private final JarResolver jarResolver; - private static final Logger log = LogManager.getLogger(ResolverFactory.class); + private static final Logger log = LoggerFactory.getLogger(ResolverFactory.class); /** * Creates a new resolver factory instance. Resolvers will not output the artifacts that they process. @@ -50,7 +50,7 @@ public void runPom(ArtifactIdent identifier, ResolutionContext ctx) { try { pomResolver.resolveArtifact(identifier, ctx); } catch (IOException | PomResolutionException e) { - log.error(e); + log.error("Error resolving pom file for {}", identifier, e); } catch (FileNotFoundException ignored) {} } @@ -64,7 +64,7 @@ public void runJar(ArtifactIdent identifier, ResolutionContext ctx) { try { jarResolver.parseJar(identifier, ctx); } catch (JarResolutionException e) { - log.error(e); + log.error("Error resolving JAR file for {}", identifier, e); } } @@ -78,14 +78,14 @@ public void runBoth(ArtifactIdent identifier, ResolutionContext ctx) { try { pomResolver.resolveArtifact(identifier, ctx); } catch (IOException | PomResolutionException e) { - log.error(e); + log.error("Error resolving pom file for {}", identifier, e); } catch(FileNotFoundException ignored){} try { jarResolver.setOutput(false); jarResolver.parseJar(identifier, ctx); } catch (JarResolutionException e) { - log.error(e); + log.error("Error resolving JAR file for {}", identifier, e); } } diff --git a/src/main/java/org/tudo/sse/utils/IndexIterator.java b/src/main/java/org/tudo/sse/utils/IndexIterator.java index edb8425..1fac507 100644 --- a/src/main/java/org/tudo/sse/utils/IndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/IndexIterator.java @@ -1,8 +1,8 @@ package org.tudo.sse.utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.maven.index.reader.IndexReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.IndexWalker; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.Package; @@ -31,7 +31,7 @@ public class IndexIterator implements Iterator { private IndexInformation nextArtifact; private boolean prevHasNext; - private static final Logger log = LogManager.getLogger(IndexIterator.class); + private static final Logger log = LoggerFactory.getLogger(IndexIterator.class); /** diff --git a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java index 651617e..58a17d6 100644 --- a/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/LibraryIndexIterator.java @@ -1,10 +1,9 @@ package org.tudo.sse.utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import org.apache.maven.index.reader.ChunkReader; import org.apache.maven.index.reader.IndexReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URI; @@ -19,7 +18,7 @@ */ public class LibraryIndexIterator implements Iterator, AutoCloseable { - private final Logger logger = LogManager.getLogger(getClass()); + private final Logger logger = LoggerFactory.getLogger(getClass()); private final Set libraryHashesSeen; private final Iterator> entryIterator; diff --git a/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java b/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java index 9eb1c8f..2cf7332 100644 --- a/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java +++ b/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java @@ -1,8 +1,8 @@ package org.tudo.sse.utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opalj.log.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Acts as a logging bridge between Log4j and OPAL. A MarinOpalLogger instance can be passed when creating an OPAL @@ -20,7 +20,7 @@ public class MarinOpalLogger implements OPALLogger { * Create a new instance that creates a new underlying Log4j backend. */ public MarinOpalLogger(){ - this.internalLog = LogManager.getLogger(MarinOpalLogger.class); + this.internalLog = LoggerFactory.getLogger(MarinOpalLogger.class); } /** @@ -90,7 +90,7 @@ public void log(LogMessage message, LogContext ctx) { } else if(message.level() == Info$.MODULE$ && infoEnabled){ internalLog.info(message.message()); } else if(message.level() == Fatal$.MODULE$){ - internalLog.fatal(message.message()); + internalLog.error("[FATAL] {}", message.message()); } } diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml deleted file mode 100644 index 96b396c..0000000 --- a/src/main/resources/log4j2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/test/java/org/tudo/sse/model/TypeStructureTest.java b/src/test/java/org/tudo/sse/model/TypeStructureTest.java index 83803c5..43bf4ac 100644 --- a/src/test/java/org/tudo/sse/model/TypeStructureTest.java +++ b/src/test/java/org/tudo/sse/model/TypeStructureTest.java @@ -2,9 +2,9 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.jar.ClassFileNode; import org.tudo.sse.model.jar.DefinedClassFileNode; import org.tudo.sse.resolution.*; @@ -26,7 +26,7 @@ class TypeStructureTest { Map json; Gson gson = new Gson(); - private static final Logger log = LogManager.getLogger(TypeStructureTest.class); + private static final Logger log = LoggerFactory.getLogger(TypeStructureTest.class); { InputStream resource = this.getClass().getClassLoader().getResourceAsStream("TypeStructure.json"); diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index a2d48f1..05e1bf3 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -2,10 +2,10 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.IndexWalker; import org.tudo.sse.model.*; @@ -37,7 +37,7 @@ class PomResolverTest { Map json; Gson gson = new Gson(); - private static final Logger log = LogManager.getLogger(PomResolverTest.class); + private static final Logger log = LoggerFactory.getLogger(PomResolverTest.class); { InputStream resource = this.getClass().getClassLoader().getResourceAsStream("PomInputs.json"); diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml new file mode 100644 index 0000000..0d5509b --- /dev/null +++ b/src/test/resources/logback.xml @@ -0,0 +1,19 @@ + + + + + + [%date{ISO8601}] [%level] [%logger] - %msg %n + + + + + 8192 + true + + + + + + + \ No newline at end of file From f345b062a60d01b6b599a356a1a62cfc587268a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 8 Dec 2025 09:31:50 +0100 Subject: [PATCH 32/55] Fix wrong logging framework being mentioned in comments --- src/main/java/org/tudo/sse/utils/MarinOpalLogger.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java b/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java index 2cf7332..de91899 100644 --- a/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java +++ b/src/main/java/org/tudo/sse/utils/MarinOpalLogger.java @@ -5,9 +5,9 @@ import org.slf4j.LoggerFactory; /** - * Acts as a logging bridge between Log4j and OPAL. A MarinOpalLogger instance can be passed when creating an OPAL - * project, and can be configured to selectively enable certain log levels. You can pass an existing Log4j logger - * to use as the underlying backend, otherwise the bridge will create its own Log4j logger backend. + * Acts as a logging bridge between SLF4J and OPAL. A MarinOpalLogger instance can be passed when creating an OPAL + * project, and can be configured to selectively enable certain log levels. You can pass an existing SLF4J logger + * to use as the underlying backend, otherwise the bridge will create its own SLF4J logger backend. */ public class MarinOpalLogger implements OPALLogger { From c63eade16ef622a0c3898b968a6217f65f7272c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 8 Dec 2025 13:41:11 +0100 Subject: [PATCH 33/55] Rework configuration system. Add configuration builder classes to programmatically build MARIN configurations --- .../MavenCentralArtifactAnalysis.java | 27 ++- .../analyses/MavenCentralLibraryAnalysis.java | 19 +- .../config/ArtifactAnalysisConfig.java | 49 +++++ .../config/ArtifactAnalysisConfigBuilder.java | 136 +++++++++++++ .../config/InvalidConfigurationException.java | 50 +++++ .../config/LibraryAnalysisConfig.java | 90 +++++++++ .../config/LibraryAnalysisConfigBuilder.java | 178 ++++++++++++++++++ .../parsing/ArtifactAnalysisConfigParser.java | 62 ++++++ .../config/parsing}/CLIParsingUtilities.java | 2 +- .../parsing/LibraryAnalysisConfigParser.java | 86 +++++++++ .../tudo/sse/utils/ArtifactConfigParser.java | 118 ------------ .../tudo/sse/utils/CommonConfigParser.java | 146 -------------- .../MavenCentralArtifactAnalysisTest.java | 20 +- .../MavenCentralLibraryAnalysisTest.java | 19 +- src/test/resources/MavenAnalysis.json | 2 +- 15 files changed, 697 insertions(+), 307 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java rename src/main/java/org/tudo/sse/{utils => analyses/config/parsing}/CLIParsingUtilities.java (98%) create mode 100644 src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java delete mode 100644 src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java delete mode 100644 src/main/java/org/tudo/sse/utils/CommonConfigParser.java diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 2446f86..bbd7f71 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -1,8 +1,9 @@ package org.tudo.sse.analyses; -import org.apache.pekko.actor.typed.ActorRef; import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; @@ -12,7 +13,7 @@ import org.tudo.sse.multithreading.WorkItem; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; -import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; import org.tudo.sse.utils.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; @@ -29,7 +30,7 @@ */ public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis { - private ArtifactConfigParser.ArtifactConfig artifactConfig; + private ArtifactAnalysisConfig artifactConfig; private ActorSystem system; private ResolverFactory resolverFactory; @@ -53,7 +54,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, super(requiresIndex, requiresPom, requiresTransitives, requiresJar); // Initialize with default config, update later - artifactConfig = new ArtifactConfigParser.ArtifactConfig(); + artifactConfig = new ArtifactAnalysisConfigBuilder().build(); } @@ -68,7 +69,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, * Returns the CLI configuration for this analysis. * @return CLI information for this analysis */ - public ArtifactConfigParser.ArtifactConfig getSetupInfo() { + public ArtifactAnalysisConfig getSetupInfo() { return this.artifactConfig; } @@ -91,8 +92,8 @@ private void printRunInfo(){ log.info("\t - Reading artifacts from Maven Central index"); if(this.artifactConfig.progressRestoreFile != null) log.info("\t - Restoring last index position from " + this.artifactConfig.progressRestoreFile); if(this.artifactConfig.progressOutputFile != null) log.info("\t - Writing last index position to " + this.artifactConfig.progressOutputFile); - if(this.artifactConfig.skip >= 0) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); - if(this.artifactConfig.take >= 0) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); + if(this.artifactConfig.hasSkip()) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); + if(this.artifactConfig.hasTake()) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); if(this.artifactConfig.since >= 0) log.info("\t - Skipping artifacts before " + this.artifactConfig.since); if(this.artifactConfig.until >= 0) log.info("\t - Taking artifacts until " + this.artifactConfig.until); @@ -105,7 +106,7 @@ private void printRunInfo(){ public final void runAnalysis(String[] args) { // Obtain CLI arguments try { - this.artifactConfig = new ArtifactConfigParser().parseArtifactConfig(args); + this.artifactConfig = new ArtifactAnalysisConfigParser().parseArtifactConfig(args); printRunInfo(); } catch(CLIException clix){ throw new RuntimeException(clix); @@ -168,7 +169,7 @@ public void indexProcessor() { final long startingPosition = getInitialPosition(); skipN(startingPosition, indexIterator); - if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1){ + if(this.artifactConfig.hasTake()){ walkPaginated(this.artifactConfig.take, indexIterator); } else if(this.artifactConfig.since != -1 && this.artifactConfig.until != -1){ walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); @@ -359,7 +360,7 @@ void processArtifactsFromInputFile() { log.info("Restoring previous progress from file (position {})", lastProgress); this.skipN(lastProgress, fileIterator); - } else if(this.artifactConfig.skip > 0){ + } else if(this.artifactConfig.hasSkip()){ // Skip configured values only if we did not restore from progress file log.info("Skipping {} entries from file", artifactConfig.skip); skipN(this.artifactConfig.skip, fileIterator); @@ -369,10 +370,8 @@ void processArtifactsFromInputFile() { if(!fileIterator.hasNext()) log.warn("No more contents left to process in input file: {}", this.artifactConfig.inputListFile); - final boolean takeLimited = this.artifactConfig.take >= 0; - long entriesTaken = 0L; - while ((!takeLimited || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { + while ((!this.artifactConfig.hasTake() || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { ArtifactIdent current = null; @@ -421,7 +420,7 @@ public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) { private long getInitialPosition() { if(artifactConfig.progressRestoreFile != null) return getStartingPos(); - else if(artifactConfig.skip > 0) return artifactConfig.skip; + else if(artifactConfig.hasSkip()) return artifactConfig.skip; else return 0L; } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index b03dc61..40a3ec8 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -2,6 +2,7 @@ import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.LibraryResolutionContext; @@ -12,7 +13,7 @@ import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; -import org.tudo.sse.utils.CommonConfigParser; +import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser; import org.tudo.sse.utils.LibraryIndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -36,7 +37,7 @@ public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis { /** * The configuration for this analysis instance. Only available after runAnalysis has been called. */ - protected CommonConfigParser.CommonConfig config; + protected LibraryAnalysisConfig config; private ActorSystem system; private long lastPositionSaved; @@ -61,7 +62,7 @@ protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTrans public final void runAnalysis(String[] args){ // Obtain CLI arguments try { - this.config = new CommonConfigParser().parseCommonConfig(args); + this.config = new LibraryAnalysisConfigParser().parseCommonConfig(args); printRunInfo(); } catch(CLIException clix){ throw new RuntimeException(clix); @@ -84,9 +85,9 @@ public final void runAnalysis(String[] args){ currentPosition += 1L; } // Notify users that skip will not be applied - if(config.skip > 0) + if(config.hasSkip()) log.info("Not applying skip value because progress was restored from previous run"); - } else if(config.skip > 0){ + } else if(config.hasSkip()){ // We only apply a skip if we did not restore previous progress log.info("Skipping {} library names", config.skip); for(int i = 0; i < config.skip; i++){ @@ -106,7 +107,7 @@ public final void runAnalysis(String[] args){ long entriesTaken = 0L; - if(config.take >= 0) + if(config.hasTake()) log.info("Taking {} library names", config.take); // Invoke the beforeRunStart lifecycle hook @@ -119,7 +120,7 @@ public final void runAnalysis(String[] args){ // If specified, we only take the configured amount of entries. If not, we process as long as the iterator // provides new entries - while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ + while((!config.hasTake()|| entriesTaken < config.take) && gaIterator.hasNext()){ processEntry(gaIterator.next()); entriesTaken += 1L; @@ -336,8 +337,8 @@ private void printRunInfo(){ log.info("\t - Reading libraries from Maven Central index"); if(config.progressRestoreFile != null) log.info("\t - Restoring last index position from " + config.progressRestoreFile); if(config.progressOutputFile != null) log.info("\t - Writing last index position to " + config.progressOutputFile); - if(config.skip >= 0) log.info("\t - Skipping " + config.skip + " artifacts"); - if(config.take >= 0) log.info("\t - Taking " + config.take + " artifacts"); + if(config.hasSkip()) log.info("\t - Skipping " + config.skip + " artifacts"); + if(config.hasTake()) log.info("\t - Taking " + config.take + " artifacts"); } else { log.info("\t - Reading libraries from GA-list at " + config.inputListFile); } diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java new file mode 100644 index 0000000..1c1545c --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -0,0 +1,49 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Path; + +/** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. + * + * @author Johannes Düsing + */ +public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { + + static final long DEFAULT_VALUE_SINCE = -1L; + static final long DEFAULT_VALUE_UNTIL = -1L; + static final Path DEFAULT_VALUE_OUTPUT = null; + + /** + * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. + */ + public long since; + + /** + * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. + */ + public long until; + + /** + * Directory to write file outputs to, or null if disabled. + */ + public Path outputDirectory; + + /** + * Whether files shall be written to the output directory. + */ + public boolean outputEnabled; + + /** + * Creates a new configuration object with default values. + */ + ArtifactAnalysisConfig(){ + super(); + + since = DEFAULT_VALUE_SINCE; + until = DEFAULT_VALUE_UNTIL; + outputEnabled = false; + outputDirectory = DEFAULT_VALUE_OUTPUT; + } + + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java new file mode 100644 index 0000000..d0e2a33 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java @@ -0,0 +1,136 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Configuration builder to obtain {@link ArtifactAnalysisConfig} instances programmatically. + * + * @author Johannes Düsing + */ +public final class ArtifactAnalysisConfigBuilder extends LibraryAnalysisConfigBuilder { + + private final ArtifactAnalysisConfig theConfig; + + private ArtifactAnalysisConfigBuilder(ArtifactAnalysisConfig theConfig) { + super(theConfig); + + this.theConfig = theConfig; + } + + /** + * Creates a new builder with default configuration values. + */ + public ArtifactAnalysisConfigBuilder(){ + this(new ArtifactAnalysisConfig()); + } + + @Override + public ArtifactAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("skip", "Cannot apply skip when time-based filtering is used"); + + super.withSkip(toSkip); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("take", "Cannot apply take when time-based filtering is used"); + + super.withTake(toTake); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withInputList(Path inputList) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("inputs", "Cannot use input list when time-based filtering is applied"); + + super.withInputList(inputList); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException { + super.withProgressOutputFile(outputFile); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException { + super.withProgressRestoreFile(restoreFile); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException { + super.withNumberOfThreads(numThreads); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException { + super.withProgressWriteInterval(progressWriteInterval); + return this; + } + + /** + * Sets the output directory to store processed artifacts in. If your analysis requires IndexInformation only, it + * will output a list of GAV triples. If pom information is required, one pom.xml file per artifact will be stored. + * If JAR information is required, the artifact's JAR will be stored in this directory. + * + * @param outDir The output directory to use. Must be a valid reference to an existing directory. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public ArtifactAnalysisConfigBuilder withOutputDirectory(Path outDir) throws InvalidConfigurationException { + if(outDir == null || !Files.isDirectory(outDir)) + throw new InvalidConfigurationException("output", "Output directory must be a valid directory reference"); + + if(this.theConfig.outputDirectory != ArtifactAnalysisConfig.DEFAULT_VALUE_OUTPUT) + log.warn("Value 'output' is set multiple times, previous value overridden."); + + this.theConfig.outputDirectory = outDir; + this.theConfig.outputEnabled = true; + return this; + } + + /** + * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have + * been released after the timestamp given in 'since', and before 'until'. + * + * @param since Timestamp marking the lower limit of the range + * @param until Timestamp marking the upper limit of the range + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration values are not valid + */ + public ArtifactAnalysisConfigBuilder withSinceUtil(long since, long until) throws InvalidConfigurationException { + if(since <= 0L || until <= since) + throw new InvalidConfigurationException("since-until", "Since and Until must define a valid range"); + + if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP || + this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE) + throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when pagination is used"); + + if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST) + throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when input list is used"); + + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + log.warn("Values 'since' and 'until' are set multiple times, previous values overridden."); + + this.theConfig.since = since; + this.theConfig.until = until; + return this; + } + + @Override + public ArtifactAnalysisConfig build(){ + return this.theConfig; + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java new file mode 100644 index 0000000..cb26b5a --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java @@ -0,0 +1,50 @@ +package org.tudo.sse.analyses.config; + +/** + * Exception thrown when a MARIN configuration value was invalid. + * + * @author Johannes Düsing + */ +public class InvalidConfigurationException extends Exception { + + /** + * Attribute the error occurred for. + */ + private final String attrName; + + /** + * Message describing the error. + */ + private final String msg; + + /** + * Creates a new instance with the given configuration attribute name and message. + * @param attrName The configuration value that was invalid + * @param msg A message describing the error + */ + InvalidConfigurationException(String attrName, String msg){ + this.attrName = attrName; + this.msg = msg; + } + + /** + * Returns the name of the attribute for which this exception occurred. + * @return The attribute name + */ + public String getAttributeName() { + return this.attrName; + } + + /** + * Returns a description on the error that occurred. + * @return The error description + */ + public String getDescription(){ + return this.msg; + } + + @Override + public String getMessage() { + return String.format("Invalid configuration value for '%s': %s", attrName, msg); + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java new file mode 100644 index 0000000..23c1b7e --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java @@ -0,0 +1,90 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * + * @author Johannes Düsing + */ +public class LibraryAnalysisConfig { + + static final long DEFAULT_VALUE_SKIP = -1L; + static final long DEFAULT_VALUE_TAKE = -1L; + static final Path DEFAULT_VALUE_INPUT_LIST = null; + static final Path DEFAULT_VALUE_PROGRESS_OUTPUT_FILE = Paths.get("marin-progress"); + static final Path DEFAULT_VALUE_PROGRESS_RESTORE_FILE = null; + static final int DEFAULT_NUM_THREADS = 1; + static final int DEFAULT_PROGRESS_WRITE_INTERVAL = 100; + + /** + * Number of artifacts to skip from the underlying source, or -1 if disabled. + */ + public long skip; + + /** + * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. + */ + public long take; + + /** + * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. + */ + public Path inputListFile; + + /** + * File path to output analysis progress to. + */ + public Path progressOutputFile; + + /** + * Path to read previous progress from, in order to restore it. Null if disabled. + */ + public Path progressRestoreFile; + + /** + * True if multiple threads shall be used, false for single-threaded analysis. + */ + public boolean multipleThreads; + + /** + * Number of threads to use. + */ + public int threadCount; + + /** + * Interval of entities after which progress shall be saved. Defaults to 100. + */ + public int progressWriteInterval; + + /** + * Creates a new configuration with default values. + */ + LibraryAnalysisConfig() { + skip = DEFAULT_VALUE_SKIP; + take = DEFAULT_VALUE_TAKE; + inputListFile = DEFAULT_VALUE_INPUT_LIST; + progressOutputFile = DEFAULT_VALUE_PROGRESS_OUTPUT_FILE; + progressRestoreFile = DEFAULT_VALUE_PROGRESS_RESTORE_FILE; + multipleThreads = DEFAULT_NUM_THREADS == 1 ? false : true; + threadCount = DEFAULT_NUM_THREADS; + progressWriteInterval = DEFAULT_PROGRESS_WRITE_INTERVAL; + } + + /** + * Checks whether there is a non-default amount of entities to skip + * @return True if there are entities to skip + */ + public boolean hasSkip() { + return this.skip != DEFAULT_VALUE_SKIP; + } + + /** + * Checks whether there is a non-default amount of entities to take + * @return True if there are limited entities to take + */ + public boolean hasTake() { + return this.take != DEFAULT_VALUE_TAKE; + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java new file mode 100644 index 0000000..7d43411 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java @@ -0,0 +1,178 @@ +package org.tudo.sse.analyses.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Configuration builder to obtain {@link LibraryAnalysisConfig} instances programmatically. + * + * @author Johannes Düsing + */ +public class LibraryAnalysisConfigBuilder { + + /** + * The logger for this instance. + */ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final LibraryAnalysisConfig theConfig; + + /** + * Creates a new builder with default configuration values. + */ + public LibraryAnalysisConfigBuilder() { + this(new LibraryAnalysisConfig()); + } + + /** + * Creates a new builder with the given baseline configuration instance. + * + * @param theConfig Baseline configuration instance for this builder + */ + protected LibraryAnalysisConfigBuilder(LibraryAnalysisConfig theConfig) { + this.theConfig = theConfig; + } + + /** + * Sets the amount of entities to skip when processing analysis inputs. Can be used for pagination. Defaults to zero. + * + * @param toSkip Amount of entities (artifacts or libraries) to skip + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid. + */ + public LibraryAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException { + if(toSkip < 0) + throw new InvalidConfigurationException("skip", "Skip must not be negative"); + + if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP) + log.warn("Value 'skip' is set multiple times, previous value overridden."); + + this.theConfig.skip = toSkip == 0 ? LibraryAnalysisConfig.DEFAULT_VALUE_SKIP : toSkip; + return this; + } + + /** + * Sets the amount of entities to take when processing analysis inputs. Can be used for pagination. By default, no + * limit is imposed and the input source is consumed fully. + * + * @param toTake Amount of entities (artifacts or libraries) to take + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid. + */ + public LibraryAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException { + if(toTake <= 0) + throw new InvalidConfigurationException("take", "Take must be positive"); + + if(this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE) + log.warn("Value 'take' is set multiple times, previous value overridden."); + + this.theConfig.take = toTake; + return this; + } + + /** + * Sets the input list of entities to process as analysis inputs. This is either a list of GAV triples (artifacts) + * or GA tuples (libraries), one entity per line. By default, the Maven Central index is used as the source of inputs. + * + * @param listPath Path to input file. Must be an existing regular file, one entity per line is required. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withInputList(Path listPath) throws InvalidConfigurationException { + if(listPath == null || !Files.isRegularFile(listPath)) + throw new InvalidConfigurationException("inputs", "Input list file must be a valid file reference"); + + if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST) + log.warn("Value 'inputs' is set multiple times, previous value overridden."); + + this.theConfig.inputListFile = listPath; + return this; + } + + /** + * Sets the output file to regularly write the analysis progress to. By default, a file called 'marin-progress.txt' + * is created in the current workdir. + * + * @param outputFile Path to the progress output file to create. Existing files will be overridden! + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException { + if(outputFile == null) + throw new InvalidConfigurationException("progress-output-file", "Output file cannot be null"); + + if(this.theConfig.progressOutputFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_OUTPUT_FILE) + log.warn("Value 'progress-output-file' is set multiple times, previous value overridden."); + + this.theConfig.progressOutputFile = outputFile; + return this; + } + + /** + * Sets the file to restore progress from. Should be a progress output file previously created by MARIN. By default, + * no progress is restored. + * + * @param restoreFile Path to the progress restore file. Must be a valid text file. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException { + if(restoreFile == null || !Files.isRegularFile(restoreFile)) + throw new InvalidConfigurationException("progress-restore-file", "Restore file must be a valid file reference"); + + if(this.theConfig.progressRestoreFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_RESTORE_FILE) + log.warn("Value 'progress-restore-file' is set multiple times, previous value overridden."); + + this.theConfig.progressRestoreFile = restoreFile; + return this; + } + + /** + * Sets the number of threads to use for analysis. By default, one thread is used. + * @param numThreads Number of threads to use + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException { + if(numThreads <= 0) + throw new InvalidConfigurationException("threads", "Number of threads must be positive"); + + if(this.theConfig.threadCount != LibraryAnalysisConfig.DEFAULT_NUM_THREADS) + log.warn("Value 'threads' is set multiple times, previous value overridden."); + + this.theConfig.threadCount = numThreads; + this.theConfig.multipleThreads = numThreads > 1; + return this; + } + + /** + * Sets the amount of entities (artifacts or libraries) after which to write progress to the progress output file. + * Default is 100. + * @param progressWriteInterval Amount of entities after which to save progress + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException { + if(progressWriteInterval <= 0) + throw new InvalidConfigurationException("save-progress-interval", "Progress write interval must be positive"); + + if(this.theConfig.progressWriteInterval != LibraryAnalysisConfig.DEFAULT_PROGRESS_WRITE_INTERVAL) + log.warn("Value 'progress-write-interval' is set multiple times, previous value overridden."); + + this.theConfig.progressWriteInterval = progressWriteInterval; + return this; + } + + /** + * Builds the configuration as specified by all previous invocations to this instance. + * + * @return The configuration object that has been constructed by this builder. + */ + public LibraryAnalysisConfig build(){ + return this.theConfig; + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java new file mode 100644 index 0000000..6b46233 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -0,0 +1,62 @@ +package org.tudo.sse.analyses.config.parsing; + +import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; + +/** + * Parser for creating {@link ArtifactAnalysisConfig} objects from CLI parameters. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ +public class ArtifactAnalysisConfigParser extends LibraryAnalysisConfigParser implements CLIParsingUtilities { + + /** + * Creates a new artifact config parser instance. + */ + public ArtifactAnalysisConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfig} object. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link ArtifactAnalysisConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ + public ArtifactAnalysisConfig parseArtifactConfig(String[] args) throws CLIException { + final ArtifactAnalysisConfigBuilder configBuilder = new ArtifactAnalysisConfigBuilder(); + + for(int i = 0; i < args.length; i += 2) { + handleArtifactParameter(args, i, configBuilder); + } + + return configBuilder.build(); + } + + + private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfigBuilder configBuilder) throws CLIException { + try { + + switch(args[i]) { + case "-su": + case "--since-until": + final long[] sinceUntil = nextArgAsLongPair(args, i); + configBuilder.withSinceUtil(sinceUntil[0], sinceUntil[1]); + break; + case "-o": + case "--output": + configBuilder.withOutputDirectory(nextArgAsDirectoryReference(args, i)); + break; + default: + super.handleParameter(args, i , configBuilder); + + } + + } catch (InvalidConfigurationException icx) { + CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName()); + wrapped.initCause(icx); + throw wrapped; + } + } + +} diff --git a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java similarity index 98% rename from src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java rename to src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java index 8d67fbc..2c175d1 100644 --- a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java @@ -1,4 +1,4 @@ -package org.tudo.sse.utils; +package org.tudo.sse.analyses.config.parsing; import org.tudo.sse.CLIException; diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java new file mode 100644 index 0000000..ba2b95e --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java @@ -0,0 +1,86 @@ +package org.tudo.sse.analyses.config.parsing; + +import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; + +/** + * Parser for {@link LibraryAnalysisConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ +public class LibraryAnalysisConfigParser implements CLIParsingUtilities { + + /** + * Creates a new library config parsers instance. + */ + public LibraryAnalysisConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link LibraryAnalysisConfig} objects. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link LibraryAnalysisConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ + public LibraryAnalysisConfig parseCommonConfig(String[] args) throws CLIException { + final LibraryAnalysisConfigBuilder configBuilder = new LibraryAnalysisConfigBuilder(); + + + for(int i = 0; i < args.length; i += 2){ + handleParameter(args, i, configBuilder); + } + + return configBuilder.build(); + } + + /** + * Parses a given parameter from the array of CLI arguments and applies it to the config object. + * @param args CLI arguments array + * @param i Current parsing position + * @param configBuilder Current configuration object + * @throws CLIException If parsing the current argument fails. + */ + protected void handleParameter(String[] args, int i, LibraryAnalysisConfigBuilder configBuilder) throws CLIException { + try { + switch (args[i]){ + case "-st": + case "--skip-take": + final long[] skipTake = nextArgAsLongPair(args, i); + configBuilder + .withSkip(skipTake[0]) + .withTake(skipTake[1]); + break; + case "-prf": + case "--progress-restore-file": + configBuilder.withProgressRestoreFile(nextArgAsRegularFileReference(args, i)); + break; + case "-spi": + case "--save-progress-interval": + configBuilder.withProgressWriteInterval(nextArgAsInt(args, i)); + break; + case "-pof": + case "--progress-output-file": + configBuilder.withProgressOutputFile(nextArgAsPath(args, i)); + break; + case "-i": + case "--inputs": + configBuilder.withInputList(nextArgAsRegularFileReference(args, i)); + break; + case "-t": + case "--threads": + final int threads = nextArgAsInt(args, i); + configBuilder.withNumberOfThreads(threads); + break; + default: + throw new CLIException(args[i]); + } + } catch (InvalidConfigurationException icx) { + CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName()); + wrapped.initCause(icx); + throw wrapped; + } + } + + +} diff --git a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java deleted file mode 100644 index 7fde305..0000000 --- a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.CLIException; - -import java.nio.file.Path; - -/** - * Parser for ArtifactConfig objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis}. - * Configuration is obtained by parsing command line arguments provided as an array of strings. - */ -public class ArtifactConfigParser extends CommonConfigParser implements CLIParsingUtilities { - - /** - * Creates a new artifact config parser instance. - */ - public ArtifactConfigParser() { super(); } - - /** - * Parses a given array containing JVM command line arguments into an {@link ArtifactConfig} object. Throws an exception - * if the given arguments are not valid. - * @param args Array of command line arguments, as passed by the JVM - * @return Corresponding {@link ArtifactConfig} if parsing was successful - * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. - */ - public ArtifactConfig parseArtifactConfig(String[] args) throws CLIException { - final ArtifactConfig artifactConfig = new ArtifactConfig(); - - for(int i = 0; i < args.length; i += 2) { - handleArtifactParameter(args, i, artifactConfig); - } - - return artifactConfig; - } - - - private void handleArtifactParameter(String[] args, int i, ArtifactConfig config) throws CLIException { - switch(args[i]) { - case "-su": - case "--since-until": - if(config.skip != -1L) - throw new CLIException("Cannot apply time-based filtering when pagination (-st) is used", args[i]); - if(config.since != -1L) - throw new CLIException("Values for since and until cannot be set twice!", args[i]); - if(config.inputListFile != null) - throw new CLIException("Cannot apply time-based filtering when input list (-i) is used", args[i]); - - final long[] sinceUntil = nextArgAsLongPair(args, i); - config.since = sinceUntil[0]; - config.until = sinceUntil[1]; - break; - - case "-o": - case "--output": - config.outputEnabled = true; - config.outputDirectory = nextArgAsDirectoryReference(args, i); - break; - - case "-st": - case "--skip-take": - if(config.since != -1L) - throw new CLIException("Cannot apply pagination when time-based filtering (-su) is used", args[i]); - super.handleParameter(args, i, config); - break; - - case "-i": - case "--inputs": - if(config.since != -1L) - throw new CLIException("Cannot use input list when time-based filtering (-su) is applied", args[i]); - super.handleParameter(args, i, config); - break; - - default: - super.handleParameter(args, i , config); - - } - } - - /** - * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. - */ - public static class ArtifactConfig extends CommonConfig { - - /** - * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. - */ - public long since; - - /** - * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. - */ - public long until; - - /** - * Directory to write file outputs to, or null if disabled. - */ - public Path outputDirectory; - - /** - * Whether files shall be written to the output directory. - */ - public boolean outputEnabled; - - /** - * Creates a new configuration object with default values. - */ - public ArtifactConfig(){ - super(); - - since = -1L; - until = -1L; - outputEnabled = false; - outputDirectory = null; - } - - - } - -} diff --git a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java deleted file mode 100644 index 474449e..0000000 --- a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java +++ /dev/null @@ -1,146 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.CLIException; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * Parser for {@link CommonConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. - * Configuration is obtained by parsing command line arguments provided as an array of strings. - */ -public class CommonConfigParser implements CLIParsingUtilities { - - /** - * Creates a new artifact config parsers instance. - */ - public CommonConfigParser() { super(); } - - /** - * Parses a given array containing JVM command line arguments into an {@link CommonConfig} objects. Throws an exception - * if the given arguments are not valid. - * @param args Array of command line arguments, as passed by the JVM - * @return Corresponding {@link CommonConfig} if parsing was successful - * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. - */ - public CommonConfig parseCommonConfig(String[] args) throws CLIException { - final CommonConfig config = new CommonConfig(); - - - for(int i = 0; i < args.length; i += 2){ - handleParameter(args, i, config); - } - - return config; - } - - /** - * Parses a given parameter from the array of CLI arguments and applies it to the config object. - * @param args CLI arguments array - * @param i Current parsing position - * @param config Current configuration object - * @throws CLIException If parsing the current argument fails. - */ - protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException { - switch (args[i]){ - case "-st": - case "--skip-take": - if(config.skip != -1L) - throw new CLIException("Values for skip and take cannot be set twice!", args[i]); - - final long[] skipTake = nextArgAsLongPair(args, i); - config.skip = skipTake[0]; - config.take = skipTake[1]; - break; - case "-prf": - case "--progress-restore-file": - if(config.progressRestoreFile != null) - throw new CLIException("Progress restore file cannot be set twice!", args[i]); - - config.progressRestoreFile = nextArgAsRegularFileReference(args, i); - break; - case "-spi": - case "--save-progress-interval": - config.progressWriteInterval = nextArgAsInt(args, i); - break; - case "-pof": - case "--progress-output-file": - config.progressOutputFile = nextArgAsPath(args, i); - break; - case "-i": - case "--inputs": - if(config.inputListFile != null) - throw new CLIException("Input file cannot be set twice!", args[i]); - config.inputListFile = nextArgAsRegularFileReference(args, i); - break; - case "-t": - case "--threads": - final int threads = nextArgAsInt(args, i); - config.multipleThreads = threads > 1; - config.threadCount = threads; - break; - default: - throw new CLIException(args[i]); - } - } - - /** - * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis} instance. - */ - public static class CommonConfig { - - /** - * Number of artifacts to skip from the underlying source, or -1 if disabled. - */ - public long skip; - - /** - * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. - */ - public long take; - - /** - * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. - */ - public Path inputListFile; - - /** - * File path to output analysis progress to. - */ - public Path progressOutputFile; - - /** - * Path to read previous progress from, in order to restore it. Null if disabled. - */ - public Path progressRestoreFile; - - /** - * True if multiple threads shall be used, false for single-threaded analysis. - */ - public boolean multipleThreads; - - /** - * Number of threads to use. - */ - public int threadCount; - - /** - * Interval of entities after which progress shall be saved. Defaults to 100. - */ - public int progressWriteInterval; - - /** - * Creates a new configuration with default values. - */ - public CommonConfig() { - skip = -1L; - take = -1L; - inputListFile = null; - progressOutputFile = Paths.get("marin-progress"); - progressRestoreFile = null; - multipleThreads = false; - threadCount = 1; - progressWriteInterval = 100; - } - } -} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 70baa56..422abf1 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -5,12 +5,13 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.Package; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralAnalysisFactory; import scala.Tuple2; @@ -43,11 +44,11 @@ class MavenCentralArtifactAnalysisTest { void parseCLIRegular() throws IOException{ Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{ parseCLI("--skip-take 500:223"), parseCLI("--inputs src/test/resources/localPom.xml"), parseCLI("--progress-restore-file src/test/resources/localPom.xml --inputs src/test/resources/localPom.xml"), - parseCLI("--since-until 53245:13243"), + parseCLI("--since-until 53245:53246"), parseCLI("--inputs src/test/resources/localPom.xml --progress-restore-file src/test/resources/localPom.xml"), parseCLI("--progress-restore-file src/test/resources/localPom.xml"), parseCLI("--output " + tmpDir.toString()) @@ -57,7 +58,7 @@ void parseCLIRegular() throws IOException{ for(int i = 0; i < configs.length; i++){ final List currExpected = expected.get(i); - final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + final ArtifactAnalysisConfig currConfig = configs[i]; assertEquals(asInt(currExpected.get(0)), currConfig.skip); assertEquals(asInt(currExpected.get(1)), currConfig.take); @@ -89,11 +90,11 @@ void parseCLIRegular() throws IOException{ void parseCLIShorthands() throws IOException{ Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{ parseCLI("-st 500:223"), parseCLI("-i src/test/resources/localPom.xml"), parseCLI("-prf src/test/resources/localPom.xml -i src/test/resources/localPom.xml"), - parseCLI("-su 53245:13243"), + parseCLI("-su 53245:53246"), parseCLI("-i src/test/resources/localPom.xml -prf src/test/resources/localPom.xml"), parseCLI("-prf src/test/resources/localPom.xml"), parseCLI("-o " + tmpDir.toString()) @@ -103,7 +104,7 @@ void parseCLIShorthands() throws IOException{ for(int i = 0; i < configs.length; i++){ final List currExpected = expected.get(i); - final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + final ArtifactAnalysisConfig currConfig = configs[i]; assertEquals(asInt(currExpected.get(0)), currConfig.skip); assertEquals(asInt(currExpected.get(1)), currConfig.take); @@ -138,6 +139,7 @@ void parseCmdLineNegative() { cliInputs.add("--inputs -/xcd/"); cliInputs.add("-st 88880000 -su --inputs path/to/file -ip path/to/file"); cliInputs.add("--inputs"); + cliInputs.add("--since-until 53245:13243"); for(String input : cliInputs) { @@ -383,9 +385,9 @@ private long getEndingIndex(Path fileName) { } } - private ArtifactConfigParser.ArtifactConfig parseCLI(String cli) { + private ArtifactAnalysisConfig parseCLI(String cli) { try { - final ArtifactConfigParser parser = new ArtifactConfigParser(); + final ArtifactAnalysisConfigParser parser = new ArtifactAnalysisConfigParser(); if(cli.isBlank()) return parser.parseArtifactConfig(new String[] {}); else return parser.parseArtifactConfig(cli.split(" ")); } catch(CLIException clix){ diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index e5c120b..3ec6545 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -3,8 +3,9 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; -import org.tudo.sse.utils.CommonConfigParser; +import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser; import java.io.IOException; import java.nio.file.Files; @@ -22,7 +23,7 @@ public class MavenCentralLibraryAnalysisTest { @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ final String validArgs = "--skip-take 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --threads 12 --save-progress-interval 20"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assert(config.multipleThreads); assertEquals(12, config.threadCount); @@ -38,7 +39,7 @@ void parseCLIRegular(){ @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") void parseCLIShorthands(){ final String validArgs = "-st 20:10000 -prf pom.xml -pof marin-progress -t 12 -spi 20"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assert(config.multipleThreads); assertEquals(12, config.threadCount); @@ -55,7 +56,7 @@ void parseCLIShorthands(){ @Test @DisplayName("The CLI parser must set the correct defaults for empty arguments") void parseCLIDefaults(){ - final CommonConfigParser.CommonConfig config = parseCLI(""); + final LibraryAnalysisConfig config = parseCLI(""); assertFalse(config.multipleThreads); assertEquals(1, config.threadCount); @@ -80,7 +81,7 @@ void parseNonExistingFile(){ void parseCustomGA(){ final String validArgs = "-st 20:10000 --inputs pom.xml"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assertEquals(Paths.get("pom.xml"), config.inputListFile); assertEquals(20, config.skip); @@ -91,7 +92,7 @@ void parseCustomGA(){ void parseNonExistingIndex(){ final String validArgs = "--progress-restore-file pom.xml --inputs pom.xml"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assertEquals(Paths.get("pom.xml"), config.inputListFile); assertEquals(Paths.get("pom.xml"), config.progressRestoreFile); @@ -336,10 +337,10 @@ private Path testResource(String pathToResource){ return null; } - private CommonConfigParser.CommonConfig parseCLI(String cli) { + private LibraryAnalysisConfig parseCLI(String cli) { try { - if(cli.isBlank()) return new CommonConfigParser().parseCommonConfig(new String[]{}); - else return new CommonConfigParser().parseCommonConfig(cli.split(" ")); + if(cli.isBlank()) return new LibraryAnalysisConfigParser().parseCommonConfig(new String[]{}); + else return new LibraryAnalysisConfigParser().parseCommonConfig(cli.split(" ")); } catch(CLIException clix) { throw new RuntimeException(clix); } diff --git a/src/test/resources/MavenAnalysis.json b/src/test/resources/MavenAnalysis.json index c7e9edd..8879f2b 100644 --- a/src/test/resources/MavenAnalysis.json +++ b/src/test/resources/MavenAnalysis.json @@ -31,7 +31,7 @@ "-1", "-1", "53245", - "13243", + "53246", "null", "null", "false" From 7b7e16b4fd535ef0ed6ec146c40c750cafe11ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 10 Dec 2025 13:53:23 +0100 Subject: [PATCH 34/55] Add artifact analysis iterator implementation (singlethreaded) --- .../org/tudo/sse/analyses/AnalysisUtils.java | 58 ++++ .../MavenCentralArtifactIterator.java | 248 ++++++++++++++++++ .../config/ArtifactAnalysisConfig.java | 8 + .../config/LibraryAnalysisConfig.java | 8 + .../sse/model/ArtifactResolutionContext.java | 10 +- .../ProcessIdentifierMessage.java | 2 +- .../sse/utils/FileBasedArtifactIterator.java | 28 ++ .../MavenCentralArtifactIteratorTest.java | 204 ++++++++++++++ .../MavenCentralLibraryAnalysisTest.java | 9 +- .../org/tudo/sse/utils/TestUtilities.java | 19 ++ src/test/resources/artifact-names-invalid.txt | 10 + 11 files changed, 594 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/AnalysisUtils.java create mode 100644 src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java create mode 100644 src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java create mode 100644 src/test/java/org/tudo/sse/utils/TestUtilities.java create mode 100644 src/test/resources/artifact-names-invalid.txt diff --git a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java new file mode 100644 index 0000000..59aad14 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java @@ -0,0 +1,58 @@ +package org.tudo.sse.analyses; + +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; + +/** + * Utility methods for general analysis implementations. + * + * @author Johannes Düsing + */ +interface AnalysisUtils { + + LibraryAnalysisConfig getConfig(); + + /** + * Retrieves the initial position to start analysis from, based on the current configuration. + * @return Initial number of entities to skip before starting + */ + default long getInitialPosition() { + final var config = getConfig(); + + if(config.progressRestoreFile != null) return getRestoreProgressValue(); + else if(config.hasSkip()) return config.skip; + else return 0L; + } + + /** + * Retrieves the last progress value to start from based on a previous run. + * @return Progress value + */ + default long getRestoreProgressValue() { + BufferedReader indexReader; + try { + indexReader = new BufferedReader(new FileReader(getConfig().progressRestoreFile.toFile())); + String line = indexReader.readLine(); + return Integer.parseInt(line); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Writes the current position to the specified progress output file. + * @param currentPosition The current progress to write + */ + default void writePosition(long currentPosition) { + final var config = getConfig(); + + try(BufferedWriter writer = Files.newBufferedWriter(config.progressOutputFile)) { + writer.write(Long.toString(currentPosition)); + } catch(IOException ignored) {} + } +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java new file mode 100644 index 0000000..741f126 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -0,0 +1,248 @@ +package org.tudo.sse.analyses; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactResolutionContext; +import org.tudo.sse.model.index.IndexInformation; +import org.tudo.sse.resolution.ResolverFactory; +import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.util.Iterator; + +/** + * Iterator that produces artifacts that are enriched with index-, pom- or jar information, as configured. This is the + * iterator equivalent to the {@link MavenCentralArtifactAnalysis}. Only supports single-threaded resolution. + * + * @author Johannes Düsing + */ +@SuppressWarnings("this-escape") +public class MavenCentralArtifactIterator implements Iterator, AnalysisUtils { + + private final Logger log = LoggerFactory.getLogger(MavenCentralArtifactIterator.class); + private final ArtifactAnalysisConfig config; + + private Iterator source; + + private boolean badSource = false; + + private long currentPosition = 0L; + private long artifactsTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + /** + * Creates a new iterator instance with the given configuration values. + * + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder} + */ + public MavenCentralArtifactIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, ArtifactAnalysisConfig config) { + this.config = config; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + if(this.config.multipleThreads){ + log.warn("Artifact iterator does no support multiple threads - using a single thread for resolution"); + } + + this.setUnderlyingSource(); + + if(this.config.outputEnabled){ + this.resolverFactory = new ResolverFactory(true, this.config.outputDirectory, resolveTransitivePoms); + } else { + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + } + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(badSource) return false; + + // If the config specifies a limited amount of artifacts to take, and that amount is reached, we have no next element + if(this.config.hasTake() && this.artifactsTaken >= this.config.take) + return false; + + // Otherwise, we have a next artifact if the underlying source has a next element + try { + return this.source.hasNext(); + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + + } + + @Override + public Artifact next(){ + if(!hasNext()) + throw new IllegalStateException("No more artifacts available"); + + final ArtifactResolutionContext ctx = this.source.next(); + final Artifact artifact = ctx.getRootArtifact(); + + if(this.resolvePom){ + resolverFactory.runPom(artifact.getIdent(), ctx); + } + + if(this.resolveJar){ + resolverFactory.runJar(artifact.getIdent(), ctx); + } + + this.artifactsTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return artifact; + } + + @Override + public LibraryAnalysisConfig getConfig(){ + return this.config; + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ + this.writePosition(this.currentPosition); + this.lastPositionSaved = this.currentPosition; + } + } + + private void skipInitial(){ + final long entitiesToSkip = getInitialPosition(); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } + + + private void setUnderlyingSource(){ + if(this.config.hasInputList()){ + try{ + this.source = new MavenCentralCustomListArtifactSource(); + } catch(IOException | IllegalArgumentException x){ + log.error("The given input list file is invalid", x); + this.badSource = true; + } + } else { + try { + this.source = new MavenCentralIndexArtifactSource(); + } catch(IOException uix){ + log.error("Failed to access Maven Central Index", uix); + this.badSource = true; + } + + } + } + + private class MavenCentralIndexArtifactSource implements Iterator { + + private final IndexIterator index; + + private boolean _needsUpdate = true; + private boolean _hasNext = false; + private IndexInformation _nextInfo = null; + + MavenCentralIndexArtifactSource() throws IOException { + this.index = new IndexIterator(MavenCentralRepository.RepoBaseURI); + } + + private void findNext(){ + _hasNext = false; + while(!_hasNext && index.hasNext()){ + var currentInfo = index.next(); + if(currentInfo != null && isValidInfo(currentInfo)){ + _hasNext = true; + _nextInfo = currentInfo; + } + } + } + + private boolean isValidInfo(IndexInformation indexInfo){ + if(!config.hasTimeBasedFiltering()) return true; + else { + long timeStamp = indexInfo.getLastModified(); + return config.since <= timeStamp && timeStamp <= config.until; + } + } + + @Override + public boolean hasNext() { + if(_needsUpdate){ + findNext(); + _needsUpdate = false; + } + return _hasNext; + } + + @Override + public ArtifactResolutionContext next() { + if(hasNext()){ + _needsUpdate = true; + + final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(_nextInfo.getIdent()); + ctx.getRootArtifact().setIndexInformation(_nextInfo); + + return ctx; + } else throw new IllegalStateException("Call to next on empty artifact source"); + } + } + + private class MavenCentralCustomListArtifactSource implements Iterator { + + private final FileBasedArtifactIterator list; + + MavenCentralCustomListArtifactSource() throws IOException { + list = new FileBasedArtifactIterator(config.inputListFile); + list.validateInput(); + } + + @Override + public boolean hasNext() { + return list.hasNext(); + } + + @Override + public ArtifactResolutionContext next() { + if(hasNext()){ + return ArtifactResolutionContext.newInstance(list.next()); + } else throw new IllegalStateException("Call to next on empty artifact source"); + } + + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java index 1c1545c..f2f3dd8 100644 --- a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -45,5 +45,13 @@ public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { outputDirectory = DEFAULT_VALUE_OUTPUT; } + /** + * Checks whether there is a custom time range to filter artifacts for. + * @return True if time based filtering is enabled + */ + public boolean hasTimeBasedFiltering(){ + return this.since != DEFAULT_VALUE_SINCE && this.until != DEFAULT_VALUE_UNTIL; + } + } diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java index 23c1b7e..5341b6d 100644 --- a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java @@ -87,4 +87,12 @@ public boolean hasSkip() { public boolean hasTake() { return this.take != DEFAULT_VALUE_TAKE; } + + /** + * Checks whether there is a custom input list of entities to process. + * @return True if there is an input list specified + */ + public boolean hasInputList() { + return this.inputListFile != DEFAULT_VALUE_INPUT_LIST; + } } diff --git a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java index be261f1..eb6ec37 100644 --- a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java @@ -26,8 +26,16 @@ public static ArtifactResolutionContext newInstance(ArtifactIdent identifier) { * Returns the identifier of the artifact for which this context was created. * @return ArtifactIdent for this context */ - public ArtifactIdent getIdentifier() { + public ArtifactIdent getRootArtifactIdentifier() { return identifier; } + /** + * Returns the artifact for which this context was initially created + * @return The root artifact of this context + */ + public Artifact getRootArtifact() { + return this.createArtifact(getRootArtifactIdentifier()); + } + } diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index 5e68bdf..2ebe7d6 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -28,7 +28,7 @@ public ProcessIdentifierMessage(ArtifactResolutionContext resolutionContext, Mav * @return The artifact identifier */ public ArtifactIdent getIdentifier() { - return this.artifactCtx.getIdentifier(); + return this.artifactCtx.getRootArtifactIdentifier(); } /** diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java index c769515..8e646c9 100644 --- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java +++ b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java @@ -1,5 +1,6 @@ package org.tudo.sse.utils; +import org.eclipse.aether.transfer.ArtifactNotFoundException; import org.tudo.sse.model.ArtifactIdent; import java.io.IOException; @@ -25,6 +26,33 @@ public FileBasedArtifactIterator(Path input){ this.inputPath = input; } + /** + * Checks whether the underlying file exists and contains only valid GAV triples. + * @throws IOException If accessing the underlying file fails + * @throws IllegalArgumentException If the underlying file contents are not valid + */ + public void validateInput() throws IOException, IllegalArgumentException { + var lines = Files.readAllLines(inputPath); + int lineCnt = 0; + + for(String line: lines){ + final String[] parts = line.split(":"); + if(parts.length != 3){ + throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + + for(String part: parts){ + if(part.isBlank()){ + throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + } + + lineCnt += 1; + } + + this.lineIterator = lines.iterator(); + } + @Override public boolean hasNext() { if(this.lineIterator == null){ diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java new file mode 100644 index 0000000..cc54a26 --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java @@ -0,0 +1,204 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MarinOpalLogger; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; + +public class MavenCentralArtifactIteratorTest { + + private final Path gavInputList = Paths.get("src/main/resources/coordinates.txt"); + + @BeforeAll + static void setUp() { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + + @Test + @DisplayName("The artifact iterator must process all inputs from custom GAV lists") + void readFromFile() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, false, true, config); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertFalse(current.hasIndexInformation()); + assertTrue(current.hasPomInformation()); + assertTrue(current.hasJarInformation()); + + artifactCount++; + } + + assertEquals(10, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must apply pagination to custom GAV lists") + void readFromFilePaginated() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .withSkip(2) + .withTake(5) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, true, false, config); + + ArtifactIdent firstEntry = getIdentInInputFile(0); + ArtifactIdent secondEntry = getIdentInInputFile(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertFalse(current.hasIndexInformation()); + assertTrue(current.hasPomInformation()); + assertFalse(current.hasJarInformation()); + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getIdent()); + assertNotEquals(secondEntry, current.getIdent()); + + artifactCount++; + } + + assertEquals(5, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must not produce any values on invalid input files") + void readFromFileInvalid() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(testResource("artifact-names-invalid.txt")) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, true, true, config); + + // If the source file contains at least one line that is not a valid GAV triple, it must not produce any elements + assertFalse(iterator.hasNext()); + assertThrows(IllegalStateException.class, iterator::next); + } + + @Test + @DisplayName("The artifact iterator must apply pagination to the Central index") + void readFromIndexPaginated() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withSkip(2) + .withTake(5) + .build(); + + var iterator = new MavenCentralArtifactIterator(false, false, true, config); + + ArtifactIdent firstEntry = getIdentInIndex(0); + ArtifactIdent secondEntry = getIdentInIndex(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertTrue(current.hasIndexInformation()); + assertFalse(current.hasPomInformation()); + assertTrue(current.hasJarInformation()); + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getIdent()); + assertNotEquals(secondEntry, current.getIdent()); + + artifactCount++; + } + + assertEquals(5, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must apply time range filtering to the Central index") + void readFromIndexFiltered() throws InvalidConfigurationException { + // We expect two entries in this range + long since = 1106902436000L; + long until = 1106902438001L; + + var config = new ArtifactAnalysisConfigBuilder() + .withSinceUtil(since, until) + .build(); + + var iterator = new MavenCentralArtifactIterator(false, false, false, config); + + int artifactCount = 0; + + while(iterator.hasNext() && artifactCount < 2){ + Artifact current = iterator.next(); + assertTrue(current.hasIndexInformation()); + assertFalse(current.hasPomInformation()); + assertFalse(current.hasJarInformation()); + + assertTrue(since <= current.getIndexInformation().getLastModified()); + assertTrue(current.getIndexInformation().getLastModified() <= until); + + assertEquals("ymsg", current.getIdent().getGroupID()); + + artifactCount++; + } + } + + private ArtifactIdent getIdentInInputFile(int pos) { + try { + var lines = Files.readAllLines(gavInputList); + + if(pos >= lines.size()) fail("Invalid position in input list: " + pos); + + var parts = lines.get(pos).split(":"); + + if(parts.length != 3) fail("Invalid GAV in input list: " + pos); + + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } catch (IOException iox) { + fail(iox); + } + + return null; + } + + private ArtifactIdent getIdentInIndex(int pos){ + try { + IndexIterator ii = new IndexIterator(MavenCentralRepository.RepoBaseURI); + + int position = 0; + + while(ii.hasNext()){ + var ident = ii.next().getIdent(); + if(position == pos){ + return ident; + } + position++; + } + + ii.closeReader(); + + } catch (IOException iox) { + fail(iox); + } + + return null; + } +} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index 3ec6545..8ca7d1f 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -16,6 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; public class MavenCentralLibraryAnalysisTest { @@ -328,14 +329,6 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertFalse(librariesHit.contains("yom:yom")); } - private Path testResource(String pathToResource){ - try { - return Path.of(Objects.requireNonNull(getClass().getClassLoader().getResource(pathToResource)).toURI()); - } catch (Exception x){ - fail("Test setup: Failed to load resource file " + pathToResource, x); - } - return null; - } private LibraryAnalysisConfig parseCLI(String cli) { try { diff --git a/src/test/java/org/tudo/sse/utils/TestUtilities.java b/src/test/java/org/tudo/sse/utils/TestUtilities.java new file mode 100644 index 0000000..7e7a686 --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/TestUtilities.java @@ -0,0 +1,19 @@ +package org.tudo.sse.utils; + +import java.nio.file.Path; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.fail; + +public class TestUtilities { + + public static Path testResource(String pathToResource){ + try { + return Path.of(Objects.requireNonNull(TestUtilities.class.getClassLoader().getResource(pathToResource)).toURI()); + } catch (Exception x){ + fail("Test setup: Failed to load resource file " + pathToResource, x); + } + return null; + } + +} diff --git a/src/test/resources/artifact-names-invalid.txt b/src/test/resources/artifact-names-invalid.txt new file mode 100644 index 0000000..827f980 --- /dev/null +++ b/src/test/resources/artifact-names-invalid.txt @@ -0,0 +1,10 @@ +org.springframework.boot:spring-boot-starter-web:2.6.3 +org.apache.commons:commons-lang3:3.12.0 +com.fasterxml.jackson.core:jackson-databind +junit:junit:4.13.2 +org.slf4j:slf4j-api:1.7.32 +org.hibernate:hibernate-core:5.6.5.Final +org.apache.httpcomponents:httpclient:4.5.13 +org.springframework:spring-core:5.3.15 +mysql:mysql-connector-java:8.0.28:: +org.apache.logging.log4j:log4j-core:2.17.1 \ No newline at end of file From 96aa62f714fa258e33bf96651b2fe7e08e24c2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 12 Dec 2025 10:21:36 +0100 Subject: [PATCH 35/55] Finish analysis iterator for libraries, some restructuring to remove duplication --- .../org/tudo/sse/analyses/AnalysisUtils.java | 24 +- .../MavenCentralArtifactAnalysis.java | 2 +- .../MavenCentralArtifactIterator.java | 24 +- .../analyses/MavenCentralLibraryIterator.java | 280 ++++++++++++++++++ .../input/AbstractInputFileIterator.java | 91 ++++++ .../input/FileBasedArtifactIterator.java | 42 +++ .../input/FileBasedLibraryIterator.java | 38 +++ .../sse/utils/FileBasedArtifactIterator.java | 81 ----- .../MavenCentralLibraryIteratorTest.java | 229 ++++++++++++++ .../resources/library-names-not-existing.txt | 1 + 10 files changed, 707 insertions(+), 105 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java create mode 100644 src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java create mode 100644 src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java create mode 100644 src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java delete mode 100644 src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java create mode 100644 src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java create mode 100644 src/test/resources/library-names-not-existing.txt diff --git a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java index 59aad14..d1d221f 100644 --- a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java +++ b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java @@ -1,6 +1,7 @@ package org.tudo.sse.analyses; import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -13,30 +14,30 @@ * * @author Johannes Düsing */ -interface AnalysisUtils { - - LibraryAnalysisConfig getConfig(); +final class AnalysisUtils { /** * Retrieves the initial position to start analysis from, based on the current configuration. + * @param config The current analysis configuration + * * @return Initial number of entities to skip before starting */ - default long getInitialPosition() { - final var config = getConfig(); - - if(config.progressRestoreFile != null) return getRestoreProgressValue(); + static long getInitialPosition(LibraryAnalysisConfig config) { + if(config.progressRestoreFile != null) return getRestoreProgressValue(config); else if(config.hasSkip()) return config.skip; else return 0L; } /** * Retrieves the last progress value to start from based on a previous run. + * @param config The current analysis configuration + * * @return Progress value */ - default long getRestoreProgressValue() { + static long getRestoreProgressValue(LibraryAnalysisConfig config) { BufferedReader indexReader; try { - indexReader = new BufferedReader(new FileReader(getConfig().progressRestoreFile.toFile())); + indexReader = new BufferedReader(new FileReader(config.progressRestoreFile.toFile())); String line = indexReader.readLine(); return Integer.parseInt(line); } catch (IOException e) { @@ -47,10 +48,9 @@ default long getRestoreProgressValue() { /** * Writes the current position to the specified progress output file. * @param currentPosition The current progress to write + * @param config The current analysis configuration */ - default void writePosition(long currentPosition) { - final var config = getConfig(); - + static void writePosition(long currentPosition, LibraryAnalysisConfig config) { try(BufferedWriter writer = Files.newBufferedWriter(config.progressOutputFile)) { writer.write(Long.toString(currentPosition)); } catch(IOException ignored) {} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index bbd7f71..35f7405 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -14,7 +14,7 @@ import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; -import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; import org.tudo.sse.utils.MavenCentralRepository; diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java index 741f126..f5b99db 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -3,12 +3,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; -import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactResolutionContext; import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.resolution.ResolverFactory; -import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -21,8 +20,7 @@ * * @author Johannes Düsing */ -@SuppressWarnings("this-escape") -public class MavenCentralArtifactIterator implements Iterator, AnalysisUtils { +public class MavenCentralArtifactIterator implements Iterator { private final Logger log = LoggerFactory.getLogger(MavenCentralArtifactIterator.class); private final ArtifactAnalysisConfig config; @@ -119,20 +117,15 @@ public Artifact next(){ return artifact; } - @Override - public LibraryAnalysisConfig getConfig(){ - return this.config; - } - private void writePositionIfNeeded(){ if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ - this.writePosition(this.currentPosition); + AnalysisUtils.writePosition(this.currentPosition, this.config); this.lastPositionSaved = this.currentPosition; } } private void skipInitial(){ - final long entitiesToSkip = getInitialPosition(); + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); if(entitiesToSkip > 0L && this.badSource) return; @@ -176,6 +169,7 @@ private class MavenCentralIndexArtifactSource implements Iterator { + + private final Logger log = LoggerFactory.getLogger(MavenCentralLibraryIterator.class); + private final LibraryAnalysisConfig config; + + private Iterator source; + + private boolean badSource = false; + private boolean sourceClosed = false; + + private long currentPosition = 0L; + private long librariesTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + private final IReleaseListProvider releaseListProvider; + private Consumer failedLibraryCallback = null; + + /** + * Creates a new iterator instance with the given configuration values. + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder} + */ + public MavenCentralLibraryIterator(boolean resolvePom, + boolean resolveTransitivePoms, + boolean resolveJar, + LibraryAnalysisConfig config) { + this.config = config; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + if(this.config.multipleThreads){ + log.warn("Library iterator does no support multiple threads - using a single thread for resolution"); + } + + this.setUnderlyingSource(); + + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(this.badSource) return false; + + // If the config specifies a limited amount of libraries to take, and that amount is reached, we have no next element + if(this.config.hasTake() && this.librariesTaken >= this.config.take){ + closeSource(); + return false; + } + + // Otherwise, we have a next library if the underlying source has a next element + try { + boolean hasNext = this.source.hasNext(); + + if(!hasNext) + closeSource(); + + return hasNext; + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + } + + @Override + public LibraryResolutionContext next() { + if(!hasNext()) + throw new IllegalStateException("No more libraries available"); + + final String nextGA = this.source.next(); + final String[] gaParts = nextGA.split(":"); // Valid tuple, ensured by source + final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(nextGA); + + // Get list of all library releases + final List allReleases = getReleaseIdentifiers(gaParts[0], gaParts[1]); + + if(allReleases != null){ + // For each release, build artifact and enrich with data + for(ArtifactIdent release : allReleases){ + final Artifact artifact = ctx.createArtifact(release); + ctx.addLibraryArtifact(artifact); + + if(this.resolvePom){ + resolverFactory.runPom(artifact.getIdent(), ctx); + } + + if(this.resolveJar){ + resolverFactory.runJar(artifact.getIdent(), ctx); + } + } + } else { + // If we failed to obtain a release list, the respective callback has been invoked. Here, we just return an + // empty context. + log.warn("No releases for library {}, returning empty context", nextGA); + } + + this.librariesTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return ctx; + } + + /** + * Sets the callback that is invoked when obtaining a release list for a given library fails. + * @param failedLibraryCallback Callback on failed libraries + */ + public void setFailedLibraryCallback(Consumer failedLibraryCallback){ + this.failedLibraryCallback = failedLibraryCallback; + } + + private List getReleaseIdentifiers(String groupId, String artifactId){ + try { + final List libraryVersions = releaseListProvider.getReleases(groupId, artifactId); + List identifiers = new ArrayList<>(); + for(String libraryVersion : libraryVersions){ + identifiers.add(new ArtifactIdent(groupId, artifactId, libraryVersion)); + } + return identifiers; + } catch (Exception x) { + log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + + if(this.failedLibraryCallback != null){ + final var event = new FailedLibraryEvent(groupId, artifactId, x); + try { this.failedLibraryCallback.accept(event); } catch (Exception ex) { + log.warn("Unexcepted exception in failed analysis callback for library {}:{}", groupId, artifactId, ex); + } + } + + return null; + } + } + + private void skipInitial(){ + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ + AnalysisUtils.writePosition(this.currentPosition, this.config); + this.lastPositionSaved = this.currentPosition; + } + } + + private void setUnderlyingSource(){ + if(this.config.hasInputList()){ + try { + var iterator = new FileBasedLibraryIterator(this.config.inputListFile); + iterator.validateInput(); + this.source = iterator; + } catch (IOException | IllegalArgumentException x){ + log.error("The given input list file is invalid", x); + this.badSource = true; + } + } else { + try { + this.source = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + } catch (IOException iox){ + log.error("Failed to access Maven Central Index", iox); + this.badSource = true; + } + + } + } + + private void closeSource(){ + if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ + try {((AutoCloseable)this.source).close();} + catch (Exception ignored){} + this.sourceClosed = true; + } + } + + /** + * Event that is thrown when obtaining a library's release list fails. + */ + public static class FailedLibraryEvent { + + private final String groupId; + private final String artifactId; + private final Throwable cause; + + /** + * Creates a new event instance. + * @param groupId The library group id + * @param artifactId The library artifact id + * @param cause The throwable that caused the failure + */ + FailedLibraryEvent(String groupId, String artifactId, Throwable cause){ + this.groupId = groupId; + this.artifactId = artifactId; + this.cause = cause; + } + + /** + * Get the failed library's groupID + * @return Maven groupID + */ + public String getGroupId() { + return this.groupId; + } + + /** + * Get the failed library's artifactID + * @return Maven artifactID + */ + public String getArtifactId() { + return this.artifactId; + } + + /** + * Get the failure cause + * @return Throwable that caused the failure + */ + public Throwable getCause() { + return this.cause; + } + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java new file mode 100644 index 0000000..d3df78b --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java @@ -0,0 +1,91 @@ +package org.tudo.sse.analyses.input; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; + +/** + * An iterator that wraps an underlying file that contains line-wise representations of entities. This wrapper produces + * one entity per line, with parsing logic defined by concrete subclasses. + * + * @param The type that entities are parsed to + * + * @author Johannes Düsing + */ +abstract class AbstractInputFileIterator implements Iterator { + + /** + * Checks whether a given line within the input file represents a valid entity + * @param line The line to check + * @return True is the line represents a valid entity, false otherwise + */ + protected abstract boolean isValidLine(String line); + + /** + * Parses a given line into its representing entity. Will only ever be called after isValidLine returned true. + * @param line The line to parse + * @return The resulting entity + */ + protected abstract T parseLine(String line); + + private final Path inputPath; + + private Iterator lineIterator; + + /** + * Creates a new iterator instance for the given file path. + * @param input Path to a text file containing one entity per line + */ + protected AbstractInputFileIterator(Path input){ + this.inputPath = input; + } + + /** + * Checks whether the underlying file exists and contains only valid entities. + * @throws IOException If accessing the underlying file fails + * @throws IllegalArgumentException If the underlying file contents are not valid + */ + public void validateInput() throws IOException, IllegalArgumentException { + var lines = Files.readAllLines(inputPath); + int lineCnt = 0; + + for(String line: lines){ + if(!isValidLine(line)){ + throw new IllegalArgumentException("Not a valid entity: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + + lineCnt += 1; + } + + this.lineIterator = lines.iterator(); + } + + @Override + public boolean hasNext() { + if(this.lineIterator == null){ + try { + var lines = Files.readAllLines(this.inputPath); + this.lineIterator = lines.iterator(); + } catch (IOException iox){ + throw new IllegalArgumentException("Failed to access input file", iox); + } + } + + return this.lineIterator.hasNext(); + } + + @Override + public T next() { + if(!hasNext()) + throw new IllegalStateException("Next on empty iterator"); + + String line = this.lineIterator.next(); + + if(isValidLine(line)) + return parseLine(line); + else + throw new IllegalStateException("Not a valid entity: " + line); + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java new file mode 100644 index 0000000..4d646a8 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java @@ -0,0 +1,42 @@ +package org.tudo.sse.analyses.input; + +import org.tudo.sse.model.ArtifactIdent; + +import java.nio.file.Path; + +/** + * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of + * groupID:artifactID:version per line. + */ +public class FileBasedArtifactIterator extends AbstractInputFileIterator { + + /** + * Creates a new artifact identifier iterator for the given input file. + * @param gavList Path to input file + */ + public FileBasedArtifactIterator(Path gavList) { + super(gavList); + } + + @Override + protected boolean isValidLine(String line) { + var parts = line.split(":"); + + if(parts.length != 3) + return false; + + for(String part: parts){ + if(part.isBlank()) + return false; + } + + return true; + } + + @Override + protected ArtifactIdent parseLine(String line) { + var parts = line.split(":"); + + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } +} diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java new file mode 100644 index 0000000..a8138e6 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java @@ -0,0 +1,38 @@ +package org.tudo.sse.analyses.input; + +import java.nio.file.Path; + +/** + * Iterator that reads Maven Central library identifiers from a text file. Expects one colon-separated tuple of + * groupID:artifactID per line. + */ +public class FileBasedLibraryIterator extends AbstractInputFileIterator{ + + /** + * Creates a new library identifier iterator for the given input file. + * @param gaList Path to input list + */ + public FileBasedLibraryIterator(Path gaList) { + super(gaList); + } + + @Override + protected boolean isValidLine(String line) { + var parts = line.split(":"); + + if(parts.length != 2) + return false; + + for(String part: parts){ + if(part.isBlank()) + return false; + } + + return true; + } + + @Override + protected String parseLine(String line) { + return line; + } +} diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java deleted file mode 100644 index 8e646c9..0000000 --- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.tudo.sse.utils; - -import org.eclipse.aether.transfer.ArtifactNotFoundException; -import org.tudo.sse.model.ArtifactIdent; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Iterator; - -/** - * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of - * groupID:artifactID:version per line. - */ -public class FileBasedArtifactIterator implements Iterator { - - private final Path inputPath; - - private Iterator lineIterator; - - /** - * Creates a new iterator instance for the given file path. - * @param input Path to a text file containing GAV triples - */ - public FileBasedArtifactIterator(Path input){ - this.inputPath = input; - } - - /** - * Checks whether the underlying file exists and contains only valid GAV triples. - * @throws IOException If accessing the underlying file fails - * @throws IllegalArgumentException If the underlying file contents are not valid - */ - public void validateInput() throws IOException, IllegalArgumentException { - var lines = Files.readAllLines(inputPath); - int lineCnt = 0; - - for(String line: lines){ - final String[] parts = line.split(":"); - if(parts.length != 3){ - throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); - } - - for(String part: parts){ - if(part.isBlank()){ - throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); - } - } - - lineCnt += 1; - } - - this.lineIterator = lines.iterator(); - } - - @Override - public boolean hasNext() { - if(this.lineIterator == null){ - try { - var lines = Files.readAllLines(this.inputPath); - this.lineIterator = lines.iterator(); - } catch (IOException iox){ - throw new IllegalArgumentException("Failed to access input file", iox); - } - } - - return this.lineIterator.hasNext(); - } - - @Override - public ArtifactIdent next() { - String line = this.lineIterator.next(); - - String[] parts = line.split(":"); - if(parts.length == 3) { - return new ArtifactIdent(parts[0], parts[1], parts[2]); - } else { - throw new IllegalStateException("Not a valid GAV-Triple: " + line); - } - } -} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java new file mode 100644 index 0000000..47daa0b --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java @@ -0,0 +1,229 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.LibraryResolutionContext; +import org.tudo.sse.resolution.FileNotFoundException; +import org.tudo.sse.utils.LibraryIndexIterator; +import org.tudo.sse.utils.MarinOpalLogger; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; + +public class MavenCentralLibraryIteratorTest { + + private static final Path validGAList = testResource("library-names-valid.txt"); + private static final Path invalidGAList = testResource("library-names-invalid.txt"); + private static final Path nonExistingGAList = testResource("library-names-not-existing.txt"); + + @BeforeAll + static void setUp() { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + + @Test + @DisplayName("The library iterator must process all inputs from custom GA lists") + void readFromFile() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(validGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + int libraryCount = 0; + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertTrue(release.hasPomInformation()); + assertFalse(release.hasJarInformation()); + } + + + libraryCount++; + } + + assertEquals(5, libraryCount); + } + + @Test + @DisplayName("The library iterator must apply pagination to custom GA lists") + void readFromFilePaginated() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(validGAList) + .withTake(2) + .withSkip(2) + .build(); + + var iterator = new MavenCentralLibraryIterator(false, false, true, config); + + int libraryCount = 0; + + String firstEntry = getGaInInputFile(0); + String secondEntry = getGaInInputFile(1); + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertFalse(release.hasPomInformation()); + assertTrue(release.hasJarInformation()); + } + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getLibraryGA()); + assertNotEquals(secondEntry, current.getLibraryGA()); + + libraryCount++; + } + + assertEquals(2, libraryCount); + } + + @Test + @DisplayName("The library iterator must not produce any values on invalid input files") + void readFromFileInvalid() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(invalidGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, true, true, config); + + // If the source file contains at least one line that is not a valid GA tuple, it must not produce any elements + assertFalse(iterator.hasNext()); + assertThrows(IllegalStateException.class, iterator::next); + } + + @Test + @DisplayName("The library iterator must apply pagination to the Central index") + void readFromIndexPaginated() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withSkip(2) + .withTake(3) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + String firstEntry = getGaInIndex(0); + String secondEntry = getGaInIndex(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertTrue(release.hasPomInformation()); + assertFalse(release.hasJarInformation()); + } + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getLibraryGA()); + assertNotEquals(secondEntry, current.getLibraryGA()); + + artifactCount++; + } + + assertEquals(3, artifactCount); + } + + @Test + @DisplayName("The library iterator must invoke the registered callback on failure") + void setCallback() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(nonExistingGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + // Define a callback for non-existing release lists + AtomicBoolean callbackInvoked = new AtomicBoolean(false); + + iterator.setFailedLibraryCallback( event -> { + assertEquals("no", event.getGroupId()); + assertEquals("lib", event.getArtifactId()); + + Throwable rootCause = event.getCause(); + while(rootCause.getCause() != null){ + rootCause = rootCause.getCause(); + } + + assertInstanceOf(FileNotFoundException.class, rootCause); + + callbackInvoked.set(true); + }); + + while(iterator.hasNext()){ + LibraryResolutionContext ctx = iterator.next(); + assertTrue(ctx.getLibraryArtifacts().isEmpty()); + } + + assertTrue(callbackInvoked.get()); + } + + + private String getGaInIndex(int pos) { + try { + LibraryIndexIterator libIt = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + + int position = 0; + while(libIt.hasNext()){ + var libGA = libIt.next(); + if(position == pos){ + return libGA; + } + position++; + } + + libIt.close(); + } catch(IOException iox) { + fail(iox); + } + + return null; + } + + private String getGaInInputFile(int pos) { + try { + assertNotNull(validGAList); + + var lines = Files.readAllLines(validGAList); + + if(pos >= lines.size()) fail("Invalid position in input list: " + pos); + + return lines.get(pos); + } catch (IOException iox) { + fail(iox); + } + + return null; + } +} diff --git a/src/test/resources/library-names-not-existing.txt b/src/test/resources/library-names-not-existing.txt new file mode 100644 index 0000000..300a466 --- /dev/null +++ b/src/test/resources/library-names-not-existing.txt @@ -0,0 +1 @@ +no:lib From 83c100e3f135e440d313798de5e92f527a9e4da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 12 Dec 2025 11:49:12 +0100 Subject: [PATCH 36/55] Refractor new iterators to extract common functionality --- .../sse/analyses/AbstractEntityIterator.java | 190 ++++++++++++++++++ .../MavenCentralArtifactIterator.java | 129 ++---------- .../analyses/MavenCentralLibraryIterator.java | 138 ++----------- 3 files changed, 223 insertions(+), 234 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java diff --git a/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java new file mode 100644 index 0000000..8d97a2b --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java @@ -0,0 +1,190 @@ +package org.tudo.sse.analyses; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ResolutionContext; +import org.tudo.sse.resolution.ResolverFactory; + +import java.util.Iterator; + +/** + * An abstract base class for analysis-equivalent iterators. This class assumes an underlying source iterator that + * produces a generic kind of source object (an "identifier" for the entity), and then transforms the source object into + * a target entity of generic type. The iterator enforces pagination and supports progress dumps and progress restores. + * + * @param Type of source objects as produced by the underlying iterator + * @param Type of target objects that are created by this iterator + * + * @author Johannes Düsing + */ +abstract class AbstractEntityIterator implements Iterator { + + /** + * Logger instance for this iterator + */ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + /** + * The base configuration for this iterator instance - subclasses may have a more specific configuration type. + */ + protected final LibraryAnalysisConfig baseConfig; + + /** + * Flag indicating whether the underlying source is invalid, i.e. because a connection attempt failed + */ + protected boolean badSource = false; + + private final Iterator source; + + private boolean sourceClosed = false; + + private long currentPosition = 0L; + private long entitiesTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + /** + * Builds the underlying source iterator for this entity iterator instance. + * @return The source iterator + */ + protected abstract Iterator buildSource(); + + /** + * Transforms a source object into the target entity. + * @param source The source object as produced by the underlying iterator + * @return The target entity + */ + protected abstract T buildEntity(S source); + + /** + * Creates a new entity iterator instance with the given configuration. Immediately sets the underlying source and + * skips all initial entities according to the configuration. + * + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param baseConfig The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder} + */ + AbstractEntityIterator(boolean resolvePom, + boolean resolveTransitivePoms, + boolean resolveJar, + LibraryAnalysisConfig baseConfig){ + this.baseConfig = baseConfig; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + this.source = buildSource(); + + if(baseConfig instanceof ArtifactAnalysisConfig && ((ArtifactAnalysisConfig)this.baseConfig).outputEnabled){ + this.resolverFactory = new ResolverFactory(true, + ((ArtifactAnalysisConfig)this.baseConfig).outputDirectory, + resolveTransitivePoms); + } else { + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + } + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public final boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(this.badSource) return false; + + // If the config specifies a limited amount of entities to take, and that amount is reached, we have no next element + if(this.baseConfig.hasTake() && this.entitiesTaken >= this.baseConfig.take){ + closeSourceIfNeeded(); + return false; + } + + // Otherwise, we have a next entity if the underlying source has a next element + try { + boolean hasNext = this.source.hasNext(); + + if(!hasNext) + closeSourceIfNeeded(); + + return hasNext; + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + } + + @Override + public final T next() { + if(!hasNext()) + throw new IllegalStateException("No more entities available"); + + final S sourceEntity = this.source.next(); + final T entity = this.buildEntity(sourceEntity); + + this.entitiesTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return entity; + } + + /** + * Enriches the given artifact with ArtifactInformation as defined by the current configuration. + * @param a The artifact to enrich + * @param ctx The current artifact resolution context + */ + protected final void enrichArtifact(Artifact a, ResolutionContext ctx){ + if(this.resolvePom) + resolverFactory.runPom(a.getIdent(), ctx); + + if(this.resolveJar) + resolverFactory.runJar(a.getIdent(), ctx); + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.baseConfig.progressWriteInterval){ + AnalysisUtils.writePosition(this.currentPosition, this.baseConfig); + this.lastPositionSaved = this.currentPosition; + } + } + + private void closeSourceIfNeeded(){ + if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ + try {((AutoCloseable)this.source).close();} + catch(Exception ignored){} + this.sourceClosed = true; + } + } + + private void skipInitial(){ + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.baseConfig); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java index f5b99db..e064e57 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -1,12 +1,9 @@ package org.tudo.sse.analyses; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactResolutionContext; import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -20,22 +17,7 @@ * * @author Johannes Düsing */ -public class MavenCentralArtifactIterator implements Iterator { - - private final Logger log = LoggerFactory.getLogger(MavenCentralArtifactIterator.class); - private final ArtifactAnalysisConfig config; - - private Iterator source; - - private boolean badSource = false; - - private long currentPosition = 0L; - private long artifactsTaken = 0L; - private long lastPositionSaved = 0L; - - private final ResolverFactory resolverFactory; - private final boolean resolvePom; - private final boolean resolveJar; +public final class MavenCentralArtifactIterator extends AbstractEntityIterator { /** * Creates a new iterator instance with the given configuration values. @@ -46,120 +28,45 @@ public class MavenCentralArtifactIterator implements Iterator { * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder} */ public MavenCentralArtifactIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, ArtifactAnalysisConfig config) { - this.config = config; - this.resolvePom = resolvePom; - this.resolveJar = resolveJar; + super(resolvePom, resolveTransitivePoms, resolveJar, config); - if(this.config.multipleThreads){ + if(this.baseConfig.multipleThreads){ log.warn("Artifact iterator does no support multiple threads - using a single thread for resolution"); } - - this.setUnderlyingSource(); - - if(this.config.outputEnabled){ - this.resolverFactory = new ResolverFactory(true, this.config.outputDirectory, resolveTransitivePoms); - } else { - this.resolverFactory = new ResolverFactory(resolveTransitivePoms); - } - - try { - if(!this.badSource) - this.skipInitial(); - } catch(Exception x){ - log.error("Failed to initialize iterator", x); - this.badSource = true; - } } @Override - public boolean hasNext() { - // If the underlying source is invalid, we have no next element - if(badSource) return false; - - // If the config specifies a limited amount of artifacts to take, and that amount is reached, we have no next element - if(this.config.hasTake() && this.artifactsTaken >= this.config.take) - return false; - - // Otherwise, we have a next artifact if the underlying source has a next element - try { - return this.source.hasNext(); - } catch(Exception x){ - // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no - // next element. - log.error("Failed to access source", x); - this.badSource = true; - return false; - } - - } - - @Override - public Artifact next(){ - if(!hasNext()) - throw new IllegalStateException("No more artifacts available"); - - final ArtifactResolutionContext ctx = this.source.next(); + protected Artifact buildEntity(ArtifactResolutionContext ctx){ final Artifact artifact = ctx.getRootArtifact(); - if(this.resolvePom){ - resolverFactory.runPom(artifact.getIdent(), ctx); - } - - if(this.resolveJar){ - resolverFactory.runJar(artifact.getIdent(), ctx); - } - - this.artifactsTaken += 1L; - this.currentPosition += 1L; - - writePositionIfNeeded(); + this.enrichArtifact(artifact, ctx); return artifact; } - private void writePositionIfNeeded(){ - if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ - AnalysisUtils.writePosition(this.currentPosition, this.config); - this.lastPositionSaved = this.currentPosition; - } - } - - private void skipInitial(){ - final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); - - if(entitiesToSkip > 0L && this.badSource) - return; - - for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ - this.source.next(); - this.currentPosition += 1L; - } - - log.debug("Successfully skipped to position {}", this.currentPosition); - - if(!this.source.hasNext()){ - log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); - } - } - - private void setUnderlyingSource(){ - if(this.config.hasInputList()){ + @Override + protected Iterator buildSource(){ + if(this.baseConfig.hasInputList()){ try{ - this.source = new MavenCentralCustomListArtifactSource(); + return new MavenCentralCustomListArtifactSource(); } catch(IOException | IllegalArgumentException x){ log.error("The given input list file is invalid", x); this.badSource = true; } } else { try { - this.source = new MavenCentralIndexArtifactSource(); + return new MavenCentralIndexArtifactSource(); } catch(IOException uix){ log.error("Failed to access Maven Central Index", uix); this.badSource = true; } - } + return null; + } + + private ArtifactAnalysisConfig getConfig(){ + return (ArtifactAnalysisConfig) this.baseConfig; } private class MavenCentralIndexArtifactSource implements Iterator { @@ -187,10 +94,10 @@ private void findNext(){ } private boolean isValidInfo(IndexInformation indexInfo){ - if(!config.hasTimeBasedFiltering()) return true; + if(!getConfig().hasTimeBasedFiltering()) return true; else { long timeStamp = indexInfo.getLastModified(); - return config.since <= timeStamp && timeStamp <= config.until; + return getConfig().since <= timeStamp && timeStamp <= getConfig().until; } } @@ -229,7 +136,7 @@ private class MavenCentralCustomListArtifactSource implements Iterator { - - private final Logger log = LoggerFactory.getLogger(MavenCentralLibraryIterator.class); - private final LibraryAnalysisConfig config; - - private Iterator source; - - private boolean badSource = false; - private boolean sourceClosed = false; - - private long currentPosition = 0L; - private long librariesTaken = 0L; - private long lastPositionSaved = 0L; - - private final ResolverFactory resolverFactory; - private final boolean resolvePom; - private final boolean resolveJar; +public final class MavenCentralLibraryIterator extends AbstractEntityIterator { private final IReleaseListProvider releaseListProvider; private Consumer failedLibraryCallback = null; @@ -57,64 +38,19 @@ public MavenCentralLibraryIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, LibraryAnalysisConfig config) { - this.config = config; - this.resolvePom = resolvePom; - this.resolveJar = resolveJar; + super(resolvePom, resolveTransitivePoms, resolveJar, config); - if(this.config.multipleThreads){ + if(this.baseConfig.multipleThreads){ log.warn("Library iterator does no support multiple threads - using a single thread for resolution"); } - this.setUnderlyingSource(); - - this.resolverFactory = new ResolverFactory(resolveTransitivePoms); this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); - - try { - if(!this.badSource) - this.skipInitial(); - } catch(Exception x){ - log.error("Failed to initialize iterator", x); - this.badSource = true; - } } @Override - public boolean hasNext() { - // If the underlying source is invalid, we have no next element - if(this.badSource) return false; - - // If the config specifies a limited amount of libraries to take, and that amount is reached, we have no next element - if(this.config.hasTake() && this.librariesTaken >= this.config.take){ - closeSource(); - return false; - } - - // Otherwise, we have a next library if the underlying source has a next element - try { - boolean hasNext = this.source.hasNext(); - - if(!hasNext) - closeSource(); - - return hasNext; - } catch(Exception x){ - // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no - // next element. - log.error("Failed to access source", x); - this.badSource = true; - return false; - } - } - - @Override - public LibraryResolutionContext next() { - if(!hasNext()) - throw new IllegalStateException("No more libraries available"); - - final String nextGA = this.source.next(); - final String[] gaParts = nextGA.split(":"); // Valid tuple, ensured by source - final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(nextGA); + protected LibraryResolutionContext buildEntity(String ga){ + final String[] gaParts = ga.split(":"); // Valid tuple, ensured by source + final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(ga); // Get list of all library releases final List allReleases = getReleaseIdentifiers(gaParts[0], gaParts[1]); @@ -125,25 +61,13 @@ public LibraryResolutionContext next() { final Artifact artifact = ctx.createArtifact(release); ctx.addLibraryArtifact(artifact); - if(this.resolvePom){ - resolverFactory.runPom(artifact.getIdent(), ctx); - } - - if(this.resolveJar){ - resolverFactory.runJar(artifact.getIdent(), ctx); - } + this.enrichArtifact(artifact, ctx); } } else { // If we failed to obtain a release list, the respective callback has been invoked. Here, we just return an // empty context. - log.warn("No releases for library {}, returning empty context", nextGA); + log.warn("No releases for library {}, returning empty context", ga); } - - this.librariesTaken += 1L; - this.currentPosition += 1L; - - writePositionIfNeeded(); - return ctx; } @@ -177,58 +101,26 @@ private List getReleaseIdentifiers(String groupId, String artifac } } - private void skipInitial(){ - final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); - - if(entitiesToSkip > 0L && this.badSource) - return; - - for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ - this.source.next(); - this.currentPosition += 1L; - } - - log.debug("Successfully skipped to position {}", this.currentPosition); - - if(!this.source.hasNext()){ - log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); - } - } - - private void writePositionIfNeeded(){ - if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ - AnalysisUtils.writePosition(this.currentPosition, this.config); - this.lastPositionSaved = this.currentPosition; - } - } - - private void setUnderlyingSource(){ - if(this.config.hasInputList()){ + @Override + protected Iterator buildSource(){ + if(this.baseConfig.hasInputList()){ try { - var iterator = new FileBasedLibraryIterator(this.config.inputListFile); + var iterator = new FileBasedLibraryIterator(this.baseConfig.inputListFile); iterator.validateInput(); - this.source = iterator; + return iterator; } catch (IOException | IllegalArgumentException x){ log.error("The given input list file is invalid", x); this.badSource = true; } } else { try { - this.source = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + return new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); } catch (IOException iox){ log.error("Failed to access Maven Central Index", iox); this.badSource = true; } - - } - } - - private void closeSource(){ - if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ - try {((AutoCloseable)this.source).close();} - catch (Exception ignored){} - this.sourceClosed = true; } + return null; } /** From 94c9f31b47e9ec22f836eb88150e81b3aa72ec84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 12 Dec 2025 11:56:48 +0100 Subject: [PATCH 37/55] Move resource file to test directory --- .../MavenCentralArtifactAnalysisTest.java | 16 ++++++++-------- .../MavenCentralArtifactIteratorTest.java | 3 +-- .../resources/artifact-names-valid.txt} | 0 3 files changed, 9 insertions(+), 10 deletions(-) rename src/{main/resources/coordinates.txt => test/resources/artifact-names-valid.txt} (100%) diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 422abf1..b5fb274 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -238,17 +238,17 @@ void indexProcessorProgress() { @DisplayName("An analysis must correctly apply CLI options when reading custom input lists") void readIdentsIn() { List cliInputs = new ArrayList<>(); - String[] args = {"--inputs", "src/main/resources/coordinates.txt"}; + String[] args = {"--inputs", "src/test/resources/artifact-names-valid.txt"}; cliInputs.add(args); - args = new String[] {"--inputs", "src/main/resources/coordinates.txt", "-pof", "src/test/resources/stop.txt"}; + args = new String[] {"--inputs", "src/test/resources/artifact-names-valid.txt", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); List> expected = (List>) json.get("readIdentsIn"); @@ -292,8 +292,8 @@ void checkMultiThreading() { singleArgs.add(new String[]{"-st", "10:1000"}); multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"}); - singleArgs.add(new String[]{"--inputs", "src/main/resources/coordinates.txt"}); - multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/main/resources/coordinates.txt"}); + singleArgs.add(new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt"}); + multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/test/resources/artifact-names-valid.txt"}); for(int i = 0; i < singleArgs.size(); i++) { diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java index cc54a26..438bef6 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java @@ -16,14 +16,13 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import static org.junit.jupiter.api.Assertions.*; import static org.tudo.sse.utils.TestUtilities.testResource; public class MavenCentralArtifactIteratorTest { - private final Path gavInputList = Paths.get("src/main/resources/coordinates.txt"); + private final Path gavInputList = testResource("artifact-names-valid.txt"); @BeforeAll static void setUp() { diff --git a/src/main/resources/coordinates.txt b/src/test/resources/artifact-names-valid.txt similarity index 100% rename from src/main/resources/coordinates.txt rename to src/test/resources/artifact-names-valid.txt From f39f665c5b55570665a2aea56ecb1e442ad160f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Tue, 16 Dec 2025 12:48:14 +0100 Subject: [PATCH 38/55] Add iterators to README --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 1344acb..b8a20ab 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,14 @@ Both analysis types provide a method called `void runAnalysis(String[] args)`. I | `-o `
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | +If you do not want to extend one of the aforementioned base classes, MARIN also provides corresponding implementations of the `java.util.Iterator` interface to enumerate and enrich artifacts or libraries. Note that those implementations are **not threadsafe** and cannot perform resolution in parallel. MARIN provides: +* The `MavenCentralArtifactIterator extends Iterator` is the equivalent to the `MavenCentralArtifactAnalysis` and supports all configuration options besides `--threads`. +* The `MavenCentralLibraryIterator extends Iterator` is the equivalent to the `MavenCentralLibraryAnalysis` and supports all configuration options besides `--threads`. + ## Usage +When using MARIN, you can decide whether to extend one of the abstract base classes available, or to rely on of the analysis-equivalent implementations of `java.util.Iterator`. + +### Extending Abstract Base Classes To use MARIN, you will need to implement two components: 1. You need an implementation of either the abstract class `MavenCentralArtifactAnalysis` or `MavenCentralLibraryAnalysis`. This is your actual analysis implementation that defines how a single artifact or library shall be processed. 2. You need a runner class that passes command line arguments to your analysis implementation. Usually, this will look like this: @@ -64,6 +71,28 @@ public class AnalysisRunner { Once this is implemented, you can run your analysis using the following command. Any of the above-mentioned CLI arguments will work for your analysis implementation. ```java -jar executableName *INSERT CLI HERE* ``` +### Using Iterators +As opposed to extending a base class, in order to use MARIN's iterator implementations you will have to first create an analysis configuration object programmatically. This can be done using the `ArtifactAnalysisConfigBuilder` or `LibraryAnalysisConfigBuilder` classes, respectively. The following example shows how to first build a configuration, and then use it to initialize a `MavenCentralArtifactIterator`. +```java +final boolean resolvePom = true; +final boolean resolveTransitive = false; +final boolean resolveJar = true; +final Path gavInputList = Paths.get("path-to-input-file"); + +final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .withSkip(2) + .withTake(5) + .build(); + +MavenCentralArtifactIterator iterator = new MavenCentralArtifactIterator(resolvePom, resolveTransitive, resolveJar, config); + +while(iterator.hasNext()){ + Artifact current = iterator.next(); + // TODO: Process artifact +} +``` + ## Example Use Cases: In the following, there are some example implementations of `MavenCentralArtifactAnalysis`. All of them can be used with the same `AnalysisRunner` implementation seen above, just replace `MyAnalysisImplementation` with the actual implementation name. You can run each example on the first 1000 Maven artifacts by invoking `java -jar executableName -st 0:1000` for the respective project executable JAR. From b71f2518b273b879ed2f5de01bd0e0bf0e363c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Tue, 16 Dec 2025 16:22:50 +0100 Subject: [PATCH 39/55] Move to OPAL 7.0.0 and Scala 3. Fix vulnerable logback release --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 835dd5c..c838afa 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ 11 UTF-8 - 2.13 + 3 @@ -78,7 +78,7 @@ ch.qos.logback logback-classic - 1.5.13 + 1.5.19 test @@ -111,7 +111,7 @@ de.opal-project framework_${scala.binary.version} - 6.0.0 + 7.0.0 From 70d6ef4a3fc9849d03ec32033f22d158711af940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 7 Jan 2026 14:12:38 +0100 Subject: [PATCH 40/55] Add semantic version parsing and sorting infrastructure --- .../sse/semver/SemanticVersionNumber.java | 229 ++++++++++++++++++ .../sse/semver/SemanticVersionParser.java | 193 +++++++++++++++ .../SemanticVersionParsingException.java | 34 +++ .../sse/semver/SemanticVersionNumberTest.java | 68 ++++++ .../sse/semver/SemanticVersionParserTest.java | 74 ++++++ 5 files changed, 598 insertions(+) create mode 100644 src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java create mode 100644 src/main/java/org/tudo/sse/semver/SemanticVersionParser.java create mode 100644 src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java create mode 100644 src/test/java/org/tudo/sse/semver/SemanticVersionNumberTest.java create mode 100644 src/test/java/org/tudo/sse/semver/SemanticVersionParserTest.java diff --git a/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java b/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java new file mode 100644 index 0000000..50402de --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java @@ -0,0 +1,229 @@ +package org.tudo.sse.semver; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public class SemanticVersionNumber implements Comparable { + + private final int majorVersion; + private final int minorVersion; + private final int patchVersion; + + private final String preReleaseData; + private final String buildMetadata; + + private List _parsedPreReleases = null; + + public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersion) { + this(majorVersion, minorVersion, patchVersion, null); + } + + public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersion, String preReleaseData) { + this(majorVersion, minorVersion, patchVersion, preReleaseData, null); + } + + public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersion, String preReleaseData, String buildMetadata) { + this.majorVersion = majorVersion; + this.minorVersion = minorVersion; + this.patchVersion = patchVersion; + + this.preReleaseData = preReleaseData; + this.buildMetadata = buildMetadata; + } + + public int getMajorVersion(){ + return this.majorVersion; + } + + public int getMinorVersion(){ + return this.minorVersion; + } + + public int getPatchVersion(){ + return this.patchVersion; + } + + public boolean hasPreRelease(){ + return this.preReleaseData != null; + } + public boolean hasBuildMetadata(){ + return this.buildMetadata != null; + } + + public String getPreRelease() { + return this.preReleaseData; + } + public String getBuildMetadata() { + return this.buildMetadata; + } + + public final int compareTo(SemanticVersionNumber other){ + // The precedence of semantic version V2 numbers is defined here: https://semver.org/#spec-item-11 + // Note that build metadata is irrelevant to precedence + // Major version takes precedence + if(this.majorVersion < other.majorVersion) return -1; + else if(this.majorVersion > other.majorVersion) return 1; + else { + // Minor version is second deciding factor + if(this.minorVersion < other.minorVersion) return -1; + else if(this.minorVersion > other.minorVersion) return 1; + else { + // Patch version is third deciding factor + if(this.patchVersion < other.patchVersion) return -1; + else if(this.patchVersion > other.patchVersion) return 1; + else { + // If major, minor and patch are equal, we proceed as follows: + // - if both versions have no prerelease data, they are equal + // - if one version has prerelease data and the other has not, prerelease data takes *lower* precedence, i.e. 1.0.0-alpha < 1.0.0 + if(!this.hasPreRelease() && !other.hasPreRelease()) return 0; + else if(!this.hasPreRelease() && other.hasPreRelease()) return 1; + else if(this.hasPreRelease() && !other.hasPreRelease()) return -1; + else { + // If both versions have prerelease data, we must parse this data. SemVer specifies that it may + // consist of dot-separated identifiers that are either numeric or textual. + List thisParts = this.getParsedPreReleaseParts(); + List otherParts = other.getParsedPreReleaseParts(); + + // The first difference in prerelease parts decides precedence + int pos = 0; + while(pos < thisParts.size() && pos < otherParts.size()) { + int compareResult = thisParts.get(pos).compareTo(otherParts.get(pos)); + if(compareResult != 0) return compareResult; + pos += 1; + } + + // If we do not find a difference for indices valid in both lists: Fewer parts equal less precedence + return thisParts.size() - otherParts.size(); + } + } + } + } + } + + private List getParsedPreReleaseParts(){ + if(this._parsedPreReleases == null) { + this._parsedPreReleases = this.parsePreReleaseData(); + } + + return this._parsedPreReleases; + } + + private List parsePreReleaseData(){ + List preReleaseParts = new ArrayList<>(); + + StringBuilder current = new StringBuilder(); + for(int i = 0; i < preReleaseData.length(); i++){ + char c = preReleaseData.charAt(i); + if(c == '.'){ + String currValue = current.toString(); + + try { + int currInt = Integer.parseInt(currValue); + preReleaseParts.add(new PreReleasePart(currInt)); + } catch(NumberFormatException ignored){ + preReleaseParts.add(new PreReleasePart(currValue)); + } + current.setLength(0); + } else { + current.append(c); + } + } + + String currValue = current.toString(); + + try { + int currInt = Integer.parseInt(currValue); + preReleaseParts.add(new PreReleasePart(currInt)); + } catch(NumberFormatException ignored){ + preReleaseParts.add(new PreReleasePart(currValue)); + } + + return preReleaseParts; + } + + /** + * Class representing a part of a prerelase identifier. The SemVer v2 spec allows prerelease identifiers to be composed + * of dot-separated parts, which can either be numbers or strings. This class implements their comparison logic + * in compliance with semantic versioning rules. + */ + private static final class PreReleasePart implements Comparable{ + + private final String stringPart; + private final Integer intPart; + + PreReleasePart(String value){ + this.stringPart = value; + this.intPart = null; + } + + PreReleasePart(int value){ + this.stringPart = null; + this.intPart = value; + } + + boolean isNum() { + return intPart != null; + } + + int numValue(){ + if(isNum()) return intPart; + else throw new IllegalStateException("Not a number"); + } + + boolean isString(){ + return stringPart != null; + } + + String stringValue(){ + if(isString()) return stringPart; + else throw new IllegalStateException("Not a string"); + } + + + @Override + public int compareTo(PreReleasePart other) { + // If both parts are numbers, they are compared numerically + if(this.isNum() && other.isNum()) return this.numValue() - other.numValue(); + // Numeric identifiers have lower precedence than text + else if(this.isNum()) return -1; + else if(other.isNum()) return 1; + else return this.stringValue().compareTo(other.stringValue()); + } + } + + public static SemanticVersionNumber parse(String value) throws SemanticVersionParsingException { + return SemanticVersionParser.parseNumber(value); + } + + public static Optional tryParse(String value) { + try { + SemanticVersionNumber number = parse(value); + return Optional.of(number); + } catch (SemanticVersionParsingException svpx){ + return Optional.empty(); + } + } + + @Override + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append(majorVersion); + sb.append("."); + sb.append(minorVersion); + sb.append("."); + sb.append(patchVersion); + + if(this.hasPreRelease()){ + sb.append("-"); + sb.append(preReleaseData); + } + + if(this.hasBuildMetadata()){ + sb.append("+"); + sb.append(buildMetadata); + } + + return sb.toString(); + } +} diff --git a/src/main/java/org/tudo/sse/semver/SemanticVersionParser.java b/src/main/java/org/tudo/sse/semver/SemanticVersionParser.java new file mode 100644 index 0000000..524a29a --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/SemanticVersionParser.java @@ -0,0 +1,193 @@ +package org.tudo.sse.semver; + +class SemanticVersionParser { + + static SemanticVersionNumber parseNumber(String value) throws SemanticVersionParsingException { + if(value.isBlank()) throw new SemanticVersionParsingException(value, "Semantic version cannot be empty", 0); + + ParsingState state = ParsingState.NUMBER; + int parsingPosition = 0; + StringBuilder currentValue = new StringBuilder(); + + int currNumberPartIdx = 0; + int[] numberParts = new int[] {-1, 0, 0}; + + String preRelease = null; + String buildMetadata = null; + + final char[] characters = value.toCharArray(); + + for(char currentChar: characters) { + + switch(state){ + case NUMBER: + if(currentChar != '.' && currentChar != '-' && currentChar != '+') { + currentValue.append(currentChar); + } else { + if(currentValue.length() == 0) + throw new SemanticVersionParsingException(value, "Numeric identifier must have at least one digit", parsingPosition); + + final String numValue = currentValue.toString(); + currentValue.setLength(0); + + if(!isNumericIdentifier(numValue)) + throw new SemanticVersionParsingException(value, "Not a valid numeric identifier", parsingPosition); + + numberParts[currNumberPartIdx] = asInt(numValue, value, parsingPosition); + + if(currentChar == '.'){ + if(currNumberPartIdx == 2) + throw new SemanticVersionParsingException(value, "Semantic version cannot have more than three parts", parsingPosition); + + currNumberPartIdx += 1; + } else if(currentChar == '-'){ + state = ParsingState.PRERELEASE; + } else { + state = ParsingState.BUILDMETADATA; + } + } + break; + + case PRERELEASE: + if(currentChar != '+'){ + currentValue.append(currentChar); + } else { + final String preReleaseValue = currentValue.toString(); + currentValue.setLength(0); + + if(!isValidPreRelease(preReleaseValue)) + throw new SemanticVersionParsingException(value, "Not a valid prerelease identifier", parsingPosition); + + preRelease = preReleaseValue; + + state = ParsingState.BUILDMETADATA; + } + break; + case BUILDMETADATA: + currentValue.append(currentChar); + + break; + } + + parsingPosition += 1; + } + + String remaining = currentValue.toString(); + + switch(state){ + case NUMBER: + numberParts[currNumberPartIdx] = asInt(remaining, value, parsingPosition); + break; + case PRERELEASE: + if(!isValidPreRelease(remaining)) + throw new SemanticVersionParsingException(value, "Not a valid prerelease identifier", parsingPosition); + + preRelease = remaining; + break; + case BUILDMETADATA: + if(!isValidBuild(remaining)) + throw new SemanticVersionParsingException(value, "Not a valid build identifier", parsingPosition); + + buildMetadata = remaining; + } + + return new SemanticVersionNumber(numberParts[0], numberParts[1], numberParts[2], preRelease, buildMetadata); + } + + private static boolean isValidBuild(String value){ + if(value.isBlank()) return false; + + final String[] parts = value.split("\\."); + + for(String part: parts){ + if(!isAlphanumericIdentifier(part) && !isDigits(part)) + return false; + } + + return true; + } + + private static boolean isValidPreRelease(String value){ + if(value.isBlank()) return false; + + final String[] parts = value.split("\\."); + + for(String part: parts){ + if(!isAlphanumericIdentifier(part) && !isNumericIdentifier(part)) + return false; + } + + return true; + } + + private static boolean isAlphanumericIdentifier(String value) { + if(value.isEmpty()) return false; + + boolean hasNonDigit = false; + + for(char c : value.toCharArray()) { + if(isNonDigit(c)) hasNonDigit = true; + + if(!isIdentifierCharacter(c)) return false; + } + + return hasNonDigit; + } + + private static boolean isNumericIdentifier(String value){ + if(value.isEmpty()) return false; + if(value.equals("0")) return true; + + char[] chars = value.toCharArray(); + + if(!isPositiveDigit(chars[0])) return false; + + for(int i = 1; i < chars.length; i++){ + if(!isDigit(chars[i])) return false; + } + + return true; + } + + private static boolean isDigits(String value){ + if(value.isEmpty()) return false; + for(char c : value.toCharArray()){ + if(!isDigit(c)) return false; + } + return true; + } + + private static boolean isIdentifierCharacter(char c){ + return isDigit(c) || isNonDigit(c); + } + + private static boolean isNonDigit(char c){ + return c == '-' || isLetter(c); + } + + private static boolean isLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + private static boolean isPositiveDigit(char c){ + return (c >= '1' && c <= '9'); + } + + private static boolean isDigit(char c) { + return (c >= '0' && c <= '9'); + } + + private static int asInt(String value, String number, int pos) throws SemanticVersionParsingException { + try { + return Integer.parseInt(value); + } catch (NumberFormatException nfx) { + throw new SemanticVersionParsingException(number, "Expected an integer", pos); + } + } + + private enum ParsingState { + NUMBER, + PRERELEASE, + BUILDMETADATA + } +} diff --git a/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java b/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java new file mode 100644 index 0000000..d3d2f81 --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java @@ -0,0 +1,34 @@ +package org.tudo.sse.semver; + +public class SemanticVersionParsingException extends Exception { + + private final String parsedValue; + private final String msg; + private final int parsingPosition; + + public SemanticVersionParsingException(String value, String description) { + this(value, description, -1); + } + + public SemanticVersionParsingException(String value, String description, int parsingPosition) { + this.parsedValue = value; + this.msg = description; + this.parsingPosition = parsingPosition; + } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder("Error parsing value '"); + sb.append(parsedValue); + sb.append("'"); + + if(parsingPosition != -1){ + sb.append(" at position ").append(parsingPosition); + } + + sb.append(" : "); + sb.append(msg); + return sb.toString(); + } + +} diff --git a/src/test/java/org/tudo/sse/semver/SemanticVersionNumberTest.java b/src/test/java/org/tudo/sse/semver/SemanticVersionNumberTest.java new file mode 100644 index 0000000..3e926c4 --- /dev/null +++ b/src/test/java/org/tudo/sse/semver/SemanticVersionNumberTest.java @@ -0,0 +1,68 @@ +package org.tudo.sse.semver; + + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SemanticVersionNumberTest { + + private final String[] validNumbersSimple = new String[]{"1.0.0", "3.0.0", "1.2.0", "1.1.1", "1054.20000.1"}; + private final int[] validNumbersSimple_Rank = new int[] {0, 3, 2, 1, 4}; + + private final String[] preReleasesSimple = new String[]{"1.0.0", "1.0.0-gamma-1", "1.0.0-beta", "1.0.0-alpha-1"}; + private final int[] preReleasesSimple_Rank = new int[] {3, 2, 1, 0}; + + // This test case is taken from: https://semver.org/#spec-item-11 + private final String[] preReleaseParts = new String[]{"1.0.0", "1.0.0-rc.1", "1.0.0-beta.11", "1.0.0-beta.2", "1.0.0-beta", "1.0.0-alpha.beta", "1.0.0-alpha.1", "1.0.0-alpha"}; + private final int[] preReleaseParts_Rank = new int[] {7,6,5,4,3,2,1,0}; + + @Test + @DisplayName("Simple version numbers must be sorted correctly") + public void sortSimple(){ + assertSorted(validNumbersSimple, validNumbersSimple_Rank); + } + + @Test + @DisplayName("Prereleases must be sorted correctly") + public void sortPrereleases(){ + assertSorted(preReleasesSimple, preReleasesSimple_Rank); + } + + @Test + @DisplayName("Prerelease parts must be parsed and sorted correctly") + public void sortPrereleaseParts(){ + assertSorted(preReleaseParts, preReleaseParts_Rank); + } + + private void assertSorted(String[] numbers, int[] ranks){ + var actual = sortSemantic(numbers); + + for(int i = 0; i < numbers.length; i++){ + var expectedNum = numbers[i]; + var expectedPos = ranks[i]; + + assertEquals(expectedNum, actual.get(expectedPos).toString()); + } + } + + private List sortSemantic(String[] numbers) { + List list = new ArrayList<>(); + + for(String number : numbers){ + var semVerOpt = SemanticVersionNumber.tryParse(number); + assert(semVerOpt.isPresent()); + list.add(semVerOpt.get()); + } + + Collections.sort(list); + + return list; + } + +} diff --git a/src/test/java/org/tudo/sse/semver/SemanticVersionParserTest.java b/src/test/java/org/tudo/sse/semver/SemanticVersionParserTest.java new file mode 100644 index 0000000..b6a0071 --- /dev/null +++ b/src/test/java/org/tudo/sse/semver/SemanticVersionParserTest.java @@ -0,0 +1,74 @@ +package org.tudo.sse.semver; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class SemanticVersionParserTest { + + private final String[] invalidSyntaxNumbers = new String []{"1a.2.3", "1.2.3.4", "foo", "1+a-n+c", "1..2", "1.0.0-"}; + private final String integerOverflow = "2131231231231231231231231231231231231231231231.1"; + + private final String[] validNumbersSimple = new String[]{"1.0.0", "3.0.0", "1.2.0", "1.1.1", "1054.20000.1"}; + private final int[][] validNumbersSimple_Expected = new int[][]{new int[]{1,0,0}, new int[]{3,0,0}, new int[]{1,2,0}, new int[]{1,1,1}, new int[]{1054,20000,1}}; + + private final String[] validNumbersComplex = new String[]{"1-a-valid-prerelase123+000000", "1.2.3+0a-def-----", "12-0+1", "0.0.0-0+0"}; + private final int[][] validNumbersComplex_Expected_Numbers = new int[][]{new int[]{1,0,0}, new int[]{1,2,3}, new int[]{12,0,0}, new int[]{0,0,0}}; + private final String[][] validNumbersComplex_Expected_Data = new String[][]{new String[]{"a-valid-prerelase123", "000000"}, new String[]{null, "0a-def-----"}, new String[]{"0","1"}, new String[]{"0", "0"}}; + + @Test + @DisplayName("Invalid syntax should lead to parsing exceptions") + public void parseInvalidSyntax(){ + for(String invalidSyntaxNumber: invalidSyntaxNumbers){ + assertThrows(SemanticVersionParsingException.class, () -> SemanticVersionNumber.parse(invalidSyntaxNumber)); + } + } + + @Test + @DisplayName("Integer overflows should lead to parsing exceptions") + public void parseOverflow(){ + assertThrows(SemanticVersionParsingException.class, () -> SemanticVersionNumber.parse(integerOverflow)); + } + + @Test + @DisplayName("Simple numbers should be parsed without exception") + public void parseSimpleNumbers(){ + try { + for(int i = 0; i < validNumbersSimple.length; i++){ + var semVer = SemanticVersionNumber.parse(validNumbersSimple[i]); + var expected = validNumbersSimple_Expected[i]; + + assertEquals(expected[0], semVer.getMajorVersion()); + assertEquals(expected[1], semVer.getMinorVersion()); + assertEquals(expected[2], semVer.getPatchVersion()); + } + } catch (SemanticVersionParsingException svpx) { + fail(svpx); + } + } + + @Test + @DisplayName("Complex numbers should be parsed without exception") + public void parseComplexNumbers(){ + try { + for(int i = 0; i < validNumbersComplex.length; i++){ + var semVer = SemanticVersionNumber.parse(validNumbersComplex[i]); + var expectedNumbers = validNumbersComplex_Expected_Numbers[i]; + var expectedData = validNumbersComplex_Expected_Data[i]; + + assertEquals(expectedNumbers[0], semVer.getMajorVersion()); + assertEquals(expectedNumbers[1], semVer.getMinorVersion()); + assertEquals(expectedNumbers[2], semVer.getPatchVersion()); + + assertEquals(expectedData[0], semVer.getPreRelease()); + assertEquals(expectedData[1], semVer.getBuildMetadata()); + } + } catch(SemanticVersionParsingException svpx){ + fail(svpx); + } + } + +} From b1e7eecf240c47ffa009d45bd47b72ba3ef5897f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 7 Jan 2026 15:25:18 +0100 Subject: [PATCH 41/55] Add semantic version numbers to artifact idents, add JavaDoc --- .../org/tudo/sse/model/ArtifactIdent.java | 51 +++++++++++ .../sse/semver/SemanticVersionNumber.java | 91 +++++++++++++++++++ .../SemanticVersionParsingException.java | 43 +++++++++ 3 files changed, 185 insertions(+) diff --git a/src/main/java/org/tudo/sse/model/ArtifactIdent.java b/src/main/java/org/tudo/sse/model/ArtifactIdent.java index 09ee7c3..78b7721 100644 --- a/src/main/java/org/tudo/sse/model/ArtifactIdent.java +++ b/src/main/java/org/tudo/sse/model/ArtifactIdent.java @@ -5,6 +5,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.tudo.sse.semver.SemanticVersionNumber; +import org.tudo.sse.semver.SemanticVersionParsingException; import org.tudo.sse.utils.MavenCentralRepository; @@ -47,6 +49,11 @@ public class ArtifactIdent { */ private String customRepository; + private boolean didParseSemVer = false; + private boolean semVerValid = false; + private SemanticVersionNumber semVer = null; + private SemanticVersionParsingException semVerException = null; + private static final Logger log = LoggerFactory.getLogger(ArtifactIdent.class); /** @@ -135,6 +142,9 @@ public String getVersion() { public void setVersion(String version) { this.version = version; this.GAV = groupID + ":" + artifactID + ":" + version; + + this.semVer = null; + this.didParseSemVer = false; } /** @@ -209,6 +219,47 @@ public URI getMavenCentralXMLUri() { } } + /** + * Parses this identifier's version according to the semantic versioning 2.0.0 standard. Returns the parsed number, + * or throws an exception if the version does not comply to the required syntax. Note that the result of this method + * invocation is cached, meaning that future calls will return the same parsed semantic version or throw the same + * exception - parsing is only ever attempted once. + * + * @return Semantic version number if parsing was successful. + * @throws SemanticVersionParsingException If version does not adhere to required syntax + */ + public SemanticVersionNumber getSemanticVersion() throws SemanticVersionParsingException { + + if(this.didParseSemVer && this.semVerValid) return this.semVer; + if(this.didParseSemVer) throw this.semVerException; + + try { + this.didParseSemVer = true; + this.semVer = SemanticVersionNumber.parse(this.version); + this.semVerValid = true; + } catch (SemanticVersionParsingException svpx){ + this.semVerException = svpx; + this.semVerValid = false; + throw this.semVerException; + } + + return this.semVer; + } + + /** + * Checks whether this identifier's version is valid according to the semantic versioning 2.0.0 standard. The results + * of this method invocation are cached so that future calls do not have to parse the version string again. + * @return True if this identifier has a valid semantic version, false otherwise + */ + public boolean hasValidSemanticVersion() { + try { + this.getSemanticVersion(); + return true; + } catch(SemanticVersionParsingException svpx){ + return false; + } + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java b/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java index 50402de..496a5f2 100644 --- a/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java +++ b/src/main/java/org/tudo/sse/semver/SemanticVersionNumber.java @@ -2,8 +2,22 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; +/** + * Class representing a version number as defined by the semantic versioning 2.0.0 standard. See this webpage for + * the formal definition. This class implements all comparison logic for version numbers as defined by the standard. + * + *

+ * Please Note: Build Metadata can be attached to semantic version numbers, but is irrelevant when comparing numbers. + * This is explicitly specified in the standard. This class adheres to those definitions and ignores build metadata when + * comparing numbers (compareTo) or checking for equality (equals, hashcode). + *

+ * + * + * @author Johannes Düsing + */ public class SemanticVersionNumber implements Comparable { private final int majorVersion; @@ -15,14 +29,35 @@ public class SemanticVersionNumber implements Comparable private List _parsedPreReleases = null; + /** + * Creates a new semantic version number with the given major, minor and patch version. + * @param majorVersion Major version of this number + * @param minorVersion Minor version of this number + * @param patchVersion Patch version of this number + */ public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersion) { this(majorVersion, minorVersion, patchVersion, null); } + /** + * Creates a new semantic version number with the given major, minor and patch versions, as well as a prerelease string. + * @param majorVersion Major version of this number + * @param minorVersion Minor version of this number + * @param patchVersion Patch version of this number + * @param preReleaseData Prelease identifier of this number + */ public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersion, String preReleaseData) { this(majorVersion, minorVersion, patchVersion, preReleaseData, null); } + /** + * Creates a new semantic version number with the given major, minor and patch versions, as well as a prerelease and build identifier. + * @param majorVersion Major version of this number + * @param minorVersion Minor version of this number + * @param patchVersion Patch version of this number + * @param preReleaseData Prelease identifier of this number + * @param buildMetadata The build metadata identifier of this number - not relevant for comparisons + */ public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersion, String preReleaseData, String buildMetadata) { this.majorVersion = majorVersion; this.minorVersion = minorVersion; @@ -32,32 +67,63 @@ public SemanticVersionNumber(int majorVersion, int minorVersion, int patchVersio this.buildMetadata = buildMetadata; } + /** + * Get this number's major version. + * @return The major version + */ public int getMajorVersion(){ return this.majorVersion; } + /** + * Get this number's minor version. + * @return The minor version + */ public int getMinorVersion(){ return this.minorVersion; } + /** + * Get this number's patch version. + * @return The patch version + */ public int getPatchVersion(){ return this.patchVersion; } + /** + * Checks whether this semantic version number includes a prerelease identifier. + * @return True if there is a prerelease identifier, false otherwise + */ public boolean hasPreRelease(){ return this.preReleaseData != null; } + + /** + * Checks whether this semantic version number includes build metadata. + * @return True if there is metadata, false otherwise + */ public boolean hasBuildMetadata(){ return this.buildMetadata != null; } + /** + * Get this number's prerelease identifier, if available. + * @return The prerelease identifier, or null, if none is available. + */ public String getPreRelease() { return this.preReleaseData; } + + /** + * Get this number's build metadata, if available. + * @return The build metadata, or null, if none is available. + */ public String getBuildMetadata() { return this.buildMetadata; } + @Override public final int compareTo(SemanticVersionNumber other){ // The precedence of semantic version V2 numbers is defined here: https://semver.org/#spec-item-11 // Note that build metadata is irrelevant to precedence @@ -192,10 +258,23 @@ public int compareTo(PreReleasePart other) { } } + /** + * Attempts to create a new semantic version number by parsing the given string value. If the string does not represent + * a valid number according to the semantic versioning 2.0.0 standard, an exception is thrown. + * @param value The string value to parse + * @return The semantic version number if parsing was successful + * @throws SemanticVersionParsingException If the value was invalid + */ public static SemanticVersionNumber parse(String value) throws SemanticVersionParsingException { return SemanticVersionParser.parseNumber(value); } + /** + * Attempts to create a new semantic version number by parsing the given string value. If the string does not represent + * a valid number according to the semantic versioning 2.0.0 standard, an empty Optional is returned. + * @param value The string value to parse + * @return Optional value containing either the parsed number (if parsing was successful), or nothing + */ public static Optional tryParse(String value) { try { SemanticVersionNumber number = parse(value); @@ -226,4 +305,16 @@ public String toString(){ return sb.toString(); } + + @Override + public boolean equals(Object other){ + if(!(other instanceof SemanticVersionNumber)) return false; + + return this.compareTo((SemanticVersionNumber)other) == 0; + } + + @Override + public int hashCode(){ + return Objects.hash(majorVersion, minorVersion, patchVersion, preReleaseData); + } } diff --git a/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java b/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java index d3d2f81..299123c 100644 --- a/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java +++ b/src/main/java/org/tudo/sse/semver/SemanticVersionParsingException.java @@ -1,15 +1,42 @@ package org.tudo.sse.semver; +/** + * Class representing an exception while parsing a semantic version number. + * + * @author Johannes Düsing + */ public class SemanticVersionParsingException extends Exception { + /** + * Value that was being parsed + */ private final String parsedValue; + + /** + * Message describing the error + */ private final String msg; + + /** + * Position at which the error occurred + */ private final int parsingPosition; + /** + * Create a new parsing exception with the given description while parsing the given value. + * @param value The value that was being parsed + * @param description A description of the error that was encountered + */ public SemanticVersionParsingException(String value, String description) { this(value, description, -1); } + /** + * Create a new parsing exception with the given description, parsing position and parsed value. + * @param value The value that was being parsed + * @param description A description of the error that was encountered + * @param parsingPosition The parsing position at which the error occurred + */ public SemanticVersionParsingException(String value, String description, int parsingPosition) { this.parsedValue = value; this.msg = description; @@ -31,4 +58,20 @@ public String getMessage() { return sb.toString(); } + /** + * Returns the position at which the error occurred in the original value. + * @return Parsing position + */ + public int getParsingPosition() { + return this.parsingPosition; + } + + /** + * Returns the value that was being parsed when the error occurred. + * @return The parsed value + */ + public String getParsedValue(){ + return this.parsedValue; + } + } From 54db833b1ea886d899fbfed1187b8287bff28709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 9 Jan 2026 17:38:10 +0100 Subject: [PATCH 42/55] Add support for semantic version ranges, parser for maven ranges, tests --- .../sse/semver/MavenVersionRangeParser.java | 107 ++++++++++++ .../semver/MultiPartSemanticVersionRange.java | 46 ++++++ .../tudo/sse/semver/SemanticVersionRange.java | 17 ++ .../semver/SimpleSemanticVersionRange.java | 152 ++++++++++++++++++ .../semver/MavenVersionRangeParserTest.java | 135 ++++++++++++++++ 5 files changed, 457 insertions(+) create mode 100644 src/main/java/org/tudo/sse/semver/MavenVersionRangeParser.java create mode 100644 src/main/java/org/tudo/sse/semver/MultiPartSemanticVersionRange.java create mode 100644 src/main/java/org/tudo/sse/semver/SemanticVersionRange.java create mode 100644 src/main/java/org/tudo/sse/semver/SimpleSemanticVersionRange.java create mode 100644 src/test/java/org/tudo/sse/semver/MavenVersionRangeParserTest.java diff --git a/src/main/java/org/tudo/sse/semver/MavenVersionRangeParser.java b/src/main/java/org/tudo/sse/semver/MavenVersionRangeParser.java new file mode 100644 index 0000000..062b45d --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/MavenVersionRangeParser.java @@ -0,0 +1,107 @@ +package org.tudo.sse.semver; + +import java.util.ArrayList; +import java.util.List; + +/** + * Parser for semantic version ranges as specified by the Maven build system. See the specification + * for details on the supported range types and syntaxes. + * + * @author Johannes Düsing + */ +public class MavenVersionRangeParser { + + private static final char START_INCLUSIVE = '['; + private static final char END_INCLUSIVE = ']'; + + private static final char START_EXCLUSIVE = '('; + private static final char END_EXCLUSIVE = ')'; + + private static final char RANGE_SEPARATOR = ','; + + private MavenVersionRangeParser(){} + + /** + * Parses the given textual representation of a Maven version range into the corresponding version range object. + * @param value Value to parse + * @return Parsed version range, if successful + * @throws SemanticVersionParsingException If either the range syntax or the version number syntax were invalid. + */ + public static SemanticVersionRange parseRange(String value) throws SemanticVersionParsingException { + final char[] chars = value.toCharArray(); + final StringBuilder currentValue = new StringBuilder(); + + boolean inRange = false; + boolean sawSeparator = false; + boolean lowerInclusive = false; + SemanticVersionNumber lowerBound = null; + + List ranges = new ArrayList<>(); + + for(int i = 0; i < chars.length; i++){ + char currentChar = chars[i]; + + // Ignore whitespaces + if(Character.isWhitespace(currentChar)){ + continue; + } + + if(!inRange){ + if(currentChar == START_INCLUSIVE){ + inRange = true; + lowerInclusive = true; + } else if(currentChar == START_EXCLUSIVE){ + inRange = true; + lowerInclusive = false; + } else if(currentChar != RANGE_SEPARATOR){ + throw new SemanticVersionParsingException(value, "Expected begin of new range or range separator but got " + currentChar, i); + } + } else { + if(currentChar == RANGE_SEPARATOR){ + String lowerBoundStr = currentValue.toString(); + currentValue.setLength(0); + + sawSeparator = true; + + if(lowerBoundStr.isBlank()) lowerBound = null; + else lowerBound = SemanticVersionNumber.parse(lowerBoundStr); + } else if(currentChar == END_INCLUSIVE || currentChar == END_EXCLUSIVE){ + String upperBoundStr = currentValue.toString(); + currentValue.setLength(0); + + SemanticVersionNumber upperBound = null; + if(!upperBoundStr.isBlank()) upperBound = SemanticVersionNumber.parse(upperBoundStr); + boolean upperInclusive = (currentChar == END_INCLUSIVE); + + // If we do not see a separator (e.g. range '[1.0]'), we have a hard requirement for one specific + // version. This means lower and upper bound are equal, and both bounds must be inclusive + if(!sawSeparator){ + lowerBound = upperBound; + + if(!lowerInclusive || !upperInclusive){ + throw new SemanticVersionParsingException(value, "Hard requirements for one specific version must have inclusive range delimiters.", i); + } + } + + SimpleSemanticVersionRange range = new SimpleSemanticVersionRange(lowerBound, lowerInclusive, upperBound, upperInclusive); + ranges.add(range); + + inRange = false; + sawSeparator = false; + } else { + currentValue.append(currentChar); + } + } + } + + if(inRange) + throw new SemanticVersionParsingException(value, "Input ended with range not being closed"); + + if(ranges.isEmpty()) + throw new SemanticVersionParsingException(value, "No ranges found in expression: " + value); + else if(ranges.size() == 1) + return ranges.get(0); + else + return new MultiPartSemanticVersionRange(ranges); + } +} diff --git a/src/main/java/org/tudo/sse/semver/MultiPartSemanticVersionRange.java b/src/main/java/org/tudo/sse/semver/MultiPartSemanticVersionRange.java new file mode 100644 index 0000000..5bee54c --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/MultiPartSemanticVersionRange.java @@ -0,0 +1,46 @@ +package org.tudo.sse.semver; + +import java.util.List; + +/** + * Implementation of composite semantic version ranges, which consist of multiple "smaller" range specifications. Those + * ranges are concatenated in a way that this composite range is the union of all partial ranges - if a given version + * number is contained in at least one of the partial ranges, it is also contained in this range. + * + * @author Johannes Düsing + */ +public class MultiPartSemanticVersionRange implements SemanticVersionRange { + + private final List partialRanges; + + /** + * Creates a new range with the given list of partial ranges. Containment of any given version number is checked + * in the same order in which partial ranges a provided here. + * + * @param ranges List of partial ranges + */ + public MultiPartSemanticVersionRange(List ranges){ + this.partialRanges = ranges; + } + + @Override + public boolean contains(SemanticVersionNumber number) { + for(SemanticVersionRange range : partialRanges){ + if(range.contains(number)) return true; + } + return false; + } + + @Override + public String toString(){ + StringBuilder sb = new StringBuilder(); + + for(int i = 0; i < partialRanges.size(); i++){ + SemanticVersionRange range = partialRanges.get(i); + sb.append(range.toString()); + if(i != partialRanges.size()-1) sb.append("."); + } + + return sb.toString(); + } +} diff --git a/src/main/java/org/tudo/sse/semver/SemanticVersionRange.java b/src/main/java/org/tudo/sse/semver/SemanticVersionRange.java new file mode 100644 index 0000000..d11f764 --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/SemanticVersionRange.java @@ -0,0 +1,17 @@ +package org.tudo.sse.semver; + +/** + * Interface that represents a range of semantic version numbers. The only requirement for implementations is that they + * must be able to decide whether any given semantic version number is contained within a range or not. + * + * @author Johannes Düsing + */ +public interface SemanticVersionRange { + + /** + * Checks whether the given semantic version number falls within this range + * @param number The number to check + * @return True if the number is contained within this range, false otherwise + */ + boolean contains(SemanticVersionNumber number); +} diff --git a/src/main/java/org/tudo/sse/semver/SimpleSemanticVersionRange.java b/src/main/java/org/tudo/sse/semver/SimpleSemanticVersionRange.java new file mode 100644 index 0000000..e2202b3 --- /dev/null +++ b/src/main/java/org/tudo/sse/semver/SimpleSemanticVersionRange.java @@ -0,0 +1,152 @@ +package org.tudo.sse.semver; + +/** + * Simple implementation of semantic version ranges. A simple range is defined by two version numbers, a lower bound and + * an upper bound. Both are optional, in which case we have (semi-) open ranges. Bounds are either inclusive or exclusive. + * + * @author Johannes Düsing + */ +public class SimpleSemanticVersionRange implements SemanticVersionRange { + + private final SemanticVersionNumber lowerBound; + private final SemanticVersionNumber upperBound; + private final boolean lowerBoundInclusive; + private final boolean upperBoundInclusive; + + /** + * Creates a new simple semantic version range with the given bounds and their inclusive-specification. Use null to + * indicate that one (or both) bounds do not exist. If a bound is not set, its 'inclusive' value is ignored. + * @param lowerBound The lower bound for this range, or null if there is no lower bound. + * @param lowerBoundInclusive Whether the lower bound is inclusive. Ignored if there is no lower bound. + * @param upperBound The upper bound for this range, or null if there is no upper bound. + * @param upperBoundInclusive Whether the upper bound is inclusive. Ignored if there is no upper bound. + */ + public SimpleSemanticVersionRange(SemanticVersionNumber lowerBound, + boolean lowerBoundInclusive, + SemanticVersionNumber upperBound, + boolean upperBoundInclusive) { + this.lowerBound = lowerBound; + this.lowerBoundInclusive = lowerBoundInclusive; + this.upperBound = upperBound; + this.upperBoundInclusive = upperBoundInclusive; + } + + /** + * Creates a new simple semantic version range that is semi-open, i.e. a range that only has a lower bound. + * @param lowerBound Lower bound for this range. + * @param lowerBoundInclusive Whether the lower bound is inclusive. + * @return Semi-Open range with only a lower bound + */ + public static SimpleSemanticVersionRange fromLowerBound(SemanticVersionNumber lowerBound, boolean lowerBoundInclusive) { + return new SimpleSemanticVersionRange(lowerBound, lowerBoundInclusive, null, false); + } + + /** + * Creates a new simple semantic version range that is semi-open, i.e. a range that only has an upper bound. + * @param upperBound Upper bound for this range. + * @param upperBoundInclusive Whether the upper bound is inclusive. + * @return Semi-Open range with only an upper bound + */ + public static SimpleSemanticVersionRange fromUpperBound(SemanticVersionNumber upperBound, boolean upperBoundInclusive) { + return new SimpleSemanticVersionRange(null, false, upperBound, upperBoundInclusive); + } + + /** + * Checks whether this range has a lower bound. + * @return True if lower bound exists. + */ + public boolean hasLowerBound() { + return lowerBound != null; + } + + /** + * Checs whether this range has an upper bound. + * @return True if upper bound exists. + */ + public boolean hasUpperBound() { + return this.upperBound != null; + } + + /** + * Checks whether the lower bound of this range is inclusive. + * @return True if lower bound is inclusive, false if exclusive. + */ + public boolean isLowerBoundInclusive() { + return lowerBoundInclusive; + } + + /** + * Checks whether the upper bound of this range is inclusive. + * @return True if upper bound is inclusive, false if exclusive. + */ + public boolean isUpperBoundInclusive() { + return upperBoundInclusive; + } + + /** + * Gets the lower bound for this range. + * @return Lower bound, or null if non exists + */ + public SemanticVersionNumber getLowerBound() { + return lowerBound; + } + + /** + * Gets the upper bound for this range. + * @return Upper bound, or null if non exists. + */ + public SemanticVersionNumber getUpperBound() { + return upperBound; + } + + /** + * Checks whether this range represents a (Maven Central) hard requirement. A hard requirement is a special range + * where lower and upper bound are equal, and both are inclusive. This range contains exactly one version number. + * @return True if this range is a hard requirement + */ + public boolean isHardRequirement(){ + return this.lowerBound.equals(this.upperBound); + } + + @Override + public boolean contains(SemanticVersionNumber number) { + if(hasLowerBound()){ + int compareResult = number.compareTo(lowerBound); + if(compareResult < 0 || (!isLowerBoundInclusive() && compareResult == 0)) return false; + } + + if(hasUpperBound()){ + int compareResult = number.compareTo(upperBound); + if(compareResult > 0 || (!isUpperBoundInclusive() && compareResult == 0)) return false; + } + + return true; + } + + @Override + public String toString(){ + StringBuilder sb = new StringBuilder(); + + if(isLowerBoundInclusive()) + sb.append('['); + else + sb.append('('); + + if(hasLowerBound()) + sb.append(lowerBound.toString()); + + if(!isHardRequirement()){ + sb.append(','); + sb.append(' '); + if(hasUpperBound()) + sb.append(upperBound.toString()); + } + + if(isUpperBoundInclusive()) + sb.append(']'); + else + sb.append(')'); + + return sb.toString(); + } +} diff --git a/src/test/java/org/tudo/sse/semver/MavenVersionRangeParserTest.java b/src/test/java/org/tudo/sse/semver/MavenVersionRangeParserTest.java new file mode 100644 index 0000000..79114a2 --- /dev/null +++ b/src/test/java/org/tudo/sse/semver/MavenVersionRangeParserTest.java @@ -0,0 +1,135 @@ +package org.tudo.sse.semver; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for version range parsing are based on the specification + */ +public class MavenVersionRangeParserTest { + + @Test + @DisplayName("The parser must parse hard requirements correctly") + void testHardRequirement_Valid(){ + SimpleSemanticVersionRange simpleRange = assertSimpleRange("[1.0]"); + + assertNotNull(simpleRange); + assertTrue(simpleRange.isHardRequirement()); + assertEquals("1.0.0", simpleRange.getLowerBound().toString()); + } + + @Test + @DisplayName("The parser must parse semi-open simple ranges like (,1.0]") + void testSemiRange1(){ + SimpleSemanticVersionRange simpleRange = assertSimpleRange("(,1.0]"); + assertNotNull(simpleRange); + assertFalse(simpleRange.hasLowerBound()); + assertTrue(simpleRange.hasUpperBound()); + assertFalse(simpleRange.isLowerBoundInclusive()); + assertTrue(simpleRange.isUpperBoundInclusive()); + assertEquals("1.0.0", simpleRange.getUpperBound().toString()); + } + + @Test + @DisplayName("The parser must parse regular simple ranges") + void testFullRange1(){ + SimpleSemanticVersionRange simpleRange = assertSimpleRange("[1.2,1.3.3-Preview+Snapshot.1]"); + assertNotNull(simpleRange); + assertTrue(simpleRange.hasLowerBound()); + assertTrue(simpleRange.hasUpperBound()); + assertTrue(simpleRange.isLowerBoundInclusive()); + assertTrue(simpleRange.isUpperBoundInclusive()); + assertTrue(simpleRange.getUpperBound().hasPreRelease()); + assertTrue(simpleRange.getUpperBound().getBuildMetadata().endsWith(".1")); + } + + @Test + @DisplayName("The parser must parse inclusivity correctly for simple ranges") + void testFullRange2(){ + SimpleSemanticVersionRange simpleRange = assertSimpleRange("[1.0, 2.0)"); + assertNotNull(simpleRange); + assertTrue(simpleRange.hasLowerBound()); + assertTrue(simpleRange.isLowerBoundInclusive()); + assertTrue(simpleRange.hasUpperBound()); + assertFalse(simpleRange.isUpperBoundInclusive()); + assertEquals(2, simpleRange.getUpperBound().getMajorVersion()); + } + + @Test + @DisplayName("The parser must parse semi-open simple ranges like [1.5,)") + void testSemiRange2(){ + SimpleSemanticVersionRange simpleRange = assertSimpleRange("[1.5,)"); + assertNotNull(simpleRange); + assertTrue(simpleRange.hasLowerBound()); + assertTrue(simpleRange.isLowerBoundInclusive()); + assertFalse(simpleRange.hasUpperBound()); + assertFalse(simpleRange.isUpperBoundInclusive()); + assertEquals(5, simpleRange.getLowerBound().getMinorVersion()); + } + + @Test + @DisplayName("The parser must parse complex composite ranges correctly") + void complexRange1(){ + MultiPartSemanticVersionRange multiRange = assertMultiRange("(,1.0],[1.2,)"); + assertNotNull(multiRange); + + var notContained = asSemVer("1.0.1"); + var contained1 = asSemVer("0.0.1"); + var contained2 = asSemVer("1.2"); + + assertFalse(multiRange.contains(notContained)); + assertTrue(multiRange.contains(contained1)); + assertTrue(multiRange.contains(contained2)); + } + + @Test + @DisplayName("The parser must handle inverse hard requirements correctly") + void testInverseHardRequirement(){ + MultiPartSemanticVersionRange multiRange = assertMultiRange("(,1.1),(1.1,)"); + assertNotNull(multiRange); + + var notContained = asSemVer("1.1"); + var contained = asSemVer("1.2"); + + assertFalse(multiRange.contains(notContained)); + assertTrue(multiRange.contains(contained)); + } + + + private SimpleSemanticVersionRange assertSimpleRange(String versionRange){ + try { + var range = MavenVersionRangeParser.parseRange(versionRange); + + assertInstanceOf(SimpleSemanticVersionRange.class, range); + + return (SimpleSemanticVersionRange)range; + } catch (SemanticVersionParsingException svpx){ + fail(svpx); + } + return null; + } + + private MultiPartSemanticVersionRange assertMultiRange(String versionRange){ + try { + var range = MavenVersionRangeParser.parseRange(versionRange); + + assertInstanceOf(MultiPartSemanticVersionRange.class, range); + + return (MultiPartSemanticVersionRange) range; + } catch (SemanticVersionParsingException svpx){ + fail(svpx); + } + return null; + } + + private SemanticVersionNumber asSemVer(String value){ + try { + return SemanticVersionNumber.parse(value); + } catch (SemanticVersionParsingException svpx){ + fail(svpx); + } + return null; + } +} From e8ca3aa8ee4e03109a760cbf0aa48db9935dbfa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 12 Jan 2026 10:44:51 +0100 Subject: [PATCH 43/55] Accept YYYY-MM-DD format for cutoff dates via CLI. Change UNIX timestamp format from milliseconds to seconds. Update documentation. --- README.md | 20 +++--- .../config/ArtifactAnalysisConfig.java | 4 +- .../parsing/ArtifactAnalysisConfigParser.java | 52 ++++++++++++++- .../config/parsing/CLIParsingUtilities.java | 22 +++++++ .../MavenCentralArtifactAnalysisTest.java | 63 ++++++++++++++++++- 5 files changed, 146 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b8a20ab..829e4b3 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,16 @@ Additionally, the `MavenCentralArtifactAnalysis` provides the `resolveIndex` opt Both analysis types provide a method called `void runAnalysis(String[] args)`. Invoking this method will start the analysis. By default, MARIN analyses will support the following CLI parameters: -| **Argument** | **Artifact Analysis** | **Library Analysis** | **Description** | **Example** | -|------------------------------------------------------------|-----------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| -| `-st :`
`--skip-take :` | Yes | Yes | Skips the first `n` inputs and then only
processes the next `t` ones. | `-st 200:10` | -| `-su :`
`--since-until :` | Yes | No | Skips artifacts released before the
timestamp `s` or after timestamp `t`. | | -| `-i `
`--inputs ` | Yes | Yes | Processes inputs from a file instead of the
Maven Central index. Expects on input
per line. Inputs are either G:A:V triple (artifacts)
or G:A tuple (libraries). | `-i libraries.txt` | -| `-pof `
`--progress-output-file ` | Yes | Yes | File to periodically write number of processed
artifacts to. Defaults to `./marin-progress`. | `-pof marin-progress` | -| `-prf `
`--progress-restore-file ` | Yes | Yes | File to load a previous run's progress from.
Inputs will be skipped until progress is restored.
Not used by default. | `-prf marin-progress` | -| `-spi `
`--save-progress-interval ` | Yes | Yes | Number of inputs after which to store progress.
Defaults to 100. | `-spi 10` | -| `-t `
`--threads ` | Yes | Yes | Number of threads to use. Defaults to 1. | `-t 8` | -| `-o `
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | +| **Argument** | **Artifact Analysis** | **Library Analysis** | **Description** | **Example** | +|------------------------------------------------------------|-----------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------| +| `-st :`
`--skip-take :` | Yes | Yes | Skips the first `` inputs and then only processes the next `` ones. | `-st 200:10` | +| `-su :`
`--since-until :` | Yes | No | Skips artifacts released before the timestamp `` or after timestamp ``. A timestamp is either a UNIX timestamp (in seconds) or a date of format YYYY-MM-DD. Note that `` is automatically parsed as end-of-day if YYYY-MM-DD is used. | `-su 1768209126:2026-02-02` | +| `-i `
`--inputs ` | Yes | Yes | Processes inputs from a file instead of the Maven Central index. Expects on input per line. Inputs are either G:A:V triple (artifacts) or G:A tuple (libraries). | `-i libraries.txt` | +| `-pof `
`--progress-output-file ` | Yes | Yes | File to periodically write number of processed artifacts to. Defaults to `./marin-progress`. | `-pof marin-progress` | +| `-prf `
`--progress-restore-file ` | Yes | Yes | File to load a previous run's progress from. Inputs will be skipped until progress is restored. Not used by default. | `-prf marin-progress` | +| `-spi `
`--save-progress-interval ` | Yes | Yes | Number of inputs after which to store progress. Defaults to 100. | `-spi 10` | +| `-t `
`--threads ` | Yes | Yes | Number of threads to use. Defaults to 1. | `-t 8` | +| `-o `
`--output ` | Yes | No | Output directory to optionally write processed artifacts to. Depending on the artifact information required by the analysis, this can be `pom.xml` files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | If you do not want to extend one of the aforementioned base classes, MARIN also provides corresponding implementations of the `java.util.Iterator` interface to enumerate and enrich artifacts or libraries. Note that those implementations are **not threadsafe** and cannot perform resolution in parallel. MARIN provides: diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java index f2f3dd8..96123ec 100644 --- a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -14,12 +14,12 @@ public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { static final Path DEFAULT_VALUE_OUTPUT = null; /** - * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. + * UNIX timestamp (in milliseconds) before which artifacts shall be excluded from analysis, or -1 if disabled. */ public long since; /** - * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. + * UNIX timestamp (in milliseconds) after which artifacts shall be excluded from analysis, or -1 if disabled. */ public long until; diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java index 6b46233..0c7d04c 100644 --- a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -5,6 +5,15 @@ import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; import org.tudo.sse.analyses.config.InvalidConfigurationException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.Date; + /** * Parser for creating {@link ArtifactAnalysisConfig} objects from CLI parameters. * Configuration is obtained by parsing command line arguments provided as an array of strings. @@ -40,8 +49,12 @@ private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfi switch(args[i]) { case "-su": case "--since-until": - final long[] sinceUntil = nextArgAsLongPair(args, i); - configBuilder.withSinceUtil(sinceUntil[0], sinceUntil[1]); + final String[] sinceUntil = nextArgAsStringPair(args, i); + + long since = toUnixTimeStampMillis(sinceUntil[0], "since", false); + long until = toUnixTimeStampMillis(sinceUntil[1], "until", true); + + configBuilder.withSinceUtil(since, until); break; case "-o": case "--output": @@ -59,4 +72,39 @@ private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfi } } + private long toUnixTimeStampMillis(String value, String attrName, boolean useEndOfDay) throws CLIException { + ZonedDateTime date; + + try { + long timestamp = Long.parseLong(value); + Instant s = Instant.ofEpochSecond(timestamp); + date = s.atZone(ZoneId.systemDefault()); + } catch (NumberFormatException ignored){ + date = parseYYYYMMDD(value, attrName); + // Parsing a date will create a (local) time of 00:00:00 - we want to add one day if requested + if(useEndOfDay) + date = date.plusHours(23).plusMinutes(59).plusSeconds(59); + } catch (DateTimeException dtx){ + throw new CLIException("Not a valid UNIX timestamp", attrName); + } + + // Do a plausibility check on the given time stamp + if(date.getYear() < 1950 || date.getYear() > 2100) + throw new CLIException("Cutoff dates must fall between the years 1950 and 2100"); + + return date.toInstant().toEpochMilli(); + } + + private ZonedDateTime parseYYYYMMDD(String value, String attrName) throws CLIException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + try { + return sdf.parse(value).toInstant().atZone(ZoneId.systemDefault()); + } catch (ParseException px){ + var exception = new CLIException("Not a valid date of format YYYY-MM-DD", attrName); + exception.initCause(px); + throw exception; + } + } + } diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java index 2c175d1..66dc4ca 100644 --- a/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java @@ -60,6 +60,28 @@ default long[] nextArgAsLongPair(String[] args, int i) throws CLIException { return toReturn; } + /** + * Parses the next argument from the given array of arguments as a pair of string values, separated by a colon. The + * pair is returned as a string-array of size 2. + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as a pair of string values + * @throws CLIException If there is no next argument, or it is not formatted as two string values separated by a colon. + */ + default String[] nextArgAsStringPair(String[] args, int i) throws CLIException { + if(i + 1 < args.length) { + String[] toReturn = args[i + 1].split(":"); + + if(toReturn.length != 2) { + throw new CLIException("Expected a colon-separated pair of strings", args[i]); + } + + return toReturn; + } else { + throw(new CLIException("Missing argument: first:second", args[i])); + } + } + /** * Parses the next argument from the given array of arguments as a Path value. * diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index b5fb274..92f8030 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -21,6 +21,9 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.*; import static org.junit.jupiter.api.Assertions.*; @@ -85,6 +88,61 @@ void parseCLIRegular() throws IOException{ } } + @Test + @DisplayName("The CLI parser must parse timestamps YYYY-MM-DD format") + void parseCLITimestamps1() { + var config = parseCLI("-su 2025-12-01:2025-12-31"); + + var since = asDate(config.since); + var until = asDate(config.until); + + assertEquals(2025, since.getYear()); + assertEquals(12, since.getMonthValue()); + assertEquals(1, since.getDayOfMonth()); + assertEquals(0, since.getHour()); + assertEquals(0, since.getMinute()); + + assertEquals(2025, until.getYear()); + assertEquals(12, until.getMonthValue()); + assertEquals(31, until.getDayOfMonth()); + assertEquals(23, until.getHour()); + assertEquals(59, until.getMinute()); + } + + @Test + @DisplayName("The CLI parser must parse UNIX timestamps") + void parseCLITimestamps2() { + var config = parseCLI("-su 2010-10-10:1321006271"); + + var since = asDate(config.since); + var until = asDate(config.until); + + assertEquals(2010, since.getYear()); + assertEquals(10, since.getMonthValue()); + assertEquals(10, since.getDayOfMonth()); + assertEquals(0, since.getHour()); + assertEquals(0, since.getMinute()); + + assertEquals(2011, until.getYear()); + assertEquals(11, until.getMonthValue()); + assertEquals(11, until.getDayOfMonth()); + assertEquals(11, until.getHour()); + assertEquals(11, until.getMinute()); + assertEquals(11, until.getSecond()); + } + + @Test + @DisplayName("The CLI parser must reject invalid ranges") + void parseCLIInvalidRanges() { + try { + parseCLI("-su 1321006271:2010-10-10"); + fail("The CLI parser must reject invalid ranges"); + } catch(Exception x){ + assertInstanceOf(RuntimeException.class, x); + assertInstanceOf(CLIException.class, x.getCause()); + } + } + @Test @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") void parseCLIShorthands() throws IOException{ @@ -400,5 +458,8 @@ private int asInt(String s){ return Integer.parseInt(s); } - + private ZonedDateTime asDate(long timestamp){ + Instant i = Instant.ofEpochMilli(timestamp); + return ZonedDateTime.ofInstant(i, ZoneId.systemDefault()); + } } \ No newline at end of file From 9316849232df69c3d0e93ee9e37cdec064171800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 12 Jan 2026 11:15:00 +0100 Subject: [PATCH 44/55] Update documentation for since-until in config builder, add convenience method --- .../config/ArtifactAnalysisConfigBuilder.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java index d0e2a33..2b628cb 100644 --- a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java @@ -2,6 +2,7 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.time.ZonedDateTime; /** * Configuration builder to obtain {@link ArtifactAnalysisConfig} instances programmatically. @@ -104,8 +105,21 @@ public ArtifactAnalysisConfigBuilder withOutputDirectory(Path outDir) throws Inv * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have * been released after the timestamp given in 'since', and before 'until'. * - * @param since Timestamp marking the lower limit of the range - * @param until Timestamp marking the upper limit of the range + * @param since DateTime marking the lower limit of the range + * @param until DateTime marking the upper limit of the range + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration values are not valid + */ + public ArtifactAnalysisConfigBuilder withSinceUntil(ZonedDateTime since, ZonedDateTime until) throws InvalidConfigurationException { + return this.withSinceUtil(since.toInstant().toEpochMilli(), until.toInstant().toEpochMilli()); + } + + /** + * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have + * been released after the timestamp given in 'since', and before 'until'. + * + * @param since UNIX timestamp in milliseconds marking the lower limit of the range + * @param until UNIX timestamp in milliseconds marking the upper limit of the range * @return The current configuration builder instance * @throws InvalidConfigurationException If the given configuration values are not valid */ From b28a4b2e45a36d856945dd54c0cf9b4089af5c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 12 Jan 2026 14:46:56 +0100 Subject: [PATCH 45/55] Fix dependence on platform time zone --- .../parsing/ArtifactAnalysisConfigParser.java | 18 +++++++++--------- .../MavenCentralArtifactAnalysisTest.java | 6 ++++-- src/test/resources/MavenAnalysis.json | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java index 0c7d04c..86d6cde 100644 --- a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -7,10 +7,8 @@ import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.DateTimeException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; +import java.time.*; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.Date; @@ -20,6 +18,8 @@ */ public class ArtifactAnalysisConfigParser extends LibraryAnalysisConfigParser implements CLIParsingUtilities { + private static final ZoneId GMT_ZONE = ZoneId.of("GMT"); + /** * Creates a new artifact config parser instance. */ @@ -78,7 +78,7 @@ private long toUnixTimeStampMillis(String value, String attrName, boolean useEnd try { long timestamp = Long.parseLong(value); Instant s = Instant.ofEpochSecond(timestamp); - date = s.atZone(ZoneId.systemDefault()); + date = s.atZone(GMT_ZONE); } catch (NumberFormatException ignored){ date = parseYYYYMMDD(value, attrName); // Parsing a date will create a (local) time of 00:00:00 - we want to add one day if requested @@ -96,13 +96,13 @@ private long toUnixTimeStampMillis(String value, String attrName, boolean useEnd } private ZonedDateTime parseYYYYMMDD(String value, String attrName) throws CLIException { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd"); try { - return sdf.parse(value).toInstant().atZone(ZoneId.systemDefault()); - } catch (ParseException px){ + return LocalDate.parse(value, dtf).atStartOfDay(GMT_ZONE); + } catch (DateTimeParseException dtpx) { var exception = new CLIException("Not a valid date of format YYYY-MM-DD", attrName); - exception.initCause(px); + exception.initCause(dtpx); throw exception; } } diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 92f8030..323c7de 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -112,11 +112,13 @@ void parseCLITimestamps1() { @Test @DisplayName("The CLI parser must parse UNIX timestamps") void parseCLITimestamps2() { - var config = parseCLI("-su 2010-10-10:1321006271"); + var config = parseCLI("-su 2010-10-10:1321009871"); var since = asDate(config.since); var until = asDate(config.until); + assertEquals(1321009871000L, config.until); + assertEquals(2010, since.getYear()); assertEquals(10, since.getMonthValue()); assertEquals(10, since.getDayOfMonth()); @@ -460,6 +462,6 @@ private int asInt(String s){ private ZonedDateTime asDate(long timestamp){ Instant i = Instant.ofEpochMilli(timestamp); - return ZonedDateTime.ofInstant(i, ZoneId.systemDefault()); + return ZonedDateTime.ofInstant(i, ZoneId.of("GMT")); } } \ No newline at end of file diff --git a/src/test/resources/MavenAnalysis.json b/src/test/resources/MavenAnalysis.json index 8879f2b..8064827 100644 --- a/src/test/resources/MavenAnalysis.json +++ b/src/test/resources/MavenAnalysis.json @@ -30,8 +30,8 @@ [ "-1", "-1", - "53245", - "53246", + "53245000", + "53246000", "null", "null", "false" From 18ff1b050295cdc9e13089d856fa4c76919fac9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 6 Mar 2026 09:57:04 +0100 Subject: [PATCH 46/55] Fix test dependence on index ordering --- src/main/java/org/tudo/sse/IndexWalker.java | 244 -------------- .../analyses/MavenCentralLibraryAnalysis.java | 2 +- .../org/tudo/sse/utils/IndexIterator.java | 11 +- .../java/org/tudo/sse/IndexWalkerTest.java | 96 ------ .../MavenCentralArtifactAnalysisTest.java | 31 +- .../MavenCentralLibraryAnalysisTest.java | 47 ++- .../tudo/sse/resolution/PomResolverTest.java | 77 +++-- .../java/org/tudo/sse/utils/JsonExporter.java | 85 +++++ .../sse/utils/LibraryIndexIteratorTest.java | 16 - .../org/tudo/sse/utils/TestUtilities.java | 28 ++ src/test/resources/PomInputs.json | 314 +++++++++++++++--- 11 files changed, 489 insertions(+), 462 deletions(-) delete mode 100644 src/main/java/org/tudo/sse/IndexWalker.java delete mode 100644 src/test/java/org/tudo/sse/IndexWalkerTest.java create mode 100644 src/test/java/org/tudo/sse/utils/JsonExporter.java diff --git a/src/main/java/org/tudo/sse/IndexWalker.java b/src/main/java/org/tudo/sse/IndexWalker.java deleted file mode 100644 index 084831c..0000000 --- a/src/main/java/org/tudo/sse/IndexWalker.java +++ /dev/null @@ -1,244 +0,0 @@ -package org.tudo.sse; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tudo.sse.model.Artifact; -import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.model.ResolutionContext; -import org.tudo.sse.utils.IndexIterator; - -import java.net.URI; -import java.util.*; -import java.util.regex.Pattern; - -/** -* This class creates an object that manages retrieving indexes from the Maven Central repository. -* Either the Identifiers can be retrieved lazily or be fully parsed, utilizing the IndexIterator. -* This class also allows for the ability to walk all indexes or walk them in a paginated fashion. -*/ - -public class IndexWalker implements Iterable { - - /** - * The string pattern at which to split index entries. - */ - public static final String splitPattern = Pattern.quote("|"); - - private IndexIterator indexIterator; - private boolean resetIterator; - private final URI base; - - private static final Logger log = LoggerFactory.getLogger(IndexWalker.class); - - /** - * Creates a new IndexWalker with the given repository base URI. - * @param base The repository base URI. - */ - public IndexWalker(URI base) { - this.base = base; - resetIterator = true; - } - - /** - * Moves the iterator to a specified index. - * - * @param position which index to move to - * @throws IOException when there is an issue opening a file - */ - public void moveIterator(long position) throws IOException { - indexIterator = new IndexIterator(base, position); - this.resetIterator = false; - } - - /** - * Produces a list of all artifact identifiers for the entire repository index. - * @return List of artifact identifiers inside the repository - * @throws IOException If connection errors occur - */ - public List lazyWalkAllIndexes() throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - List idents = new ArrayList<>(); - while(indexIterator.hasNext()) { - idents.add(indexIterator.next().getIdent()); - } - - indexIterator.closeReader(); - return idents; - } - - /** - * Produces a list of all artifacts for the entire repository index. All artifacts are annotated with index information. - * @return List of artifacts (with index information) - * @throws IOException If connection errors occur - */ - public List walkAllIndexes() throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); - - List artifacts = new ArrayList<>(); - while(indexIterator.hasNext()) { - final IndexInformation indexInformation = indexIterator.next(); - final Artifact theArtifact = ctx.createArtifact(indexInformation.getIdent()); - - theArtifact.setIndexInformation(indexInformation); - - artifacts.add(theArtifact); - } - - if(artifacts.size() % 500000 == 0) { - log.info("{} artifacts have been parsed.", artifacts.size()); - } - indexIterator.closeReader(); - return artifacts; - } - - /** - * Produces a list of artifacts from the repository index with the given pagination values. - * @param skip Number of artifact identifiers to skip - * @param take Number of artifact identifiers to take - * @return List of artifact identifiers - * @throws IOException If connection errors occur - */ - public List lazyWalkPaginated(long skip, long take) throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - List idents = new ArrayList<>(); - - int count = 0; - int fromFront = 0; - while(indexIterator.hasNext() && count < take) { - if(fromFront >= skip) { - idents.add(indexIterator.next().getIdent()); - count++; - } else { - indexIterator.next(); - } - fromFront++; - } - indexIterator.closeReader(); - - return idents; - } - - /** - * Produces a list of artifact identifiers from the repository index with the given pagination values. All artifacts - * are annotated with index information. - * @param skip Number of artifacts to skip - * @param take Number of artifacts to take - * @return List of artifacts with index information - * @throws IOException If connection errors occur - */ - public List walkPaginated(long skip, long take) throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); - - List artifacts = new ArrayList<>(); - - int count = 0; - int fromFront = 0; - while(indexIterator.hasNext() && count < take) { - if(fromFront >= skip) { - final IndexInformation indexInformation = indexIterator.next(); - final Artifact theArtifact = ctx.createArtifact(indexInformation.getIdent()); - - theArtifact.setIndexInformation(indexInformation); - - artifacts.add(theArtifact); - count++; - } else { - indexIterator.next(); - } - fromFront++; - } - indexIterator.closeReader(); - return artifacts; - } - - /** - * Produces a list of artifact identifiers from the repository index within the given time bounds. - * @param since Timestamp marking the lower bound for release dates - * @param until Timestamp marking the upper bound for release dates - * @return List of artifact identifiers - * @throws IOException If connection errors occur - */ - public List walkDates(long since, long until) throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); - - List artifacts = new ArrayList<>(); - - long currentToSince = 0; - long sinceToUntil = since; - while(indexIterator.hasNext() && sinceToUntil < until) { - final IndexInformation temp = indexIterator.next(); - if(currentToSince >= since) { - sinceToUntil = temp.getLastModified(); - - final Artifact theArtifact = ctx.createArtifact(temp.getIdent()); - theArtifact.setIndexInformation(temp); - - artifacts.add(theArtifact); - } else { - currentToSince = temp.getLastModified(); - } - } - indexIterator.closeReader(); - return artifacts; - } - - /** - * Produces a list of artifacts from the repository index within the given time bounds. All artifacts - * are annotated with index information. - * @param since Timestamp marking the lower bound for release dates - * @param until Timestamp marking the upper bound for release dates - * @return List of artifacts with index information - * @throws IOException If connection errors occur - */ - public List lazyWalkDates(long since, long until) throws IOException{ - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - List idents = new ArrayList<>(); - - long currentToSince = 0; - long sinceToUntil = since; - while(indexIterator.hasNext() && sinceToUntil < until) { - IndexInformation temp = indexIterator.next(); - if(currentToSince >= since) { - sinceToUntil = temp.getLastModified(); - idents.add(temp.getIdent()); - } else { - currentToSince = temp.getLastModified(); - } - } - indexIterator.closeReader(); - - return idents; - } - - @Override - public Iterator iterator() { - try { - return new IndexIterator(base); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 40a3ec8..7032b55 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -276,7 +276,7 @@ private List getReleaseIdentifiers(String groupId, String artifac } return identifiers; } catch (Exception x) { - log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + log.warn("Failed to obtain version list for library {}:{} ({})", groupId, artifactId, x.getCause().getMessage()); try { this.onVersionListError(groupId, artifactId, x); } catch(Exception inner) { diff --git a/src/main/java/org/tudo/sse/utils/IndexIterator.java b/src/main/java/org/tudo/sse/utils/IndexIterator.java index 1fac507..75780a0 100644 --- a/src/main/java/org/tudo/sse/utils/IndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/IndexIterator.java @@ -3,7 +3,6 @@ import org.apache.maven.index.reader.IndexReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.tudo.sse.IndexWalker; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.Package; import org.tudo.sse.model.index.IndexInformation; @@ -13,6 +12,7 @@ import java.net.URI; import java.util.Iterator; import java.util.Map; +import java.util.regex.Pattern; /** * This class creates an iterator for iterating over indexes and returning IndexArtifact objects. @@ -22,6 +22,9 @@ */ public class IndexIterator implements Iterator { + + public static final String splitPattern = Pattern.quote("|"); + private long index; private final URI baseUri; @@ -96,7 +99,7 @@ private void recoverConnectionReset() throws IOException{ * @see ArtifactIdent */ public ArtifactIdent processArtifactIdent(String gav) { - String[] parts = gav.split(IndexWalker.splitPattern); + String[] parts = gav.split(splitPattern); return new ArtifactIdent(parts[0], parts[1], parts[2]); } @@ -110,7 +113,7 @@ public ArtifactIdent processArtifactIdent(String gav) { */ public Package processPackage(String information, String checksum) { if(information != null) { - String[] parts = information.split(IndexWalker.splitPattern); + String[] parts = information.split(splitPattern); return new Package(parts[0], Long.parseLong(parts[1]), Long.parseLong(parts[2]), Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5]), checksum); } return null; @@ -141,7 +144,7 @@ private IndexInformation processIndex(Map item, ArtifactIdent id //Create an artifact using the values found in the 'i' and '1' tags if(iVal != null) { - String[] parts = iVal.split(IndexWalker.splitPattern); + String[] parts = iVal.split(splitPattern); Package tmpPackage = new Package(parts[0], Long.parseLong(parts[1]), Long.parseLong(parts[2]), Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5]), item.get("1")); diff --git a/src/test/java/org/tudo/sse/IndexWalkerTest.java b/src/test/java/org/tudo/sse/IndexWalkerTest.java deleted file mode 100644 index 99347c8..0000000 --- a/src/test/java/org/tudo/sse/IndexWalkerTest.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.tudo.sse; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.tudo.sse.model.Artifact; -import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.index.Package; -import org.tudo.sse.model.index.IndexInformation; - -import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - -@SuppressWarnings("unchecked") -class IndexWalkerTest { - - IndexWalker indexWalker; - - final Map json; - final Gson gson = new Gson(); - - { - InputStream resource = this.getClass().getClassLoader().getResourceAsStream("IndexInputs.json"); - assert resource != null; - Reader targetReader = new InputStreamReader(resource); - json = gson.fromJson(targetReader, new TypeToken>() {}.getType()); - } - - @BeforeEach - void setUp() { - try { - indexWalker = new IndexWalker(new URI("https://repo1.maven.org/maven2/")); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - - @Test - void walkPaginatedEndings() { - int[][] startNtake = {{20, 30}, {20, 1}, {100000, 200000}, {500, 6000}, {6000, 10}}; - List idents; - List indexArtifacts; - - for (int[] ints : startNtake) { - try { - indexArtifacts = indexWalker.walkPaginated(ints[0], ints[1]); - idents = indexWalker.lazyWalkPaginated(ints[0], ints[1]); - } catch (IOException e) { - throw new RuntimeException(e); - } - assertEquals(ints[1], indexArtifacts.size()); - assertEquals(ints[1], idents.size()); - } - } - - @Test - void walkPaginated() { - ArrayList> expectedValues = (ArrayList>) json.get("walkPaginated"); - List results; - try { - results = indexWalker.walkPaginated(1000, 10); - } catch (IOException e) { - throw new RuntimeException(e); - } - - int i = 3; - for(Map value: expectedValues) { - IndexInformation currentArtifact = results.get(i).getIndexInformation(); - - assertEquals(value.get("ident"), currentArtifact.getIdent().getCoordinates()); - assertEquals(value.get("name"), currentArtifact.getName()); - assertEquals(Long.parseLong((String) value.get("index")), currentArtifact.getIndex()); - - ArrayList> expectedpackages = (ArrayList>) value.get("packages"); - List packages = currentArtifact.getPackages(); - for(int j = 0; j < expectedpackages.size(); j++) { - assertEquals(expectedpackages.get(j).get("packaging"), packages.get(j).getPackaging()); - assertEquals(Long.parseLong(expectedpackages.get(j).get("lastModified")), packages.get(j).getLastModified()); - assertEquals(Long.parseLong(expectedpackages.get(j).get("size")), packages.get(j).getSize()); - assertEquals(Integer.parseInt(expectedpackages.get(j).get("sourcesExist")), packages.get(j).getSourcesExist()); - assertEquals(Integer.parseInt(expectedpackages.get(j).get("javadocExist")), packages.get(j).getJavadocExists()); - assertEquals(Integer.parseInt(expectedpackages.get(j).get("signatureExist")), packages.get(j).getSignatureExists()); - assertEquals(expectedpackages.get(j).get("sha1checksum"), packages.get(j).getSha1checksum()); - } - i += 2; - } - - } -} \ No newline at end of file diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index b5fb274..8bae9c1 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -159,8 +159,6 @@ public void analyzeArtifact(Artifact current) { } }; - - List> inputs = new ArrayList<>(); inputs.add(new Tuple2<>(500, 10)); inputs.add(new Tuple2<>(0, 10)); @@ -168,35 +166,22 @@ public void analyzeArtifact(Artifact current) { inputs.add(new Tuple2<>(763, 20)); for(Tuple2 input : inputs) { - int start1 = input._1; + int skip = input._1; int take = input._2; - int start2 = (start1 + take) - 1; - try { - IndexIterator iterator = new IndexIterator(new URI(base), start1); + IndexIterator iterator = new IndexIterator(new URI(base), skip); theAnalysis.walkPaginated(take, iterator); assertFalse(artifactsSeen.isEmpty()); + assertEquals(take, artifactsSeen.size()); - Artifact lastOne = artifactsSeen.get(artifactsSeen.size() - 1); - - assertNotNull(lastOne); - - int i = 2; - while(lastOne.getIndexInformation().getIndex() > start2) { - lastOne = artifactsSeen.get(artifactsSeen.size() - i); - assertNotNull(lastOne); - i++; + for(Artifact a : artifactsSeen){ + assertTrue(a.hasIndexInformation()); + assertTrue(a.getIndexInformation().getIndex() >= (skip - 1)); } - iterator = new IndexIterator(new URI(base), start2); - artifactsSeen.clear(); - theAnalysis.walkPaginated(1, iterator); - - long lastOne2 = artifactsSeen.get(0).getIndexInformation().getIndex(); - assertEquals(lastOne.getIndexInformation().getIndex(), lastOne2); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); } @@ -289,8 +274,8 @@ void checkMultiThreading() { List singleArgs = new ArrayList<>(); List multiArgs = new ArrayList<>(); - singleArgs.add(new String[]{"-st", "10:1000"}); - multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"}); + singleArgs.add(new String[]{"-st", "10:200"}); + multiArgs.add(new String[]{"--threads", "5", "-st", "10:200"}); singleArgs.add(new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt"}); multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/test/resources/artifact-names-valid.txt"}); diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index 8ca7d1f..84e1c8c 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -101,10 +101,17 @@ void parseNonExistingIndex(){ @Test @DisplayName("An analysis with no requirements must not produce information instances") - void simpleIndexAnalysis(){ + void simpleIndexAnalysis() throws IOException { final List librariesHit = new java.util.ArrayList<>(); + final Path validLibraryInput = testResource("library-names-valid.txt"); + + assertNotNull(validLibraryInput); + assert(Files.exists(validLibraryInput)); + + final List expectedLibraries = Files.readAllLines(validLibraryInput); + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { @Override protected void analyzeLibrary(String libraryGA, List releases) { @@ -116,20 +123,28 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 0:3"; + final String args = "-st 2:2 --inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); - assertEquals(3, librariesHit.size()); - assert(librariesHit.contains("yom:yom")); + assertEquals(2, librariesHit.size()); + + assertTrue(librariesHit.contains(expectedLibraries.get(2))); + assertTrue(librariesHit.contains(expectedLibraries.get(3))); } @Test @DisplayName("An analysis with JAR requirements must produce JAR information instances") - void simpleIndexWithJarAnalysis(){ + void simpleIndexWithJarAnalysis() throws IOException { final List librariesHit = new java.util.ArrayList<>(); + final Path validLibraryInput = testResource("library-names-valid.txt"); + + assertNotNull(validLibraryInput); + assert(Files.exists(validLibraryInput)); + + final List expectedLibraries = Files.readAllLines(validLibraryInput); final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, true) { @Override @@ -142,14 +157,15 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 3:3"; + final String args = "-st 2:2 --inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); - assertEquals(3, librariesHit.size()); - assertFalse(librariesHit.contains("yom:yom")); - assert(librariesHit.contains("yan:yan")); + assertEquals(2, librariesHit.size()); + + assertTrue(librariesHit.contains(expectedLibraries.get(2))); + assertTrue(librariesHit.contains(expectedLibraries.get(3))); } @Test @@ -167,6 +183,13 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertNull(releases.get(0).getJarInformation()); librariesHit.add(libraryGA); } + + @Override + protected void onVersionListError(String g, String a, Exception x){ + assertTrue(x.getCause().getMessage().contains("was not found")); + String ga = g+":"+a; + librariesHit.add(ga); + } }; final String args = "-st 5000:10 --save-progress-interval 5"; @@ -198,6 +221,12 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertNull(releases.get(0).getJarInformation()); count.incrementAndGet(); } + + @Override + protected void onVersionListError(String g, String a, Exception x){ + assertTrue(x.getCause().getMessage().contains("was not found")); + count.incrementAndGet(); + } }; final String args = "-st 5000:500 --threads 8"; diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index 05e1bf3..9b39d1c 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -1,22 +1,22 @@ package org.tudo.sse.resolution; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.tudo.sse.IndexWalker; import org.tudo.sse.model.*; import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; +import java.nio.file.Files; +import java.util.*; import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Map; -import java.util.ArrayList; import java.util.stream.IntStream; import org.apache.commons.io.IOUtils; @@ -25,9 +25,11 @@ import org.tudo.sse.model.pom.RawPomFeatures; import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.resolution.releases.IReleaseListProvider; +import org.tudo.sse.utils.JsonExporter; import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; @SuppressWarnings("unchecked") class PomResolverTest { @@ -73,37 +75,62 @@ void processRawPomFeatures() { } @Test - void processArtifacts() { - //walk 10 indexes - List idents; - try { - IndexWalker walker = new IndexWalker(new URI("https://repo1.maven.org/maven2/")); - idents = walker.lazyWalkPaginated(200000, 10); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); + @DisplayName("Assert correct raw pom feature contents") + void processArtifacts() throws IOException { + List identStrings = Files.readAllLines(Objects.requireNonNull(testResource("artifact-names-valid.txt"))); + + List idents = new ArrayList<>(); + + for(String ident : identStrings) { + String[] parts = ident.split(":"); + idents.add(new ArtifactIdent(parts[0], parts[1], parts[2])); } ArrayList> expected = (ArrayList>) json.get("resolveArtifacts"); List poms = pomResolver.resolveArtifacts(idents, testContext); + assertEquals(10, poms.size()); + //check to see that the parsed data from the POM for these indexes are correct for actually retrieving the files from url for(int i = 0; i < expected.size(); i++) { - RawPomFeatures current = poms.get(i).getPomInformation().getRawPomFeatures(); - checkRawFeatures(current, expected, i); + assertNotNull(poms.get(i).getPomInformation()); + assertNotNull(poms.get(i).getPomInformation().getRawPomFeatures()); + checkRawFeatures(poms.get(i).getPomInformation().getRawPomFeatures(), expected, i); } } - //touch this up to throw all exceptions, should have 100% coverage for the pomResolver class - void processArtifactsCov() { - List idents; - try { - IndexWalker walker = new IndexWalker(new URI("https://repo1.maven.org/maven2/")); - idents = walker.lazyWalkPaginated(0, 10000); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); + @Test + @DisplayName("Export new expected values for: Assert correct raw pom feature contents") + @Disabled + void exportRawPomFeatures() throws IOException { + final Gson exportGson = new GsonBuilder().setPrettyPrinting().create(); + List identStrings = Files.readAllLines(Objects.requireNonNull(testResource("artifact-names-valid.txt"))); + + List idents = new ArrayList<>(); + + for(String ident : identStrings) { + String[] parts = ident.split(":"); + idents.add(new ArtifactIdent(parts[0], parts[1], parts[2])); + } + + ArrayList> expected = (ArrayList>) json.get("resolveArtifacts"); + + List poms = pomResolver.resolveArtifacts(idents, testContext); + + assertEquals(10, poms.size()); + + JsonArray exportArray = new JsonArray(); + + //check to see that the parsed data from the POM for these indexes are correct for actually retrieving the files from url + for(int i = 0; i < expected.size(); i++) { + assertNotNull(poms.get(i).getPomInformation()); + assertNotNull(poms.get(i).getPomInformation().getRawPomFeatures()); + JsonObject exportObject = JsonExporter.exportRawPomFeatures(poms.get(i).getPomInformation().getRawPomFeatures()); + exportArray.add(exportObject); } + exportGson.toJson(exportArray, System.out); } @Test diff --git a/src/test/java/org/tudo/sse/utils/JsonExporter.java b/src/test/java/org/tudo/sse/utils/JsonExporter.java new file mode 100644 index 0000000..2edf2e5 --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/JsonExporter.java @@ -0,0 +1,85 @@ +package org.tudo.sse.utils; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.tudo.sse.model.pom.RawPomFeatures; + +public class JsonExporter { + + public static JsonObject exportRawPomFeatures(RawPomFeatures features) { + JsonObject json = new JsonObject(); + + String parent = "null"; + if(features.getParent() != null){ + parent = features.getParent().getCoordinates(); + } + + json.add("parent", new JsonPrimitive(parent)); + + String name = "null"; + if(features.getName() != null){ + name = features.getName(); + } + + json.add("name", new JsonPrimitive(name)); + + String description = "null"; + if(features.getDescription() != null){ + description = features.getDescription(); + } + + json.add("description", new JsonPrimitive(description)); + + String url = "null"; + if(features.getUrl() != null){ + url = features.getUrl(); + } + + json.add("url", new JsonPrimitive(url)); + + String packaging = "null"; + if(features.getPackaging() != null){ + packaging = features.getPackaging(); + } + + json.add("packaging", new JsonPrimitive(packaging)); + + String inceptionYear = "null"; + if(features.getInceptionYear() != null){ + inceptionYear = features.getInceptionYear(); + } + + json.add("inceptionYear", new JsonPrimitive(inceptionYear)); + + if(!features.getProperties().isEmpty()){ + JsonArray properties = new JsonArray(features.getProperties().size()); + features.getProperties().forEach((k,v)->{ + JsonArray propArray = new JsonArray(2); + propArray.add(new JsonPrimitive(k)); + propArray.add(new JsonPrimitive(v)); + properties.add(propArray); + }); + json.add("properties", properties); + } + + JsonArray dependencies = new JsonArray(); + features.getDependencies().forEach(dependency -> dependencies.add(new JsonPrimitive(dependency.getIdent().getCoordinates()))); + + json.add("dependencies", dependencies); + + JsonArray licenses = new JsonArray(); + features.getLicenses().forEach(license -> licenses.add(new JsonPrimitive(license.getUrl()))); + json.add("licenses", licenses); + + if(features.getDependencyManagement() != null){ + JsonArray dependencyManagement = new JsonArray(); + features.getDependencyManagement().forEach(dependency -> dependencyManagement.add(new JsonPrimitive(dependency.getIdent().getCoordinates()))); + json.add("dependencyManagement", dependencyManagement); + } else { + json.add("dependencyManagement", new JsonPrimitive("null")); + } + + return json; + } +} diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java index 8d3b53e..50b17ff 100644 --- a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -60,22 +60,6 @@ void uniqueGAs() { } } - @Test - @DisplayName("The Library Index Iterator must apply custom starting positions") - void customStartingPositions() { - iteratorUnderTest.setPosition(42000); - - int cutoff = 1000; - int idx = 0; - - while(iteratorUnderTest.hasNext() && idx < cutoff){ - final String ga = iteratorUnderTest.next(); - assert(!ga.equals("yom:yom")); // Make sure we skipped the first library in index! - idx += 1; - } - - } - @Test @Disabled @DisplayName("The Library Index Iterator must terminate") diff --git a/src/test/java/org/tudo/sse/utils/TestUtilities.java b/src/test/java/org/tudo/sse/utils/TestUtilities.java index 7e7a686..ad34151 100644 --- a/src/test/java/org/tudo/sse/utils/TestUtilities.java +++ b/src/test/java/org/tudo/sse/utils/TestUtilities.java @@ -1,6 +1,14 @@ package org.tudo.sse.utils; +import org.tudo.sse.analyses.MavenCentralArtifactIterator; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.model.Artifact; + import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import static org.junit.jupiter.api.Assertions.fail; @@ -16,4 +24,24 @@ public static Path testResource(String pathToResource){ return null; } + public static List getFromIndex(int skip, int take, boolean resolvePom, boolean resolveTransitives, boolean resolveJar){ + try { + final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder().withSkip(skip).withTake(take).build(); + + final MavenCentralArtifactIterator it = new MavenCentralArtifactIterator(resolvePom, resolveTransitives, resolveJar, config); + + List artifacts = new ArrayList<>(); + + while(it.hasNext()){ + artifacts.add(it.next()); + } + + return artifacts; + } catch(InvalidConfigurationException icx){ + fail(icx); + } + + return null; + } + } diff --git a/src/test/resources/PomInputs.json b/src/test/resources/PomInputs.json index bab2673..e03f9f7 100644 --- a/src/test/resources/PomInputs.json +++ b/src/test/resources/PomInputs.json @@ -206,75 +206,301 @@ }, "resolveArtifacts": [ { - "parent": "net.officefloor.plugin:plugins:1.3.0", - "name": "WAR Plug-in", - "description": "OfficeFloor plug-in for utilising WAR", - "url": "null", + "parent": "null", + "name": "spring-boot-starter-web", + "description": "Starter for building web, including RESTful, applications using Spring MVC. Uses Tomcat as the default embedded container", + "url": "https://spring.io/projects/spring-boot", "packaging": "jar", - "properties": [ - ["project.build.sourceEncoding", "UTF-8"] - ], "inceptionYear": "null", "dependencies": [ - "net.officefloor.core:officebuilding:null", - "${project.groupId}:officeplugin_servlet:null", - "org.eclipse.jetty:jetty-servlet:null", - "org.apache.httpcomponents:httpclient:null" + "org.springframework.boot:spring-boot-starter:2.6.3", + "org.springframework.boot:spring-boot-starter-json:2.6.3", + "org.springframework.boot:spring-boot-starter-tomcat:2.6.3", + "org.springframework:spring-web:5.3.15", + "org.springframework:spring-webmvc:5.3.15" + ], + "licenses": [ + "https://www.apache.org/licenses/LICENSE-2.0" ], - "licenses": [], "dependencyManagement": "null" }, { - "parent": "net.officefloor.plugin:plugins:1.2.0", - "name": "WAR Plug-in", - "description": "OfficeFloor plug-in for utilising WAR", - "url": "null", + "parent": "org.apache.commons:commons-parent:52", + "name": "Apache Commons Lang", + "description": "Apache Commons Lang, a package of Java utility classes for the\n classes that are in java.lang\u0027s hierarchy, or are considered to be so\n standard as to justify existence in java.lang.", + "url": "https://commons.apache.org/proper/commons-lang/", "packaging": "jar", + "inceptionYear": "2001", "properties": [ - ["project.build.sourceEncoding", "UTF-8"] + [ + "jmh.version", + "1.27" + ], + [ + "commons.release.version", + "3.12.0" + ], + [ + "commons.encoding", + "utf-8" + ], + [ + "commons.bc.version", + "3.11" + ], + [ + "commons.site.path", + "lang" + ], + [ + "commons.jira.id", + "LANG" + ], + [ + "commons.scmPubCheckoutDirectory", + "site-content" + ], + [ + "commons.releaseManagerName", + "Gary Gregory" + ], + [ + "commons.releaseManagerKey", + "86fdc7e2a11262cb" + ], + [ + "commons.componentid", + "lang" + ], + [ + "uberjar.name", + "benchmarks" + ], + [ + "commons.javadoc.version", + "3.2.0" + ], + [ + "commons.japicmp.version", + "0.15.2" + ], + [ + "project.build.sourceEncoding", + "ISO-8859-1" + ], + [ + "commons.module.name", + "org.apache.commons.lang3" + ], + [ + "spotbugs.impl.version", + "4.2.1" + ], + [ + "commons.release.2.version", + "2.6" + ], + [ + "clirr.skip", + "true" + ], + [ + "commons.release.desc", + "(Java 8+)" + ], + [ + "maven.compiler.source", + "1.8" + ], + [ + "checkstyle.configdir", + "src/site/resources/checkstyle" + ], + [ + "commons.jira.pid", + "12310481" + ], + [ + "commons.distSvnStagingUrl", + "scm:svn:https://dist.apache.org/repos/dist/dev/commons/lang" + ], + [ + "project.reporting.outputEncoding", + "UTF-8" + ], + [ + "commons.rc.version", + "RC1" + ], + [ + "commons.surefire.version", + "3.0.0-M5" + ], + [ + "maven.compiler.target", + "1.8" + ], + [ + "commons.release.2.name", + "commons-lang-${commons.release.2.version}" + ], + [ + "spotbugs.plugin.version", + "4.2.0" + ], + [ + "commons.scmPubUrl", + "https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-lang" + ], + [ + "checkstyle.version", + "8.40" + ], + [ + "japicmp.skip", + "false" + ], + [ + "checkstyle.plugin.version", + "3.1.2" + ], + [ + "commons.jacoco.version", + "0.8.6" + ], + [ + "commons.release.isDistModule", + "true" + ], + [ + "commons.release.2.desc", + "(Requires Java 1.2 or later)" + ], + [ + "commons.packageId", + "lang3" + ], + [ + "argLine", + "-Xmx512m" + ] ], - "inceptionYear": "null", "dependencies": [ - "net.officefloor.core:officebuilding:null", - "${project.groupId}:officeplugin_servlet:null", - "org.eclipse.jetty:jetty-servlet:null", - "org.apache.httpcomponents:httpclient:null" + "org.junit.jupiter:junit-jupiter:null", + "org.junit-pioneer:junit-pioneer:1.3.0", + "org.hamcrest:hamcrest:2.2", + "org.easymock:easymock:4.2", + "org.openjdk.jmh:jmh-core:${jmh.version}", + "org.openjdk.jmh:jmh-generator-annprocess:${jmh.version}", + "com.google.code.findbugs:jsr305:3.0.2" ], "licenses": [], - "dependencyManagement": "null" + "dependencyManagement": [ + "org.junit:junit-bom:5.7.1" + ] }, { - "parent": "net.officefloor.plugin:plugins:1.1.0", - "name": "WAR Plug-in", - "description": "OfficeFloor plug-in for utilising WAR", - "url": "null", - "packaging": "jar", + "parent": "com.fasterxml.jackson:jackson-base:2.13.1", + "name": "jackson-databind", + "description": "General data-binding functionality for Jackson: works on core streaming API", + "url": "http://github.com/FasterXML/jackson", + "packaging": "bundle", + "inceptionYear": "2008", "properties": [ - ["project.build.sourceEncoding", "UTF-8"] + [ + "javac.src.version", + "1.8" + ], + [ + "packageVersion.dir", + "com/fasterxml/jackson/databind/cfg" + ], + [ + "javac.target.version", + "1.8" + ], + [ + "osgi.import", + "org.w3c.dom.bootstrap;resolution:\u003doptional,\n *" + ], + [ + "packageVersion.package", + "com.fasterxml.jackson.databind.cfg" + ], + [ + "osgi.export", + "com.fasterxml.jackson.databind.*;version\u003d${project.version}" + ], + [ + "version.powermock", + "2.0.0" + ] ], - "inceptionYear": "null", "dependencies": [ - "net.officefloor.core:officebuilding:null", - "${project.groupId}:officeplugin_servlet:null", - "org.apache.httpcomponents:httpclient:null" + "com.fasterxml.jackson.core:jackson-annotations:${jackson.version.annotations}", + "com.fasterxml.jackson.core:jackson-core:${jackson.version.core}", + "org.powermock:powermock-core:${version.powermock}", + "org.powermock:powermock-module-junit4:${version.powermock}", + "org.powermock:powermock-api-mockito2:${version.powermock}", + "javax.measure:jsr-275:0.9.1" + ], + "licenses": [ + "http://www.apache.org/licenses/LICENSE-2.0.txt" ], - "licenses": [], "dependencyManagement": "null" }, { - "parent" : "net.officefloor.plugin:plugins:1.4.0", - "name" : "Spring Plug-in", - "description" : "Integration with Spring to allow use of its BeanFactory (specifically XmlBeanFactory).", - "url" : "null", - "packaging" : "jar", - "properties" : [ - ["project.build.sourceEncoding", "UTF-8"] + "parent": "null", + "name": "JUnit", + "description": "JUnit is a unit testing framework for Java, created by Erich Gamma and Kent Beck.", + "url": "http://junit.org", + "packaging": "jar", + "inceptionYear": "2002", + "properties": [ + [ + "surefireVersion", + "2.19.1" + ], + [ + "jdkVersion", + "1.5" + ], + [ + "project.build.sourceEncoding", + "ISO-8859-1" + ], + [ + "jarPluginVersion", + "2.6" + ], + [ + "javadocPluginVersion", + "2.10.3" + ], + [ + "enforcerPluginVersion", + "1.4" + ], + [ + "arguments", + "" + ], + [ + "hamcrestVersion", + "1.3" + ], + [ + "gpg.keyname", + "67893CC4" + ] ], - "inceptionYear": "null", "dependencies": [ - "org.springframework:spring:null" + "org.hamcrest:hamcrest-core:${hamcrestVersion}", + "org.hamcrest:hamcrest-library:${hamcrestVersion}" + ], + "licenses": [ + "http://www.eclipse.org/legal/epl-v10.html" ], - "licenses": [], "dependencyManagement": "null" } ], From 3eb79e266d67ee6b919d02975a5dc4a7ab5be576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 6 Mar 2026 09:58:49 +0100 Subject: [PATCH 47/55] Fix Javadoc warning --- src/main/java/org/tudo/sse/utils/IndexIterator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/tudo/sse/utils/IndexIterator.java b/src/main/java/org/tudo/sse/utils/IndexIterator.java index 75780a0..fdb9a2d 100644 --- a/src/main/java/org/tudo/sse/utils/IndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/IndexIterator.java @@ -23,7 +23,10 @@ public class IndexIterator implements Iterator { - public static final String splitPattern = Pattern.quote("|"); + /** + * Pattern to split Lucene Index entries at + */ + private static final String splitPattern = Pattern.quote("|"); private long index; From 8a8c772f715f753dbb63ceebc2d49fc02276dae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 9 Mar 2026 10:11:49 +0100 Subject: [PATCH 48/55] Add fallback version list provider based on HTML parsing to compensate for missing metadata files --- pom.xml | 6 ++ .../DefaultMavenReleaseListProvider.java | 22 +++++- .../HTMLBasedMavenReleaseListProvider.java | 76 +++++++++++++++++++ .../MavenCentralLibraryAnalysisTest.java | 5 +- .../DefaultMavenReleaseListProviderTest.java | 28 +++++++ ...HTMLBasedMavenReleaseListProviderTest.java | 58 ++++++++++++++ 6 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java create mode 100644 src/test/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProviderTest.java create mode 100644 src/test/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProviderTest.java diff --git a/pom.xml b/pom.xml index c838afa..b0247dc 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,12 @@ 5.10.0 test + + org.jsoup + jsoup + 1.22.1 + compile + org.apache.maven.indexer indexer-reader diff --git a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java index 0a83c83..9e2cb59 100644 --- a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java +++ b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java @@ -4,6 +4,8 @@ import org.apache.maven.artifact.repository.metadata.Versioning; import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader; import org.codehaus.plexus.util.xml.pull.XmlPullParserException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.resolution.FileNotFoundException; import org.tudo.sse.utils.MavenCentralRepository; @@ -26,6 +28,8 @@ */ public class DefaultMavenReleaseListProvider implements IReleaseListProvider{ + private final Logger log = LoggerFactory.getLogger(this.getClass()); + private final MetadataXpp3Reader reader = new MetadataXpp3Reader(); private final MavenCentralRepository mavenRepo = MavenCentralRepository.getInstance(); @@ -50,17 +54,27 @@ public List getReleases(String groupId, String artifactId) throws IOExce Versioning versioning = versionListData.getVersioning(); - if(versioning != null) { + if (versioning != null) { List versions = new ArrayList<>(); - for(Object version : versioning.getVersions()) { - versions.add((String)version); + for (Object version : versioning.getVersions()) { + versions.add((String) version); } return versions; } else { return List.of(); } + } + catch(FileNotFoundException fnfx){ + log.warn("No `maven-metadata.xml` found for {}:{}, using fallback HTML-based version list provider",groupId,artifactId); + final IReleaseListProvider fallbackProvider = HTMLBasedMavenReleaseListProvider.getInstance(); - } catch (XmlPullParserException | IOException | FileNotFoundException | URISyntaxException x) { + try { + return fallbackProvider.getReleases(groupId, artifactId); + } catch (IOException iox){ + throw new IOException("Failed to obtain version list for " + groupId + ":" + artifactId, iox); + } + } + catch (XmlPullParserException | IOException | URISyntaxException x) { throw new IOException("Failed to obtain version list for " + groupId + ":" + artifactId, x); } } diff --git a/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java new file mode 100644 index 0000000..ca61ab8 --- /dev/null +++ b/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java @@ -0,0 +1,76 @@ +package org.tudo.sse.resolution.releases; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.tudo.sse.resolution.FileNotFoundException; +import org.tudo.sse.utils.MavenCentralRepository; +import org.tudo.sse.utils.ResourceConnections; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; + +public class HTMLBasedMavenReleaseListProvider implements IReleaseListProvider { + + // Singleton pattern implementation + private static final HTMLBasedMavenReleaseListProvider instance = new HTMLBasedMavenReleaseListProvider(); + + /** + * Access the singleton instance of HtmlBasedMavenReleaseListProvider + * @return The only instance of this class + */ + public static HTMLBasedMavenReleaseListProvider getInstance() { + return instance; + } + + // Singleton pattern implementation + private HTMLBasedMavenReleaseListProvider(){} + + @Override + public List getReleases(String groupId, String artifactId) throws IOException { + + InputStream content = null; + try { + URI libraryBase = MavenCentralRepository.buildLibraryBaseURI(groupId, artifactId); + content = ResourceConnections.openInputStream(libraryBase); + + if(content != null){ + //https://repo1.maven.org/maven2/uk/org/retep/tools/math/ + Document document = Jsoup.parse(content, "UTF-8", "https://www.example.com"); + Element listElem = document.getElementById("contents"); + + if(listElem == null){ + // If there is only one version, the "list" seems to be in => 
+                    final Element body = document.getElementsByTag("body").first();
+                    listElem = body == null ? null : body.getElementsByTag("pre").first();
+
+                    if(listElem == null)
+                        throw new IOException("Could not locate HTML element with ID 'contents' in version list HTML");
+                }
+
+                List versions = new ArrayList();
+                listElem.getElementsByTag("a").forEach( linkElem -> {
+                    String hrefValue = linkElem.attr("href");
+
+                    if(!hrefValue.equals("../") && hrefValue.endsWith("/")){
+                        versions.add(hrefValue.substring(0, hrefValue.length() - 1));
+                    }
+                });
+
+                return versions;
+            } else {
+                throw new IOException("Failed to open HTML connection to " + libraryBase);
+            }
+
+        } catch (URISyntaxException | FileNotFoundException x){
+            throw new IOException(x);
+        } finally {
+            if(content != null)
+                content.close();
+        }
+    }
+
+}
diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
index 84e1c8c..f8b28b9 100644
--- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
+++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
@@ -186,7 +186,7 @@ protected void analyzeLibrary(String libraryGA, List releases) {
 
             @Override
             protected void onVersionListError(String g, String a, Exception x){
-                assertTrue(x.getCause().getMessage().contains("was not found"));
+                assertFalse(x.getCause().getMessage().contains("was not found"));
                 String ga = g+":"+a;
                 librariesHit.add(ga);
             }
@@ -224,7 +224,8 @@ protected void analyzeLibrary(String libraryGA, List releases) {
 
             @Override
             protected void onVersionListError(String g, String a, Exception x){
-                assertTrue(x.getCause().getMessage().contains("was not found"));
+
+                assertFalse(x.getCause().getMessage().contains("was not found"));
                 count.incrementAndGet();
             }
         };
diff --git a/src/test/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProviderTest.java b/src/test/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProviderTest.java
new file mode 100644
index 0000000..1a44edc
--- /dev/null
+++ b/src/test/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProviderTest.java
@@ -0,0 +1,28 @@
+package org.tudo.sse.resolution.releases;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class DefaultMavenReleaseListProviderTest {
+
+    private final IReleaseListProvider provider = DefaultMavenReleaseListProvider.getInstance();
+
+    @Test
+    @DisplayName("The Default Release List Provider must fall back to the HTML provider if no metadata file is present")
+    void testNoExtraFiles(){
+        try {
+            List versions = provider.getReleases("uk.org.retep.tools", "math");
+            assertTrue(versions.size() >= 6);
+            assertTrue(versions.contains("9.11.1"));
+        } catch (Exception x) {
+            fail(x);
+        }
+
+    }
+
+}
diff --git a/src/test/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProviderTest.java b/src/test/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProviderTest.java
new file mode 100644
index 0000000..52c6aec
--- /dev/null
+++ b/src/test/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProviderTest.java
@@ -0,0 +1,58 @@
+package org.tudo.sse.resolution.releases;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class HTMLBasedMavenReleaseListProviderTest {
+
+    private final IReleaseListProvider provider = HTMLBasedMavenReleaseListProvider.getInstance();
+
+    @Test
+    @DisplayName("The HTML Release List Provider must extract version lists for simple cases")
+    void testNoExtraFiles(){
+        try {
+            List versions = provider.getReleases("uk.org.retep.tools", "math");
+            assertTrue(versions.size() >= 6);
+            assertTrue(versions.contains("9.11.1"));
+        } catch (Exception x) {
+            fail(x);
+        }
+
+    }
+
+    @Test
+    @DisplayName("The HTML Release List Provider must extract version lists for real use-cases")
+    void testExtraFiles(){
+        try {
+            List versions = provider.getReleases("junit", "junit");
+            assertTrue(versions.size() >= 32);
+
+            assertTrue(versions.contains("3.7"));
+            assertTrue(versions.contains("4.9"));
+            assertTrue(versions.contains("4.13-beta-1"));
+
+            assertFalse(versions.contains("maven-metadata.xml"));
+            assertFalse(versions.contains("maven-metadata.xml.md5"));
+            assertFalse(versions.contains(".."));
+        } catch (Exception x) {
+            fail(x);
+        }
+    }
+
+    @Test
+    @DisplayName("The HTML Release List Provider must extract version lists one-element lists (with particular HTML structure)")
+    void testSingletonList(){
+        try {
+            List versions = provider.getReleases("uk.nominet", "dnsjnio");
+            assertFalse(versions.isEmpty());
+            assertTrue(versions.contains("1.0.3"));
+
+        } catch (Exception x) {
+            fail(x);
+        }
+    }
+}

From 7fa7d661c965b0c91d1965fe70d82ba62a57f7f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 9 Mar 2026 10:32:13 +0100
Subject: [PATCH 49/55] Fix missing JavaDoc

---
 .../releases/DefaultMavenReleaseListProvider.java           | 5 +++--
 .../releases/HTMLBasedMavenReleaseListProvider.java         | 6 ++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java
index 9e2cb59..6eeb038 100644
--- a/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java
+++ b/src/main/java/org/tudo/sse/resolution/releases/DefaultMavenReleaseListProvider.java
@@ -21,8 +21,9 @@
 
 /**
  * This default implementation of IReleaseListProvider obtains a list of releases for a given GA-Tuple by
- * querying the maven-metadata.xml file hosted on Maven Central. It will throw an IOException if file
- * access files for any given reason.
+ * querying the maven-metadata.xml file hosted on Maven Central. If the file it not present, it will attempt
+ * to obtain a list of release versions from the HTML list provided by Maven Central. If both attempts fail,
+ * an IOException will be thrown.
  *
  * @author Johannes Düsing
  */
diff --git a/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java b/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java
index ca61ab8..af140da 100644
--- a/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java
+++ b/src/main/java/org/tudo/sse/resolution/releases/HTMLBasedMavenReleaseListProvider.java
@@ -13,6 +13,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * This implementation of IReleaseListProvider obtains a list of releases for a given GA-Tuple by parsing the
+ * HTML overview generated by Maven Central when fetching a GA's base URI.
+ *
+ * @author Johannes Düsing
+ */
 public class HTMLBasedMavenReleaseListProvider implements IReleaseListProvider {
 
     // Singleton pattern implementation

From 25ccd7d2d8a85bec254b084fbd97d1116aa95f39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 9 Mar 2026 11:45:32 +0100
Subject: [PATCH 50/55] Fix stack overflow error for circular dependencies

---
 .../org/tudo/sse/model/ResolutionContext.java | 32 +++++++++
 .../org/tudo/sse/resolution/PomResolver.java  | 70 +++++++++++++------
 .../tudo/sse/resolution/PomResolverTest.java  | 14 ++++
 3 files changed, 93 insertions(+), 23 deletions(-)

diff --git a/src/main/java/org/tudo/sse/model/ResolutionContext.java b/src/main/java/org/tudo/sse/model/ResolutionContext.java
index 7b66f4f..1b3544d 100644
--- a/src/main/java/org/tudo/sse/model/ResolutionContext.java
+++ b/src/main/java/org/tudo/sse/model/ResolutionContext.java
@@ -27,11 +27,18 @@ public static ResolutionContext createAnonymousContext(){
      */
     protected final Map artifactsResolved;
 
+    /**
+     * Set of artifact identifiers that are currently being under resolution by any Resolver. Helps avoid infinite
+     * resolution loops.
+     */
+    protected final Set currentlyResolving;
+
     /**
      * Builds a new ResolutionContext instance.
      */
     protected ResolutionContext(){
         this.artifactsResolved = new HashMap<>();
+        this.currentlyResolving = new HashSet<>();
     }
 
     /**
@@ -67,4 +74,29 @@ public Set getAllArtifactsResolved(){
         return new HashSet<>(artifactsResolved.values());
     }
 
+    /**
+     * Check whether the given artifact identifier is currently being resolved in this context.
+     * @param ident Artifact identifier to check
+     * @return True if this identifier is currently being resolved, false otherwise
+     */
+    public boolean isCurrentlyResolving(ArtifactIdent ident){
+        return this.currentlyResolving.contains(ident);
+    }
+
+    /**
+     * Mark the given artifact identifier as currently being resolved in this context.
+     * @param ident The identifier to mark
+     */
+    public void setIsCurrentlyResolving(ArtifactIdent ident){
+        this.currentlyResolving.add(ident);
+    }
+
+    /**
+     * Mark the given artifact identifier as not being resolved anymore in this context.
+     * @param ident The identifier to mark
+     */
+    public void setIsFinishedResolving(ArtifactIdent ident){
+        this.currentlyResolving.remove(ident);
+    }
+
 }
diff --git a/src/main/java/org/tudo/sse/resolution/PomResolver.java b/src/main/java/org/tudo/sse/resolution/PomResolver.java
index 6c8312a..05f1a55 100644
--- a/src/main/java/org/tudo/sse/resolution/PomResolver.java
+++ b/src/main/java/org/tudo/sse/resolution/PomResolver.java
@@ -352,17 +352,30 @@ public List getDependencies(List
     public Artifact processArtifact(ArtifactIdent identifier, ResolutionContext ctx) throws PomResolutionException, FileNotFoundException, IOException {
         final Artifact currentArtifact = ctx.createArtifact(identifier);
 
+        // If resolution on this artifact already begun (because, e.g. we are in an infinite dependency loop),
+        // we do not re-start resolution, but just return the artifact object - it will be fully resolved
+        // eventually
+        if(ctx.isCurrentlyResolving(identifier))
+            return currentArtifact;
+
         // Only build pom information if it is not already present for the current resolution context
         if(currentArtifact.hasPomInformation())
             return currentArtifact;
 
+        // Record that we are currently resolving this artifact
+        ctx.setIsCurrentlyResolving(identifier);
+
         PomInformation pomInformation = new PomInformation(identifier);
 
         RawPomFeatures rawPomFeatures;
         try(InputStream is = MavenRepo.openPomFileInputStream(identifier) ) {
             rawPomFeatures = processRawPomFeatures(is, identifier);
         } catch (SocketException e) {
+            ctx.setIsFinishedResolving(identifier);
             throw new PomResolutionException(e.getMessage(), identifier, e);
+        } catch(Exception x){
+            ctx.setIsFinishedResolving(identifier);
+            throw x;
         }
 
         ArtifactIdent relocation = null;
@@ -382,6 +395,9 @@ public Artifact processArtifact(ArtifactIdent identifier, ResolutionContext ctx)
         // Finally set pom information for current artifact, then return
         currentArtifact.setPomInformation(pomInformation);
 
+        // Record that we are no longer currently resolving this artifact
+        ctx.setIsFinishedResolving(identifier);
+
         return currentArtifact;
     }
 
@@ -540,18 +556,22 @@ private org.tudo.sse.model.pom.Dependency resolveScope(org.tudo.sse.model.pom.De
 
             if(current.getImports() != null) {
                 for(Artifact anImport : current.getImports()) {
-                    if(anImport.getPomInformation().getRawPomFeatures().getDependencyManagement() != null) {
-                        Tuple2 depStuff = findGA(missingVersion, anImport.getPomInformation().getRawPomFeatures().getDependencyManagement());
-                        if(depStuff != null) {
-                            if(depStuff._2 != null) {
-                                toReturn.setScope(depStuff._2);
-                                return toReturn;
+                    // Import resolution may have failed because of invalid import specifications. In that case, no POM
+                    // information will be present. We only try to expand imports for which we have pom information.
+                    if(anImport.hasPomInformation()) {
+                        if(anImport.getPomInformation().getRawPomFeatures().getDependencyManagement() != null) {
+                            Tuple2 depStuff = findGA(missingVersion, anImport.getPomInformation().getRawPomFeatures().getDependencyManagement());
+                            if(depStuff != null) {
+                                if(depStuff._2 != null) {
+                                    toReturn.setScope(depStuff._2);
+                                    return toReturn;
+                                }
                             }
                         }
-                    }
-                    toReturn = recursiveHandler(anImport.getPomInformation(), toReturn);
-                    if(toReturn.getScope() != null) {
-                        return toReturn;
+                        toReturn = recursiveHandler(anImport.getPomInformation(), toReturn);
+                        if(toReturn.getScope() != null) {
+                            return toReturn;
+                        }
                     }
                 }
             }
@@ -629,21 +649,25 @@ private org.tudo.sse.model.pom.Dependency resolveVersion(org.tudo.sse.model.pom.
 
             if(!parentResolved && current.getImports() != null) {
                 for(Artifact anImport : current.getImports()) {
-                    if(anImport.getPomInformation().getRawPomFeatures().getDependencyManagement() != null) {
-                        Tuple2 depStuff = findGA(missingVersion, anImport.getPomInformation().getRawPomFeatures().getDependencyManagement());
-                        if(depStuff != null) {
-                            version = depStuff._1;
-                            if(depStuff._2 != null && toReturn.getScope() == null) {
-                                toReturn.setScope(depStuff._2);
+                    // Import resolution may have failed because of invalid import specifications. In that case, no POM
+                    // information will be present. We only try to expand imports for which we have pom information.
+                    if(anImport.hasPomInformation()){
+                        if(anImport.getPomInformation().getRawPomFeatures().getDependencyManagement() != null) {
+                            Tuple2 depStuff = findGA(missingVersion, anImport.getPomInformation().getRawPomFeatures().getDependencyManagement());
+                            if(depStuff != null) {
+                                version = depStuff._1;
+                                if(depStuff._2 != null && toReturn.getScope() == null) {
+                                    toReturn.setScope(depStuff._2);
+                                }
+                                toReturn.getIdent().setVersion(version);
+                                current = anImport.getPomInformation();
                             }
-                            toReturn.getIdent().setVersion(version);
-                            current = anImport.getPomInformation();
                         }
-                    }
-                    if(version == null) {
-                        toReturn = recursiveHandler(anImport.getPomInformation(), toReturn);
-                        curIdent = toReturn.getIdent();
-                        version = curIdent.getVersion();
+                        if(version == null) {
+                            toReturn = recursiveHandler(anImport.getPomInformation(), toReturn);
+                            curIdent = toReturn.getIdent();
+                            version = curIdent.getVersion();
+                        }
                     }
                 }
             }
diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java
index 9b39d1c..c890911 100644
--- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java
+++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java
@@ -401,4 +401,18 @@ void resolveFromSecondaryRepository() {
             }
         }
     }
+
+    @Test
+    @DisplayName("The POM resolver must not loop for circular dependencies")
+    void resolveDependencyLoop() {
+        // Based on https://github.com/sse-labs/marin/issues/52
+        final ArtifactIdent loopingArtifact = new ArtifactIdent("net.wicp.tams", "ts-maven-plugin", "8.0.2");
+        final ArtifactResolutionContext freshCtx = ArtifactResolutionContext.newInstance(loopingArtifact);
+
+        final Artifact artifact = pomResolver.resolveArtifacts(List.of(loopingArtifact), freshCtx).get(0);
+
+        assertNotNull(artifact);
+
+
+    }
 }
\ No newline at end of file

From b1272e66ceddf83a0799eb86a9d3c993ec73bf8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 30 Mar 2026 13:31:36 +0200
Subject: [PATCH 51/55] Create progress restore file with default value if it
 does not exist on startup

---
 .../config/LibraryAnalysisConfigBuilder.java  | 17 ++++++++++++-
 .../parsing/LibraryAnalysisConfigParser.java  |  2 +-
 .../MavenCentralLibraryAnalysisTest.java      | 24 +++++++++++++++----
 3 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java
index 7d43411..94c1e94 100644
--- a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java
+++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java
@@ -3,8 +3,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 
 /**
  * Configuration builder to obtain {@link LibraryAnalysisConfig} instances programmatically.
@@ -120,7 +122,20 @@ public LibraryAnalysisConfigBuilder withProgressOutputFile(Path outputFile) thro
      * @throws InvalidConfigurationException If the given configuration value is not valid
      */
     public LibraryAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException {
-        if(restoreFile == null || !Files.isRegularFile(restoreFile))
+
+        boolean fileExists = Files.isRegularFile(restoreFile);
+
+        if(!fileExists){
+            try {
+                log.info("Progress restore file does not yet exists, creating default at {}", restoreFile.toAbsolutePath());
+                Files.write(restoreFile, "0".getBytes(), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+                fileExists = true;
+            } catch(IOException iox){
+                log.warn("Failed to create restore file", iox);
+            }
+        }
+
+        if(!fileExists)
             throw new InvalidConfigurationException("progress-restore-file", "Restore file must be a valid file reference");
 
         if(this.theConfig.progressRestoreFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_RESTORE_FILE)
diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java
index ba2b95e..49667f3 100644
--- a/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java
+++ b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java
@@ -53,7 +53,7 @@ protected void handleParameter(String[] args, int i, LibraryAnalysisConfigBuilde
                     break;
                 case "-prf":
                 case "--progress-restore-file":
-                    configBuilder.withProgressRestoreFile(nextArgAsRegularFileReference(args, i));
+                    configBuilder.withProgressRestoreFile(nextArgAsPath(args, i));
                     break;
                 case "-spi":
                 case "--save-progress-interval":
diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
index f8b28b9..2d07a2f 100644
--- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
+++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
@@ -7,6 +7,7 @@
 import org.tudo.sse.model.Artifact;
 import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -70,11 +71,26 @@ void parseCLIDefaults(){
     }
 
     @Test
-    @DisplayName("The CLI parser must fail if input files do not exist")
-    void parseNonExistingFile(){
-        final String validArgs = "--progress-restore-file foo.input";
+    @DisplayName("The CLI parser must create a default progress restore file with progress 0 if file does not exist")
+    void parseNonExistingFile() throws IOException {
+        final String fileRef = "foo.input";
+        final File nonExistingInput = new File(fileRef);
 
-        assertThrows(RuntimeException.class, () -> parseCLI(validArgs));
+        Files.deleteIfExists(nonExistingInput.toPath());
+
+        assertFalse(nonExistingInput.exists());
+
+        final String validArgs = "--progress-restore-file " + fileRef;
+
+        final LibraryAnalysisConfig config = parseCLI(validArgs);
+
+        assertEquals(nonExistingInput.toPath(), config.progressRestoreFile);
+        assertTrue(nonExistingInput.exists());
+
+        String content = Files.readString(nonExistingInput.toPath());
+        assertEquals("0", content);
+
+        Files.deleteIfExists(nonExistingInput.toPath());
     }
 
     @Test

From d09a95ee248409d6f9600640f561ff2e4b66e945 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 30 Mar 2026 14:29:35 +0200
Subject: [PATCH 52/55] Improve documentation of logging requirements, improve
 examples in readme, change visibility of core analysis class

---
 README.md                                     | 63 +++++++++++++------
 pom.xml                                       |  1 +
 .../sse/analyses/MavenCentralAnalysis.java    |  7 ++-
 3 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 829e4b3..ce40972 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,16 @@ Add the following dependency to your `pom.xml` to add MARIN to your project:
 
 ```
 
+Ideally, make sure that you have a logging backend like `logback-classic` included in your project as well:
+
+```
+
+    ch.qos.logback
+    logback-classic
+    1.5.32
+
+```
+
 ## Required Java Version
 You will need Java 11 or higher to build MARIN yourself.
 
@@ -74,22 +84,24 @@ Once this is implemented, you can run your analysis using the following command.
 ### Using Iterators
 As opposed to extending a base class, in order to use MARIN's iterator implementations you will have to first create an analysis configuration object programmatically. This can be done using the `ArtifactAnalysisConfigBuilder` or `LibraryAnalysisConfigBuilder` classes, respectively. The following example shows how to first build a configuration, and then use it to initialize a `MavenCentralArtifactIterator`.
 ```java
-final boolean resolvePom = true;
-final boolean resolveTransitive = false;
-final boolean resolveJar = true;
-final Path gavInputList = Paths.get("path-to-input-file");
-
-final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder()
-                .withInputList(gavInputList)
-                .withSkip(2)
-                .withTake(5)
-                .build();
-
-MavenCentralArtifactIterator iterator = new MavenCentralArtifactIterator(resolvePom, resolveTransitive, resolveJar, config);
-
-while(iterator.hasNext()){
-    Artifact current = iterator.next();
-    // TODO: Process artifact
+static void analyze() throws InvalidConfigurationException {
+    final boolean resolvePom = true;
+    final boolean resolveTransitive = false;
+    final boolean resolveJar = true;
+    final Path gavInputList = Paths.get("path-to-input-file");
+
+    final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder()
+            .withInputList(gavInputList)
+            .withSkip(2)
+            .withTake(5)
+            .build();
+
+    MavenCentralArtifactIterator iterator = new MavenCentralArtifactIterator(resolvePom, resolveTransitive, resolveJar, config);
+
+    while(iterator.hasNext()){
+        Artifact current = iterator.next();
+        // TODO: Process artifact
+    }
 }
 ```
 
@@ -99,6 +111,9 @@ You can run each example on the first 1000 Maven artifacts by invoking `java -ja
 
 ### Counting all classFiles from jar artifacts:
 ``` java
+import org.tudo.sse.analyses.MavenCentralArtifactAnalysis;
+import org.tudo.sse.model.Artifact;
+
 public class ClassFileCountImplementation extends MavenCentralArtifactAnalysis {
 
     private long numberOfClassfiles;
@@ -123,6 +138,13 @@ public class ClassFileCountImplementation extends MavenCentralArtifactAnalysis {
 
 ### Find all Unique Licenses from pom artifacts
 ``` java
+import org.tudo.sse.analyses.MavenCentralArtifactAnalysis;
+import org.tudo.sse.model.Artifact;
+import org.tudo.sse.model.pom.License;
+import org.tudo.sse.model.pom.PomInformation;
+
+import java.util.*;
+
 public class LicenseImplementation extends MavenCentralArtifactAnalysis {
     private final Set uniqueLicenses;
 
@@ -151,6 +173,11 @@ public class LicenseImplementation extends MavenCentralArtifactAnalysis {
 
 ### Collect all artifacts that have javadocs
 ``` java
+import org.tudo.sse.analyses.MavenCentralArtifactAnalysis;
+import org.tudo.sse.model.Artifact;
+
+import java.util.*;
+
 public class JavaDocImplementation extends MavenCentralArtifactAnalysis {
 
     private final Set hasJavadocs;
@@ -163,8 +190,8 @@ public class JavaDocImplementation extends MavenCentralArtifactAnalysis {
     @Override
     public void analyzeArtifact(Artifact toAnalyze) {
         if(toAnalyze.getIndexInformation() != null) {
-            List packages = toAnalyze.getIndexInformation().getPackages();
-            for(Package current : packages) {
+            List packages = toAnalyze.getIndexInformation().getPackages();
+            for(org.tudo.sse.model.index.Package current : packages) {
                 if(current.getJavadocExists() > 0) {
                     hasJavadocs.add(toAnalyze);
                     break;
diff --git a/pom.xml b/pom.xml
index b0247dc..796837b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,6 +80,7 @@
             org.slf4j
             slf4j-api
             2.0.17
+            compile
         
         
             ch.qos.logback
diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java
index 435c192..dfa12ee 100644
--- a/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java
+++ b/src/main/java/org/tudo/sse/analyses/MavenCentralAnalysis.java
@@ -6,7 +6,12 @@
 import org.slf4j.LoggerFactory;
 import org.tudo.sse.utils.MarinOpalLogger;
 
-abstract class MavenCentralAnalysis {
+/**
+ * An analysis that processes entities (artifacts or libraries) from the Maven Central ecosystem.
+ *
+ * @author Johannes Düsing
+ */
+public abstract class MavenCentralAnalysis {
 
     /**
      * Defines whether this analysis requires artifacts to have index information annotated.

From 6b5c2216e2ab194a5739c6aff5a28f52d91b8111 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 30 Mar 2026 14:35:24 +0200
Subject: [PATCH 53/55] Fix vulnerability in dependencies

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 796837b..50134e0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,7 +85,7 @@
         
             ch.qos.logback
             logback-classic
-            1.5.19
+            1.5.25
             test
         
         
@@ -97,7 +97,7 @@
         
             org.apache.maven
             maven-model
-            3.9.9
+            3.9.14
         
         
         

From 4377825b99c718278a908e07c910c1341316aa01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 30 Mar 2026 15:21:04 +0200
Subject: [PATCH 54/55] Update version numbers following release 2.0.0

---
 README.md | 2 +-
 pom.xml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ce40972..3ed5afb 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@ Add the following dependency to your `pom.xml` to add MARIN to your project:
 
     eu.sse-labs
     marin
-    1.0.0
+    2.0.0
 
 ```
 
diff --git a/pom.xml b/pom.xml
index 50134e0..ff20462 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
 
     eu.sse-labs
     marin
-    2.0.0-SNAPSHOT
+    2.0.0
 
     jar
 

From 1d4e3827e46899f88c8d108dab5504901ae2babf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20D=C3=BCsing?= 
Date: Mon, 30 Mar 2026 15:24:24 +0200
Subject: [PATCH 55/55] Add release guide

---
 docs/RELEASE.md | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 docs/RELEASE.md

diff --git a/docs/RELEASE.md b/docs/RELEASE.md
new file mode 100644
index 0000000..3dc194d
--- /dev/null
+++ b/docs/RELEASE.md
@@ -0,0 +1,39 @@
+# Creating a MARIN release
+
+1. Make sure the last CI build on `develop` was successful
+2. Checkout branch `develop` locally, and create a new branch `feature/prepare-release-X.X.X` off of `develop`
+3. Check [the dependabot logs](https://github.com/sse-labs/marin/security/dependabot) for vulnerabilities in the current build. If relevant vulnerabilities exist:
+    * Update dependencies according to security advisories.
+    * Run `mvn clean package` and make sure the dependency updates did not break compilation or tests
+    * If the update breaks the MARIN build or tests, postpone release and implement dependency updates on `develop`
+    * Add dependency updates as a commit to the release preparation branch
+4. Verify that all code snippets in the README file actually work. To do so:
+    * Run `mvn clean install -DskipTests` to install the current snapshot version locally
+    * Create a demo client project that depends on the current snapshot version of MARIN
+    * Copy-Paste all snippets from the MARIN README to the client project
+    * Ensure the client projects builds successfully, and all executable code can in fact be executed.
+    * Update README snippets if necessary, and add a commit to the release preparation branch
+5. If the release preparation branch contains at least one new commit, push it to the remote repo and create a PR on GitHub to merge your changes **back into develop**
+6. Update your local version of MARIN with the last changes: `git checkout develop` and `git pull`
+7. Create a new branch `release/X.X.X` off of develop
+8. Update the `` tag within `pom.xml` to contain the actual version you want to release (i.e., remove `-SNAPSHOT`)
+9. Perform a dry run for deployment:
+    * Ensure `` is set to `true` in `pom.xml`
+    * Ensure `settings.xml` with the publishing credentials is located in your current user's `.m2` directory
+    * Ensure you have the GPG signing key installed and the passphrase is available
+    * Run `mvn clean deploy -DskipTests -P release`, enter the GPG passphrase when prompted
+    * If everything is configured correctly, Maven will report `BUILD SUCCESS` without actually publishing anything yet
+10. Deploy the new version to Maven Central:
+    * Set `` to `false` in `pom.xml`
+    * Run `mvn clean deploy -DskipTests -P release`, enter the GPG passphrase when prompted
+    * Set `` to back to `true` in `pom.xml`
+11. Publish the deployment:
+    * Go to the [Sonatype Website](https://central.sonatype.com/publishing/deployments) and log in to the publishing account
+    * Select the recent deployment, assert that the status says `Validated`
+    * Click on "Publish"
+    * Publishing can take a few minutes - check back periodically until status says "Published"
+12. Change current MARIN version in README.md to newly released version
+13. Push your `release/X.X.X` branch and create a PR merging you changes **into the main branch**.
+14. Merge your `main` branch back into `develop`, increase `` attribute in `pom.xml` (either minor or patch version), and add a `-SNAPSHOT` suffix
+15. Create a GitHub release on the `main` branch
+16. Update your local repo with `git fetch` and delete the release branch
\ No newline at end of file