From c63eade16ef622a0c3898b968a6217f65f7272c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 8 Dec 2025 13:41:11 +0100 Subject: [PATCH 1/6] Rework configuration system. Add configuration builder classes to programmatically build MARIN configurations --- .../MavenCentralArtifactAnalysis.java | 27 ++- .../analyses/MavenCentralLibraryAnalysis.java | 19 +- .../config/ArtifactAnalysisConfig.java | 49 +++++ .../config/ArtifactAnalysisConfigBuilder.java | 136 +++++++++++++ .../config/InvalidConfigurationException.java | 50 +++++ .../config/LibraryAnalysisConfig.java | 90 +++++++++ .../config/LibraryAnalysisConfigBuilder.java | 178 ++++++++++++++++++ .../parsing/ArtifactAnalysisConfigParser.java | 62 ++++++ .../config/parsing}/CLIParsingUtilities.java | 2 +- .../parsing/LibraryAnalysisConfigParser.java | 86 +++++++++ .../tudo/sse/utils/ArtifactConfigParser.java | 118 ------------ .../tudo/sse/utils/CommonConfigParser.java | 146 -------------- .../MavenCentralArtifactAnalysisTest.java | 20 +- .../MavenCentralLibraryAnalysisTest.java | 19 +- src/test/resources/MavenAnalysis.json | 2 +- 15 files changed, 697 insertions(+), 307 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java create mode 100644 src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java rename src/main/java/org/tudo/sse/{utils => analyses/config/parsing}/CLIParsingUtilities.java (98%) create mode 100644 src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java delete mode 100644 src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java delete mode 100644 src/main/java/org/tudo/sse/utils/CommonConfigParser.java diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 2446f86..bbd7f71 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -1,8 +1,9 @@ package org.tudo.sse.analyses; -import org.apache.pekko.actor.typed.ActorRef; import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; @@ -12,7 +13,7 @@ import org.tudo.sse.multithreading.WorkItem; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; -import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; import org.tudo.sse.utils.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; @@ -29,7 +30,7 @@ */ public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis { - private ArtifactConfigParser.ArtifactConfig artifactConfig; + private ArtifactAnalysisConfig artifactConfig; private ActorSystem system; private ResolverFactory resolverFactory; @@ -53,7 +54,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, super(requiresIndex, requiresPom, requiresTransitives, requiresJar); // Initialize with default config, update later - artifactConfig = new ArtifactConfigParser.ArtifactConfig(); + artifactConfig = new ArtifactAnalysisConfigBuilder().build(); } @@ -68,7 +69,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, * Returns the CLI configuration for this analysis. * @return CLI information for this analysis */ - public ArtifactConfigParser.ArtifactConfig getSetupInfo() { + public ArtifactAnalysisConfig getSetupInfo() { return this.artifactConfig; } @@ -91,8 +92,8 @@ private void printRunInfo(){ log.info("\t - Reading artifacts from Maven Central index"); if(this.artifactConfig.progressRestoreFile != null) log.info("\t - Restoring last index position from " + this.artifactConfig.progressRestoreFile); if(this.artifactConfig.progressOutputFile != null) log.info("\t - Writing last index position to " + this.artifactConfig.progressOutputFile); - if(this.artifactConfig.skip >= 0) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); - if(this.artifactConfig.take >= 0) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); + if(this.artifactConfig.hasSkip()) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); + if(this.artifactConfig.hasTake()) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); if(this.artifactConfig.since >= 0) log.info("\t - Skipping artifacts before " + this.artifactConfig.since); if(this.artifactConfig.until >= 0) log.info("\t - Taking artifacts until " + this.artifactConfig.until); @@ -105,7 +106,7 @@ private void printRunInfo(){ public final void runAnalysis(String[] args) { // Obtain CLI arguments try { - this.artifactConfig = new ArtifactConfigParser().parseArtifactConfig(args); + this.artifactConfig = new ArtifactAnalysisConfigParser().parseArtifactConfig(args); printRunInfo(); } catch(CLIException clix){ throw new RuntimeException(clix); @@ -168,7 +169,7 @@ public void indexProcessor() { final long startingPosition = getInitialPosition(); skipN(startingPosition, indexIterator); - if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1){ + if(this.artifactConfig.hasTake()){ walkPaginated(this.artifactConfig.take, indexIterator); } else if(this.artifactConfig.since != -1 && this.artifactConfig.until != -1){ walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); @@ -359,7 +360,7 @@ void processArtifactsFromInputFile() { log.info("Restoring previous progress from file (position {})", lastProgress); this.skipN(lastProgress, fileIterator); - } else if(this.artifactConfig.skip > 0){ + } else if(this.artifactConfig.hasSkip()){ // Skip configured values only if we did not restore from progress file log.info("Skipping {} entries from file", artifactConfig.skip); skipN(this.artifactConfig.skip, fileIterator); @@ -369,10 +370,8 @@ void processArtifactsFromInputFile() { if(!fileIterator.hasNext()) log.warn("No more contents left to process in input file: {}", this.artifactConfig.inputListFile); - final boolean takeLimited = this.artifactConfig.take >= 0; - long entriesTaken = 0L; - while ((!takeLimited || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { + while ((!this.artifactConfig.hasTake() || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { ArtifactIdent current = null; @@ -421,7 +420,7 @@ public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) { private long getInitialPosition() { if(artifactConfig.progressRestoreFile != null) return getStartingPos(); - else if(artifactConfig.skip > 0) return artifactConfig.skip; + else if(artifactConfig.hasSkip()) return artifactConfig.skip; else return 0L; } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index b03dc61..40a3ec8 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -2,6 +2,7 @@ import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.LibraryResolutionContext; @@ -12,7 +13,7 @@ import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; -import org.tudo.sse.utils.CommonConfigParser; +import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser; import org.tudo.sse.utils.LibraryIndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -36,7 +37,7 @@ public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis { /** * The configuration for this analysis instance. Only available after runAnalysis has been called. */ - protected CommonConfigParser.CommonConfig config; + protected LibraryAnalysisConfig config; private ActorSystem system; private long lastPositionSaved; @@ -61,7 +62,7 @@ protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTrans public final void runAnalysis(String[] args){ // Obtain CLI arguments try { - this.config = new CommonConfigParser().parseCommonConfig(args); + this.config = new LibraryAnalysisConfigParser().parseCommonConfig(args); printRunInfo(); } catch(CLIException clix){ throw new RuntimeException(clix); @@ -84,9 +85,9 @@ public final void runAnalysis(String[] args){ currentPosition += 1L; } // Notify users that skip will not be applied - if(config.skip > 0) + if(config.hasSkip()) log.info("Not applying skip value because progress was restored from previous run"); - } else if(config.skip > 0){ + } else if(config.hasSkip()){ // We only apply a skip if we did not restore previous progress log.info("Skipping {} library names", config.skip); for(int i = 0; i < config.skip; i++){ @@ -106,7 +107,7 @@ public final void runAnalysis(String[] args){ long entriesTaken = 0L; - if(config.take >= 0) + if(config.hasTake()) log.info("Taking {} library names", config.take); // Invoke the beforeRunStart lifecycle hook @@ -119,7 +120,7 @@ public final void runAnalysis(String[] args){ // If specified, we only take the configured amount of entries. If not, we process as long as the iterator // provides new entries - while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ + while((!config.hasTake()|| entriesTaken < config.take) && gaIterator.hasNext()){ processEntry(gaIterator.next()); entriesTaken += 1L; @@ -336,8 +337,8 @@ private void printRunInfo(){ log.info("\t - Reading libraries from Maven Central index"); if(config.progressRestoreFile != null) log.info("\t - Restoring last index position from " + config.progressRestoreFile); if(config.progressOutputFile != null) log.info("\t - Writing last index position to " + config.progressOutputFile); - if(config.skip >= 0) log.info("\t - Skipping " + config.skip + " artifacts"); - if(config.take >= 0) log.info("\t - Taking " + config.take + " artifacts"); + if(config.hasSkip()) log.info("\t - Skipping " + config.skip + " artifacts"); + if(config.hasTake()) log.info("\t - Taking " + config.take + " artifacts"); } else { log.info("\t - Reading libraries from GA-list at " + config.inputListFile); } diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java new file mode 100644 index 0000000..1c1545c --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -0,0 +1,49 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Path; + +/** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. + * + * @author Johannes Düsing + */ +public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { + + static final long DEFAULT_VALUE_SINCE = -1L; + static final long DEFAULT_VALUE_UNTIL = -1L; + static final Path DEFAULT_VALUE_OUTPUT = null; + + /** + * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. + */ + public long since; + + /** + * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. + */ + public long until; + + /** + * Directory to write file outputs to, or null if disabled. + */ + public Path outputDirectory; + + /** + * Whether files shall be written to the output directory. + */ + public boolean outputEnabled; + + /** + * Creates a new configuration object with default values. + */ + ArtifactAnalysisConfig(){ + super(); + + since = DEFAULT_VALUE_SINCE; + until = DEFAULT_VALUE_UNTIL; + outputEnabled = false; + outputDirectory = DEFAULT_VALUE_OUTPUT; + } + + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java new file mode 100644 index 0000000..d0e2a33 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java @@ -0,0 +1,136 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Configuration builder to obtain {@link ArtifactAnalysisConfig} instances programmatically. + * + * @author Johannes Düsing + */ +public final class ArtifactAnalysisConfigBuilder extends LibraryAnalysisConfigBuilder { + + private final ArtifactAnalysisConfig theConfig; + + private ArtifactAnalysisConfigBuilder(ArtifactAnalysisConfig theConfig) { + super(theConfig); + + this.theConfig = theConfig; + } + + /** + * Creates a new builder with default configuration values. + */ + public ArtifactAnalysisConfigBuilder(){ + this(new ArtifactAnalysisConfig()); + } + + @Override + public ArtifactAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("skip", "Cannot apply skip when time-based filtering is used"); + + super.withSkip(toSkip); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("take", "Cannot apply take when time-based filtering is used"); + + super.withTake(toTake); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withInputList(Path inputList) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("inputs", "Cannot use input list when time-based filtering is applied"); + + super.withInputList(inputList); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException { + super.withProgressOutputFile(outputFile); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException { + super.withProgressRestoreFile(restoreFile); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException { + super.withNumberOfThreads(numThreads); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException { + super.withProgressWriteInterval(progressWriteInterval); + return this; + } + + /** + * Sets the output directory to store processed artifacts in. If your analysis requires IndexInformation only, it + * will output a list of GAV triples. If pom information is required, one pom.xml file per artifact will be stored. + * If JAR information is required, the artifact's JAR will be stored in this directory. + * + * @param outDir The output directory to use. Must be a valid reference to an existing directory. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public ArtifactAnalysisConfigBuilder withOutputDirectory(Path outDir) throws InvalidConfigurationException { + if(outDir == null || !Files.isDirectory(outDir)) + throw new InvalidConfigurationException("output", "Output directory must be a valid directory reference"); + + if(this.theConfig.outputDirectory != ArtifactAnalysisConfig.DEFAULT_VALUE_OUTPUT) + log.warn("Value 'output' is set multiple times, previous value overridden."); + + this.theConfig.outputDirectory = outDir; + this.theConfig.outputEnabled = true; + return this; + } + + /** + * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have + * been released after the timestamp given in 'since', and before 'until'. + * + * @param since Timestamp marking the lower limit of the range + * @param until Timestamp marking the upper limit of the range + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration values are not valid + */ + public ArtifactAnalysisConfigBuilder withSinceUtil(long since, long until) throws InvalidConfigurationException { + if(since <= 0L || until <= since) + throw new InvalidConfigurationException("since-until", "Since and Until must define a valid range"); + + if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP || + this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE) + throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when pagination is used"); + + if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST) + throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when input list is used"); + + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + log.warn("Values 'since' and 'until' are set multiple times, previous values overridden."); + + this.theConfig.since = since; + this.theConfig.until = until; + return this; + } + + @Override + public ArtifactAnalysisConfig build(){ + return this.theConfig; + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java new file mode 100644 index 0000000..cb26b5a --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java @@ -0,0 +1,50 @@ +package org.tudo.sse.analyses.config; + +/** + * Exception thrown when a MARIN configuration value was invalid. + * + * @author Johannes Düsing + */ +public class InvalidConfigurationException extends Exception { + + /** + * Attribute the error occurred for. + */ + private final String attrName; + + /** + * Message describing the error. + */ + private final String msg; + + /** + * Creates a new instance with the given configuration attribute name and message. + * @param attrName The configuration value that was invalid + * @param msg A message describing the error + */ + InvalidConfigurationException(String attrName, String msg){ + this.attrName = attrName; + this.msg = msg; + } + + /** + * Returns the name of the attribute for which this exception occurred. + * @return The attribute name + */ + public String getAttributeName() { + return this.attrName; + } + + /** + * Returns a description on the error that occurred. + * @return The error description + */ + public String getDescription(){ + return this.msg; + } + + @Override + public String getMessage() { + return String.format("Invalid configuration value for '%s': %s", attrName, msg); + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java new file mode 100644 index 0000000..23c1b7e --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java @@ -0,0 +1,90 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * + * @author Johannes Düsing + */ +public class LibraryAnalysisConfig { + + static final long DEFAULT_VALUE_SKIP = -1L; + static final long DEFAULT_VALUE_TAKE = -1L; + static final Path DEFAULT_VALUE_INPUT_LIST = null; + static final Path DEFAULT_VALUE_PROGRESS_OUTPUT_FILE = Paths.get("marin-progress"); + static final Path DEFAULT_VALUE_PROGRESS_RESTORE_FILE = null; + static final int DEFAULT_NUM_THREADS = 1; + static final int DEFAULT_PROGRESS_WRITE_INTERVAL = 100; + + /** + * Number of artifacts to skip from the underlying source, or -1 if disabled. + */ + public long skip; + + /** + * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. + */ + public long take; + + /** + * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. + */ + public Path inputListFile; + + /** + * File path to output analysis progress to. + */ + public Path progressOutputFile; + + /** + * Path to read previous progress from, in order to restore it. Null if disabled. + */ + public Path progressRestoreFile; + + /** + * True if multiple threads shall be used, false for single-threaded analysis. + */ + public boolean multipleThreads; + + /** + * Number of threads to use. + */ + public int threadCount; + + /** + * Interval of entities after which progress shall be saved. Defaults to 100. + */ + public int progressWriteInterval; + + /** + * Creates a new configuration with default values. + */ + LibraryAnalysisConfig() { + skip = DEFAULT_VALUE_SKIP; + take = DEFAULT_VALUE_TAKE; + inputListFile = DEFAULT_VALUE_INPUT_LIST; + progressOutputFile = DEFAULT_VALUE_PROGRESS_OUTPUT_FILE; + progressRestoreFile = DEFAULT_VALUE_PROGRESS_RESTORE_FILE; + multipleThreads = DEFAULT_NUM_THREADS == 1 ? false : true; + threadCount = DEFAULT_NUM_THREADS; + progressWriteInterval = DEFAULT_PROGRESS_WRITE_INTERVAL; + } + + /** + * Checks whether there is a non-default amount of entities to skip + * @return True if there are entities to skip + */ + public boolean hasSkip() { + return this.skip != DEFAULT_VALUE_SKIP; + } + + /** + * Checks whether there is a non-default amount of entities to take + * @return True if there are limited entities to take + */ + public boolean hasTake() { + return this.take != DEFAULT_VALUE_TAKE; + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java new file mode 100644 index 0000000..7d43411 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java @@ -0,0 +1,178 @@ +package org.tudo.sse.analyses.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Configuration builder to obtain {@link LibraryAnalysisConfig} instances programmatically. + * + * @author Johannes Düsing + */ +public class LibraryAnalysisConfigBuilder { + + /** + * The logger for this instance. + */ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final LibraryAnalysisConfig theConfig; + + /** + * Creates a new builder with default configuration values. + */ + public LibraryAnalysisConfigBuilder() { + this(new LibraryAnalysisConfig()); + } + + /** + * Creates a new builder with the given baseline configuration instance. + * + * @param theConfig Baseline configuration instance for this builder + */ + protected LibraryAnalysisConfigBuilder(LibraryAnalysisConfig theConfig) { + this.theConfig = theConfig; + } + + /** + * Sets the amount of entities to skip when processing analysis inputs. Can be used for pagination. Defaults to zero. + * + * @param toSkip Amount of entities (artifacts or libraries) to skip + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid. + */ + public LibraryAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException { + if(toSkip < 0) + throw new InvalidConfigurationException("skip", "Skip must not be negative"); + + if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP) + log.warn("Value 'skip' is set multiple times, previous value overridden."); + + this.theConfig.skip = toSkip == 0 ? LibraryAnalysisConfig.DEFAULT_VALUE_SKIP : toSkip; + return this; + } + + /** + * Sets the amount of entities to take when processing analysis inputs. Can be used for pagination. By default, no + * limit is imposed and the input source is consumed fully. + * + * @param toTake Amount of entities (artifacts or libraries) to take + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid. + */ + public LibraryAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException { + if(toTake <= 0) + throw new InvalidConfigurationException("take", "Take must be positive"); + + if(this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE) + log.warn("Value 'take' is set multiple times, previous value overridden."); + + this.theConfig.take = toTake; + return this; + } + + /** + * Sets the input list of entities to process as analysis inputs. This is either a list of GAV triples (artifacts) + * or GA tuples (libraries), one entity per line. By default, the Maven Central index is used as the source of inputs. + * + * @param listPath Path to input file. Must be an existing regular file, one entity per line is required. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withInputList(Path listPath) throws InvalidConfigurationException { + if(listPath == null || !Files.isRegularFile(listPath)) + throw new InvalidConfigurationException("inputs", "Input list file must be a valid file reference"); + + if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST) + log.warn("Value 'inputs' is set multiple times, previous value overridden."); + + this.theConfig.inputListFile = listPath; + return this; + } + + /** + * Sets the output file to regularly write the analysis progress to. By default, a file called 'marin-progress.txt' + * is created in the current workdir. + * + * @param outputFile Path to the progress output file to create. Existing files will be overridden! + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException { + if(outputFile == null) + throw new InvalidConfigurationException("progress-output-file", "Output file cannot be null"); + + if(this.theConfig.progressOutputFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_OUTPUT_FILE) + log.warn("Value 'progress-output-file' is set multiple times, previous value overridden."); + + this.theConfig.progressOutputFile = outputFile; + return this; + } + + /** + * Sets the file to restore progress from. Should be a progress output file previously created by MARIN. By default, + * no progress is restored. + * + * @param restoreFile Path to the progress restore file. Must be a valid text file. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException { + if(restoreFile == null || !Files.isRegularFile(restoreFile)) + throw new InvalidConfigurationException("progress-restore-file", "Restore file must be a valid file reference"); + + if(this.theConfig.progressRestoreFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_RESTORE_FILE) + log.warn("Value 'progress-restore-file' is set multiple times, previous value overridden."); + + this.theConfig.progressRestoreFile = restoreFile; + return this; + } + + /** + * Sets the number of threads to use for analysis. By default, one thread is used. + * @param numThreads Number of threads to use + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException { + if(numThreads <= 0) + throw new InvalidConfigurationException("threads", "Number of threads must be positive"); + + if(this.theConfig.threadCount != LibraryAnalysisConfig.DEFAULT_NUM_THREADS) + log.warn("Value 'threads' is set multiple times, previous value overridden."); + + this.theConfig.threadCount = numThreads; + this.theConfig.multipleThreads = numThreads > 1; + return this; + } + + /** + * Sets the amount of entities (artifacts or libraries) after which to write progress to the progress output file. + * Default is 100. + * @param progressWriteInterval Amount of entities after which to save progress + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException { + if(progressWriteInterval <= 0) + throw new InvalidConfigurationException("save-progress-interval", "Progress write interval must be positive"); + + if(this.theConfig.progressWriteInterval != LibraryAnalysisConfig.DEFAULT_PROGRESS_WRITE_INTERVAL) + log.warn("Value 'progress-write-interval' is set multiple times, previous value overridden."); + + this.theConfig.progressWriteInterval = progressWriteInterval; + return this; + } + + /** + * Builds the configuration as specified by all previous invocations to this instance. + * + * @return The configuration object that has been constructed by this builder. + */ + public LibraryAnalysisConfig build(){ + return this.theConfig; + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java new file mode 100644 index 0000000..6b46233 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -0,0 +1,62 @@ +package org.tudo.sse.analyses.config.parsing; + +import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; + +/** + * Parser for creating {@link ArtifactAnalysisConfig} objects from CLI parameters. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ +public class ArtifactAnalysisConfigParser extends LibraryAnalysisConfigParser implements CLIParsingUtilities { + + /** + * Creates a new artifact config parser instance. + */ + public ArtifactAnalysisConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfig} object. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link ArtifactAnalysisConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ + public ArtifactAnalysisConfig parseArtifactConfig(String[] args) throws CLIException { + final ArtifactAnalysisConfigBuilder configBuilder = new ArtifactAnalysisConfigBuilder(); + + for(int i = 0; i < args.length; i += 2) { + handleArtifactParameter(args, i, configBuilder); + } + + return configBuilder.build(); + } + + + private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfigBuilder configBuilder) throws CLIException { + try { + + switch(args[i]) { + case "-su": + case "--since-until": + final long[] sinceUntil = nextArgAsLongPair(args, i); + configBuilder.withSinceUtil(sinceUntil[0], sinceUntil[1]); + break; + case "-o": + case "--output": + configBuilder.withOutputDirectory(nextArgAsDirectoryReference(args, i)); + break; + default: + super.handleParameter(args, i , configBuilder); + + } + + } catch (InvalidConfigurationException icx) { + CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName()); + wrapped.initCause(icx); + throw wrapped; + } + } + +} diff --git a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java similarity index 98% rename from src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java rename to src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java index 8d67fbc..2c175d1 100644 --- a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java @@ -1,4 +1,4 @@ -package org.tudo.sse.utils; +package org.tudo.sse.analyses.config.parsing; import org.tudo.sse.CLIException; diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java new file mode 100644 index 0000000..ba2b95e --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java @@ -0,0 +1,86 @@ +package org.tudo.sse.analyses.config.parsing; + +import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; + +/** + * Parser for {@link LibraryAnalysisConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ +public class LibraryAnalysisConfigParser implements CLIParsingUtilities { + + /** + * Creates a new library config parsers instance. + */ + public LibraryAnalysisConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link LibraryAnalysisConfig} objects. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link LibraryAnalysisConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ + public LibraryAnalysisConfig parseCommonConfig(String[] args) throws CLIException { + final LibraryAnalysisConfigBuilder configBuilder = new LibraryAnalysisConfigBuilder(); + + + for(int i = 0; i < args.length; i += 2){ + handleParameter(args, i, configBuilder); + } + + return configBuilder.build(); + } + + /** + * Parses a given parameter from the array of CLI arguments and applies it to the config object. + * @param args CLI arguments array + * @param i Current parsing position + * @param configBuilder Current configuration object + * @throws CLIException If parsing the current argument fails. + */ + protected void handleParameter(String[] args, int i, LibraryAnalysisConfigBuilder configBuilder) throws CLIException { + try { + switch (args[i]){ + case "-st": + case "--skip-take": + final long[] skipTake = nextArgAsLongPair(args, i); + configBuilder + .withSkip(skipTake[0]) + .withTake(skipTake[1]); + break; + case "-prf": + case "--progress-restore-file": + configBuilder.withProgressRestoreFile(nextArgAsRegularFileReference(args, i)); + break; + case "-spi": + case "--save-progress-interval": + configBuilder.withProgressWriteInterval(nextArgAsInt(args, i)); + break; + case "-pof": + case "--progress-output-file": + configBuilder.withProgressOutputFile(nextArgAsPath(args, i)); + break; + case "-i": + case "--inputs": + configBuilder.withInputList(nextArgAsRegularFileReference(args, i)); + break; + case "-t": + case "--threads": + final int threads = nextArgAsInt(args, i); + configBuilder.withNumberOfThreads(threads); + break; + default: + throw new CLIException(args[i]); + } + } catch (InvalidConfigurationException icx) { + CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName()); + wrapped.initCause(icx); + throw wrapped; + } + } + + +} diff --git a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java deleted file mode 100644 index 7fde305..0000000 --- a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.CLIException; - -import java.nio.file.Path; - -/** - * Parser for ArtifactConfig objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis}. - * Configuration is obtained by parsing command line arguments provided as an array of strings. - */ -public class ArtifactConfigParser extends CommonConfigParser implements CLIParsingUtilities { - - /** - * Creates a new artifact config parser instance. - */ - public ArtifactConfigParser() { super(); } - - /** - * Parses a given array containing JVM command line arguments into an {@link ArtifactConfig} object. Throws an exception - * if the given arguments are not valid. - * @param args Array of command line arguments, as passed by the JVM - * @return Corresponding {@link ArtifactConfig} if parsing was successful - * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. - */ - public ArtifactConfig parseArtifactConfig(String[] args) throws CLIException { - final ArtifactConfig artifactConfig = new ArtifactConfig(); - - for(int i = 0; i < args.length; i += 2) { - handleArtifactParameter(args, i, artifactConfig); - } - - return artifactConfig; - } - - - private void handleArtifactParameter(String[] args, int i, ArtifactConfig config) throws CLIException { - switch(args[i]) { - case "-su": - case "--since-until": - if(config.skip != -1L) - throw new CLIException("Cannot apply time-based filtering when pagination (-st) is used", args[i]); - if(config.since != -1L) - throw new CLIException("Values for since and until cannot be set twice!", args[i]); - if(config.inputListFile != null) - throw new CLIException("Cannot apply time-based filtering when input list (-i) is used", args[i]); - - final long[] sinceUntil = nextArgAsLongPair(args, i); - config.since = sinceUntil[0]; - config.until = sinceUntil[1]; - break; - - case "-o": - case "--output": - config.outputEnabled = true; - config.outputDirectory = nextArgAsDirectoryReference(args, i); - break; - - case "-st": - case "--skip-take": - if(config.since != -1L) - throw new CLIException("Cannot apply pagination when time-based filtering (-su) is used", args[i]); - super.handleParameter(args, i, config); - break; - - case "-i": - case "--inputs": - if(config.since != -1L) - throw new CLIException("Cannot use input list when time-based filtering (-su) is applied", args[i]); - super.handleParameter(args, i, config); - break; - - default: - super.handleParameter(args, i , config); - - } - } - - /** - * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. - */ - public static class ArtifactConfig extends CommonConfig { - - /** - * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. - */ - public long since; - - /** - * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. - */ - public long until; - - /** - * Directory to write file outputs to, or null if disabled. - */ - public Path outputDirectory; - - /** - * Whether files shall be written to the output directory. - */ - public boolean outputEnabled; - - /** - * Creates a new configuration object with default values. - */ - public ArtifactConfig(){ - super(); - - since = -1L; - until = -1L; - outputEnabled = false; - outputDirectory = null; - } - - - } - -} diff --git a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java deleted file mode 100644 index 474449e..0000000 --- a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java +++ /dev/null @@ -1,146 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.CLIException; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * Parser for {@link CommonConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. - * Configuration is obtained by parsing command line arguments provided as an array of strings. - */ -public class CommonConfigParser implements CLIParsingUtilities { - - /** - * Creates a new artifact config parsers instance. - */ - public CommonConfigParser() { super(); } - - /** - * Parses a given array containing JVM command line arguments into an {@link CommonConfig} objects. Throws an exception - * if the given arguments are not valid. - * @param args Array of command line arguments, as passed by the JVM - * @return Corresponding {@link CommonConfig} if parsing was successful - * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. - */ - public CommonConfig parseCommonConfig(String[] args) throws CLIException { - final CommonConfig config = new CommonConfig(); - - - for(int i = 0; i < args.length; i += 2){ - handleParameter(args, i, config); - } - - return config; - } - - /** - * Parses a given parameter from the array of CLI arguments and applies it to the config object. - * @param args CLI arguments array - * @param i Current parsing position - * @param config Current configuration object - * @throws CLIException If parsing the current argument fails. - */ - protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException { - switch (args[i]){ - case "-st": - case "--skip-take": - if(config.skip != -1L) - throw new CLIException("Values for skip and take cannot be set twice!", args[i]); - - final long[] skipTake = nextArgAsLongPair(args, i); - config.skip = skipTake[0]; - config.take = skipTake[1]; - break; - case "-prf": - case "--progress-restore-file": - if(config.progressRestoreFile != null) - throw new CLIException("Progress restore file cannot be set twice!", args[i]); - - config.progressRestoreFile = nextArgAsRegularFileReference(args, i); - break; - case "-spi": - case "--save-progress-interval": - config.progressWriteInterval = nextArgAsInt(args, i); - break; - case "-pof": - case "--progress-output-file": - config.progressOutputFile = nextArgAsPath(args, i); - break; - case "-i": - case "--inputs": - if(config.inputListFile != null) - throw new CLIException("Input file cannot be set twice!", args[i]); - config.inputListFile = nextArgAsRegularFileReference(args, i); - break; - case "-t": - case "--threads": - final int threads = nextArgAsInt(args, i); - config.multipleThreads = threads > 1; - config.threadCount = threads; - break; - default: - throw new CLIException(args[i]); - } - } - - /** - * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis} instance. - */ - public static class CommonConfig { - - /** - * Number of artifacts to skip from the underlying source, or -1 if disabled. - */ - public long skip; - - /** - * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. - */ - public long take; - - /** - * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. - */ - public Path inputListFile; - - /** - * File path to output analysis progress to. - */ - public Path progressOutputFile; - - /** - * Path to read previous progress from, in order to restore it. Null if disabled. - */ - public Path progressRestoreFile; - - /** - * True if multiple threads shall be used, false for single-threaded analysis. - */ - public boolean multipleThreads; - - /** - * Number of threads to use. - */ - public int threadCount; - - /** - * Interval of entities after which progress shall be saved. Defaults to 100. - */ - public int progressWriteInterval; - - /** - * Creates a new configuration with default values. - */ - public CommonConfig() { - skip = -1L; - take = -1L; - inputListFile = null; - progressOutputFile = Paths.get("marin-progress"); - progressRestoreFile = null; - multipleThreads = false; - threadCount = 1; - progressWriteInterval = 100; - } - } -} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 70baa56..422abf1 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -5,12 +5,13 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.Package; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralAnalysisFactory; import scala.Tuple2; @@ -43,11 +44,11 @@ class MavenCentralArtifactAnalysisTest { void parseCLIRegular() throws IOException{ Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{ parseCLI("--skip-take 500:223"), parseCLI("--inputs src/test/resources/localPom.xml"), parseCLI("--progress-restore-file src/test/resources/localPom.xml --inputs src/test/resources/localPom.xml"), - parseCLI("--since-until 53245:13243"), + parseCLI("--since-until 53245:53246"), parseCLI("--inputs src/test/resources/localPom.xml --progress-restore-file src/test/resources/localPom.xml"), parseCLI("--progress-restore-file src/test/resources/localPom.xml"), parseCLI("--output " + tmpDir.toString()) @@ -57,7 +58,7 @@ void parseCLIRegular() throws IOException{ for(int i = 0; i < configs.length; i++){ final List currExpected = expected.get(i); - final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + final ArtifactAnalysisConfig currConfig = configs[i]; assertEquals(asInt(currExpected.get(0)), currConfig.skip); assertEquals(asInt(currExpected.get(1)), currConfig.take); @@ -89,11 +90,11 @@ void parseCLIRegular() throws IOException{ void parseCLIShorthands() throws IOException{ Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{ parseCLI("-st 500:223"), parseCLI("-i src/test/resources/localPom.xml"), parseCLI("-prf src/test/resources/localPom.xml -i src/test/resources/localPom.xml"), - parseCLI("-su 53245:13243"), + parseCLI("-su 53245:53246"), parseCLI("-i src/test/resources/localPom.xml -prf src/test/resources/localPom.xml"), parseCLI("-prf src/test/resources/localPom.xml"), parseCLI("-o " + tmpDir.toString()) @@ -103,7 +104,7 @@ void parseCLIShorthands() throws IOException{ for(int i = 0; i < configs.length; i++){ final List currExpected = expected.get(i); - final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + final ArtifactAnalysisConfig currConfig = configs[i]; assertEquals(asInt(currExpected.get(0)), currConfig.skip); assertEquals(asInt(currExpected.get(1)), currConfig.take); @@ -138,6 +139,7 @@ void parseCmdLineNegative() { cliInputs.add("--inputs -/xcd/"); cliInputs.add("-st 88880000 -su --inputs path/to/file -ip path/to/file"); cliInputs.add("--inputs"); + cliInputs.add("--since-until 53245:13243"); for(String input : cliInputs) { @@ -383,9 +385,9 @@ private long getEndingIndex(Path fileName) { } } - private ArtifactConfigParser.ArtifactConfig parseCLI(String cli) { + private ArtifactAnalysisConfig parseCLI(String cli) { try { - final ArtifactConfigParser parser = new ArtifactConfigParser(); + final ArtifactAnalysisConfigParser parser = new ArtifactAnalysisConfigParser(); if(cli.isBlank()) return parser.parseArtifactConfig(new String[] {}); else return parser.parseArtifactConfig(cli.split(" ")); } catch(CLIException clix){ diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index e5c120b..3ec6545 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -3,8 +3,9 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; -import org.tudo.sse.utils.CommonConfigParser; +import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser; import java.io.IOException; import java.nio.file.Files; @@ -22,7 +23,7 @@ public class MavenCentralLibraryAnalysisTest { @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ final String validArgs = "--skip-take 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --threads 12 --save-progress-interval 20"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assert(config.multipleThreads); assertEquals(12, config.threadCount); @@ -38,7 +39,7 @@ void parseCLIRegular(){ @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") void parseCLIShorthands(){ final String validArgs = "-st 20:10000 -prf pom.xml -pof marin-progress -t 12 -spi 20"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assert(config.multipleThreads); assertEquals(12, config.threadCount); @@ -55,7 +56,7 @@ void parseCLIShorthands(){ @Test @DisplayName("The CLI parser must set the correct defaults for empty arguments") void parseCLIDefaults(){ - final CommonConfigParser.CommonConfig config = parseCLI(""); + final LibraryAnalysisConfig config = parseCLI(""); assertFalse(config.multipleThreads); assertEquals(1, config.threadCount); @@ -80,7 +81,7 @@ void parseNonExistingFile(){ void parseCustomGA(){ final String validArgs = "-st 20:10000 --inputs pom.xml"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assertEquals(Paths.get("pom.xml"), config.inputListFile); assertEquals(20, config.skip); @@ -91,7 +92,7 @@ void parseCustomGA(){ void parseNonExistingIndex(){ final String validArgs = "--progress-restore-file pom.xml --inputs pom.xml"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assertEquals(Paths.get("pom.xml"), config.inputListFile); assertEquals(Paths.get("pom.xml"), config.progressRestoreFile); @@ -336,10 +337,10 @@ private Path testResource(String pathToResource){ return null; } - private CommonConfigParser.CommonConfig parseCLI(String cli) { + private LibraryAnalysisConfig parseCLI(String cli) { try { - if(cli.isBlank()) return new CommonConfigParser().parseCommonConfig(new String[]{}); - else return new CommonConfigParser().parseCommonConfig(cli.split(" ")); + if(cli.isBlank()) return new LibraryAnalysisConfigParser().parseCommonConfig(new String[]{}); + else return new LibraryAnalysisConfigParser().parseCommonConfig(cli.split(" ")); } catch(CLIException clix) { throw new RuntimeException(clix); } diff --git a/src/test/resources/MavenAnalysis.json b/src/test/resources/MavenAnalysis.json index c7e9edd..8879f2b 100644 --- a/src/test/resources/MavenAnalysis.json +++ b/src/test/resources/MavenAnalysis.json @@ -31,7 +31,7 @@ "-1", "-1", "53245", - "13243", + "53246", "null", "null", "false" From 7b7e16b4fd535ef0ed6ec146c40c750cafe11ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Wed, 10 Dec 2025 13:53:23 +0100 Subject: [PATCH 2/6] Add artifact analysis iterator implementation (singlethreaded) --- .../org/tudo/sse/analyses/AnalysisUtils.java | 58 ++++ .../MavenCentralArtifactIterator.java | 248 ++++++++++++++++++ .../config/ArtifactAnalysisConfig.java | 8 + .../config/LibraryAnalysisConfig.java | 8 + .../sse/model/ArtifactResolutionContext.java | 10 +- .../ProcessIdentifierMessage.java | 2 +- .../sse/utils/FileBasedArtifactIterator.java | 28 ++ .../MavenCentralArtifactIteratorTest.java | 204 ++++++++++++++ .../MavenCentralLibraryAnalysisTest.java | 9 +- .../org/tudo/sse/utils/TestUtilities.java | 19 ++ src/test/resources/artifact-names-invalid.txt | 10 + 11 files changed, 594 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/AnalysisUtils.java create mode 100644 src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java create mode 100644 src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java create mode 100644 src/test/java/org/tudo/sse/utils/TestUtilities.java create mode 100644 src/test/resources/artifact-names-invalid.txt diff --git a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java new file mode 100644 index 0000000..59aad14 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java @@ -0,0 +1,58 @@ +package org.tudo.sse.analyses; + +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; + +/** + * Utility methods for general analysis implementations. + * + * @author Johannes Düsing + */ +interface AnalysisUtils { + + LibraryAnalysisConfig getConfig(); + + /** + * Retrieves the initial position to start analysis from, based on the current configuration. + * @return Initial number of entities to skip before starting + */ + default long getInitialPosition() { + final var config = getConfig(); + + if(config.progressRestoreFile != null) return getRestoreProgressValue(); + else if(config.hasSkip()) return config.skip; + else return 0L; + } + + /** + * Retrieves the last progress value to start from based on a previous run. + * @return Progress value + */ + default long getRestoreProgressValue() { + BufferedReader indexReader; + try { + indexReader = new BufferedReader(new FileReader(getConfig().progressRestoreFile.toFile())); + String line = indexReader.readLine(); + return Integer.parseInt(line); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Writes the current position to the specified progress output file. + * @param currentPosition The current progress to write + */ + default void writePosition(long currentPosition) { + final var config = getConfig(); + + try(BufferedWriter writer = Files.newBufferedWriter(config.progressOutputFile)) { + writer.write(Long.toString(currentPosition)); + } catch(IOException ignored) {} + } +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java new file mode 100644 index 0000000..741f126 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -0,0 +1,248 @@ +package org.tudo.sse.analyses; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactResolutionContext; +import org.tudo.sse.model.index.IndexInformation; +import org.tudo.sse.resolution.ResolverFactory; +import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.util.Iterator; + +/** + * Iterator that produces artifacts that are enriched with index-, pom- or jar information, as configured. This is the + * iterator equivalent to the {@link MavenCentralArtifactAnalysis}. Only supports single-threaded resolution. + * + * @author Johannes Düsing + */ +@SuppressWarnings("this-escape") +public class MavenCentralArtifactIterator implements Iterator, AnalysisUtils { + + private final Logger log = LoggerFactory.getLogger(MavenCentralArtifactIterator.class); + private final ArtifactAnalysisConfig config; + + private Iterator source; + + private boolean badSource = false; + + private long currentPosition = 0L; + private long artifactsTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + /** + * Creates a new iterator instance with the given configuration values. + * + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder} + */ + public MavenCentralArtifactIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, ArtifactAnalysisConfig config) { + this.config = config; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + if(this.config.multipleThreads){ + log.warn("Artifact iterator does no support multiple threads - using a single thread for resolution"); + } + + this.setUnderlyingSource(); + + if(this.config.outputEnabled){ + this.resolverFactory = new ResolverFactory(true, this.config.outputDirectory, resolveTransitivePoms); + } else { + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + } + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(badSource) return false; + + // If the config specifies a limited amount of artifacts to take, and that amount is reached, we have no next element + if(this.config.hasTake() && this.artifactsTaken >= this.config.take) + return false; + + // Otherwise, we have a next artifact if the underlying source has a next element + try { + return this.source.hasNext(); + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + + } + + @Override + public Artifact next(){ + if(!hasNext()) + throw new IllegalStateException("No more artifacts available"); + + final ArtifactResolutionContext ctx = this.source.next(); + final Artifact artifact = ctx.getRootArtifact(); + + if(this.resolvePom){ + resolverFactory.runPom(artifact.getIdent(), ctx); + } + + if(this.resolveJar){ + resolverFactory.runJar(artifact.getIdent(), ctx); + } + + this.artifactsTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return artifact; + } + + @Override + public LibraryAnalysisConfig getConfig(){ + return this.config; + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ + this.writePosition(this.currentPosition); + this.lastPositionSaved = this.currentPosition; + } + } + + private void skipInitial(){ + final long entitiesToSkip = getInitialPosition(); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } + + + private void setUnderlyingSource(){ + if(this.config.hasInputList()){ + try{ + this.source = new MavenCentralCustomListArtifactSource(); + } catch(IOException | IllegalArgumentException x){ + log.error("The given input list file is invalid", x); + this.badSource = true; + } + } else { + try { + this.source = new MavenCentralIndexArtifactSource(); + } catch(IOException uix){ + log.error("Failed to access Maven Central Index", uix); + this.badSource = true; + } + + } + } + + private class MavenCentralIndexArtifactSource implements Iterator { + + private final IndexIterator index; + + private boolean _needsUpdate = true; + private boolean _hasNext = false; + private IndexInformation _nextInfo = null; + + MavenCentralIndexArtifactSource() throws IOException { + this.index = new IndexIterator(MavenCentralRepository.RepoBaseURI); + } + + private void findNext(){ + _hasNext = false; + while(!_hasNext && index.hasNext()){ + var currentInfo = index.next(); + if(currentInfo != null && isValidInfo(currentInfo)){ + _hasNext = true; + _nextInfo = currentInfo; + } + } + } + + private boolean isValidInfo(IndexInformation indexInfo){ + if(!config.hasTimeBasedFiltering()) return true; + else { + long timeStamp = indexInfo.getLastModified(); + return config.since <= timeStamp && timeStamp <= config.until; + } + } + + @Override + public boolean hasNext() { + if(_needsUpdate){ + findNext(); + _needsUpdate = false; + } + return _hasNext; + } + + @Override + public ArtifactResolutionContext next() { + if(hasNext()){ + _needsUpdate = true; + + final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(_nextInfo.getIdent()); + ctx.getRootArtifact().setIndexInformation(_nextInfo); + + return ctx; + } else throw new IllegalStateException("Call to next on empty artifact source"); + } + } + + private class MavenCentralCustomListArtifactSource implements Iterator { + + private final FileBasedArtifactIterator list; + + MavenCentralCustomListArtifactSource() throws IOException { + list = new FileBasedArtifactIterator(config.inputListFile); + list.validateInput(); + } + + @Override + public boolean hasNext() { + return list.hasNext(); + } + + @Override + public ArtifactResolutionContext next() { + if(hasNext()){ + return ArtifactResolutionContext.newInstance(list.next()); + } else throw new IllegalStateException("Call to next on empty artifact source"); + } + + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java index 1c1545c..f2f3dd8 100644 --- a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -45,5 +45,13 @@ public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { outputDirectory = DEFAULT_VALUE_OUTPUT; } + /** + * Checks whether there is a custom time range to filter artifacts for. + * @return True if time based filtering is enabled + */ + public boolean hasTimeBasedFiltering(){ + return this.since != DEFAULT_VALUE_SINCE && this.until != DEFAULT_VALUE_UNTIL; + } + } diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java index 23c1b7e..5341b6d 100644 --- a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java @@ -87,4 +87,12 @@ public boolean hasSkip() { public boolean hasTake() { return this.take != DEFAULT_VALUE_TAKE; } + + /** + * Checks whether there is a custom input list of entities to process. + * @return True if there is an input list specified + */ + public boolean hasInputList() { + return this.inputListFile != DEFAULT_VALUE_INPUT_LIST; + } } diff --git a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java index be261f1..eb6ec37 100644 --- a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java @@ -26,8 +26,16 @@ public static ArtifactResolutionContext newInstance(ArtifactIdent identifier) { * Returns the identifier of the artifact for which this context was created. * @return ArtifactIdent for this context */ - public ArtifactIdent getIdentifier() { + public ArtifactIdent getRootArtifactIdentifier() { return identifier; } + /** + * Returns the artifact for which this context was initially created + * @return The root artifact of this context + */ + public Artifact getRootArtifact() { + return this.createArtifact(getRootArtifactIdentifier()); + } + } diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index 5e68bdf..2ebe7d6 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -28,7 +28,7 @@ public ProcessIdentifierMessage(ArtifactResolutionContext resolutionContext, Mav * @return The artifact identifier */ public ArtifactIdent getIdentifier() { - return this.artifactCtx.getIdentifier(); + return this.artifactCtx.getRootArtifactIdentifier(); } /** diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java index c769515..8e646c9 100644 --- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java +++ b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java @@ -1,5 +1,6 @@ package org.tudo.sse.utils; +import org.eclipse.aether.transfer.ArtifactNotFoundException; import org.tudo.sse.model.ArtifactIdent; import java.io.IOException; @@ -25,6 +26,33 @@ public FileBasedArtifactIterator(Path input){ this.inputPath = input; } + /** + * Checks whether the underlying file exists and contains only valid GAV triples. + * @throws IOException If accessing the underlying file fails + * @throws IllegalArgumentException If the underlying file contents are not valid + */ + public void validateInput() throws IOException, IllegalArgumentException { + var lines = Files.readAllLines(inputPath); + int lineCnt = 0; + + for(String line: lines){ + final String[] parts = line.split(":"); + if(parts.length != 3){ + throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + + for(String part: parts){ + if(part.isBlank()){ + throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + } + + lineCnt += 1; + } + + this.lineIterator = lines.iterator(); + } + @Override public boolean hasNext() { if(this.lineIterator == null){ diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java new file mode 100644 index 0000000..cc54a26 --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java @@ -0,0 +1,204 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MarinOpalLogger; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; + +public class MavenCentralArtifactIteratorTest { + + private final Path gavInputList = Paths.get("src/main/resources/coordinates.txt"); + + @BeforeAll + static void setUp() { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + + @Test + @DisplayName("The artifact iterator must process all inputs from custom GAV lists") + void readFromFile() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, false, true, config); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertFalse(current.hasIndexInformation()); + assertTrue(current.hasPomInformation()); + assertTrue(current.hasJarInformation()); + + artifactCount++; + } + + assertEquals(10, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must apply pagination to custom GAV lists") + void readFromFilePaginated() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .withSkip(2) + .withTake(5) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, true, false, config); + + ArtifactIdent firstEntry = getIdentInInputFile(0); + ArtifactIdent secondEntry = getIdentInInputFile(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertFalse(current.hasIndexInformation()); + assertTrue(current.hasPomInformation()); + assertFalse(current.hasJarInformation()); + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getIdent()); + assertNotEquals(secondEntry, current.getIdent()); + + artifactCount++; + } + + assertEquals(5, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must not produce any values on invalid input files") + void readFromFileInvalid() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(testResource("artifact-names-invalid.txt")) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, true, true, config); + + // If the source file contains at least one line that is not a valid GAV triple, it must not produce any elements + assertFalse(iterator.hasNext()); + assertThrows(IllegalStateException.class, iterator::next); + } + + @Test + @DisplayName("The artifact iterator must apply pagination to the Central index") + void readFromIndexPaginated() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withSkip(2) + .withTake(5) + .build(); + + var iterator = new MavenCentralArtifactIterator(false, false, true, config); + + ArtifactIdent firstEntry = getIdentInIndex(0); + ArtifactIdent secondEntry = getIdentInIndex(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertTrue(current.hasIndexInformation()); + assertFalse(current.hasPomInformation()); + assertTrue(current.hasJarInformation()); + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getIdent()); + assertNotEquals(secondEntry, current.getIdent()); + + artifactCount++; + } + + assertEquals(5, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must apply time range filtering to the Central index") + void readFromIndexFiltered() throws InvalidConfigurationException { + // We expect two entries in this range + long since = 1106902436000L; + long until = 1106902438001L; + + var config = new ArtifactAnalysisConfigBuilder() + .withSinceUtil(since, until) + .build(); + + var iterator = new MavenCentralArtifactIterator(false, false, false, config); + + int artifactCount = 0; + + while(iterator.hasNext() && artifactCount < 2){ + Artifact current = iterator.next(); + assertTrue(current.hasIndexInformation()); + assertFalse(current.hasPomInformation()); + assertFalse(current.hasJarInformation()); + + assertTrue(since <= current.getIndexInformation().getLastModified()); + assertTrue(current.getIndexInformation().getLastModified() <= until); + + assertEquals("ymsg", current.getIdent().getGroupID()); + + artifactCount++; + } + } + + private ArtifactIdent getIdentInInputFile(int pos) { + try { + var lines = Files.readAllLines(gavInputList); + + if(pos >= lines.size()) fail("Invalid position in input list: " + pos); + + var parts = lines.get(pos).split(":"); + + if(parts.length != 3) fail("Invalid GAV in input list: " + pos); + + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } catch (IOException iox) { + fail(iox); + } + + return null; + } + + private ArtifactIdent getIdentInIndex(int pos){ + try { + IndexIterator ii = new IndexIterator(MavenCentralRepository.RepoBaseURI); + + int position = 0; + + while(ii.hasNext()){ + var ident = ii.next().getIdent(); + if(position == pos){ + return ident; + } + position++; + } + + ii.closeReader(); + + } catch (IOException iox) { + fail(iox); + } + + return null; + } +} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index 3ec6545..8ca7d1f 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -16,6 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; public class MavenCentralLibraryAnalysisTest { @@ -328,14 +329,6 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertFalse(librariesHit.contains("yom:yom")); } - private Path testResource(String pathToResource){ - try { - return Path.of(Objects.requireNonNull(getClass().getClassLoader().getResource(pathToResource)).toURI()); - } catch (Exception x){ - fail("Test setup: Failed to load resource file " + pathToResource, x); - } - return null; - } private LibraryAnalysisConfig parseCLI(String cli) { try { diff --git a/src/test/java/org/tudo/sse/utils/TestUtilities.java b/src/test/java/org/tudo/sse/utils/TestUtilities.java new file mode 100644 index 0000000..7e7a686 --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/TestUtilities.java @@ -0,0 +1,19 @@ +package org.tudo.sse.utils; + +import java.nio.file.Path; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.fail; + +public class TestUtilities { + + public static Path testResource(String pathToResource){ + try { + return Path.of(Objects.requireNonNull(TestUtilities.class.getClassLoader().getResource(pathToResource)).toURI()); + } catch (Exception x){ + fail("Test setup: Failed to load resource file " + pathToResource, x); + } + return null; + } + +} diff --git a/src/test/resources/artifact-names-invalid.txt b/src/test/resources/artifact-names-invalid.txt new file mode 100644 index 0000000..827f980 --- /dev/null +++ b/src/test/resources/artifact-names-invalid.txt @@ -0,0 +1,10 @@ +org.springframework.boot:spring-boot-starter-web:2.6.3 +org.apache.commons:commons-lang3:3.12.0 +com.fasterxml.jackson.core:jackson-databind +junit:junit:4.13.2 +org.slf4j:slf4j-api:1.7.32 +org.hibernate:hibernate-core:5.6.5.Final +org.apache.httpcomponents:httpclient:4.5.13 +org.springframework:spring-core:5.3.15 +mysql:mysql-connector-java:8.0.28:: +org.apache.logging.log4j:log4j-core:2.17.1 \ No newline at end of file From 96aa62f714fa258e33bf96651b2fe7e08e24c2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 12 Dec 2025 10:21:36 +0100 Subject: [PATCH 3/6] Finish analysis iterator for libraries, some restructuring to remove duplication --- .../org/tudo/sse/analyses/AnalysisUtils.java | 24 +- .../MavenCentralArtifactAnalysis.java | 2 +- .../MavenCentralArtifactIterator.java | 24 +- .../analyses/MavenCentralLibraryIterator.java | 280 ++++++++++++++++++ .../input/AbstractInputFileIterator.java | 91 ++++++ .../input/FileBasedArtifactIterator.java | 42 +++ .../input/FileBasedLibraryIterator.java | 38 +++ .../sse/utils/FileBasedArtifactIterator.java | 81 ----- .../MavenCentralLibraryIteratorTest.java | 229 ++++++++++++++ .../resources/library-names-not-existing.txt | 1 + 10 files changed, 707 insertions(+), 105 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java create mode 100644 src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java create mode 100644 src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java create mode 100644 src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java delete mode 100644 src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java create mode 100644 src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java create mode 100644 src/test/resources/library-names-not-existing.txt diff --git a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java index 59aad14..d1d221f 100644 --- a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java +++ b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java @@ -1,6 +1,7 @@ package org.tudo.sse.analyses; import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -13,30 +14,30 @@ * * @author Johannes Düsing */ -interface AnalysisUtils { - - LibraryAnalysisConfig getConfig(); +final class AnalysisUtils { /** * Retrieves the initial position to start analysis from, based on the current configuration. + * @param config The current analysis configuration + * * @return Initial number of entities to skip before starting */ - default long getInitialPosition() { - final var config = getConfig(); - - if(config.progressRestoreFile != null) return getRestoreProgressValue(); + static long getInitialPosition(LibraryAnalysisConfig config) { + if(config.progressRestoreFile != null) return getRestoreProgressValue(config); else if(config.hasSkip()) return config.skip; else return 0L; } /** * Retrieves the last progress value to start from based on a previous run. + * @param config The current analysis configuration + * * @return Progress value */ - default long getRestoreProgressValue() { + static long getRestoreProgressValue(LibraryAnalysisConfig config) { BufferedReader indexReader; try { - indexReader = new BufferedReader(new FileReader(getConfig().progressRestoreFile.toFile())); + indexReader = new BufferedReader(new FileReader(config.progressRestoreFile.toFile())); String line = indexReader.readLine(); return Integer.parseInt(line); } catch (IOException e) { @@ -47,10 +48,9 @@ default long getRestoreProgressValue() { /** * Writes the current position to the specified progress output file. * @param currentPosition The current progress to write + * @param config The current analysis configuration */ - default void writePosition(long currentPosition) { - final var config = getConfig(); - + static void writePosition(long currentPosition, LibraryAnalysisConfig config) { try(BufferedWriter writer = Files.newBufferedWriter(config.progressOutputFile)) { writer.write(Long.toString(currentPosition)); } catch(IOException ignored) {} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index bbd7f71..35f7405 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -14,7 +14,7 @@ import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; -import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; import org.tudo.sse.utils.MavenCentralRepository; diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java index 741f126..f5b99db 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -3,12 +3,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; -import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactResolutionContext; import org.tudo.sse.model.index.IndexInformation; import org.tudo.sse.resolution.ResolverFactory; -import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -21,8 +20,7 @@ * * @author Johannes Düsing */ -@SuppressWarnings("this-escape") -public class MavenCentralArtifactIterator implements Iterator, AnalysisUtils { +public class MavenCentralArtifactIterator implements Iterator { private final Logger log = LoggerFactory.getLogger(MavenCentralArtifactIterator.class); private final ArtifactAnalysisConfig config; @@ -119,20 +117,15 @@ public Artifact next(){ return artifact; } - @Override - public LibraryAnalysisConfig getConfig(){ - return this.config; - } - private void writePositionIfNeeded(){ if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ - this.writePosition(this.currentPosition); + AnalysisUtils.writePosition(this.currentPosition, this.config); this.lastPositionSaved = this.currentPosition; } } private void skipInitial(){ - final long entitiesToSkip = getInitialPosition(); + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); if(entitiesToSkip > 0L && this.badSource) return; @@ -176,6 +169,7 @@ private class MavenCentralIndexArtifactSource implements Iterator { + + private final Logger log = LoggerFactory.getLogger(MavenCentralLibraryIterator.class); + private final LibraryAnalysisConfig config; + + private Iterator source; + + private boolean badSource = false; + private boolean sourceClosed = false; + + private long currentPosition = 0L; + private long librariesTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + private final IReleaseListProvider releaseListProvider; + private Consumer failedLibraryCallback = null; + + /** + * Creates a new iterator instance with the given configuration values. + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder} + */ + public MavenCentralLibraryIterator(boolean resolvePom, + boolean resolveTransitivePoms, + boolean resolveJar, + LibraryAnalysisConfig config) { + this.config = config; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + if(this.config.multipleThreads){ + log.warn("Library iterator does no support multiple threads - using a single thread for resolution"); + } + + this.setUnderlyingSource(); + + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(this.badSource) return false; + + // If the config specifies a limited amount of libraries to take, and that amount is reached, we have no next element + if(this.config.hasTake() && this.librariesTaken >= this.config.take){ + closeSource(); + return false; + } + + // Otherwise, we have a next library if the underlying source has a next element + try { + boolean hasNext = this.source.hasNext(); + + if(!hasNext) + closeSource(); + + return hasNext; + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + } + + @Override + public LibraryResolutionContext next() { + if(!hasNext()) + throw new IllegalStateException("No more libraries available"); + + final String nextGA = this.source.next(); + final String[] gaParts = nextGA.split(":"); // Valid tuple, ensured by source + final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(nextGA); + + // Get list of all library releases + final List allReleases = getReleaseIdentifiers(gaParts[0], gaParts[1]); + + if(allReleases != null){ + // For each release, build artifact and enrich with data + for(ArtifactIdent release : allReleases){ + final Artifact artifact = ctx.createArtifact(release); + ctx.addLibraryArtifact(artifact); + + if(this.resolvePom){ + resolverFactory.runPom(artifact.getIdent(), ctx); + } + + if(this.resolveJar){ + resolverFactory.runJar(artifact.getIdent(), ctx); + } + } + } else { + // If we failed to obtain a release list, the respective callback has been invoked. Here, we just return an + // empty context. + log.warn("No releases for library {}, returning empty context", nextGA); + } + + this.librariesTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return ctx; + } + + /** + * Sets the callback that is invoked when obtaining a release list for a given library fails. + * @param failedLibraryCallback Callback on failed libraries + */ + public void setFailedLibraryCallback(Consumer failedLibraryCallback){ + this.failedLibraryCallback = failedLibraryCallback; + } + + private List getReleaseIdentifiers(String groupId, String artifactId){ + try { + final List libraryVersions = releaseListProvider.getReleases(groupId, artifactId); + List identifiers = new ArrayList<>(); + for(String libraryVersion : libraryVersions){ + identifiers.add(new ArtifactIdent(groupId, artifactId, libraryVersion)); + } + return identifiers; + } catch (Exception x) { + log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + + if(this.failedLibraryCallback != null){ + final var event = new FailedLibraryEvent(groupId, artifactId, x); + try { this.failedLibraryCallback.accept(event); } catch (Exception ex) { + log.warn("Unexcepted exception in failed analysis callback for library {}:{}", groupId, artifactId, ex); + } + } + + return null; + } + } + + private void skipInitial(){ + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ + AnalysisUtils.writePosition(this.currentPosition, this.config); + this.lastPositionSaved = this.currentPosition; + } + } + + private void setUnderlyingSource(){ + if(this.config.hasInputList()){ + try { + var iterator = new FileBasedLibraryIterator(this.config.inputListFile); + iterator.validateInput(); + this.source = iterator; + } catch (IOException | IllegalArgumentException x){ + log.error("The given input list file is invalid", x); + this.badSource = true; + } + } else { + try { + this.source = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + } catch (IOException iox){ + log.error("Failed to access Maven Central Index", iox); + this.badSource = true; + } + + } + } + + private void closeSource(){ + if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ + try {((AutoCloseable)this.source).close();} + catch (Exception ignored){} + this.sourceClosed = true; + } + } + + /** + * Event that is thrown when obtaining a library's release list fails. + */ + public static class FailedLibraryEvent { + + private final String groupId; + private final String artifactId; + private final Throwable cause; + + /** + * Creates a new event instance. + * @param groupId The library group id + * @param artifactId The library artifact id + * @param cause The throwable that caused the failure + */ + FailedLibraryEvent(String groupId, String artifactId, Throwable cause){ + this.groupId = groupId; + this.artifactId = artifactId; + this.cause = cause; + } + + /** + * Get the failed library's groupID + * @return Maven groupID + */ + public String getGroupId() { + return this.groupId; + } + + /** + * Get the failed library's artifactID + * @return Maven artifactID + */ + public String getArtifactId() { + return this.artifactId; + } + + /** + * Get the failure cause + * @return Throwable that caused the failure + */ + public Throwable getCause() { + return this.cause; + } + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java new file mode 100644 index 0000000..d3df78b --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java @@ -0,0 +1,91 @@ +package org.tudo.sse.analyses.input; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; + +/** + * An iterator that wraps an underlying file that contains line-wise representations of entities. This wrapper produces + * one entity per line, with parsing logic defined by concrete subclasses. + * + * @param The type that entities are parsed to + * + * @author Johannes Düsing + */ +abstract class AbstractInputFileIterator implements Iterator { + + /** + * Checks whether a given line within the input file represents a valid entity + * @param line The line to check + * @return True is the line represents a valid entity, false otherwise + */ + protected abstract boolean isValidLine(String line); + + /** + * Parses a given line into its representing entity. Will only ever be called after isValidLine returned true. + * @param line The line to parse + * @return The resulting entity + */ + protected abstract T parseLine(String line); + + private final Path inputPath; + + private Iterator lineIterator; + + /** + * Creates a new iterator instance for the given file path. + * @param input Path to a text file containing one entity per line + */ + protected AbstractInputFileIterator(Path input){ + this.inputPath = input; + } + + /** + * Checks whether the underlying file exists and contains only valid entities. + * @throws IOException If accessing the underlying file fails + * @throws IllegalArgumentException If the underlying file contents are not valid + */ + public void validateInput() throws IOException, IllegalArgumentException { + var lines = Files.readAllLines(inputPath); + int lineCnt = 0; + + for(String line: lines){ + if(!isValidLine(line)){ + throw new IllegalArgumentException("Not a valid entity: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + + lineCnt += 1; + } + + this.lineIterator = lines.iterator(); + } + + @Override + public boolean hasNext() { + if(this.lineIterator == null){ + try { + var lines = Files.readAllLines(this.inputPath); + this.lineIterator = lines.iterator(); + } catch (IOException iox){ + throw new IllegalArgumentException("Failed to access input file", iox); + } + } + + return this.lineIterator.hasNext(); + } + + @Override + public T next() { + if(!hasNext()) + throw new IllegalStateException("Next on empty iterator"); + + String line = this.lineIterator.next(); + + if(isValidLine(line)) + return parseLine(line); + else + throw new IllegalStateException("Not a valid entity: " + line); + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java new file mode 100644 index 0000000..4d646a8 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java @@ -0,0 +1,42 @@ +package org.tudo.sse.analyses.input; + +import org.tudo.sse.model.ArtifactIdent; + +import java.nio.file.Path; + +/** + * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of + * groupID:artifactID:version per line. + */ +public class FileBasedArtifactIterator extends AbstractInputFileIterator { + + /** + * Creates a new artifact identifier iterator for the given input file. + * @param gavList Path to input file + */ + public FileBasedArtifactIterator(Path gavList) { + super(gavList); + } + + @Override + protected boolean isValidLine(String line) { + var parts = line.split(":"); + + if(parts.length != 3) + return false; + + for(String part: parts){ + if(part.isBlank()) + return false; + } + + return true; + } + + @Override + protected ArtifactIdent parseLine(String line) { + var parts = line.split(":"); + + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } +} diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java new file mode 100644 index 0000000..a8138e6 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java @@ -0,0 +1,38 @@ +package org.tudo.sse.analyses.input; + +import java.nio.file.Path; + +/** + * Iterator that reads Maven Central library identifiers from a text file. Expects one colon-separated tuple of + * groupID:artifactID per line. + */ +public class FileBasedLibraryIterator extends AbstractInputFileIterator{ + + /** + * Creates a new library identifier iterator for the given input file. + * @param gaList Path to input list + */ + public FileBasedLibraryIterator(Path gaList) { + super(gaList); + } + + @Override + protected boolean isValidLine(String line) { + var parts = line.split(":"); + + if(parts.length != 2) + return false; + + for(String part: parts){ + if(part.isBlank()) + return false; + } + + return true; + } + + @Override + protected String parseLine(String line) { + return line; + } +} diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java deleted file mode 100644 index 8e646c9..0000000 --- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.tudo.sse.utils; - -import org.eclipse.aether.transfer.ArtifactNotFoundException; -import org.tudo.sse.model.ArtifactIdent; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Iterator; - -/** - * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of - * groupID:artifactID:version per line. - */ -public class FileBasedArtifactIterator implements Iterator { - - private final Path inputPath; - - private Iterator lineIterator; - - /** - * Creates a new iterator instance for the given file path. - * @param input Path to a text file containing GAV triples - */ - public FileBasedArtifactIterator(Path input){ - this.inputPath = input; - } - - /** - * Checks whether the underlying file exists and contains only valid GAV triples. - * @throws IOException If accessing the underlying file fails - * @throws IllegalArgumentException If the underlying file contents are not valid - */ - public void validateInput() throws IOException, IllegalArgumentException { - var lines = Files.readAllLines(inputPath); - int lineCnt = 0; - - for(String line: lines){ - final String[] parts = line.split(":"); - if(parts.length != 3){ - throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); - } - - for(String part: parts){ - if(part.isBlank()){ - throw new IllegalArgumentException("Not a valid GAV-Triple: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); - } - } - - lineCnt += 1; - } - - this.lineIterator = lines.iterator(); - } - - @Override - public boolean hasNext() { - if(this.lineIterator == null){ - try { - var lines = Files.readAllLines(this.inputPath); - this.lineIterator = lines.iterator(); - } catch (IOException iox){ - throw new IllegalArgumentException("Failed to access input file", iox); - } - } - - return this.lineIterator.hasNext(); - } - - @Override - public ArtifactIdent next() { - String line = this.lineIterator.next(); - - String[] parts = line.split(":"); - if(parts.length == 3) { - return new ArtifactIdent(parts[0], parts[1], parts[2]); - } else { - throw new IllegalStateException("Not a valid GAV-Triple: " + line); - } - } -} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java new file mode 100644 index 0000000..47daa0b --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java @@ -0,0 +1,229 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.LibraryResolutionContext; +import org.tudo.sse.resolution.FileNotFoundException; +import org.tudo.sse.utils.LibraryIndexIterator; +import org.tudo.sse.utils.MarinOpalLogger; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; + +public class MavenCentralLibraryIteratorTest { + + private static final Path validGAList = testResource("library-names-valid.txt"); + private static final Path invalidGAList = testResource("library-names-invalid.txt"); + private static final Path nonExistingGAList = testResource("library-names-not-existing.txt"); + + @BeforeAll + static void setUp() { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + + @Test + @DisplayName("The library iterator must process all inputs from custom GA lists") + void readFromFile() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(validGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + int libraryCount = 0; + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertTrue(release.hasPomInformation()); + assertFalse(release.hasJarInformation()); + } + + + libraryCount++; + } + + assertEquals(5, libraryCount); + } + + @Test + @DisplayName("The library iterator must apply pagination to custom GA lists") + void readFromFilePaginated() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(validGAList) + .withTake(2) + .withSkip(2) + .build(); + + var iterator = new MavenCentralLibraryIterator(false, false, true, config); + + int libraryCount = 0; + + String firstEntry = getGaInInputFile(0); + String secondEntry = getGaInInputFile(1); + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertFalse(release.hasPomInformation()); + assertTrue(release.hasJarInformation()); + } + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getLibraryGA()); + assertNotEquals(secondEntry, current.getLibraryGA()); + + libraryCount++; + } + + assertEquals(2, libraryCount); + } + + @Test + @DisplayName("The library iterator must not produce any values on invalid input files") + void readFromFileInvalid() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(invalidGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, true, true, config); + + // If the source file contains at least one line that is not a valid GA tuple, it must not produce any elements + assertFalse(iterator.hasNext()); + assertThrows(IllegalStateException.class, iterator::next); + } + + @Test + @DisplayName("The library iterator must apply pagination to the Central index") + void readFromIndexPaginated() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withSkip(2) + .withTake(3) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + String firstEntry = getGaInIndex(0); + String secondEntry = getGaInIndex(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertTrue(release.hasPomInformation()); + assertFalse(release.hasJarInformation()); + } + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getLibraryGA()); + assertNotEquals(secondEntry, current.getLibraryGA()); + + artifactCount++; + } + + assertEquals(3, artifactCount); + } + + @Test + @DisplayName("The library iterator must invoke the registered callback on failure") + void setCallback() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(nonExistingGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + // Define a callback for non-existing release lists + AtomicBoolean callbackInvoked = new AtomicBoolean(false); + + iterator.setFailedLibraryCallback( event -> { + assertEquals("no", event.getGroupId()); + assertEquals("lib", event.getArtifactId()); + + Throwable rootCause = event.getCause(); + while(rootCause.getCause() != null){ + rootCause = rootCause.getCause(); + } + + assertInstanceOf(FileNotFoundException.class, rootCause); + + callbackInvoked.set(true); + }); + + while(iterator.hasNext()){ + LibraryResolutionContext ctx = iterator.next(); + assertTrue(ctx.getLibraryArtifacts().isEmpty()); + } + + assertTrue(callbackInvoked.get()); + } + + + private String getGaInIndex(int pos) { + try { + LibraryIndexIterator libIt = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + + int position = 0; + while(libIt.hasNext()){ + var libGA = libIt.next(); + if(position == pos){ + return libGA; + } + position++; + } + + libIt.close(); + } catch(IOException iox) { + fail(iox); + } + + return null; + } + + private String getGaInInputFile(int pos) { + try { + assertNotNull(validGAList); + + var lines = Files.readAllLines(validGAList); + + if(pos >= lines.size()) fail("Invalid position in input list: " + pos); + + return lines.get(pos); + } catch (IOException iox) { + fail(iox); + } + + return null; + } +} diff --git a/src/test/resources/library-names-not-existing.txt b/src/test/resources/library-names-not-existing.txt new file mode 100644 index 0000000..300a466 --- /dev/null +++ b/src/test/resources/library-names-not-existing.txt @@ -0,0 +1 @@ +no:lib From 83c100e3f135e440d313798de5e92f527a9e4da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 12 Dec 2025 11:49:12 +0100 Subject: [PATCH 4/6] Refractor new iterators to extract common functionality --- .../sse/analyses/AbstractEntityIterator.java | 190 ++++++++++++++++++ .../MavenCentralArtifactIterator.java | 129 ++---------- .../analyses/MavenCentralLibraryIterator.java | 138 ++----------- 3 files changed, 223 insertions(+), 234 deletions(-) create mode 100644 src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java diff --git a/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java new file mode 100644 index 0000000..8d97a2b --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java @@ -0,0 +1,190 @@ +package org.tudo.sse.analyses; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ResolutionContext; +import org.tudo.sse.resolution.ResolverFactory; + +import java.util.Iterator; + +/** + * An abstract base class for analysis-equivalent iterators. This class assumes an underlying source iterator that + * produces a generic kind of source object (an "identifier" for the entity), and then transforms the source object into + * a target entity of generic type. The iterator enforces pagination and supports progress dumps and progress restores. + * + * @param Type of source objects as produced by the underlying iterator + * @param Type of target objects that are created by this iterator + * + * @author Johannes Düsing + */ +abstract class AbstractEntityIterator implements Iterator { + + /** + * Logger instance for this iterator + */ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + /** + * The base configuration for this iterator instance - subclasses may have a more specific configuration type. + */ + protected final LibraryAnalysisConfig baseConfig; + + /** + * Flag indicating whether the underlying source is invalid, i.e. because a connection attempt failed + */ + protected boolean badSource = false; + + private final Iterator source; + + private boolean sourceClosed = false; + + private long currentPosition = 0L; + private long entitiesTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + /** + * Builds the underlying source iterator for this entity iterator instance. + * @return The source iterator + */ + protected abstract Iterator buildSource(); + + /** + * Transforms a source object into the target entity. + * @param source The source object as produced by the underlying iterator + * @return The target entity + */ + protected abstract T buildEntity(S source); + + /** + * Creates a new entity iterator instance with the given configuration. Immediately sets the underlying source and + * skips all initial entities according to the configuration. + * + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param baseConfig The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder} + */ + AbstractEntityIterator(boolean resolvePom, + boolean resolveTransitivePoms, + boolean resolveJar, + LibraryAnalysisConfig baseConfig){ + this.baseConfig = baseConfig; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + this.source = buildSource(); + + if(baseConfig instanceof ArtifactAnalysisConfig && ((ArtifactAnalysisConfig)this.baseConfig).outputEnabled){ + this.resolverFactory = new ResolverFactory(true, + ((ArtifactAnalysisConfig)this.baseConfig).outputDirectory, + resolveTransitivePoms); + } else { + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + } + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public final boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(this.badSource) return false; + + // If the config specifies a limited amount of entities to take, and that amount is reached, we have no next element + if(this.baseConfig.hasTake() && this.entitiesTaken >= this.baseConfig.take){ + closeSourceIfNeeded(); + return false; + } + + // Otherwise, we have a next entity if the underlying source has a next element + try { + boolean hasNext = this.source.hasNext(); + + if(!hasNext) + closeSourceIfNeeded(); + + return hasNext; + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + } + + @Override + public final T next() { + if(!hasNext()) + throw new IllegalStateException("No more entities available"); + + final S sourceEntity = this.source.next(); + final T entity = this.buildEntity(sourceEntity); + + this.entitiesTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return entity; + } + + /** + * Enriches the given artifact with ArtifactInformation as defined by the current configuration. + * @param a The artifact to enrich + * @param ctx The current artifact resolution context + */ + protected final void enrichArtifact(Artifact a, ResolutionContext ctx){ + if(this.resolvePom) + resolverFactory.runPom(a.getIdent(), ctx); + + if(this.resolveJar) + resolverFactory.runJar(a.getIdent(), ctx); + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.baseConfig.progressWriteInterval){ + AnalysisUtils.writePosition(this.currentPosition, this.baseConfig); + this.lastPositionSaved = this.currentPosition; + } + } + + private void closeSourceIfNeeded(){ + if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ + try {((AutoCloseable)this.source).close();} + catch(Exception ignored){} + this.sourceClosed = true; + } + } + + private void skipInitial(){ + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.baseConfig); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java index f5b99db..e064e57 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -1,12 +1,9 @@ package org.tudo.sse.analyses; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactResolutionContext; import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -20,22 +17,7 @@ * * @author Johannes Düsing */ -public class MavenCentralArtifactIterator implements Iterator { - - private final Logger log = LoggerFactory.getLogger(MavenCentralArtifactIterator.class); - private final ArtifactAnalysisConfig config; - - private Iterator source; - - private boolean badSource = false; - - private long currentPosition = 0L; - private long artifactsTaken = 0L; - private long lastPositionSaved = 0L; - - private final ResolverFactory resolverFactory; - private final boolean resolvePom; - private final boolean resolveJar; +public final class MavenCentralArtifactIterator extends AbstractEntityIterator { /** * Creates a new iterator instance with the given configuration values. @@ -46,120 +28,45 @@ public class MavenCentralArtifactIterator implements Iterator { * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder} */ public MavenCentralArtifactIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, ArtifactAnalysisConfig config) { - this.config = config; - this.resolvePom = resolvePom; - this.resolveJar = resolveJar; + super(resolvePom, resolveTransitivePoms, resolveJar, config); - if(this.config.multipleThreads){ + if(this.baseConfig.multipleThreads){ log.warn("Artifact iterator does no support multiple threads - using a single thread for resolution"); } - - this.setUnderlyingSource(); - - if(this.config.outputEnabled){ - this.resolverFactory = new ResolverFactory(true, this.config.outputDirectory, resolveTransitivePoms); - } else { - this.resolverFactory = new ResolverFactory(resolveTransitivePoms); - } - - try { - if(!this.badSource) - this.skipInitial(); - } catch(Exception x){ - log.error("Failed to initialize iterator", x); - this.badSource = true; - } } @Override - public boolean hasNext() { - // If the underlying source is invalid, we have no next element - if(badSource) return false; - - // If the config specifies a limited amount of artifacts to take, and that amount is reached, we have no next element - if(this.config.hasTake() && this.artifactsTaken >= this.config.take) - return false; - - // Otherwise, we have a next artifact if the underlying source has a next element - try { - return this.source.hasNext(); - } catch(Exception x){ - // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no - // next element. - log.error("Failed to access source", x); - this.badSource = true; - return false; - } - - } - - @Override - public Artifact next(){ - if(!hasNext()) - throw new IllegalStateException("No more artifacts available"); - - final ArtifactResolutionContext ctx = this.source.next(); + protected Artifact buildEntity(ArtifactResolutionContext ctx){ final Artifact artifact = ctx.getRootArtifact(); - if(this.resolvePom){ - resolverFactory.runPom(artifact.getIdent(), ctx); - } - - if(this.resolveJar){ - resolverFactory.runJar(artifact.getIdent(), ctx); - } - - this.artifactsTaken += 1L; - this.currentPosition += 1L; - - writePositionIfNeeded(); + this.enrichArtifact(artifact, ctx); return artifact; } - private void writePositionIfNeeded(){ - if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ - AnalysisUtils.writePosition(this.currentPosition, this.config); - this.lastPositionSaved = this.currentPosition; - } - } - - private void skipInitial(){ - final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); - - if(entitiesToSkip > 0L && this.badSource) - return; - - for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ - this.source.next(); - this.currentPosition += 1L; - } - - log.debug("Successfully skipped to position {}", this.currentPosition); - - if(!this.source.hasNext()){ - log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); - } - } - - private void setUnderlyingSource(){ - if(this.config.hasInputList()){ + @Override + protected Iterator buildSource(){ + if(this.baseConfig.hasInputList()){ try{ - this.source = new MavenCentralCustomListArtifactSource(); + return new MavenCentralCustomListArtifactSource(); } catch(IOException | IllegalArgumentException x){ log.error("The given input list file is invalid", x); this.badSource = true; } } else { try { - this.source = new MavenCentralIndexArtifactSource(); + return new MavenCentralIndexArtifactSource(); } catch(IOException uix){ log.error("Failed to access Maven Central Index", uix); this.badSource = true; } - } + return null; + } + + private ArtifactAnalysisConfig getConfig(){ + return (ArtifactAnalysisConfig) this.baseConfig; } private class MavenCentralIndexArtifactSource implements Iterator { @@ -187,10 +94,10 @@ private void findNext(){ } private boolean isValidInfo(IndexInformation indexInfo){ - if(!config.hasTimeBasedFiltering()) return true; + if(!getConfig().hasTimeBasedFiltering()) return true; else { long timeStamp = indexInfo.getLastModified(); - return config.since <= timeStamp && timeStamp <= config.until; + return getConfig().since <= timeStamp && timeStamp <= getConfig().until; } } @@ -229,7 +136,7 @@ private class MavenCentralCustomListArtifactSource implements Iterator { - - private final Logger log = LoggerFactory.getLogger(MavenCentralLibraryIterator.class); - private final LibraryAnalysisConfig config; - - private Iterator source; - - private boolean badSource = false; - private boolean sourceClosed = false; - - private long currentPosition = 0L; - private long librariesTaken = 0L; - private long lastPositionSaved = 0L; - - private final ResolverFactory resolverFactory; - private final boolean resolvePom; - private final boolean resolveJar; +public final class MavenCentralLibraryIterator extends AbstractEntityIterator { private final IReleaseListProvider releaseListProvider; private Consumer failedLibraryCallback = null; @@ -57,64 +38,19 @@ public MavenCentralLibraryIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, LibraryAnalysisConfig config) { - this.config = config; - this.resolvePom = resolvePom; - this.resolveJar = resolveJar; + super(resolvePom, resolveTransitivePoms, resolveJar, config); - if(this.config.multipleThreads){ + if(this.baseConfig.multipleThreads){ log.warn("Library iterator does no support multiple threads - using a single thread for resolution"); } - this.setUnderlyingSource(); - - this.resolverFactory = new ResolverFactory(resolveTransitivePoms); this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); - - try { - if(!this.badSource) - this.skipInitial(); - } catch(Exception x){ - log.error("Failed to initialize iterator", x); - this.badSource = true; - } } @Override - public boolean hasNext() { - // If the underlying source is invalid, we have no next element - if(this.badSource) return false; - - // If the config specifies a limited amount of libraries to take, and that amount is reached, we have no next element - if(this.config.hasTake() && this.librariesTaken >= this.config.take){ - closeSource(); - return false; - } - - // Otherwise, we have a next library if the underlying source has a next element - try { - boolean hasNext = this.source.hasNext(); - - if(!hasNext) - closeSource(); - - return hasNext; - } catch(Exception x){ - // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no - // next element. - log.error("Failed to access source", x); - this.badSource = true; - return false; - } - } - - @Override - public LibraryResolutionContext next() { - if(!hasNext()) - throw new IllegalStateException("No more libraries available"); - - final String nextGA = this.source.next(); - final String[] gaParts = nextGA.split(":"); // Valid tuple, ensured by source - final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(nextGA); + protected LibraryResolutionContext buildEntity(String ga){ + final String[] gaParts = ga.split(":"); // Valid tuple, ensured by source + final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(ga); // Get list of all library releases final List allReleases = getReleaseIdentifiers(gaParts[0], gaParts[1]); @@ -125,25 +61,13 @@ public LibraryResolutionContext next() { final Artifact artifact = ctx.createArtifact(release); ctx.addLibraryArtifact(artifact); - if(this.resolvePom){ - resolverFactory.runPom(artifact.getIdent(), ctx); - } - - if(this.resolveJar){ - resolverFactory.runJar(artifact.getIdent(), ctx); - } + this.enrichArtifact(artifact, ctx); } } else { // If we failed to obtain a release list, the respective callback has been invoked. Here, we just return an // empty context. - log.warn("No releases for library {}, returning empty context", nextGA); + log.warn("No releases for library {}, returning empty context", ga); } - - this.librariesTaken += 1L; - this.currentPosition += 1L; - - writePositionIfNeeded(); - return ctx; } @@ -177,58 +101,26 @@ private List getReleaseIdentifiers(String groupId, String artifac } } - private void skipInitial(){ - final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.config); - - if(entitiesToSkip > 0L && this.badSource) - return; - - for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ - this.source.next(); - this.currentPosition += 1L; - } - - log.debug("Successfully skipped to position {}", this.currentPosition); - - if(!this.source.hasNext()){ - log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); - } - } - - private void writePositionIfNeeded(){ - if(this.currentPosition - this.lastPositionSaved > this.config.progressWriteInterval){ - AnalysisUtils.writePosition(this.currentPosition, this.config); - this.lastPositionSaved = this.currentPosition; - } - } - - private void setUnderlyingSource(){ - if(this.config.hasInputList()){ + @Override + protected Iterator buildSource(){ + if(this.baseConfig.hasInputList()){ try { - var iterator = new FileBasedLibraryIterator(this.config.inputListFile); + var iterator = new FileBasedLibraryIterator(this.baseConfig.inputListFile); iterator.validateInput(); - this.source = iterator; + return iterator; } catch (IOException | IllegalArgumentException x){ log.error("The given input list file is invalid", x); this.badSource = true; } } else { try { - this.source = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + return new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); } catch (IOException iox){ log.error("Failed to access Maven Central Index", iox); this.badSource = true; } - - } - } - - private void closeSource(){ - if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ - try {((AutoCloseable)this.source).close();} - catch (Exception ignored){} - this.sourceClosed = true; } + return null; } /** From 94c9f31b47e9ec22f836eb88150e81b3aa72ec84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 12 Dec 2025 11:56:48 +0100 Subject: [PATCH 5/6] Move resource file to test directory --- .../MavenCentralArtifactAnalysisTest.java | 16 ++++++++-------- .../MavenCentralArtifactIteratorTest.java | 3 +-- .../resources/artifact-names-valid.txt} | 0 3 files changed, 9 insertions(+), 10 deletions(-) rename src/{main/resources/coordinates.txt => test/resources/artifact-names-valid.txt} (100%) diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 422abf1..b5fb274 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -238,17 +238,17 @@ void indexProcessorProgress() { @DisplayName("An analysis must correctly apply CLI options when reading custom input lists") void readIdentsIn() { List cliInputs = new ArrayList<>(); - String[] args = {"--inputs", "src/main/resources/coordinates.txt"}; + String[] args = {"--inputs", "src/test/resources/artifact-names-valid.txt"}; cliInputs.add(args); - args = new String[] {"--inputs", "src/main/resources/coordinates.txt", "-pof", "src/test/resources/stop.txt"}; + args = new String[] {"--inputs", "src/test/resources/artifact-names-valid.txt", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); List> expected = (List>) json.get("readIdentsIn"); @@ -292,8 +292,8 @@ void checkMultiThreading() { singleArgs.add(new String[]{"-st", "10:1000"}); multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"}); - singleArgs.add(new String[]{"--inputs", "src/main/resources/coordinates.txt"}); - multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/main/resources/coordinates.txt"}); + singleArgs.add(new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt"}); + multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/test/resources/artifact-names-valid.txt"}); for(int i = 0; i < singleArgs.size(); i++) { diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java index cc54a26..438bef6 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java @@ -16,14 +16,13 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import static org.junit.jupiter.api.Assertions.*; import static org.tudo.sse.utils.TestUtilities.testResource; public class MavenCentralArtifactIteratorTest { - private final Path gavInputList = Paths.get("src/main/resources/coordinates.txt"); + private final Path gavInputList = testResource("artifact-names-valid.txt"); @BeforeAll static void setUp() { diff --git a/src/main/resources/coordinates.txt b/src/test/resources/artifact-names-valid.txt similarity index 100% rename from src/main/resources/coordinates.txt rename to src/test/resources/artifact-names-valid.txt From f39f665c5b55570665a2aea56ecb1e442ad160f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Tue, 16 Dec 2025 12:48:14 +0100 Subject: [PATCH 6/6] Add iterators to README --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 1344acb..b8a20ab 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,14 @@ Both analysis types provide a method called `void runAnalysis(String[] args)`. I | `-o `
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | +If you do not want to extend one of the aforementioned base classes, MARIN also provides corresponding implementations of the `java.util.Iterator` interface to enumerate and enrich artifacts or libraries. Note that those implementations are **not threadsafe** and cannot perform resolution in parallel. MARIN provides: +* The `MavenCentralArtifactIterator extends Iterator` is the equivalent to the `MavenCentralArtifactAnalysis` and supports all configuration options besides `--threads`. +* The `MavenCentralLibraryIterator extends Iterator` is the equivalent to the `MavenCentralLibraryAnalysis` and supports all configuration options besides `--threads`. + ## Usage +When using MARIN, you can decide whether to extend one of the abstract base classes available, or to rely on of the analysis-equivalent implementations of `java.util.Iterator`. + +### Extending Abstract Base Classes To use MARIN, you will need to implement two components: 1. You need an implementation of either the abstract class `MavenCentralArtifactAnalysis` or `MavenCentralLibraryAnalysis`. This is your actual analysis implementation that defines how a single artifact or library shall be processed. 2. You need a runner class that passes command line arguments to your analysis implementation. Usually, this will look like this: @@ -64,6 +71,28 @@ public class AnalysisRunner { Once this is implemented, you can run your analysis using the following command. Any of the above-mentioned CLI arguments will work for your analysis implementation. ```java -jar executableName *INSERT CLI HERE* ``` +### Using Iterators +As opposed to extending a base class, in order to use MARIN's iterator implementations you will have to first create an analysis configuration object programmatically. This can be done using the `ArtifactAnalysisConfigBuilder` or `LibraryAnalysisConfigBuilder` classes, respectively. The following example shows how to first build a configuration, and then use it to initialize a `MavenCentralArtifactIterator`. +```java +final boolean resolvePom = true; +final boolean resolveTransitive = false; +final boolean resolveJar = true; +final Path gavInputList = Paths.get("path-to-input-file"); + +final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .withSkip(2) + .withTake(5) + .build(); + +MavenCentralArtifactIterator iterator = new MavenCentralArtifactIterator(resolvePom, resolveTransitive, resolveJar, config); + +while(iterator.hasNext()){ + Artifact current = iterator.next(); + // TODO: Process artifact +} +``` + ## Example Use Cases: In the following, there are some example implementations of `MavenCentralArtifactAnalysis`. All of them can be used with the same `AnalysisRunner` implementation seen above, just replace `MyAnalysisImplementation` with the actual implementation name. You can run each example on the first 1000 Maven artifacts by invoking `java -jar executableName -st 0:1000` for the respective project executable JAR.