diff --git a/README.md b/README.md
index 1344acb..b8a20ab 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,14 @@ Both analysis types provide a method called `void runAnalysis(String[] args)`. I
| `-o
`
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` |
+If you do not want to extend one of the aforementioned base classes, MARIN also provides corresponding implementations of the `java.util.Iterator` interface to enumerate and enrich artifacts or libraries. Note that those implementations are **not threadsafe** and cannot perform resolution in parallel. MARIN provides:
+* The `MavenCentralArtifactIterator extends Iterator` is the equivalent to the `MavenCentralArtifactAnalysis` and supports all configuration options besides `--threads`.
+* The `MavenCentralLibraryIterator extends Iterator` is the equivalent to the `MavenCentralLibraryAnalysis` and supports all configuration options besides `--threads`.
+
## Usage
+When using MARIN, you can decide whether to extend one of the abstract base classes available, or to rely on of the analysis-equivalent implementations of `java.util.Iterator`.
+
+### Extending Abstract Base Classes
To use MARIN, you will need to implement two components:
1. You need an implementation of either the abstract class `MavenCentralArtifactAnalysis` or `MavenCentralLibraryAnalysis`. This is your actual analysis implementation that defines how a single artifact or library shall be processed.
2. You need a runner class that passes command line arguments to your analysis implementation. Usually, this will look like this:
@@ -64,6 +71,28 @@ public class AnalysisRunner {
Once this is implemented, you can run your analysis using the following command. Any of the above-mentioned CLI arguments will work for your analysis implementation.
```java -jar executableName *INSERT CLI HERE* ```
+### Using Iterators
+As opposed to extending a base class, in order to use MARIN's iterator implementations you will have to first create an analysis configuration object programmatically. This can be done using the `ArtifactAnalysisConfigBuilder` or `LibraryAnalysisConfigBuilder` classes, respectively. The following example shows how to first build a configuration, and then use it to initialize a `MavenCentralArtifactIterator`.
+```java
+final boolean resolvePom = true;
+final boolean resolveTransitive = false;
+final boolean resolveJar = true;
+final Path gavInputList = Paths.get("path-to-input-file");
+
+final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder()
+ .withInputList(gavInputList)
+ .withSkip(2)
+ .withTake(5)
+ .build();
+
+MavenCentralArtifactIterator iterator = new MavenCentralArtifactIterator(resolvePom, resolveTransitive, resolveJar, config);
+
+while(iterator.hasNext()){
+ Artifact current = iterator.next();
+ // TODO: Process artifact
+}
+```
+
## Example Use Cases:
In the following, there are some example implementations of `MavenCentralArtifactAnalysis`. All of them can be used with the same `AnalysisRunner` implementation seen above, just replace `MyAnalysisImplementation` with the actual implementation name.
You can run each example on the first 1000 Maven artifacts by invoking `java -jar executableName -st 0:1000` for the respective project executable JAR.
diff --git a/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java
new file mode 100644
index 0000000..8d97a2b
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java
@@ -0,0 +1,190 @@
+package org.tudo.sse.analyses;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfig;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfig;
+import org.tudo.sse.model.Artifact;
+import org.tudo.sse.model.ResolutionContext;
+import org.tudo.sse.resolution.ResolverFactory;
+
+import java.util.Iterator;
+
+/**
+ * An abstract base class for analysis-equivalent iterators. This class assumes an underlying source iterator that
+ * produces a generic kind of source object (an "identifier" for the entity), and then transforms the source object into
+ * a target entity of generic type. The iterator enforces pagination and supports progress dumps and progress restores.
+ *
+ * @param Type of source objects as produced by the underlying iterator
+ * @param Type of target objects that are created by this iterator
+ *
+ * @author Johannes Düsing
+ */
+abstract class AbstractEntityIterator implements Iterator {
+
+ /**
+ * Logger instance for this iterator
+ */
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ /**
+ * The base configuration for this iterator instance - subclasses may have a more specific configuration type.
+ */
+ protected final LibraryAnalysisConfig baseConfig;
+
+ /**
+ * Flag indicating whether the underlying source is invalid, i.e. because a connection attempt failed
+ */
+ protected boolean badSource = false;
+
+ private final Iterator source;
+
+ private boolean sourceClosed = false;
+
+ private long currentPosition = 0L;
+ private long entitiesTaken = 0L;
+ private long lastPositionSaved = 0L;
+
+ private final ResolverFactory resolverFactory;
+ private final boolean resolvePom;
+ private final boolean resolveJar;
+
+ /**
+ * Builds the underlying source iterator for this entity iterator instance.
+ * @return The source iterator
+ */
+ protected abstract Iterator buildSource();
+
+ /**
+ * Transforms a source object into the target entity.
+ * @param source The source object as produced by the underlying iterator
+ * @return The target entity
+ */
+ protected abstract T buildEntity(S source);
+
+ /**
+ * Creates a new entity iterator instance with the given configuration. Immediately sets the underlying source and
+ * skips all initial entities according to the configuration.
+ *
+ * @param resolvePom Whether information on artifact pom files shall be resolved
+ * @param resolveTransitivePoms Whether transitive poms should also be resolved
+ * @param resolveJar Whether information on jar files shall be resolved
+ * @param baseConfig The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder}
+ */
+ AbstractEntityIterator(boolean resolvePom,
+ boolean resolveTransitivePoms,
+ boolean resolveJar,
+ LibraryAnalysisConfig baseConfig){
+ this.baseConfig = baseConfig;
+ this.resolvePom = resolvePom;
+ this.resolveJar = resolveJar;
+
+ this.source = buildSource();
+
+ if(baseConfig instanceof ArtifactAnalysisConfig && ((ArtifactAnalysisConfig)this.baseConfig).outputEnabled){
+ this.resolverFactory = new ResolverFactory(true,
+ ((ArtifactAnalysisConfig)this.baseConfig).outputDirectory,
+ resolveTransitivePoms);
+ } else {
+ this.resolverFactory = new ResolverFactory(resolveTransitivePoms);
+ }
+
+ try {
+ if(!this.badSource)
+ this.skipInitial();
+ } catch(Exception x){
+ log.error("Failed to initialize iterator", x);
+ this.badSource = true;
+ }
+ }
+
+ @Override
+ public final boolean hasNext() {
+ // If the underlying source is invalid, we have no next element
+ if(this.badSource) return false;
+
+ // If the config specifies a limited amount of entities to take, and that amount is reached, we have no next element
+ if(this.baseConfig.hasTake() && this.entitiesTaken >= this.baseConfig.take){
+ closeSourceIfNeeded();
+ return false;
+ }
+
+ // Otherwise, we have a next entity if the underlying source has a next element
+ try {
+ boolean hasNext = this.source.hasNext();
+
+ if(!hasNext)
+ closeSourceIfNeeded();
+
+ return hasNext;
+ } catch(Exception x){
+ // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no
+ // next element.
+ log.error("Failed to access source", x);
+ this.badSource = true;
+ return false;
+ }
+ }
+
+ @Override
+ public final T next() {
+ if(!hasNext())
+ throw new IllegalStateException("No more entities available");
+
+ final S sourceEntity = this.source.next();
+ final T entity = this.buildEntity(sourceEntity);
+
+ this.entitiesTaken += 1L;
+ this.currentPosition += 1L;
+
+ writePositionIfNeeded();
+
+ return entity;
+ }
+
+ /**
+ * Enriches the given artifact with ArtifactInformation as defined by the current configuration.
+ * @param a The artifact to enrich
+ * @param ctx The current artifact resolution context
+ */
+ protected final void enrichArtifact(Artifact a, ResolutionContext ctx){
+ if(this.resolvePom)
+ resolverFactory.runPom(a.getIdent(), ctx);
+
+ if(this.resolveJar)
+ resolverFactory.runJar(a.getIdent(), ctx);
+ }
+
+ private void writePositionIfNeeded(){
+ if(this.currentPosition - this.lastPositionSaved > this.baseConfig.progressWriteInterval){
+ AnalysisUtils.writePosition(this.currentPosition, this.baseConfig);
+ this.lastPositionSaved = this.currentPosition;
+ }
+ }
+
+ private void closeSourceIfNeeded(){
+ if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){
+ try {((AutoCloseable)this.source).close();}
+ catch(Exception ignored){}
+ this.sourceClosed = true;
+ }
+ }
+
+ private void skipInitial(){
+ final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.baseConfig);
+
+ if(entitiesToSkip > 0L && this.badSource)
+ return;
+
+ for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){
+ this.source.next();
+ this.currentPosition += 1L;
+ }
+
+ log.debug("Successfully skipped to position {}", this.currentPosition);
+
+ if(!this.source.hasNext()){
+ log.warn("Reached end of input source while skipping to position {}", entitiesToSkip);
+ }
+ }
+}
diff --git a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java
new file mode 100644
index 0000000..d1d221f
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java
@@ -0,0 +1,58 @@
+package org.tudo.sse.analyses;
+
+import org.tudo.sse.analyses.config.LibraryAnalysisConfig;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.file.Files;
+
+/**
+ * Utility methods for general analysis implementations.
+ *
+ * @author Johannes Düsing
+ */
+final class AnalysisUtils {
+
+ /**
+ * Retrieves the initial position to start analysis from, based on the current configuration.
+ * @param config The current analysis configuration
+ *
+ * @return Initial number of entities to skip before starting
+ */
+ static long getInitialPosition(LibraryAnalysisConfig config) {
+ if(config.progressRestoreFile != null) return getRestoreProgressValue(config);
+ else if(config.hasSkip()) return config.skip;
+ else return 0L;
+ }
+
+ /**
+ * Retrieves the last progress value to start from based on a previous run.
+ * @param config The current analysis configuration
+ *
+ * @return Progress value
+ */
+ static long getRestoreProgressValue(LibraryAnalysisConfig config) {
+ BufferedReader indexReader;
+ try {
+ indexReader = new BufferedReader(new FileReader(config.progressRestoreFile.toFile()));
+ String line = indexReader.readLine();
+ return Integer.parseInt(line);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Writes the current position to the specified progress output file.
+ * @param currentPosition The current progress to write
+ * @param config The current analysis configuration
+ */
+ static void writePosition(long currentPosition, LibraryAnalysisConfig config) {
+ try(BufferedWriter writer = Files.newBufferedWriter(config.progressOutputFile)) {
+ writer.write(Long.toString(currentPosition));
+ } catch(IOException ignored) {}
+ }
+}
diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java
index 2446f86..35f7405 100644
--- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java
+++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java
@@ -1,8 +1,9 @@
package org.tudo.sse.analyses;
-import org.apache.pekko.actor.typed.ActorRef;
import org.apache.pekko.actor.typed.ActorSystem;
import org.tudo.sse.CLIException;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfig;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder;
import org.tudo.sse.model.Artifact;
import org.tudo.sse.model.ArtifactIdent;
import org.tudo.sse.model.index.IndexInformation;
@@ -12,8 +13,8 @@
import org.tudo.sse.multithreading.WorkItem;
import org.tudo.sse.multithreading.WorkloadIsFinalMessage;
import org.tudo.sse.resolution.ResolverFactory;
-import org.tudo.sse.utils.ArtifactConfigParser;
-import org.tudo.sse.utils.FileBasedArtifactIterator;
+import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser;
+import org.tudo.sse.analyses.input.FileBasedArtifactIterator;
import org.tudo.sse.utils.IndexIterator;
import org.tudo.sse.multithreading.QueueActor;
import org.tudo.sse.utils.MavenCentralRepository;
@@ -29,7 +30,7 @@
*/
public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis {
- private ArtifactConfigParser.ArtifactConfig artifactConfig;
+ private ArtifactAnalysisConfig artifactConfig;
private ActorSystem system;
private ResolverFactory resolverFactory;
@@ -53,7 +54,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex,
super(requiresIndex, requiresPom, requiresTransitives, requiresJar);
// Initialize with default config, update later
- artifactConfig = new ArtifactConfigParser.ArtifactConfig();
+ artifactConfig = new ArtifactAnalysisConfigBuilder().build();
}
@@ -68,7 +69,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex,
* Returns the CLI configuration for this analysis.
* @return CLI information for this analysis
*/
- public ArtifactConfigParser.ArtifactConfig getSetupInfo() {
+ public ArtifactAnalysisConfig getSetupInfo() {
return this.artifactConfig;
}
@@ -91,8 +92,8 @@ private void printRunInfo(){
log.info("\t - Reading artifacts from Maven Central index");
if(this.artifactConfig.progressRestoreFile != null) log.info("\t - Restoring last index position from " + this.artifactConfig.progressRestoreFile);
if(this.artifactConfig.progressOutputFile != null) log.info("\t - Writing last index position to " + this.artifactConfig.progressOutputFile);
- if(this.artifactConfig.skip >= 0) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts");
- if(this.artifactConfig.take >= 0) log.info("\t - Taking " + this.artifactConfig.take + " artifacts");
+ if(this.artifactConfig.hasSkip()) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts");
+ if(this.artifactConfig.hasTake()) log.info("\t - Taking " + this.artifactConfig.take + " artifacts");
if(this.artifactConfig.since >= 0) log.info("\t - Skipping artifacts before " + this.artifactConfig.since);
if(this.artifactConfig.until >= 0) log.info("\t - Taking artifacts until " + this.artifactConfig.until);
@@ -105,7 +106,7 @@ private void printRunInfo(){
public final void runAnalysis(String[] args) {
// Obtain CLI arguments
try {
- this.artifactConfig = new ArtifactConfigParser().parseArtifactConfig(args);
+ this.artifactConfig = new ArtifactAnalysisConfigParser().parseArtifactConfig(args);
printRunInfo();
} catch(CLIException clix){
throw new RuntimeException(clix);
@@ -168,7 +169,7 @@ public void indexProcessor() {
final long startingPosition = getInitialPosition();
skipN(startingPosition, indexIterator);
- if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1){
+ if(this.artifactConfig.hasTake()){
walkPaginated(this.artifactConfig.take, indexIterator);
} else if(this.artifactConfig.since != -1 && this.artifactConfig.until != -1){
walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator);
@@ -359,7 +360,7 @@ void processArtifactsFromInputFile() {
log.info("Restoring previous progress from file (position {})", lastProgress);
this.skipN(lastProgress, fileIterator);
- } else if(this.artifactConfig.skip > 0){
+ } else if(this.artifactConfig.hasSkip()){
// Skip configured values only if we did not restore from progress file
log.info("Skipping {} entries from file", artifactConfig.skip);
skipN(this.artifactConfig.skip, fileIterator);
@@ -369,10 +370,8 @@ void processArtifactsFromInputFile() {
if(!fileIterator.hasNext())
log.warn("No more contents left to process in input file: {}", this.artifactConfig.inputListFile);
- final boolean takeLimited = this.artifactConfig.take >= 0;
-
long entriesTaken = 0L;
- while ((!takeLimited || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) {
+ while ((!this.artifactConfig.hasTake() || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) {
ArtifactIdent current = null;
@@ -421,7 +420,7 @@ public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) {
private long getInitialPosition() {
if(artifactConfig.progressRestoreFile != null) return getStartingPos();
- else if(artifactConfig.skip > 0) return artifactConfig.skip;
+ else if(artifactConfig.hasSkip()) return artifactConfig.skip;
else return 0L;
}
diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java
new file mode 100644
index 0000000..e064e57
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java
@@ -0,0 +1,157 @@
+package org.tudo.sse.analyses;
+
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfig;
+import org.tudo.sse.model.Artifact;
+import org.tudo.sse.model.ArtifactResolutionContext;
+import org.tudo.sse.model.index.IndexInformation;
+import org.tudo.sse.analyses.input.FileBasedArtifactIterator;
+import org.tudo.sse.utils.IndexIterator;
+import org.tudo.sse.utils.MavenCentralRepository;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Iterator that produces artifacts that are enriched with index-, pom- or jar information, as configured. This is the
+ * iterator equivalent to the {@link MavenCentralArtifactAnalysis}. Only supports single-threaded resolution.
+ *
+ * @author Johannes Düsing
+ */
+public final class MavenCentralArtifactIterator extends AbstractEntityIterator {
+
+ /**
+ * Creates a new iterator instance with the given configuration values.
+ *
+ * @param resolvePom Whether information on artifact pom files shall be resolved
+ * @param resolveTransitivePoms Whether transitive poms should also be resolved
+ * @param resolveJar Whether information on jar files shall be resolved
+ * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder}
+ */
+ public MavenCentralArtifactIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, ArtifactAnalysisConfig config) {
+ super(resolvePom, resolveTransitivePoms, resolveJar, config);
+
+ if(this.baseConfig.multipleThreads){
+ log.warn("Artifact iterator does no support multiple threads - using a single thread for resolution");
+ }
+ }
+
+ @Override
+ protected Artifact buildEntity(ArtifactResolutionContext ctx){
+ final Artifact artifact = ctx.getRootArtifact();
+
+ this.enrichArtifact(artifact, ctx);
+
+ return artifact;
+ }
+
+
+ @Override
+ protected Iterator buildSource(){
+ if(this.baseConfig.hasInputList()){
+ try{
+ return new MavenCentralCustomListArtifactSource();
+ } catch(IOException | IllegalArgumentException x){
+ log.error("The given input list file is invalid", x);
+ this.badSource = true;
+ }
+ } else {
+ try {
+ return new MavenCentralIndexArtifactSource();
+ } catch(IOException uix){
+ log.error("Failed to access Maven Central Index", uix);
+ this.badSource = true;
+ }
+ }
+ return null;
+ }
+
+ private ArtifactAnalysisConfig getConfig(){
+ return (ArtifactAnalysisConfig) this.baseConfig;
+ }
+
+ private class MavenCentralIndexArtifactSource implements Iterator {
+
+ private final IndexIterator index;
+
+ private boolean _needsUpdate = true;
+ private boolean _hasNext = false;
+ private IndexInformation _nextInfo = null;
+ private boolean _indexClosed = false;
+
+ MavenCentralIndexArtifactSource() throws IOException {
+ this.index = new IndexIterator(MavenCentralRepository.RepoBaseURI);
+ }
+
+ private void findNext(){
+ _hasNext = false;
+ while(!_hasNext && index.hasNext()){
+ var currentInfo = index.next();
+ if(currentInfo != null && isValidInfo(currentInfo)){
+ _hasNext = true;
+ _nextInfo = currentInfo;
+ }
+ }
+ }
+
+ private boolean isValidInfo(IndexInformation indexInfo){
+ if(!getConfig().hasTimeBasedFiltering()) return true;
+ else {
+ long timeStamp = indexInfo.getLastModified();
+ return getConfig().since <= timeStamp && timeStamp <= getConfig().until;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(_needsUpdate){
+ findNext();
+ _needsUpdate = false;
+ }
+
+ // If we have no more entries on the underlying index, we should close it to release resources
+ if(!_hasNext && !_indexClosed){
+ try { this.index.closeReader(); }
+ catch (IOException ignored) {}
+ this._indexClosed = true;
+ }
+
+ return _hasNext;
+ }
+
+ @Override
+ public ArtifactResolutionContext next() {
+ if(hasNext()){
+ _needsUpdate = true;
+
+ final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(_nextInfo.getIdent());
+ ctx.getRootArtifact().setIndexInformation(_nextInfo);
+
+ return ctx;
+ } else throw new IllegalStateException("Call to next on empty artifact source");
+ }
+ }
+
+ private class MavenCentralCustomListArtifactSource implements Iterator {
+
+ private final FileBasedArtifactIterator list;
+
+ MavenCentralCustomListArtifactSource() throws IOException {
+ list = new FileBasedArtifactIterator(baseConfig.inputListFile);
+ list.validateInput();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return list.hasNext();
+ }
+
+ @Override
+ public ArtifactResolutionContext next() {
+ if(hasNext()){
+ return ArtifactResolutionContext.newInstance(list.next());
+ } else throw new IllegalStateException("Call to next on empty artifact source");
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java
index b03dc61..40a3ec8 100644
--- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java
+++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java
@@ -2,6 +2,7 @@
import org.apache.pekko.actor.typed.ActorSystem;
import org.tudo.sse.CLIException;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfig;
import org.tudo.sse.model.Artifact;
import org.tudo.sse.model.ArtifactIdent;
import org.tudo.sse.model.LibraryResolutionContext;
@@ -12,7 +13,7 @@
import org.tudo.sse.resolution.ResolverFactory;
import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider;
import org.tudo.sse.resolution.releases.IReleaseListProvider;
-import org.tudo.sse.utils.CommonConfigParser;
+import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser;
import org.tudo.sse.utils.LibraryIndexIterator;
import org.tudo.sse.utils.MavenCentralRepository;
@@ -36,7 +37,7 @@ public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis {
/**
* The configuration for this analysis instance. Only available after runAnalysis has been called.
*/
- protected CommonConfigParser.CommonConfig config;
+ protected LibraryAnalysisConfig config;
private ActorSystem system;
private long lastPositionSaved;
@@ -61,7 +62,7 @@ protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTrans
public final void runAnalysis(String[] args){
// Obtain CLI arguments
try {
- this.config = new CommonConfigParser().parseCommonConfig(args);
+ this.config = new LibraryAnalysisConfigParser().parseCommonConfig(args);
printRunInfo();
} catch(CLIException clix){
throw new RuntimeException(clix);
@@ -84,9 +85,9 @@ public final void runAnalysis(String[] args){
currentPosition += 1L;
}
// Notify users that skip will not be applied
- if(config.skip > 0)
+ if(config.hasSkip())
log.info("Not applying skip value because progress was restored from previous run");
- } else if(config.skip > 0){
+ } else if(config.hasSkip()){
// We only apply a skip if we did not restore previous progress
log.info("Skipping {} library names", config.skip);
for(int i = 0; i < config.skip; i++){
@@ -106,7 +107,7 @@ public final void runAnalysis(String[] args){
long entriesTaken = 0L;
- if(config.take >= 0)
+ if(config.hasTake())
log.info("Taking {} library names", config.take);
// Invoke the beforeRunStart lifecycle hook
@@ -119,7 +120,7 @@ public final void runAnalysis(String[] args){
// If specified, we only take the configured amount of entries. If not, we process as long as the iterator
// provides new entries
- while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){
+ while((!config.hasTake()|| entriesTaken < config.take) && gaIterator.hasNext()){
processEntry(gaIterator.next());
entriesTaken += 1L;
@@ -336,8 +337,8 @@ private void printRunInfo(){
log.info("\t - Reading libraries from Maven Central index");
if(config.progressRestoreFile != null) log.info("\t - Restoring last index position from " + config.progressRestoreFile);
if(config.progressOutputFile != null) log.info("\t - Writing last index position to " + config.progressOutputFile);
- if(config.skip >= 0) log.info("\t - Skipping " + config.skip + " artifacts");
- if(config.take >= 0) log.info("\t - Taking " + config.take + " artifacts");
+ if(config.hasSkip()) log.info("\t - Skipping " + config.skip + " artifacts");
+ if(config.hasTake()) log.info("\t - Taking " + config.take + " artifacts");
} else {
log.info("\t - Reading libraries from GA-list at " + config.inputListFile);
}
diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java
new file mode 100644
index 0000000..e6c08df
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java
@@ -0,0 +1,172 @@
+package org.tudo.sse.analyses;
+
+import org.tudo.sse.analyses.config.LibraryAnalysisConfig;
+import org.tudo.sse.analyses.input.FileBasedLibraryIterator;
+import org.tudo.sse.model.Artifact;
+import org.tudo.sse.model.ArtifactIdent;
+import org.tudo.sse.model.LibraryResolutionContext;
+import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider;
+import org.tudo.sse.resolution.releases.IReleaseListProvider;
+import org.tudo.sse.utils.LibraryIndexIterator;
+import org.tudo.sse.utils.MavenCentralRepository;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * Iterator that produces libraries that are enriched with index-, pom- or jar information, as configured. This is the
+ * iterator equivalent to the {@link MavenCentralLibraryAnalysis}. Only supports single-threaded resolution.
+ *
+ * @author Johannes Düsing
+ */
+public final class MavenCentralLibraryIterator extends AbstractEntityIterator {
+
+ private final IReleaseListProvider releaseListProvider;
+ private Consumer failedLibraryCallback = null;
+
+ /**
+ * Creates a new iterator instance with the given configuration values.
+ * @param resolvePom Whether information on artifact pom files shall be resolved
+ * @param resolveTransitivePoms Whether transitive poms should also be resolved
+ * @param resolveJar Whether information on jar files shall be resolved
+ * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder}
+ */
+ public MavenCentralLibraryIterator(boolean resolvePom,
+ boolean resolveTransitivePoms,
+ boolean resolveJar,
+ LibraryAnalysisConfig config) {
+ super(resolvePom, resolveTransitivePoms, resolveJar, config);
+
+ if(this.baseConfig.multipleThreads){
+ log.warn("Library iterator does no support multiple threads - using a single thread for resolution");
+ }
+
+ this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance();
+ }
+
+ @Override
+ protected LibraryResolutionContext buildEntity(String ga){
+ final String[] gaParts = ga.split(":"); // Valid tuple, ensured by source
+ final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(ga);
+
+ // Get list of all library releases
+ final List allReleases = getReleaseIdentifiers(gaParts[0], gaParts[1]);
+
+ if(allReleases != null){
+ // For each release, build artifact and enrich with data
+ for(ArtifactIdent release : allReleases){
+ final Artifact artifact = ctx.createArtifact(release);
+ ctx.addLibraryArtifact(artifact);
+
+ this.enrichArtifact(artifact, ctx);
+ }
+ } else {
+ // If we failed to obtain a release list, the respective callback has been invoked. Here, we just return an
+ // empty context.
+ log.warn("No releases for library {}, returning empty context", ga);
+ }
+ return ctx;
+ }
+
+ /**
+ * Sets the callback that is invoked when obtaining a release list for a given library fails.
+ * @param failedLibraryCallback Callback on failed libraries
+ */
+ public void setFailedLibraryCallback(Consumer failedLibraryCallback){
+ this.failedLibraryCallback = failedLibraryCallback;
+ }
+
+ private List getReleaseIdentifiers(String groupId, String artifactId){
+ try {
+ final List libraryVersions = releaseListProvider.getReleases(groupId, artifactId);
+ List identifiers = new ArrayList<>();
+ for(String libraryVersion : libraryVersions){
+ identifiers.add(new ArtifactIdent(groupId, artifactId, libraryVersion));
+ }
+ return identifiers;
+ } catch (Exception x) {
+ log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x);
+
+ if(this.failedLibraryCallback != null){
+ final var event = new FailedLibraryEvent(groupId, artifactId, x);
+ try { this.failedLibraryCallback.accept(event); } catch (Exception ex) {
+ log.warn("Unexcepted exception in failed analysis callback for library {}:{}", groupId, artifactId, ex);
+ }
+ }
+
+ return null;
+ }
+ }
+
+ @Override
+ protected Iterator buildSource(){
+ if(this.baseConfig.hasInputList()){
+ try {
+ var iterator = new FileBasedLibraryIterator(this.baseConfig.inputListFile);
+ iterator.validateInput();
+ return iterator;
+ } catch (IOException | IllegalArgumentException x){
+ log.error("The given input list file is invalid", x);
+ this.badSource = true;
+ }
+ } else {
+ try {
+ return new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI);
+ } catch (IOException iox){
+ log.error("Failed to access Maven Central Index", iox);
+ this.badSource = true;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Event that is thrown when obtaining a library's release list fails.
+ */
+ public static class FailedLibraryEvent {
+
+ private final String groupId;
+ private final String artifactId;
+ private final Throwable cause;
+
+ /**
+ * Creates a new event instance.
+ * @param groupId The library group id
+ * @param artifactId The library artifact id
+ * @param cause The throwable that caused the failure
+ */
+ FailedLibraryEvent(String groupId, String artifactId, Throwable cause){
+ this.groupId = groupId;
+ this.artifactId = artifactId;
+ this.cause = cause;
+ }
+
+ /**
+ * Get the failed library's groupID
+ * @return Maven groupID
+ */
+ public String getGroupId() {
+ return this.groupId;
+ }
+
+ /**
+ * Get the failed library's artifactID
+ * @return Maven artifactID
+ */
+ public String getArtifactId() {
+ return this.artifactId;
+ }
+
+ /**
+ * Get the failure cause
+ * @return Throwable that caused the failure
+ */
+ public Throwable getCause() {
+ return this.cause;
+ }
+ }
+
+}
diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java
new file mode 100644
index 0000000..f2f3dd8
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java
@@ -0,0 +1,57 @@
+package org.tudo.sse.analyses.config;
+
+import java.nio.file.Path;
+
+/**
+ * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance.
+ *
+ * @author Johannes Düsing
+ */
+public class ArtifactAnalysisConfig extends LibraryAnalysisConfig {
+
+ static final long DEFAULT_VALUE_SINCE = -1L;
+ static final long DEFAULT_VALUE_UNTIL = -1L;
+ static final Path DEFAULT_VALUE_OUTPUT = null;
+
+ /**
+ * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled.
+ */
+ public long since;
+
+ /**
+ * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled.
+ */
+ public long until;
+
+ /**
+ * Directory to write file outputs to, or null if disabled.
+ */
+ public Path outputDirectory;
+
+ /**
+ * Whether files shall be written to the output directory.
+ */
+ public boolean outputEnabled;
+
+ /**
+ * Creates a new configuration object with default values.
+ */
+ ArtifactAnalysisConfig(){
+ super();
+
+ since = DEFAULT_VALUE_SINCE;
+ until = DEFAULT_VALUE_UNTIL;
+ outputEnabled = false;
+ outputDirectory = DEFAULT_VALUE_OUTPUT;
+ }
+
+ /**
+ * Checks whether there is a custom time range to filter artifacts for.
+ * @return True if time based filtering is enabled
+ */
+ public boolean hasTimeBasedFiltering(){
+ return this.since != DEFAULT_VALUE_SINCE && this.until != DEFAULT_VALUE_UNTIL;
+ }
+
+
+}
diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java
new file mode 100644
index 0000000..d0e2a33
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java
@@ -0,0 +1,136 @@
+package org.tudo.sse.analyses.config;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Configuration builder to obtain {@link ArtifactAnalysisConfig} instances programmatically.
+ *
+ * @author Johannes Düsing
+ */
+public final class ArtifactAnalysisConfigBuilder extends LibraryAnalysisConfigBuilder {
+
+ private final ArtifactAnalysisConfig theConfig;
+
+ private ArtifactAnalysisConfigBuilder(ArtifactAnalysisConfig theConfig) {
+ super(theConfig);
+
+ this.theConfig = theConfig;
+ }
+
+ /**
+ * Creates a new builder with default configuration values.
+ */
+ public ArtifactAnalysisConfigBuilder(){
+ this(new ArtifactAnalysisConfig());
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException {
+ if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE ||
+ this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL)
+ throw new InvalidConfigurationException("skip", "Cannot apply skip when time-based filtering is used");
+
+ super.withSkip(toSkip);
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException {
+ if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE ||
+ this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL)
+ throw new InvalidConfigurationException("take", "Cannot apply take when time-based filtering is used");
+
+ super.withTake(toTake);
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withInputList(Path inputList) throws InvalidConfigurationException {
+ if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE ||
+ this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL)
+ throw new InvalidConfigurationException("inputs", "Cannot use input list when time-based filtering is applied");
+
+ super.withInputList(inputList);
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException {
+ super.withProgressOutputFile(outputFile);
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException {
+ super.withProgressRestoreFile(restoreFile);
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException {
+ super.withNumberOfThreads(numThreads);
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException {
+ super.withProgressWriteInterval(progressWriteInterval);
+ return this;
+ }
+
+ /**
+ * Sets the output directory to store processed artifacts in. If your analysis requires IndexInformation only, it
+ * will output a list of GAV triples. If pom information is required, one pom.xml file per artifact will be stored.
+ * If JAR information is required, the artifact's JAR will be stored in this directory.
+ *
+ * @param outDir The output directory to use. Must be a valid reference to an existing directory.
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid
+ */
+ public ArtifactAnalysisConfigBuilder withOutputDirectory(Path outDir) throws InvalidConfigurationException {
+ if(outDir == null || !Files.isDirectory(outDir))
+ throw new InvalidConfigurationException("output", "Output directory must be a valid directory reference");
+
+ if(this.theConfig.outputDirectory != ArtifactAnalysisConfig.DEFAULT_VALUE_OUTPUT)
+ log.warn("Value 'output' is set multiple times, previous value overridden.");
+
+ this.theConfig.outputDirectory = outDir;
+ this.theConfig.outputEnabled = true;
+ return this;
+ }
+
+ /**
+ * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have
+ * been released after the timestamp given in 'since', and before 'until'.
+ *
+ * @param since Timestamp marking the lower limit of the range
+ * @param until Timestamp marking the upper limit of the range
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration values are not valid
+ */
+ public ArtifactAnalysisConfigBuilder withSinceUtil(long since, long until) throws InvalidConfigurationException {
+ if(since <= 0L || until <= since)
+ throw new InvalidConfigurationException("since-until", "Since and Until must define a valid range");
+
+ if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP ||
+ this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE)
+ throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when pagination is used");
+
+ if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST)
+ throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when input list is used");
+
+ if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE ||
+ this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL)
+ log.warn("Values 'since' and 'until' are set multiple times, previous values overridden.");
+
+ this.theConfig.since = since;
+ this.theConfig.until = until;
+ return this;
+ }
+
+ @Override
+ public ArtifactAnalysisConfig build(){
+ return this.theConfig;
+ }
+}
diff --git a/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java
new file mode 100644
index 0000000..cb26b5a
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java
@@ -0,0 +1,50 @@
+package org.tudo.sse.analyses.config;
+
+/**
+ * Exception thrown when a MARIN configuration value was invalid.
+ *
+ * @author Johannes Düsing
+ */
+public class InvalidConfigurationException extends Exception {
+
+ /**
+ * Attribute the error occurred for.
+ */
+ private final String attrName;
+
+ /**
+ * Message describing the error.
+ */
+ private final String msg;
+
+ /**
+ * Creates a new instance with the given configuration attribute name and message.
+ * @param attrName The configuration value that was invalid
+ * @param msg A message describing the error
+ */
+ InvalidConfigurationException(String attrName, String msg){
+ this.attrName = attrName;
+ this.msg = msg;
+ }
+
+ /**
+ * Returns the name of the attribute for which this exception occurred.
+ * @return The attribute name
+ */
+ public String getAttributeName() {
+ return this.attrName;
+ }
+
+ /**
+ * Returns a description on the error that occurred.
+ * @return The error description
+ */
+ public String getDescription(){
+ return this.msg;
+ }
+
+ @Override
+ public String getMessage() {
+ return String.format("Invalid configuration value for '%s': %s", attrName, msg);
+ }
+}
diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java
new file mode 100644
index 0000000..5341b6d
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java
@@ -0,0 +1,98 @@
+package org.tudo.sse.analyses.config;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}.
+ *
+ * @author Johannes Düsing
+ */
+public class LibraryAnalysisConfig {
+
+ static final long DEFAULT_VALUE_SKIP = -1L;
+ static final long DEFAULT_VALUE_TAKE = -1L;
+ static final Path DEFAULT_VALUE_INPUT_LIST = null;
+ static final Path DEFAULT_VALUE_PROGRESS_OUTPUT_FILE = Paths.get("marin-progress");
+ static final Path DEFAULT_VALUE_PROGRESS_RESTORE_FILE = null;
+ static final int DEFAULT_NUM_THREADS = 1;
+ static final int DEFAULT_PROGRESS_WRITE_INTERVAL = 100;
+
+ /**
+ * Number of artifacts to skip from the underlying source, or -1 if disabled.
+ */
+ public long skip;
+
+ /**
+ * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied.
+ */
+ public long take;
+
+ /**
+ * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used.
+ */
+ public Path inputListFile;
+
+ /**
+ * File path to output analysis progress to.
+ */
+ public Path progressOutputFile;
+
+ /**
+ * Path to read previous progress from, in order to restore it. Null if disabled.
+ */
+ public Path progressRestoreFile;
+
+ /**
+ * True if multiple threads shall be used, false for single-threaded analysis.
+ */
+ public boolean multipleThreads;
+
+ /**
+ * Number of threads to use.
+ */
+ public int threadCount;
+
+ /**
+ * Interval of entities after which progress shall be saved. Defaults to 100.
+ */
+ public int progressWriteInterval;
+
+ /**
+ * Creates a new configuration with default values.
+ */
+ LibraryAnalysisConfig() {
+ skip = DEFAULT_VALUE_SKIP;
+ take = DEFAULT_VALUE_TAKE;
+ inputListFile = DEFAULT_VALUE_INPUT_LIST;
+ progressOutputFile = DEFAULT_VALUE_PROGRESS_OUTPUT_FILE;
+ progressRestoreFile = DEFAULT_VALUE_PROGRESS_RESTORE_FILE;
+ multipleThreads = DEFAULT_NUM_THREADS == 1 ? false : true;
+ threadCount = DEFAULT_NUM_THREADS;
+ progressWriteInterval = DEFAULT_PROGRESS_WRITE_INTERVAL;
+ }
+
+ /**
+ * Checks whether there is a non-default amount of entities to skip
+ * @return True if there are entities to skip
+ */
+ public boolean hasSkip() {
+ return this.skip != DEFAULT_VALUE_SKIP;
+ }
+
+ /**
+ * Checks whether there is a non-default amount of entities to take
+ * @return True if there are limited entities to take
+ */
+ public boolean hasTake() {
+ return this.take != DEFAULT_VALUE_TAKE;
+ }
+
+ /**
+ * Checks whether there is a custom input list of entities to process.
+ * @return True if there is an input list specified
+ */
+ public boolean hasInputList() {
+ return this.inputListFile != DEFAULT_VALUE_INPUT_LIST;
+ }
+}
diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java
new file mode 100644
index 0000000..7d43411
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java
@@ -0,0 +1,178 @@
+package org.tudo.sse.analyses.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Configuration builder to obtain {@link LibraryAnalysisConfig} instances programmatically.
+ *
+ * @author Johannes Düsing
+ */
+public class LibraryAnalysisConfigBuilder {
+
+ /**
+ * The logger for this instance.
+ */
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ private final LibraryAnalysisConfig theConfig;
+
+ /**
+ * Creates a new builder with default configuration values.
+ */
+ public LibraryAnalysisConfigBuilder() {
+ this(new LibraryAnalysisConfig());
+ }
+
+ /**
+ * Creates a new builder with the given baseline configuration instance.
+ *
+ * @param theConfig Baseline configuration instance for this builder
+ */
+ protected LibraryAnalysisConfigBuilder(LibraryAnalysisConfig theConfig) {
+ this.theConfig = theConfig;
+ }
+
+ /**
+ * Sets the amount of entities to skip when processing analysis inputs. Can be used for pagination. Defaults to zero.
+ *
+ * @param toSkip Amount of entities (artifacts or libraries) to skip
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid.
+ */
+ public LibraryAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException {
+ if(toSkip < 0)
+ throw new InvalidConfigurationException("skip", "Skip must not be negative");
+
+ if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP)
+ log.warn("Value 'skip' is set multiple times, previous value overridden.");
+
+ this.theConfig.skip = toSkip == 0 ? LibraryAnalysisConfig.DEFAULT_VALUE_SKIP : toSkip;
+ return this;
+ }
+
+ /**
+ * Sets the amount of entities to take when processing analysis inputs. Can be used for pagination. By default, no
+ * limit is imposed and the input source is consumed fully.
+ *
+ * @param toTake Amount of entities (artifacts or libraries) to take
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid.
+ */
+ public LibraryAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException {
+ if(toTake <= 0)
+ throw new InvalidConfigurationException("take", "Take must be positive");
+
+ if(this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE)
+ log.warn("Value 'take' is set multiple times, previous value overridden.");
+
+ this.theConfig.take = toTake;
+ return this;
+ }
+
+ /**
+ * Sets the input list of entities to process as analysis inputs. This is either a list of GAV triples (artifacts)
+ * or GA tuples (libraries), one entity per line. By default, the Maven Central index is used as the source of inputs.
+ *
+ * @param listPath Path to input file. Must be an existing regular file, one entity per line is required.
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid
+ */
+ public LibraryAnalysisConfigBuilder withInputList(Path listPath) throws InvalidConfigurationException {
+ if(listPath == null || !Files.isRegularFile(listPath))
+ throw new InvalidConfigurationException("inputs", "Input list file must be a valid file reference");
+
+ if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST)
+ log.warn("Value 'inputs' is set multiple times, previous value overridden.");
+
+ this.theConfig.inputListFile = listPath;
+ return this;
+ }
+
+ /**
+ * Sets the output file to regularly write the analysis progress to. By default, a file called 'marin-progress.txt'
+ * is created in the current workdir.
+ *
+ * @param outputFile Path to the progress output file to create. Existing files will be overridden!
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid
+ */
+ public LibraryAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException {
+ if(outputFile == null)
+ throw new InvalidConfigurationException("progress-output-file", "Output file cannot be null");
+
+ if(this.theConfig.progressOutputFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_OUTPUT_FILE)
+ log.warn("Value 'progress-output-file' is set multiple times, previous value overridden.");
+
+ this.theConfig.progressOutputFile = outputFile;
+ return this;
+ }
+
+ /**
+ * Sets the file to restore progress from. Should be a progress output file previously created by MARIN. By default,
+ * no progress is restored.
+ *
+ * @param restoreFile Path to the progress restore file. Must be a valid text file.
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid
+ */
+ public LibraryAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException {
+ if(restoreFile == null || !Files.isRegularFile(restoreFile))
+ throw new InvalidConfigurationException("progress-restore-file", "Restore file must be a valid file reference");
+
+ if(this.theConfig.progressRestoreFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_RESTORE_FILE)
+ log.warn("Value 'progress-restore-file' is set multiple times, previous value overridden.");
+
+ this.theConfig.progressRestoreFile = restoreFile;
+ return this;
+ }
+
+ /**
+ * Sets the number of threads to use for analysis. By default, one thread is used.
+ * @param numThreads Number of threads to use
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid
+ */
+ public LibraryAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException {
+ if(numThreads <= 0)
+ throw new InvalidConfigurationException("threads", "Number of threads must be positive");
+
+ if(this.theConfig.threadCount != LibraryAnalysisConfig.DEFAULT_NUM_THREADS)
+ log.warn("Value 'threads' is set multiple times, previous value overridden.");
+
+ this.theConfig.threadCount = numThreads;
+ this.theConfig.multipleThreads = numThreads > 1;
+ return this;
+ }
+
+ /**
+ * Sets the amount of entities (artifacts or libraries) after which to write progress to the progress output file.
+ * Default is 100.
+ * @param progressWriteInterval Amount of entities after which to save progress
+ * @return The current configuration builder instance
+ * @throws InvalidConfigurationException If the given configuration value is not valid
+ */
+ public LibraryAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException {
+ if(progressWriteInterval <= 0)
+ throw new InvalidConfigurationException("save-progress-interval", "Progress write interval must be positive");
+
+ if(this.theConfig.progressWriteInterval != LibraryAnalysisConfig.DEFAULT_PROGRESS_WRITE_INTERVAL)
+ log.warn("Value 'progress-write-interval' is set multiple times, previous value overridden.");
+
+ this.theConfig.progressWriteInterval = progressWriteInterval;
+ return this;
+ }
+
+ /**
+ * Builds the configuration as specified by all previous invocations to this instance.
+ *
+ * @return The configuration object that has been constructed by this builder.
+ */
+ public LibraryAnalysisConfig build(){
+ return this.theConfig;
+ }
+
+}
diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java
new file mode 100644
index 0000000..6b46233
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java
@@ -0,0 +1,62 @@
+package org.tudo.sse.analyses.config.parsing;
+
+import org.tudo.sse.CLIException;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfig;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder;
+import org.tudo.sse.analyses.config.InvalidConfigurationException;
+
+/**
+ * Parser for creating {@link ArtifactAnalysisConfig} objects from CLI parameters.
+ * Configuration is obtained by parsing command line arguments provided as an array of strings.
+ */
+public class ArtifactAnalysisConfigParser extends LibraryAnalysisConfigParser implements CLIParsingUtilities {
+
+ /**
+ * Creates a new artifact config parser instance.
+ */
+ public ArtifactAnalysisConfigParser() { super(); }
+
+ /**
+ * Parses a given array containing JVM command line arguments into an {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfig} object. Throws an exception
+ * if the given arguments are not valid.
+ * @param args Array of command line arguments, as passed by the JVM
+ * @return Corresponding {@link ArtifactAnalysisConfig} if parsing was successful
+ * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid.
+ */
+ public ArtifactAnalysisConfig parseArtifactConfig(String[] args) throws CLIException {
+ final ArtifactAnalysisConfigBuilder configBuilder = new ArtifactAnalysisConfigBuilder();
+
+ for(int i = 0; i < args.length; i += 2) {
+ handleArtifactParameter(args, i, configBuilder);
+ }
+
+ return configBuilder.build();
+ }
+
+
+ private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfigBuilder configBuilder) throws CLIException {
+ try {
+
+ switch(args[i]) {
+ case "-su":
+ case "--since-until":
+ final long[] sinceUntil = nextArgAsLongPair(args, i);
+ configBuilder.withSinceUtil(sinceUntil[0], sinceUntil[1]);
+ break;
+ case "-o":
+ case "--output":
+ configBuilder.withOutputDirectory(nextArgAsDirectoryReference(args, i));
+ break;
+ default:
+ super.handleParameter(args, i , configBuilder);
+
+ }
+
+ } catch (InvalidConfigurationException icx) {
+ CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName());
+ wrapped.initCause(icx);
+ throw wrapped;
+ }
+ }
+
+}
diff --git a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java
similarity index 98%
rename from src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java
rename to src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java
index 8d67fbc..2c175d1 100644
--- a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java
+++ b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java
@@ -1,4 +1,4 @@
-package org.tudo.sse.utils;
+package org.tudo.sse.analyses.config.parsing;
import org.tudo.sse.CLIException;
diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java
new file mode 100644
index 0000000..ba2b95e
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java
@@ -0,0 +1,86 @@
+package org.tudo.sse.analyses.config.parsing;
+
+import org.tudo.sse.CLIException;
+import org.tudo.sse.analyses.config.InvalidConfigurationException;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfig;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder;
+
+/**
+ * Parser for {@link LibraryAnalysisConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}.
+ * Configuration is obtained by parsing command line arguments provided as an array of strings.
+ */
+public class LibraryAnalysisConfigParser implements CLIParsingUtilities {
+
+ /**
+ * Creates a new library config parsers instance.
+ */
+ public LibraryAnalysisConfigParser() { super(); }
+
+ /**
+ * Parses a given array containing JVM command line arguments into an {@link LibraryAnalysisConfig} objects. Throws an exception
+ * if the given arguments are not valid.
+ * @param args Array of command line arguments, as passed by the JVM
+ * @return Corresponding {@link LibraryAnalysisConfig} if parsing was successful
+ * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid.
+ */
+ public LibraryAnalysisConfig parseCommonConfig(String[] args) throws CLIException {
+ final LibraryAnalysisConfigBuilder configBuilder = new LibraryAnalysisConfigBuilder();
+
+
+ for(int i = 0; i < args.length; i += 2){
+ handleParameter(args, i, configBuilder);
+ }
+
+ return configBuilder.build();
+ }
+
+ /**
+ * Parses a given parameter from the array of CLI arguments and applies it to the config object.
+ * @param args CLI arguments array
+ * @param i Current parsing position
+ * @param configBuilder Current configuration object
+ * @throws CLIException If parsing the current argument fails.
+ */
+ protected void handleParameter(String[] args, int i, LibraryAnalysisConfigBuilder configBuilder) throws CLIException {
+ try {
+ switch (args[i]){
+ case "-st":
+ case "--skip-take":
+ final long[] skipTake = nextArgAsLongPair(args, i);
+ configBuilder
+ .withSkip(skipTake[0])
+ .withTake(skipTake[1]);
+ break;
+ case "-prf":
+ case "--progress-restore-file":
+ configBuilder.withProgressRestoreFile(nextArgAsRegularFileReference(args, i));
+ break;
+ case "-spi":
+ case "--save-progress-interval":
+ configBuilder.withProgressWriteInterval(nextArgAsInt(args, i));
+ break;
+ case "-pof":
+ case "--progress-output-file":
+ configBuilder.withProgressOutputFile(nextArgAsPath(args, i));
+ break;
+ case "-i":
+ case "--inputs":
+ configBuilder.withInputList(nextArgAsRegularFileReference(args, i));
+ break;
+ case "-t":
+ case "--threads":
+ final int threads = nextArgAsInt(args, i);
+ configBuilder.withNumberOfThreads(threads);
+ break;
+ default:
+ throw new CLIException(args[i]);
+ }
+ } catch (InvalidConfigurationException icx) {
+ CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName());
+ wrapped.initCause(icx);
+ throw wrapped;
+ }
+ }
+
+
+}
diff --git a/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java
new file mode 100644
index 0000000..d3df78b
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java
@@ -0,0 +1,91 @@
+package org.tudo.sse.analyses.input;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+
+/**
+ * An iterator that wraps an underlying file that contains line-wise representations of entities. This wrapper produces
+ * one entity per line, with parsing logic defined by concrete subclasses.
+ *
+ * @param The type that entities are parsed to
+ *
+ * @author Johannes Düsing
+ */
+abstract class AbstractInputFileIterator implements Iterator {
+
+ /**
+ * Checks whether a given line within the input file represents a valid entity
+ * @param line The line to check
+ * @return True is the line represents a valid entity, false otherwise
+ */
+ protected abstract boolean isValidLine(String line);
+
+ /**
+ * Parses a given line into its representing entity. Will only ever be called after isValidLine returned true.
+ * @param line The line to parse
+ * @return The resulting entity
+ */
+ protected abstract T parseLine(String line);
+
+ private final Path inputPath;
+
+ private Iterator lineIterator;
+
+ /**
+ * Creates a new iterator instance for the given file path.
+ * @param input Path to a text file containing one entity per line
+ */
+ protected AbstractInputFileIterator(Path input){
+ this.inputPath = input;
+ }
+
+ /**
+ * Checks whether the underlying file exists and contains only valid entities.
+ * @throws IOException If accessing the underlying file fails
+ * @throws IllegalArgumentException If the underlying file contents are not valid
+ */
+ public void validateInput() throws IOException, IllegalArgumentException {
+ var lines = Files.readAllLines(inputPath);
+ int lineCnt = 0;
+
+ for(String line: lines){
+ if(!isValidLine(line)){
+ throw new IllegalArgumentException("Not a valid entity: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")");
+ }
+
+ lineCnt += 1;
+ }
+
+ this.lineIterator = lines.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(this.lineIterator == null){
+ try {
+ var lines = Files.readAllLines(this.inputPath);
+ this.lineIterator = lines.iterator();
+ } catch (IOException iox){
+ throw new IllegalArgumentException("Failed to access input file", iox);
+ }
+ }
+
+ return this.lineIterator.hasNext();
+ }
+
+ @Override
+ public T next() {
+ if(!hasNext())
+ throw new IllegalStateException("Next on empty iterator");
+
+ String line = this.lineIterator.next();
+
+ if(isValidLine(line))
+ return parseLine(line);
+ else
+ throw new IllegalStateException("Not a valid entity: " + line);
+ }
+
+}
diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java
new file mode 100644
index 0000000..4d646a8
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java
@@ -0,0 +1,42 @@
+package org.tudo.sse.analyses.input;
+
+import org.tudo.sse.model.ArtifactIdent;
+
+import java.nio.file.Path;
+
+/**
+ * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of
+ * groupID:artifactID:version per line.
+ */
+public class FileBasedArtifactIterator extends AbstractInputFileIterator {
+
+ /**
+ * Creates a new artifact identifier iterator for the given input file.
+ * @param gavList Path to input file
+ */
+ public FileBasedArtifactIterator(Path gavList) {
+ super(gavList);
+ }
+
+ @Override
+ protected boolean isValidLine(String line) {
+ var parts = line.split(":");
+
+ if(parts.length != 3)
+ return false;
+
+ for(String part: parts){
+ if(part.isBlank())
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ protected ArtifactIdent parseLine(String line) {
+ var parts = line.split(":");
+
+ return new ArtifactIdent(parts[0], parts[1], parts[2]);
+ }
+}
diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java
new file mode 100644
index 0000000..a8138e6
--- /dev/null
+++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java
@@ -0,0 +1,38 @@
+package org.tudo.sse.analyses.input;
+
+import java.nio.file.Path;
+
+/**
+ * Iterator that reads Maven Central library identifiers from a text file. Expects one colon-separated tuple of
+ * groupID:artifactID per line.
+ */
+public class FileBasedLibraryIterator extends AbstractInputFileIterator{
+
+ /**
+ * Creates a new library identifier iterator for the given input file.
+ * @param gaList Path to input list
+ */
+ public FileBasedLibraryIterator(Path gaList) {
+ super(gaList);
+ }
+
+ @Override
+ protected boolean isValidLine(String line) {
+ var parts = line.split(":");
+
+ if(parts.length != 2)
+ return false;
+
+ for(String part: parts){
+ if(part.isBlank())
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ protected String parseLine(String line) {
+ return line;
+ }
+}
diff --git a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java
index be261f1..eb6ec37 100644
--- a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java
+++ b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java
@@ -26,8 +26,16 @@ public static ArtifactResolutionContext newInstance(ArtifactIdent identifier) {
* Returns the identifier of the artifact for which this context was created.
* @return ArtifactIdent for this context
*/
- public ArtifactIdent getIdentifier() {
+ public ArtifactIdent getRootArtifactIdentifier() {
return identifier;
}
+ /**
+ * Returns the artifact for which this context was initially created
+ * @return The root artifact of this context
+ */
+ public Artifact getRootArtifact() {
+ return this.createArtifact(getRootArtifactIdentifier());
+ }
+
}
diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java
index 5e68bdf..2ebe7d6 100644
--- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java
+++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java
@@ -28,7 +28,7 @@ public ProcessIdentifierMessage(ArtifactResolutionContext resolutionContext, Mav
* @return The artifact identifier
*/
public ArtifactIdent getIdentifier() {
- return this.artifactCtx.getIdentifier();
+ return this.artifactCtx.getRootArtifactIdentifier();
}
/**
diff --git a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java
deleted file mode 100644
index 7fde305..0000000
--- a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package org.tudo.sse.utils;
-
-import org.tudo.sse.CLIException;
-
-import java.nio.file.Path;
-
-/**
- * Parser for ArtifactConfig objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis}.
- * Configuration is obtained by parsing command line arguments provided as an array of strings.
- */
-public class ArtifactConfigParser extends CommonConfigParser implements CLIParsingUtilities {
-
- /**
- * Creates a new artifact config parser instance.
- */
- public ArtifactConfigParser() { super(); }
-
- /**
- * Parses a given array containing JVM command line arguments into an {@link ArtifactConfig} object. Throws an exception
- * if the given arguments are not valid.
- * @param args Array of command line arguments, as passed by the JVM
- * @return Corresponding {@link ArtifactConfig} if parsing was successful
- * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid.
- */
- public ArtifactConfig parseArtifactConfig(String[] args) throws CLIException {
- final ArtifactConfig artifactConfig = new ArtifactConfig();
-
- for(int i = 0; i < args.length; i += 2) {
- handleArtifactParameter(args, i, artifactConfig);
- }
-
- return artifactConfig;
- }
-
-
- private void handleArtifactParameter(String[] args, int i, ArtifactConfig config) throws CLIException {
- switch(args[i]) {
- case "-su":
- case "--since-until":
- if(config.skip != -1L)
- throw new CLIException("Cannot apply time-based filtering when pagination (-st) is used", args[i]);
- if(config.since != -1L)
- throw new CLIException("Values for since and until cannot be set twice!", args[i]);
- if(config.inputListFile != null)
- throw new CLIException("Cannot apply time-based filtering when input list (-i) is used", args[i]);
-
- final long[] sinceUntil = nextArgAsLongPair(args, i);
- config.since = sinceUntil[0];
- config.until = sinceUntil[1];
- break;
-
- case "-o":
- case "--output":
- config.outputEnabled = true;
- config.outputDirectory = nextArgAsDirectoryReference(args, i);
- break;
-
- case "-st":
- case "--skip-take":
- if(config.since != -1L)
- throw new CLIException("Cannot apply pagination when time-based filtering (-su) is used", args[i]);
- super.handleParameter(args, i, config);
- break;
-
- case "-i":
- case "--inputs":
- if(config.since != -1L)
- throw new CLIException("Cannot use input list when time-based filtering (-su) is applied", args[i]);
- super.handleParameter(args, i, config);
- break;
-
- default:
- super.handleParameter(args, i , config);
-
- }
- }
-
- /**
- * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance.
- */
- public static class ArtifactConfig extends CommonConfig {
-
- /**
- * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled.
- */
- public long since;
-
- /**
- * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled.
- */
- public long until;
-
- /**
- * Directory to write file outputs to, or null if disabled.
- */
- public Path outputDirectory;
-
- /**
- * Whether files shall be written to the output directory.
- */
- public boolean outputEnabled;
-
- /**
- * Creates a new configuration object with default values.
- */
- public ArtifactConfig(){
- super();
-
- since = -1L;
- until = -1L;
- outputEnabled = false;
- outputDirectory = null;
- }
-
-
- }
-
-}
diff --git a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java
deleted file mode 100644
index 474449e..0000000
--- a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java
+++ /dev/null
@@ -1,146 +0,0 @@
-package org.tudo.sse.utils;
-
-import org.tudo.sse.CLIException;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Parser for {@link CommonConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}.
- * Configuration is obtained by parsing command line arguments provided as an array of strings.
- */
-public class CommonConfigParser implements CLIParsingUtilities {
-
- /**
- * Creates a new artifact config parsers instance.
- */
- public CommonConfigParser() { super(); }
-
- /**
- * Parses a given array containing JVM command line arguments into an {@link CommonConfig} objects. Throws an exception
- * if the given arguments are not valid.
- * @param args Array of command line arguments, as passed by the JVM
- * @return Corresponding {@link CommonConfig} if parsing was successful
- * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid.
- */
- public CommonConfig parseCommonConfig(String[] args) throws CLIException {
- final CommonConfig config = new CommonConfig();
-
-
- for(int i = 0; i < args.length; i += 2){
- handleParameter(args, i, config);
- }
-
- return config;
- }
-
- /**
- * Parses a given parameter from the array of CLI arguments and applies it to the config object.
- * @param args CLI arguments array
- * @param i Current parsing position
- * @param config Current configuration object
- * @throws CLIException If parsing the current argument fails.
- */
- protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException {
- switch (args[i]){
- case "-st":
- case "--skip-take":
- if(config.skip != -1L)
- throw new CLIException("Values for skip and take cannot be set twice!", args[i]);
-
- final long[] skipTake = nextArgAsLongPair(args, i);
- config.skip = skipTake[0];
- config.take = skipTake[1];
- break;
- case "-prf":
- case "--progress-restore-file":
- if(config.progressRestoreFile != null)
- throw new CLIException("Progress restore file cannot be set twice!", args[i]);
-
- config.progressRestoreFile = nextArgAsRegularFileReference(args, i);
- break;
- case "-spi":
- case "--save-progress-interval":
- config.progressWriteInterval = nextArgAsInt(args, i);
- break;
- case "-pof":
- case "--progress-output-file":
- config.progressOutputFile = nextArgAsPath(args, i);
- break;
- case "-i":
- case "--inputs":
- if(config.inputListFile != null)
- throw new CLIException("Input file cannot be set twice!", args[i]);
- config.inputListFile = nextArgAsRegularFileReference(args, i);
- break;
- case "-t":
- case "--threads":
- final int threads = nextArgAsInt(args, i);
- config.multipleThreads = threads > 1;
- config.threadCount = threads;
- break;
- default:
- throw new CLIException(args[i]);
- }
- }
-
- /**
- * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis} instance.
- */
- public static class CommonConfig {
-
- /**
- * Number of artifacts to skip from the underlying source, or -1 if disabled.
- */
- public long skip;
-
- /**
- * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied.
- */
- public long take;
-
- /**
- * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used.
- */
- public Path inputListFile;
-
- /**
- * File path to output analysis progress to.
- */
- public Path progressOutputFile;
-
- /**
- * Path to read previous progress from, in order to restore it. Null if disabled.
- */
- public Path progressRestoreFile;
-
- /**
- * True if multiple threads shall be used, false for single-threaded analysis.
- */
- public boolean multipleThreads;
-
- /**
- * Number of threads to use.
- */
- public int threadCount;
-
- /**
- * Interval of entities after which progress shall be saved. Defaults to 100.
- */
- public int progressWriteInterval;
-
- /**
- * Creates a new configuration with default values.
- */
- public CommonConfig() {
- skip = -1L;
- take = -1L;
- inputListFile = null;
- progressOutputFile = Paths.get("marin-progress");
- progressRestoreFile = null;
- multipleThreads = false;
- threadCount = 1;
- progressWriteInterval = 100;
- }
- }
-}
diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java
deleted file mode 100644
index c769515..0000000
--- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.tudo.sse.utils;
-
-import org.tudo.sse.model.ArtifactIdent;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Iterator;
-
-/**
- * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of
- * groupID:artifactID:version per line.
- */
-public class FileBasedArtifactIterator implements Iterator {
-
- private final Path inputPath;
-
- private Iterator lineIterator;
-
- /**
- * Creates a new iterator instance for the given file path.
- * @param input Path to a text file containing GAV triples
- */
- public FileBasedArtifactIterator(Path input){
- this.inputPath = input;
- }
-
- @Override
- public boolean hasNext() {
- if(this.lineIterator == null){
- try {
- var lines = Files.readAllLines(this.inputPath);
- this.lineIterator = lines.iterator();
- } catch (IOException iox){
- throw new IllegalArgumentException("Failed to access input file", iox);
- }
- }
-
- return this.lineIterator.hasNext();
- }
-
- @Override
- public ArtifactIdent next() {
- String line = this.lineIterator.next();
-
- String[] parts = line.split(":");
- if(parts.length == 3) {
- return new ArtifactIdent(parts[0], parts[1], parts[2]);
- } else {
- throw new IllegalStateException("Not a valid GAV-Triple: " + line);
- }
- }
-}
diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java
index 70baa56..b5fb274 100644
--- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java
+++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java
@@ -5,12 +5,13 @@
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.tudo.sse.CLIException;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfig;
import org.tudo.sse.model.Artifact;
import org.tudo.sse.model.ArtifactIdent;
import org.tudo.sse.model.index.Package;
import org.tudo.sse.model.pom.License;
import org.tudo.sse.model.pom.PomInformation;
-import org.tudo.sse.utils.ArtifactConfigParser;
+import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser;
import org.tudo.sse.utils.IndexIterator;
import org.tudo.sse.utils.MavenCentralAnalysisFactory;
import scala.Tuple2;
@@ -43,11 +44,11 @@ class MavenCentralArtifactAnalysisTest {
void parseCLIRegular() throws IOException{
Path tmpDir = Files.createTempDirectory("maven-resolution-files");
- ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{
+ ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{
parseCLI("--skip-take 500:223"),
parseCLI("--inputs src/test/resources/localPom.xml"),
parseCLI("--progress-restore-file src/test/resources/localPom.xml --inputs src/test/resources/localPom.xml"),
- parseCLI("--since-until 53245:13243"),
+ parseCLI("--since-until 53245:53246"),
parseCLI("--inputs src/test/resources/localPom.xml --progress-restore-file src/test/resources/localPom.xml"),
parseCLI("--progress-restore-file src/test/resources/localPom.xml"),
parseCLI("--output " + tmpDir.toString())
@@ -57,7 +58,7 @@ void parseCLIRegular() throws IOException{
for(int i = 0; i < configs.length; i++){
final List currExpected = expected.get(i);
- final ArtifactConfigParser.ArtifactConfig currConfig = configs[i];
+ final ArtifactAnalysisConfig currConfig = configs[i];
assertEquals(asInt(currExpected.get(0)), currConfig.skip);
assertEquals(asInt(currExpected.get(1)), currConfig.take);
@@ -89,11 +90,11 @@ void parseCLIRegular() throws IOException{
void parseCLIShorthands() throws IOException{
Path tmpDir = Files.createTempDirectory("maven-resolution-files");
- ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{
+ ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{
parseCLI("-st 500:223"),
parseCLI("-i src/test/resources/localPom.xml"),
parseCLI("-prf src/test/resources/localPom.xml -i src/test/resources/localPom.xml"),
- parseCLI("-su 53245:13243"),
+ parseCLI("-su 53245:53246"),
parseCLI("-i src/test/resources/localPom.xml -prf src/test/resources/localPom.xml"),
parseCLI("-prf src/test/resources/localPom.xml"),
parseCLI("-o " + tmpDir.toString())
@@ -103,7 +104,7 @@ void parseCLIShorthands() throws IOException{
for(int i = 0; i < configs.length; i++){
final List currExpected = expected.get(i);
- final ArtifactConfigParser.ArtifactConfig currConfig = configs[i];
+ final ArtifactAnalysisConfig currConfig = configs[i];
assertEquals(asInt(currExpected.get(0)), currConfig.skip);
assertEquals(asInt(currExpected.get(1)), currConfig.take);
@@ -138,6 +139,7 @@ void parseCmdLineNegative() {
cliInputs.add("--inputs -/xcd/");
cliInputs.add("-st 88880000 -su --inputs path/to/file -ip path/to/file");
cliInputs.add("--inputs");
+ cliInputs.add("--since-until 53245:13243");
for(String input : cliInputs) {
@@ -236,17 +238,17 @@ void indexProcessorProgress() {
@DisplayName("An analysis must correctly apply CLI options when reading custom input lists")
void readIdentsIn() {
List cliInputs = new ArrayList<>();
- String[] args = {"--inputs", "src/main/resources/coordinates.txt"};
+ String[] args = {"--inputs", "src/test/resources/artifact-names-valid.txt"};
cliInputs.add(args);
- args = new String[] {"--inputs", "src/main/resources/coordinates.txt", "-pof", "src/test/resources/stop.txt"};
+ args = new String[] {"--inputs", "src/test/resources/artifact-names-valid.txt", "-pof", "src/test/resources/stop.txt"};
cliInputs.add(args);
- args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5"};
+ args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5"};
cliInputs.add(args);
- args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"};
+ args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"};
cliInputs.add(args);
- args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt"};
+ args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt"};
cliInputs.add(args);
- args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"};
+ args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"};
cliInputs.add(args);
List> expected = (List>) json.get("readIdentsIn");
@@ -290,8 +292,8 @@ void checkMultiThreading() {
singleArgs.add(new String[]{"-st", "10:1000"});
multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"});
- singleArgs.add(new String[]{"--inputs", "src/main/resources/coordinates.txt"});
- multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/main/resources/coordinates.txt"});
+ singleArgs.add(new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt"});
+ multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/test/resources/artifact-names-valid.txt"});
for(int i = 0; i < singleArgs.size(); i++) {
@@ -383,9 +385,9 @@ private long getEndingIndex(Path fileName) {
}
}
- private ArtifactConfigParser.ArtifactConfig parseCLI(String cli) {
+ private ArtifactAnalysisConfig parseCLI(String cli) {
try {
- final ArtifactConfigParser parser = new ArtifactConfigParser();
+ final ArtifactAnalysisConfigParser parser = new ArtifactAnalysisConfigParser();
if(cli.isBlank()) return parser.parseArtifactConfig(new String[] {});
else return parser.parseArtifactConfig(cli.split(" "));
} catch(CLIException clix){
diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java
new file mode 100644
index 0000000..438bef6
--- /dev/null
+++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java
@@ -0,0 +1,203 @@
+package org.tudo.sse.analyses;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.opalj.log.GlobalLogContext$;
+import org.opalj.log.OPALLogger;
+import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder;
+import org.tudo.sse.analyses.config.InvalidConfigurationException;
+import org.tudo.sse.model.Artifact;
+import org.tudo.sse.model.ArtifactIdent;
+import org.tudo.sse.utils.IndexIterator;
+import org.tudo.sse.utils.MarinOpalLogger;
+import org.tudo.sse.utils.MavenCentralRepository;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.tudo.sse.utils.TestUtilities.testResource;
+
+public class MavenCentralArtifactIteratorTest {
+
+ private final Path gavInputList = testResource("artifact-names-valid.txt");
+
+ @BeforeAll
+ static void setUp() {
+ // Use global OPAL Logger - will only forward OPAL messages with level error or fatal
+ OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger());
+ }
+
+ @Test
+ @DisplayName("The artifact iterator must process all inputs from custom GAV lists")
+ void readFromFile() throws InvalidConfigurationException {
+ var config = new ArtifactAnalysisConfigBuilder()
+ .withInputList(gavInputList)
+ .build();
+
+ var iterator = new MavenCentralArtifactIterator(true, false, true, config);
+
+ int artifactCount = 0;
+
+ while(iterator.hasNext()){
+ Artifact current = iterator.next();
+ assertFalse(current.hasIndexInformation());
+ assertTrue(current.hasPomInformation());
+ assertTrue(current.hasJarInformation());
+
+ artifactCount++;
+ }
+
+ assertEquals(10, artifactCount);
+ }
+
+ @Test
+ @DisplayName("The artifact iterator must apply pagination to custom GAV lists")
+ void readFromFilePaginated() throws InvalidConfigurationException {
+ var config = new ArtifactAnalysisConfigBuilder()
+ .withInputList(gavInputList)
+ .withSkip(2)
+ .withTake(5)
+ .build();
+
+ var iterator = new MavenCentralArtifactIterator(true, true, false, config);
+
+ ArtifactIdent firstEntry = getIdentInInputFile(0);
+ ArtifactIdent secondEntry = getIdentInInputFile(1);
+
+ int artifactCount = 0;
+
+ while(iterator.hasNext()){
+ Artifact current = iterator.next();
+ assertFalse(current.hasIndexInformation());
+ assertTrue(current.hasPomInformation());
+ assertFalse(current.hasJarInformation());
+
+ // Assert that we are not seeing the skipped entries
+ assertNotEquals(firstEntry, current.getIdent());
+ assertNotEquals(secondEntry, current.getIdent());
+
+ artifactCount++;
+ }
+
+ assertEquals(5, artifactCount);
+ }
+
+ @Test
+ @DisplayName("The artifact iterator must not produce any values on invalid input files")
+ void readFromFileInvalid() throws InvalidConfigurationException {
+ var config = new ArtifactAnalysisConfigBuilder()
+ .withInputList(testResource("artifact-names-invalid.txt"))
+ .build();
+
+ var iterator = new MavenCentralArtifactIterator(true, true, true, config);
+
+ // If the source file contains at least one line that is not a valid GAV triple, it must not produce any elements
+ assertFalse(iterator.hasNext());
+ assertThrows(IllegalStateException.class, iterator::next);
+ }
+
+ @Test
+ @DisplayName("The artifact iterator must apply pagination to the Central index")
+ void readFromIndexPaginated() throws InvalidConfigurationException {
+ var config = new ArtifactAnalysisConfigBuilder()
+ .withSkip(2)
+ .withTake(5)
+ .build();
+
+ var iterator = new MavenCentralArtifactIterator(false, false, true, config);
+
+ ArtifactIdent firstEntry = getIdentInIndex(0);
+ ArtifactIdent secondEntry = getIdentInIndex(1);
+
+ int artifactCount = 0;
+
+ while(iterator.hasNext()){
+ Artifact current = iterator.next();
+ assertTrue(current.hasIndexInformation());
+ assertFalse(current.hasPomInformation());
+ assertTrue(current.hasJarInformation());
+
+ // Assert that we are not seeing the skipped entries
+ assertNotEquals(firstEntry, current.getIdent());
+ assertNotEquals(secondEntry, current.getIdent());
+
+ artifactCount++;
+ }
+
+ assertEquals(5, artifactCount);
+ }
+
+ @Test
+ @DisplayName("The artifact iterator must apply time range filtering to the Central index")
+ void readFromIndexFiltered() throws InvalidConfigurationException {
+ // We expect two entries in this range
+ long since = 1106902436000L;
+ long until = 1106902438001L;
+
+ var config = new ArtifactAnalysisConfigBuilder()
+ .withSinceUtil(since, until)
+ .build();
+
+ var iterator = new MavenCentralArtifactIterator(false, false, false, config);
+
+ int artifactCount = 0;
+
+ while(iterator.hasNext() && artifactCount < 2){
+ Artifact current = iterator.next();
+ assertTrue(current.hasIndexInformation());
+ assertFalse(current.hasPomInformation());
+ assertFalse(current.hasJarInformation());
+
+ assertTrue(since <= current.getIndexInformation().getLastModified());
+ assertTrue(current.getIndexInformation().getLastModified() <= until);
+
+ assertEquals("ymsg", current.getIdent().getGroupID());
+
+ artifactCount++;
+ }
+ }
+
+ private ArtifactIdent getIdentInInputFile(int pos) {
+ try {
+ var lines = Files.readAllLines(gavInputList);
+
+ if(pos >= lines.size()) fail("Invalid position in input list: " + pos);
+
+ var parts = lines.get(pos).split(":");
+
+ if(parts.length != 3) fail("Invalid GAV in input list: " + pos);
+
+ return new ArtifactIdent(parts[0], parts[1], parts[2]);
+ } catch (IOException iox) {
+ fail(iox);
+ }
+
+ return null;
+ }
+
+ private ArtifactIdent getIdentInIndex(int pos){
+ try {
+ IndexIterator ii = new IndexIterator(MavenCentralRepository.RepoBaseURI);
+
+ int position = 0;
+
+ while(ii.hasNext()){
+ var ident = ii.next().getIdent();
+ if(position == pos){
+ return ident;
+ }
+ position++;
+ }
+
+ ii.closeReader();
+
+ } catch (IOException iox) {
+ fail(iox);
+ }
+
+ return null;
+ }
+}
diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
index e5c120b..8ca7d1f 100644
--- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
+++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java
@@ -3,8 +3,9 @@
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.tudo.sse.CLIException;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfig;
import org.tudo.sse.model.Artifact;
-import org.tudo.sse.utils.CommonConfigParser;
+import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser;
import java.io.IOException;
import java.nio.file.Files;
@@ -15,6 +16,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.*;
+import static org.tudo.sse.utils.TestUtilities.testResource;
public class MavenCentralLibraryAnalysisTest {
@@ -22,7 +24,7 @@ public class MavenCentralLibraryAnalysisTest {
@DisplayName("The CLI parser must parse common argument values correctly")
void parseCLIRegular(){
final String validArgs = "--skip-take 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --threads 12 --save-progress-interval 20";
- final CommonConfigParser.CommonConfig config = parseCLI(validArgs);
+ final LibraryAnalysisConfig config = parseCLI(validArgs);
assert(config.multipleThreads);
assertEquals(12, config.threadCount);
@@ -38,7 +40,7 @@ void parseCLIRegular(){
@DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names")
void parseCLIShorthands(){
final String validArgs = "-st 20:10000 -prf pom.xml -pof marin-progress -t 12 -spi 20";
- final CommonConfigParser.CommonConfig config = parseCLI(validArgs);
+ final LibraryAnalysisConfig config = parseCLI(validArgs);
assert(config.multipleThreads);
assertEquals(12, config.threadCount);
@@ -55,7 +57,7 @@ void parseCLIShorthands(){
@Test
@DisplayName("The CLI parser must set the correct defaults for empty arguments")
void parseCLIDefaults(){
- final CommonConfigParser.CommonConfig config = parseCLI("");
+ final LibraryAnalysisConfig config = parseCLI("");
assertFalse(config.multipleThreads);
assertEquals(1, config.threadCount);
@@ -80,7 +82,7 @@ void parseNonExistingFile(){
void parseCustomGA(){
final String validArgs = "-st 20:10000 --inputs pom.xml";
- final CommonConfigParser.CommonConfig config = parseCLI(validArgs);
+ final LibraryAnalysisConfig config = parseCLI(validArgs);
assertEquals(Paths.get("pom.xml"), config.inputListFile);
assertEquals(20, config.skip);
@@ -91,7 +93,7 @@ void parseCustomGA(){
void parseNonExistingIndex(){
final String validArgs = "--progress-restore-file pom.xml --inputs pom.xml";
- final CommonConfigParser.CommonConfig config = parseCLI(validArgs);
+ final LibraryAnalysisConfig config = parseCLI(validArgs);
assertEquals(Paths.get("pom.xml"), config.inputListFile);
assertEquals(Paths.get("pom.xml"), config.progressRestoreFile);
@@ -327,19 +329,11 @@ protected void analyzeLibrary(String libraryGA, List releases) {
assertFalse(librariesHit.contains("yom:yom"));
}
- private Path testResource(String pathToResource){
- try {
- return Path.of(Objects.requireNonNull(getClass().getClassLoader().getResource(pathToResource)).toURI());
- } catch (Exception x){
- fail("Test setup: Failed to load resource file " + pathToResource, x);
- }
- return null;
- }
- private CommonConfigParser.CommonConfig parseCLI(String cli) {
+ private LibraryAnalysisConfig parseCLI(String cli) {
try {
- if(cli.isBlank()) return new CommonConfigParser().parseCommonConfig(new String[]{});
- else return new CommonConfigParser().parseCommonConfig(cli.split(" "));
+ if(cli.isBlank()) return new LibraryAnalysisConfigParser().parseCommonConfig(new String[]{});
+ else return new LibraryAnalysisConfigParser().parseCommonConfig(cli.split(" "));
} catch(CLIException clix) {
throw new RuntimeException(clix);
}
diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java
new file mode 100644
index 0000000..47daa0b
--- /dev/null
+++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java
@@ -0,0 +1,229 @@
+package org.tudo.sse.analyses;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.opalj.log.GlobalLogContext$;
+import org.opalj.log.OPALLogger;
+import org.tudo.sse.analyses.config.InvalidConfigurationException;
+import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder;
+import org.tudo.sse.model.Artifact;
+import org.tudo.sse.model.LibraryResolutionContext;
+import org.tudo.sse.resolution.FileNotFoundException;
+import org.tudo.sse.utils.LibraryIndexIterator;
+import org.tudo.sse.utils.MarinOpalLogger;
+import org.tudo.sse.utils.MavenCentralRepository;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.tudo.sse.utils.TestUtilities.testResource;
+
+public class MavenCentralLibraryIteratorTest {
+
+ private static final Path validGAList = testResource("library-names-valid.txt");
+ private static final Path invalidGAList = testResource("library-names-invalid.txt");
+ private static final Path nonExistingGAList = testResource("library-names-not-existing.txt");
+
+ @BeforeAll
+ static void setUp() {
+ // Use global OPAL Logger - will only forward OPAL messages with level error or fatal
+ OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger());
+ }
+
+ @Test
+ @DisplayName("The library iterator must process all inputs from custom GA lists")
+ void readFromFile() throws InvalidConfigurationException {
+ var config = new LibraryAnalysisConfigBuilder()
+ .withInputList(validGAList)
+ .build();
+
+ var iterator = new MavenCentralLibraryIterator(true, false, false, config);
+
+ int libraryCount = 0;
+
+ while(iterator.hasNext()){
+ final LibraryResolutionContext current = iterator.next();
+ final List releases = current.getLibraryArtifacts();
+
+ assertFalse(releases.isEmpty());
+
+
+ for(Artifact release: releases){
+ assertFalse(release.hasIndexInformation());
+ assertTrue(release.hasPomInformation());
+ assertFalse(release.hasJarInformation());
+ }
+
+
+ libraryCount++;
+ }
+
+ assertEquals(5, libraryCount);
+ }
+
+ @Test
+ @DisplayName("The library iterator must apply pagination to custom GA lists")
+ void readFromFilePaginated() throws InvalidConfigurationException {
+ var config = new LibraryAnalysisConfigBuilder()
+ .withInputList(validGAList)
+ .withTake(2)
+ .withSkip(2)
+ .build();
+
+ var iterator = new MavenCentralLibraryIterator(false, false, true, config);
+
+ int libraryCount = 0;
+
+ String firstEntry = getGaInInputFile(0);
+ String secondEntry = getGaInInputFile(1);
+
+ while(iterator.hasNext()){
+ final LibraryResolutionContext current = iterator.next();
+ final List releases = current.getLibraryArtifacts();
+
+ assertFalse(releases.isEmpty());
+
+ for(Artifact release: releases){
+ assertFalse(release.hasIndexInformation());
+ assertFalse(release.hasPomInformation());
+ assertTrue(release.hasJarInformation());
+ }
+
+ // Assert that we are not seeing the skipped entries
+ assertNotEquals(firstEntry, current.getLibraryGA());
+ assertNotEquals(secondEntry, current.getLibraryGA());
+
+ libraryCount++;
+ }
+
+ assertEquals(2, libraryCount);
+ }
+
+ @Test
+ @DisplayName("The library iterator must not produce any values on invalid input files")
+ void readFromFileInvalid() throws InvalidConfigurationException {
+ var config = new LibraryAnalysisConfigBuilder()
+ .withInputList(invalidGAList)
+ .build();
+
+ var iterator = new MavenCentralLibraryIterator(true, true, true, config);
+
+ // If the source file contains at least one line that is not a valid GA tuple, it must not produce any elements
+ assertFalse(iterator.hasNext());
+ assertThrows(IllegalStateException.class, iterator::next);
+ }
+
+ @Test
+ @DisplayName("The library iterator must apply pagination to the Central index")
+ void readFromIndexPaginated() throws InvalidConfigurationException {
+ var config = new LibraryAnalysisConfigBuilder()
+ .withSkip(2)
+ .withTake(3)
+ .build();
+
+ var iterator = new MavenCentralLibraryIterator(true, false, false, config);
+
+ String firstEntry = getGaInIndex(0);
+ String secondEntry = getGaInIndex(1);
+
+ int artifactCount = 0;
+
+ while(iterator.hasNext()){
+ final LibraryResolutionContext current = iterator.next();
+ final List releases = current.getLibraryArtifacts();
+
+ assertFalse(releases.isEmpty());
+
+ for(Artifact release: releases){
+ assertFalse(release.hasIndexInformation());
+ assertTrue(release.hasPomInformation());
+ assertFalse(release.hasJarInformation());
+ }
+
+ // Assert that we are not seeing the skipped entries
+ assertNotEquals(firstEntry, current.getLibraryGA());
+ assertNotEquals(secondEntry, current.getLibraryGA());
+
+ artifactCount++;
+ }
+
+ assertEquals(3, artifactCount);
+ }
+
+ @Test
+ @DisplayName("The library iterator must invoke the registered callback on failure")
+ void setCallback() throws InvalidConfigurationException {
+ var config = new LibraryAnalysisConfigBuilder()
+ .withInputList(nonExistingGAList)
+ .build();
+
+ var iterator = new MavenCentralLibraryIterator(true, false, false, config);
+
+ // Define a callback for non-existing release lists
+ AtomicBoolean callbackInvoked = new AtomicBoolean(false);
+
+ iterator.setFailedLibraryCallback( event -> {
+ assertEquals("no", event.getGroupId());
+ assertEquals("lib", event.getArtifactId());
+
+ Throwable rootCause = event.getCause();
+ while(rootCause.getCause() != null){
+ rootCause = rootCause.getCause();
+ }
+
+ assertInstanceOf(FileNotFoundException.class, rootCause);
+
+ callbackInvoked.set(true);
+ });
+
+ while(iterator.hasNext()){
+ LibraryResolutionContext ctx = iterator.next();
+ assertTrue(ctx.getLibraryArtifacts().isEmpty());
+ }
+
+ assertTrue(callbackInvoked.get());
+ }
+
+
+ private String getGaInIndex(int pos) {
+ try {
+ LibraryIndexIterator libIt = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI);
+
+ int position = 0;
+ while(libIt.hasNext()){
+ var libGA = libIt.next();
+ if(position == pos){
+ return libGA;
+ }
+ position++;
+ }
+
+ libIt.close();
+ } catch(IOException iox) {
+ fail(iox);
+ }
+
+ return null;
+ }
+
+ private String getGaInInputFile(int pos) {
+ try {
+ assertNotNull(validGAList);
+
+ var lines = Files.readAllLines(validGAList);
+
+ if(pos >= lines.size()) fail("Invalid position in input list: " + pos);
+
+ return lines.get(pos);
+ } catch (IOException iox) {
+ fail(iox);
+ }
+
+ return null;
+ }
+}
diff --git a/src/test/java/org/tudo/sse/utils/TestUtilities.java b/src/test/java/org/tudo/sse/utils/TestUtilities.java
new file mode 100644
index 0000000..7e7a686
--- /dev/null
+++ b/src/test/java/org/tudo/sse/utils/TestUtilities.java
@@ -0,0 +1,19 @@
+package org.tudo.sse.utils;
+
+import java.nio.file.Path;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class TestUtilities {
+
+ public static Path testResource(String pathToResource){
+ try {
+ return Path.of(Objects.requireNonNull(TestUtilities.class.getClassLoader().getResource(pathToResource)).toURI());
+ } catch (Exception x){
+ fail("Test setup: Failed to load resource file " + pathToResource, x);
+ }
+ return null;
+ }
+
+}
diff --git a/src/test/resources/MavenAnalysis.json b/src/test/resources/MavenAnalysis.json
index c7e9edd..8879f2b 100644
--- a/src/test/resources/MavenAnalysis.json
+++ b/src/test/resources/MavenAnalysis.json
@@ -31,7 +31,7 @@
"-1",
"-1",
"53245",
- "13243",
+ "53246",
"null",
"null",
"false"
diff --git a/src/test/resources/artifact-names-invalid.txt b/src/test/resources/artifact-names-invalid.txt
new file mode 100644
index 0000000..827f980
--- /dev/null
+++ b/src/test/resources/artifact-names-invalid.txt
@@ -0,0 +1,10 @@
+org.springframework.boot:spring-boot-starter-web:2.6.3
+org.apache.commons:commons-lang3:3.12.0
+com.fasterxml.jackson.core:jackson-databind
+junit:junit:4.13.2
+org.slf4j:slf4j-api:1.7.32
+org.hibernate:hibernate-core:5.6.5.Final
+org.apache.httpcomponents:httpclient:4.5.13
+org.springframework:spring-core:5.3.15
+mysql:mysql-connector-java:8.0.28::
+org.apache.logging.log4j:log4j-core:2.17.1
\ No newline at end of file
diff --git a/src/main/resources/coordinates.txt b/src/test/resources/artifact-names-valid.txt
similarity index 100%
rename from src/main/resources/coordinates.txt
rename to src/test/resources/artifact-names-valid.txt
diff --git a/src/test/resources/library-names-not-existing.txt b/src/test/resources/library-names-not-existing.txt
new file mode 100644
index 0000000..300a466
--- /dev/null
+++ b/src/test/resources/library-names-not-existing.txt
@@ -0,0 +1 @@
+no:lib