diff --git a/README.md b/README.md index 1344acb..b8a20ab 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,14 @@ Both analysis types provide a method called `void runAnalysis(String[] args)`. I | `-o `
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | +If you do not want to extend one of the aforementioned base classes, MARIN also provides corresponding implementations of the `java.util.Iterator` interface to enumerate and enrich artifacts or libraries. Note that those implementations are **not threadsafe** and cannot perform resolution in parallel. MARIN provides: +* The `MavenCentralArtifactIterator extends Iterator` is the equivalent to the `MavenCentralArtifactAnalysis` and supports all configuration options besides `--threads`. +* The `MavenCentralLibraryIterator extends Iterator` is the equivalent to the `MavenCentralLibraryAnalysis` and supports all configuration options besides `--threads`. + ## Usage +When using MARIN, you can decide whether to extend one of the abstract base classes available, or to rely on of the analysis-equivalent implementations of `java.util.Iterator`. + +### Extending Abstract Base Classes To use MARIN, you will need to implement two components: 1. You need an implementation of either the abstract class `MavenCentralArtifactAnalysis` or `MavenCentralLibraryAnalysis`. This is your actual analysis implementation that defines how a single artifact or library shall be processed. 2. You need a runner class that passes command line arguments to your analysis implementation. Usually, this will look like this: @@ -64,6 +71,28 @@ public class AnalysisRunner { Once this is implemented, you can run your analysis using the following command. Any of the above-mentioned CLI arguments will work for your analysis implementation. ```java -jar executableName *INSERT CLI HERE* ``` +### Using Iterators +As opposed to extending a base class, in order to use MARIN's iterator implementations you will have to first create an analysis configuration object programmatically. This can be done using the `ArtifactAnalysisConfigBuilder` or `LibraryAnalysisConfigBuilder` classes, respectively. The following example shows how to first build a configuration, and then use it to initialize a `MavenCentralArtifactIterator`. +```java +final boolean resolvePom = true; +final boolean resolveTransitive = false; +final boolean resolveJar = true; +final Path gavInputList = Paths.get("path-to-input-file"); + +final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .withSkip(2) + .withTake(5) + .build(); + +MavenCentralArtifactIterator iterator = new MavenCentralArtifactIterator(resolvePom, resolveTransitive, resolveJar, config); + +while(iterator.hasNext()){ + Artifact current = iterator.next(); + // TODO: Process artifact +} +``` + ## Example Use Cases: In the following, there are some example implementations of `MavenCentralArtifactAnalysis`. All of them can be used with the same `AnalysisRunner` implementation seen above, just replace `MyAnalysisImplementation` with the actual implementation name. You can run each example on the first 1000 Maven artifacts by invoking `java -jar executableName -st 0:1000` for the respective project executable JAR. diff --git a/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java new file mode 100644 index 0000000..8d97a2b --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/AbstractEntityIterator.java @@ -0,0 +1,190 @@ +package org.tudo.sse.analyses; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ResolutionContext; +import org.tudo.sse.resolution.ResolverFactory; + +import java.util.Iterator; + +/** + * An abstract base class for analysis-equivalent iterators. This class assumes an underlying source iterator that + * produces a generic kind of source object (an "identifier" for the entity), and then transforms the source object into + * a target entity of generic type. The iterator enforces pagination and supports progress dumps and progress restores. + * + * @param Type of source objects as produced by the underlying iterator + * @param Type of target objects that are created by this iterator + * + * @author Johannes Düsing + */ +abstract class AbstractEntityIterator implements Iterator { + + /** + * Logger instance for this iterator + */ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + /** + * The base configuration for this iterator instance - subclasses may have a more specific configuration type. + */ + protected final LibraryAnalysisConfig baseConfig; + + /** + * Flag indicating whether the underlying source is invalid, i.e. because a connection attempt failed + */ + protected boolean badSource = false; + + private final Iterator source; + + private boolean sourceClosed = false; + + private long currentPosition = 0L; + private long entitiesTaken = 0L; + private long lastPositionSaved = 0L; + + private final ResolverFactory resolverFactory; + private final boolean resolvePom; + private final boolean resolveJar; + + /** + * Builds the underlying source iterator for this entity iterator instance. + * @return The source iterator + */ + protected abstract Iterator buildSource(); + + /** + * Transforms a source object into the target entity. + * @param source The source object as produced by the underlying iterator + * @return The target entity + */ + protected abstract T buildEntity(S source); + + /** + * Creates a new entity iterator instance with the given configuration. Immediately sets the underlying source and + * skips all initial entities according to the configuration. + * + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param baseConfig The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder} + */ + AbstractEntityIterator(boolean resolvePom, + boolean resolveTransitivePoms, + boolean resolveJar, + LibraryAnalysisConfig baseConfig){ + this.baseConfig = baseConfig; + this.resolvePom = resolvePom; + this.resolveJar = resolveJar; + + this.source = buildSource(); + + if(baseConfig instanceof ArtifactAnalysisConfig && ((ArtifactAnalysisConfig)this.baseConfig).outputEnabled){ + this.resolverFactory = new ResolverFactory(true, + ((ArtifactAnalysisConfig)this.baseConfig).outputDirectory, + resolveTransitivePoms); + } else { + this.resolverFactory = new ResolverFactory(resolveTransitivePoms); + } + + try { + if(!this.badSource) + this.skipInitial(); + } catch(Exception x){ + log.error("Failed to initialize iterator", x); + this.badSource = true; + } + } + + @Override + public final boolean hasNext() { + // If the underlying source is invalid, we have no next element + if(this.badSource) return false; + + // If the config specifies a limited amount of entities to take, and that amount is reached, we have no next element + if(this.baseConfig.hasTake() && this.entitiesTaken >= this.baseConfig.take){ + closeSourceIfNeeded(); + return false; + } + + // Otherwise, we have a next entity if the underlying source has a next element + try { + boolean hasNext = this.source.hasNext(); + + if(!hasNext) + closeSourceIfNeeded(); + + return hasNext; + } catch(Exception x){ + // If accessing the source fails, we report an error and mark the source as bad - also we assume we have no + // next element. + log.error("Failed to access source", x); + this.badSource = true; + return false; + } + } + + @Override + public final T next() { + if(!hasNext()) + throw new IllegalStateException("No more entities available"); + + final S sourceEntity = this.source.next(); + final T entity = this.buildEntity(sourceEntity); + + this.entitiesTaken += 1L; + this.currentPosition += 1L; + + writePositionIfNeeded(); + + return entity; + } + + /** + * Enriches the given artifact with ArtifactInformation as defined by the current configuration. + * @param a The artifact to enrich + * @param ctx The current artifact resolution context + */ + protected final void enrichArtifact(Artifact a, ResolutionContext ctx){ + if(this.resolvePom) + resolverFactory.runPom(a.getIdent(), ctx); + + if(this.resolveJar) + resolverFactory.runJar(a.getIdent(), ctx); + } + + private void writePositionIfNeeded(){ + if(this.currentPosition - this.lastPositionSaved > this.baseConfig.progressWriteInterval){ + AnalysisUtils.writePosition(this.currentPosition, this.baseConfig); + this.lastPositionSaved = this.currentPosition; + } + } + + private void closeSourceIfNeeded(){ + if(!this.sourceClosed && !this.badSource && this.source instanceof AutoCloseable){ + try {((AutoCloseable)this.source).close();} + catch(Exception ignored){} + this.sourceClosed = true; + } + } + + private void skipInitial(){ + final long entitiesToSkip = AnalysisUtils.getInitialPosition(this.baseConfig); + + if(entitiesToSkip > 0L && this.badSource) + return; + + for(int i = 0; i < entitiesToSkip && this.source.hasNext(); i++){ + this.source.next(); + this.currentPosition += 1L; + } + + log.debug("Successfully skipped to position {}", this.currentPosition); + + if(!this.source.hasNext()){ + log.warn("Reached end of input source while skipping to position {}", entitiesToSkip); + } + } +} diff --git a/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java new file mode 100644 index 0000000..d1d221f --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/AnalysisUtils.java @@ -0,0 +1,58 @@ +package org.tudo.sse.analyses; + +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; + +/** + * Utility methods for general analysis implementations. + * + * @author Johannes Düsing + */ +final class AnalysisUtils { + + /** + * Retrieves the initial position to start analysis from, based on the current configuration. + * @param config The current analysis configuration + * + * @return Initial number of entities to skip before starting + */ + static long getInitialPosition(LibraryAnalysisConfig config) { + if(config.progressRestoreFile != null) return getRestoreProgressValue(config); + else if(config.hasSkip()) return config.skip; + else return 0L; + } + + /** + * Retrieves the last progress value to start from based on a previous run. + * @param config The current analysis configuration + * + * @return Progress value + */ + static long getRestoreProgressValue(LibraryAnalysisConfig config) { + BufferedReader indexReader; + try { + indexReader = new BufferedReader(new FileReader(config.progressRestoreFile.toFile())); + String line = indexReader.readLine(); + return Integer.parseInt(line); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Writes the current position to the specified progress output file. + * @param currentPosition The current progress to write + * @param config The current analysis configuration + */ + static void writePosition(long currentPosition, LibraryAnalysisConfig config) { + try(BufferedWriter writer = Files.newBufferedWriter(config.progressOutputFile)) { + writer.write(Long.toString(currentPosition)); + } catch(IOException ignored) {} + } +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java index 2446f86..35f7405 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysis.java @@ -1,8 +1,9 @@ package org.tudo.sse.analyses; -import org.apache.pekko.actor.typed.ActorRef; import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.IndexInformation; @@ -12,8 +13,8 @@ import org.tudo.sse.multithreading.WorkItem; import org.tudo.sse.multithreading.WorkloadIsFinalMessage; import org.tudo.sse.resolution.ResolverFactory; -import org.tudo.sse.utils.ArtifactConfigParser; -import org.tudo.sse.utils.FileBasedArtifactIterator; +import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; +import org.tudo.sse.analyses.input.FileBasedArtifactIterator; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.multithreading.QueueActor; import org.tudo.sse.utils.MavenCentralRepository; @@ -29,7 +30,7 @@ */ public abstract class MavenCentralArtifactAnalysis extends MavenCentralAnalysis { - private ArtifactConfigParser.ArtifactConfig artifactConfig; + private ArtifactAnalysisConfig artifactConfig; private ActorSystem system; private ResolverFactory resolverFactory; @@ -53,7 +54,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, super(requiresIndex, requiresPom, requiresTransitives, requiresJar); // Initialize with default config, update later - artifactConfig = new ArtifactConfigParser.ArtifactConfig(); + artifactConfig = new ArtifactAnalysisConfigBuilder().build(); } @@ -68,7 +69,7 @@ protected MavenCentralArtifactAnalysis(boolean requiresIndex, * Returns the CLI configuration for this analysis. * @return CLI information for this analysis */ - public ArtifactConfigParser.ArtifactConfig getSetupInfo() { + public ArtifactAnalysisConfig getSetupInfo() { return this.artifactConfig; } @@ -91,8 +92,8 @@ private void printRunInfo(){ log.info("\t - Reading artifacts from Maven Central index"); if(this.artifactConfig.progressRestoreFile != null) log.info("\t - Restoring last index position from " + this.artifactConfig.progressRestoreFile); if(this.artifactConfig.progressOutputFile != null) log.info("\t - Writing last index position to " + this.artifactConfig.progressOutputFile); - if(this.artifactConfig.skip >= 0) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); - if(this.artifactConfig.take >= 0) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); + if(this.artifactConfig.hasSkip()) log.info("\t - Skipping " + this.artifactConfig.skip + " artifacts"); + if(this.artifactConfig.hasTake()) log.info("\t - Taking " + this.artifactConfig.take + " artifacts"); if(this.artifactConfig.since >= 0) log.info("\t - Skipping artifacts before " + this.artifactConfig.since); if(this.artifactConfig.until >= 0) log.info("\t - Taking artifacts until " + this.artifactConfig.until); @@ -105,7 +106,7 @@ private void printRunInfo(){ public final void runAnalysis(String[] args) { // Obtain CLI arguments try { - this.artifactConfig = new ArtifactConfigParser().parseArtifactConfig(args); + this.artifactConfig = new ArtifactAnalysisConfigParser().parseArtifactConfig(args); printRunInfo(); } catch(CLIException clix){ throw new RuntimeException(clix); @@ -168,7 +169,7 @@ public void indexProcessor() { final long startingPosition = getInitialPosition(); skipN(startingPosition, indexIterator); - if(this.artifactConfig.skip != -1 && this.artifactConfig.take != -1){ + if(this.artifactConfig.hasTake()){ walkPaginated(this.artifactConfig.take, indexIterator); } else if(this.artifactConfig.since != -1 && this.artifactConfig.until != -1){ walkDates(this.artifactConfig.since, this.artifactConfig.until, indexIterator); @@ -359,7 +360,7 @@ void processArtifactsFromInputFile() { log.info("Restoring previous progress from file (position {})", lastProgress); this.skipN(lastProgress, fileIterator); - } else if(this.artifactConfig.skip > 0){ + } else if(this.artifactConfig.hasSkip()){ // Skip configured values only if we did not restore from progress file log.info("Skipping {} entries from file", artifactConfig.skip); skipN(this.artifactConfig.skip, fileIterator); @@ -369,10 +370,8 @@ void processArtifactsFromInputFile() { if(!fileIterator.hasNext()) log.warn("No more contents left to process in input file: {}", this.artifactConfig.inputListFile); - final boolean takeLimited = this.artifactConfig.take >= 0; - long entriesTaken = 0L; - while ((!takeLimited || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { + while ((!this.artifactConfig.hasTake() || entriesTaken < this.artifactConfig.take) && fileIterator.hasNext()) { ArtifactIdent current = null; @@ -421,7 +420,7 @@ public void callResolver(ArtifactIdent identifier, ResolutionContext ctx) { private long getInitialPosition() { if(artifactConfig.progressRestoreFile != null) return getStartingPos(); - else if(artifactConfig.skip > 0) return artifactConfig.skip; + else if(artifactConfig.hasSkip()) return artifactConfig.skip; else return 0L; } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java new file mode 100644 index 0000000..e064e57 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralArtifactIterator.java @@ -0,0 +1,157 @@ +package org.tudo.sse.analyses; + +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactResolutionContext; +import org.tudo.sse.model.index.IndexInformation; +import org.tudo.sse.analyses.input.FileBasedArtifactIterator; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.util.Iterator; + +/** + * Iterator that produces artifacts that are enriched with index-, pom- or jar information, as configured. This is the + * iterator equivalent to the {@link MavenCentralArtifactAnalysis}. Only supports single-threaded resolution. + * + * @author Johannes Düsing + */ +public final class MavenCentralArtifactIterator extends AbstractEntityIterator { + + /** + * Creates a new iterator instance with the given configuration values. + * + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder} + */ + public MavenCentralArtifactIterator(boolean resolvePom, boolean resolveTransitivePoms, boolean resolveJar, ArtifactAnalysisConfig config) { + super(resolvePom, resolveTransitivePoms, resolveJar, config); + + if(this.baseConfig.multipleThreads){ + log.warn("Artifact iterator does no support multiple threads - using a single thread for resolution"); + } + } + + @Override + protected Artifact buildEntity(ArtifactResolutionContext ctx){ + final Artifact artifact = ctx.getRootArtifact(); + + this.enrichArtifact(artifact, ctx); + + return artifact; + } + + + @Override + protected Iterator buildSource(){ + if(this.baseConfig.hasInputList()){ + try{ + return new MavenCentralCustomListArtifactSource(); + } catch(IOException | IllegalArgumentException x){ + log.error("The given input list file is invalid", x); + this.badSource = true; + } + } else { + try { + return new MavenCentralIndexArtifactSource(); + } catch(IOException uix){ + log.error("Failed to access Maven Central Index", uix); + this.badSource = true; + } + } + return null; + } + + private ArtifactAnalysisConfig getConfig(){ + return (ArtifactAnalysisConfig) this.baseConfig; + } + + private class MavenCentralIndexArtifactSource implements Iterator { + + private final IndexIterator index; + + private boolean _needsUpdate = true; + private boolean _hasNext = false; + private IndexInformation _nextInfo = null; + private boolean _indexClosed = false; + + MavenCentralIndexArtifactSource() throws IOException { + this.index = new IndexIterator(MavenCentralRepository.RepoBaseURI); + } + + private void findNext(){ + _hasNext = false; + while(!_hasNext && index.hasNext()){ + var currentInfo = index.next(); + if(currentInfo != null && isValidInfo(currentInfo)){ + _hasNext = true; + _nextInfo = currentInfo; + } + } + } + + private boolean isValidInfo(IndexInformation indexInfo){ + if(!getConfig().hasTimeBasedFiltering()) return true; + else { + long timeStamp = indexInfo.getLastModified(); + return getConfig().since <= timeStamp && timeStamp <= getConfig().until; + } + } + + @Override + public boolean hasNext() { + if(_needsUpdate){ + findNext(); + _needsUpdate = false; + } + + // If we have no more entries on the underlying index, we should close it to release resources + if(!_hasNext && !_indexClosed){ + try { this.index.closeReader(); } + catch (IOException ignored) {} + this._indexClosed = true; + } + + return _hasNext; + } + + @Override + public ArtifactResolutionContext next() { + if(hasNext()){ + _needsUpdate = true; + + final ArtifactResolutionContext ctx = ArtifactResolutionContext.newInstance(_nextInfo.getIdent()); + ctx.getRootArtifact().setIndexInformation(_nextInfo); + + return ctx; + } else throw new IllegalStateException("Call to next on empty artifact source"); + } + } + + private class MavenCentralCustomListArtifactSource implements Iterator { + + private final FileBasedArtifactIterator list; + + MavenCentralCustomListArtifactSource() throws IOException { + list = new FileBasedArtifactIterator(baseConfig.inputListFile); + list.validateInput(); + } + + @Override + public boolean hasNext() { + return list.hasNext(); + } + + @Override + public ArtifactResolutionContext next() { + if(hasNext()){ + return ArtifactResolutionContext.newInstance(list.next()); + } else throw new IllegalStateException("Call to next on empty artifact source"); + } + + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index b03dc61..40a3ec8 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -2,6 +2,7 @@ import org.apache.pekko.actor.typed.ActorSystem; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.LibraryResolutionContext; @@ -12,7 +13,7 @@ import org.tudo.sse.resolution.ResolverFactory; import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; import org.tudo.sse.resolution.releases.IReleaseListProvider; -import org.tudo.sse.utils.CommonConfigParser; +import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser; import org.tudo.sse.utils.LibraryIndexIterator; import org.tudo.sse.utils.MavenCentralRepository; @@ -36,7 +37,7 @@ public abstract class MavenCentralLibraryAnalysis extends MavenCentralAnalysis { /** * The configuration for this analysis instance. Only available after runAnalysis has been called. */ - protected CommonConfigParser.CommonConfig config; + protected LibraryAnalysisConfig config; private ActorSystem system; private long lastPositionSaved; @@ -61,7 +62,7 @@ protected MavenCentralLibraryAnalysis(boolean requiresPom, boolean requiresTrans public final void runAnalysis(String[] args){ // Obtain CLI arguments try { - this.config = new CommonConfigParser().parseCommonConfig(args); + this.config = new LibraryAnalysisConfigParser().parseCommonConfig(args); printRunInfo(); } catch(CLIException clix){ throw new RuntimeException(clix); @@ -84,9 +85,9 @@ public final void runAnalysis(String[] args){ currentPosition += 1L; } // Notify users that skip will not be applied - if(config.skip > 0) + if(config.hasSkip()) log.info("Not applying skip value because progress was restored from previous run"); - } else if(config.skip > 0){ + } else if(config.hasSkip()){ // We only apply a skip if we did not restore previous progress log.info("Skipping {} library names", config.skip); for(int i = 0; i < config.skip; i++){ @@ -106,7 +107,7 @@ public final void runAnalysis(String[] args){ long entriesTaken = 0L; - if(config.take >= 0) + if(config.hasTake()) log.info("Taking {} library names", config.take); // Invoke the beforeRunStart lifecycle hook @@ -119,7 +120,7 @@ public final void runAnalysis(String[] args){ // If specified, we only take the configured amount of entries. If not, we process as long as the iterator // provides new entries - while((config.take < 0 || entriesTaken < config.take) && gaIterator.hasNext()){ + while((!config.hasTake()|| entriesTaken < config.take) && gaIterator.hasNext()){ processEntry(gaIterator.next()); entriesTaken += 1L; @@ -336,8 +337,8 @@ private void printRunInfo(){ log.info("\t - Reading libraries from Maven Central index"); if(config.progressRestoreFile != null) log.info("\t - Restoring last index position from " + config.progressRestoreFile); if(config.progressOutputFile != null) log.info("\t - Writing last index position to " + config.progressOutputFile); - if(config.skip >= 0) log.info("\t - Skipping " + config.skip + " artifacts"); - if(config.take >= 0) log.info("\t - Taking " + config.take + " artifacts"); + if(config.hasSkip()) log.info("\t - Skipping " + config.skip + " artifacts"); + if(config.hasTake()) log.info("\t - Taking " + config.take + " artifacts"); } else { log.info("\t - Reading libraries from GA-list at " + config.inputListFile); } diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java new file mode 100644 index 0000000..e6c08df --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryIterator.java @@ -0,0 +1,172 @@ +package org.tudo.sse.analyses; + +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.input.FileBasedLibraryIterator; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.model.LibraryResolutionContext; +import org.tudo.sse.resolution.releases.DefaultMavenReleaseListProvider; +import org.tudo.sse.resolution.releases.IReleaseListProvider; +import org.tudo.sse.utils.LibraryIndexIterator; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; + +/** + * Iterator that produces libraries that are enriched with index-, pom- or jar information, as configured. This is the + * iterator equivalent to the {@link MavenCentralLibraryAnalysis}. Only supports single-threaded resolution. + * + * @author Johannes Düsing + */ +public final class MavenCentralLibraryIterator extends AbstractEntityIterator { + + private final IReleaseListProvider releaseListProvider; + private Consumer failedLibraryCallback = null; + + /** + * Creates a new iterator instance with the given configuration values. + * @param resolvePom Whether information on artifact pom files shall be resolved + * @param resolveTransitivePoms Whether transitive poms should also be resolved + * @param resolveJar Whether information on jar files shall be resolved + * @param config The analysis configuration, see {@link org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder} + */ + public MavenCentralLibraryIterator(boolean resolvePom, + boolean resolveTransitivePoms, + boolean resolveJar, + LibraryAnalysisConfig config) { + super(resolvePom, resolveTransitivePoms, resolveJar, config); + + if(this.baseConfig.multipleThreads){ + log.warn("Library iterator does no support multiple threads - using a single thread for resolution"); + } + + this.releaseListProvider = DefaultMavenReleaseListProvider.getInstance(); + } + + @Override + protected LibraryResolutionContext buildEntity(String ga){ + final String[] gaParts = ga.split(":"); // Valid tuple, ensured by source + final LibraryResolutionContext ctx = LibraryResolutionContext.newInstance(ga); + + // Get list of all library releases + final List allReleases = getReleaseIdentifiers(gaParts[0], gaParts[1]); + + if(allReleases != null){ + // For each release, build artifact and enrich with data + for(ArtifactIdent release : allReleases){ + final Artifact artifact = ctx.createArtifact(release); + ctx.addLibraryArtifact(artifact); + + this.enrichArtifact(artifact, ctx); + } + } else { + // If we failed to obtain a release list, the respective callback has been invoked. Here, we just return an + // empty context. + log.warn("No releases for library {}, returning empty context", ga); + } + return ctx; + } + + /** + * Sets the callback that is invoked when obtaining a release list for a given library fails. + * @param failedLibraryCallback Callback on failed libraries + */ + public void setFailedLibraryCallback(Consumer failedLibraryCallback){ + this.failedLibraryCallback = failedLibraryCallback; + } + + private List getReleaseIdentifiers(String groupId, String artifactId){ + try { + final List libraryVersions = releaseListProvider.getReleases(groupId, artifactId); + List identifiers = new ArrayList<>(); + for(String libraryVersion : libraryVersions){ + identifiers.add(new ArtifactIdent(groupId, artifactId, libraryVersion)); + } + return identifiers; + } catch (Exception x) { + log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + + if(this.failedLibraryCallback != null){ + final var event = new FailedLibraryEvent(groupId, artifactId, x); + try { this.failedLibraryCallback.accept(event); } catch (Exception ex) { + log.warn("Unexcepted exception in failed analysis callback for library {}:{}", groupId, artifactId, ex); + } + } + + return null; + } + } + + @Override + protected Iterator buildSource(){ + if(this.baseConfig.hasInputList()){ + try { + var iterator = new FileBasedLibraryIterator(this.baseConfig.inputListFile); + iterator.validateInput(); + return iterator; + } catch (IOException | IllegalArgumentException x){ + log.error("The given input list file is invalid", x); + this.badSource = true; + } + } else { + try { + return new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + } catch (IOException iox){ + log.error("Failed to access Maven Central Index", iox); + this.badSource = true; + } + } + return null; + } + + /** + * Event that is thrown when obtaining a library's release list fails. + */ + public static class FailedLibraryEvent { + + private final String groupId; + private final String artifactId; + private final Throwable cause; + + /** + * Creates a new event instance. + * @param groupId The library group id + * @param artifactId The library artifact id + * @param cause The throwable that caused the failure + */ + FailedLibraryEvent(String groupId, String artifactId, Throwable cause){ + this.groupId = groupId; + this.artifactId = artifactId; + this.cause = cause; + } + + /** + * Get the failed library's groupID + * @return Maven groupID + */ + public String getGroupId() { + return this.groupId; + } + + /** + * Get the failed library's artifactID + * @return Maven artifactID + */ + public String getArtifactId() { + return this.artifactId; + } + + /** + * Get the failure cause + * @return Throwable that caused the failure + */ + public Throwable getCause() { + return this.cause; + } + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java new file mode 100644 index 0000000..f2f3dd8 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -0,0 +1,57 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Path; + +/** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. + * + * @author Johannes Düsing + */ +public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { + + static final long DEFAULT_VALUE_SINCE = -1L; + static final long DEFAULT_VALUE_UNTIL = -1L; + static final Path DEFAULT_VALUE_OUTPUT = null; + + /** + * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. + */ + public long since; + + /** + * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. + */ + public long until; + + /** + * Directory to write file outputs to, or null if disabled. + */ + public Path outputDirectory; + + /** + * Whether files shall be written to the output directory. + */ + public boolean outputEnabled; + + /** + * Creates a new configuration object with default values. + */ + ArtifactAnalysisConfig(){ + super(); + + since = DEFAULT_VALUE_SINCE; + until = DEFAULT_VALUE_UNTIL; + outputEnabled = false; + outputDirectory = DEFAULT_VALUE_OUTPUT; + } + + /** + * Checks whether there is a custom time range to filter artifacts for. + * @return True if time based filtering is enabled + */ + public boolean hasTimeBasedFiltering(){ + return this.since != DEFAULT_VALUE_SINCE && this.until != DEFAULT_VALUE_UNTIL; + } + + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java new file mode 100644 index 0000000..d0e2a33 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java @@ -0,0 +1,136 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Configuration builder to obtain {@link ArtifactAnalysisConfig} instances programmatically. + * + * @author Johannes Düsing + */ +public final class ArtifactAnalysisConfigBuilder extends LibraryAnalysisConfigBuilder { + + private final ArtifactAnalysisConfig theConfig; + + private ArtifactAnalysisConfigBuilder(ArtifactAnalysisConfig theConfig) { + super(theConfig); + + this.theConfig = theConfig; + } + + /** + * Creates a new builder with default configuration values. + */ + public ArtifactAnalysisConfigBuilder(){ + this(new ArtifactAnalysisConfig()); + } + + @Override + public ArtifactAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("skip", "Cannot apply skip when time-based filtering is used"); + + super.withSkip(toSkip); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("take", "Cannot apply take when time-based filtering is used"); + + super.withTake(toTake); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withInputList(Path inputList) throws InvalidConfigurationException { + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + throw new InvalidConfigurationException("inputs", "Cannot use input list when time-based filtering is applied"); + + super.withInputList(inputList); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException { + super.withProgressOutputFile(outputFile); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException { + super.withProgressRestoreFile(restoreFile); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException { + super.withNumberOfThreads(numThreads); + return this; + } + + @Override + public ArtifactAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException { + super.withProgressWriteInterval(progressWriteInterval); + return this; + } + + /** + * Sets the output directory to store processed artifacts in. If your analysis requires IndexInformation only, it + * will output a list of GAV triples. If pom information is required, one pom.xml file per artifact will be stored. + * If JAR information is required, the artifact's JAR will be stored in this directory. + * + * @param outDir The output directory to use. Must be a valid reference to an existing directory. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public ArtifactAnalysisConfigBuilder withOutputDirectory(Path outDir) throws InvalidConfigurationException { + if(outDir == null || !Files.isDirectory(outDir)) + throw new InvalidConfigurationException("output", "Output directory must be a valid directory reference"); + + if(this.theConfig.outputDirectory != ArtifactAnalysisConfig.DEFAULT_VALUE_OUTPUT) + log.warn("Value 'output' is set multiple times, previous value overridden."); + + this.theConfig.outputDirectory = outDir; + this.theConfig.outputEnabled = true; + return this; + } + + /** + * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have + * been released after the timestamp given in 'since', and before 'until'. + * + * @param since Timestamp marking the lower limit of the range + * @param until Timestamp marking the upper limit of the range + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration values are not valid + */ + public ArtifactAnalysisConfigBuilder withSinceUtil(long since, long until) throws InvalidConfigurationException { + if(since <= 0L || until <= since) + throw new InvalidConfigurationException("since-until", "Since and Until must define a valid range"); + + if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP || + this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE) + throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when pagination is used"); + + if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST) + throw new InvalidConfigurationException("since-until", "Cannot apply time-based filtering when input list is used"); + + if(this.theConfig.since != ArtifactAnalysisConfig.DEFAULT_VALUE_SINCE || + this.theConfig.until != ArtifactAnalysisConfig.DEFAULT_VALUE_UNTIL) + log.warn("Values 'since' and 'until' are set multiple times, previous values overridden."); + + this.theConfig.since = since; + this.theConfig.until = until; + return this; + } + + @Override + public ArtifactAnalysisConfig build(){ + return this.theConfig; + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java new file mode 100644 index 0000000..cb26b5a --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/InvalidConfigurationException.java @@ -0,0 +1,50 @@ +package org.tudo.sse.analyses.config; + +/** + * Exception thrown when a MARIN configuration value was invalid. + * + * @author Johannes Düsing + */ +public class InvalidConfigurationException extends Exception { + + /** + * Attribute the error occurred for. + */ + private final String attrName; + + /** + * Message describing the error. + */ + private final String msg; + + /** + * Creates a new instance with the given configuration attribute name and message. + * @param attrName The configuration value that was invalid + * @param msg A message describing the error + */ + InvalidConfigurationException(String attrName, String msg){ + this.attrName = attrName; + this.msg = msg; + } + + /** + * Returns the name of the attribute for which this exception occurred. + * @return The attribute name + */ + public String getAttributeName() { + return this.attrName; + } + + /** + * Returns a description on the error that occurred. + * @return The error description + */ + public String getDescription(){ + return this.msg; + } + + @Override + public String getMessage() { + return String.format("Invalid configuration value for '%s': %s", attrName, msg); + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java new file mode 100644 index 0000000..5341b6d --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfig.java @@ -0,0 +1,98 @@ +package org.tudo.sse.analyses.config; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * + * @author Johannes Düsing + */ +public class LibraryAnalysisConfig { + + static final long DEFAULT_VALUE_SKIP = -1L; + static final long DEFAULT_VALUE_TAKE = -1L; + static final Path DEFAULT_VALUE_INPUT_LIST = null; + static final Path DEFAULT_VALUE_PROGRESS_OUTPUT_FILE = Paths.get("marin-progress"); + static final Path DEFAULT_VALUE_PROGRESS_RESTORE_FILE = null; + static final int DEFAULT_NUM_THREADS = 1; + static final int DEFAULT_PROGRESS_WRITE_INTERVAL = 100; + + /** + * Number of artifacts to skip from the underlying source, or -1 if disabled. + */ + public long skip; + + /** + * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. + */ + public long take; + + /** + * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. + */ + public Path inputListFile; + + /** + * File path to output analysis progress to. + */ + public Path progressOutputFile; + + /** + * Path to read previous progress from, in order to restore it. Null if disabled. + */ + public Path progressRestoreFile; + + /** + * True if multiple threads shall be used, false for single-threaded analysis. + */ + public boolean multipleThreads; + + /** + * Number of threads to use. + */ + public int threadCount; + + /** + * Interval of entities after which progress shall be saved. Defaults to 100. + */ + public int progressWriteInterval; + + /** + * Creates a new configuration with default values. + */ + LibraryAnalysisConfig() { + skip = DEFAULT_VALUE_SKIP; + take = DEFAULT_VALUE_TAKE; + inputListFile = DEFAULT_VALUE_INPUT_LIST; + progressOutputFile = DEFAULT_VALUE_PROGRESS_OUTPUT_FILE; + progressRestoreFile = DEFAULT_VALUE_PROGRESS_RESTORE_FILE; + multipleThreads = DEFAULT_NUM_THREADS == 1 ? false : true; + threadCount = DEFAULT_NUM_THREADS; + progressWriteInterval = DEFAULT_PROGRESS_WRITE_INTERVAL; + } + + /** + * Checks whether there is a non-default amount of entities to skip + * @return True if there are entities to skip + */ + public boolean hasSkip() { + return this.skip != DEFAULT_VALUE_SKIP; + } + + /** + * Checks whether there is a non-default amount of entities to take + * @return True if there are limited entities to take + */ + public boolean hasTake() { + return this.take != DEFAULT_VALUE_TAKE; + } + + /** + * Checks whether there is a custom input list of entities to process. + * @return True if there is an input list specified + */ + public boolean hasInputList() { + return this.inputListFile != DEFAULT_VALUE_INPUT_LIST; + } +} diff --git a/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java new file mode 100644 index 0000000..7d43411 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/LibraryAnalysisConfigBuilder.java @@ -0,0 +1,178 @@ +package org.tudo.sse.analyses.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Configuration builder to obtain {@link LibraryAnalysisConfig} instances programmatically. + * + * @author Johannes Düsing + */ +public class LibraryAnalysisConfigBuilder { + + /** + * The logger for this instance. + */ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final LibraryAnalysisConfig theConfig; + + /** + * Creates a new builder with default configuration values. + */ + public LibraryAnalysisConfigBuilder() { + this(new LibraryAnalysisConfig()); + } + + /** + * Creates a new builder with the given baseline configuration instance. + * + * @param theConfig Baseline configuration instance for this builder + */ + protected LibraryAnalysisConfigBuilder(LibraryAnalysisConfig theConfig) { + this.theConfig = theConfig; + } + + /** + * Sets the amount of entities to skip when processing analysis inputs. Can be used for pagination. Defaults to zero. + * + * @param toSkip Amount of entities (artifacts or libraries) to skip + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid. + */ + public LibraryAnalysisConfigBuilder withSkip(long toSkip) throws InvalidConfigurationException { + if(toSkip < 0) + throw new InvalidConfigurationException("skip", "Skip must not be negative"); + + if(this.theConfig.skip != LibraryAnalysisConfig.DEFAULT_VALUE_SKIP) + log.warn("Value 'skip' is set multiple times, previous value overridden."); + + this.theConfig.skip = toSkip == 0 ? LibraryAnalysisConfig.DEFAULT_VALUE_SKIP : toSkip; + return this; + } + + /** + * Sets the amount of entities to take when processing analysis inputs. Can be used for pagination. By default, no + * limit is imposed and the input source is consumed fully. + * + * @param toTake Amount of entities (artifacts or libraries) to take + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid. + */ + public LibraryAnalysisConfigBuilder withTake(long toTake) throws InvalidConfigurationException { + if(toTake <= 0) + throw new InvalidConfigurationException("take", "Take must be positive"); + + if(this.theConfig.take != LibraryAnalysisConfig.DEFAULT_VALUE_TAKE) + log.warn("Value 'take' is set multiple times, previous value overridden."); + + this.theConfig.take = toTake; + return this; + } + + /** + * Sets the input list of entities to process as analysis inputs. This is either a list of GAV triples (artifacts) + * or GA tuples (libraries), one entity per line. By default, the Maven Central index is used as the source of inputs. + * + * @param listPath Path to input file. Must be an existing regular file, one entity per line is required. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withInputList(Path listPath) throws InvalidConfigurationException { + if(listPath == null || !Files.isRegularFile(listPath)) + throw new InvalidConfigurationException("inputs", "Input list file must be a valid file reference"); + + if(this.theConfig.inputListFile != LibraryAnalysisConfig.DEFAULT_VALUE_INPUT_LIST) + log.warn("Value 'inputs' is set multiple times, previous value overridden."); + + this.theConfig.inputListFile = listPath; + return this; + } + + /** + * Sets the output file to regularly write the analysis progress to. By default, a file called 'marin-progress.txt' + * is created in the current workdir. + * + * @param outputFile Path to the progress output file to create. Existing files will be overridden! + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressOutputFile(Path outputFile) throws InvalidConfigurationException { + if(outputFile == null) + throw new InvalidConfigurationException("progress-output-file", "Output file cannot be null"); + + if(this.theConfig.progressOutputFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_OUTPUT_FILE) + log.warn("Value 'progress-output-file' is set multiple times, previous value overridden."); + + this.theConfig.progressOutputFile = outputFile; + return this; + } + + /** + * Sets the file to restore progress from. Should be a progress output file previously created by MARIN. By default, + * no progress is restored. + * + * @param restoreFile Path to the progress restore file. Must be a valid text file. + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressRestoreFile(Path restoreFile) throws InvalidConfigurationException { + if(restoreFile == null || !Files.isRegularFile(restoreFile)) + throw new InvalidConfigurationException("progress-restore-file", "Restore file must be a valid file reference"); + + if(this.theConfig.progressRestoreFile != LibraryAnalysisConfig.DEFAULT_VALUE_PROGRESS_RESTORE_FILE) + log.warn("Value 'progress-restore-file' is set multiple times, previous value overridden."); + + this.theConfig.progressRestoreFile = restoreFile; + return this; + } + + /** + * Sets the number of threads to use for analysis. By default, one thread is used. + * @param numThreads Number of threads to use + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withNumberOfThreads(int numThreads) throws InvalidConfigurationException { + if(numThreads <= 0) + throw new InvalidConfigurationException("threads", "Number of threads must be positive"); + + if(this.theConfig.threadCount != LibraryAnalysisConfig.DEFAULT_NUM_THREADS) + log.warn("Value 'threads' is set multiple times, previous value overridden."); + + this.theConfig.threadCount = numThreads; + this.theConfig.multipleThreads = numThreads > 1; + return this; + } + + /** + * Sets the amount of entities (artifacts or libraries) after which to write progress to the progress output file. + * Default is 100. + * @param progressWriteInterval Amount of entities after which to save progress + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration value is not valid + */ + public LibraryAnalysisConfigBuilder withProgressWriteInterval(int progressWriteInterval) throws InvalidConfigurationException { + if(progressWriteInterval <= 0) + throw new InvalidConfigurationException("save-progress-interval", "Progress write interval must be positive"); + + if(this.theConfig.progressWriteInterval != LibraryAnalysisConfig.DEFAULT_PROGRESS_WRITE_INTERVAL) + log.warn("Value 'progress-write-interval' is set multiple times, previous value overridden."); + + this.theConfig.progressWriteInterval = progressWriteInterval; + return this; + } + + /** + * Builds the configuration as specified by all previous invocations to this instance. + * + * @return The configuration object that has been constructed by this builder. + */ + public LibraryAnalysisConfig build(){ + return this.theConfig; + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java new file mode 100644 index 0000000..6b46233 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -0,0 +1,62 @@ +package org.tudo.sse.analyses.config.parsing; + +import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; + +/** + * Parser for creating {@link ArtifactAnalysisConfig} objects from CLI parameters. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ +public class ArtifactAnalysisConfigParser extends LibraryAnalysisConfigParser implements CLIParsingUtilities { + + /** + * Creates a new artifact config parser instance. + */ + public ArtifactAnalysisConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link org.tudo.sse.analyses.config.ArtifactAnalysisConfig} object. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link ArtifactAnalysisConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ + public ArtifactAnalysisConfig parseArtifactConfig(String[] args) throws CLIException { + final ArtifactAnalysisConfigBuilder configBuilder = new ArtifactAnalysisConfigBuilder(); + + for(int i = 0; i < args.length; i += 2) { + handleArtifactParameter(args, i, configBuilder); + } + + return configBuilder.build(); + } + + + private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfigBuilder configBuilder) throws CLIException { + try { + + switch(args[i]) { + case "-su": + case "--since-until": + final long[] sinceUntil = nextArgAsLongPair(args, i); + configBuilder.withSinceUtil(sinceUntil[0], sinceUntil[1]); + break; + case "-o": + case "--output": + configBuilder.withOutputDirectory(nextArgAsDirectoryReference(args, i)); + break; + default: + super.handleParameter(args, i , configBuilder); + + } + + } catch (InvalidConfigurationException icx) { + CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName()); + wrapped.initCause(icx); + throw wrapped; + } + } + +} diff --git a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java similarity index 98% rename from src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java rename to src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java index 8d67fbc..2c175d1 100644 --- a/src/main/java/org/tudo/sse/utils/CLIParsingUtilities.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java @@ -1,4 +1,4 @@ -package org.tudo.sse.utils; +package org.tudo.sse.analyses.config.parsing; import org.tudo.sse.CLIException; diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java new file mode 100644 index 0000000..ba2b95e --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/LibraryAnalysisConfigParser.java @@ -0,0 +1,86 @@ +package org.tudo.sse.analyses.config.parsing; + +import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; + +/** + * Parser for {@link LibraryAnalysisConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. + * Configuration is obtained by parsing command line arguments provided as an array of strings. + */ +public class LibraryAnalysisConfigParser implements CLIParsingUtilities { + + /** + * Creates a new library config parsers instance. + */ + public LibraryAnalysisConfigParser() { super(); } + + /** + * Parses a given array containing JVM command line arguments into an {@link LibraryAnalysisConfig} objects. Throws an exception + * if the given arguments are not valid. + * @param args Array of command line arguments, as passed by the JVM + * @return Corresponding {@link LibraryAnalysisConfig} if parsing was successful + * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. + */ + public LibraryAnalysisConfig parseCommonConfig(String[] args) throws CLIException { + final LibraryAnalysisConfigBuilder configBuilder = new LibraryAnalysisConfigBuilder(); + + + for(int i = 0; i < args.length; i += 2){ + handleParameter(args, i, configBuilder); + } + + return configBuilder.build(); + } + + /** + * Parses a given parameter from the array of CLI arguments and applies it to the config object. + * @param args CLI arguments array + * @param i Current parsing position + * @param configBuilder Current configuration object + * @throws CLIException If parsing the current argument fails. + */ + protected void handleParameter(String[] args, int i, LibraryAnalysisConfigBuilder configBuilder) throws CLIException { + try { + switch (args[i]){ + case "-st": + case "--skip-take": + final long[] skipTake = nextArgAsLongPair(args, i); + configBuilder + .withSkip(skipTake[0]) + .withTake(skipTake[1]); + break; + case "-prf": + case "--progress-restore-file": + configBuilder.withProgressRestoreFile(nextArgAsRegularFileReference(args, i)); + break; + case "-spi": + case "--save-progress-interval": + configBuilder.withProgressWriteInterval(nextArgAsInt(args, i)); + break; + case "-pof": + case "--progress-output-file": + configBuilder.withProgressOutputFile(nextArgAsPath(args, i)); + break; + case "-i": + case "--inputs": + configBuilder.withInputList(nextArgAsRegularFileReference(args, i)); + break; + case "-t": + case "--threads": + final int threads = nextArgAsInt(args, i); + configBuilder.withNumberOfThreads(threads); + break; + default: + throw new CLIException(args[i]); + } + } catch (InvalidConfigurationException icx) { + CLIException wrapped = new CLIException(icx.getMessage(), icx.getAttributeName()); + wrapped.initCause(icx); + throw wrapped; + } + } + + +} diff --git a/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java new file mode 100644 index 0000000..d3df78b --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/AbstractInputFileIterator.java @@ -0,0 +1,91 @@ +package org.tudo.sse.analyses.input; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; + +/** + * An iterator that wraps an underlying file that contains line-wise representations of entities. This wrapper produces + * one entity per line, with parsing logic defined by concrete subclasses. + * + * @param The type that entities are parsed to + * + * @author Johannes Düsing + */ +abstract class AbstractInputFileIterator implements Iterator { + + /** + * Checks whether a given line within the input file represents a valid entity + * @param line The line to check + * @return True is the line represents a valid entity, false otherwise + */ + protected abstract boolean isValidLine(String line); + + /** + * Parses a given line into its representing entity. Will only ever be called after isValidLine returned true. + * @param line The line to parse + * @return The resulting entity + */ + protected abstract T parseLine(String line); + + private final Path inputPath; + + private Iterator lineIterator; + + /** + * Creates a new iterator instance for the given file path. + * @param input Path to a text file containing one entity per line + */ + protected AbstractInputFileIterator(Path input){ + this.inputPath = input; + } + + /** + * Checks whether the underlying file exists and contains only valid entities. + * @throws IOException If accessing the underlying file fails + * @throws IllegalArgumentException If the underlying file contents are not valid + */ + public void validateInput() throws IOException, IllegalArgumentException { + var lines = Files.readAllLines(inputPath); + int lineCnt = 0; + + for(String line: lines){ + if(!isValidLine(line)){ + throw new IllegalArgumentException("Not a valid entity: " + line + " (" + inputPath.getFileName() + ":" + lineCnt + ")"); + } + + lineCnt += 1; + } + + this.lineIterator = lines.iterator(); + } + + @Override + public boolean hasNext() { + if(this.lineIterator == null){ + try { + var lines = Files.readAllLines(this.inputPath); + this.lineIterator = lines.iterator(); + } catch (IOException iox){ + throw new IllegalArgumentException("Failed to access input file", iox); + } + } + + return this.lineIterator.hasNext(); + } + + @Override + public T next() { + if(!hasNext()) + throw new IllegalStateException("Next on empty iterator"); + + String line = this.lineIterator.next(); + + if(isValidLine(line)) + return parseLine(line); + else + throw new IllegalStateException("Not a valid entity: " + line); + } + +} diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java new file mode 100644 index 0000000..4d646a8 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedArtifactIterator.java @@ -0,0 +1,42 @@ +package org.tudo.sse.analyses.input; + +import org.tudo.sse.model.ArtifactIdent; + +import java.nio.file.Path; + +/** + * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of + * groupID:artifactID:version per line. + */ +public class FileBasedArtifactIterator extends AbstractInputFileIterator { + + /** + * Creates a new artifact identifier iterator for the given input file. + * @param gavList Path to input file + */ + public FileBasedArtifactIterator(Path gavList) { + super(gavList); + } + + @Override + protected boolean isValidLine(String line) { + var parts = line.split(":"); + + if(parts.length != 3) + return false; + + for(String part: parts){ + if(part.isBlank()) + return false; + } + + return true; + } + + @Override + protected ArtifactIdent parseLine(String line) { + var parts = line.split(":"); + + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } +} diff --git a/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java new file mode 100644 index 0000000..a8138e6 --- /dev/null +++ b/src/main/java/org/tudo/sse/analyses/input/FileBasedLibraryIterator.java @@ -0,0 +1,38 @@ +package org.tudo.sse.analyses.input; + +import java.nio.file.Path; + +/** + * Iterator that reads Maven Central library identifiers from a text file. Expects one colon-separated tuple of + * groupID:artifactID per line. + */ +public class FileBasedLibraryIterator extends AbstractInputFileIterator{ + + /** + * Creates a new library identifier iterator for the given input file. + * @param gaList Path to input list + */ + public FileBasedLibraryIterator(Path gaList) { + super(gaList); + } + + @Override + protected boolean isValidLine(String line) { + var parts = line.split(":"); + + if(parts.length != 2) + return false; + + for(String part: parts){ + if(part.isBlank()) + return false; + } + + return true; + } + + @Override + protected String parseLine(String line) { + return line; + } +} diff --git a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java index be261f1..eb6ec37 100644 --- a/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java +++ b/src/main/java/org/tudo/sse/model/ArtifactResolutionContext.java @@ -26,8 +26,16 @@ public static ArtifactResolutionContext newInstance(ArtifactIdent identifier) { * Returns the identifier of the artifact for which this context was created. * @return ArtifactIdent for this context */ - public ArtifactIdent getIdentifier() { + public ArtifactIdent getRootArtifactIdentifier() { return identifier; } + /** + * Returns the artifact for which this context was initially created + * @return The root artifact of this context + */ + public Artifact getRootArtifact() { + return this.createArtifact(getRootArtifactIdentifier()); + } + } diff --git a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java index 5e68bdf..2ebe7d6 100644 --- a/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java +++ b/src/main/java/org/tudo/sse/multithreading/ProcessIdentifierMessage.java @@ -28,7 +28,7 @@ public ProcessIdentifierMessage(ArtifactResolutionContext resolutionContext, Mav * @return The artifact identifier */ public ArtifactIdent getIdentifier() { - return this.artifactCtx.getIdentifier(); + return this.artifactCtx.getRootArtifactIdentifier(); } /** diff --git a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java b/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java deleted file mode 100644 index 7fde305..0000000 --- a/src/main/java/org/tudo/sse/utils/ArtifactConfigParser.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.CLIException; - -import java.nio.file.Path; - -/** - * Parser for ArtifactConfig objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis}. - * Configuration is obtained by parsing command line arguments provided as an array of strings. - */ -public class ArtifactConfigParser extends CommonConfigParser implements CLIParsingUtilities { - - /** - * Creates a new artifact config parser instance. - */ - public ArtifactConfigParser() { super(); } - - /** - * Parses a given array containing JVM command line arguments into an {@link ArtifactConfig} object. Throws an exception - * if the given arguments are not valid. - * @param args Array of command line arguments, as passed by the JVM - * @return Corresponding {@link ArtifactConfig} if parsing was successful - * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. - */ - public ArtifactConfig parseArtifactConfig(String[] args) throws CLIException { - final ArtifactConfig artifactConfig = new ArtifactConfig(); - - for(int i = 0; i < args.length; i += 2) { - handleArtifactParameter(args, i, artifactConfig); - } - - return artifactConfig; - } - - - private void handleArtifactParameter(String[] args, int i, ArtifactConfig config) throws CLIException { - switch(args[i]) { - case "-su": - case "--since-until": - if(config.skip != -1L) - throw new CLIException("Cannot apply time-based filtering when pagination (-st) is used", args[i]); - if(config.since != -1L) - throw new CLIException("Values for since and until cannot be set twice!", args[i]); - if(config.inputListFile != null) - throw new CLIException("Cannot apply time-based filtering when input list (-i) is used", args[i]); - - final long[] sinceUntil = nextArgAsLongPair(args, i); - config.since = sinceUntil[0]; - config.until = sinceUntil[1]; - break; - - case "-o": - case "--output": - config.outputEnabled = true; - config.outputDirectory = nextArgAsDirectoryReference(args, i); - break; - - case "-st": - case "--skip-take": - if(config.since != -1L) - throw new CLIException("Cannot apply pagination when time-based filtering (-su) is used", args[i]); - super.handleParameter(args, i, config); - break; - - case "-i": - case "--inputs": - if(config.since != -1L) - throw new CLIException("Cannot use input list when time-based filtering (-su) is applied", args[i]); - super.handleParameter(args, i, config); - break; - - default: - super.handleParameter(args, i , config); - - } - } - - /** - * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralArtifactAnalysis} instance. - */ - public static class ArtifactConfig extends CommonConfig { - - /** - * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. - */ - public long since; - - /** - * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. - */ - public long until; - - /** - * Directory to write file outputs to, or null if disabled. - */ - public Path outputDirectory; - - /** - * Whether files shall be written to the output directory. - */ - public boolean outputEnabled; - - /** - * Creates a new configuration object with default values. - */ - public ArtifactConfig(){ - super(); - - since = -1L; - until = -1L; - outputEnabled = false; - outputDirectory = null; - } - - - } - -} diff --git a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java b/src/main/java/org/tudo/sse/utils/CommonConfigParser.java deleted file mode 100644 index 474449e..0000000 --- a/src/main/java/org/tudo/sse/utils/CommonConfigParser.java +++ /dev/null @@ -1,146 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.CLIException; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * Parser for {@link CommonConfig} objects, i.e. configurations of the {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis}. - * Configuration is obtained by parsing command line arguments provided as an array of strings. - */ -public class CommonConfigParser implements CLIParsingUtilities { - - /** - * Creates a new artifact config parsers instance. - */ - public CommonConfigParser() { super(); } - - /** - * Parses a given array containing JVM command line arguments into an {@link CommonConfig} objects. Throws an exception - * if the given arguments are not valid. - * @param args Array of command line arguments, as passed by the JVM - * @return Corresponding {@link CommonConfig} if parsing was successful - * @throws CLIException If parsing failed. Contains details about the specific argument that was invalid. - */ - public CommonConfig parseCommonConfig(String[] args) throws CLIException { - final CommonConfig config = new CommonConfig(); - - - for(int i = 0; i < args.length; i += 2){ - handleParameter(args, i, config); - } - - return config; - } - - /** - * Parses a given parameter from the array of CLI arguments and applies it to the config object. - * @param args CLI arguments array - * @param i Current parsing position - * @param config Current configuration object - * @throws CLIException If parsing the current argument fails. - */ - protected void handleParameter(String[] args, int i, CommonConfig config) throws CLIException { - switch (args[i]){ - case "-st": - case "--skip-take": - if(config.skip != -1L) - throw new CLIException("Values for skip and take cannot be set twice!", args[i]); - - final long[] skipTake = nextArgAsLongPair(args, i); - config.skip = skipTake[0]; - config.take = skipTake[1]; - break; - case "-prf": - case "--progress-restore-file": - if(config.progressRestoreFile != null) - throw new CLIException("Progress restore file cannot be set twice!", args[i]); - - config.progressRestoreFile = nextArgAsRegularFileReference(args, i); - break; - case "-spi": - case "--save-progress-interval": - config.progressWriteInterval = nextArgAsInt(args, i); - break; - case "-pof": - case "--progress-output-file": - config.progressOutputFile = nextArgAsPath(args, i); - break; - case "-i": - case "--inputs": - if(config.inputListFile != null) - throw new CLIException("Input file cannot be set twice!", args[i]); - config.inputListFile = nextArgAsRegularFileReference(args, i); - break; - case "-t": - case "--threads": - final int threads = nextArgAsInt(args, i); - config.multipleThreads = threads > 1; - config.threadCount = threads; - break; - default: - throw new CLIException(args[i]); - } - } - - /** - * Class representing the configuration of a {@link org.tudo.sse.analyses.MavenCentralLibraryAnalysis} instance. - */ - public static class CommonConfig { - - /** - * Number of artifacts to skip from the underlying source, or -1 if disabled. - */ - public long skip; - - /** - * Number of artifacts to take from the underlying source, or -1 if no limit shall be applied. - */ - public long take; - - /** - * Path to read inputs from, instead of accessing the Maven Central index. Null if the index shall be used. - */ - public Path inputListFile; - - /** - * File path to output analysis progress to. - */ - public Path progressOutputFile; - - /** - * Path to read previous progress from, in order to restore it. Null if disabled. - */ - public Path progressRestoreFile; - - /** - * True if multiple threads shall be used, false for single-threaded analysis. - */ - public boolean multipleThreads; - - /** - * Number of threads to use. - */ - public int threadCount; - - /** - * Interval of entities after which progress shall be saved. Defaults to 100. - */ - public int progressWriteInterval; - - /** - * Creates a new configuration with default values. - */ - public CommonConfig() { - skip = -1L; - take = -1L; - inputListFile = null; - progressOutputFile = Paths.get("marin-progress"); - progressRestoreFile = null; - multipleThreads = false; - threadCount = 1; - progressWriteInterval = 100; - } - } -} diff --git a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java b/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java deleted file mode 100644 index c769515..0000000 --- a/src/main/java/org/tudo/sse/utils/FileBasedArtifactIterator.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.tudo.sse.utils; - -import org.tudo.sse.model.ArtifactIdent; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Iterator; - -/** - * Iterator that reads Maven Central artifact identifiers from a text file. Expects one colon-separated triple of - * groupID:artifactID:version per line. - */ -public class FileBasedArtifactIterator implements Iterator { - - private final Path inputPath; - - private Iterator lineIterator; - - /** - * Creates a new iterator instance for the given file path. - * @param input Path to a text file containing GAV triples - */ - public FileBasedArtifactIterator(Path input){ - this.inputPath = input; - } - - @Override - public boolean hasNext() { - if(this.lineIterator == null){ - try { - var lines = Files.readAllLines(this.inputPath); - this.lineIterator = lines.iterator(); - } catch (IOException iox){ - throw new IllegalArgumentException("Failed to access input file", iox); - } - } - - return this.lineIterator.hasNext(); - } - - @Override - public ArtifactIdent next() { - String line = this.lineIterator.next(); - - String[] parts = line.split(":"); - if(parts.length == 3) { - return new ArtifactIdent(parts[0], parts[1], parts[2]); - } else { - throw new IllegalStateException("Not a valid GAV-Triple: " + line); - } - } -} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 70baa56..b5fb274 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -5,12 +5,13 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; import org.tudo.sse.model.Artifact; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.Package; import org.tudo.sse.model.pom.License; import org.tudo.sse.model.pom.PomInformation; -import org.tudo.sse.utils.ArtifactConfigParser; +import org.tudo.sse.analyses.config.parsing.ArtifactAnalysisConfigParser; import org.tudo.sse.utils.IndexIterator; import org.tudo.sse.utils.MavenCentralAnalysisFactory; import scala.Tuple2; @@ -43,11 +44,11 @@ class MavenCentralArtifactAnalysisTest { void parseCLIRegular() throws IOException{ Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{ parseCLI("--skip-take 500:223"), parseCLI("--inputs src/test/resources/localPom.xml"), parseCLI("--progress-restore-file src/test/resources/localPom.xml --inputs src/test/resources/localPom.xml"), - parseCLI("--since-until 53245:13243"), + parseCLI("--since-until 53245:53246"), parseCLI("--inputs src/test/resources/localPom.xml --progress-restore-file src/test/resources/localPom.xml"), parseCLI("--progress-restore-file src/test/resources/localPom.xml"), parseCLI("--output " + tmpDir.toString()) @@ -57,7 +58,7 @@ void parseCLIRegular() throws IOException{ for(int i = 0; i < configs.length; i++){ final List currExpected = expected.get(i); - final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + final ArtifactAnalysisConfig currConfig = configs[i]; assertEquals(asInt(currExpected.get(0)), currConfig.skip); assertEquals(asInt(currExpected.get(1)), currConfig.take); @@ -89,11 +90,11 @@ void parseCLIRegular() throws IOException{ void parseCLIShorthands() throws IOException{ Path tmpDir = Files.createTempDirectory("maven-resolution-files"); - ArtifactConfigParser.ArtifactConfig[] configs = new ArtifactConfigParser.ArtifactConfig[]{ + ArtifactAnalysisConfig[] configs = new ArtifactAnalysisConfig[]{ parseCLI("-st 500:223"), parseCLI("-i src/test/resources/localPom.xml"), parseCLI("-prf src/test/resources/localPom.xml -i src/test/resources/localPom.xml"), - parseCLI("-su 53245:13243"), + parseCLI("-su 53245:53246"), parseCLI("-i src/test/resources/localPom.xml -prf src/test/resources/localPom.xml"), parseCLI("-prf src/test/resources/localPom.xml"), parseCLI("-o " + tmpDir.toString()) @@ -103,7 +104,7 @@ void parseCLIShorthands() throws IOException{ for(int i = 0; i < configs.length; i++){ final List currExpected = expected.get(i); - final ArtifactConfigParser.ArtifactConfig currConfig = configs[i]; + final ArtifactAnalysisConfig currConfig = configs[i]; assertEquals(asInt(currExpected.get(0)), currConfig.skip); assertEquals(asInt(currExpected.get(1)), currConfig.take); @@ -138,6 +139,7 @@ void parseCmdLineNegative() { cliInputs.add("--inputs -/xcd/"); cliInputs.add("-st 88880000 -su --inputs path/to/file -ip path/to/file"); cliInputs.add("--inputs"); + cliInputs.add("--since-until 53245:13243"); for(String input : cliInputs) { @@ -236,17 +238,17 @@ void indexProcessorProgress() { @DisplayName("An analysis must correctly apply CLI options when reading custom input lists") void readIdentsIn() { List cliInputs = new ArrayList<>(); - String[] args = {"--inputs", "src/main/resources/coordinates.txt"}; + String[] args = {"--inputs", "src/test/resources/artifact-names-valid.txt"}; cliInputs.add(args); - args = new String[] {"--inputs", "src/main/resources/coordinates.txt", "-pof", "src/test/resources/stop.txt"}; + args = new String[] {"--inputs", "src/test/resources/artifact-names-valid.txt", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-st", "4:5", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt"}; cliInputs.add(args); - args = new String[]{"--inputs", "src/main/resources/coordinates.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; + args = new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt", "-prf", "src/test/resources/testingIndexPosition.txt", "-pof", "src/test/resources/stop.txt"}; cliInputs.add(args); List> expected = (List>) json.get("readIdentsIn"); @@ -290,8 +292,8 @@ void checkMultiThreading() { singleArgs.add(new String[]{"-st", "10:1000"}); multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"}); - singleArgs.add(new String[]{"--inputs", "src/main/resources/coordinates.txt"}); - multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/main/resources/coordinates.txt"}); + singleArgs.add(new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt"}); + multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/test/resources/artifact-names-valid.txt"}); for(int i = 0; i < singleArgs.size(); i++) { @@ -383,9 +385,9 @@ private long getEndingIndex(Path fileName) { } } - private ArtifactConfigParser.ArtifactConfig parseCLI(String cli) { + private ArtifactAnalysisConfig parseCLI(String cli) { try { - final ArtifactConfigParser parser = new ArtifactConfigParser(); + final ArtifactAnalysisConfigParser parser = new ArtifactAnalysisConfigParser(); if(cli.isBlank()) return parser.parseArtifactConfig(new String[] {}); else return parser.parseArtifactConfig(cli.split(" ")); } catch(CLIException clix){ diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java new file mode 100644 index 0000000..438bef6 --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactIteratorTest.java @@ -0,0 +1,203 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.ArtifactIdent; +import org.tudo.sse.utils.IndexIterator; +import org.tudo.sse.utils.MarinOpalLogger; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; + +public class MavenCentralArtifactIteratorTest { + + private final Path gavInputList = testResource("artifact-names-valid.txt"); + + @BeforeAll + static void setUp() { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + + @Test + @DisplayName("The artifact iterator must process all inputs from custom GAV lists") + void readFromFile() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, false, true, config); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertFalse(current.hasIndexInformation()); + assertTrue(current.hasPomInformation()); + assertTrue(current.hasJarInformation()); + + artifactCount++; + } + + assertEquals(10, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must apply pagination to custom GAV lists") + void readFromFilePaginated() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(gavInputList) + .withSkip(2) + .withTake(5) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, true, false, config); + + ArtifactIdent firstEntry = getIdentInInputFile(0); + ArtifactIdent secondEntry = getIdentInInputFile(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertFalse(current.hasIndexInformation()); + assertTrue(current.hasPomInformation()); + assertFalse(current.hasJarInformation()); + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getIdent()); + assertNotEquals(secondEntry, current.getIdent()); + + artifactCount++; + } + + assertEquals(5, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must not produce any values on invalid input files") + void readFromFileInvalid() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withInputList(testResource("artifact-names-invalid.txt")) + .build(); + + var iterator = new MavenCentralArtifactIterator(true, true, true, config); + + // If the source file contains at least one line that is not a valid GAV triple, it must not produce any elements + assertFalse(iterator.hasNext()); + assertThrows(IllegalStateException.class, iterator::next); + } + + @Test + @DisplayName("The artifact iterator must apply pagination to the Central index") + void readFromIndexPaginated() throws InvalidConfigurationException { + var config = new ArtifactAnalysisConfigBuilder() + .withSkip(2) + .withTake(5) + .build(); + + var iterator = new MavenCentralArtifactIterator(false, false, true, config); + + ArtifactIdent firstEntry = getIdentInIndex(0); + ArtifactIdent secondEntry = getIdentInIndex(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + Artifact current = iterator.next(); + assertTrue(current.hasIndexInformation()); + assertFalse(current.hasPomInformation()); + assertTrue(current.hasJarInformation()); + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getIdent()); + assertNotEquals(secondEntry, current.getIdent()); + + artifactCount++; + } + + assertEquals(5, artifactCount); + } + + @Test + @DisplayName("The artifact iterator must apply time range filtering to the Central index") + void readFromIndexFiltered() throws InvalidConfigurationException { + // We expect two entries in this range + long since = 1106902436000L; + long until = 1106902438001L; + + var config = new ArtifactAnalysisConfigBuilder() + .withSinceUtil(since, until) + .build(); + + var iterator = new MavenCentralArtifactIterator(false, false, false, config); + + int artifactCount = 0; + + while(iterator.hasNext() && artifactCount < 2){ + Artifact current = iterator.next(); + assertTrue(current.hasIndexInformation()); + assertFalse(current.hasPomInformation()); + assertFalse(current.hasJarInformation()); + + assertTrue(since <= current.getIndexInformation().getLastModified()); + assertTrue(current.getIndexInformation().getLastModified() <= until); + + assertEquals("ymsg", current.getIdent().getGroupID()); + + artifactCount++; + } + } + + private ArtifactIdent getIdentInInputFile(int pos) { + try { + var lines = Files.readAllLines(gavInputList); + + if(pos >= lines.size()) fail("Invalid position in input list: " + pos); + + var parts = lines.get(pos).split(":"); + + if(parts.length != 3) fail("Invalid GAV in input list: " + pos); + + return new ArtifactIdent(parts[0], parts[1], parts[2]); + } catch (IOException iox) { + fail(iox); + } + + return null; + } + + private ArtifactIdent getIdentInIndex(int pos){ + try { + IndexIterator ii = new IndexIterator(MavenCentralRepository.RepoBaseURI); + + int position = 0; + + while(ii.hasNext()){ + var ident = ii.next().getIdent(); + if(position == pos){ + return ident; + } + position++; + } + + ii.closeReader(); + + } catch (IOException iox) { + fail(iox); + } + + return null; + } +} diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index e5c120b..8ca7d1f 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -3,8 +3,9 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.tudo.sse.CLIException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfig; import org.tudo.sse.model.Artifact; -import org.tudo.sse.utils.CommonConfigParser; +import org.tudo.sse.analyses.config.parsing.LibraryAnalysisConfigParser; import java.io.IOException; import java.nio.file.Files; @@ -15,6 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; public class MavenCentralLibraryAnalysisTest { @@ -22,7 +24,7 @@ public class MavenCentralLibraryAnalysisTest { @DisplayName("The CLI parser must parse common argument values correctly") void parseCLIRegular(){ final String validArgs = "--skip-take 20:10000 --progress-restore-file pom.xml --progress-output-file marin-progress --threads 12 --save-progress-interval 20"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assert(config.multipleThreads); assertEquals(12, config.threadCount); @@ -38,7 +40,7 @@ void parseCLIRegular(){ @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") void parseCLIShorthands(){ final String validArgs = "-st 20:10000 -prf pom.xml -pof marin-progress -t 12 -spi 20"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assert(config.multipleThreads); assertEquals(12, config.threadCount); @@ -55,7 +57,7 @@ void parseCLIShorthands(){ @Test @DisplayName("The CLI parser must set the correct defaults for empty arguments") void parseCLIDefaults(){ - final CommonConfigParser.CommonConfig config = parseCLI(""); + final LibraryAnalysisConfig config = parseCLI(""); assertFalse(config.multipleThreads); assertEquals(1, config.threadCount); @@ -80,7 +82,7 @@ void parseNonExistingFile(){ void parseCustomGA(){ final String validArgs = "-st 20:10000 --inputs pom.xml"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assertEquals(Paths.get("pom.xml"), config.inputListFile); assertEquals(20, config.skip); @@ -91,7 +93,7 @@ void parseCustomGA(){ void parseNonExistingIndex(){ final String validArgs = "--progress-restore-file pom.xml --inputs pom.xml"; - final CommonConfigParser.CommonConfig config = parseCLI(validArgs); + final LibraryAnalysisConfig config = parseCLI(validArgs); assertEquals(Paths.get("pom.xml"), config.inputListFile); assertEquals(Paths.get("pom.xml"), config.progressRestoreFile); @@ -327,19 +329,11 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertFalse(librariesHit.contains("yom:yom")); } - private Path testResource(String pathToResource){ - try { - return Path.of(Objects.requireNonNull(getClass().getClassLoader().getResource(pathToResource)).toURI()); - } catch (Exception x){ - fail("Test setup: Failed to load resource file " + pathToResource, x); - } - return null; - } - private CommonConfigParser.CommonConfig parseCLI(String cli) { + private LibraryAnalysisConfig parseCLI(String cli) { try { - if(cli.isBlank()) return new CommonConfigParser().parseCommonConfig(new String[]{}); - else return new CommonConfigParser().parseCommonConfig(cli.split(" ")); + if(cli.isBlank()) return new LibraryAnalysisConfigParser().parseCommonConfig(new String[]{}); + else return new LibraryAnalysisConfigParser().parseCommonConfig(cli.split(" ")); } catch(CLIException clix) { throw new RuntimeException(clix); } diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java new file mode 100644 index 0000000..47daa0b --- /dev/null +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryIteratorTest.java @@ -0,0 +1,229 @@ +package org.tudo.sse.analyses; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opalj.log.GlobalLogContext$; +import org.opalj.log.OPALLogger; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.analyses.config.LibraryAnalysisConfigBuilder; +import org.tudo.sse.model.Artifact; +import org.tudo.sse.model.LibraryResolutionContext; +import org.tudo.sse.resolution.FileNotFoundException; +import org.tudo.sse.utils.LibraryIndexIterator; +import org.tudo.sse.utils.MarinOpalLogger; +import org.tudo.sse.utils.MavenCentralRepository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; + +public class MavenCentralLibraryIteratorTest { + + private static final Path validGAList = testResource("library-names-valid.txt"); + private static final Path invalidGAList = testResource("library-names-invalid.txt"); + private static final Path nonExistingGAList = testResource("library-names-not-existing.txt"); + + @BeforeAll + static void setUp() { + // Use global OPAL Logger - will only forward OPAL messages with level error or fatal + OPALLogger.updateLogger(GlobalLogContext$.MODULE$, MarinOpalLogger.getGlobalLogger()); + } + + @Test + @DisplayName("The library iterator must process all inputs from custom GA lists") + void readFromFile() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(validGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + int libraryCount = 0; + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertTrue(release.hasPomInformation()); + assertFalse(release.hasJarInformation()); + } + + + libraryCount++; + } + + assertEquals(5, libraryCount); + } + + @Test + @DisplayName("The library iterator must apply pagination to custom GA lists") + void readFromFilePaginated() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(validGAList) + .withTake(2) + .withSkip(2) + .build(); + + var iterator = new MavenCentralLibraryIterator(false, false, true, config); + + int libraryCount = 0; + + String firstEntry = getGaInInputFile(0); + String secondEntry = getGaInInputFile(1); + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertFalse(release.hasPomInformation()); + assertTrue(release.hasJarInformation()); + } + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getLibraryGA()); + assertNotEquals(secondEntry, current.getLibraryGA()); + + libraryCount++; + } + + assertEquals(2, libraryCount); + } + + @Test + @DisplayName("The library iterator must not produce any values on invalid input files") + void readFromFileInvalid() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(invalidGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, true, true, config); + + // If the source file contains at least one line that is not a valid GA tuple, it must not produce any elements + assertFalse(iterator.hasNext()); + assertThrows(IllegalStateException.class, iterator::next); + } + + @Test + @DisplayName("The library iterator must apply pagination to the Central index") + void readFromIndexPaginated() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withSkip(2) + .withTake(3) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + String firstEntry = getGaInIndex(0); + String secondEntry = getGaInIndex(1); + + int artifactCount = 0; + + while(iterator.hasNext()){ + final LibraryResolutionContext current = iterator.next(); + final List releases = current.getLibraryArtifacts(); + + assertFalse(releases.isEmpty()); + + for(Artifact release: releases){ + assertFalse(release.hasIndexInformation()); + assertTrue(release.hasPomInformation()); + assertFalse(release.hasJarInformation()); + } + + // Assert that we are not seeing the skipped entries + assertNotEquals(firstEntry, current.getLibraryGA()); + assertNotEquals(secondEntry, current.getLibraryGA()); + + artifactCount++; + } + + assertEquals(3, artifactCount); + } + + @Test + @DisplayName("The library iterator must invoke the registered callback on failure") + void setCallback() throws InvalidConfigurationException { + var config = new LibraryAnalysisConfigBuilder() + .withInputList(nonExistingGAList) + .build(); + + var iterator = new MavenCentralLibraryIterator(true, false, false, config); + + // Define a callback for non-existing release lists + AtomicBoolean callbackInvoked = new AtomicBoolean(false); + + iterator.setFailedLibraryCallback( event -> { + assertEquals("no", event.getGroupId()); + assertEquals("lib", event.getArtifactId()); + + Throwable rootCause = event.getCause(); + while(rootCause.getCause() != null){ + rootCause = rootCause.getCause(); + } + + assertInstanceOf(FileNotFoundException.class, rootCause); + + callbackInvoked.set(true); + }); + + while(iterator.hasNext()){ + LibraryResolutionContext ctx = iterator.next(); + assertTrue(ctx.getLibraryArtifacts().isEmpty()); + } + + assertTrue(callbackInvoked.get()); + } + + + private String getGaInIndex(int pos) { + try { + LibraryIndexIterator libIt = new LibraryIndexIterator(MavenCentralRepository.RepoBaseURI); + + int position = 0; + while(libIt.hasNext()){ + var libGA = libIt.next(); + if(position == pos){ + return libGA; + } + position++; + } + + libIt.close(); + } catch(IOException iox) { + fail(iox); + } + + return null; + } + + private String getGaInInputFile(int pos) { + try { + assertNotNull(validGAList); + + var lines = Files.readAllLines(validGAList); + + if(pos >= lines.size()) fail("Invalid position in input list: " + pos); + + return lines.get(pos); + } catch (IOException iox) { + fail(iox); + } + + return null; + } +} diff --git a/src/test/java/org/tudo/sse/utils/TestUtilities.java b/src/test/java/org/tudo/sse/utils/TestUtilities.java new file mode 100644 index 0000000..7e7a686 --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/TestUtilities.java @@ -0,0 +1,19 @@ +package org.tudo.sse.utils; + +import java.nio.file.Path; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.fail; + +public class TestUtilities { + + public static Path testResource(String pathToResource){ + try { + return Path.of(Objects.requireNonNull(TestUtilities.class.getClassLoader().getResource(pathToResource)).toURI()); + } catch (Exception x){ + fail("Test setup: Failed to load resource file " + pathToResource, x); + } + return null; + } + +} diff --git a/src/test/resources/MavenAnalysis.json b/src/test/resources/MavenAnalysis.json index c7e9edd..8879f2b 100644 --- a/src/test/resources/MavenAnalysis.json +++ b/src/test/resources/MavenAnalysis.json @@ -31,7 +31,7 @@ "-1", "-1", "53245", - "13243", + "53246", "null", "null", "false" diff --git a/src/test/resources/artifact-names-invalid.txt b/src/test/resources/artifact-names-invalid.txt new file mode 100644 index 0000000..827f980 --- /dev/null +++ b/src/test/resources/artifact-names-invalid.txt @@ -0,0 +1,10 @@ +org.springframework.boot:spring-boot-starter-web:2.6.3 +org.apache.commons:commons-lang3:3.12.0 +com.fasterxml.jackson.core:jackson-databind +junit:junit:4.13.2 +org.slf4j:slf4j-api:1.7.32 +org.hibernate:hibernate-core:5.6.5.Final +org.apache.httpcomponents:httpclient:4.5.13 +org.springframework:spring-core:5.3.15 +mysql:mysql-connector-java:8.0.28:: +org.apache.logging.log4j:log4j-core:2.17.1 \ No newline at end of file diff --git a/src/main/resources/coordinates.txt b/src/test/resources/artifact-names-valid.txt similarity index 100% rename from src/main/resources/coordinates.txt rename to src/test/resources/artifact-names-valid.txt diff --git a/src/test/resources/library-names-not-existing.txt b/src/test/resources/library-names-not-existing.txt new file mode 100644 index 0000000..300a466 --- /dev/null +++ b/src/test/resources/library-names-not-existing.txt @@ -0,0 +1 @@ +no:lib