From e8ca3aa8ee4e03109a760cbf0aa48db9935dbfa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 12 Jan 2026 10:44:51 +0100 Subject: [PATCH 1/5] Accept YYYY-MM-DD format for cutoff dates via CLI. Change UNIX timestamp format from milliseconds to seconds. Update documentation. --- README.md | 20 +++--- .../config/ArtifactAnalysisConfig.java | 4 +- .../parsing/ArtifactAnalysisConfigParser.java | 52 ++++++++++++++- .../config/parsing/CLIParsingUtilities.java | 22 +++++++ .../MavenCentralArtifactAnalysisTest.java | 63 ++++++++++++++++++- 5 files changed, 146 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b8a20ab..829e4b3 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,16 @@ Additionally, the `MavenCentralArtifactAnalysis` provides the `resolveIndex` opt Both analysis types provide a method called `void runAnalysis(String[] args)`. Invoking this method will start the analysis. By default, MARIN analyses will support the following CLI parameters: -| **Argument** | **Artifact Analysis** | **Library Analysis** | **Description** | **Example** | -|------------------------------------------------------------|-----------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| -| `-st :`
`--skip-take :` | Yes | Yes | Skips the first `n` inputs and then only
processes the next `t` ones. | `-st 200:10` | -| `-su :`
`--since-until :` | Yes | No | Skips artifacts released before the
timestamp `s` or after timestamp `t`. | | -| `-i `
`--inputs ` | Yes | Yes | Processes inputs from a file instead of the
Maven Central index. Expects on input
per line. Inputs are either G:A:V triple (artifacts)
or G:A tuple (libraries). | `-i libraries.txt` | -| `-pof `
`--progress-output-file ` | Yes | Yes | File to periodically write number of processed
artifacts to. Defaults to `./marin-progress`. | `-pof marin-progress` | -| `-prf `
`--progress-restore-file ` | Yes | Yes | File to load a previous run's progress from.
Inputs will be skipped until progress is restored.
Not used by default. | `-prf marin-progress` | -| `-spi `
`--save-progress-interval ` | Yes | Yes | Number of inputs after which to store progress.
Defaults to 100. | `-spi 10` | -| `-t `
`--threads ` | Yes | Yes | Number of threads to use. Defaults to 1. | `-t 8` | -| `-o `
`--output ` | Yes | No | Output directory to optionally write processed
artifacts to. Depending on the artifact information
required by the analysis, this can be `pom.xml`
files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | +| **Argument** | **Artifact Analysis** | **Library Analysis** | **Description** | **Example** | +|------------------------------------------------------------|-----------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------| +| `-st :`
`--skip-take :` | Yes | Yes | Skips the first `` inputs and then only processes the next `` ones. | `-st 200:10` | +| `-su :`
`--since-until :` | Yes | No | Skips artifacts released before the timestamp `` or after timestamp ``. A timestamp is either a UNIX timestamp (in seconds) or a date of format YYYY-MM-DD. Note that `` is automatically parsed as end-of-day if YYYY-MM-DD is used. | `-su 1768209126:2026-02-02` | +| `-i `
`--inputs ` | Yes | Yes | Processes inputs from a file instead of the Maven Central index. Expects on input per line. Inputs are either G:A:V triple (artifacts) or G:A tuple (libraries). | `-i libraries.txt` | +| `-pof `
`--progress-output-file ` | Yes | Yes | File to periodically write number of processed artifacts to. Defaults to `./marin-progress`. | `-pof marin-progress` | +| `-prf `
`--progress-restore-file ` | Yes | Yes | File to load a previous run's progress from. Inputs will be skipped until progress is restored. Not used by default. | `-prf marin-progress` | +| `-spi `
`--save-progress-interval ` | Yes | Yes | Number of inputs after which to store progress. Defaults to 100. | `-spi 10` | +| `-t `
`--threads ` | Yes | Yes | Number of threads to use. Defaults to 1. | `-t 8` | +| `-o `
`--output ` | Yes | No | Output directory to optionally write processed artifacts to. Depending on the artifact information required by the analysis, this can be `pom.xml` files, `*.jar` files or GAV triple. Not used by default. | `-o ./jars-processed/` | If you do not want to extend one of the aforementioned base classes, MARIN also provides corresponding implementations of the `java.util.Iterator` interface to enumerate and enrich artifacts or libraries. Note that those implementations are **not threadsafe** and cannot perform resolution in parallel. MARIN provides: diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java index f2f3dd8..96123ec 100644 --- a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfig.java @@ -14,12 +14,12 @@ public class ArtifactAnalysisConfig extends LibraryAnalysisConfig { static final Path DEFAULT_VALUE_OUTPUT = null; /** - * Timestamp before which artifacts shall be excluded from analysis, or -1 if disabled. + * UNIX timestamp (in milliseconds) before which artifacts shall be excluded from analysis, or -1 if disabled. */ public long since; /** - * Timestamp after which artifacts shall be excluded from analysis, or -1 if disabled. + * UNIX timestamp (in milliseconds) after which artifacts shall be excluded from analysis, or -1 if disabled. */ public long until; diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java index 6b46233..0c7d04c 100644 --- a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -5,6 +5,15 @@ import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; import org.tudo.sse.analyses.config.InvalidConfigurationException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.Date; + /** * Parser for creating {@link ArtifactAnalysisConfig} objects from CLI parameters. * Configuration is obtained by parsing command line arguments provided as an array of strings. @@ -40,8 +49,12 @@ private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfi switch(args[i]) { case "-su": case "--since-until": - final long[] sinceUntil = nextArgAsLongPair(args, i); - configBuilder.withSinceUtil(sinceUntil[0], sinceUntil[1]); + final String[] sinceUntil = nextArgAsStringPair(args, i); + + long since = toUnixTimeStampMillis(sinceUntil[0], "since", false); + long until = toUnixTimeStampMillis(sinceUntil[1], "until", true); + + configBuilder.withSinceUtil(since, until); break; case "-o": case "--output": @@ -59,4 +72,39 @@ private void handleArtifactParameter(String[] args, int i, ArtifactAnalysisConfi } } + private long toUnixTimeStampMillis(String value, String attrName, boolean useEndOfDay) throws CLIException { + ZonedDateTime date; + + try { + long timestamp = Long.parseLong(value); + Instant s = Instant.ofEpochSecond(timestamp); + date = s.atZone(ZoneId.systemDefault()); + } catch (NumberFormatException ignored){ + date = parseYYYYMMDD(value, attrName); + // Parsing a date will create a (local) time of 00:00:00 - we want to add one day if requested + if(useEndOfDay) + date = date.plusHours(23).plusMinutes(59).plusSeconds(59); + } catch (DateTimeException dtx){ + throw new CLIException("Not a valid UNIX timestamp", attrName); + } + + // Do a plausibility check on the given time stamp + if(date.getYear() < 1950 || date.getYear() > 2100) + throw new CLIException("Cutoff dates must fall between the years 1950 and 2100"); + + return date.toInstant().toEpochMilli(); + } + + private ZonedDateTime parseYYYYMMDD(String value, String attrName) throws CLIException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + try { + return sdf.parse(value).toInstant().atZone(ZoneId.systemDefault()); + } catch (ParseException px){ + var exception = new CLIException("Not a valid date of format YYYY-MM-DD", attrName); + exception.initCause(px); + throw exception; + } + } + } diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java index 2c175d1..66dc4ca 100644 --- a/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/CLIParsingUtilities.java @@ -60,6 +60,28 @@ default long[] nextArgAsLongPair(String[] args, int i) throws CLIException { return toReturn; } + /** + * Parses the next argument from the given array of arguments as a pair of string values, separated by a colon. The + * pair is returned as a string-array of size 2. + * @param args The CLI arguments + * @param i The current parsing position + * @return The next argument at position i+1 as a pair of string values + * @throws CLIException If there is no next argument, or it is not formatted as two string values separated by a colon. + */ + default String[] nextArgAsStringPair(String[] args, int i) throws CLIException { + if(i + 1 < args.length) { + String[] toReturn = args[i + 1].split(":"); + + if(toReturn.length != 2) { + throw new CLIException("Expected a colon-separated pair of strings", args[i]); + } + + return toReturn; + } else { + throw(new CLIException("Missing argument: first:second", args[i])); + } + } + /** * Parses the next argument from the given array of arguments as a Path value. * diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index b5fb274..92f8030 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -21,6 +21,9 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.*; import static org.junit.jupiter.api.Assertions.*; @@ -85,6 +88,61 @@ void parseCLIRegular() throws IOException{ } } + @Test + @DisplayName("The CLI parser must parse timestamps YYYY-MM-DD format") + void parseCLITimestamps1() { + var config = parseCLI("-su 2025-12-01:2025-12-31"); + + var since = asDate(config.since); + var until = asDate(config.until); + + assertEquals(2025, since.getYear()); + assertEquals(12, since.getMonthValue()); + assertEquals(1, since.getDayOfMonth()); + assertEquals(0, since.getHour()); + assertEquals(0, since.getMinute()); + + assertEquals(2025, until.getYear()); + assertEquals(12, until.getMonthValue()); + assertEquals(31, until.getDayOfMonth()); + assertEquals(23, until.getHour()); + assertEquals(59, until.getMinute()); + } + + @Test + @DisplayName("The CLI parser must parse UNIX timestamps") + void parseCLITimestamps2() { + var config = parseCLI("-su 2010-10-10:1321006271"); + + var since = asDate(config.since); + var until = asDate(config.until); + + assertEquals(2010, since.getYear()); + assertEquals(10, since.getMonthValue()); + assertEquals(10, since.getDayOfMonth()); + assertEquals(0, since.getHour()); + assertEquals(0, since.getMinute()); + + assertEquals(2011, until.getYear()); + assertEquals(11, until.getMonthValue()); + assertEquals(11, until.getDayOfMonth()); + assertEquals(11, until.getHour()); + assertEquals(11, until.getMinute()); + assertEquals(11, until.getSecond()); + } + + @Test + @DisplayName("The CLI parser must reject invalid ranges") + void parseCLIInvalidRanges() { + try { + parseCLI("-su 1321006271:2010-10-10"); + fail("The CLI parser must reject invalid ranges"); + } catch(Exception x){ + assertInstanceOf(RuntimeException.class, x); + assertInstanceOf(CLIException.class, x.getCause()); + } + } + @Test @DisplayName("The CLI parser must parse common argument values correctly using shorthand argument names") void parseCLIShorthands() throws IOException{ @@ -400,5 +458,8 @@ private int asInt(String s){ return Integer.parseInt(s); } - + private ZonedDateTime asDate(long timestamp){ + Instant i = Instant.ofEpochMilli(timestamp); + return ZonedDateTime.ofInstant(i, ZoneId.systemDefault()); + } } \ No newline at end of file From 9316849232df69c3d0e93ee9e37cdec064171800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 12 Jan 2026 11:15:00 +0100 Subject: [PATCH 2/5] Update documentation for since-until in config builder, add convenience method --- .../config/ArtifactAnalysisConfigBuilder.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java index d0e2a33..2b628cb 100644 --- a/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java +++ b/src/main/java/org/tudo/sse/analyses/config/ArtifactAnalysisConfigBuilder.java @@ -2,6 +2,7 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.time.ZonedDateTime; /** * Configuration builder to obtain {@link ArtifactAnalysisConfig} instances programmatically. @@ -104,8 +105,21 @@ public ArtifactAnalysisConfigBuilder withOutputDirectory(Path outDir) throws Inv * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have * been released after the timestamp given in 'since', and before 'until'. * - * @param since Timestamp marking the lower limit of the range - * @param until Timestamp marking the upper limit of the range + * @param since DateTime marking the lower limit of the range + * @param until DateTime marking the upper limit of the range + * @return The current configuration builder instance + * @throws InvalidConfigurationException If the given configuration values are not valid + */ + public ArtifactAnalysisConfigBuilder withSinceUntil(ZonedDateTime since, ZonedDateTime until) throws InvalidConfigurationException { + return this.withSinceUtil(since.toInstant().toEpochMilli(), until.toInstant().toEpochMilli()); + } + + /** + * Sets a time-based range to filter artifacts for. The resulting analysis will only process artifacts that have + * been released after the timestamp given in 'since', and before 'until'. + * + * @param since UNIX timestamp in milliseconds marking the lower limit of the range + * @param until UNIX timestamp in milliseconds marking the upper limit of the range * @return The current configuration builder instance * @throws InvalidConfigurationException If the given configuration values are not valid */ From b28a4b2e45a36d856945dd54c0cf9b4089af5c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Mon, 12 Jan 2026 14:46:56 +0100 Subject: [PATCH 3/5] Fix dependence on platform time zone --- .../parsing/ArtifactAnalysisConfigParser.java | 18 +++++++++--------- .../MavenCentralArtifactAnalysisTest.java | 6 ++++-- src/test/resources/MavenAnalysis.json | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java index 0c7d04c..86d6cde 100644 --- a/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java +++ b/src/main/java/org/tudo/sse/analyses/config/parsing/ArtifactAnalysisConfigParser.java @@ -7,10 +7,8 @@ import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.DateTimeException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; +import java.time.*; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.Date; @@ -20,6 +18,8 @@ */ public class ArtifactAnalysisConfigParser extends LibraryAnalysisConfigParser implements CLIParsingUtilities { + private static final ZoneId GMT_ZONE = ZoneId.of("GMT"); + /** * Creates a new artifact config parser instance. */ @@ -78,7 +78,7 @@ private long toUnixTimeStampMillis(String value, String attrName, boolean useEnd try { long timestamp = Long.parseLong(value); Instant s = Instant.ofEpochSecond(timestamp); - date = s.atZone(ZoneId.systemDefault()); + date = s.atZone(GMT_ZONE); } catch (NumberFormatException ignored){ date = parseYYYYMMDD(value, attrName); // Parsing a date will create a (local) time of 00:00:00 - we want to add one day if requested @@ -96,13 +96,13 @@ private long toUnixTimeStampMillis(String value, String attrName, boolean useEnd } private ZonedDateTime parseYYYYMMDD(String value, String attrName) throws CLIException { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd"); try { - return sdf.parse(value).toInstant().atZone(ZoneId.systemDefault()); - } catch (ParseException px){ + return LocalDate.parse(value, dtf).atStartOfDay(GMT_ZONE); + } catch (DateTimeParseException dtpx) { var exception = new CLIException("Not a valid date of format YYYY-MM-DD", attrName); - exception.initCause(px); + exception.initCause(dtpx); throw exception; } } diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index 92f8030..323c7de 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -112,11 +112,13 @@ void parseCLITimestamps1() { @Test @DisplayName("The CLI parser must parse UNIX timestamps") void parseCLITimestamps2() { - var config = parseCLI("-su 2010-10-10:1321006271"); + var config = parseCLI("-su 2010-10-10:1321009871"); var since = asDate(config.since); var until = asDate(config.until); + assertEquals(1321009871000L, config.until); + assertEquals(2010, since.getYear()); assertEquals(10, since.getMonthValue()); assertEquals(10, since.getDayOfMonth()); @@ -460,6 +462,6 @@ private int asInt(String s){ private ZonedDateTime asDate(long timestamp){ Instant i = Instant.ofEpochMilli(timestamp); - return ZonedDateTime.ofInstant(i, ZoneId.systemDefault()); + return ZonedDateTime.ofInstant(i, ZoneId.of("GMT")); } } \ No newline at end of file diff --git a/src/test/resources/MavenAnalysis.json b/src/test/resources/MavenAnalysis.json index 8879f2b..8064827 100644 --- a/src/test/resources/MavenAnalysis.json +++ b/src/test/resources/MavenAnalysis.json @@ -30,8 +30,8 @@ [ "-1", "-1", - "53245", - "53246", + "53245000", + "53246000", "null", "null", "false" From 18ff1b050295cdc9e13089d856fa4c76919fac9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 6 Mar 2026 09:57:04 +0100 Subject: [PATCH 4/5] Fix test dependence on index ordering --- src/main/java/org/tudo/sse/IndexWalker.java | 244 -------------- .../analyses/MavenCentralLibraryAnalysis.java | 2 +- .../org/tudo/sse/utils/IndexIterator.java | 11 +- .../java/org/tudo/sse/IndexWalkerTest.java | 96 ------ .../MavenCentralArtifactAnalysisTest.java | 31 +- .../MavenCentralLibraryAnalysisTest.java | 47 ++- .../tudo/sse/resolution/PomResolverTest.java | 77 +++-- .../java/org/tudo/sse/utils/JsonExporter.java | 85 +++++ .../sse/utils/LibraryIndexIteratorTest.java | 16 - .../org/tudo/sse/utils/TestUtilities.java | 28 ++ src/test/resources/PomInputs.json | 314 +++++++++++++++--- 11 files changed, 489 insertions(+), 462 deletions(-) delete mode 100644 src/main/java/org/tudo/sse/IndexWalker.java delete mode 100644 src/test/java/org/tudo/sse/IndexWalkerTest.java create mode 100644 src/test/java/org/tudo/sse/utils/JsonExporter.java diff --git a/src/main/java/org/tudo/sse/IndexWalker.java b/src/main/java/org/tudo/sse/IndexWalker.java deleted file mode 100644 index 084831c..0000000 --- a/src/main/java/org/tudo/sse/IndexWalker.java +++ /dev/null @@ -1,244 +0,0 @@ -package org.tudo.sse; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tudo.sse.model.Artifact; -import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.index.IndexInformation; -import org.tudo.sse.model.ResolutionContext; -import org.tudo.sse.utils.IndexIterator; - -import java.net.URI; -import java.util.*; -import java.util.regex.Pattern; - -/** -* This class creates an object that manages retrieving indexes from the Maven Central repository. -* Either the Identifiers can be retrieved lazily or be fully parsed, utilizing the IndexIterator. -* This class also allows for the ability to walk all indexes or walk them in a paginated fashion. -*/ - -public class IndexWalker implements Iterable { - - /** - * The string pattern at which to split index entries. - */ - public static final String splitPattern = Pattern.quote("|"); - - private IndexIterator indexIterator; - private boolean resetIterator; - private final URI base; - - private static final Logger log = LoggerFactory.getLogger(IndexWalker.class); - - /** - * Creates a new IndexWalker with the given repository base URI. - * @param base The repository base URI. - */ - public IndexWalker(URI base) { - this.base = base; - resetIterator = true; - } - - /** - * Moves the iterator to a specified index. - * - * @param position which index to move to - * @throws IOException when there is an issue opening a file - */ - public void moveIterator(long position) throws IOException { - indexIterator = new IndexIterator(base, position); - this.resetIterator = false; - } - - /** - * Produces a list of all artifact identifiers for the entire repository index. - * @return List of artifact identifiers inside the repository - * @throws IOException If connection errors occur - */ - public List lazyWalkAllIndexes() throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - List idents = new ArrayList<>(); - while(indexIterator.hasNext()) { - idents.add(indexIterator.next().getIdent()); - } - - indexIterator.closeReader(); - return idents; - } - - /** - * Produces a list of all artifacts for the entire repository index. All artifacts are annotated with index information. - * @return List of artifacts (with index information) - * @throws IOException If connection errors occur - */ - public List walkAllIndexes() throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); - - List artifacts = new ArrayList<>(); - while(indexIterator.hasNext()) { - final IndexInformation indexInformation = indexIterator.next(); - final Artifact theArtifact = ctx.createArtifact(indexInformation.getIdent()); - - theArtifact.setIndexInformation(indexInformation); - - artifacts.add(theArtifact); - } - - if(artifacts.size() % 500000 == 0) { - log.info("{} artifacts have been parsed.", artifacts.size()); - } - indexIterator.closeReader(); - return artifacts; - } - - /** - * Produces a list of artifacts from the repository index with the given pagination values. - * @param skip Number of artifact identifiers to skip - * @param take Number of artifact identifiers to take - * @return List of artifact identifiers - * @throws IOException If connection errors occur - */ - public List lazyWalkPaginated(long skip, long take) throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - List idents = new ArrayList<>(); - - int count = 0; - int fromFront = 0; - while(indexIterator.hasNext() && count < take) { - if(fromFront >= skip) { - idents.add(indexIterator.next().getIdent()); - count++; - } else { - indexIterator.next(); - } - fromFront++; - } - indexIterator.closeReader(); - - return idents; - } - - /** - * Produces a list of artifact identifiers from the repository index with the given pagination values. All artifacts - * are annotated with index information. - * @param skip Number of artifacts to skip - * @param take Number of artifacts to take - * @return List of artifacts with index information - * @throws IOException If connection errors occur - */ - public List walkPaginated(long skip, long take) throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); - - List artifacts = new ArrayList<>(); - - int count = 0; - int fromFront = 0; - while(indexIterator.hasNext() && count < take) { - if(fromFront >= skip) { - final IndexInformation indexInformation = indexIterator.next(); - final Artifact theArtifact = ctx.createArtifact(indexInformation.getIdent()); - - theArtifact.setIndexInformation(indexInformation); - - artifacts.add(theArtifact); - count++; - } else { - indexIterator.next(); - } - fromFront++; - } - indexIterator.closeReader(); - return artifacts; - } - - /** - * Produces a list of artifact identifiers from the repository index within the given time bounds. - * @param since Timestamp marking the lower bound for release dates - * @param until Timestamp marking the upper bound for release dates - * @return List of artifact identifiers - * @throws IOException If connection errors occur - */ - public List walkDates(long since, long until) throws IOException { - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - - final ResolutionContext ctx = ResolutionContext.createAnonymousContext(); - - List artifacts = new ArrayList<>(); - - long currentToSince = 0; - long sinceToUntil = since; - while(indexIterator.hasNext() && sinceToUntil < until) { - final IndexInformation temp = indexIterator.next(); - if(currentToSince >= since) { - sinceToUntil = temp.getLastModified(); - - final Artifact theArtifact = ctx.createArtifact(temp.getIdent()); - theArtifact.setIndexInformation(temp); - - artifacts.add(theArtifact); - } else { - currentToSince = temp.getLastModified(); - } - } - indexIterator.closeReader(); - return artifacts; - } - - /** - * Produces a list of artifacts from the repository index within the given time bounds. All artifacts - * are annotated with index information. - * @param since Timestamp marking the lower bound for release dates - * @param until Timestamp marking the upper bound for release dates - * @return List of artifacts with index information - * @throws IOException If connection errors occur - */ - public List lazyWalkDates(long since, long until) throws IOException{ - if(resetIterator) { - indexIterator = new IndexIterator(base); - } - List idents = new ArrayList<>(); - - long currentToSince = 0; - long sinceToUntil = since; - while(indexIterator.hasNext() && sinceToUntil < until) { - IndexInformation temp = indexIterator.next(); - if(currentToSince >= since) { - sinceToUntil = temp.getLastModified(); - idents.add(temp.getIdent()); - } else { - currentToSince = temp.getLastModified(); - } - } - indexIterator.closeReader(); - - return idents; - } - - @Override - public Iterator iterator() { - try { - return new IndexIterator(base); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java index 40a3ec8..7032b55 100644 --- a/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java +++ b/src/main/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysis.java @@ -276,7 +276,7 @@ private List getReleaseIdentifiers(String groupId, String artifac } return identifiers; } catch (Exception x) { - log.warn("Failed to obtain version list for library {}:{}", groupId, artifactId, x); + log.warn("Failed to obtain version list for library {}:{} ({})", groupId, artifactId, x.getCause().getMessage()); try { this.onVersionListError(groupId, artifactId, x); } catch(Exception inner) { diff --git a/src/main/java/org/tudo/sse/utils/IndexIterator.java b/src/main/java/org/tudo/sse/utils/IndexIterator.java index 1fac507..75780a0 100644 --- a/src/main/java/org/tudo/sse/utils/IndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/IndexIterator.java @@ -3,7 +3,6 @@ import org.apache.maven.index.reader.IndexReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.tudo.sse.IndexWalker; import org.tudo.sse.model.ArtifactIdent; import org.tudo.sse.model.index.Package; import org.tudo.sse.model.index.IndexInformation; @@ -13,6 +12,7 @@ import java.net.URI; import java.util.Iterator; import java.util.Map; +import java.util.regex.Pattern; /** * This class creates an iterator for iterating over indexes and returning IndexArtifact objects. @@ -22,6 +22,9 @@ */ public class IndexIterator implements Iterator { + + public static final String splitPattern = Pattern.quote("|"); + private long index; private final URI baseUri; @@ -96,7 +99,7 @@ private void recoverConnectionReset() throws IOException{ * @see ArtifactIdent */ public ArtifactIdent processArtifactIdent(String gav) { - String[] parts = gav.split(IndexWalker.splitPattern); + String[] parts = gav.split(splitPattern); return new ArtifactIdent(parts[0], parts[1], parts[2]); } @@ -110,7 +113,7 @@ public ArtifactIdent processArtifactIdent(String gav) { */ public Package processPackage(String information, String checksum) { if(information != null) { - String[] parts = information.split(IndexWalker.splitPattern); + String[] parts = information.split(splitPattern); return new Package(parts[0], Long.parseLong(parts[1]), Long.parseLong(parts[2]), Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5]), checksum); } return null; @@ -141,7 +144,7 @@ private IndexInformation processIndex(Map item, ArtifactIdent id //Create an artifact using the values found in the 'i' and '1' tags if(iVal != null) { - String[] parts = iVal.split(IndexWalker.splitPattern); + String[] parts = iVal.split(splitPattern); Package tmpPackage = new Package(parts[0], Long.parseLong(parts[1]), Long.parseLong(parts[2]), Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5]), item.get("1")); diff --git a/src/test/java/org/tudo/sse/IndexWalkerTest.java b/src/test/java/org/tudo/sse/IndexWalkerTest.java deleted file mode 100644 index 99347c8..0000000 --- a/src/test/java/org/tudo/sse/IndexWalkerTest.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.tudo.sse; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.tudo.sse.model.Artifact; -import org.tudo.sse.model.ArtifactIdent; -import org.tudo.sse.model.index.Package; -import org.tudo.sse.model.index.IndexInformation; - -import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - -@SuppressWarnings("unchecked") -class IndexWalkerTest { - - IndexWalker indexWalker; - - final Map json; - final Gson gson = new Gson(); - - { - InputStream resource = this.getClass().getClassLoader().getResourceAsStream("IndexInputs.json"); - assert resource != null; - Reader targetReader = new InputStreamReader(resource); - json = gson.fromJson(targetReader, new TypeToken>() {}.getType()); - } - - @BeforeEach - void setUp() { - try { - indexWalker = new IndexWalker(new URI("https://repo1.maven.org/maven2/")); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - - @Test - void walkPaginatedEndings() { - int[][] startNtake = {{20, 30}, {20, 1}, {100000, 200000}, {500, 6000}, {6000, 10}}; - List idents; - List indexArtifacts; - - for (int[] ints : startNtake) { - try { - indexArtifacts = indexWalker.walkPaginated(ints[0], ints[1]); - idents = indexWalker.lazyWalkPaginated(ints[0], ints[1]); - } catch (IOException e) { - throw new RuntimeException(e); - } - assertEquals(ints[1], indexArtifacts.size()); - assertEquals(ints[1], idents.size()); - } - } - - @Test - void walkPaginated() { - ArrayList> expectedValues = (ArrayList>) json.get("walkPaginated"); - List results; - try { - results = indexWalker.walkPaginated(1000, 10); - } catch (IOException e) { - throw new RuntimeException(e); - } - - int i = 3; - for(Map value: expectedValues) { - IndexInformation currentArtifact = results.get(i).getIndexInformation(); - - assertEquals(value.get("ident"), currentArtifact.getIdent().getCoordinates()); - assertEquals(value.get("name"), currentArtifact.getName()); - assertEquals(Long.parseLong((String) value.get("index")), currentArtifact.getIndex()); - - ArrayList> expectedpackages = (ArrayList>) value.get("packages"); - List packages = currentArtifact.getPackages(); - for(int j = 0; j < expectedpackages.size(); j++) { - assertEquals(expectedpackages.get(j).get("packaging"), packages.get(j).getPackaging()); - assertEquals(Long.parseLong(expectedpackages.get(j).get("lastModified")), packages.get(j).getLastModified()); - assertEquals(Long.parseLong(expectedpackages.get(j).get("size")), packages.get(j).getSize()); - assertEquals(Integer.parseInt(expectedpackages.get(j).get("sourcesExist")), packages.get(j).getSourcesExist()); - assertEquals(Integer.parseInt(expectedpackages.get(j).get("javadocExist")), packages.get(j).getJavadocExists()); - assertEquals(Integer.parseInt(expectedpackages.get(j).get("signatureExist")), packages.get(j).getSignatureExists()); - assertEquals(expectedpackages.get(j).get("sha1checksum"), packages.get(j).getSha1checksum()); - } - i += 2; - } - - } -} \ No newline at end of file diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java index b5fb274..8bae9c1 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralArtifactAnalysisTest.java @@ -159,8 +159,6 @@ public void analyzeArtifact(Artifact current) { } }; - - List> inputs = new ArrayList<>(); inputs.add(new Tuple2<>(500, 10)); inputs.add(new Tuple2<>(0, 10)); @@ -168,35 +166,22 @@ public void analyzeArtifact(Artifact current) { inputs.add(new Tuple2<>(763, 20)); for(Tuple2 input : inputs) { - int start1 = input._1; + int skip = input._1; int take = input._2; - int start2 = (start1 + take) - 1; - try { - IndexIterator iterator = new IndexIterator(new URI(base), start1); + IndexIterator iterator = new IndexIterator(new URI(base), skip); theAnalysis.walkPaginated(take, iterator); assertFalse(artifactsSeen.isEmpty()); + assertEquals(take, artifactsSeen.size()); - Artifact lastOne = artifactsSeen.get(artifactsSeen.size() - 1); - - assertNotNull(lastOne); - - int i = 2; - while(lastOne.getIndexInformation().getIndex() > start2) { - lastOne = artifactsSeen.get(artifactsSeen.size() - i); - assertNotNull(lastOne); - i++; + for(Artifact a : artifactsSeen){ + assertTrue(a.hasIndexInformation()); + assertTrue(a.getIndexInformation().getIndex() >= (skip - 1)); } - iterator = new IndexIterator(new URI(base), start2); - artifactsSeen.clear(); - theAnalysis.walkPaginated(1, iterator); - - long lastOne2 = artifactsSeen.get(0).getIndexInformation().getIndex(); - assertEquals(lastOne.getIndexInformation().getIndex(), lastOne2); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); } @@ -289,8 +274,8 @@ void checkMultiThreading() { List singleArgs = new ArrayList<>(); List multiArgs = new ArrayList<>(); - singleArgs.add(new String[]{"-st", "10:1000"}); - multiArgs.add(new String[]{"--threads", "5", "-st", "10:1000"}); + singleArgs.add(new String[]{"-st", "10:200"}); + multiArgs.add(new String[]{"--threads", "5", "-st", "10:200"}); singleArgs.add(new String[]{"--inputs", "src/test/resources/artifact-names-valid.txt"}); multiArgs.add(new String[]{"--threads", "5", "--inputs", "src/test/resources/artifact-names-valid.txt"}); diff --git a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java index 8ca7d1f..84e1c8c 100644 --- a/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java +++ b/src/test/java/org/tudo/sse/analyses/MavenCentralLibraryAnalysisTest.java @@ -101,10 +101,17 @@ void parseNonExistingIndex(){ @Test @DisplayName("An analysis with no requirements must not produce information instances") - void simpleIndexAnalysis(){ + void simpleIndexAnalysis() throws IOException { final List librariesHit = new java.util.ArrayList<>(); + final Path validLibraryInput = testResource("library-names-valid.txt"); + + assertNotNull(validLibraryInput); + assert(Files.exists(validLibraryInput)); + + final List expectedLibraries = Files.readAllLines(validLibraryInput); + final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, false) { @Override protected void analyzeLibrary(String libraryGA, List releases) { @@ -116,20 +123,28 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 0:3"; + final String args = "-st 2:2 --inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); - assertEquals(3, librariesHit.size()); - assert(librariesHit.contains("yom:yom")); + assertEquals(2, librariesHit.size()); + + assertTrue(librariesHit.contains(expectedLibraries.get(2))); + assertTrue(librariesHit.contains(expectedLibraries.get(3))); } @Test @DisplayName("An analysis with JAR requirements must produce JAR information instances") - void simpleIndexWithJarAnalysis(){ + void simpleIndexWithJarAnalysis() throws IOException { final List librariesHit = new java.util.ArrayList<>(); + final Path validLibraryInput = testResource("library-names-valid.txt"); + + assertNotNull(validLibraryInput); + assert(Files.exists(validLibraryInput)); + + final List expectedLibraries = Files.readAllLines(validLibraryInput); final MavenCentralLibraryAnalysis analysis = new MavenCentralLibraryAnalysis(false, false, true) { @Override @@ -142,14 +157,15 @@ protected void analyzeLibrary(String libraryGA, List releases) { } }; - final String args = "-st 3:3"; + final String args = "-st 2:2 --inputs " + validLibraryInput.toAbsolutePath(); final String[] argsArray = args.split(" "); analysis.runAnalysis(argsArray); - assertEquals(3, librariesHit.size()); - assertFalse(librariesHit.contains("yom:yom")); - assert(librariesHit.contains("yan:yan")); + assertEquals(2, librariesHit.size()); + + assertTrue(librariesHit.contains(expectedLibraries.get(2))); + assertTrue(librariesHit.contains(expectedLibraries.get(3))); } @Test @@ -167,6 +183,13 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertNull(releases.get(0).getJarInformation()); librariesHit.add(libraryGA); } + + @Override + protected void onVersionListError(String g, String a, Exception x){ + assertTrue(x.getCause().getMessage().contains("was not found")); + String ga = g+":"+a; + librariesHit.add(ga); + } }; final String args = "-st 5000:10 --save-progress-interval 5"; @@ -198,6 +221,12 @@ protected void analyzeLibrary(String libraryGA, List releases) { assertNull(releases.get(0).getJarInformation()); count.incrementAndGet(); } + + @Override + protected void onVersionListError(String g, String a, Exception x){ + assertTrue(x.getCause().getMessage().contains("was not found")); + count.incrementAndGet(); + } }; final String args = "-st 5000:500 --threads 8"; diff --git a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java index 05e1bf3..9b39d1c 100644 --- a/src/test/java/org/tudo/sse/resolution/PomResolverTest.java +++ b/src/test/java/org/tudo/sse/resolution/PomResolverTest.java @@ -1,22 +1,22 @@ package org.tudo.sse.resolution; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.tudo.sse.IndexWalker; import org.tudo.sse.model.*; import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; +import java.nio.file.Files; +import java.util.*; import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Map; -import java.util.ArrayList; import java.util.stream.IntStream; import org.apache.commons.io.IOUtils; @@ -25,9 +25,11 @@ import org.tudo.sse.model.pom.RawPomFeatures; import org.tudo.sse.model.ResolutionContext; import org.tudo.sse.resolution.releases.IReleaseListProvider; +import org.tudo.sse.utils.JsonExporter; import static org.junit.jupiter.api.Assertions.*; +import static org.tudo.sse.utils.TestUtilities.testResource; @SuppressWarnings("unchecked") class PomResolverTest { @@ -73,37 +75,62 @@ void processRawPomFeatures() { } @Test - void processArtifacts() { - //walk 10 indexes - List idents; - try { - IndexWalker walker = new IndexWalker(new URI("https://repo1.maven.org/maven2/")); - idents = walker.lazyWalkPaginated(200000, 10); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); + @DisplayName("Assert correct raw pom feature contents") + void processArtifacts() throws IOException { + List identStrings = Files.readAllLines(Objects.requireNonNull(testResource("artifact-names-valid.txt"))); + + List idents = new ArrayList<>(); + + for(String ident : identStrings) { + String[] parts = ident.split(":"); + idents.add(new ArtifactIdent(parts[0], parts[1], parts[2])); } ArrayList> expected = (ArrayList>) json.get("resolveArtifacts"); List poms = pomResolver.resolveArtifacts(idents, testContext); + assertEquals(10, poms.size()); + //check to see that the parsed data from the POM for these indexes are correct for actually retrieving the files from url for(int i = 0; i < expected.size(); i++) { - RawPomFeatures current = poms.get(i).getPomInformation().getRawPomFeatures(); - checkRawFeatures(current, expected, i); + assertNotNull(poms.get(i).getPomInformation()); + assertNotNull(poms.get(i).getPomInformation().getRawPomFeatures()); + checkRawFeatures(poms.get(i).getPomInformation().getRawPomFeatures(), expected, i); } } - //touch this up to throw all exceptions, should have 100% coverage for the pomResolver class - void processArtifactsCov() { - List idents; - try { - IndexWalker walker = new IndexWalker(new URI("https://repo1.maven.org/maven2/")); - idents = walker.lazyWalkPaginated(0, 10000); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); + @Test + @DisplayName("Export new expected values for: Assert correct raw pom feature contents") + @Disabled + void exportRawPomFeatures() throws IOException { + final Gson exportGson = new GsonBuilder().setPrettyPrinting().create(); + List identStrings = Files.readAllLines(Objects.requireNonNull(testResource("artifact-names-valid.txt"))); + + List idents = new ArrayList<>(); + + for(String ident : identStrings) { + String[] parts = ident.split(":"); + idents.add(new ArtifactIdent(parts[0], parts[1], parts[2])); + } + + ArrayList> expected = (ArrayList>) json.get("resolveArtifacts"); + + List poms = pomResolver.resolveArtifacts(idents, testContext); + + assertEquals(10, poms.size()); + + JsonArray exportArray = new JsonArray(); + + //check to see that the parsed data from the POM for these indexes are correct for actually retrieving the files from url + for(int i = 0; i < expected.size(); i++) { + assertNotNull(poms.get(i).getPomInformation()); + assertNotNull(poms.get(i).getPomInformation().getRawPomFeatures()); + JsonObject exportObject = JsonExporter.exportRawPomFeatures(poms.get(i).getPomInformation().getRawPomFeatures()); + exportArray.add(exportObject); } + exportGson.toJson(exportArray, System.out); } @Test diff --git a/src/test/java/org/tudo/sse/utils/JsonExporter.java b/src/test/java/org/tudo/sse/utils/JsonExporter.java new file mode 100644 index 0000000..2edf2e5 --- /dev/null +++ b/src/test/java/org/tudo/sse/utils/JsonExporter.java @@ -0,0 +1,85 @@ +package org.tudo.sse.utils; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import org.tudo.sse.model.pom.RawPomFeatures; + +public class JsonExporter { + + public static JsonObject exportRawPomFeatures(RawPomFeatures features) { + JsonObject json = new JsonObject(); + + String parent = "null"; + if(features.getParent() != null){ + parent = features.getParent().getCoordinates(); + } + + json.add("parent", new JsonPrimitive(parent)); + + String name = "null"; + if(features.getName() != null){ + name = features.getName(); + } + + json.add("name", new JsonPrimitive(name)); + + String description = "null"; + if(features.getDescription() != null){ + description = features.getDescription(); + } + + json.add("description", new JsonPrimitive(description)); + + String url = "null"; + if(features.getUrl() != null){ + url = features.getUrl(); + } + + json.add("url", new JsonPrimitive(url)); + + String packaging = "null"; + if(features.getPackaging() != null){ + packaging = features.getPackaging(); + } + + json.add("packaging", new JsonPrimitive(packaging)); + + String inceptionYear = "null"; + if(features.getInceptionYear() != null){ + inceptionYear = features.getInceptionYear(); + } + + json.add("inceptionYear", new JsonPrimitive(inceptionYear)); + + if(!features.getProperties().isEmpty()){ + JsonArray properties = new JsonArray(features.getProperties().size()); + features.getProperties().forEach((k,v)->{ + JsonArray propArray = new JsonArray(2); + propArray.add(new JsonPrimitive(k)); + propArray.add(new JsonPrimitive(v)); + properties.add(propArray); + }); + json.add("properties", properties); + } + + JsonArray dependencies = new JsonArray(); + features.getDependencies().forEach(dependency -> dependencies.add(new JsonPrimitive(dependency.getIdent().getCoordinates()))); + + json.add("dependencies", dependencies); + + JsonArray licenses = new JsonArray(); + features.getLicenses().forEach(license -> licenses.add(new JsonPrimitive(license.getUrl()))); + json.add("licenses", licenses); + + if(features.getDependencyManagement() != null){ + JsonArray dependencyManagement = new JsonArray(); + features.getDependencyManagement().forEach(dependency -> dependencyManagement.add(new JsonPrimitive(dependency.getIdent().getCoordinates()))); + json.add("dependencyManagement", dependencyManagement); + } else { + json.add("dependencyManagement", new JsonPrimitive("null")); + } + + return json; + } +} diff --git a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java index 8d3b53e..50b17ff 100644 --- a/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java +++ b/src/test/java/org/tudo/sse/utils/LibraryIndexIteratorTest.java @@ -60,22 +60,6 @@ void uniqueGAs() { } } - @Test - @DisplayName("The Library Index Iterator must apply custom starting positions") - void customStartingPositions() { - iteratorUnderTest.setPosition(42000); - - int cutoff = 1000; - int idx = 0; - - while(iteratorUnderTest.hasNext() && idx < cutoff){ - final String ga = iteratorUnderTest.next(); - assert(!ga.equals("yom:yom")); // Make sure we skipped the first library in index! - idx += 1; - } - - } - @Test @Disabled @DisplayName("The Library Index Iterator must terminate") diff --git a/src/test/java/org/tudo/sse/utils/TestUtilities.java b/src/test/java/org/tudo/sse/utils/TestUtilities.java index 7e7a686..ad34151 100644 --- a/src/test/java/org/tudo/sse/utils/TestUtilities.java +++ b/src/test/java/org/tudo/sse/utils/TestUtilities.java @@ -1,6 +1,14 @@ package org.tudo.sse.utils; +import org.tudo.sse.analyses.MavenCentralArtifactIterator; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfig; +import org.tudo.sse.analyses.config.ArtifactAnalysisConfigBuilder; +import org.tudo.sse.analyses.config.InvalidConfigurationException; +import org.tudo.sse.model.Artifact; + import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import static org.junit.jupiter.api.Assertions.fail; @@ -16,4 +24,24 @@ public static Path testResource(String pathToResource){ return null; } + public static List getFromIndex(int skip, int take, boolean resolvePom, boolean resolveTransitives, boolean resolveJar){ + try { + final ArtifactAnalysisConfig config = new ArtifactAnalysisConfigBuilder().withSkip(skip).withTake(take).build(); + + final MavenCentralArtifactIterator it = new MavenCentralArtifactIterator(resolvePom, resolveTransitives, resolveJar, config); + + List artifacts = new ArrayList<>(); + + while(it.hasNext()){ + artifacts.add(it.next()); + } + + return artifacts; + } catch(InvalidConfigurationException icx){ + fail(icx); + } + + return null; + } + } diff --git a/src/test/resources/PomInputs.json b/src/test/resources/PomInputs.json index bab2673..e03f9f7 100644 --- a/src/test/resources/PomInputs.json +++ b/src/test/resources/PomInputs.json @@ -206,75 +206,301 @@ }, "resolveArtifacts": [ { - "parent": "net.officefloor.plugin:plugins:1.3.0", - "name": "WAR Plug-in", - "description": "OfficeFloor plug-in for utilising WAR", - "url": "null", + "parent": "null", + "name": "spring-boot-starter-web", + "description": "Starter for building web, including RESTful, applications using Spring MVC. Uses Tomcat as the default embedded container", + "url": "https://spring.io/projects/spring-boot", "packaging": "jar", - "properties": [ - ["project.build.sourceEncoding", "UTF-8"] - ], "inceptionYear": "null", "dependencies": [ - "net.officefloor.core:officebuilding:null", - "${project.groupId}:officeplugin_servlet:null", - "org.eclipse.jetty:jetty-servlet:null", - "org.apache.httpcomponents:httpclient:null" + "org.springframework.boot:spring-boot-starter:2.6.3", + "org.springframework.boot:spring-boot-starter-json:2.6.3", + "org.springframework.boot:spring-boot-starter-tomcat:2.6.3", + "org.springframework:spring-web:5.3.15", + "org.springframework:spring-webmvc:5.3.15" + ], + "licenses": [ + "https://www.apache.org/licenses/LICENSE-2.0" ], - "licenses": [], "dependencyManagement": "null" }, { - "parent": "net.officefloor.plugin:plugins:1.2.0", - "name": "WAR Plug-in", - "description": "OfficeFloor plug-in for utilising WAR", - "url": "null", + "parent": "org.apache.commons:commons-parent:52", + "name": "Apache Commons Lang", + "description": "Apache Commons Lang, a package of Java utility classes for the\n classes that are in java.lang\u0027s hierarchy, or are considered to be so\n standard as to justify existence in java.lang.", + "url": "https://commons.apache.org/proper/commons-lang/", "packaging": "jar", + "inceptionYear": "2001", "properties": [ - ["project.build.sourceEncoding", "UTF-8"] + [ + "jmh.version", + "1.27" + ], + [ + "commons.release.version", + "3.12.0" + ], + [ + "commons.encoding", + "utf-8" + ], + [ + "commons.bc.version", + "3.11" + ], + [ + "commons.site.path", + "lang" + ], + [ + "commons.jira.id", + "LANG" + ], + [ + "commons.scmPubCheckoutDirectory", + "site-content" + ], + [ + "commons.releaseManagerName", + "Gary Gregory" + ], + [ + "commons.releaseManagerKey", + "86fdc7e2a11262cb" + ], + [ + "commons.componentid", + "lang" + ], + [ + "uberjar.name", + "benchmarks" + ], + [ + "commons.javadoc.version", + "3.2.0" + ], + [ + "commons.japicmp.version", + "0.15.2" + ], + [ + "project.build.sourceEncoding", + "ISO-8859-1" + ], + [ + "commons.module.name", + "org.apache.commons.lang3" + ], + [ + "spotbugs.impl.version", + "4.2.1" + ], + [ + "commons.release.2.version", + "2.6" + ], + [ + "clirr.skip", + "true" + ], + [ + "commons.release.desc", + "(Java 8+)" + ], + [ + "maven.compiler.source", + "1.8" + ], + [ + "checkstyle.configdir", + "src/site/resources/checkstyle" + ], + [ + "commons.jira.pid", + "12310481" + ], + [ + "commons.distSvnStagingUrl", + "scm:svn:https://dist.apache.org/repos/dist/dev/commons/lang" + ], + [ + "project.reporting.outputEncoding", + "UTF-8" + ], + [ + "commons.rc.version", + "RC1" + ], + [ + "commons.surefire.version", + "3.0.0-M5" + ], + [ + "maven.compiler.target", + "1.8" + ], + [ + "commons.release.2.name", + "commons-lang-${commons.release.2.version}" + ], + [ + "spotbugs.plugin.version", + "4.2.0" + ], + [ + "commons.scmPubUrl", + "https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-lang" + ], + [ + "checkstyle.version", + "8.40" + ], + [ + "japicmp.skip", + "false" + ], + [ + "checkstyle.plugin.version", + "3.1.2" + ], + [ + "commons.jacoco.version", + "0.8.6" + ], + [ + "commons.release.isDistModule", + "true" + ], + [ + "commons.release.2.desc", + "(Requires Java 1.2 or later)" + ], + [ + "commons.packageId", + "lang3" + ], + [ + "argLine", + "-Xmx512m" + ] ], - "inceptionYear": "null", "dependencies": [ - "net.officefloor.core:officebuilding:null", - "${project.groupId}:officeplugin_servlet:null", - "org.eclipse.jetty:jetty-servlet:null", - "org.apache.httpcomponents:httpclient:null" + "org.junit.jupiter:junit-jupiter:null", + "org.junit-pioneer:junit-pioneer:1.3.0", + "org.hamcrest:hamcrest:2.2", + "org.easymock:easymock:4.2", + "org.openjdk.jmh:jmh-core:${jmh.version}", + "org.openjdk.jmh:jmh-generator-annprocess:${jmh.version}", + "com.google.code.findbugs:jsr305:3.0.2" ], "licenses": [], - "dependencyManagement": "null" + "dependencyManagement": [ + "org.junit:junit-bom:5.7.1" + ] }, { - "parent": "net.officefloor.plugin:plugins:1.1.0", - "name": "WAR Plug-in", - "description": "OfficeFloor plug-in for utilising WAR", - "url": "null", - "packaging": "jar", + "parent": "com.fasterxml.jackson:jackson-base:2.13.1", + "name": "jackson-databind", + "description": "General data-binding functionality for Jackson: works on core streaming API", + "url": "http://github.com/FasterXML/jackson", + "packaging": "bundle", + "inceptionYear": "2008", "properties": [ - ["project.build.sourceEncoding", "UTF-8"] + [ + "javac.src.version", + "1.8" + ], + [ + "packageVersion.dir", + "com/fasterxml/jackson/databind/cfg" + ], + [ + "javac.target.version", + "1.8" + ], + [ + "osgi.import", + "org.w3c.dom.bootstrap;resolution:\u003doptional,\n *" + ], + [ + "packageVersion.package", + "com.fasterxml.jackson.databind.cfg" + ], + [ + "osgi.export", + "com.fasterxml.jackson.databind.*;version\u003d${project.version}" + ], + [ + "version.powermock", + "2.0.0" + ] ], - "inceptionYear": "null", "dependencies": [ - "net.officefloor.core:officebuilding:null", - "${project.groupId}:officeplugin_servlet:null", - "org.apache.httpcomponents:httpclient:null" + "com.fasterxml.jackson.core:jackson-annotations:${jackson.version.annotations}", + "com.fasterxml.jackson.core:jackson-core:${jackson.version.core}", + "org.powermock:powermock-core:${version.powermock}", + "org.powermock:powermock-module-junit4:${version.powermock}", + "org.powermock:powermock-api-mockito2:${version.powermock}", + "javax.measure:jsr-275:0.9.1" + ], + "licenses": [ + "http://www.apache.org/licenses/LICENSE-2.0.txt" ], - "licenses": [], "dependencyManagement": "null" }, { - "parent" : "net.officefloor.plugin:plugins:1.4.0", - "name" : "Spring Plug-in", - "description" : "Integration with Spring to allow use of its BeanFactory (specifically XmlBeanFactory).", - "url" : "null", - "packaging" : "jar", - "properties" : [ - ["project.build.sourceEncoding", "UTF-8"] + "parent": "null", + "name": "JUnit", + "description": "JUnit is a unit testing framework for Java, created by Erich Gamma and Kent Beck.", + "url": "http://junit.org", + "packaging": "jar", + "inceptionYear": "2002", + "properties": [ + [ + "surefireVersion", + "2.19.1" + ], + [ + "jdkVersion", + "1.5" + ], + [ + "project.build.sourceEncoding", + "ISO-8859-1" + ], + [ + "jarPluginVersion", + "2.6" + ], + [ + "javadocPluginVersion", + "2.10.3" + ], + [ + "enforcerPluginVersion", + "1.4" + ], + [ + "arguments", + "" + ], + [ + "hamcrestVersion", + "1.3" + ], + [ + "gpg.keyname", + "67893CC4" + ] ], - "inceptionYear": "null", "dependencies": [ - "org.springframework:spring:null" + "org.hamcrest:hamcrest-core:${hamcrestVersion}", + "org.hamcrest:hamcrest-library:${hamcrestVersion}" + ], + "licenses": [ + "http://www.eclipse.org/legal/epl-v10.html" ], - "licenses": [], "dependencyManagement": "null" } ], From 3eb79e266d67ee6b919d02975a5dc4a7ab5be576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20D=C3=BCsing?= Date: Fri, 6 Mar 2026 09:58:49 +0100 Subject: [PATCH 5/5] Fix Javadoc warning --- src/main/java/org/tudo/sse/utils/IndexIterator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/tudo/sse/utils/IndexIterator.java b/src/main/java/org/tudo/sse/utils/IndexIterator.java index 75780a0..fdb9a2d 100644 --- a/src/main/java/org/tudo/sse/utils/IndexIterator.java +++ b/src/main/java/org/tudo/sse/utils/IndexIterator.java @@ -23,7 +23,10 @@ public class IndexIterator implements Iterator { - public static final String splitPattern = Pattern.quote("|"); + /** + * Pattern to split Lucene Index entries at + */ + private static final String splitPattern = Pattern.quote("|"); private long index;