diff --git a/.gitignore b/.gitignore index 98f7f7d..5cbda34 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ /.project /.classpath /.settings +.idea/* +/.proxyai/ +/.idea/ +/crawl-rss.iml diff --git a/pom.xml b/pom.xml index ccdccc0..529df67 100644 --- a/pom.xml +++ b/pom.xml @@ -1,239 +1,323 @@ - - - 4.0.0 - - - org.sonatype.oss - oss-parent - 9 - - - is.landsbokasafn.crawl-rss - crawl-rss - 0.0.1-SNAPSHOT - Crawler RSS - RSS crawling add-on for Heritrix 3 - - - GitHub Issues - https://github.com/Landsbokasafn/crawlrss/issues - - - - Kristinn - Kristinn Sigurðsson - kristinn at landsbokasafn.is - National and University Library of Iceland - - Developer - - +0 - - - - - scm:git://github.com/Landsbokasafn/crawlrss.git - scm:https://kris-sigur@github.com/Landsbokasafn/crawlrss.git - https://github.com/Landsbokasafn/crawlrss - - - National and University Library of Iceland - http://www.landsbokasafn.is - - - UTF-8 - 3.4.0-20190205 - - - - - - github-pages-site - Deployment through GitHub's site deployment plugin - http://landsbokasafn.github.io/CrawlRSS/${project.version}/ - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - maven-assembly-plugin - 2.4 - - - - src/main/assemblies/dist.xml - - - ${project.artifactId}-${project.version} - - - - make-assembly - package - - single - - - - - - maven-site-plugin - 3.3 - - en - - - - com.github.github - site-maven-plugin - 0.10 - - - github - - - Creating site for - ${project.artifactId}-${project.version} - - - ${project.distributionManagement.site.url} - - true - - - - - - site - - site-deploy - - - - - - - - - rome - rome - 1.0 - - - org.archive.heritrix - heritrix-commons - ${org.archive.heritrix.version} - provided - - - org.archive.heritrix - heritrix-modules - ${org.archive.heritrix.version} - provided - - - org.archive.heritrix - heritrix-engine - ${org.archive.heritrix.version} - provided - - - mysql - mysql-connector-java - 5.1.27 - - - org.hibernate - hibernate-annotations - 3.3.1.GA - - - commons-logging - commons-logging - - - hibernate - org.hibernate - - - - - org.slf4j - jcl-over-slf4j - 1.7.2 - runtime - - - org.slf4j - slf4j-log4j12 - 1.7.2 - runtime - - - log4j - log4j - 1.2.14 - runtime - - - org.springframework - spring-orm - 3.1.2.RELEASE - - - org.hibernate - hibernate-core - 3.3.1.GA - - - javassist - javassist - 3.12.1.GA - - - c3p0 - c3p0 - 0.9.1.2 - - - org.hibernate - hibernate-ehcache - 3.3.2.GA - - - javax.annotation - javax.annotation-api - 1.3.2 - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.8.1 - - ${basedir}/src/main/java/is/landsbokasafn/deduplicator/overview.html - - - - - + + + 4.0.0 + + + is.landsbokasafn.crawl-rss + crawl-rss + 0.0.4-SNAPSHOT + Crawler RSS + RSS crawling add-on for Heritrix 3 + + + GitHub Issues + https://github.com/Landsbokasafn/crawlrss/issues + + + + Kristinn + Kristinn Sigurðsson + kristinn at landsbokasafn.is + National and University Library of Iceland + + Developer + + +0 + + + + + scm:git:ssh://github.com/netarchivesuite/crawlrss.git + + + + National and University Library of Iceland + http://www.landsbokasafn.is + + + UTF-8 + 3.14.1 + + + + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/releases + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/snapshots/ + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + UTF-8 + + + + + maven-assembly-plugin + 2.4 + + + + src/main/assemblies/dist.xml + + + ${project.artifactId}-${project.version} + + + + make-assembly + package + + single + + + + + + maven-site-plugin + 3.3 + + en + + + + com.github.github + site-maven-plugin + 0.10 + + + github + + + Creating site for + ${project.artifactId}-${project.version} + + + ${project.distributionManagement.site.url} + + true + + + + + + site + + site-deploy + + + + + + + + + + joda-time + joda-time + 2.14.1 + compile + + + + commons-httpclient + commons-httpclient + 3.1 + compile + + + rome + rome + 1.0 + + + org.archive.heritrix + heritrix-commons + ${heritrix3.version} + provided + + + org.archive.heritrix + heritrix-engine + ${heritrix3.version} + + + + org.archive.heritrix + heritrix + ${heritrix3.version} + dist + zip + + + + org.archive.heritrix + heritrix-contrib + ${heritrix3.version} + + + + org.archive.heritrix + heritrix-modules + ${heritrix3.version} + provided + + + + mysql + mysql-connector-java + 5.1.27 + + + org.hibernate + hibernate-annotations + 3.3.1.GA + + + commons-logging + commons-logging + + + hibernate + org.hibernate + + + + + org.slf4j + jcl-over-slf4j + 1.7.2 + runtime + + + org.slf4j + slf4j-log4j12 + 1.7.2 + runtime + + + log4j + log4j + 1.2.14 + runtime + + + org.springframework + spring-orm + + 3.0.5.RELEASE + + + org.hibernate + hibernate-core + 3.3.1.GA + + + javassist + javassist + 3.12.1.GA + + + c3p0 + c3p0 + 0.9.1.2 + + + org.hibernate + hibernate-ehcache + 3.3.2.GA + + + + + jakarta.annotation + jakarta.annotation-api + 3.0.0 + compile + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.8.1 + + ${basedir}/src/main/java/is/landsbokasafn/deduplicator/overview.html + + + + + + + + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org/maven2 + default + + true + daily + warn + + + + + + sbforge-nexus + https://sbforge.org/nexus/content/groups/public + + true + + + true + + + + + \ No newline at end of file diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml new file mode 100644 index 0000000..ed2141f --- /dev/null +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml @@ -0,0 +1,601 @@ + + + + + + + + + + + + +# This Properties map is specified in the Java 'property list' text format +# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29 + +### +### some of these overrides is actually just the default value, so they can be skipped +### + +## Q: can overrides like 'fetchDns.enabled=false' be used to disable the beans? + +metadata.jobName=crawlrss_cxml +metadata.description=Default Profile with CrawlRSS +metadata.operator=Admin +metadata.userAgentTemplate=Mozilla/5.0 (compatible; heritrix/3.13.0 +@OPERATOR_CONTACT_URL@) +## Edit the two following lines to match your setup. +metadata.operatorContactUrl=http://netarkivet.dk/webcrawler/ +metadata.operatorFrom=info@netarkivet.dk + +loggerModule.path=logs + +crawlLimiter.maxBytesDownload=0 +crawlLimiter.maxDocumentsDownload=0 +## MaxTimeseconds inserted by NetarchiveSuite (Delete line, if behaviour unwanted) +crawlLimiter.maxTimeSeconds=%{MAX_TIME_SECONDS_PLACEHOLDER} + +crawlController.maxToeThreads=50 +crawlController.recorderOutBufferBytes=4096 +crawlController.recorderInBufferBytes=65536 +crawlController.pauseAtStart=false +crawlController.runWhileEmpty=true +crawlController.scratchDir=scratch + +## org.archive.bdb.BdbModule overrides +bdb.dir=state +bdb.cachePercent=40 + +## seeds properties +seeds.sourceTagSeeds=false + +## Politeness overrides +disposition.delayFactor=1.0 +disposition.maxDelayMs=1000 +disposition.minDelayMs=300 +disposition.maxPerHostBandwidthUsageKbSec=500 + +## preparer.preferenceEmbedHops=1 +## preparer.preferenceDepthHops=-1 + +frontier.maxRetries=3 +frontier.retryDelaySeconds=300 +frontier.recoveryLogEnabled=false +frontier.balanceReplenishAmount=3000 +frontier.errorPenaltyAmount=100 +## Can be used instead of the QuotaEnforcer module. In this case the following line should look +## like: frontier.queueTotalBudget=%{FRONTIER_QUEUE_TOTAL_BUDGET_PLACEHOLDER} +## instead of: frontier.queueTotalBudget= + +frontier.queueTotalBudget=%{FRONTIER_QUEUE_TOTAL_BUDGET_PLACEHOLDER} +frontier.snoozeLongMs=300000 + +preselector.enabled=true +preselector.logToFile=false +preselector.recheckScope=true +preselector.blockAll=false + +preconditions.enabled=true +preconditions.ipValidityDurationSeconds=21600 +preconditions.robotsValidityDurationSeconds=86400 +preconditions.calculateRobotsOnly=false + +fetchDns.enabled=true +fetchDns.acceptNonDnsResolves=false +fetchDns.digestContent=true +fetchDns.digestAlgorithm=sha1 + +fetchHttp.enabled=true +fetchHttp.timeoutSeconds=1200 +fetchHttp.soTimeoutMs=20000 +fetchHttp.maxFetchKBSec=0 +fetchHttp.maxLengthBytes=0 +fetchHttp.ignoreCookies=false +fetchHttp.sslTrustLevel=OPEN +fetchHttp.defaultEncoding=ISO-8859-1 +fetchHttp.digestContent=true +fetchHttp.digestAlgorithm=sha1 +fetchHttp.sendIfModifiedSince=true +fetchHttp.sendIfNoneMatch=true +fetchHttp.sendConnectionClose=true +fetchHttp.sendReferer=true +fetchHttp.sendRange=false +extractorHttp.enabled=true +extractorHtml.enabled=true +extractorHtml.extractJavascript=true +extractorHtml.treatFramesAsEmbedLinks=false +extractorHtml.ignoreFormActionUrls=true +extractorHtml.extractValueAttributes=false +extractorHtml.ignoreUnexpectedHtml=true +extractorCss.enabled=true +extractorJs.enabled=true + extractorSwf.enabled=true + candidates.seedsRedirectNewSeeds=false + statisticsTracker.intervalSeconds=20 + warcWriter.compress=true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +%{ARCHIVER_PROCESSOR_BEAN_PLACEHOLDER} + + + + + + + + + + + + + + + %{ARCHIVER_BEAN_REFERENCE_PLACEHOLDER} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https://www.dr.dk/nyheder/ + http://www.dr.dk/nyheder/allenyheder/indland + + + + + + + + http://www.dr.dk/nyheder/allenyheder/udland + + + + + + + + http://www.dr.dk/nyheder/allenyheder/penge + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java b/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java index 95d779b..b6084d5 100644 --- a/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java +++ b/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java @@ -26,13 +26,15 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.archive.crawler.datamodel.UriUniqFilter; import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.framework.CrawlController; @@ -53,13 +55,6 @@ public class RssCrawlController implements { private static final Logger log = Logger.getLogger(RssCrawlController.class.getName()); - static { - CrawlURI.getPersistentDataKeys().add(RSS_SITE); - CrawlURI.getPersistentDataKeys().add(RSS_URI_TYPE); - CrawlURI.getPersistentDataKeys().add(RSS_MOST_RECENTLY_SEEN); - CrawlURI.getPersistentDataKeys().add(RSS_IMPLIED_LINKS); - } - boolean shouldStop = false; boolean started = false; @@ -149,7 +144,7 @@ public void setCrawlLogToPreloadUriUniqFilter(String log) { protected void startManagerThread() { managerThread = new Thread(this+".managerThread") { public void run() { - log.fine("Starting manager thread"); + log.info("Starting manager thread"); RssCrawlController.this.controlTasks(); } }; @@ -184,12 +179,12 @@ public void aboutToSchedule(CrawlURI curi) { protected void handleRssLink(CrawlURI curi) { // Need to ensure that all feeds associated with the site are finished before scheduling with the // frontier. Add it to the discovered items. - getSiteFor(curi).addDiscoveredItems(curi); + Optional.ofNullable(getSiteFor(curi)).ifPresent(site -> site.addDiscoveredItems(curi)); } protected void handleRssDerived(CrawlURI curi) { // Just note that the URI is in progress for the site - getSiteFor(curi).incrementInProgressURLs(); + Optional.ofNullable(getSiteFor(curi)).ifPresent(RssSite::incrementInProgressURLs); } protected void controlTasks() { @@ -201,7 +196,7 @@ protected void controlTasks() { log.log(Level.FINE,"Interrupted",e); } } - log.fine("Frontier is now running"); + log.info("Frontier is now running"); while (!shouldStop) { State state = (State)controller.getState(); if (state==State.RUNNING || state==State.EMPTY) { @@ -209,7 +204,9 @@ protected void controlTasks() { readConfig(); // Check for new sites // Trigger updates in all WAITING and ENDED sites for (RssSite site : sites.values()) { + log.info("Checking " + site.name); if (site.getState()==RssSiteState.WAITING || site.getState()==RssSiteState.ENDED) { + log.info("Updating " + site.name); site.doUpdate(); } } @@ -217,7 +214,7 @@ protected void controlTasks() { } for (RssSite site : sites.values()) { for (CrawlURI curi : site.emitReadyFeeds()) { - log.fine("Scheduling: " + curi.getURI()); + log.info("Scheduling: " + curi.getURI()); frontier.schedule(curi); } } @@ -228,27 +225,27 @@ protected void controlTasks() { log.log(Level.WARNING,"Unexpected interruption of RssCrawlController control thread", e); } } - log.fine("EXITING"); + log.info("EXITING"); } private void readConfig() { for (RssSite site : configurationManager.getSites()){ if (!sites.containsKey(site.getName())) { - log.fine("Site found in configuration: " + site.getName()); + log.info("Site found in configuration: " + site.getName()); this.sites.put(site.getName(), site); } } } - + @Override @PostConstruct public void start() { if (started) { + log.info("RssCrawlController already started."); return; } - log.fine("Starting RssCrawlController"); + log.info("Starting RssCrawlController"); started = true; - readConfig(); lastCheckedConfig=System.currentTimeMillis(); recheckConfig=configurationManager.supportsRuntimeChanges(); @@ -281,6 +278,8 @@ public void stop() { public void onApplicationEvent(ApplicationEvent event) { if(event instanceof CrawlURIDispositionEvent) { CrawlURIDispositionEvent dvent = (CrawlURIDispositionEvent)event; + log.fine(this.getClass().getName() + + " responding to " + dvent.getCrawlURI() + " " + dvent.getDisposition()); switch(dvent.getDisposition()) { case SUCCEEDED: case FAILED: @@ -299,25 +298,37 @@ public void onApplicationEvent(ApplicationEvent event) { } private void finished(CrawlURI curi){ - log.fine(curi.getURI()); + log.fine(this.getClass().getName() + " finished with " + curi.getURI() + + " " + getUriType(curi)); + RssSite siteFor = getSiteFor(curi); switch (getUriType(curi)) { - case RSS_FEED : - getSiteFor(curi).noteFeedCrawled(curi); + case RSS_FEED : + if (siteFor != null) { + siteFor.noteFeedCrawled(curi); + } break; case RSS_LINK : case RSS_INFERRED : case RSS_DERIVED : - getSiteFor(curi).decrementInProgressURLs(); + if (siteFor != null) { + siteFor.decrementInProgressURLs(); + } break; } } private RssSite getSiteFor(CrawlURI curi) { - String siteName = (String)curi.getData().get(RssAttributeConstants.RSS_SITE); - return sites.get(siteName); + try { + String siteName = (String)curi.getData().get(RssAttributeConstants.RSS_SITE); + return sites.get(siteName); + } catch (Exception e) { + return null; + } } private RssUriType getUriType(CrawlURI curi) { + log.info("Starting RssCrawlController from getUriType()"); + this.start(); Object o = curi.getData().get(RSS_URI_TYPE); if (o==null || !(o instanceof RssUriType) ) { // This is an error, log and assume derived (extracted from regular content, not RSS feed) @@ -330,7 +341,7 @@ private RssUriType getUriType(CrawlURI curi) { @Override public void receiveDuplicate(CrawlURI curi) { log.fine(curi.getURI()); - getSiteFor(curi).decrementInProgressURLs(); + Optional.ofNullable(getSiteFor(curi)).ifPresent(RssSite::decrementInProgressURLs); } diff --git a/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java b/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java index 719cd21..5d86ae8 100644 --- a/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java +++ b/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java @@ -27,9 +27,9 @@ import java.util.Date; import java.util.List; -import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; import org.archive.net.UURIFactory; +import org.archive.url.URIException; import org.springframework.beans.factory.annotation.Autowired; public class RssFeed {