From 4f37ae7fe0faaa3dc898397d17db7256096a30dc Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 5 Oct 2016 09:21:40 +0200 Subject: [PATCH 01/24] Set to use sbforge nexus for h3 dependency and to push snapshots. --- pom.xml | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 04cddc0..7f635db 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,8 @@ UTF-8 3.3.0-SNAPSHOT + 3.3.0-LBS-2016-02 + 1.0.0 - + + + sbforge-nexus + https://sbforge.org/nexus/content/groups/public + + true + + + true + + + + \ No newline at end of file From fc3bc56aa7b72c123611ce68af0d24e685e216a9 Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 5 Oct 2016 09:22:31 +0200 Subject: [PATCH 02/24] Added .idea files to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 98f7f7d..e89ffd1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /.project /.classpath /.settings +.idea/* \ No newline at end of file From 612e6891d8719b53d001f41ded3f8999bd53f110 Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 5 Oct 2016 10:40:39 +0200 Subject: [PATCH 03/24] Downgraded spring dependency. --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7f635db..6e70c10 100644 --- a/pom.xml +++ b/pom.xml @@ -215,7 +215,8 @@ org.springframework spring-orm - 3.1.2.RELEASE + + 3.0.5.RELEASE org.hibernate From 12db1548b3866dc6874cebe376314ff1cfc32cd5 Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 5 Oct 2016 10:46:35 +0200 Subject: [PATCH 04/24] Fixed dependency management --- pom.xml | 15 +- .../netarkivet-crawlrss.cxml | 595 ++++++++++++++++++ 2 files changed, 608 insertions(+), 2 deletions(-) create mode 100644 src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml diff --git a/pom.xml b/pom.xml index 6e70c10..8b1b03f 100644 --- a/pom.xml +++ b/pom.xml @@ -51,11 +51,22 @@ - + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/releases + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/snapshots/ + diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml new file mode 100644 index 0000000..6bc75d2 --- /dev/null +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml @@ -0,0 +1,595 @@ + + + + + + + + + + + + +# This Properties map is specified in the Java 'property list' text format +# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29 + +### +### some of these overrides is actually just the default value, so they can be skipped +### + +## Q: can overrides like 'fetchDns.enabled=false' be used to disable the beans? + +metadata.jobName=crawlrss_cxml +metadata.description=Default Profile +metadata.operator=Admin +metadata.userAgentTemplate=Mozilla/5.0 (compatible; heritrix/3.3.0 +@OPERATOR_CONTACT_URL@) +## Edit the two following lines to match your setup. +metadata.operatorContactUrl=http://netarkivet.dk/webcrawler/ +metadata.operatorFrom=info@netarkivet.dk + +loggerModule.path=logs + +crawlLimiter.maxBytesDownload=0 +crawlLimiter.maxDocumentsDownload=0 +## MaxTimeseconds inserted by NetarchiveSuite (Delete line, if behaviour unwanted) +crawlLimiter.maxTimeSeconds=%{MAX_TIME_SECONDS_PLACEHOLDER} + +crawlController.maxToeThreads=50 +crawlController.recorderOutBufferBytes=4096 +crawlController.recorderInBufferBytes=65536 +crawlController.pauseAtStart=false +crawlController.scratchDir=scratch + +## org.archive.bdb.BdbModule overrides +bdb.dir=state +bdb.cachePercent=40 + +## seeds properties +seeds.sourceTagSeeds=false + +scope.rules[2].maxHops=25 +scope.rules[6].maxRepetitions=3 +scope.rules[3].maxTransHops=5 +scope.rules[3].maxSpeculativeHops=1 + +## Politeness overrides +disposition.delayFactor=1.0 +disposition.maxDelayMs=1000 +disposition.minDelayMs=300 +disposition.maxPerHostBandwidthUsageKbSec=500 + +preparer.preferenceEmbedHops=1 +preparer.preferenceDepthHops=-1 + +frontier.maxRetries=3 +frontier.retryDelaySeconds=300 +frontier.recoveryLogEnabled=false +frontier.balanceReplenishAmount=3000 +frontier.errorPenaltyAmount=100 +## Can be used instead of the QuotaEnforcer module. In this case the following line should look +## like: frontier.queueTotalBudget=%{FRONTIER_QUEUE_TOTAL_BUDGET_PLACEHOLDER} +## instead of: frontier.queueTotalBudget= + +frontier.queueTotalBudget=%{FRONTIER_QUEUE_TOTAL_BUDGET_PLACEHOLDER} +frontier.snoozeLongMs=300000 + +preselector.enabled=true +preselector.logToFile=false +preselector.recheckScope=true +preselector.blockAll=false + +preconditions.enabled=true +preconditions.ipValidityDurationSeconds=21600 +preconditions.robotsValidityDurationSeconds=86400 +preconditions.calculateRobotsOnly=false + +fetchDns.enabled=true +fetchDns.acceptNonDnsResolves=false +fetchDns.digestContent=true +fetchDns.digestAlgorithm=sha1 + +fetchHttp.enabled=true +fetchHttp.timeoutSeconds=1200 +fetchHttp.soTimeoutMs=20000 +fetchHttp.maxFetchKBSec=0 +fetchHttp.maxLengthBytes=0 +fetchHttp.ignoreCookies=false +fetchHttp.sslTrustLevel=OPEN +fetchHttp.defaultEncoding=ISO-8859-1 +fetchHttp.digestContent=true +fetchHttp.digestAlgorithm=sha1 +fetchHttp.sendIfModifiedSince=true +fetchHttp.sendIfNoneMatch=true +fetchHttp.sendConnectionClose=true +fetchHttp.sendReferer=true +fetchHttp.sendRange=false +extractorHttp.enabled=true +extractorHtml.enabled=true +extractorHtml.extractJavascript=true +extractorHtml.treatFramesAsEmbedLinks=false +extractorHtml.ignoreFormActionUrls=true +extractorHtml.extractValueAttributes=false +extractorHtml.ignoreUnexpectedHtml=true +extractorCss.enabled=true +extractorJs.enabled=true + extractorSwf.enabled=true + candidates.seedsRedirectNewSeeds=false + statisticsTracker.intervalSeconds=20 + warcWriter.compress=true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +%{ARCHIVER_PROCESSOR_BEAN_PLACEHOLDER} + + + + + + + + + + + + + + + + %{ARCHIVER_BEAN_REFERENCE_PLACEHOLDER} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + http://www.ruv.is/ + + + + + + + + http://www.ruv.is/ + http://www.ruv.is/innlent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 2b9a6e14a18a9744adf5aefb4381e0b515202981 Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 5 Oct 2016 18:16:19 +0200 Subject: [PATCH 05/24] Fixed sample crawler-bean template. --- .../netarkivet-crawlrss.cxml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml index 6bc75d2..f49fe88 100644 --- a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml @@ -77,10 +77,6 @@ bdb.cachePercent=40 ## seeds properties seeds.sourceTagSeeds=false -scope.rules[2].maxHops=25 -scope.rules[6].maxRepetitions=3 -scope.rules[3].maxTransHops=5 -scope.rules[3].maxSpeculativeHops=1 ## Politeness overrides disposition.delayFactor=1.0 @@ -376,19 +372,20 @@ extractorJs.enabled=true - - + + - + - http://www.ruv.is/ + https://www.dr.dk/nyheder/ + From fb0045d5d28c3df0254acbd6ec54d36f007894cc Mon Sep 17 00:00:00 2001 From: Colin Rosenthal Date: Thu, 6 Oct 2016 09:49:31 +0200 Subject: [PATCH 06/24] Removed some parametere overrides which messed up the harvest. --- ...ss.cxml => netarkivet-crawlrss.dr.dk.cxml} | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) rename src/main/conf/jobs/CrawlRSS-Sample-Profile/{netarkivet-crawlrss.cxml => netarkivet-crawlrss.dr.dk.cxml} (92%) diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml similarity index 92% rename from src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml rename to src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml index f49fe88..e6284ba 100644 --- a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.cxml +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml @@ -84,8 +84,8 @@ disposition.maxDelayMs=1000 disposition.minDelayMs=300 disposition.maxPerHostBandwidthUsageKbSec=500 -preparer.preferenceEmbedHops=1 -preparer.preferenceDepthHops=-1 +## preparer.preferenceEmbedHops=1 +## preparer.preferenceDepthHops=-1 frontier.maxRetries=3 frontier.retryDelaySeconds=300 @@ -372,32 +372,39 @@ extractorJs.enabled=true - - - - - - - - - - https://www.dr.dk/nyheder/ - + + + + + + + + + + https://www.dr.dk/nyheder/ + http://www.dr.dk/nyheder/allenyheder/indland + + + + + + + + http://www.dr.dk/nyheder/allenyheder/udland + + + + + + + + http://www.dr.dk/nyheder/allenyheder/penge + + + + - - - - - + 1.0.0 diff --git a/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java b/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java index 95d779b..5e0f69d 100644 --- a/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java +++ b/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java @@ -53,13 +53,6 @@ public class RssCrawlController implements { private static final Logger log = Logger.getLogger(RssCrawlController.class.getName()); - static { - CrawlURI.getPersistentDataKeys().add(RSS_SITE); - CrawlURI.getPersistentDataKeys().add(RSS_URI_TYPE); - CrawlURI.getPersistentDataKeys().add(RSS_MOST_RECENTLY_SEEN); - CrawlURI.getPersistentDataKeys().add(RSS_IMPLIED_LINKS); - } - boolean shouldStop = false; boolean started = false; @@ -300,21 +293,30 @@ public void onApplicationEvent(ApplicationEvent event) { private void finished(CrawlURI curi){ log.fine(curi.getURI()); + RssSite siteFor = getSiteFor(curi); switch (getUriType(curi)) { - case RSS_FEED : - getSiteFor(curi).noteFeedCrawled(curi); + case RSS_FEED : + if (siteFor != null) { + siteFor.noteFeedCrawled(curi); + } break; case RSS_LINK : case RSS_INFERRED : case RSS_DERIVED : - getSiteFor(curi).decrementInProgressURLs(); + if (siteFor != null) { + siteFor.decrementInProgressURLs(); + } break; } } private RssSite getSiteFor(CrawlURI curi) { - String siteName = (String)curi.getData().get(RssAttributeConstants.RSS_SITE); - return sites.get(siteName); + try { + String siteName = (String)curi.getData().get(RssAttributeConstants.RSS_SITE); + return sites.get(siteName); + } catch (Exception e) { + return null; + } } private RssUriType getUriType(CrawlURI curi) { From 892aefbe85d109efbab30df903c3cd45d54e151f Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 4 May 2022 15:01:13 +0200 Subject: [PATCH 08/24] Fixed the sample config so it functions with the latest DR feed urls --- ...xml => netarkivet-crawlrss.dr.dk.cxml.txt} | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) rename src/main/conf/jobs/CrawlRSS-Sample-Profile/{netarkivet-crawlrss.dr.dk.cxml => netarkivet-crawlrss.dr.dk.cxml.txt} (97%) diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml.txt similarity index 97% rename from src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml rename to src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml.txt index e6284ba..e7ffad1 100644 --- a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml.txt @@ -174,9 +174,9 @@ extractorJs.enabled=true - - - + + + @@ -265,14 +265,16 @@ extractorJs.enabled=true - - + + + + @@ -286,6 +288,7 @@ extractorJs.enabled=true + @@ -299,7 +302,7 @@ extractorJs.enabled=true - + @@ -378,7 +381,7 @@ extractorJs.enabled=true - + https://www.dr.dk/nyheder/ @@ -387,7 +390,7 @@ extractorJs.enabled=true - + http://www.dr.dk/nyheder/allenyheder/udland @@ -395,7 +398,7 @@ extractorJs.enabled=true - + http://www.dr.dk/nyheder/allenyheder/penge From 143bab9658a4f1074edeee07fd228d84bc35783b Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 4 May 2022 15:02:04 +0200 Subject: [PATCH 09/24] Renamed cxml file --- ...vet-crawlrss.dr.dk.cxml.txt => netarkivet-crawlrss.dr.dk.cxml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/conf/jobs/CrawlRSS-Sample-Profile/{netarkivet-crawlrss.dr.dk.cxml.txt => netarkivet-crawlrss.dr.dk.cxml} (100%) diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml.txt b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml similarity index 100% rename from src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml.txt rename to src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml From a4b03d6976933ddea6efe77eabf25822443558cf Mon Sep 17 00:00:00 2001 From: csrster Date: Fri, 15 Jul 2022 11:46:37 +0200 Subject: [PATCH 10/24] [maven-release-plugin] rollback the release of crawl-rss-0.0.2 --- pom.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index c21c208..f95bca6 100644 --- a/pom.xml +++ b/pom.xml @@ -34,9 +34,9 @@ - scm:git://github.com/Landsbokasafn/crawlrss.git - scm:https://kris-sigur@github.com/Landsbokasafn/crawlrss.git - https://github.com/Landsbokasafn/crawlrss + scm:git://github.com/netarchivesuite/crawlrss.git + scm:git:https://github.com/netarchivesuite/crawlrss.git + https://github.com/netarchivesuite/crawlrss National and University Library of Iceland @@ -44,10 +44,10 @@ UTF-8 - 3.4.0-20210923 - 3.4.0-20210923 + + 3.4.0-NAS-7.4 - 1.0.0 + 1.0.5 is.landsbokasafn.crawl-rss crawl-rss From 22cfd1bb94765ebcc84eefdeab3a2a068f541ab5 Mon Sep 17 00:00:00 2001 From: csrster Date: Fri, 15 Jul 2022 12:13:23 +0200 Subject: [PATCH 12/24] [maven-release-plugin] rollback the release of crawl-rss-0.0.2 --- pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pom.xml b/pom.xml index a829d05..f95bca6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - is.landsbokasafn.crawl-rss crawl-rss From 03d64a1bb511ebcfe7afe3f8fc0308535936f436 Mon Sep 17 00:00:00 2001 From: csrster Date: Fri, 15 Jul 2022 12:18:04 +0200 Subject: [PATCH 13/24] Remove sonatype parent --- pom.xml | 602 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 301 insertions(+), 301 deletions(-) diff --git a/pom.xml b/pom.xml index f95bca6..3e30954 100644 --- a/pom.xml +++ b/pom.xml @@ -1,302 +1,302 @@ - - - 4.0.0 - - - org.sonatype.oss - oss-parent - 9 - - - is.landsbokasafn.crawl-rss - crawl-rss - 0.0.2-SNAPSHOT - Crawler RSS - RSS crawling add-on for Heritrix 3 - - - GitHub Issues - https://github.com/Landsbokasafn/crawlrss/issues - - - - Kristinn - Kristinn Sigurðsson - kristinn at landsbokasafn.is - National and University Library of Iceland - - Developer - - +0 - - - - - scm:git://github.com/netarchivesuite/crawlrss.git - scm:git:https://github.com/netarchivesuite/crawlrss.git - https://github.com/netarchivesuite/crawlrss - - - National and University Library of Iceland - http://www.landsbokasafn.is - - - UTF-8 - - 3.4.0-NAS-7.4 - - 1.0.5 - - - - - - - sbforge-nexus - SBForge Nexus Repo manager - https://sbforge.org/nexus/content/repositories/releases - - - - sbforge-nexus - SBForge Nexus Repo manager - https://sbforge.org/nexus/content/repositories/snapshots/ - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - maven-assembly-plugin - 2.4 - - - - src/main/assemblies/dist.xml - - - ${project.artifactId}-${project.version} - - - - make-assembly - package - - single - - - - - - maven-site-plugin - 3.3 - - en - - - - com.github.github - site-maven-plugin - 0.10 - - - github - - - Creating site for - ${project.artifactId}-${project.version} - - - ${project.distributionManagement.site.url} - - true - - - - - - site - - site-deploy - - - - - - - - - rome - rome - 1.0 - - - org.archive.heritrix - heritrix-commons - ${heritrix3.version} - provided - - - org.archive.heritrix - heritrix-engine - ${heritrix3.version} - - - - org.archive.heritrix - heritrix - ${heritrix3.version} - dist - zip - - - - org.archive.heritrix - heritrix-contrib - ${heritrix3.version} - - - - org.archive.heritrix - heritrix-modules - ${heritrix3.version} - provided - - - - mysql - mysql-connector-java - 5.1.27 - - - org.hibernate - hibernate-annotations - 3.3.1.GA - - - commons-logging - commons-logging - - - hibernate - org.hibernate - - - - - org.slf4j - jcl-over-slf4j - 1.7.2 - runtime - - - org.slf4j - slf4j-log4j12 - 1.7.2 - runtime - - - log4j - log4j - 1.2.14 - runtime - - - org.springframework - spring-orm - - 3.0.5.RELEASE - - - org.hibernate - hibernate-core - 3.3.1.GA - - - javassist - javassist - 3.12.1.GA - - - c3p0 - c3p0 - 0.9.1.2 - - - org.hibernate - hibernate-ehcache - 3.3.2.GA - - - javax.annotation - javax.annotation-api - 1.3.2 - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.8.1 - - ${basedir}/src/main/java/is/landsbokasafn/deduplicator/overview.html - - - - - - - - - internetarchive - Internet Archive Maven Repository - http://builds.archive.org/maven2 - default - - true - daily - warn - - - - - - sbforge-nexus - https://sbforge.org/nexus/content/groups/public - - true - - - true - - - - - + + + 4.0.0 + + + + is.landsbokasafn.crawl-rss + crawl-rss + 0.0.2 + Crawler RSS + RSS crawling add-on for Heritrix 3 + + + GitHub Issues + https://github.com/Landsbokasafn/crawlrss/issues + + + + Kristinn + Kristinn Sigurðsson + kristinn at landsbokasafn.is + National and University Library of Iceland + + Developer + + +0 + + + + + scm:git://github.com/netarchivesuite/crawlrss.git + scm:git:https://github.com/netarchivesuite/crawlrss.git + https://github.com/netarchivesuite/crawlrss + + + National and University Library of Iceland + http://www.landsbokasafn.is + + + UTF-8 + + 3.4.0-NAS-7.4 + + 1.0.5 + + + + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/releases + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/snapshots/ + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + UTF-8 + + + + + maven-assembly-plugin + 2.4 + + + + src/main/assemblies/dist.xml + + + ${project.artifactId}-${project.version} + + + + make-assembly + package + + single + + + + + + maven-site-plugin + 3.3 + + en + + + + com.github.github + site-maven-plugin + 0.10 + + + github + + + Creating site for + ${project.artifactId}-${project.version} + + + ${project.distributionManagement.site.url} + + true + + + + + + site + + site-deploy + + + + + + + + + rome + rome + 1.0 + + + org.archive.heritrix + heritrix-commons + ${heritrix3.version} + provided + + + org.archive.heritrix + heritrix-engine + ${heritrix3.version} + + + + org.archive.heritrix + heritrix + ${heritrix3.version} + dist + zip + + + + org.archive.heritrix + heritrix-contrib + ${heritrix3.version} + + + + org.archive.heritrix + heritrix-modules + ${heritrix3.version} + provided + + + + mysql + mysql-connector-java + 5.1.27 + + + org.hibernate + hibernate-annotations + 3.3.1.GA + + + commons-logging + commons-logging + + + hibernate + org.hibernate + + + + + org.slf4j + jcl-over-slf4j + 1.7.2 + runtime + + + org.slf4j + slf4j-log4j12 + 1.7.2 + runtime + + + log4j + log4j + 1.2.14 + runtime + + + org.springframework + spring-orm + + 3.0.5.RELEASE + + + org.hibernate + hibernate-core + 3.3.1.GA + + + javassist + javassist + 3.12.1.GA + + + c3p0 + c3p0 + 0.9.1.2 + + + org.hibernate + hibernate-ehcache + 3.3.2.GA + + + javax.annotation + javax.annotation-api + 1.3.2 + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.8.1 + + ${basedir}/src/main/java/is/landsbokasafn/deduplicator/overview.html + + + + + + + + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org/maven2 + default + + true + daily + warn + + + + + + sbforge-nexus + https://sbforge.org/nexus/content/groups/public + + true + + + true + + + + + \ No newline at end of file From c6faf62fdf7b55278c308018f467dccaeed2c538 Mon Sep 17 00:00:00 2001 From: csrster Date: Fri, 15 Jul 2022 12:20:15 +0200 Subject: [PATCH 14/24] ssh github --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 3e30954..88d3d9e 100644 --- a/pom.xml +++ b/pom.xml @@ -34,9 +34,9 @@ - scm:git://github.com/netarchivesuite/crawlrss.git - scm:git:https://github.com/netarchivesuite/crawlrss.git - https://github.com/netarchivesuite/crawlrss + scm:git:ssh://github.com/netarchivesuite/crawlrss.git + National and University Library of Iceland From f6c91c8f155d1514dee98d29c67165b7828241b8 Mon Sep 17 00:00:00 2001 From: csrster Date: Fri, 15 Jul 2022 12:29:03 +0200 Subject: [PATCH 15/24] Preparing for release --- pom.xml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index c21c208..8d2b28a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,12 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + is.landsbokasafn.crawl-rss crawl-rss 0.0.2-SNAPSHOT @@ -34,9 +35,9 @@ - scm:git://github.com/Landsbokasafn/crawlrss.git - scm:https://kris-sigur@github.com/Landsbokasafn/crawlrss.git - https://github.com/Landsbokasafn/crawlrss + scm:git:ssh://github.com/netarchivesuite/crawlrss.git + National and University Library of Iceland @@ -44,10 +45,10 @@ UTF-8 - 3.4.0-20210923 - 3.4.0-20210923 - - 1.0.0 + + 3.4.0-NAS-7.4 + 1.0.5 - is.landsbokasafn.crawl-rss - crawl-rss - 0.0.2-SNAPSHOT - Crawler RSS - RSS crawling add-on for Heritrix 3 - - - GitHub Issues - https://github.com/Landsbokasafn/crawlrss/issues - - - - Kristinn - Kristinn Sigurðsson - kristinn at landsbokasafn.is - National and University Library of Iceland - - Developer - - +0 - - - - - scm:git:ssh://github.com/netarchivesuite/crawlrss.git - - - - National and University Library of Iceland - http://www.landsbokasafn.is - - - UTF-8 - - 3.4.0-NAS-7.4 - 1.0.5 - - - - - - - sbforge-nexus - SBForge Nexus Repo manager - https://sbforge.org/nexus/content/repositories/releases - - - - sbforge-nexus - SBForge Nexus Repo manager - https://sbforge.org/nexus/content/repositories/snapshots/ - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - maven-assembly-plugin - 2.4 - - - - src/main/assemblies/dist.xml - - - ${project.artifactId}-${project.version} - - - - make-assembly - package - - single - - - - - - maven-site-plugin - 3.3 - - en - - - - com.github.github - site-maven-plugin - 0.10 - - - github - - - Creating site for - ${project.artifactId}-${project.version} - - - ${project.distributionManagement.site.url} - - true - - - - - - site - - site-deploy - - - - - - - - - rome - rome - 1.0 - - - org.archive.heritrix - heritrix-commons - ${heritrix3.version} - provided - - - org.archive.heritrix - heritrix-engine - ${heritrix3.version} - - - - org.archive.heritrix - heritrix - ${heritrix3.version} - dist - zip - - - - org.archive.heritrix - heritrix-contrib - ${heritrix3.version} - - - - org.archive.heritrix - heritrix-modules - ${heritrix3.version} - provided - - - - mysql - mysql-connector-java - 5.1.27 - - - org.hibernate - hibernate-annotations - 3.3.1.GA - - - commons-logging - commons-logging - - - hibernate - org.hibernate - - - - - org.slf4j - jcl-over-slf4j - 1.7.2 - runtime - - - org.slf4j - slf4j-log4j12 - 1.7.2 - runtime - - - log4j - log4j - 1.2.14 - runtime - - - org.springframework - spring-orm - - 3.0.5.RELEASE - - - org.hibernate - hibernate-core - 3.3.1.GA - - - javassist - javassist - 3.12.1.GA - - - c3p0 - c3p0 - 0.9.1.2 - - - org.hibernate - hibernate-ehcache - 3.3.2.GA - - - javax.annotation - javax.annotation-api - 1.3.2 - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.8.1 - - ${basedir}/src/main/java/is/landsbokasafn/deduplicator/overview.html - - - - - - - - - internetarchive - Internet Archive Maven Repository - http://builds.archive.org/maven2 - default - - true - daily - warn - - - - - - sbforge-nexus - https://sbforge.org/nexus/content/groups/public - - true - - - true - - - - - + + + 4.0.0 + + + is.landsbokasafn.crawl-rss + crawl-rss + 0.0.2 + Crawler RSS + RSS crawling add-on for Heritrix 3 + + + GitHub Issues + https://github.com/Landsbokasafn/crawlrss/issues + + + + Kristinn + Kristinn Sigurðsson + kristinn at landsbokasafn.is + National and University Library of Iceland + + Developer + + +0 + + + + + scm:git:ssh://github.com/netarchivesuite/crawlrss.git + + crawl-rss-0.0.2 + + + National and University Library of Iceland + http://www.landsbokasafn.is + + + UTF-8 + + 3.4.0-NAS-7.4 + 1.0.5 + + + + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/releases + + + + sbforge-nexus + SBForge Nexus Repo manager + https://sbforge.org/nexus/content/repositories/snapshots/ + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + UTF-8 + + + + + maven-assembly-plugin + 2.4 + + + + src/main/assemblies/dist.xml + + + ${project.artifactId}-${project.version} + + + + make-assembly + package + + single + + + + + + maven-site-plugin + 3.3 + + en + + + + com.github.github + site-maven-plugin + 0.10 + + + github + + + Creating site for + ${project.artifactId}-${project.version} + + + ${project.distributionManagement.site.url} + + true + + + + + + site + + site-deploy + + + + + + + + + rome + rome + 1.0 + + + org.archive.heritrix + heritrix-commons + ${heritrix3.version} + provided + + + org.archive.heritrix + heritrix-engine + ${heritrix3.version} + + + + org.archive.heritrix + heritrix + ${heritrix3.version} + dist + zip + + + + org.archive.heritrix + heritrix-contrib + ${heritrix3.version} + + + + org.archive.heritrix + heritrix-modules + ${heritrix3.version} + provided + + + + mysql + mysql-connector-java + 5.1.27 + + + org.hibernate + hibernate-annotations + 3.3.1.GA + + + commons-logging + commons-logging + + + hibernate + org.hibernate + + + + + org.slf4j + jcl-over-slf4j + 1.7.2 + runtime + + + org.slf4j + slf4j-log4j12 + 1.7.2 + runtime + + + log4j + log4j + 1.2.14 + runtime + + + org.springframework + spring-orm + + 3.0.5.RELEASE + + + org.hibernate + hibernate-core + 3.3.1.GA + + + javassist + javassist + 3.12.1.GA + + + c3p0 + c3p0 + 0.9.1.2 + + + org.hibernate + hibernate-ehcache + 3.3.2.GA + + + javax.annotation + javax.annotation-api + 1.3.2 + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.8.1 + + ${basedir}/src/main/java/is/landsbokasafn/deduplicator/overview.html + + + + + + + + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org/maven2 + default + + true + daily + warn + + + + + + sbforge-nexus + https://sbforge.org/nexus/content/groups/public + + true + + + true + + + + + \ No newline at end of file From 5382f1c2d345adafa54df707fed8a105b4da1f86 Mon Sep 17 00:00:00 2001 From: csrster Date: Wed, 9 Oct 2024 15:47:53 +0200 Subject: [PATCH 17/24] [maven-release-plugin] rollback the release of crawl-rss-0.0.2 --- pom.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2c51d2b..e58c05a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 is.landsbokasafn.crawl-rss crawl-rss - 0.0.2 + 0.0.2-SNAPSHOT Crawler RSS RSS crawling add-on for Heritrix 3 @@ -36,7 +38,6 @@ scm:git:ssh://github.com/netarchivesuite/crawlrss.git - crawl-rss-0.0.2 National and University Library of Iceland From 8ff5ac67c56f8e41ea8e6cc7ecc12731c2e8da51 Mon Sep 17 00:00:00 2001 From: csrster Date: Mon, 14 Oct 2024 13:23:06 +0200 Subject: [PATCH 18/24] Release 0.0.3 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e58c05a..ef4d08c 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ --> is.landsbokasafn.crawl-rss crawl-rss - 0.0.2-SNAPSHOT + 0.0.3 Crawler RSS RSS crawling add-on for Heritrix 3 @@ -47,7 +47,7 @@ UTF-8 - 3.4.0-NAS-7.4 + 3.4.0-NAS-7.6 1.0.5 From 18146dc6d9e80eebb02a9d5eff7df8f0f400fbfc Mon Sep 17 00:00:00 2001 From: csrster Date: Tue, 5 May 2026 15:11:43 +0200 Subject: [PATCH 19/24] Minor changes related to Heritrix 3.13.0 --- pom.xml | 29 +++++++++++++-- .../crawler/rss/RssCrawlController.java | 37 ++++++++++++------- .../is/landsbokasafn/crawler/rss/RssFeed.java | 2 +- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index ef4d08c..c956e50 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ --> is.landsbokasafn.crawl-rss crawl-rss - 0.0.3 + 0.0.4-SNAPSHOT Crawler RSS RSS crawling add-on for Heritrix 3 @@ -47,8 +47,8 @@ UTF-8 - 3.4.0-NAS-7.6 - 1.0.5 + 3.13.0 + 1.1.0 + + joda-time + joda-time + 2.14.1 + compile + + + + commons-httpclient + commons-httpclient + 3.1 + compile + rome rome @@ -248,11 +262,20 @@ hibernate-ehcache 3.3.2.GA + + + + jakarta.annotation + jakarta.annotation-api + 3.0.0 + compile + diff --git a/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java b/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java index 5e0f69d..b6084d5 100644 --- a/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java +++ b/src/main/java/is/landsbokasafn/crawler/rss/RssCrawlController.java @@ -26,13 +26,15 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.archive.crawler.datamodel.UriUniqFilter; import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.framework.CrawlController; @@ -142,7 +144,7 @@ public void setCrawlLogToPreloadUriUniqFilter(String log) { protected void startManagerThread() { managerThread = new Thread(this+".managerThread") { public void run() { - log.fine("Starting manager thread"); + log.info("Starting manager thread"); RssCrawlController.this.controlTasks(); } }; @@ -177,12 +179,12 @@ public void aboutToSchedule(CrawlURI curi) { protected void handleRssLink(CrawlURI curi) { // Need to ensure that all feeds associated with the site are finished before scheduling with the // frontier. Add it to the discovered items. - getSiteFor(curi).addDiscoveredItems(curi); + Optional.ofNullable(getSiteFor(curi)).ifPresent(site -> site.addDiscoveredItems(curi)); } protected void handleRssDerived(CrawlURI curi) { // Just note that the URI is in progress for the site - getSiteFor(curi).incrementInProgressURLs(); + Optional.ofNullable(getSiteFor(curi)).ifPresent(RssSite::incrementInProgressURLs); } protected void controlTasks() { @@ -194,7 +196,7 @@ protected void controlTasks() { log.log(Level.FINE,"Interrupted",e); } } - log.fine("Frontier is now running"); + log.info("Frontier is now running"); while (!shouldStop) { State state = (State)controller.getState(); if (state==State.RUNNING || state==State.EMPTY) { @@ -202,7 +204,9 @@ protected void controlTasks() { readConfig(); // Check for new sites // Trigger updates in all WAITING and ENDED sites for (RssSite site : sites.values()) { + log.info("Checking " + site.name); if (site.getState()==RssSiteState.WAITING || site.getState()==RssSiteState.ENDED) { + log.info("Updating " + site.name); site.doUpdate(); } } @@ -210,7 +214,7 @@ protected void controlTasks() { } for (RssSite site : sites.values()) { for (CrawlURI curi : site.emitReadyFeeds()) { - log.fine("Scheduling: " + curi.getURI()); + log.info("Scheduling: " + curi.getURI()); frontier.schedule(curi); } } @@ -221,27 +225,27 @@ protected void controlTasks() { log.log(Level.WARNING,"Unexpected interruption of RssCrawlController control thread", e); } } - log.fine("EXITING"); + log.info("EXITING"); } private void readConfig() { for (RssSite site : configurationManager.getSites()){ if (!sites.containsKey(site.getName())) { - log.fine("Site found in configuration: " + site.getName()); + log.info("Site found in configuration: " + site.getName()); this.sites.put(site.getName(), site); } } } - + @Override @PostConstruct public void start() { if (started) { + log.info("RssCrawlController already started."); return; } - log.fine("Starting RssCrawlController"); + log.info("Starting RssCrawlController"); started = true; - readConfig(); lastCheckedConfig=System.currentTimeMillis(); recheckConfig=configurationManager.supportsRuntimeChanges(); @@ -274,6 +278,8 @@ public void stop() { public void onApplicationEvent(ApplicationEvent event) { if(event instanceof CrawlURIDispositionEvent) { CrawlURIDispositionEvent dvent = (CrawlURIDispositionEvent)event; + log.fine(this.getClass().getName() + + " responding to " + dvent.getCrawlURI() + " " + dvent.getDisposition()); switch(dvent.getDisposition()) { case SUCCEEDED: case FAILED: @@ -292,7 +298,8 @@ public void onApplicationEvent(ApplicationEvent event) { } private void finished(CrawlURI curi){ - log.fine(curi.getURI()); + log.fine(this.getClass().getName() + " finished with " + curi.getURI() + + " " + getUriType(curi)); RssSite siteFor = getSiteFor(curi); switch (getUriType(curi)) { case RSS_FEED : @@ -320,6 +327,8 @@ private RssSite getSiteFor(CrawlURI curi) { } private RssUriType getUriType(CrawlURI curi) { + log.info("Starting RssCrawlController from getUriType()"); + this.start(); Object o = curi.getData().get(RSS_URI_TYPE); if (o==null || !(o instanceof RssUriType) ) { // This is an error, log and assume derived (extracted from regular content, not RSS feed) @@ -332,7 +341,7 @@ private RssUriType getUriType(CrawlURI curi) { @Override public void receiveDuplicate(CrawlURI curi) { log.fine(curi.getURI()); - getSiteFor(curi).decrementInProgressURLs(); + Optional.ofNullable(getSiteFor(curi)).ifPresent(RssSite::decrementInProgressURLs); } diff --git a/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java b/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java index 719cd21..5d86ae8 100644 --- a/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java +++ b/src/main/java/is/landsbokasafn/crawler/rss/RssFeed.java @@ -27,9 +27,9 @@ import java.util.Date; import java.util.List; -import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; import org.archive.net.UURIFactory; +import org.archive.url.URIException; import org.springframework.beans.factory.annotation.Autowired; public class RssFeed { From d8656b0b22b377d1acf4a523de1b7703128b4739 Mon Sep 17 00:00:00 2001 From: halh Date: Wed, 6 May 2026 11:40:44 +0200 Subject: [PATCH 20/24] Update of the CXML file with corrections from the running file in the production environment. --- .../netarkivet-crawlrss.dr.dk.cxml | 544 +++++++++++++++++- 1 file changed, 522 insertions(+), 22 deletions(-) diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml index e7ffad1..12954fe 100644 --- a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml @@ -50,9 +50,9 @@ ## Q: can overrides like 'fetchDns.enabled=false' be used to disable the beans? metadata.jobName=crawlrss_cxml -metadata.description=Default Profile +metadata.description=Default Profile with CrawlRSS metadata.operator=Admin -metadata.userAgentTemplate=Mozilla/5.0 (compatible; heritrix/3.3.0 +@OPERATOR_CONTACT_URL@) +metadata.userAgentTemplate=Mozilla/5.0 (compatible; heritrix/3.13.0 +@OPERATOR_CONTACT_URL@) ## Edit the two following lines to match your setup. metadata.operatorContactUrl=http://netarkivet.dk/webcrawler/ metadata.operatorFrom=info@netarkivet.dk @@ -64,10 +64,11 @@ crawlLimiter.maxDocumentsDownload=0 ## MaxTimeseconds inserted by NetarchiveSuite (Delete line, if behaviour unwanted) crawlLimiter.maxTimeSeconds=%{MAX_TIME_SECONDS_PLACEHOLDER} -crawlController.maxToeThreads=50 +crawlController.maxToeThreads=250 crawlController.recorderOutBufferBytes=4096 crawlController.recorderInBufferBytes=65536 crawlController.pauseAtStart=false +crawlController.runWhileEmpty=true crawlController.scratchDir=scratch ## org.archive.bdb.BdbModule overrides @@ -77,7 +78,6 @@ bdb.cachePercent=40 ## seeds properties seeds.sourceTagSeeds=false - ## Politeness overrides disposition.delayFactor=1.0 disposition.maxDelayMs=1000 @@ -132,7 +132,7 @@ fetchHttp.sendRange=false extractorHttp.enabled=true extractorHtml.enabled=true extractorHtml.extractJavascript=true -extractorHtml.treatFramesAsEmbedLinks=false +extractorHtml.treatFramesAsEmbedLinks=true extractorHtml.ignoreFormActionUrls=true extractorHtml.extractValueAttributes=false extractorHtml.ignoreUnexpectedHtml=true @@ -192,6 +192,104 @@ extractorJs.enabled=true + + + + + + + + ^https:\/\/media\.gettyimages\.com\/.* + ^https:\/\/transform-cf1\.nws\.ai\/.* + ^https:\/\/api\.nws\.ai\/.* + ^https:\/\/cdn\.ampproject\.org\/.* + ^https:\/\/consent\.cookiebot\.com\/.* + ^https:\/\/svr\.nws\.ai\/.* + ^https:\/\/trackerapi\.ebxcdn\.com\/.* + ^https:\/\/cdn\.thenewsroom\.io\/.* + ^https?:\/\/content\.jwplatform\.com\/.* + ^https?:\/\/videos-cloudfront\.jwpsrv\.com\/.* + ^https?:\/\/([^\.]+\.)?mc\.tritondigital\.com\/.* + ^https?:\/\/([^\.]+\.)?akamaized\.net\/.* + ^https?:\/\/cdn\.videosyndicate\.io.* + ^https?:\/\/cdn\.jwplayer\.com\/videos\/.* + ^https?:\/\/([^\.]+\.)cloudfront\.net\/binary + ^https?:\/\/mcdn\.podbean\.com\/mf\/web\/.* + ^https?:\/\/media\.blubrry\.com\/.*mp[34].* + ^https?:\/\/intetnytfravestfronten\.dk\/.*mp3.* + + + + + + + + + + + + + + + + + + + + + + + .*ekstrabladet\.dk.*\/eb_(noconsent|forbrug|forside|krimi|nyheder|penge|ros|sport|underholdning)(?:_|$)(?:(halfpage[1-4]|mediumrectangle|megaboard|monster|monster[1-9]|topscroll)|$)(?:_|$)(?:(scribble|artikel|dynamisk[1-4]|play|top|bund)|$) + .*ekstrabladet\.dk.*\/ebjs\.(setReady|run|ready)$ + .*ekstrabladet\.dk.*\/cog6RsN8rxSFyjh63sHFo9Uoj53Nlltr3wk8Te4ujVP\.S7$ + .*ekstrabladet\.dk/.*\/ekstrabladet\/(sport|nyheder|underholdning|forbrug|musik|ferie|nationen|plus|play)\/.*$ + ^https?:\/\/(www\.)?[a-z_-]+\.\w{2,3}.*\/mejs\.[a-z_-]+.* + .*\/gtm\.[^\/]+$ + .*\/developer_id\.dZTNiMT$ + .*\/current(_add)?\.(rf|fd|fd|typ|trm|src|mdm|cnt|cmp|tct|plt|fmt|ep)$ + .*\/udata\.(uag|vst)$ + .*\/(jw)?player\.(volume|started|ended)$ + .*\/OneSignalSDK(Updater)?Worker\.js$ + .*\/\?search=$ + .*\/mm\/dd\/yyyy$ + .*\/ekstra_bladet\/eb_noconsent\/eb_noconsent_megaboard_top_new$ + .*\.[^/]+\/.*\/wp-includes\/js\/dist\/a11y\.js$ + .*\/elementor\/lazyload\/observe$ + .*\/gform\/theme\/scripts_loaded$ + .*\/includes\/js\/index\.js$ + .*\/locations\/cities\/\?orderby=ip$ + ^https?:\/\/www\.podbean\.com\/site\/podcatcher\/.* + ^https?:\/\/podbean\.com$ + ^https?:\/\/www\.podbean\.com\/free-audiobooks$ + ^https?:\/\/www\.podbean\.com\/careers$ + ^https?:\/\/www\.podbean\.com\/$ + ^https?:\/\/media\.borsen\.dk\/size\/615\/borsen-dk\/images\/.*\.jpg\.jpg$ + ^https?:\/\/media\.borsen\.dk\/size\/615\/borsen-dk\/images\/landscape\/.*\.(JPG\.jpg|png\.png)$ + ^https?:\/\/admin5\.podbean\.com\/.* + ^https?:\/\/www\.podbean\.com\/podbean-terms$ + ^https?:\/\/www\.podbean\.com\/Podbean-partner-privacy$ + ^https?:\/\/www\.podbean\.com\/(about-us|advertise-your-podcast-on-podbean-app|affiliate|all|amwayina3rdji7t7|apple-podcasts-subscriptions|arts-podcasts|audio-recorder|awm|blog-to-podcast|BOGP|borsen27c568sv|Brand-Resources|business-podcasts|careers|christian-podcasting|church-audio-live-stream|comedy-podcasts|contact-us|cookies|create-a-podcast|customer-service-video-training-podcast-platform|dctm|dennikn|education-podcast|education-podcasts|favicon.ico|fiction-podcasts|free-audiobooks|free-guide-internal-communications-podcast|gatewaylife|government-podcasts|green-initiative|guide-to-podcast-advertising|headphone-guide-for-podcasting|health-and-fitness-podcasts|herocomplexwae35tgr|history-podcasts|HooverInstitution|how-to-monetize-a-podcast|how-to-promote-your-podcast|how-to-record-your-podcast|how-to-start-a-corporate-podcast|how-to-start-a-live-podcast|how-to-start-a-podcast|information|kids-and-family-podcasts|KPMG|leisure-podcasts|live-list|livestream|mayoclinic|missiontothemoon|mitchellinstituteaerospaceadvantageuybxqarb|mobile-podcast-recording-guide|music-podcasts|mustreadalaska|new-features|new-noteworthy-podcasts|news-podcasts|paychex|Podbean-ad-terms|Podbean-aup|Podbean-general-TOU|Podbean-infringement|Podbean-partner-privacy|Podbean-podcaster-monetize-terms|Podbean-podcaster-terms|podbean-terms|podbean-terms-ppnu|podcast-academy|podcast-ai|podcast-app-iphone-android-mobile|podcast-hosting-features|podcast-hosting-pricing|press|privacy|privacy-ppnu|private-video-hosting|religion-and-spirituality-podcasts|remote-podcast-recording|sales-training-video-podcast-plastform|science-podcasts|sermon-audio|sjxkpz2gmnu9r7|soc2-type2|society-and-culture-podcasts|sports-podcasts|start-audio-blog|start-podcast|start-video-podcast|switch-podcast-to-podbean|technology-podcasts|true-crime-podcasts|tv-and-film-podcasts|youtube-to-podcast)$ + ^https?:\/\/server\.symfony\.berlingske\.aws\.berlingskemedia\.net\/.* + ^https?:\/\/server\.symfony\.euroinvestor\.aws\.berlingskemedia\.net\/.* + ^https?:\/\/jyllands-posten\.dk\/[^\/]+\/[^\/]+\/.*\/static\/chunks\/[^\/]+\.js$ + ^https?:\/\/www\.altinget\.dk\/[^\/]+\/artikel\/.*\.[jJ][pP][gG]$ + ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*module\.js$ + ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/payment\/plus$ + ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/Europe\/Berlin$ + ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/ekstrabladet\/(krimi|penge|nyheder|sport|underholdning|side9|sex_og_samliv|plus|musik|incoming|forbrug)$ + ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/side9\/maanedens$ + ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/eb_forside\/eb_forside_megaboard_top_new$ + ^https?:\/\/(www\.)?tv2kosmopol\.dk\/.*\/%22$ + ^https?:\/\/(www\.)?danmarksfrihedsraad\.com\/.*\/core\/[^\/]+$ + ^https?:\/\/(www\.)?nordfront\.dk\/.*\/(admin-ajax\.php|matomo\.js|matomo\.php|Nordfront\.dk)$ + ^https?:\/\/(www\.)?nordfront\.dk\/.*\/(Billedtekster\/Undertekster|contact-form-7\/v1|Interview\/reportage|Multikultur\/masseindvandring|plugins\/tubepress|wp-content\/tubepress-content)$ + .*\/contact-form-7\/v1$ + + ^https?:\/\/(www\.)?pure\..+\.dk.* + + + @@ -231,13 +329,13 @@ extractorJs.enabled=true - + - + @@ -278,7 +376,278 @@ extractorJs.enabled=true - + + + + + + + ^script\.js$ + ^includes/js/index\.js$ + ^_next/static/chunks/.* + ^static/chunks/.* + ^wrapperMessagingWithoutDetection\.js$ + ^bootstrap\.js($|\?ver=.*) + ^googlemap\.js($|\?ver=.*) + ^i18n\.[^/]+$ + ^actions\.[^/]+$ + ^state\.[^/]+$ + ^react\.[^/]+$ + ^internal\.[^/]+$ + ^svStandardLib_time\.[^/]+$ + ^joomla\.[^/]+$ + ^system\.[^/]+$ + ^Period\.[^/]+$ + ^0\.[^/]+$ + ^v2\.[^/]+$ + ^fields\.[^/]+$ + ^1\.[^/]+$ + ^TextSearch\.[^/]+$ + ^4\.[^/]+$ + ^-0\.[^/]+$ + ^Areas\.[^/]+$ + ^bootstrap\.[^/]+$ + ^web\.[^/]+$ + ^12\.[^/]+$ + ^-1\.[^/]+$ + ^oneclick\.[^/]+$ + ^app\.[^/]+$ + ^slideshow\.[^/]+$ + ^widget\.[^/]+$ + ^master\.[^/]+$ + ^dualCurrency\.[^/]+$ + ^easing\.[^/]+$ + ^cookie-categories\.[^/]+$ + ^classes\.[^/]+$ + ^selectors\.[^/]+$ + ^domain-only\.[^/]+$ + ^ui\.[^/]+$ + ^wordfence_lh=.+$ + ^api/[^/]+/called$ + ^contact-form-[0-9]/v[0-9]$ + ^current_add\.[a-z][a-z]$ + ^jetpack_vendor/.*$ + ^wp/v[0-9]+/$ + + ^a\.[^/]+$ + ^p\.[^/]+$ + ^dialog\.[^/]+$ + ^change\.[^/]+$ + ^loaded\.[^/]+$ + ^div\.[^/]+$ + ^document\.[^/]+$ + ^img\.[^/]+$ + ^input\.[^/]+$ + ^window\.[^/]+$ + ^edge\.[^/]+$ + ^form\.[^/]+$ + ^li\.[^/]+$ + ^ls\.[^/]+$ + ^experimental\.[^/]+$ + ^span\.[^/]+$ + ^table\.[^/]+$ + ^ul\.[^/]+$ + ^click\.[^/]+$ + ^keydown\.[^/]+$ + ^keyup\.[^/]+$ + ^blur\.[^/]+$ + ^body\.[^/]+$ + ^paste\.[^/]+$ + ^scroll\.[^/]+$ + ^submit\.[^/]+$ + ^cut\.[^/]+$ + ^filter\.[^/]+$ + ^focus\.[^/]+$ + ^touchend\.[^/]+$ + ^touchmove\.[^/]+$ + ^touchstart\.[^/]+$ + ^mouseout\.[^/]+$ + ^mouseover\.[^/]+$ + ^mouseleave\.[^/]+$ + ^mouseenter\.[^/]+$ + ^mouseup\.[^/]+$ + ^mousedown\.[^/]+$ + ^translations\.[^/]+$ + ^sw-modern\.[^/]+$ + ^owl\.[^/]+$ + ^sw-legacy\.[^/]+$ + ^a11y\.[^/]+$ + ^gtm4wp\.[^/]+$ + ^resize\.[^/]+$ + ^mousewheel\.[^/]+$ + ^gbar(\.|_)[^/]+$ + ^stg\.[^/]+$ + ^Number\.[^/]+$ + ^View\.[^/]+$ + ^labels\.[^/]+$ + ^geometry\.[^/]+$ + ^landscape\.[^/]+$ + ^road\.[^/]+$ + ^newrelic\.[^/]+$ + ^Array\.[^/]+$ + ^Symbol\.[^/]+$ + ^Reflect\.[^/]+$ + ^browserPerformance\.[^/]+$ + ^WIZ_global_data\.[^/]+$ + ^Math\.[^/]+$ + ^core\.[^/]+$ + ^gapi\.[^/]+$ + ^error\.[^/]+$ + ^ready\.[^/]+$ + ^Expo\.[^/]+$ + ^Drone\.[^/]+$ + ^load\.[^/]+$ + ^bs\.[^/]+$ + ^NavigationControl\.[^/]+$ + ^modal\.[^/]+$ + ^HHmmss\.[^/]+$ + ^moment\.[^/]+$ + ^XFBML\.[^/]+$ + ^Sslac\.[^/]+$ + ^BaseLayout\.[^/]+$ + ^dp\.[^/]+$ + ^options\.[^/]+$ + ^layout\.[^/]+$ + ^matomo\.[^/]+$ + ^page\.[^/]+$ + ^prod\.[^/]+$ + ^comments\.[^/]+$ + ^people\.[^/]+$ + ^peoplesearch\.[^/]+$ + ^networkupdates\.[^/]+$ + ^memberupdates\.[^/]+$ + ^groups\.[^/]+$ + ^connections\.[^/]+$ + ^xdoor\.[^/]+$ + ^IN\.[^/]+$ + ^urls\.[^/]+$ + ^item\.[^/]+$ + ^additional\.[^/]+$ + ^nav\.[^/]+$ + ^current\.[^/]+$ + ^session\.[^/]+$ + ^support\.[^/]+$ + ^udata\.[^/]+$ + ^DC\.[^/]+$ + ^post-actions\.[^/]+$ + ^share-link\.[^/]+$ + ^share-form\.[^/]+$ + ^pagination\.[^/]+$ + ^like-button\.[^/]+$ + ^itemDataObject\.[^/]+$ + ^shared\.[^/]+$ + ^jquery\.[^/]+$ + ^server\.[^/]+$ + ^smartmediaClient\.[^/]+$ + ^[^/]+\.default$ + ^[^/]+\.classic$ + ^post-form(\.|-)[^/]+$ + ^[^/]+\.js$ + ^[0-9a-zA-Z][a-zA-Z]\.[0-9a-zA-Z][a-zA-Z]$ + + ^\d\d/\d\d/\d\d(\d\d)?$ + + ^.*\|.*$ + ^.*\(.*$ + ^.*\).*$ + ^.*@.*$ + ^.*%40.*$ + + ^-?\d+\.\d+$ + + ^-?\d+\.?\d*(r?em|vw|vh|px|KB|MB|GB|kg|cm|mm|s|ms)$ + + ^[a-f0-9]+\.\d+$ + ^.*\?variant=.*$ + + ^\..*$ + ^-.*$ + + ^(video|audio)/[^/]+$ + + ^[a-zA-Z-]+\.dk$ + + ^[0-9]+\.([0-9]+\.)*[0-9]+$ + ^v([0-9]+\.)*[0-9]+$ + ^[a-zA-Z]+-[0-9]+((\.|-)[0-9]+)*$ + ^Microsoft\.XMLHTTP$ + + ^mejs\.[a-zA-Z0-9-]+$ + + ^.*gtm\..*$ + + ^developer_id\..+$ + + ^Atlantic/[a-zA-Z_]+$ + ^Africa/[a-zA-Z_]+$ + ^America/[a-zA-Z_]+$ + ^Asia/[a-zA-Z_]+$ + ^Australia/[a-zA-Z_]+$ + ^Canada/[a-zA-Z_]+$ + ^Europe/[a-zA-Z_]+$ + ^node_modules/.*$ + + + + + + ace/mode/less + ace/theme/textmate + ace/ext/language_tools + api/start/ + API/ + API/noticeError/called + application.version + browser/logs + browser/blobs + Captions/Subtitles + Billedtekster/Undertekster + d/m/Y + dd.mm.yy + dd.mm.yyyy + dd/mm/yy + dd/mm/yyyy + YYYY/MM/DD + dd/mm/yyyy + DD/MM/YYYY + MM/DD/YYYY + DD/M/YYYY + MM/D/YYYY + m/d/Y + Header/Seen + init/start + elementor/frontend/init + elementor/lazyload/observe + elementor/popup/show + Filter/Dispatcher + Firefox/ + g.doubleclick + Generic/CSPViolation/Detected + gform/post_render + gform/postRender + gform/theme/scripts_loaded + heading.default + litespeed/javascript + multipart/form-data + privacy.cookies_enabled + session_trace.enabled + sitekit.withgoogle.com + ss.is_SU + ss.O_ID + ss.U_ID + text/javascript + text/css + track/ + ical=1 + n/j/Y + tribe/events + events/tag + ?rest_route= + + + @@ -301,8 +670,8 @@ extractorJs.enabled=true - - + + @@ -324,7 +693,6 @@ extractorJs.enabled=true %{ARCHIVER_PROCESSOR_BEAN_PLACEHOLDER} - @@ -355,8 +723,8 @@ extractorJs.enabled=true - + + @@ -465,7 +833,7 @@ default H3 policy is org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPo - @@ -564,13 +932,145 @@ default H3 policy is org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPo - - - - + + + + + + + + + + + + + http://(dk,politiken, + + + + + changeUserAgent + + + + + + + http://(dk,version2, + http://(dk,ing, + + + + + ipplus + + + + + + + http://(com,jwpsrv,videos-cloudfront, + http://(net,akamaized, + http://(net,cloudfront, + + + + + + prioritySheet + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + disableExtractorJS + + + + + + + + .*\?nocache=.* + .*\/wp-json\/?$ + .*\/wp-includes\/$ + .*\/cart\/?$ + .*\.min\.js\?.* + .*\.min\.css\?.* + .*\/wp-json\/wp\/v[0-9]+\/posts$ + .*\/wp-json\/wp\/v[0-9]+\/media(\/|\?parent).* + .*\/wp-json\/wp\/v[0-9]+\/pages\/[0-9]+ + .*\/wp-json\/.*media.* + .*\/wp-json\/wp\/v[0-9]+\/users\/[0-9]+ + .*\/wp-json\/wp\/v[0-9]+\/types\/.* + .*\/wp-json\/wp\/v[0-9]+\/taxonomies\.* + .*\/wp-json\/wp\/v[0-9]+\/tags\/.* + .*\/wc-blocks-data\.js[^\/]*$ + .*\/cart-frontend\.js[^\/]*$ + .*\/blocks-checkout\.js[^\/]*$ + .*\/woocommerce\/assets\/client\/blocks\/.* + .*\/wp-json\/tribe\/(tickets|events)\/v[0-9]\/$ + .*\/gtag\/js\?.* + .*\/events\/202[0-9]-[0-1][0-9]-[0-3][0-9]\/$ + .*\/_next\/static\/.* + .*rest_route=.* + .*instagram\.com\/embed\.js + .*platform\.twitter\.com\/widgets\.js + (?i).*(\/(image|font|static|chunks|app)(?=\/)){3,}.* + (?i).*(\/(Billedtekster|elementor|lazyload|includes|js)(?=\/)){3,}.* + (?i).*\/search(\/|\?).* + .*\/includes\/js\/index\.js + (?i).*\/phpMyAdmin\/.* + .*\/API\/logging\/.* + .*\/API\/register\/.* + (?i).*\/catalogsearch(\/|\?).* + .*\/wp-json\/semplice\/v1\/frontend + .*\/wrapperMessagingWithoutDetection\.js$ + + + + + + + + + + + + + + + + Date: Wed, 6 May 2026 11:44:30 +0200 Subject: [PATCH 21/24] add some files to gitignore and update --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e89ffd1..f320cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ /.project /.classpath /.settings -.idea/* \ No newline at end of file +.idea/* +/.proxyai/ +/.idea/ From f3349c73a2cd0963e276413a0356639f1675d2bf Mon Sep 17 00:00:00 2001 From: halh Date: Wed, 6 May 2026 11:45:06 +0200 Subject: [PATCH 22/24] add some files to gitignore and update --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f320cd5..5cbda34 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ .idea/* /.proxyai/ /.idea/ +/crawl-rss.iml From 946b021610ad2a9252e03d3cc0e3afc3e448d70f Mon Sep 17 00:00:00 2001 From: halh Date: Wed, 6 May 2026 14:01:02 +0200 Subject: [PATCH 23/24] Fixes after review --- .../netarkivet-crawlrss.dr.dk.cxml | 527 +----------------- 1 file changed, 13 insertions(+), 514 deletions(-) diff --git a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml index 12954fe..ed2141f 100644 --- a/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml +++ b/src/main/conf/jobs/CrawlRSS-Sample-Profile/netarkivet-crawlrss.dr.dk.cxml @@ -64,7 +64,7 @@ crawlLimiter.maxDocumentsDownload=0 ## MaxTimeseconds inserted by NetarchiveSuite (Delete line, if behaviour unwanted) crawlLimiter.maxTimeSeconds=%{MAX_TIME_SECONDS_PLACEHOLDER} -crawlController.maxToeThreads=250 +crawlController.maxToeThreads=50 crawlController.recorderOutBufferBytes=4096 crawlController.recorderInBufferBytes=65536 crawlController.pauseAtStart=false @@ -132,7 +132,7 @@ fetchHttp.sendRange=false extractorHttp.enabled=true extractorHtml.enabled=true extractorHtml.extractJavascript=true -extractorHtml.treatFramesAsEmbedLinks=true +extractorHtml.treatFramesAsEmbedLinks=false extractorHtml.ignoreFormActionUrls=true extractorHtml.extractValueAttributes=false extractorHtml.ignoreUnexpectedHtml=true @@ -192,104 +192,6 @@ extractorJs.enabled=true - - - - - - - - ^https:\/\/media\.gettyimages\.com\/.* - ^https:\/\/transform-cf1\.nws\.ai\/.* - ^https:\/\/api\.nws\.ai\/.* - ^https:\/\/cdn\.ampproject\.org\/.* - ^https:\/\/consent\.cookiebot\.com\/.* - ^https:\/\/svr\.nws\.ai\/.* - ^https:\/\/trackerapi\.ebxcdn\.com\/.* - ^https:\/\/cdn\.thenewsroom\.io\/.* - ^https?:\/\/content\.jwplatform\.com\/.* - ^https?:\/\/videos-cloudfront\.jwpsrv\.com\/.* - ^https?:\/\/([^\.]+\.)?mc\.tritondigital\.com\/.* - ^https?:\/\/([^\.]+\.)?akamaized\.net\/.* - ^https?:\/\/cdn\.videosyndicate\.io.* - ^https?:\/\/cdn\.jwplayer\.com\/videos\/.* - ^https?:\/\/([^\.]+\.)cloudfront\.net\/binary - ^https?:\/\/mcdn\.podbean\.com\/mf\/web\/.* - ^https?:\/\/media\.blubrry\.com\/.*mp[34].* - ^https?:\/\/intetnytfravestfronten\.dk\/.*mp3.* - - - - - - - - - - - - - - - - - - - - - - - .*ekstrabladet\.dk.*\/eb_(noconsent|forbrug|forside|krimi|nyheder|penge|ros|sport|underholdning)(?:_|$)(?:(halfpage[1-4]|mediumrectangle|megaboard|monster|monster[1-9]|topscroll)|$)(?:_|$)(?:(scribble|artikel|dynamisk[1-4]|play|top|bund)|$) - .*ekstrabladet\.dk.*\/ebjs\.(setReady|run|ready)$ - .*ekstrabladet\.dk.*\/cog6RsN8rxSFyjh63sHFo9Uoj53Nlltr3wk8Te4ujVP\.S7$ - .*ekstrabladet\.dk/.*\/ekstrabladet\/(sport|nyheder|underholdning|forbrug|musik|ferie|nationen|plus|play)\/.*$ - ^https?:\/\/(www\.)?[a-z_-]+\.\w{2,3}.*\/mejs\.[a-z_-]+.* - .*\/gtm\.[^\/]+$ - .*\/developer_id\.dZTNiMT$ - .*\/current(_add)?\.(rf|fd|fd|typ|trm|src|mdm|cnt|cmp|tct|plt|fmt|ep)$ - .*\/udata\.(uag|vst)$ - .*\/(jw)?player\.(volume|started|ended)$ - .*\/OneSignalSDK(Updater)?Worker\.js$ - .*\/\?search=$ - .*\/mm\/dd\/yyyy$ - .*\/ekstra_bladet\/eb_noconsent\/eb_noconsent_megaboard_top_new$ - .*\.[^/]+\/.*\/wp-includes\/js\/dist\/a11y\.js$ - .*\/elementor\/lazyload\/observe$ - .*\/gform\/theme\/scripts_loaded$ - .*\/includes\/js\/index\.js$ - .*\/locations\/cities\/\?orderby=ip$ - ^https?:\/\/www\.podbean\.com\/site\/podcatcher\/.* - ^https?:\/\/podbean\.com$ - ^https?:\/\/www\.podbean\.com\/free-audiobooks$ - ^https?:\/\/www\.podbean\.com\/careers$ - ^https?:\/\/www\.podbean\.com\/$ - ^https?:\/\/media\.borsen\.dk\/size\/615\/borsen-dk\/images\/.*\.jpg\.jpg$ - ^https?:\/\/media\.borsen\.dk\/size\/615\/borsen-dk\/images\/landscape\/.*\.(JPG\.jpg|png\.png)$ - ^https?:\/\/admin5\.podbean\.com\/.* - ^https?:\/\/www\.podbean\.com\/podbean-terms$ - ^https?:\/\/www\.podbean\.com\/Podbean-partner-privacy$ - ^https?:\/\/www\.podbean\.com\/(about-us|advertise-your-podcast-on-podbean-app|affiliate|all|amwayina3rdji7t7|apple-podcasts-subscriptions|arts-podcasts|audio-recorder|awm|blog-to-podcast|BOGP|borsen27c568sv|Brand-Resources|business-podcasts|careers|christian-podcasting|church-audio-live-stream|comedy-podcasts|contact-us|cookies|create-a-podcast|customer-service-video-training-podcast-platform|dctm|dennikn|education-podcast|education-podcasts|favicon.ico|fiction-podcasts|free-audiobooks|free-guide-internal-communications-podcast|gatewaylife|government-podcasts|green-initiative|guide-to-podcast-advertising|headphone-guide-for-podcasting|health-and-fitness-podcasts|herocomplexwae35tgr|history-podcasts|HooverInstitution|how-to-monetize-a-podcast|how-to-promote-your-podcast|how-to-record-your-podcast|how-to-start-a-corporate-podcast|how-to-start-a-live-podcast|how-to-start-a-podcast|information|kids-and-family-podcasts|KPMG|leisure-podcasts|live-list|livestream|mayoclinic|missiontothemoon|mitchellinstituteaerospaceadvantageuybxqarb|mobile-podcast-recording-guide|music-podcasts|mustreadalaska|new-features|new-noteworthy-podcasts|news-podcasts|paychex|Podbean-ad-terms|Podbean-aup|Podbean-general-TOU|Podbean-infringement|Podbean-partner-privacy|Podbean-podcaster-monetize-terms|Podbean-podcaster-terms|podbean-terms|podbean-terms-ppnu|podcast-academy|podcast-ai|podcast-app-iphone-android-mobile|podcast-hosting-features|podcast-hosting-pricing|press|privacy|privacy-ppnu|private-video-hosting|religion-and-spirituality-podcasts|remote-podcast-recording|sales-training-video-podcast-plastform|science-podcasts|sermon-audio|sjxkpz2gmnu9r7|soc2-type2|society-and-culture-podcasts|sports-podcasts|start-audio-blog|start-podcast|start-video-podcast|switch-podcast-to-podbean|technology-podcasts|true-crime-podcasts|tv-and-film-podcasts|youtube-to-podcast)$ - ^https?:\/\/server\.symfony\.berlingske\.aws\.berlingskemedia\.net\/.* - ^https?:\/\/server\.symfony\.euroinvestor\.aws\.berlingskemedia\.net\/.* - ^https?:\/\/jyllands-posten\.dk\/[^\/]+\/[^\/]+\/.*\/static\/chunks\/[^\/]+\.js$ - ^https?:\/\/www\.altinget\.dk\/[^\/]+\/artikel\/.*\.[jJ][pP][gG]$ - ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*module\.js$ - ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/payment\/plus$ - ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/Europe\/Berlin$ - ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/ekstrabladet\/(krimi|penge|nyheder|sport|underholdning|side9|sex_og_samliv|plus|musik|incoming|forbrug)$ - ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/side9\/maanedens$ - ^https?:\/\/ekstrabladet\.dk\/[^\/]+\/.*\/eb_forside\/eb_forside_megaboard_top_new$ - ^https?:\/\/(www\.)?tv2kosmopol\.dk\/.*\/%22$ - ^https?:\/\/(www\.)?danmarksfrihedsraad\.com\/.*\/core\/[^\/]+$ - ^https?:\/\/(www\.)?nordfront\.dk\/.*\/(admin-ajax\.php|matomo\.js|matomo\.php|Nordfront\.dk)$ - ^https?:\/\/(www\.)?nordfront\.dk\/.*\/(Billedtekster\/Undertekster|contact-form-7\/v1|Interview\/reportage|Multikultur\/masseindvandring|plugins\/tubepress|wp-content\/tubepress-content)$ - .*\/contact-form-7\/v1$ - - ^https?:\/\/(www\.)?pure\..+\.dk.* - - - @@ -333,9 +235,9 @@ extractorJs.enabled=true - + @@ -376,278 +278,7 @@ extractorJs.enabled=true - - - - - - - ^script\.js$ - ^includes/js/index\.js$ - ^_next/static/chunks/.* - ^static/chunks/.* - ^wrapperMessagingWithoutDetection\.js$ - ^bootstrap\.js($|\?ver=.*) - ^googlemap\.js($|\?ver=.*) - ^i18n\.[^/]+$ - ^actions\.[^/]+$ - ^state\.[^/]+$ - ^react\.[^/]+$ - ^internal\.[^/]+$ - ^svStandardLib_time\.[^/]+$ - ^joomla\.[^/]+$ - ^system\.[^/]+$ - ^Period\.[^/]+$ - ^0\.[^/]+$ - ^v2\.[^/]+$ - ^fields\.[^/]+$ - ^1\.[^/]+$ - ^TextSearch\.[^/]+$ - ^4\.[^/]+$ - ^-0\.[^/]+$ - ^Areas\.[^/]+$ - ^bootstrap\.[^/]+$ - ^web\.[^/]+$ - ^12\.[^/]+$ - ^-1\.[^/]+$ - ^oneclick\.[^/]+$ - ^app\.[^/]+$ - ^slideshow\.[^/]+$ - ^widget\.[^/]+$ - ^master\.[^/]+$ - ^dualCurrency\.[^/]+$ - ^easing\.[^/]+$ - ^cookie-categories\.[^/]+$ - ^classes\.[^/]+$ - ^selectors\.[^/]+$ - ^domain-only\.[^/]+$ - ^ui\.[^/]+$ - ^wordfence_lh=.+$ - ^api/[^/]+/called$ - ^contact-form-[0-9]/v[0-9]$ - ^current_add\.[a-z][a-z]$ - ^jetpack_vendor/.*$ - ^wp/v[0-9]+/$ - - ^a\.[^/]+$ - ^p\.[^/]+$ - ^dialog\.[^/]+$ - ^change\.[^/]+$ - ^loaded\.[^/]+$ - ^div\.[^/]+$ - ^document\.[^/]+$ - ^img\.[^/]+$ - ^input\.[^/]+$ - ^window\.[^/]+$ - ^edge\.[^/]+$ - ^form\.[^/]+$ - ^li\.[^/]+$ - ^ls\.[^/]+$ - ^experimental\.[^/]+$ - ^span\.[^/]+$ - ^table\.[^/]+$ - ^ul\.[^/]+$ - ^click\.[^/]+$ - ^keydown\.[^/]+$ - ^keyup\.[^/]+$ - ^blur\.[^/]+$ - ^body\.[^/]+$ - ^paste\.[^/]+$ - ^scroll\.[^/]+$ - ^submit\.[^/]+$ - ^cut\.[^/]+$ - ^filter\.[^/]+$ - ^focus\.[^/]+$ - ^touchend\.[^/]+$ - ^touchmove\.[^/]+$ - ^touchstart\.[^/]+$ - ^mouseout\.[^/]+$ - ^mouseover\.[^/]+$ - ^mouseleave\.[^/]+$ - ^mouseenter\.[^/]+$ - ^mouseup\.[^/]+$ - ^mousedown\.[^/]+$ - ^translations\.[^/]+$ - ^sw-modern\.[^/]+$ - ^owl\.[^/]+$ - ^sw-legacy\.[^/]+$ - ^a11y\.[^/]+$ - ^gtm4wp\.[^/]+$ - ^resize\.[^/]+$ - ^mousewheel\.[^/]+$ - ^gbar(\.|_)[^/]+$ - ^stg\.[^/]+$ - ^Number\.[^/]+$ - ^View\.[^/]+$ - ^labels\.[^/]+$ - ^geometry\.[^/]+$ - ^landscape\.[^/]+$ - ^road\.[^/]+$ - ^newrelic\.[^/]+$ - ^Array\.[^/]+$ - ^Symbol\.[^/]+$ - ^Reflect\.[^/]+$ - ^browserPerformance\.[^/]+$ - ^WIZ_global_data\.[^/]+$ - ^Math\.[^/]+$ - ^core\.[^/]+$ - ^gapi\.[^/]+$ - ^error\.[^/]+$ - ^ready\.[^/]+$ - ^Expo\.[^/]+$ - ^Drone\.[^/]+$ - ^load\.[^/]+$ - ^bs\.[^/]+$ - ^NavigationControl\.[^/]+$ - ^modal\.[^/]+$ - ^HHmmss\.[^/]+$ - ^moment\.[^/]+$ - ^XFBML\.[^/]+$ - ^Sslac\.[^/]+$ - ^BaseLayout\.[^/]+$ - ^dp\.[^/]+$ - ^options\.[^/]+$ - ^layout\.[^/]+$ - ^matomo\.[^/]+$ - ^page\.[^/]+$ - ^prod\.[^/]+$ - ^comments\.[^/]+$ - ^people\.[^/]+$ - ^peoplesearch\.[^/]+$ - ^networkupdates\.[^/]+$ - ^memberupdates\.[^/]+$ - ^groups\.[^/]+$ - ^connections\.[^/]+$ - ^xdoor\.[^/]+$ - ^IN\.[^/]+$ - ^urls\.[^/]+$ - ^item\.[^/]+$ - ^additional\.[^/]+$ - ^nav\.[^/]+$ - ^current\.[^/]+$ - ^session\.[^/]+$ - ^support\.[^/]+$ - ^udata\.[^/]+$ - ^DC\.[^/]+$ - ^post-actions\.[^/]+$ - ^share-link\.[^/]+$ - ^share-form\.[^/]+$ - ^pagination\.[^/]+$ - ^like-button\.[^/]+$ - ^itemDataObject\.[^/]+$ - ^shared\.[^/]+$ - ^jquery\.[^/]+$ - ^server\.[^/]+$ - ^smartmediaClient\.[^/]+$ - ^[^/]+\.default$ - ^[^/]+\.classic$ - ^post-form(\.|-)[^/]+$ - ^[^/]+\.js$ - ^[0-9a-zA-Z][a-zA-Z]\.[0-9a-zA-Z][a-zA-Z]$ - - ^\d\d/\d\d/\d\d(\d\d)?$ - - ^.*\|.*$ - ^.*\(.*$ - ^.*\).*$ - ^.*@.*$ - ^.*%40.*$ - - ^-?\d+\.\d+$ - - ^-?\d+\.?\d*(r?em|vw|vh|px|KB|MB|GB|kg|cm|mm|s|ms)$ - - ^[a-f0-9]+\.\d+$ - ^.*\?variant=.*$ - - ^\..*$ - ^-.*$ - - ^(video|audio)/[^/]+$ - - ^[a-zA-Z-]+\.dk$ - - ^[0-9]+\.([0-9]+\.)*[0-9]+$ - ^v([0-9]+\.)*[0-9]+$ - ^[a-zA-Z]+-[0-9]+((\.|-)[0-9]+)*$ - ^Microsoft\.XMLHTTP$ - - ^mejs\.[a-zA-Z0-9-]+$ - - ^.*gtm\..*$ - - ^developer_id\..+$ - - ^Atlantic/[a-zA-Z_]+$ - ^Africa/[a-zA-Z_]+$ - ^America/[a-zA-Z_]+$ - ^Asia/[a-zA-Z_]+$ - ^Australia/[a-zA-Z_]+$ - ^Canada/[a-zA-Z_]+$ - ^Europe/[a-zA-Z_]+$ - ^node_modules/.*$ - - - - - - ace/mode/less - ace/theme/textmate - ace/ext/language_tools - api/start/ - API/ - API/noticeError/called - application.version - browser/logs - browser/blobs - Captions/Subtitles - Billedtekster/Undertekster - d/m/Y - dd.mm.yy - dd.mm.yyyy - dd/mm/yy - dd/mm/yyyy - YYYY/MM/DD - dd/mm/yyyy - DD/MM/YYYY - MM/DD/YYYY - DD/M/YYYY - MM/D/YYYY - m/d/Y - Header/Seen - init/start - elementor/frontend/init - elementor/lazyload/observe - elementor/popup/show - Filter/Dispatcher - Firefox/ - g.doubleclick - Generic/CSPViolation/Detected - gform/post_render - gform/postRender - gform/theme/scripts_loaded - heading.default - litespeed/javascript - multipart/form-data - privacy.cookies_enabled - session_trace.enabled - sitekit.withgoogle.com - ss.is_SU - ss.O_ID - ss.U_ID - text/javascript - text/css - track/ - ical=1 - n/j/Y - tribe/events - events/tag - ?rest_route= - - - + @@ -723,8 +354,8 @@ extractorJs.enabled=true - - + @@ -932,145 +563,13 @@ default H3 policy is org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPo - - - - - - - - - - - - http://(dk,politiken, - - - - - changeUserAgent - - - - - - - http://(dk,version2, - http://(dk,ing, - - - - - ipplus - - - - - - - http://(com,jwpsrv,videos-cloudfront, - http://(net,akamaized, - http://(net,cloudfront, - - - - - - prioritySheet - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - disableExtractorJS - - - - - - - - .*\?nocache=.* - .*\/wp-json\/?$ - .*\/wp-includes\/$ - .*\/cart\/?$ - .*\.min\.js\?.* - .*\.min\.css\?.* - .*\/wp-json\/wp\/v[0-9]+\/posts$ - .*\/wp-json\/wp\/v[0-9]+\/media(\/|\?parent).* - .*\/wp-json\/wp\/v[0-9]+\/pages\/[0-9]+ - .*\/wp-json\/.*media.* - .*\/wp-json\/wp\/v[0-9]+\/users\/[0-9]+ - .*\/wp-json\/wp\/v[0-9]+\/types\/.* - .*\/wp-json\/wp\/v[0-9]+\/taxonomies\.* - .*\/wp-json\/wp\/v[0-9]+\/tags\/.* - .*\/wc-blocks-data\.js[^\/]*$ - .*\/cart-frontend\.js[^\/]*$ - .*\/blocks-checkout\.js[^\/]*$ - .*\/woocommerce\/assets\/client\/blocks\/.* - .*\/wp-json\/tribe\/(tickets|events)\/v[0-9]\/$ - .*\/gtag\/js\?.* - .*\/events\/202[0-9]-[0-1][0-9]-[0-3][0-9]\/$ - .*\/_next\/static\/.* - .*rest_route=.* - .*instagram\.com\/embed\.js - .*platform\.twitter\.com\/widgets\.js - (?i).*(\/(image|font|static|chunks|app)(?=\/)){3,}.* - (?i).*(\/(Billedtekster|elementor|lazyload|includes|js)(?=\/)){3,}.* - (?i).*\/search(\/|\?).* - .*\/includes\/js\/index\.js - (?i).*\/phpMyAdmin\/.* - .*\/API\/logging\/.* - .*\/API\/register\/.* - (?i).*\/catalogsearch(\/|\?).* - .*\/wp-json\/semplice\/v1\/frontend - .*\/wrapperMessagingWithoutDetection\.js$ - - - - - - - - - - - - - - - - + + + Date: Wed, 6 May 2026 14:05:08 +0200 Subject: [PATCH 24/24] pam update to heritrix 3.14.1 --- pom.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index c956e50..529df67 100644 --- a/pom.xml +++ b/pom.xml @@ -45,10 +45,7 @@ UTF-8 - - 3.13.0 - 1.1.0 + 3.14.1