diff --git a/.gitignore b/.gitignore index a14af547..5bfc248f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ project/plugins/project/ .classpath .project /bin/ + +#IntelliJ +.idea/ diff --git a/Figaro/.gitignore b/Figaro/.gitignore index ae3c1726..e6c44a3c 100644 --- a/Figaro/.gitignore +++ b/Figaro/.gitignore @@ -1 +1,4 @@ /bin/ +/.cache-main +/.cache-tests +/build.properties diff --git a/Figaro/META-INF/MANIFEST.MF b/Figaro/META-INF/MANIFEST.MF index 52ab34a2..7347f6ef 100644 --- a/Figaro/META-INF/MANIFEST.MF +++ b/Figaro/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Figaro Bundle-SymbolicName: com.cra.figaro -Bundle-Version: 3.2.0 +Bundle-Version: 4.1.0 Export-Package: com.cra.figaro.algorithm, com.cra.figaro.algorithm.decision, com.cra.figaro.algorithm.decision.index, @@ -13,6 +13,7 @@ Export-Package: com.cra.figaro.algorithm, com.cra.figaro.language, com.cra.figaro.library.atomic.continuous, com.cra.figaro.library.atomic.discrete, + com.cra.figaro.library.collection, com.cra.figaro.library.compound, com.cra.figaro.library.decision, com.cra.figaro.util @@ -20,9 +21,8 @@ Bundle-Vendor: Charles River Analytics Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Require-Bundle: org.scala-lang.scala-library, org.scala-lang.scala-reflect, - com.typesafe.akka.actor;bundle-version="2.3.3", - com.typesafe.config;bundle-version="1.2.1", - org.scalatest;bundle-version="2.1.6" + com.typesafe.config, + com.typesafe.akka.actor Import-Package: org.apache.commons.math3.distribution;version="3.3.0" diff --git a/Figaro/figaro_build.properties b/Figaro/figaro_build.properties index cdb3799a..358c657e 100644 --- a/Figaro/figaro_build.properties +++ b/Figaro/figaro_build.properties @@ -1 +1 @@ -version=3.2.0.0 +version=4.1.0.0 diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/Algorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/Algorithm.scala index f28d70a2..ab0fa9a9 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/Algorithm.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/Algorithm.scala @@ -26,7 +26,6 @@ class UnsupportedAlgorithmException(element: Element[_]) extends RuntimeExceptio class AlgorithmException extends RuntimeException class AlgorithmInactiveException extends AlgorithmException class AlgorithmActiveException extends AlgorithmException -class NotATargetException[T](target: Element[T]) extends AlgorithmException /** * The general class of Figaro algorithms. The Algorithm class is defined to generalize both @@ -47,22 +46,22 @@ trait Algorithm { /* * Start the algorithm. After it returns, the algorithm must be ready to provide answers. */ - protected def doStart(): Unit + protected[algorithm] def doStart(): Unit /* * Stop the algorithm from computing. The algorithm is still ready to provide answers after it returns. */ - protected def doStop(): Unit + protected[algorithm] def doStop(): Unit /* * Resume the computation of the algorithm, if it has been stopped. */ - protected def doResume(): Unit + protected[algorithm] def doResume(): Unit /* * Kill the algorithm so that it is inactive. It will no longer be able to provide answers. */ - protected def doKill(): Unit + protected[algorithm] def doKill(): Unit protected var active = false diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/Anytime.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/Anytime.scala index e885a1bb..644b0920 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/Anytime.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/Anytime.scala @@ -16,6 +16,13 @@ package com.cra.figaro.algorithm import com.cra.figaro.language._ import akka.actor._ import com.typesafe.config.ConfigFactory +import akka.util.Timeout +import java.util.concurrent.TimeUnit +import akka.pattern.{ ask } +import scala.concurrent.Await +import scala.concurrent.Future +import java.util.concurrent.TimeoutException +import scala.concurrent.duration.Duration /** * Class of services implemented by the anytime algorithm. @@ -26,8 +33,14 @@ abstract class Service * Class of responses to services. */ abstract class Response + +/** + * Ack Response (String) + */ +case object AckResponse extends Response + /** - * General Response (String) + * Exception Response (String) */ case class ExceptionResponse(msg: String) extends Response @@ -40,6 +53,9 @@ sealed abstract class Message */ case class Handle(service: Service) extends Message + +class AnytimeAlgorithmException(s: String) extends RuntimeException(s) + /** * An anytime algorithm is able to improve its estimated answers over time. Anytime algorithms run in their * own thread using an actor. @@ -50,15 +66,6 @@ case class Handle(service: Service) extends Message */ trait Anytime extends Algorithm { - /** - * Run a single step of the algorithm. The algorithm must be able to provide answers after each step. - */ - def runStep(): Unit - - /** - * Optional function to run when the algorithm is stopped (not killed). Used in samplers to update lazy values. - */ - def stopUpdate(): Unit = { } /** * A class representing the actor running the algorithm. @@ -71,7 +78,8 @@ trait Anytime extends Algorithm { sender ! handle(service) case "stop" => stopUpdate() - become (inactive) + sender ! AckResponse + become(inactive) case "next" => runStep() self ! "next" @@ -89,9 +97,11 @@ trait Anytime extends Algorithm { case "resume" => resume() become(active) - self ! "next" + self ! "next" case "kill" => - become(shuttingDown) + cleanUp() + sender ! AckResponse + become(shuttingDown) case _ => sender ! ExceptionResponse("Algorithm is stopped") } @@ -102,8 +112,6 @@ trait Anytime extends Algorithm { } def receive = inactive - - } /** @@ -121,28 +129,43 @@ trait Anytime extends Algorithm { var runner: ActorRef = null var running = false; + /** + * default message timeout. Increase if queries to the algorithm fail due to timeout + */ + implicit var messageTimeout = Timeout(5000, TimeUnit.MILLISECONDS) + + /** + * Run a single step of the algorithm. The algorithm must be able to provide answers after each step. + */ + def runStep(): Unit + + /** + * Optional function to run when the algorithm is stopped (not killed). Used in samplers to update lazy values. + */ + def stopUpdate(): Unit = {} + /** * A handler of services provided by the algorithm. */ def handle(service: Service): Response - protected def doStart() = { + protected[algorithm] def doStart() = { if (!running) { - system = ActorSystem("Anytime", ConfigFactory.load(customConf)) - runner = system.actorOf(Props(new Runner)) - initialize() - running = true + system = ActorSystem("Anytime", ConfigFactory.load(customConf)) + runner = system.actorOf(Props(new Runner)) + initialize() + running = true } runner ! "start" } - protected def doStop() = runner ! "stop" + protected[algorithm] def doStop() = runner ! "stop" - protected def doResume() = runner ! "resume" + protected[algorithm] def doResume() = runner ! "resume" - protected def doKill() = { + protected[algorithm] def doKill() = { shutdown } @@ -150,12 +173,36 @@ trait Anytime extends Algorithm { * Release all resources from this anytime algorithm. */ def shutdown { - cleanUp() - if (running) - { - runner ! "kill" - system.stop(runner) - system.shutdown + if (running) { + awaitResponse(runner ? "kill", messageTimeout.duration) + system.stop(runner) + system.shutdown } } + + /* + * A helper function to query the running thread and await a response. + * In the case that it times out, it will print a message that it timed out and return an exception response. + * Note, on a time, it does NOT throw an exception. + */ + protected def awaitResponse(response: Future[Any], duration: Duration): Response = { + try { + val result = Await.result(response, duration) + result match { + case e: ExceptionResponse => { + println(e.msg) + e + } + case r: Response => r + case _ => throw new AnytimeAlgorithmException("Unknown Response") + } + } catch { + case to: TimeoutException => { + println("Error! Did not receive a response from algorithm thread - it may be hanging or taking an exceptionally long time to respond. Try increasing messageTimeout.") + ExceptionResponse("Timeout") + } + case e: Exception => throw e + } + } + } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeMPE.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeMPE.scala index b0d1c98f..d6038ec9 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeMPE.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeMPE.scala @@ -14,6 +14,7 @@ package com.cra.figaro.algorithm import com.cra.figaro.language._ +import akka.pattern.{ask} /** * Anytime algorithms that compute most likely values of elements. @@ -25,7 +26,7 @@ trait AnytimeMPE extends MPEAlgorithm with Anytime { * A message instructing the handler to compute the most likely value of the target element. */ case class ComputeMostLikelyValue[T](target: Element[T]) extends Service - + /** * A message from the handler containing the most likely value of the previously requested element. */ @@ -36,4 +37,14 @@ trait AnytimeMPE extends MPEAlgorithm with Anytime { case ComputeMostLikelyValue(target) => MostLikelyValue(mostLikelyValue(target)) } + + protected def doMostLikelyValue[T](target: Element[T]): T = { + awaitResponse(runner ? Handle(ComputeMostLikelyValue(target)), messageTimeout.duration) match { + case MostLikelyValue(result) => result.asInstanceOf[T] + case _ => { + println("Error: Response not recognized from algorithm") + target.value + } + } + } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbEvidence.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbEvidence.scala index 3c1c953d..9178d438 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbEvidence.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbEvidence.scala @@ -37,18 +37,12 @@ trait AnytimeProbEvidence extends ProbEvidenceAlgorithm with Anytime { /** * Returns the probability of evidence of the universe on which the algorithm operates. * Throws AlgorithmInactiveException if the algorithm is not active. - */ - implicit val timeout = Timeout(5000, TimeUnit.MILLISECONDS) + */ def probabilityOfEvidence(): Double = { - if (!active) throw new AlgorithmInactiveException - val response = runner ? Handle(ComputeProbEvidence) - Await.result(response, timeout.duration ).asInstanceOf[Response] match { + awaitResponse(runner ? Handle(ComputeProbEvidence), messageTimeout.duration) match { case ProbEvidence(result) => result - case ExceptionResponse(msg) => - println(msg) - 0.0 case _ => 0.0 - } + } } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbQuery.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbQuery.scala index 29339a0f..c30d515c 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbQuery.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/AnytimeProbQuery.scala @@ -50,6 +50,14 @@ trait AnytimeProbQuery extends ProbQueryAlgorithm with Anytime { * A message from the handler containing the probability of the previously requested predicate and element. */ case class Probability(probability: Double) extends Response + /** + * A message instructing the handler to compute the projection of the target element. + */ + case class ComputeProjection[T](target: Element[T]) extends Service + /** + * A message from the handler containing the projection of the previously requested element. + */ + case class Projection[T](projection: List[(T, Double)]) extends Response def handle(service: Service): Response = service match { @@ -60,38 +68,33 @@ trait AnytimeProbQuery extends ProbQueryAlgorithm with Anytime { case ComputeProbability(target, predicate) => Probability(computeProbability(target, predicate)) } - - implicit val timeout = Timeout(5000, TimeUnit.MILLISECONDS) + protected def doDistribution[T](target: Element[T]): Stream[(Double, T)] = { - val response = runner ? Handle(ComputeDistribution(target)) - Await.result(response, timeout.duration ).asInstanceOf[Response] match { + awaitResponse(runner ? Handle(ComputeDistribution(target)), messageTimeout.duration) match { case Distribution(result) => result.asInstanceOf[Stream[(Double, T)]] - case ExceptionResponse(msg) => - println(msg) - Stream() case _ => Stream() - } + } } protected def doExpectation[T](target: Element[T], function: T => Double): Double = { - val response = runner ? Handle(ComputeExpectation(target, function)) - Await.result(response, timeout.duration ).asInstanceOf[Response] match { + awaitResponse(runner ? Handle(ComputeExpectation(target, function)), messageTimeout.duration) match { case Expectation(result) => result - case ExceptionResponse(msg) => - println(msg) - 0.0 case _ => 0.0 - } + } } protected override def doProbability[T](target: Element[T], predicate: T => Boolean): Double = { - val response = runner ? Handle(ComputeProbability(target, predicate)) - Await.result(response, timeout.duration ).asInstanceOf[Response] match { + awaitResponse(runner ? Handle(ComputeProbability(target, predicate)), messageTimeout.duration) match { case Probability(result) => result - case ExceptionResponse(msg) => - println(msg) - 0.0 case _ => 0.0 - } + } } + + protected override def doProjection[T](target: Element[T]): List[(T, Double)] = { + awaitResponse(runner ? Handle(ComputeProjection(target)), messageTimeout.duration) match { + case Projection(result) => result.asInstanceOf[List[(T, Double)]] + case _ => List() + } + } + } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/MPEAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/MPEAlgorithm.scala index 95bceb1c..b70b00e9 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/MPEAlgorithm.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/MPEAlgorithm.scala @@ -28,4 +28,7 @@ trait MPEAlgorithm extends Algorithm { * Returns the most likely value for the target element. */ def mostLikelyValue[T](target: Element[T]): T + + // Defined in one time or anytime versions + protected def doMostLikelyValue[T](target: Element[T]): T } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTime.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTime.scala index 61ad309c..5f00699a 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTime.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTime.scala @@ -23,16 +23,16 @@ trait OneTime extends Algorithm { */ def run(): Unit - protected def doStart() = { + protected[algorithm] def doStart() = { initialize() run() } - protected def doStop() = () + protected[algorithm] def doStop() = () - protected def doResume() = () + protected[algorithm] def doResume() = () - protected def doKill() = { + protected[algorithm] def doKill() = { cleanUp() } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeMPE.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeMPE.scala index f89a7505..3d590a86 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeMPE.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeMPE.scala @@ -19,4 +19,6 @@ import com.cra.figaro.language._ * One-time algorithms that compute the most likely values of elements. * A class that implements this trait must implement run and mostLikelyValue methods. */ -trait OneTimeMPE extends MPEAlgorithm with OneTime +trait OneTimeMPE extends MPEAlgorithm with OneTime { + protected def doMostLikelyValue[T](target: Element[T]): T = mostLikelyValue(target) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeProbQuery.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeProbQuery.scala index 48cb5dab..2cfe00f3 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeProbQuery.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/OneTimeProbQuery.scala @@ -29,5 +29,7 @@ trait OneTimeProbQuery extends ProbQueryAlgorithm with OneTime { protected def doProbability[T](target: Element[T], predicate: T => Boolean): Double = { computeProbability(target, predicate) } + + override protected def doProjection[T](target: Element[T]): List[(T, Double)] = computeProjection(target) } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbEvidenceQuery.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbEvidenceQuery.scala new file mode 100644 index 00000000..592c29c2 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbEvidenceQuery.scala @@ -0,0 +1,29 @@ +/* + * ProbEvidenceQuery.scala + * Algorithms that provide a query for the probability of some named evidence. + * + * Created By: Lee Kellogg + * Creation Date: May 11, 2015 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm + +import com.cra.figaro.language.NamedEvidence + +/** + * Algorithms that provide a query for the probability of some named evidence + */ +trait ProbEvidenceQuery { + + /** + * Compute the probability of the given named evidence. + * Takes the conditions and constraints in the model as part of the model definition. + * This method takes care of creating and running the necessary algorithms. + */ + def probabilityOfEvidence(evidence: List[NamedEvidence[_]]): Double +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbQueryAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbQueryAlgorithm.scala index 68c302c0..fd5a0228 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbQueryAlgorithm.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/ProbQueryAlgorithm.scala @@ -14,18 +14,39 @@ package com.cra.figaro.algorithm import com.cra.figaro.language._ +import scala.language.higherKinds /** - * Algorithms that compute conditional probabilities of queries. - * + * Algorithms that compute conditional probabilities of queries over elements in a universe. */ -trait ProbQueryAlgorithm - extends Algorithm { +trait ProbQueryAlgorithm extends BaseProbQueryAlgorithm[Element] { + val universe: Universe + + /** + * Return an element representing the posterior probability distribution of the given element. + */ + def posteriorElement[T](target: Element[T], universe: Universe = Universe.universe): Element[T] = { + Select(distribution(target).toList:_*)("", universe) + } + + universe.registerAlgorithm(this) +} + + +/** + * Algorithms that compute conditional probabilities of queries. This is a base trait, to provide + * support for both elements in a single universe, or references across multiple universes. + * Generic type U is either an Element or a Reference. T is the type of the element or reference. + */ +trait BaseProbQueryAlgorithm[U[_]] + extends Algorithm { + + class NotATargetException[T](target: U[T]) extends AlgorithmException /* * @param targets List of elements that can be queried after running the algorithm. */ - val queryTargets: Seq[Element[_]] + val queryTargets: Seq[U[_]] /* * Particular implementations of algorithm must provide the following two methods. */ @@ -35,21 +56,29 @@ trait ProbQueryAlgorithm * with its probability. The result is a lazy stream. It is up to the algorithm how the stream is * ordered. */ - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] + def computeDistribution[T](target: U[T]): Stream[(Double, T)] /** * Return an estimate of the expectation of the function under the marginal probability distribution * of the target. */ - def computeExpectation[T](target: Element[T], function: T => Double): Double + def computeExpectation[T](target: U[T], function: T => Double): Double /** * Return an estimate of the probability of the predicate under the marginal probability distribution * of the target. */ - def computeProbability[T](target: Element[T], predicate: T => Boolean): Double = { + def computeProbability[T](target: U[T], predicate: T => Boolean): Double = { computeExpectation(target, (t: T) => if (predicate(t)) 1.0; else 0.0) } + + protected[algorithm] def computeProjection[T](target: U[T]): List[(T, Double)] = { + projectDistribution(computeDistribution(target)) + } + + private def projectDistribution[T](distribution: Stream[(Double, T)]): List[(T, Double)] = { + (distribution map (_.swap)).toList + } /* @@ -57,13 +86,17 @@ trait ProbQueryAlgorithm * and do not need to be defined by particular algorithm implementations. */ - protected def doDistribution[T](target: Element[T]): Stream[(Double, T)] + protected def doDistribution[T](target: U[T]): Stream[(Double, T)] + + protected def doExpectation[T](target: U[T], function: T => Double): Double - protected def doExpectation[T](target: Element[T], function: T => Double): Double + protected def doProbability[T](target: U[T], predicate: T => Boolean): Double - protected def doProbability[T](target: Element[T], predicate: T => Boolean): Double + protected def doProjection[T](target: U[T]): List[(T, Double)] = { + projectDistribution(doDistribution(target)) + } - private def check[T](target: Element[T]): Unit = { + private def check[T](target: U[T]): Unit = { if (!active) throw new AlgorithmInactiveException if (!(queryTargets contains target)) throw new NotATargetException(target) } @@ -76,7 +109,7 @@ trait ProbQueryAlgorithm * targets of the algorithm. * Throws AlgorithmInactiveException if the algorithm is inactive. */ - def distribution[T](target: Element[T]): Stream[(Double, T)] = { + def distribution[T](target: U[T]): Stream[(Double, T)] = { check(target) doDistribution(target) } @@ -88,7 +121,19 @@ trait ProbQueryAlgorithm * targets of the algorithm. * Throws AlgorithmInactiveException if the algorithm is inactive. */ - def expectation[T](target: Element[T], function: T => Double): Double = { + def expectation[T](target: U[T], function: T => Double): Double = { + check(target) + doExpectation(target, function) + } + + /** + * Return an estimate of the expectation of the function under the marginal probability distribution + * of the target. + * Throws NotATargetException if called on a target that is not in the list of + * targets of the algorithm. + * Throws AlgorithmInactiveException if the algorithm is inactive. + */ + def expectation[T](target: U[T])(function: T => Double, c: Any = DummyImplicit): Double = { check(target) doExpectation(target, function) } @@ -96,26 +141,19 @@ trait ProbQueryAlgorithm /** * Return the mean of the probability density function for the given continuous element. */ - def mean(target: Element[Double]): Double = { + def mean(target: U[Double]): Double = { expectation(target, (d: Double) => d) } /** * Return the variance of the probability density function for the given continuous element. */ - def variance(target: Element[Double]): Double = { + def variance(target: U[Double]): Double = { val m = mean(target) val ex2 = expectation(target, (d: Double) => d * d) ex2 - m*m } - /** - * Return an element representing the posterior probability distribution of the given element. - */ - def posteriorElement[T](target: Element[T], universe: Universe = Universe.universe): Element[T] = { - Select(distribution(target).toList:_*)("", universe) - } - /** * Return an estimate of the probability of the predicate under the marginal probability distribution * of the target. @@ -123,21 +161,40 @@ trait ProbQueryAlgorithm * targets of the algorithm. * Throws AlgorithmInactiveException if the algorithm is inactive. */ - def probability[T](target: Element[T], predicate: T => Boolean): Double = { + def probability[T](target: U[T], predicate: T => Boolean): Double = { check(target) doProbability(target, predicate) } + /** + * Return an estimate of the probability of the predicate under the marginal probability distribution + * of the target. + * Throws NotATargetException if called on a target that is not in the list of + * targets of the algorithm. + * Throws AlgorithmInactiveException if the algorithm is inactive. + */ + def probability[T](target: U[T])(predicate: T => Boolean, c: Any = DummyImplicit): Double = { + probability(target, predicate) + } + + /** * Return an estimate of the probability that the target produces the value. * Throws NotATargetException if called on a target that is not in the list of * targets of the algorithm. * Throws AlgorithmInactiveException if the algorithm is inactive. */ - def probability[T](target: Element[T], value: T): Double = { + def probability[T](target: U[T], value: T): Double = { check(target) doProbability(target, (t: T) => t == value) } +} - universe.registerAlgorithm(this) + +trait StreamableProbQueryAlgorithm extends ProbQueryAlgorithm { + /** + * Sample an value from the posterior of this element + */ + def sampleFromPosterior[T](element: Element[T]): Stream[T] } + diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionImportance.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionImportance.scala index b8aef04e..ef8a0fe6 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionImportance.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionImportance.scala @@ -75,6 +75,7 @@ abstract class DecisionImportance[T, U] private (override val universe: Universe // override doSample so can update the local utilities override protected def doSample(): Unit = { val s = sample() + universe.clearTemporaries() totalWeight = logSum(s._1, totalWeight) allWeightsSeen foreach (updateWeightSeenForTarget(s, _)) allUtilitiesSeen foreach (updateWeightSeenForTargetNoLog((math.exp(s._1) * utilitySum, s._2), _)) diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionVariableElimination.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionVariableElimination.scala index e9ce5c16..c2dd1a83 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionVariableElimination.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/decision/DecisionVariableElimination.scala @@ -1,13 +1,13 @@ /* * DecisionVariableElimination.scala * Variable elimination for Decisions algorithm. - * + * * Created By: Brian Ruttenberg (bruttenberg@cra.com) * Creation Date: Oct 1, 2012 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -16,7 +16,6 @@ package com.cra.figaro.algorithm.decision import com.cra.figaro.algorithm._ import com.cra.figaro.algorithm.factored._ import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.factored.factors.factory._ import com.cra.figaro.algorithm.sampling._ import com.cra.figaro.language._ import com.cra.figaro.library.decision._ @@ -25,10 +24,11 @@ import com.cra.figaro.algorithm.lazyfactored.Extended import annotation.tailrec import scala.collection.mutable.{ Map, Set } import scala.language.existentials +import com.cra.figaro.algorithm.factored.factors.factory.Factory /* Trait only extends for double utilities. User needs to provide another trait or convert utilities to double * in order to use - * + * */ /** * Trait for Decision based Variable Elimination. This implementation is hardcoded to use. @@ -39,12 +39,12 @@ trait ProbabilisticVariableEliminationDecision extends VariableElimination[(Doub */ /* Implementations must define this */ def getUtilityNodes: List[Element[_]] - + /** * Semiring for Decisions uses a sum-product-utility semiring. */ override val semiring = SumProductUtilitySemiring() - + /** * Makes a utility factor an element designated as a utility. This is factor of a tuple (Double, Double) * where the first value is 1.0 and the second is a possible utility of the element. @@ -60,24 +60,23 @@ trait ProbabilisticVariableEliminationDecision extends VariableElimination[(Doub override def starterElements = getUtilityNodes ::: targetElements /** - * Create the factors for decision factors. Each factor is hardcoded as a tuple of (Double, Double), - * where the first value is the probability and the second is the utility. - */ + * Create the factors for decision factors. Each factor is hardcoded as a tuple of (Double, Double), + * where the first value is the probability and the second is the utility. + */ def getFactors(neededElements: List[Element[_]], targetElements: List[Element[_]], upper: Boolean = false): List[Factor[(Double, Double)]] = { if (debug) { println("Elements (other than utilities) appearing in factors and their ranges:") - for { element <- neededElements } { - println(Variable(element).id + "(" + element.name.string + "@" + element.hashCode + ")" + ": " + element + ": " + Variable(element).range.mkString(",")) + for { element <- neededElements } { + println(Variable(element).id + "(" + element.name.string + "@" + element.hashCode + ")" + ": " + element + ": " + Variable(element).range.mkString(",")) } } - Factory.removeFactors() - val thisUniverseFactorsExceptUtil = neededElements flatMap (Factory.make(_)) + val thisUniverseFactorsExceptUtil = neededElements flatMap (Factory.makeFactorsForElement(_)) // Make special utility factors for utility elements val thisUniverseFactorsUtil = getUtilityNodes map (makeUtilFactor(_)) val dependentUniverseFactors = - for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) + for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(Variable.cc, universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) // Convert all non-utility factors from standard factors to decision factors, ie, factors are now tuples of (Double, _) val thisUniverseFactorsExceptUtil_conv = thisUniverseFactorsExceptUtil.map(s => convert(s, false)) @@ -99,7 +98,7 @@ trait ProbabilisticVariableEliminationDecision extends VariableElimination[(Doub f.mapTo[(Double, Double)]((d: Double) => (d, 0.0), semiring.asInstanceOf[Semiring[(Double, Double)]]) } else { if (f.variables.length > 1) throw new IllegalUtilityNodeException - + val newF = f.mapTo[(Double, Double)]((d: Double) => (d, 0.0), semiring.asInstanceOf[Semiring[(Double, Double)]]) for {i <- 0 until f.variables(0).range.size} { newF.set(List(i), (newF.get(List(i))._1, f.variables(0).range(i).asInstanceOf[Double])) @@ -125,20 +124,20 @@ class ProbQueryVariableEliminationDecision[T, U](override val universe: Universe lazy val queryTargets = List(target) /** - * The variable elimination eliminates all variables except on all decision nodes and their parents. + * The variable elimination eliminates all variables except on all decision nodes and their parents. * Thus the target elements is both the decision element and the parent element. */ - val targetElements = List(target, target.args(0)) + val targetElements = List(target, target.args(0)) def getUtilityNodes = utilityNodes private var finalFactors: Factor[(Double, Double)] = Factory.defaultFactor[(Double, Double)](List(), List(), semiring) /* Marginalizes the final factor using the semiring for decisions - * + * */ private def marginalizeToTarget(factor: Factor[(Double, Double)], target: Element[_]): Unit = { - val unnormalizedTargetFactor = factor.marginalizeTo(semiring, Variable(target)) + val unnormalizedTargetFactor = factor.marginalizeTo(Variable(target)) val z = unnormalizedTargetFactor.foldLeft(semiring.zero, (x: (Double, Double), y: (Double, Double)) => (x._1 + y._1, 0.0)) //val targetFactor = Factory.make[(Double, Double)](unnormalizedTargetFactor.variables) val targetFactor = unnormalizedTargetFactor.mapTo((d: (Double, Double)) => (d._1 / z._1, d._2)) @@ -148,13 +147,13 @@ class ProbQueryVariableEliminationDecision[T, U](override val universe: Universe private def marginalize(resultFactor: Factor[(Double, Double)]) = queryTargets foreach (marginalizeToTarget(resultFactor, _)) - private def makeResultFactor(factorsAfterElimination: Set[Factor[(Double, Double)]]): Factor[(Double, Double)] = { + private def makeResultFactor(factorsAfterElimination: MultiSet[Factor[(Double, Double)]]): Factor[(Double, Double)] = { // It is possible that there are no factors (this will happen if there are no decisions or utilities). // Therefore, we start with the unit factor and use foldLeft, instead of simply reducing the factorsAfterElimination. factorsAfterElimination.foldLeft(Factory.unit(semiring))(_.product(_)) } - def finish(factorsAfterElimination: Set[Factor[(Double, Double)]], eliminationOrder: List[Variable[_]]) = + def finish(factorsAfterElimination: MultiSet[Factor[(Double, Double)]], eliminationOrder: List[Variable[_]]) = finalFactors = makeResultFactor(factorsAfterElimination) /** @@ -176,7 +175,7 @@ class ProbQueryVariableEliminationDecision[T, U](override val universe: Universe (0.0 /: computeDistribution(target))(_ + get(_)) } - /** + /** * Returns the computed utility of all parent/decision tuple values. For VE, these are not samples * but the actual computed expected utility for all combinations of the parent and decision. */ @@ -194,14 +193,14 @@ class ProbQueryVariableEliminationDecision[T, U](override val universe: Universe // find the variables of the parents. val parentVariable = factor.variables.filterNot(_ == decisionVariable)(0) - // index of the decision variable + // index of the decision variable val indexOfDecision = indices(factor.variables, decisionVariable) val indexOfParent = indices(factor.variables, parentVariable) for { indices <- factor.getIndices} { - /* for each index in the list of indices, strip out the decision variable index, + /* for each index in the list of indices, strip out the decision variable index, * and retrieve the map entry for the parents. If the factor value is greater than * what is currently stored in the strategy map, replace the decision with the new one from the factor */ @@ -257,7 +256,7 @@ object DecisionVariableElimination { } /** * Create a decision variable elimination algorithm with the given decision variables and indicated utility - * nodes and using the given dependent universes in the current default universe. Use the given dependent + * nodes and using the given dependent universes in the current default universe. Use the given dependent * algorithm function to determine the algorithm to use to compute probability of evidence in each dependent universe. */ def apply[T, U]( diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/FactoredAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/FactoredAlgorithm.scala index 5747db53..3e0b16e9 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/FactoredAlgorithm.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/FactoredAlgorithm.scala @@ -23,12 +23,14 @@ import scala.collection._ import com.cra.figaro.algorithm.lazyfactored._ import scala.collection.immutable.Set import com.cra.figaro.algorithm.UnsupportedAlgorithmException +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.library.collection.MakeArray /** * Trait for algorithms that use factors. */ trait FactoredAlgorithm[T] extends Algorithm { - + /** * Get the elements that are needed by the query target variables and the evidence variables. Also compute the values * of those variables to the given depth. @@ -39,7 +41,7 @@ trait FactoredAlgorithm[T] extends Algorithm { * If any of these elements has * in its range, the lower and upper bounds of factors will be different, so we need to compute both. * If they don't, we don't need to compute bounds. */ - def getNeededElements(starterElements: List[Element[_]], depth: Int): (List[Element[_]], Boolean) = { + def getNeededElements(starterElements: List[Element[_]], depth: Int, parameterized: Boolean = false): (List[Element[_]], Boolean) = { // Since there may be evidence on the dependent universes, we have to include their parents as important elements val dependentUniverseParents = for { @@ -61,7 +63,7 @@ trait FactoredAlgorithm[T] extends Algorithm { } // Make sure we compute values from scratch in case the elements have changed LazyValues.clear(universe) - val values = LazyValues(universe) + val values = LazyValues(universe, parameterized) /* * Beginning with the given element at the given depth, find all elements that the given element is used by within the depth. @@ -86,7 +88,8 @@ trait FactoredAlgorithm[T] extends Algorithm { * * */ val newlyNeededElements = - Element.closeUnderContingencies(starterElements.toSet).map((elem: Element[_]) => (elem, depth)) + Element.closeUnderContingencies(starterElements.toSet ++ boundsInducingElements.toSet).map((elem: Element[_]) => (elem, depth)) + @tailrec def expandElements(curr: Set[(Element[_], Int)]): Unit = { @@ -96,12 +99,13 @@ trait FactoredAlgorithm[T] extends Algorithm { val (uni, set) = pair val values = LazyValues(uni) values.expandAll(set) - val currentlyExpanded = values.expandedElements.toSet - val currentDepths = currentlyExpanded.map(d => (d, values.expandedDepth(d).getOrElse(0))) - val others = currentDepths.flatMap(e => chaseDown(e._1, e._2, currentlyExpanded)) - val filteredElements = others.filter(o => boundsInducingElements.contains(o._1)) - val neededElements = filteredElements.flatMap(f => (f._1.elementsIAmContingentOn + f._1).map((_, f._2))) - neededElements + //val currentlyExpanded = values.expandedElements.toSet + //val currentDepths = currentlyExpanded.map(d => (d, values.expandedDepth(d).getOrElse(0))) + //val others = currentDepths.flatMap(e => chaseDown(e._1, e._2, currentlyExpanded)) + //val filteredElements = others.filter(o => boundsInducingElements.contains(o._1)) + //val neededElements = filteredElements.flatMap(f => (f._1.elementsIAmContingentOn + f._1).map((_, f._2))) + //neededElements + List() }) expandElements(allNeededElements.toSet) } @@ -110,14 +114,6 @@ trait FactoredAlgorithm[T] extends Algorithm { // We only include elements from other universes if they are specified in the starter elements. val resultElements = values.expandedElements.toList ::: starterElements.filter(_.universe != universe) - - /* We support non caching chains now using sampling - * resultElements.foreach(p => p match { - * case n: NonCachingChain[_,_] => throw new UnsupportedAlgorithmException(n) - * case _ => - }* ) - * - */ // Only conditions and constraints produce distinct lower and upper bounds for *. So, if there are not such elements with * as one // of their values, we don't need to compute bounds and can save computation. @@ -125,9 +121,14 @@ trait FactoredAlgorithm[T] extends Algorithm { // We make sure to clear the variable cache now, once all values have been computed. // This ensures that any created variables will be consistent with the computed values. Variable.clearCache() + + // Even though we just cleared the variables, we generate all the variables so we can create the factors + resultElements.foreach(Variable(_)) + (resultElements, boundsNeeded) } + /** * All implementations of factored algorithms must specify a way to get the factors from the given universe and * dependent universes. diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/MPEVariableElimination.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/MPEVariableElimination.scala index a8e1f2c3..709c3194 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/MPEVariableElimination.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/MPEVariableElimination.scala @@ -1,13 +1,13 @@ /* * MPEVariableElimination.scala * Variable elimination algorithm for computing most probable explanation. - * + * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jan 1, 2009 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -20,10 +20,11 @@ import com.cra.figaro.algorithm.factored.factors._ import com.cra.figaro.util import scala.collection.mutable.{ Set, Map } import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.util.MultiSet /** * Variable elimination algorithm to compute the most probable explanation. - * + * * @param showTiming Produce timing information on steps of the algorithm */ class MPEVariableElimination(override val universe: Universe)( @@ -33,12 +34,12 @@ class MPEVariableElimination(override val universe: Universe)( override val comparator = Some((x: Double, y: Double) => x < y) override val semiring = MaxProductSemiring() - + /* * We are trying to find a configuration of all the elements, so we must make them all starter elements for expansion. */ override val starterElements = universe.activeElements - + /** * Empty for MPE Algorithms. */ @@ -47,15 +48,15 @@ class MPEVariableElimination(override val universe: Universe)( private val maximizers: Map[Variable[_], Any] = Map() private def getMaximizer[T](variable: Variable[T]): T = maximizers(variable).asInstanceOf[variable.Value] - + /* * Convert factors to use MaxProduct */ override def getFactors(allElements: List[Element[_]], targetElements: List[Element[_]], upper: Boolean = false): List[Factor[Double]] = { - val factors = super.getFactors(allElements, targetElements, upper) + val factors = super.getFactors(allElements, targetElements, upper) factors.map (_.mapTo(x => x, semiring)) } - + def mostLikelyValue[T](target: Element[T]): T = getMaximizer(Variable(target)) private def backtrackOne[T](factor: Factor[_], variable: Variable[T]): Unit = { @@ -64,7 +65,7 @@ class MPEVariableElimination(override val universe: Universe)( maximizers += variable -> factor.get(indices) } - def finish(factorsAfterElimination: Set[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = + def finish(factorsAfterElimination: MultiSet[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = for { (variable, factor) <- eliminationOrder.reverse.zip(recordingFactors) } { backtrackOne(factor, variable) } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/ParticleGenerator.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/ParticleGenerator.scala index ba397427..bfcfd3b0 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/ParticleGenerator.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/ParticleGenerator.scala @@ -24,13 +24,21 @@ import com.cra.figaro.util.MapResampler import com.cra.figaro.algorithm.factored.factors.Factor /** - * Class to handle sampling from continuous elements in PBP - * @param argSamples Maximum number of samples to take from atomic elements - * @param totalSamples Maximum number of samples on the output of chains + * Class to handle sampling from continuous elements to make factors + * @param numSamplesFromAtomics Maximum number of samples to take from atomic elements + * @param maxNumSamplesAtChain Maximum number of samples on the output of chains * @param de An instance to compute the density estimate of point during resampling */ -class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTotalSamples: Int) { +class ParticleGenerator(de: DensityEstimator, val numSamplesFromAtomics: Int, val maxNumSamplesAtChain: Int) { + @deprecated("numArgSamples is deprecated. Please use numSamplesFromAtomics", "4.1") + val numArgSamples = numSamplesFromAtomics + + @deprecated("numTotalSamples is deprecated. Please use maxNumSamplesAtChain", "4.1") + val numTotalSamples = maxNumSamplesAtChain + + var warningIssued = false + // Caches the samples for an element private val sampleMap = Map[Element[_], (List[(Double, _)], Int)]() @@ -52,7 +60,7 @@ class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTot /** * Retrieves the samples for an element using the default number of samples. */ - def apply[T](elem: Element[T]): List[(Double, T)] = apply(elem, numArgSamples) + def apply[T](elem: Element[T]): List[(Double, T)] = apply(elem, numSamplesFromAtomics) /** * Retrieves the samples for an element using the indicated number of samples @@ -63,6 +71,10 @@ class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTot e.asInstanceOf[(List[(Double, T)], Int)]._1 } case None => { + if (!warningIssued) { + println("Warning: you are using a factored algorithm with continuous or infinite elements. The element will be sampled " + numSamples + " times") + warningIssued = true + } val sampler = ElementSampler(elem, numSamples) sampler.start val result = sampler.computeDistribution(elem).toList @@ -86,9 +98,9 @@ class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTot def nextDouble(d: Double) = random.nextGaussian() * proposalVariance + d val numSamples = sampleMap(elem)._2 - + val sampleDensity: Double = 1.0 / numSamples - + // Generate new samples given the old samples for an element val newSamples = elem match { /* If the element is an instance of OneShifter (Geometric, poisson, etc), @@ -97,7 +109,7 @@ class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTot case o: OneShifter => { val toResample = if (beliefs.size < numSamples) { val resampler = new MapResampler(beliefs.map(s => (s._1, s._2))) - List.fill(numSamples)(1.0/numSamples, resampler.resample) + List.fill(numSamples)(1.0 / numSamples, resampler.resample) } else { beliefs } @@ -118,7 +130,7 @@ class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTot // return the new particles samples.groupBy(_._2).toList.map(s => (s._2.unzip._1.sum, s._2.head._2)) } - + /* For atomic doubles, we do the same thing as the OneShifters, but we assume * that we never need to resample since the number of particles equals numSamples. * We propose a new double and check its acceptance. Note the proposal is symmetric. @@ -146,9 +158,9 @@ class ParticleGenerator(de: DensityEstimator, val numArgSamples: Int, val numTot * we estimate the density using the density estimator, then multiple all of the estimates together. Finally, since * we only sample atomic elements, we multiple each result but the density of the values in the original element */ - private def accept[T](elem: Atomic[_], oldValue: T, newValue: T, proposalProb: Double, beliefs: List[List[(Double, T)]]): T = { - val oldDensity = beliefs.map(de.getDensity(oldValue, _)).product*elem.asInstanceOf[Atomic[T]].density(oldValue) - val newDensity = beliefs.map(de.getDensity(newValue, _)).product*elem.asInstanceOf[Atomic[T]].density(newValue) + private def accept[T](elem: Atomic[_], oldValue: T, newValue: T, proposalProb: Double, beliefs: List[List[(Double, T)]]): T = { + val oldDensity = beliefs.map(de.getDensity(oldValue, _)).product * elem.asInstanceOf[Atomic[T]].density(oldValue) + val newDensity = beliefs.map(de.getDensity(newValue, _)).product * elem.asInstanceOf[Atomic[T]].density(newValue) val ratio = (newDensity / oldDensity) * proposalProb val nextValue = if (ratio > 1) { @@ -164,12 +176,18 @@ object ParticleGenerator { /** * Maximum number of particles to generate per atomic */ - var defaultArgSamples = 15 + var defaultNumSamplesFromAtomics = 15 + @deprecated("defaultArgSamples is deprecated. Please use defaultNumSamplesFromAtomics", "4.1") + var defaultArgSamples = defaultNumSamplesFromAtomics + /** * Maximum number of particles to generate through a chain. */ - var defaultTotalSamples = 15 + var defaultMaxNumSamplesAtChain = 15 + + @deprecated("defaultTotalSamples is deprecated. Please use defaultMaxNumSamplesAtChain", "4.1") + var defaultTotalSamples = defaultMaxNumSamplesAtChain private val samplerMap: Map[Universe, ParticleGenerator] = Map() @@ -186,11 +204,11 @@ object ParticleGenerator { /** * Create a new particle generator for the given universe, using the given density estimatore, number of argument samples and total number of samples */ - def apply(univ: Universe, de: DensityEstimator, numArgSamples: Int, numTotalSamples: Int): ParticleGenerator = + def apply(univ: Universe, de: DensityEstimator, numSamplesFromAtomics: Int, maxNumSamplesAtChain: Int): ParticleGenerator = samplerMap.get(univ) match { case Some(e) => e case None => { - samplerMap += (univ -> new ParticleGenerator(de, numArgSamples, numTotalSamples)) + samplerMap += (univ -> new ParticleGenerator(de, numSamplesFromAtomics, maxNumSamplesAtChain)) univ.registerUniverse(samplerMap) samplerMap(univ) } @@ -200,7 +218,13 @@ object ParticleGenerator { * Create a new particle generate for a universe using a constant density estimator and default samples */ def apply(univ: Universe): ParticleGenerator = apply(univ, new ConstantDensityEstimator, - defaultArgSamples, defaultTotalSamples) + defaultNumSamplesFromAtomics, defaultMaxNumSamplesAtChain) + + /** + * Create a new particle generator for a universe using a constant density estimator and the number of argument samples and total number of samples + */ + def apply(univ: Universe, numSamplesFromAtomics: Int, maxNumSamplesAtChain: Int): ParticleGenerator = apply(univ, new ConstantDensityEstimator, + numSamplesFromAtomics, maxNumSamplesAtChain) /** * Check if a particle generate exists for this universe diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/SufficientStatisticsVariableElimination.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/SufficientStatisticsVariableElimination.scala index 3cf5786d..0788f451 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/SufficientStatisticsVariableElimination.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/SufficientStatisticsVariableElimination.scala @@ -1,13 +1,13 @@ /* * SufficientStatisticsVariableElimination.scala * Variable elimination algorithm for sufficient statistics factors - * + * * Created By: Michael Howard (mhoward@cra.com) * Creation Date: Jun 1, 2013 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -16,15 +16,18 @@ package com.cra.figaro.algorithm.factored import com.cra.figaro.algorithm._ import com.cra.figaro.algorithm.learning._ import com.cra.figaro.language._ -import com.cra.figaro.algorithm.factored.factors._ import scala.collection._ -import scala.collection.mutable.{ Map, Set } +import scala.collection.mutable.{ Set } +import scala.collection.immutable.Map +import com.cra.figaro.util.MultiSet +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.factored.factors.factory.Factory /** - * Variable elimination for sufficient statistics factors. + * Variable elimination for sufficient statistics factors. * The final factor resulting from variable elimination contains a mapping of parameters to sufficient statistics vectors * which can be used to maximize parameter values. - * + * * @param parameterMap A map of parameters to their sufficient statistics. */ class SufficientStatisticsVariableElimination( @@ -43,25 +46,24 @@ class SufficientStatisticsVariableElimination( * Clear the sufficient statistics factors used by this algorithm. */ private def removeFactors() { - statFactor.removeFactors + Variable.clearCache } /** - * Particular implementations of probability of evidence algorithms must define the following method. + * Particular implementations of probability of evidence algorithms must define the following method. */ - def getFactors(neededElements: List[Element[_]], targetElements: List[Element[_]], upper: Boolean = false): List[Factor[(Double, mutable.Map[Parameter[_], Seq[Double]])]] = { + def getFactors(neededElements: List[Element[_]], targetElements: List[Element[_]], upper: Boolean = false): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { val allElements = neededElements.filter(p => p.isInstanceOf[Parameter[_]] == false) if (debug) { println("Elements appearing in factors and their ranges:") - for { element <- allElements } { - println(Variable(element).id + "(" + element.name.string + "@" + element.hashCode + ")" + ": " + element + ": " + Variable(element).range.mkString(",")) + for { element <- allElements } { + println(Variable(element).id + "(" + element.name.string + "@" + element.hashCode + ")" + ": " + element + ": " + Variable(element).range.mkString(",")) } } - - Factory.removeFactors() + val thisUniverseFactors = allElements flatMap (statFactor.make(_)) val dependentUniverseFactors = - for { (dependentUniverse, evidence) <- dependentUniverses } yield statFactor.makeDependentFactor(universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) + for { (dependentUniverse, evidence) <- dependentUniverses } yield statFactor.makeDependentFactor(Variable.cc, universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) dependentUniverseFactors ::: thisUniverseFactors } @@ -70,12 +72,12 @@ class SufficientStatisticsVariableElimination( * Empty for this algorithm. */ val targetElements = List[Element[_]]() - + override def starterElements = universe.conditionedElements ++ universe.constrainedElements private var result: (Double, Map[Parameter[_], Seq[Double]]) = _ - def finish(factorsAfterElimination: Set[Factor[(Double, Map[Parameter[_], Seq[Double]])]], eliminationOrder: List[Variable[_]]): Unit = { + def finish(factorsAfterElimination: MultiSet[Factor[(Double, Map[Parameter[_], Seq[Double]])]], eliminationOrder: List[Variable[_]]): Unit = { // It is possible that there are no factors (this will happen if there is no evidence). // Therefore, we start with the unit factor and use foldLeft, instead of simply reducing the factorsAfterElimination. val finalFactor = factorsAfterElimination.foldLeft(Factory.unit(semiring))(_.product(_)) @@ -86,15 +88,14 @@ class SufficientStatisticsVariableElimination( } /** - * Returns a mapping of parameters to sufficient statistics resulting from + * Returns a mapping of parameters to sufficient statistics resulting from * elimination of the factors. */ def getSufficientStatisticsForAllParameters = { result._2.toMap } val semiring = SufficientStatisticsSemiring(parameterMap) - override def cleanUp() = { - statFactor.removeFactors + override def cleanUp() = { super.cleanUp() } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VEGraph.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VEGraph.scala index 90ef0a19..05338dd6 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VEGraph.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VEGraph.scala @@ -48,7 +48,7 @@ class VEGraph private (val info: Map[Variable[_], VariableInfo]) { * variables appearing in a factor with the eliminated variable, and excludes all factors in which the * eliminated variable appears. */ - def eliminate(variable: Variable[_]): VEGraph = { + def eliminate(variable: Variable[_]): (VEGraph, Double) = { val VariableInfo(oldFactors, allVars) = info(variable) val newFactor = AbstractFactor((allVars - variable).toList) var newInfo = VEGraph.makeInfo(info, List(newFactor), oldFactors) @@ -57,7 +57,7 @@ class VEGraph private (val info: Map[Variable[_], VariableInfo]) { newInfo += neighbor -> VariableInfo(oldNeighborFactors, oldNeighborNeighbors - variable) } newInfo(variable).neighbors foreach (removeNeighbor(_)) - (new VEGraph(newInfo)) + (new VEGraph(newInfo), VEGraph.cost(newFactor)) } /** diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VariableElimination.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VariableElimination.scala index a1d49c97..db0ccb96 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VariableElimination.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/VariableElimination.scala @@ -21,6 +21,9 @@ import com.cra.figaro.util._ import annotation.tailrec import scala.collection.mutable.{ Map, Set } import scala.language.postfixOps +import scala.util.control.TailCalls._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.factored.factors.factory.Factory /** * Trait of algorithms that perform variable elimination. @@ -66,13 +69,17 @@ trait VariableElimination[T] extends FactoredAlgorithm[T] with OneTime { // The first element of FactorMap is the complete set of factors. // The second element maps variables to the factors mentioning that variable. - private type FactorMap[T] = Map[Variable[_], Set[Factor[T]]] + // The previous implementation used sets, but that resulted in bugs where an identical factor appeared more than once. + // The new implementation uses multisets. + private type FactorMap[T] = Map[Variable[_], MultiSet[Factor[T]]] + // Add a factor to the list, even if it appears already. private def addFactor[T](factor: Factor[T], map: FactorMap[T]): Unit = - factor.variables foreach (v => map += v -> (map.getOrElse(v, Set()) + factor)) + factor.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).addOne(factor))) + // Remove one instance of the factor from the list. private def removeFactor[T](factor: Factor[T], map: FactorMap[T]): Unit = - factor.variables foreach (v => map += v -> (map.getOrElse(v, Set()) - factor)) + factor.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).removeOne(factor))) protected def initialFactorMap(factors: Traversable[Factor[T]]): FactorMap[T] = { val map: FactorMap[T] = Map() @@ -95,7 +102,7 @@ trait VariableElimination[T] extends FactoredAlgorithm[T] with OneTime { private def eliminate( variable: Variable[_], - factors: Set[Factor[T]], + factors: MultiSet[Factor[T]], map: FactorMap[T]): Unit = { val varFactors = map(variable) if (debug) { @@ -106,31 +113,45 @@ trait VariableElimination[T] extends FactoredAlgorithm[T] with OneTime { if (varFactors nonEmpty) { val productFactor = varFactors reduceLeft (_.product(_)) val resultFactor = productFactor.sumOver(variable) - varFactors foreach (removeFactor(_, map)) - addFactor(resultFactor, map) + if (debug) println("Result factor\n" + resultFactor.toReadableString) comparator match { case None => () case Some(recorder) => recordingFactors ::= productFactor.recordArgMax(variable, recorder) } + varFactors.foreach(factors.removeOne(_)) + factors.addOne(resultFactor) + varFactors.foreach(removeFactor(_, map)) map -= variable - factors --= varFactors - if (debug) println("Result factor\n" + resultFactor.toReadableString) - factors += resultFactor + addFactor(resultFactor, map) } } + // Wraps the TailRec class and returns the result protected def eliminateInOrder( order: List[Variable[_]], - factors: Set[Factor[T]], - map: FactorMap[T]): Set[Factor[T]] = + factors: MultiSet[Factor[T]], + map: FactorMap[T]): MultiSet[Factor[T]] = { + callEliminateInOrder(order, factors, map).result + } + + /* + * TailRec class turns a tail-recursive method into a while loop + * The result needs to be extracted explicitly + */ + private def callEliminateInOrder( + order: List[Variable[_]], + factors: MultiSet[Factor[T]], + map: FactorMap[T]): TailRec[MultiSet[Factor[T]]] = { order match { case Nil => - factors + done(factors) case first :: rest => eliminate(first, factors, map) - eliminateInOrder(rest, factors, map) + tailcall(callEliminateInOrder(rest, factors, map)) } - + } + + private[figaro] def ve(): Unit = { //expand() val (neededElements, _) = getNeededElements(starterElements, Int.MaxValue) @@ -145,9 +166,10 @@ trait VariableElimination[T] extends FactoredAlgorithm[T] with OneTime { println("*****************\nStarting factors\n") allFactors.foreach((f: Factor[_]) => println(f.toReadableString)) } - val (_, order) = optionallyShowTiming(VariableElimination.eliminationOrder(allFactors, targetVariables), "Computing elimination order") + val (score, order) = optionallyShowTiming(VariableElimination.eliminationOrder(allFactors, targetVariables), "Computing elimination order") + if (debug) println("***************** Eliminition Score: " + score) val factorsAfterElimination = - optionallyShowTiming(eliminateInOrder(order, Set(allFactors: _*), initialFactorMap(allFactors)), "Elimination") + optionallyShowTiming(eliminateInOrder(order, HashMultiSet(allFactors: _*), initialFactorMap(allFactors)), "Elimination") if (debug) println("*****************") if (debug) factorsAfterElimination foreach (f => println(f.toReadableString)) optionallyShowTiming(finish(factorsAfterElimination, order), "Finalizing") @@ -159,7 +181,7 @@ trait VariableElimination[T] extends FactoredAlgorithm[T] with OneTime { /** * All implementation of variable elimination must specify what to do after variables have been eliminated. */ - def finish(factorsAfterElimination: Set[Factor[T]], eliminationOrder: List[Variable[_]]): Unit + def finish(factorsAfterElimination: MultiSet[Factor[T]], eliminationOrder: List[Variable[_]]): Unit def run() = ve() @@ -176,10 +198,9 @@ trait ProbabilisticVariableElimination extends VariableElimination[Double] { println(Variable(element).id + "(" + element.name.string + "@" + element.hashCode + ")" + ": " + element + ": " + Variable(element).range.mkString(",")) } } - Factory.removeFactors() - val thisUniverseFactors = allElements flatMap (Factory.make(_)) + val thisUniverseFactors = allElements flatMap(Factory.makeFactorsForElement(_)) val dependentUniverseFactors = - for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) + for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(Variable.cc, universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) dependentUniverseFactors ::: thisUniverseFactors } @@ -201,7 +222,7 @@ class ProbQueryVariableElimination(override val universe: Universe, targets: Ele val semiring = SumProductSemiring() private def marginalizeToTarget(factor: Factor[Double], target: Element[_]): Unit = { - val unnormalizedTargetFactor = factor.marginalizeTo(semiring.asInstanceOf[Semiring[Double]], Variable(target)) + val unnormalizedTargetFactor = factor.marginalizeTo(Variable(target)) val z = unnormalizedTargetFactor.foldLeft(semiring.zero, _ + _) //val targetFactor = Factory.make[Double](unnormalizedTargetFactor.variables) val targetFactor = unnormalizedTargetFactor.mapTo((d: Double) => d / z) @@ -211,13 +232,13 @@ class ProbQueryVariableElimination(override val universe: Universe, targets: Ele private def marginalize(resultFactor: Factor[Double]) = targets foreach (marginalizeToTarget(resultFactor, _)) - private def makeResultFactor(factorsAfterElimination: Set[Factor[Double]]): Factor[Double] = { + private def makeResultFactor(factorsAfterElimination: MultiSet[Factor[Double]]): Factor[Double] = { // It is possible that there are no factors (this will happen if there are no queries or evidence). // Therefore, we start with the unit factor and use foldLeft, instead of simply reducing the factorsAfterElimination. factorsAfterElimination.foldLeft(Factory.unit(semiring))(_.product(_)) } - def finish(factorsAfterElimination: Set[Factor[Double]], eliminationOrder: List[Variable[_]]) = + def finish(factorsAfterElimination: MultiSet[Factor[Double]], eliminationOrder: List[Variable[_]]) = marginalize(makeResultFactor(factorsAfterElimination)) /** @@ -232,7 +253,7 @@ class ProbQueryVariableElimination(override val universe: Universe, targets: Ele dist.toStream } - /** + /** * Computes the expectation of a given function for single target element. */ def computeExpectation[T](target: Element[T], function: T => Double): Double = { @@ -248,29 +269,32 @@ object VariableElimination { * minimizes the number of extra factor entries that would be created when it is eliminated. * Override this method if you want a different rule. * - * Returns the score of the ordering as well as the ordering. + * Returns the score of the ordering as well as the ordering. If useBestScore is set to false, then it returns the total score of the + * entire eliminiation operation */ - def eliminationOrder[T](factors: Traversable[Factor[T]], toPreserve: Traversable[Variable[_]]): (Double, List[Variable[_]]) = { + def eliminationOrder[T](factors: Traversable[Factor[T]], toPreserve: Traversable[Variable[_]], useBestScore: Boolean = true): (Double, List[Variable[_]]) = { val eliminableVars = (Set[Variable[_]]() /: factors)(_ ++ _.variables) -- toPreserve var initialGraph = new VEGraph(factors) val candidates = new HeapPriorityMap[Variable[_], Double] eliminableVars foreach (v => candidates += v -> initialGraph.score(v)) - eliminationOrderHelper(candidates, toPreserve, initialGraph, Double.NegativeInfinity, List()) + val initScore = if (useBestScore) Double.NegativeInfinity else 0.0 + eliminationOrderHelper(candidates, toPreserve, initialGraph, initScore, List(), useBestScore) } @tailrec private def eliminationOrderHelper(candidates: PriorityMap[Variable[_], Double], toPreserve: Traversable[Variable[_]], graph: VEGraph, currentScore: Double, - accum: List[Variable[_]]): (Double, List[Variable[_]]) = { + accum: List[Variable[_]], useBestScore: Boolean): (Double, List[Variable[_]]) = { if (candidates.isEmpty) (currentScore, accum.reverse) else { val (best, bestScore) = candidates.extractMin() // do not read the best variable after it has been removed, and do not add the preserved variables val touched = graph.info(best).neighbors - best -- toPreserve - val nextGraph = graph.eliminate(best) + val (nextGraph, newCost) = graph.eliminate(best) touched foreach (v => candidates += v -> graph.score(v)) - eliminationOrderHelper(candidates, toPreserve, nextGraph, bestScore max currentScore, best :: accum) + val nextScore = if (useBestScore) bestScore max currentScore else newCost+currentScore + eliminationOrderHelper(candidates, toPreserve, nextGraph, nextScore, best :: accum, useBestScore) } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/BeliefPropagation.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/BeliefPropagation.scala index c5b63ac7..4f9e688c 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/BeliefPropagation.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/BeliefPropagation.scala @@ -21,15 +21,14 @@ import com.cra.figaro.util._ import annotation.tailrec import com.cra.figaro.algorithm.OneTimeProbQuery import com.cra.figaro.algorithm.ProbQueryAlgorithm -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.factored.factors.factory._ import com.cra.figaro.algorithm.factored._ +import com.cra.figaro.algorithm.factored.factors._ import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler -import com.cra.figaro.language.Element -import com.cra.figaro.language.Universe +import com.cra.figaro.language._ import com.cra.figaro.algorithm.lazyfactored.LazyValues import com.cra.figaro.algorithm.lazyfactored.BoundedProbFactor import scala.collection.mutable.Map +import com.cra.figaro.algorithm.factored.factors.factory.Factory /** * Trait for performing belief propagation. @@ -56,8 +55,18 @@ trait BeliefPropagation[T] extends FactoredAlgorithm[T] { /** * Since BP uses division to compute messages, the semiring has to have a division function defined + * and must be log convertable. + * Note that BP operates in log space and any semiring must be log convertible + * If you define a non-log semiring, it will automatically convert, and convert it back to normal space at the end + * If you define a log semiring, it won't convert to log or convert from log. + * In other words, it outputs the answer in the space specified by the semiring + */ + override val semiring: LogConvertibleSemiRing[T] + + /** + * Returns the log space version of the semiring (or the semiring if already in log space) */ - override val semiring: DivideableSemiRing[T] + protected def logSpaceSemiring(): LogConvertibleSemiRing[T] = if (semiring.isLog) semiring else semiring.convert /** * Elements towards which queries are directed. By default, these are the target elements. @@ -95,11 +104,11 @@ trait BeliefPropagation[T] extends FactoredAlgorithm[T] { * messages from all other Nodes (except the destination node), * marginalized over all variables except the variable: */ - private def getNewMessageFactorToVar(fn: FactorNode, vn: VariableNode) = { + protected def getNewMessageFactorToVar(fn: FactorNode, vn: VariableNode) = { val vnFactor = factorGraph.getLastMessage(vn, fn) - val total = beliefMap(fn).combination(vnFactor, semiring.divide) - total.marginalizeTo(semiring, vn.variable) + val total = beliefMap(fn).combination(vnFactor, logSpaceSemiring().divide) + total.marginalizeTo(vn.variable) } /* @@ -107,10 +116,10 @@ trait BeliefPropagation[T] extends FactoredAlgorithm[T] { * all other neighboring factor Nodes (except the recipient; alternatively one can say the * recipient sends the message "1"): */ - private def getNewMessageVarToFactor(vn: VariableNode, fn: FactorNode) = { + protected def getNewMessageVarToFactor(vn: VariableNode, fn: FactorNode) = { val fnFactor = factorGraph.getLastMessage(fn, vn) - val total = beliefMap(vn).combination(fnFactor, semiring.divide) + val total = beliefMap(vn).combination(fnFactor, logSpaceSemiring().divide) total } @@ -122,7 +131,7 @@ trait BeliefPropagation[T] extends FactoredAlgorithm[T] { if (messageList.isEmpty) { source match { - case fn: FactorNode => factorGraph.getFactorForNode(fn)//factorGraph.uniformFactor(fn.variables.toList) + case fn: FactorNode => factorGraph.getFactorForNode(fn) //factorGraph.uniformFactor(fn.variables.toList) case vn: VariableNode => factorGraph.uniformFactor(List(vn.variable)) } } else { @@ -185,9 +194,9 @@ trait ProbabilisticBeliefPropagation extends BeliefPropagation[Double] { * Normalize a factor. */ def normalize(factor: Factor[Double]): Factor[Double] = { - val z = semiring.sumMany(factor.contents.values) + val z = logSpaceSemiring().sumMany(factor.contents.values) // Since we're in log space, d - z = log(exp(d)/exp(z)) - factor.mapTo((d: Double) => if (z != semiring.zero) d - z else semiring.zero) + factor.mapTo((d: Double) => if (z != logSpaceSemiring().zero) d - z else logSpaceSemiring().zero) } /* @@ -203,21 +212,50 @@ trait ProbabilisticBeliefPropagation extends BeliefPropagation[Double] { * for all elements in the universe. */ def getFactors(neededElements: List[Element[_]], targetElements: List[Element[_]], upperBounds: Boolean = false): List[Factor[Double]] = { - - val thisUniverseFactors = (neededElements flatMap (BoundedProbFactor.make(_, upperBounds))).filterNot(_.isEmpty) + val parameterized = this match { + case p: ParameterLearner => true + case _ => false + } + val thisUniverseFactors = (neededElements flatMap (Factory.makeFactorsForElement(_, upperBounds, parameterized))).filterNot(_.isEmpty) val dependentUniverseFactors = - for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) + for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(Variable.cc, universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) val factors = dependentUniverseFactors ::: thisUniverseFactors - // To prevent underflow, we do all computation in log space - factors.map(makeLogarithmic(_)) + // To prevent underflow, we do all computation in log space + convertFactors(factors) + } + + /* + * Convert factors to log space if necessary + */ + protected def convertFactors(factors: List[Factor[Double]]) = { + if (factors.size == 0) { + factors + } else { + if (factors.head.semiring == semiring && semiring.isLog == false) { + // semiring in factors matches semiring for BP, and not in log space so need to convert + factors.map(makeLogarithmic(_)) + } else if (factors.head.semiring == semiring && semiring.isLog == true) { + // semiring in factors matches semiring for BP, and already in log space so no need to convert + factors + } else if (factors.head.semiring == semiring.convert() && semiring.isLog == false) { + // semiring in factors matches the converted semiring, and not in log space. But the factors are already in log space so no need to convert + factors + } else if (factors.head.semiring == semiring.convert() && semiring.isLog == true) { + // semiring in factors matches the converted semiring, and in log space. Need to convert + factors.map(makeLogarithmic(_)) + } else { + // Incompatible case. This might cause a problem if they are not converted eventually + factors.map(makeLogarithmic(_)) + } + } } private[figaro] def makeLogarithmic(factor: Factor[Double]): Factor[Double] = { - factor.mapTo((d: Double) => Math.log(d), LogSumProductSemiring()) + factor.mapTo((d: Double) => Math.log(d), logSpaceSemiring()) } private[figaro] def unmakeLogarithmic(factor: Factor[Double]): Factor[Double] = { - factor.mapTo((d: Double) => Math.exp(d), SumProductSemiring()) + factor.mapTo((d: Double) => Math.exp(d), logSpaceSemiring().convert) } /** @@ -259,7 +297,11 @@ trait ProbabilisticBeliefPropagation extends BeliefPropagation[Double] { val variable = factor.variables(0) val ff = normalize(factor) - ff.getIndices.filter(f => variable.range(f.head).isRegular).map(f => (Math.exp(ff.get(f)), variable.range(f.head).value)).toList + val inOriginalSpace = semiring.isLog match { + case true => ff + case false => unmakeLogarithmic(ff) + } + inOriginalSpace.getIndices.filter(f => variable.range(f.head).isRegular).map(f => (inOriginalSpace.get(f), variable.range(f.head).value)).toList } /** @@ -275,7 +317,7 @@ trait ProbabilisticBeliefPropagation extends BeliefPropagation[Double] { * Trait for One Time BP algorithms. */ trait OneTimeProbabilisticBeliefPropagation extends ProbabilisticBeliefPropagation with OneTime { - val iterations: Int + def iterations: Int def run() = { if (debug) { val varNodes = factorGraph.getNodes.filter(_.isInstanceOf[VariableNode]) @@ -328,13 +370,17 @@ abstract class ProbQueryBeliefPropagation(override val universe: Universe, targe val queryTargets = targetElements - val semiring = LogSumProductSemiring() + val semiring = SumProductSemiring() var neededElements: List[Element[_]] = _ var needsBounds: Boolean = _ def generateGraph() = { - val needs = getNeededElements(starterElements, depth) + val parameterized = this match { + case p: ParameterLearner => true + case _ => false + } + val needs = getNeededElements(starterElements, depth, parameterized) neededElements = needs._1 needsBounds = needs._2 @@ -345,7 +391,7 @@ abstract class ProbQueryBeliefPropagation(override val universe: Universe, targe getFactors(neededElements, targetElements) } - factorGraph = new BasicFactorGraph(factors, semiring): FactorGraph[Double] + factorGraph = new BasicFactorGraph(factors, logSpaceSemiring()): FactorGraph[Double] } override def initialize() = { diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/MPEBeliefPropagation.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/MPEBeliefPropagation.scala index 1c9defa9..507fdab7 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/MPEBeliefPropagation.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/MPEBeliefPropagation.scala @@ -29,7 +29,7 @@ abstract class MPEBeliefPropagation(override val universe: Universe)( val dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double) extends MPEAlgorithm with ProbabilisticBeliefPropagation { - override val semiring = LogMaxProductSemiring() + override val semiring = MaxProductSemiring() /* * Empty for MPE Algorithms */ @@ -37,8 +37,7 @@ abstract class MPEBeliefPropagation(override val universe: Universe)( override def initialize() = { val (neededElements, _) = getNeededElements(universe.activeElements, Int.MaxValue) - - factorGraph = new BasicFactorGraph(getFactors(neededElements, targetElements), semiring): FactorGraph[Double] + factorGraph = new BasicFactorGraph(getFactors(neededElements, targetElements), logSpaceSemiring()): FactorGraph[Double] super.initialize } @@ -46,8 +45,10 @@ abstract class MPEBeliefPropagation(override val universe: Universe)( * Convert factors to use MaxProduct */ override def getFactors(allElements: List[Element[_]], targetElements: List[Element[_]], upper: Boolean = false): List[Factor[Double]] = { - val factors = super.getFactors(allElements, targetElements, upper) - factors.map (_.mapTo(x => x, semiring)) + val factors = super.getFactors(allElements, targetElements, upper) + // Not needed since BP now converts factors to log space of the defined semiring + //factors.map (_.mapTo(x => x, logSpaceSemiring())) + factors } def mostLikelyValue[T](target: Element[T]): T = { diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/ProbEvidenceBeliefPropagation.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/ProbEvidenceBeliefPropagation.scala index 6eaf7216..faf709f6 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/ProbEvidenceBeliefPropagation.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/beliefpropagation/ProbEvidenceBeliefPropagation.scala @@ -33,13 +33,13 @@ import scala.collection.mutable.Map trait ProbEvidenceBeliefPropagation extends ProbabilisticBeliefPropagation with ProbEvidenceAlgorithm { - private def logFcn: (Double => Double) = (semiring: DivideableSemiRing[Double]) match { + private def logFcn: (Double => Double) = (logSpaceSemiring(): DivideableSemiRing[Double]) match { case LogSumProductSemiring() => (d: Double) => d case SumProductSemiring() => (d: Double) => if (d == semiring.zero) Double.NegativeInfinity else math.log(d) } - private def probFcn: (Double => Double) = (semiring: DivideableSemiRing[Double]) match { - case LogSumProductSemiring() => (d: Double) => if (d == semiring.zero) 0 else math.exp(d) + private def probFcn: (Double => Double) = (logSpaceSemiring(): DivideableSemiRing[Double]) match { + case LogSumProductSemiring() => (d: Double) => if (d == logSpaceSemiring().zero) 0 else math.exp(d) case SumProductSemiring() => (d: Double) => d } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/BasicFactor.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/BasicFactor.scala index 571dbd49..a4e90994 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/BasicFactor.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/BasicFactor.scala @@ -36,10 +36,6 @@ class BasicFactor[T](val parents: List[Variable[_]], val output: List[Variable[_ override def createFactor[T](parents: List[Variable[_]], output: List[Variable[_]], _semiring: Semiring[T] = semiring): Factor[T] = new BasicFactor[T](parents, output, _semiring) - override def convert[U](semiring: Semiring[U]): Factor[U] = { - createFactor[U](parents, output, semiring) - } - /** * Get the value associated with a row. The row is identified by an list of indices * into the ranges of the variables over which the factor is defined. @@ -116,7 +112,8 @@ class BasicFactor[T](val parents: List[Variable[_]], val output: List[Variable[_ that: Factor[T], op: (T, T) => T): Factor[T] = { that match { - case _:SparseFactor[T] => that.combination(this, op) + // Switch the order of op because it might not be commutative + case _:SparseFactor[T] => that.combination(this, (a,b) => op(b, a)) case _ => { val (allParents, allChildren, indexMap1, indexMap2) = unionVars(that) val result: Factor[T] = that.createFactor(allParents, allChildren) @@ -135,21 +132,9 @@ class BasicFactor[T](val parents: List[Variable[_]], val output: List[Variable[_ } } - private def computeSum( - resultIndices: List[Int], - summedVariable: Variable[_], - summedVariableIndices: List[Int], - semiring: Semiring[T]): T = { - val values = - for { i <- 0 until summedVariable.size } yield { - val sourceIndices = insertAtIndices(resultIndices, summedVariableIndices, i) - get(sourceIndices) - } - semiring.sumMany(values) - } - override def sumOver( - variable: Variable[_]): Factor[T] = { + variable: Variable[_], + sum: (T, T) => T = semiring.sum): Factor[T] = { if (variables contains variable) { // The summed over variable does not necessarily appear exactly once in the factor. val indicesOfSummedVariable = indices(variables, variable) @@ -174,7 +159,7 @@ class BasicFactor[T](val parents: List[Variable[_]], val output: List[Variable[_ val value = get(index) val newIndices: List[Int] = indexMap map (index(_)) val oldValue = result.get(newIndices) - result.set(newIndices, semiring.sum(oldValue, value)) + result.set(newIndices, sum(oldValue, value)) } } result @@ -226,13 +211,13 @@ class BasicFactor[T](val parents: List[Variable[_]], val output: List[Variable[_ result } - override def marginalizeTo( - semiring: Semiring[T], + override def marginalizeToWithSum( + sum: (T, T) => T, targets: Variable[_]*): Factor[T] = { val marginalized = (this.asInstanceOf[Factor[T]] /: variables)((factor: Factor[T], variable: Variable[_]) => if (targets contains variable) factor - else factor.sumOver(variable)) + else factor.sumOver(variable, sum)) // It's possible that the target variable appears more than once in this factor. If so, we need to reduce it to // one column by eliminating any rows in which the target variable values do not agree. deDuplicate(marginalized) diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factor.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factor.scala index 23abbe63..a9de74d1 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factor.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factor.scala @@ -135,8 +135,12 @@ trait Factor[T] { * The result is associated with all the variables in the * input except for the summed over variable and the value for a set of assignments is the * sum of the values of the corresponding assignments in the input. + * + * If no funciton is provided, this defaults to the sum function in this factor's semiring. */ - def sumOver(variable: Variable[_]): Factor[T] + def sumOver( + variable: Variable[_], + sum: (T, T) => T = semiring.sum): Factor[T] /** * Returns a factor that maps values of the other variables to the value of the given variable that @@ -150,11 +154,22 @@ trait Factor[T] { def recordArgMax[U](variable: Variable[U], comparator: (T, T) => Boolean, _semiring: Semiring[U] = semiring.asInstanceOf[Semiring[U]]): Factor[U] /** - * Returns the marginalization of the factor to a variable according to the given addition function. - * This involves summing out all other variables. + * Returns the marginalization of the factor to a variable according to the addition + * function in this factor's semiring. This involves summing out all other variables. */ - def marginalizeTo( - semiring: Semiring[T], + def marginalizeTo(targets: Variable[_]*): Factor[T] = marginalizeToWithSum(semiring.sum, targets:_*) + + /** + * Returns the marginalization of the factor to a variable according to the given + * addition function. Unlike marginalizeTo, this uses the provided sum function. + * This is useful e.g. to easily switch between max-product and sum-product operations + * when the data within are unchanged and the product operation is the same. + * + * The returned factor uses the semiring associated with this factor; it does not + * override the sum function of the semiring with the function given here. + */ + def marginalizeToWithSum( + sum: (T, T) => T, targets: Variable[_]*): Factor[T] /** @@ -162,11 +177,6 @@ trait Factor[T] { */ def deDuplicate(): Factor[T] - /** - * Creates a new Factor of the same class with a different type and semiring - */ - def convert[U](semiring: Semiring[U]): Factor[U] - /** * Produce a readable string representation of the factor */ diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Semiring.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Semiring.scala index 5194ad4b..bde3ee45 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Semiring.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Semiring.scala @@ -56,6 +56,18 @@ trait DivideableSemiRing[T] extends Semiring[T] { def divide(x: T, y: T): T } +trait LogConvertibleSemiRing[T] extends DivideableSemiRing[T] { + /** + * Is true if this semiring is in log space + */ + def isLog(): Boolean + /** + * Converts this semiring into its log inverse + * For non-log space semirings, converts to log, and for log semirings exponentiates + */ + def convert(): LogConvertibleSemiRing[T] +} + case class SumProductUtilitySemiring() extends DivideableSemiRing[(Double, Double)] { /** * Decision joint factor combination. @@ -102,7 +114,7 @@ case class BooleanSemiring() extends Semiring[Boolean] { val one = true } -case class SumProductSemiring() extends DivideableSemiRing[Double] { +case class SumProductSemiring() extends DivideableSemiRing[Double] with LogConvertibleSemiRing[Double] { /** * Standard multiplication */ @@ -125,12 +137,16 @@ case class SumProductSemiring() extends DivideableSemiRing[Double] { * 1 */ val one = 1.0 + + val isLog = false + + def convert() = LogSumProductSemiring() } /** * Semiring for computing sums and products with log probabilities. */ -case class LogSumProductSemiring() extends DivideableSemiRing[Double] { +case class LogSumProductSemiring() extends DivideableSemiRing[Double] with LogConvertibleSemiRing[Double] { val zero = Double.NegativeInfinity val one = 0.0 @@ -150,12 +166,16 @@ case class LogSumProductSemiring() extends DivideableSemiRing[Double] { } def sum(x: Double, y: Double) = sumMany(List(x, y)) + + val isLog = true + + def convert() = SumProductSemiring() } /** * Semiring for computing maxs and products with log probabilities. */ -case class LogMaxProductSemiring() extends DivideableSemiRing[Double] { +case class LogMaxProductSemiring() extends DivideableSemiRing[Double] with LogConvertibleSemiRing[Double] { val zero = Double.NegativeInfinity val one = 0.0 @@ -165,6 +185,10 @@ case class LogMaxProductSemiring() extends DivideableSemiRing[Double] { def divide(x: Double, y: Double) = if (y == zero) zero else x - y def sum(x: Double, y: Double) = x max y + + val isLog = true + + def convert() = MaxProductSemiring() } /** @@ -194,7 +218,8 @@ case class BoundsSumProductSemiring() extends DivideableSemiRing[(Double, Double val one = (1.0, 1.0) } -case class MaxProductSemiring() extends DivideableSemiRing[Double] { +case class MaxProductSemiring() extends DivideableSemiRing[Double] with LogConvertibleSemiRing[Double] { + /** * Standard multiplication */ @@ -218,5 +243,9 @@ case class MaxProductSemiring() extends DivideableSemiRing[Double] { * 1 */ val one = 1.0 + + val isLog = false + + def convert() = LogMaxProductSemiring() } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/SufficientStatisticsSemiring.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/SufficientStatisticsSemiring.scala index 5d425dae..3b0480de 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/SufficientStatisticsSemiring.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/SufficientStatisticsSemiring.scala @@ -15,7 +15,7 @@ package com.cra.figaro.algorithm.factored.factors import com.cra.figaro.language._ import scala.collection._ -import scala.collection.mutable.{ Set, Map } +import scala.collection.immutable.{ Set, Map } /** * Sum and product operations defined for sufficient statistics. @@ -25,19 +25,19 @@ import scala.collection.mutable.{ Set, Map } * Maximization determines the parameterMap automatically from the parameters. */ class SufficientStatisticsSemiring(parameterMap: immutable.Map[Parameter[_], Seq[Double]]) - extends Semiring[(Double, mutable.Map[Parameter[_], Seq[Double]])] { + extends Semiring[(Double, immutable.Map[Parameter[_], Seq[Double]])] { /** * 0 probability and a vector of zeros for all parameters. The vector for a parameter * must be of length equal to number of possible observations of the parameter. */ - val zero = (0.0, mutable.Map(parameterMap.toSeq: _*)) + val zero = (0.0, immutable.Map(parameterMap.toSeq: _*)) /** * 1 probability and a vector of zeros for all parameters. The vector for a parameter * must be of length equal to number of possible observations of the parameter. */ - val one = (1.0, mutable.Map(parameterMap.toSeq: _*)) + val one = (1.0, immutable.Map(parameterMap.toSeq: _*)) /** * Probabilities are multiplied using standard multiplication. @@ -57,22 +57,17 @@ class SufficientStatisticsSemiring(parameterMap: immutable.Map[Parameter[_], Seq } private def mapSum(xVector: (Double, Map[Parameter[_], Seq[Double]]), yVector: (Double, Map[Parameter[_], Seq[Double]])): Map[Parameter[_], Seq[Double]] = { - require(xVector._2.size == yVector._2.size) - val result: Map[Parameter[_], Seq[Double]] = Map() - for (x <- xVector._2.keys) { - result += x -> weightedComponentSum(xVector._2(x), yVector._2(x), xVector._1, yVector._1) - } - result + require(xVector._2.size == yVector._2.size) + Map() ++ {for (x <- xVector._2.keys) yield { + x -> weightedComponentSum(xVector._2(x), yVector._2(x), xVector._1, yVector._1) + }} } - private def mapProduct(xVector: (Double, Map[Parameter[_], Seq[Double]]), yVector: (Double, Map[Parameter[_], Seq[Double]])): Map[Parameter[_], Seq[Double]] = { - val result: Map[Parameter[_], Seq[Double]] = Map() + private def mapProduct(xVector: (Double, Map[Parameter[_], Seq[Double]]), yVector: (Double, Map[Parameter[_], Seq[Double]])): Map[Parameter[_], Seq[Double]] = { require(xVector._2.size == yVector._2.size) - for (x <- xVector._2.keys) { - result += x -> simpleComponentSum(xVector._2(x), yVector._2(x)) - } - result - + Map() ++ {for (x <- xVector._2.keys) yield { + x -> simpleComponentSum(xVector._2(x), yVector._2(x)) + }} } private def simpleComponentSum(xVector: Seq[Double], yVector: Seq[Double]): Seq[Double] = { diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Variable.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Variable.scala index 0c1c02d4..2eea5b22 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Variable.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Variable.scala @@ -1,13 +1,13 @@ /* * Variable.scala * Variables that appear in factors. - * + * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jan 1, 2009 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -17,6 +17,9 @@ import com.cra.figaro.algorithm._ import com.cra.figaro.language._ import scala.collection.mutable.Map import com.cra.figaro.algorithm.lazyfactored.{ LazyValues, Extended, ValueSet } +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.library.collection.MakeArray +import scala.collection.mutable.HashMap /** * Variables that appear in factors. @@ -37,7 +40,7 @@ class Variable[T](val valueSet: ValueSet[T]) { * Size of the range of the variable. */ val size = range.size - + /** * Override equality of Variable. Variables are the same if their id's are the same */ @@ -73,7 +76,7 @@ class ParameterizedVariable[T](override val element: Parameterized[T]) extends E /** * Variables that are internal to Factors. - * + * * This is the same as a temporary variable, but is more explicitly identified * * @param range The range of values of the variable @@ -82,13 +85,27 @@ class InternalVariable[T](values: ValueSet[T]) extends Variable(values) { override def toString = "Internal variable:" + values.toString } -/* Variables generated from sufficient statistics of parameters */ +class InternalChainVariable[U](values: ValueSet[U], val chain: Chain[_, _], val chainVar: Variable[_]) extends InternalVariable(values) + object Variable { - // An element should always map to the same variable - private val memoMake: Map[Element[_], Variable[_]] = Map() + + // The global component collection to support legacy factor creation with SFI factor generation + private[figaro] var cc: ComponentCollection = new ComponentCollection + private[figaro] var problem = new Problem(cc, List()) + + private def variableExists(elem: Element[_]) = cc.contains(elem) && (cc(elem).variable != null) + + private def makeComponent[T](elem: Element[T]): ProblemComponent[T] = elem match { + case chain: Chain[_, T] => new ChainComponent(problem, chain) + case makeArray: MakeArray[_] => new MakeArrayComponent(problem, makeArray).asInstanceOf[ProblemComponent[T]] + case apply: Apply[_] => new ApplyComponent(problem, apply) + case _ => new ProblemComponent(problem, elem) + } // Make sure to register this map (or replace the memoMake) - private val idCache: Map[Element[_], Int] = Map() + private val idCache: Map[Element[_], Int] = new HashMap[Element[_], Int]() { + override def hashCode = 2 + } private var idState: Int = 0 @@ -108,15 +125,19 @@ object Variable { } private def make[T](elem: Element[T]): Variable[T] = { - // Make sure that the element will be removed from the memoMake map when it is inactivated - elem.universe.register(memoMake) + // Make sure that the element will be removed from the cc.components map when it is inactivated + elem.universe.register(cc.components) elem.universe.register(idCache) - new ElementVariable(elem) + elem match { + case p: Parameterized[T] => new ParameterizedVariable(p) + case _ => new ElementVariable(elem) + } } private def make[T](p: Parameterized[T]): Variable[T] = { // Make sure that the element will be removed from the memoMake map when it is inactivated - p.universe.register(memoMake) + //p.universe.register(memoMake) + p.universe.register(cc.components) p.universe.register(idCache) new ParameterizedVariable(p) } @@ -125,22 +146,40 @@ object Variable { * Create the variable associated with an element. This method is memoized. */ def apply[T](elem: Element[T]): Variable[T] = - memoMake.get(elem) match { - case Some(v) => v.asInstanceOf[Variable[T]] - case None => - val result = make(elem) - memoMake += elem -> result - result + if (variableExists(elem)) { + cc(elem).variable + } else { + val result = make(elem) + val comp = makeComponent(elem) + comp.setVariable(result) + comp.range = result.valueSet + cc.components += elem -> comp + result } + /** + * Create the variable associated with a parame. This method is memoized. + */ def apply[T](elem: Parameterized[T]): Variable[T] = - memoMake.get(elem) match { - case Some(v) => v.asInstanceOf[Variable[T]] - case None => - val result = make(elem) - memoMake += elem -> result - result + if (variableExists(elem)) { + cc(elem).variable + } else { + val result = make(elem) + val comp = makeComponent(elem) + comp.setVariable(result) + comp.range = result.valueSet + cc.components += elem -> comp + result } - def clearCache() { memoMake.clear() } + /** + * Clear the variable cache + */ + def clearCache() { + cc.components.foreach{c => c._1.universe.deregister(cc.components)} + cc = new ComponentCollection + problem = new Problem(cc, List()) + } + + } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ApplyFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ApplyFactory.scala index 382beba2..f0e4774f 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ApplyFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ApplyFactory.scala @@ -1,13 +1,13 @@ /* * ApplyFactory.scala - * Description needed - * + * Methods to create factors associated with Apply elements. + * * Created By: Glenn Takata (gtakata@cra.com) * Creation Date: Dec 15, 2014 - * + * * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -15,40 +15,37 @@ package com.cra.figaro.algorithm.factored.factors.factory import com.cra.figaro.algorithm.PointMapper import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.lazyfactored._ import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.ComponentCollection /** * A Sub-Factory for Apply Elements */ object ApplyFactory { - + /** * Factor constructor for an Apply Element that has one input */ - def makeFactors[T, U](apply: Apply1[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val applyMap: scala.collection.mutable.Map[T, U] = LazyValues(apply.universe).getMap(apply) - val arg1Var = Variable(apply.arg1) - val resultVar = Variable(apply) - val applyValues = LazyValues(apply.universe).storedValues(apply) + def makeFactors[T, U](cc: ComponentCollection, apply: Apply1[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val arg1Var = Factory.getVariable(cc, apply.arg1) + val resultVar = Factory.getVariable(cc, apply) + cc.variableParents(resultVar) += arg1Var + val applyComponent = cc(apply) + val applyMap = applyComponent.getMap + val applyValues = applyComponent.range val factor = new SparseFactor[Double](List(arg1Var), List(resultVar)) val arg1Indices = arg1Var.range.zipWithIndex for { (arg1Val, arg1Index) <- arg1Indices - //(resultVal, resultIndex) <- resultIndices } { - // See logic in makeCares if (arg1Val.isRegular) { - // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. - // By using applyMap, we can make sure that any contained elements in the result of the apply are the same - // now as they were when values were computed. val resultVal = mapper.map(applyMap(arg1Val.value), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(_.value == resultVal) + val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) factor.set(List(arg1Index, resultIndex), 1.0) } else if (!arg1Val.isRegular && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) + val resultIndex = resultVar.range.indexWhere(!_.isRegular) factor.set(List(arg1Index, resultIndex), 1.0) - } + } } List(factor) } @@ -56,12 +53,14 @@ object ApplyFactory { /** * Factor constructor for an Apply Element that has two inputs */ - def makeFactors[T1, T2, U](apply: Apply2[T1, T2, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val applyMap: scala.collection.mutable.Map[(T1, T2), U] = LazyValues(apply.universe).getMap(apply) - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val resultVar = Variable(apply) - val applyValues = LazyValues(apply.universe).storedValues(apply) + def makeFactors[T1, T2, U](cc: ComponentCollection, apply: Apply2[T1, T2, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val arg1Var = Factory.getVariable(cc, apply.arg1) + val arg2Var = Factory.getVariable(cc, apply.arg2) + val resultVar = Factory.getVariable(cc, apply) + cc.variableParents(resultVar) ++= Set(arg1Var, arg2Var) + val applyComponent = cc(apply) + val applyMap = applyComponent.getMap + val applyValues = cc(apply).range val factor = new SparseFactor[Double](List(arg1Var, arg2Var), List(resultVar)) val arg1Indices = arg1Var.range.zipWithIndex val arg2Indices = arg2Var.range.zipWithIndex @@ -69,19 +68,16 @@ object ApplyFactory { for { (arg1Val, arg1Index) <- arg1Indices (arg2Val, arg2Index) <- arg2Indices - //(resultVal, resultIndex) <- resultIndices } { if (arg1Val.isRegular && arg2Val.isRegular) { - // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. - // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value)), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(_.value == resultVal) + val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) factor.set(List(arg1Index, arg2Index, resultIndex), 1.0) } else if ((!arg1Val.isRegular || !arg2Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) + val resultIndex = resultVar.range.indexWhere(!_.isRegular) factor.set(List(arg1Index, arg2Index, resultIndex), 1.0) - } - + } + } List(factor) } @@ -89,34 +85,32 @@ object ApplyFactory { /** * Factor constructor for an Apply Element that has three inputs */ - def makeFactors[T1, T2, T3, U](apply: Apply3[T1, T2, T3, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val applyMap: scala.collection.mutable.Map[(T1, T2, T3), U] = LazyValues(apply.universe).getMap(apply) - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val arg3Var = Variable(apply.arg3) - val resultVar = Variable(apply) - val applyValues = LazyValues(apply.universe).storedValues(apply) + def makeFactors[T1, T2, T3, U](cc: ComponentCollection, apply: Apply3[T1, T2, T3, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val arg1Var = Factory.getVariable(cc, apply.arg1) + val arg2Var = Factory.getVariable(cc, apply.arg2) + val arg3Var = Factory.getVariable(cc, apply.arg3) + val resultVar = Factory.getVariable(cc, apply) + cc.variableParents(resultVar) ++= Set(arg1Var, arg2Var, arg3Var) + val applyComponent = cc(apply) + val applyMap = applyComponent.getMap + val applyValues = cc(apply).range val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var), List(resultVar)) val arg1Indices = arg1Var.range.zipWithIndex val arg2Indices = arg2Var.range.zipWithIndex val arg3Indices = arg3Var.range.zipWithIndex - //val resultIndices = resultVar.range.zipWithIndex for { (arg1Val, arg1Index) <- arg1Indices (arg2Val, arg2Index) <- arg2Indices (arg3Val, arg3Index) <- arg3Indices - //(resultVal, resultIndex) <- resultIndices - } { + } { if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular) { - // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. - // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value, arg3Val.value)), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(_.value == resultVal) + val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), 1.0) } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) + val resultIndex = resultVar.range.indexWhere(!_.isRegular) factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), 1.0) - } + } } List(factor) } @@ -124,37 +118,35 @@ object ApplyFactory { /** * Factor constructor for an Apply Element that has four inputs */ - def makeFactors[T1, T2, T3, T4, U](apply: Apply4[T1, T2, T3, T4, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val applyMap: scala.collection.mutable.Map[(T1, T2, T3, T4), U] = LazyValues(apply.universe).getMap(apply) - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val arg3Var = Variable(apply.arg3) - val arg4Var = Variable(apply.arg4) - val resultVar = Variable(apply) - val applyValues = LazyValues(apply.universe).storedValues(apply) + def makeFactors[T1, T2, T3, T4, U](cc: ComponentCollection, apply: Apply4[T1, T2, T3, T4, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val arg1Var = Factory.getVariable(cc, apply.arg1) + val arg2Var = Factory.getVariable(cc, apply.arg2) + val arg3Var = Factory.getVariable(cc, apply.arg3) + val arg4Var = Factory.getVariable(cc, apply.arg4) + val resultVar = Factory.getVariable(cc, apply) + cc.variableParents(resultVar) ++= Set(arg1Var, arg2Var, arg3Var, arg4Var) + val applyComponent = cc(apply) + val applyMap = applyComponent.getMap + val applyValues = cc(apply).range val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var, arg4Var), List(resultVar)) val arg1Indices = arg1Var.range.zipWithIndex val arg2Indices = arg2Var.range.zipWithIndex val arg3Indices = arg3Var.range.zipWithIndex val arg4Indices = arg4Var.range.zipWithIndex - //val resultIndices = resultVar.range.zipWithIndex for { (arg1Val, arg1Index) <- arg1Indices (arg2Val, arg2Index) <- arg2Indices (arg3Val, arg3Index) <- arg3Indices (arg4Val, arg4Index) <- arg4Indices - //(resultVal, resultIndex) <- resultIndices } { - if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular) { - // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. - // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. + if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular) { val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value)), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(_.value == resultVal) + val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), 1.0) } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular || !arg4Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) + val resultIndex = resultVar.range.indexWhere(!_.isRegular) factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), 1.0) - } + } } List(factor) } @@ -162,42 +154,42 @@ object ApplyFactory { /** * Factor constructor for an Apply Element that has five inputs */ - def makeFactors[T1, T2, T3, T4, T5, U](apply: Apply5[T1, T2, T3, T4, T5, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val applyMap: scala.collection.mutable.Map[(T1, T2, T3, T4, T5), U] = LazyValues(apply.universe).getMap(apply) - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val arg3Var = Variable(apply.arg3) - val arg4Var = Variable(apply.arg4) - val arg5Var = Variable(apply.arg5) - val resultVar = Variable(apply) - val applyValues = LazyValues(apply.universe).storedValues(apply) + def makeFactors[T1, T2, T3, T4, T5, U](cc: ComponentCollection, apply: Apply5[T1, T2, T3, T4, T5, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val arg1Var = Factory.getVariable(cc, apply.arg1) + val arg2Var = Factory.getVariable(cc, apply.arg2) + val arg3Var = Factory.getVariable(cc, apply.arg3) + val arg4Var = Factory.getVariable(cc, apply.arg4) + val arg5Var = Factory.getVariable(cc, apply.arg5) + val resultVar = Factory.getVariable(cc, apply) + cc.variableParents(resultVar) ++= Set(arg1Var, arg2Var, arg3Var, arg4Var, arg5Var) + val applyComponent = cc(apply) + val applyMap = applyComponent.getMap + val applyValues = cc(apply).range val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var, arg4Var, arg5Var), List(resultVar)) val arg1Indices = arg1Var.range.zipWithIndex val arg2Indices = arg2Var.range.zipWithIndex val arg3Indices = arg3Var.range.zipWithIndex val arg4Indices = arg4Var.range.zipWithIndex val arg5Indices = arg5Var.range.zipWithIndex -// val resultIndices = resultVar.range.zipWithIndex + val resultIndices = resultVar.range.zipWithIndex for { (arg1Val, arg1Index) <- arg1Indices (arg2Val, arg2Index) <- arg2Indices (arg3Val, arg3Index) <- arg3Indices (arg4Val, arg4Index) <- arg4Indices (arg5Val, arg5Index) <- arg5Indices -// (resultVal, resultIndex) <- resultIndices + (resultVal, resultIndex) <- resultIndices } { if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular && arg5Val.isRegular) { - // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. - // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value, arg5Val.value)), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(_.value == resultVal) + val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), 1.0) } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular || !arg4Val.isRegular || !arg5Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) + val resultIndex = resultVar.range.indexWhere(!_.isRegular) factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), 1.0) - } + } } List(factor) } -} \ No newline at end of file +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ChainFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ChainFactory.scala index 4660cb1b..42bad4c5 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ChainFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ChainFactory.scala @@ -1,156 +1,114 @@ /* * ChainFactory.scala - * Create a factor from a Chain element. - * - * Created By: Glenn Takata (gtakata@cra.com) - * Creation Date: Mar 22, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * Methods to create factors associated with Chain elements. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ package com.cra.figaro.algorithm.factored.factors.factory import com.cra.figaro.algorithm.PointMapper -import com.cra.figaro.algorithm.factored.factors.{ BasicFactor, Factor, SparseFactor, Variable, InternalVariable } -import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored.ValueSet import com.cra.figaro.language._ -import scala.collection.immutable.SortedSet -import scala.collection.mutable.ListBuffer -import scala.reflect.runtime.universe -import com.cra.figaro.algorithm.factored.factors.ConditionalSelector +import com.cra.figaro.algorithm.structured.ComponentCollection +import com.cra.figaro.util._ /** - * @author Glenn Takata Mar 22, 2015 + * @author Glenn Takata Feb 19, 2015 * */ object ChainFactory { - /** - * Factor constructor for a Chain Element + * Make the factors associated with a chain element. */ - def makeFactors[T, U](chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val chainMap: scala.collection.mutable.Map[T, Element[U]] = LazyValues(chain.universe).getMap(chain) - val outputVar = Variable(chain) - val parentVar = Variable(chain.parent) - // create a selectionVar and Factor to take the place of the parent/output combination - val selectorVar = makeSelectorVariable(parentVar, outputVar) - val selector = new SparseFactor[Double](List(parentVar, selectorVar), List(outputVar)) - val parentSize = parentVar.range.size + import com.cra.figaro.algorithm.factored.factors.factory.Factory + + def makeFactors[T, U](cc: ComponentCollection, chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val chainComp = cc(chain) + if (chainComp.allSubproblemsEliminatedCompletely && cc.useSingleChainFactor) { + makeSingleFactor[T, U](cc, chain)(mapper) + } else { + makeMultipleFactors[T, U](cc, chain)(mapper) + } + } - var tempFactors = new ListBuffer[Factor[Double]] - tempFactors.append(selector) - // selector must be passed to makeSelection as it is updated based on the resultVar/chainVar match - for { - (parent, parentIndex) <- parentVar.range.zipWithIndex - } { - // parentVal.value should have been placed in applyMap at the time the values of this apply were computed. - // By using chainMap, we can make sure that the result element is the same now as they were when values were computed. - if (parent.isRegular) { - tempFactors.append(makeSelection(chain, selectorVar, parentSize, parentIndex, Variable(chainMap(parent.value)), selector)(mapper)) + def makeMultipleFactors[T, U](cc: ComponentCollection, chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val chainComp = cc(chain) + val parentVar = Factory.getVariable(cc, chain.parent) + val chainVar = Factory.getVariable(cc, chain) + val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, Some(chain), parentVar, chainVar) + cc.variableParents(pairVar) = Set(parentVar, chainVar) + var tempFactors = parentVar.range.zipWithIndex flatMap (pair => { + val (parentVal, parentIndex) = pair + if (parentVal.isRegular) { + // We need to create an actual variable to represent the outcome of the chain. + // This will have the same range as the formal variable associated with the expansion. + // The formal variable will be replaced with the actual variable in the solution. + // There are two possibilities. + // If the outcome element is defined inside the chain, it will actually be different in every expansion, + // even though the formal variable is the same. But if the outcome element is defined outside the chain, + // it is a global that will be the same every time, so the actual variable is the variable of this global. + val nestedProblem = cc.expansions((chain.chainFunction, parentVal.value)) + val outcomeElem = nestedProblem.target.asInstanceOf[Element[U]] + val formalVar = Factory.getVariable(cc, outcomeElem) + val actualVar = if (!nestedProblem.global(formalVar)) Factory.makeVariable(cc, formalVar.valueSet) else formalVar + cc.variableParents(chainVar) += actualVar + chainComp.actualSubproblemVariables += parentVal.value -> actualVar + List(Factory.makeConditionalSelector(pairVar, parentVal, actualVar, chainComp.range.regularValues)(mapper)) } else { // We create a dummy variable for the outcome variable whose value is always star. // We create a dummy factor for that variable. - // Then we use makeSelection with the dummy variable - val dummy = new Variable(ValueSet.withStar[U](Set())) - val dummyFactor = new BasicFactor[Double](List(), List(dummy)) - dummyFactor.set(List(0), 1.0) - tempFactors.append(makeSelection(chain, selectorVar, parentSize, parentIndex, dummy, selector), dummyFactor) + // Then we use makeConditionalSelector with the dummy variable + val dummy = Factory.makeVariable(cc, ValueSet.withStar[U](Set())) + List(Factory.makeConditionalSelector(pairVar, parentVal, dummy, chainComp.range.regularValues)) } - } - - tempFactors.toList + }) + pairFactor :: tempFactors } - /* - * Create a temporary variable representing the combination of the parent variable and the chain - * variable - */ - private def makeSelectorVariable(parent: Variable[_], overallVar: Variable[_]): Variable[Int] = { - val selectorSize = parent.size * overallVar.size + def makeSingleFactor[T, U](cc: ComponentCollection, chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val chainComp = cc(chain) + val parentVar = Factory.getVariable(cc, chain.parent) + val childVar = Factory.getVariable(cc, chain) + val factor = new BasicFactor[Double](List(parentVar), List(childVar)) + for { parentIndex <- 0 until parentVar.range.length } { + val parentXV = parentVar.range(parentIndex) + if (parentXV.isRegular && chainComp.subproblems.contains(parentXV.value) && !chainComp.subproblems(parentXV.value).solution.isEmpty) { + val subproblem = chainComp.subproblems(parentXV.value) + // Need to normalize subsolution in case there's any nested evidence + val subsolution = subproblem.solution.reduce(_.product(_)) + //val sum = subsolution.foldLeft(subsolution.semiring.zero, subsolution.semiring.sum(_, _)) + val subVars = subsolution.variables + if (subVars.length == 1) { + val subVar = subVars(0) + for { subVal <- subVar.range } { + val childIndex = childVar.range.indexOf(subVal) + val subIndex = subVar.range.indexOf(subVal) + //val entry = subsolution.semiring.product(subsolution.get(List(subIndex)), 1.0 / sum) + factor.set(List(parentIndex, childIndex), subsolution.get(List(subIndex))) + } + } else { // This should be a case where the subproblem is empty and the value is * + val starIndex = childVar.range.indexWhere(!_.isRegular) + factor.set(List(parentIndex, starIndex), factor.semiring.one) + } - val values = SortedSet[Int]((0 until selectorSize): _*) - new InternalVariable(ValueSet.withoutStar(values)) - } - - /** - * Make a selection factor used in the decomposition of chain. - * A chain defines a factor over the parent element, each of the possible result elements of the chain, - * and the overall chain element. This can produce a very large factor when there are many result elements. - * This is solved by decomposing the chain factor into a product of factors, each of which contains a - * selector variable and the result variable. - * - * The conditional selector creates a factor in which, when the selector's value is such that the result - * element is relevant to the final result, the result element and chain element must have the same - * value (handled by makeCares). Otherwise, the result element and chain element can take on any - * value (handled by makeDontCares) - * - */ - private def makeSelection[U](chain: Element[U], selectorVar: Variable[_], parentSize: Int, - parentIndex: Int, resultVar: Variable[U], selector: Factor[Double])(implicit mapper: PointMapper[U]): Factor[Double] = { - val chainVar = Variable(chain) - val chainValues = LazyValues(chain.universe).storedValues(chain) - val factor = new ConditionalSelector[Double](List(selectorVar), List(resultVar)) - - makeCares(factor, parentIndex, resultVar, chainVar, chainValues.regularValues, selector)(mapper) - for (index <- 0 until parentSize) { - if (index != parentIndex) - makeDontCares[U](factor, index, resultVar, chainVar, selector) + } else { + for { childIndex <- 0 until childVar.range.length } { + val entry = if (childVar.range(childIndex).isRegular) factor.semiring.zero else factor.semiring.one + factor.set(List(parentIndex, childIndex), entry) + } + val childIndex = childVar.range.indexWhere(!_.isRegular) + factor.set(List(parentIndex, childIndex), factor.semiring.one) + } } - factor - } + List(factor) - /* - * Values that specify the potential for the selector variable and the specified output - */ - private def makeCares[U](factor: ConditionalSelector[Double], parentIndex: Int, resultVar: Variable[U], - chainVar: Variable[U], choices: Set[U], selector: Factor[Double])(implicit mapper: PointMapper[U]): Unit = { - // We care to match up overallVar with outcomeVar - val chainSize = chainVar.size - for { - (resultVal, j) <- resultVar.range.zipWithIndex - (chainVal, k) <- chainVar.range.zipWithIndex - } { - // Star stands for "something". If outcomeVal is Star and overallVal is Star, we know something will match something, so the entry is (1,1). - // If outcomeVal is Star and overallVal is a regular value, then maybe there will be a match, so the entry is (0,1). - // If outcomeVal is regular, all the probability mass associated with that outcome should be on regular values of overallVal, so the entry is (0,0). - val entry = - if (chainVal.isRegular && resultVal.isRegular) { - if (chainVal.value == mapper.map(resultVal.value, choices)) 1.0 - else 0.0 - } else if (!chainVal.isRegular && !resultVal.isRegular) 1.0 - else 0.0 - - // calculate the relevant index of the selector Factor and update both - // the selector Factor and the current (conditional selector) Factor - val selectorIndex = parentIndex * chainSize + k - selector.set(List(parentIndex, selectorIndex, k), 1.0) - factor.set(List(selectorIndex, j), entry) - } } - /* - * Values that are irrelevant to the selector variable and specified output - */ - private def makeDontCares[U](factor: ConditionalSelector[Double], - parentIndex: Int, - outcomeVar: Variable[U], - chainVar: Variable[U], selector: Factor[Double]): Unit = { - - val chainSize = chainVar.size - - // If we don't care, we assign 1.0 to all combinations of the distVar and outcomeVar - for { - j <- 0 until outcomeVar.size - k <- 0 until chainSize - } { - - // calculate the relevant index of the selector Factor and update - // current (conditional selector) Factor - // The selector factor is sparse and needs no update - val selectorIndex = parentIndex * chainSize + k - factor.set(List(selectorIndex, j), 1.0) - } - } -} \ No newline at end of file +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ComplexFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ComplexFactory.scala index 679b8b35..9c1e0c38 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ComplexFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ComplexFactory.scala @@ -1,22 +1,25 @@ /* * ComplexFactory.scala - * Description needed - * + * Methods to create factors associated with a variety of complex elements. + * * Created By: Glenn Takata (gtakata@cra.com) * Creation Date: Dec 15, 2014 - * + * * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ package com.cra.figaro.algorithm.factored.factors.factory -import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.lazyfactored.{ValueSet, Extended, Regular} +import com.cra.figaro.algorithm.lazyfactored.ValueSet._ import com.cra.figaro.language._ import com.cra.figaro.library.compound._ import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.structured.{ComponentCollection, ProblemComponent, Range} +import com.cra.figaro.util.{MultiSet, HashMultiSet, homogeneousCartesianProduct} /** * A Sub-Factory for Complex Elements @@ -26,158 +29,229 @@ object ComplexFactory { /** * Factor constructor for a SingleValuedReferenceElement */ - def makeFactors[T](element: SingleValuedReferenceElement[T]): List[Factor[Double]] = { - val (first, rest) = element.collection.getFirst(element.reference) - rest match { - case None => - val elementVar = Variable(element) - val firstVar = Variable(first) - val factor = new BasicFactor[Double](List(firstVar), List(elementVar)) - for { - i <- 0 until firstVar.range.size - j <- 0 until elementVar.range.size - } { - factor.set(List(i, j), (if (i == j) 1.0; else 0.0)) - } - List(factor) - case Some(restRef) => - val firstVar = Variable(first) - val selectedFactors = + def makeFactors[T](cc: ComponentCollection, element: SingleValuedReferenceElement[T]): List[Factor[Double]] = { + // Make the list of factors starting from the given element collection with the given reference. + // startVar represents the variable whose value is the value of the reference. + def make(ec: ElementCollection, startVar: Variable[T], reference: Reference[T]): List[Factor[Double]] = { + val (firstElem, restRefOpt) = ec.getFirst(reference) + restRefOpt match { + case None => + val firstVar = Factory.getVariable(cc, firstElem) + val factor = new SparseFactor[Double](List(firstVar), List(startVar)) for { - (firstXvalue, firstIndex) <- firstVar.range.zipWithIndex - firstCollection = firstXvalue.value.asInstanceOf[ElementCollection] - restElement = element.embeddedElements(firstCollection) - } yield { - Factory.makeConditionalSelector(element, firstVar, firstIndex, Variable(restElement)) :: makeFactors(restElement) + (firstVal, firstIndex) <- firstVar.range.zipWithIndex + } { + val startIndex = startVar.range.indexOf(firstVal) + factor.set(List(firstIndex, startIndex), 1.0) } - selectedFactors.flatten + List(factor) + + case Some(restRef) => + val firstVar: Variable[firstElem.Value] = Factory.getVariable(cc, firstElem).asInstanceOf[Variable[firstElem.Value]] + // For each possible first element collection, we: + // - create an element named restVar to denote the value of the reference starting from that collection + // - get the factors associated with restVar by calling make(firstCollection, restVar, restRef) + // - use a conditional selector to select thos factors appropriately + // Then we take all of the factors for all first element collections. + val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, None, firstVar, startVar) + val selectedFactors = + for { + (firstXValue, firstIndex) <- firstVar.range.zipWithIndex + } yield { + if (firstXValue.isRegular) { + val firstCollection = firstXValue.value.asInstanceOf[ElementCollection] + val restRange = Range.getRangeOfSingleValuedReference(cc, firstCollection, restRef) + val restVar: Variable[T] = Factory.makeVariable(cc, restRange) + val restFactors = make(firstCollection, restVar, restRef) + + Factory.makeConditionalSelector(pairVar, firstXValue, restVar, Set()) :: restFactors + } else { + val dummy = Factory.makeVariable(cc, ValueSet.withStar[T](Set())) + List(Factory.makeConditionalSelector(pairVar, firstXValue, dummy, Set())) + } + } + pairFactor :: selectedFactors.flatten + } } + + make(element.collection, Factory.getVariable(cc, element), element.reference) } - + /** * Factor constructor for a MultiValuedReferenceElement */ - def makeFactors[T](element: MultiValuedReferenceElement[T]): List[Factor[Double]] = { - val (first, rest) = element.collection.getFirst(element.reference) - val selectionFactors: List[List[Factor[Double]]] = { - rest match { - case None => - val elementVar = Variable(element) - val firstVar = Variable(first) - val factor = new BasicFactor[Double](List(firstVar), List(elementVar)) + def makeFactors[T](cc: ComponentCollection, element: MultiValuedReferenceElement[T]): List[Factor[Double]] = { + def makeEmbeddedInject(inputVariables: List[Variable[MultiSet[T]]]): (Variable[List[MultiSet[T]]], Factor[Double]) = { + def rule(values: List[Extended[_]]) = { + val inputXvalues :+ resultXvalue = values + if (inputXvalues.exists(!_.isRegular)) { + if (!resultXvalue.isRegular) 1.0; else 0.0 + } else if (resultXvalue.isRegular) { + if (resultXvalue.value.asInstanceOf[List[T]] == inputXvalues.map(_.value.asInstanceOf[T])) 1.0; else 0.0 + } else 0.0 + } + val argVSs = inputVariables.map(_.valueSet) + val incomplete = argVSs.exists(_.hasStar) + val argChoices = argVSs.toList.map(_.regularValues.toList) + val resultValues: Set[List[MultiSet[T]]] = homogeneousCartesianProduct(argChoices: _*).toSet + val injectRange = if (incomplete) withStar(resultValues); else withoutStar(resultValues) + + val resultVariable = Factory.makeVariable(cc, injectRange) + val factor = new BasicFactor[Double](inputVariables, List(resultVariable)) + factor.fillByRule(rule _) + (resultVariable, factor) + } + + def makeEmbeddedApply(injectVar: Variable[List[MultiSet[T]]]): (Variable[MultiSet[T]], Factor[Double]) = { + def rule(sets: List[MultiSet[T]]): MultiSet[T] = { + val starter: MultiSet[T] = HashMultiSet[T]() + sets.foldLeft(starter)(_ union _) + } + val applyVS: ValueSet[MultiSet[T]] = injectVar.valueSet.map(rule(_)) + val applyVar = Factory.makeVariable(cc, applyVS) + val factor = new SparseFactor[Double](List(injectVar), List(applyVar)) + for { (injectVal, injectIndex) <- injectVar.range.zipWithIndex } { + if (injectVal.isRegular) { + val resultVal = rule(injectVal.value) + val resultIndex = applyVar.range.indexWhere(_.value == resultVal) + factor.set(List(injectIndex, resultIndex), 1.0) + } else if (!injectVal.isRegular && applyVar.range.exists(!_.isRegular)) { + val resultIndex = applyVar.range.indexWhere(!_.isRegular) + factor.set(List(injectIndex, resultIndex), 1.0) + } + } + (applyVar, factor) + } + + // Make the list of factors starting from the given element collection with the given reference. + // startVar represents the variable whose value is the value of the reference. + def make(ec: ElementCollection, startVar: Variable[MultiSet[T]], reference: Reference[T]): List[Factor[Double]] = { + val (firstElem, restRefOpt) = ec.getFirst(reference) + val firstVar = Factory.getVariable(cc, firstElem).asInstanceOf[Variable[firstElem.Value]] + restRefOpt match { + case None => { + // When the reference is simple, it only refers to a single element, so we just get the factor mapping + // values of that element to values of startVar + val factor = new SparseFactor[Double](List(firstVar), List(startVar)) for { - i <- 0 until firstVar.range.size - j <- 0 until elementVar.range.size + (firstVal, firstIndex) <- firstVar.range.zipWithIndex } { - factor.set(List(i, j), (if (i == j) 1.0; else 0.0)) + val startIndex = + if (firstVal.isRegular) startVar.range.indexWhere(_.value == HashMultiSet(firstVal.value)) + else startVar.range.indexWhere(!_.isRegular) + factor.set(List(firstIndex, startIndex), 1.0) } - List(List(factor)) - case Some(restRef) => - val firstVar = Variable(first) - for { - (firstXvalue, firstIndex) <- firstVar.range.zipWithIndex - } yield { - if (firstXvalue.isRegular) { - firstXvalue.value match { - case firstCollection: ElementCollection => - val restElement = element.embeddedElements(firstCollection) - val result: List[Factor[Double]] = - Factory.makeConditionalSelector(element, firstVar, firstIndex, Variable(restElement)) :: Factory.make(restElement) - result - case cs: Traversable[_] => - // Create a multi-valued reference element (MVRE) for each collection in the value of the first name. - // Since the first name is multi-valued, its value is the union of the values of all these MVREs. - val collections = cs.asInstanceOf[Traversable[ElementCollection]].toList.distinct // Set semantics - val multis: List[MultiValuedReferenceElement[T]] = collections.map(element.embeddedElements(_)).toList - // Create the element that takes the union of the values of the all the MVREs. - // The combination and setMaker elements are encapsulated within this object and are created now, so we need to create factors for them. - // Finally, we create a conditional selector (see ProbFactor) to select the appropriate result value when the first - // name's value is these MVREs. - val combination = element.embeddedInject(collections) - val setMaker = element.embeddedApply(collections) - val result: List[Factor[Double]] = - Factory.makeConditionalSelector(element, firstVar, firstIndex, Variable(setMaker)) :: Factory.make(combination) ::: - Factory.make(setMaker) - result + List(factor) + } + + case Some(restRef) => { + // When the reference is indirect, we get all factors associated with all values of the first name in the reference. + // Each first value is either a single element collection, in which case we recurse simply, just like for a single-valued reference. + // Otherwise, the first value is a traversable of element collections, in which case see below. + val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, None, firstVar, startVar) + val selectionFactors: List[List[Factor[Double]]] = { + val selectedFactors = + for { + (firstXvalue, firstIndex) <- firstVar.range.zipWithIndex + } yield { + if (!firstXvalue.isRegular) { + val dummy = Factory.makeVariable(cc, ValueSet.withStar[MultiSet[T]](Set())) + List(Factory.makeConditionalSelector(pairVar, firstXvalue, dummy, Set())) + } + else { + firstXvalue.value match { + case firstCollection: ElementCollection => + val restRange = Range.getRangeOfMultiValuedReference(cc, firstCollection, restRef) + val restVar: Variable[MultiSet[T]] = Factory.makeVariable(cc, restRange) + val restFactors = make(firstCollection, restVar, restRef) + Factory.makeConditionalSelector(pairVar, firstXvalue, restVar, Set()) :: restFactors + case cs: Traversable[_] => + // If the first value consists of multiple element collections, we first get a list of distinct collections. + // This is because multi-valued references use set semantics, whereby if an element is pointed to more than once, + // its value only counts once in the multiset value of the reference element. + // So, a multiset value consists of a value for each of the distinct element collections. + // For each collection, we get the factors for it using make(firstCollection, restVar, restRef) + // We add a conditional selector on the first value to determine when these factors are relevant. + // Then, we essentially create an embedded Inject on the variables representing these collections, + // and then an Apply that takes a list of values from these collections and turns them into a multiset. + // However, unlike the old implementation, we do not actually create these as elements, as that would + // break the atomicity of makeFactors required by structured factored inference. + // We just create a variable for the Inject and a variable for the Apply and create the factors directly. + val collections = cs.asInstanceOf[Traversable[ElementCollection]].toList.distinct // Set semantics + val factorsForCollections = + for { firstCollection <- collections } yield { + val restRange = Range.getRangeOfMultiValuedReference(cc, firstCollection, restRef) + val restVar: Variable[MultiSet[T]] = Factory.makeVariable(cc, restRange) + val restFactors = make(firstCollection, restVar, restRef) + (restVar, restFactors) + } + val (injectVar, injectFactor) = makeEmbeddedInject(factorsForCollections.map(_._1)) + val (applyVar, applyFactor) = makeEmbeddedApply(injectVar) + val valueFactors = applyFactor :: injectFactor :: factorsForCollections.map(_._2).flatten + Factory.makeConditionalSelector(pairVar, firstXvalue, applyVar, Set()) :: valueFactors + } + } } - } else StarFactory.makeStarFactor(element) + selectedFactors } + pairFactor :: selectionFactors.flatten + } } } - selectionFactors.flatten + + make(element.collection, Factory.getVariable(cc, element), element.reference) } /** * Factor constructor for an Aggregate Element */ - def makeFactors[T, U](element: Aggregate[T, U]): List[Factor[Double]] = { - val elementVar = Variable(element) - val mvreVar = Variable(element.mvre) - val factor = new BasicFactor[Double](List(mvreVar), List(elementVar)) + def makeFactors[T, U](cc: ComponentCollection, element: Aggregate[T, U]): List[Factor[Double]] = { + val elementVar = Factory.getVariable(cc, element) + val mvreVar = Factory.getVariable(cc, element.mvre) + val factor = new SparseFactor[Double](List(mvreVar), List(elementVar)) for { (mvreXvalue, mvreIndex) <- mvreVar.range.zipWithIndex - (elementXvalue, elementIndex) <- elementVar.range.zipWithIndex } { - if (elementXvalue.isRegular && mvreXvalue.isRegular) factor.set(List(mvreIndex, elementIndex), if (element.aggregate(mvreXvalue.value) == elementXvalue.value) 1.0; else 0.0) + if (mvreXvalue.isRegular) { + val aggValue = element.aggregate(mvreXvalue.value) + val elementIndex = elementVar.range.indexOf(Regular(aggValue)) + factor.set(List(mvreIndex, elementIndex), 1.0) + } else factor.set(List(mvreIndex, elementVar.range.indexWhere(!_.isRegular)), 1.0) } // The MultiValuedReferenceElement for this aggregate is generated when values is called. // Therefore, it will be included in the expansion and have factors made for it automatically, so we do not create factors for it here. List(factor) } - /** - * Constructor for a MakeList Element - */ - def makeFactors[T](element: MakeList[T]): List[Factor[Double]] = { - val parentVar = Variable(element.numItems) - // We need to create factors for the items and the lists themselves, which are encapsulated in this MakeList - val regularParents = parentVar.range.filter(_.isRegular).map(_.value) - val maxItem = regularParents.reduce(_ max _) - val itemFactors = List.tabulate(maxItem)((i: Int) => Factory.make(element.items(i))) - val indexedResultElemsAndFactors = - for { i <- regularParents } yield { - val elem = element.embeddedInject(i) - val factors = Factory.make(elem) - (Regular(i), elem, factors) - } - val conditionalFactors = - parentVar.range.zipWithIndex map (pair => - Factory.makeConditionalSelector(element, parentVar, pair._2, Variable(indexedResultElemsAndFactors.find(_._1 == pair._1).get._2))) - conditionalFactors ::: itemFactors.flatten ::: indexedResultElemsAndFactors.flatMap(_._3) - } - - // adapted from Apply1 /** * Factor constructor for a MakeArray Element */ - def makeFactors[T](element: com.cra.figaro.library.collection.MakeArray[T]): List[Factor[Double]] = { - val arg1Var = Variable(element.numItems) - val resultVar = Variable(element) - val factor = new BasicFactor[Double](List(arg1Var), List(resultVar)) + def makeFactors[T](cc: ComponentCollection, element: com.cra.figaro.library.collection.MakeArray[T]): List[Factor[Double]] = { + val arg1Var = Factory.getVariable(cc, element.numItems) + val resultVar = Factory.getVariable(cc, element) + val makeArrayComponent = cc(element) + val factor = new SparseFactor[Double](List(arg1Var), List(resultVar)) val arg1Indices = arg1Var.range.zipWithIndex val resultIndices = resultVar.range.zipWithIndex for { (arg1Val, arg1Index) <- arg1Indices (resultVal, resultIndex) <- resultIndices } { - val entry = - if (arg1Val.isRegular && resultVal.isRegular) { - if (resultVal.value == element.arrays(arg1Val.value)) 1.0 - else 0.0 - } else if (!arg1Val.isRegular && !resultVal.isRegular) 1.0 - else if (!arg1Val.isRegular && resultVal.isRegular) 0.0 - else 0.0 - factor.set(List(arg1Index, resultIndex), entry) + if ((arg1Val.isRegular && arg1Val.value <= makeArrayComponent.maxExpanded && + resultVal.isRegular && resultVal.value == element.arrays(arg1Val.value) || + (!arg1Val.isRegular || arg1Val.value > makeArrayComponent.maxExpanded) && !resultVal.isRegular)) { + factor.set(List(arg1Index, resultIndex), 1.0) + } } List(factor) } - + /** * Factor constructor for a FoldLeft Element */ - def makeFactors[T,U](fold: FoldLeft[T,U]): List[Factor[Double]] = { + def makeFactors[T,U](cc: ComponentCollection, fold: FoldLeft[T,U]): List[Factor[Double]] = { def makeOneFactor(currentAccumVar: Variable[U], elemVar: Variable[T], nextAccumVar: Variable[U]): Factor[Double] = { - val result = new BasicFactor[Double](List(currentAccumVar, elemVar), List(nextAccumVar)) + val result = new SparseFactor[Double](List(currentAccumVar, elemVar), List(nextAccumVar)) val currentAccumIndices = currentAccumVar.range.zipWithIndex val elemIndices = elemVar.range.zipWithIndex val nextAccumIndices = nextAccumVar.range.zipWithIndex @@ -188,11 +262,11 @@ object ComplexFactory { } { val entry = if (currentAccumVal.isRegular && elemVal.isRegular && nextAccumVal.isRegular) { - if (nextAccumVal.value == fold.function(currentAccumVal.value, elemVal.value)) 1.0 - else 0.0 - } else if ((!currentAccumVal.isRegular || !elemVal.isRegular) && !nextAccumVal.isRegular) 1.0 - else 0.0 - result.set(List(currentAccumIndex, elemIndex, nextAccumIndex), entry) + if (nextAccumVal.value == fold.function(currentAccumVal.value, elemVal.value)) + result.set(List(currentAccumIndex, elemIndex, nextAccumIndex), 1.0) + } else if ((!currentAccumVal.isRegular || !elemVal.isRegular) && !nextAccumVal.isRegular) { + result.set(List(currentAccumIndex, elemIndex, nextAccumIndex), 1.0) + } } result } @@ -200,27 +274,27 @@ object ComplexFactory { def makeFactorSequence(currentAccumVar: Variable[U], remaining: Seq[Element[T]]): List[Factor[Double]] = { if (remaining.isEmpty) List() else { - val firstVar = Variable(remaining.head) + val firstVar = Factory.getVariable(cc, remaining.head) val rest = remaining.tail val nextAccumVar = - if (rest.isEmpty) Variable(fold) + if (rest.isEmpty) Factory.getVariable(cc, fold) else { - val currentAccumRegular = currentAccumVar.range.filter(_.isRegular).map(_.value) - val firstRegular = firstVar.range.filter(_.isRegular).map(_.value) - val nextVals = - for { - accum <- currentAccumRegular - first <- firstRegular - } yield fold.function(accum, first) - val nextHasStar = currentAccumVar.range.exists(!_.isRegular) || firstVar.range.exists(!_.isRegular) - val nextVS = if (nextHasStar) ValueSet.withStar(nextVals.toSet) else ValueSet.withoutStar(nextVals.toSet) - new Variable(nextVS) + val currentAccumRegular = currentAccumVar.range.filter(_.isRegular).map(_.value) + val firstRegular = firstVar.range.filter(_.isRegular).map(_.value) + val nextVals = + for { + accum <- currentAccumRegular + first <- firstRegular + } yield fold.function(accum, first) + val nextHasStar = currentAccumVar.range.exists(!_.isRegular) || firstVar.range.exists(!_.isRegular) + val nextVS = if (nextHasStar) ValueSet.withStar(nextVals.toSet) else ValueSet.withoutStar(nextVals.toSet) + Factory.makeVariable(cc, nextVS) } val nextFactor = makeOneFactor(currentAccumVar, firstVar, nextAccumVar) nextFactor :: makeFactorSequence(nextAccumVar, rest) } } - val startVar = new Variable(ValueSet.withoutStar(Set(fold.start))) + val startVar = Factory.makeVariable(cc, ValueSet.withoutStar(Set(fold.start))) makeFactorSequence(startVar, fold.elements) } } diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ConstraintFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ConstraintFactory.scala similarity index 97% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ConstraintFactory.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ConstraintFactory.scala index 920f4f77..7dd2f0c3 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ConstraintFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/ConstraintFactory.scala @@ -10,10 +10,10 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured.factory +package com.cra.figaro.algorithm.factored.factors.factory import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured.ComponentCollection +import com.cra.figaro.algorithm.structured.ComponentCollection import com.cra.figaro.algorithm.factored.factors._ import com.cra.figaro.algorithm.lazyfactored.Regular diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/DistributionFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/DistributionFactory.scala index 26bd68af..dc76fb54 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/DistributionFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/DistributionFactory.scala @@ -1,13 +1,13 @@ /* * DistributionFactory.scala - * Description needed - * + * Methods to create factors for simple distributions. + * * Created By: Glenn Takata (gtakata@cra.com) * Creation Date: Dec 15, 2014 - * + * * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -16,21 +16,22 @@ package com.cra.figaro.algorithm.factored.factors.factory import com.cra.figaro.language._ import com.cra.figaro.library.atomic.discrete._ import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.lazyfactored.Regular +import com.cra.figaro.algorithm.structured.ComponentCollection /** * A Sub-Factory for simple probability distribution Elements */ object DistributionFactory { - + /** - * Factor constructor for an AtomicFlip + * Factor constructor for an AtomicFlip */ - def makeFactors(flip: AtomicFlip): List[Factor[Double]] = { - val flipVar = Variable(flip) + def makeFactors(cc: ComponentCollection, flip: AtomicFlip): List[Factor[Double]] = { + val flipVar = Factory.getVariable(cc, flip) if (flipVar.range.exists(!_.isRegular)) { assert(flipVar.range.size == 1) // Flip's range must either be {T,F} or {*} - StarFactory.makeStarFactor(flip) + StarFactory.makeStarFactor(cc, flip) } else { val factor = new BasicFactor[Double](List(), List(flipVar)) val i = flipVar.range.indexOf(Regular(true)) @@ -43,48 +44,77 @@ object DistributionFactory { /** * Factor constructor for a CompoundFlip */ - def makeFactors(flip: CompoundFlip): List[Factor[Double]] = { - val flipVar = Variable(flip) + def makeFactors(cc: ComponentCollection, flip: CompoundFlip): List[Factor[Double]] = { + val flipVar = Factory.getVariable(cc, flip) + val probVar = Factory.getVariable(cc, flip.prob) + makeCompoundFlip(flipVar, probVar) + } + + private def makeCompoundFlip(flipVar: Variable[Boolean], probVar: Variable[Double]): List[Factor[Double]] = { + val factor = new BasicFactor[Double](List(probVar), List(flipVar)) + val parentVals = probVar.range if (flipVar.range.exists(!_.isRegular)) { - assert(flipVar.range.size == 1) // Flip's range must either be {T,F} or {*} - StarFactory.makeStarFactor(flip) - } else { - val probVar = Variable(flip.prob) - val factor = new BasicFactor[Double](List(probVar), List(flipVar)) - val parentVals = probVar.range - val i = flipVar.range.indexOf(Regular(true)) + val falseIndex = flipVar.range.indexOf(Regular(false)) + val trueIndex = flipVar.range.indexOf(Regular(true)) + val starIndex = flipVar.range.indexWhere(!_.isRegular) for { j <- 0 until parentVals.size } { if (parentVals(j).isRegular) { val value = parentVals(j).value - factor.set(List(j, i), value) - factor.set(List(j, 1 - i), 1.0 - value) + factor.set(List(j, trueIndex), value) + factor.set(List(j, falseIndex), 1.0 - value) + factor.set(List(j, starIndex), 0.0) } else { - factor.set(List(j, 0), 0.0) - factor.set(List(j, 1), 0.0) + factor.set(List(j, trueIndex), 0.0) + factor.set(List(j, falseIndex), 0.0) + factor.set(List(j, starIndex), 1.0) } } List(factor) + } else { + val trueIndex = flipVar.range.indexOf(Regular(true)) + val falseIndex = 1 - trueIndex + for { j <- 0 until parentVals.size } { + val value = parentVals(j).value + factor.set(List(j, trueIndex), value) + factor.set(List(j, falseIndex), 1.0 - value) + } + List(factor) } } - + /** * Factor constructor for a ParameterizedFlip */ - def makeFactors(flip: ParameterizedFlip): List[Factor[Double]] = { - val flipVar = Variable(flip) - val factor = new BasicFactor[Double](List(),List(flipVar)) - val prob = flip.parameter.MAPValue - val i = flipVar.range.indexOf(Regular(true)) - factor.set(List(i), prob) - factor.set(List(1 - i), 1.0 - prob) - List(factor) + def makeFactors(cc: ComponentCollection, flip: ParameterizedFlip, parameterized: Boolean): List[Factor[Double]] = { + if (parameterized) { + val flipVar = Factory.getVariable(cc, flip) + val factor = new BasicFactor[Double](List(),List(flipVar)) + val prob = flip.parameter.MAPValue + if (flipVar.range.forall(_.isRegular)) { + val i = flipVar.range.indexOf(Regular(true)) + factor.set(List(i), prob) + factor.set(List(1 - i), 1.0 - prob) + } else { + val trueIndex = flipVar.range.indexOf(Regular(true)) + val falseIndex = flipVar.range.indexOf(Regular(false)) + val starIndex = flipVar.range.indexWhere(!_.isRegular) + factor.set(List(trueIndex), prob) + factor.set(List(falseIndex), 1.0 - prob) + factor.set(List(starIndex), 0.0) + } + List(factor) + } else { + val flipVar = Factory.getVariable(cc, flip) + val probVar = Factory.getVariable(cc, flip.parameter) + makeCompoundFlip(flipVar, probVar) + } } - + /** * Factor constructor for an AtomicBinomial */ - def makeFactors(binomial: AtomicBinomial): List[Factor[Double]] = { - val binVar = Variable(binomial) + def makeFactors(cc: ComponentCollection, binomial: AtomicBinomial): List[Factor[Double]] = { + val binVar = Factory.getVariable(cc, binomial) val factor = new BasicFactor[Double](List(), List(binVar)) for { (xvalue, index) <- binVar.range.zipWithIndex } { factor.set(List(index), binomial.density(xvalue.value)) @@ -92,4 +122,38 @@ object DistributionFactory { List(factor) } -} \ No newline at end of file + /** + * Factor constructor for a parameterized binomial + */ + def makeFactors(cc: ComponentCollection, binomial: ParameterizedBinomialFixedNumTrials, parameterized: Boolean): List[Factor[Double]] = { + if (parameterized) { + val binVar = Factory.getVariable(cc, binomial) + val factor = new BasicFactor[Double](List(),List(binVar)) + if (binVar.range.exists(!_.isRegular)) { // parameter must not have been added since it's an atomic beta + for { (xvalue, index) <- binVar.range.zipWithIndex } { + val entry = if (xvalue.isRegular) 0.0 else 1.0 + factor.set(List(index), entry) + } + } else { + val probSuccess = binomial.parameter.MAPValue + for { (xvalue, index) <- binVar.range.zipWithIndex } { + factor.set(List(index), Util.binomialDensity(binomial.numTrials, probSuccess, xvalue.value)) + } + } + List(factor) + } else { + val binVar = Factory.getVariable(cc, binomial) + if (binVar.range.exists(!_.isRegular)) { // parameter must not have been added since it's an atomic beta + val factor = new BasicFactor[Double](List(),List(binVar)) + for { (xvalue, index) <- binVar.range.zipWithIndex } { + val entry = if (xvalue.isRegular) 0.0 else 1.0 + factor.set(List(index), entry) + } + List(factor) + } else { + ChainFactory.makeFactors(cc, binomial) + } + } + } + +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/FactorMaker.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/FactorMaker.scala new file mode 100644 index 00000000..d7d547b3 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/FactorMaker.scala @@ -0,0 +1,20 @@ +/* + * FactorMaker.scala + * A trait for elements that are able to construct their own Factor. + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: December 30, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.factored.factors.factory + +import com.cra.figaro.algorithm.factored.factors.Factor + +trait FactorMaker[T] { + def makeFactors[T]: List[Factor[Double]] +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/Factory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/Factory.scala similarity index 59% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/Factory.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/Factory.scala index 881522d8..7671e710 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/Factory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/Factory.scala @@ -11,22 +11,21 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured.factory +package com.cra.figaro.algorithm.factored.factors.factory import com.cra.figaro.algorithm._ import com.cra.figaro.language._ import com.cra.figaro.util._ import com.cra.figaro.algorithm.lazyfactored._ -import ValueSet._ +import com.cra.figaro.algorithm.lazyfactored.ValueSet._ import com.cra.figaro.algorithm.factored._ -import scala.collection.mutable.ListBuffer import com.cra.figaro.algorithm.factored.factors._ import com.cra.figaro.library.compound._ import com.cra.figaro.library.collection._ import com.cra.figaro.library.atomic.discrete._ +import com.cra.figaro.algorithm.structured._ import scala.reflect.runtime.universe.{typeTag, TypeTag} -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.experimental.structured.ProblemComponent + /** * Methods for creating probabilistic factors associated with elements. @@ -54,26 +53,35 @@ object Factory { v } + def makeVariable[T](cc: ComponentCollection, valueSet: ValueSet[T], chain: Chain[_, _]): Variable[T] = { + val v = new InternalChainVariable(valueSet, chain, getVariable(cc, chain)) + cc.intermediates += v + v + } + /** * Create a new factor in which one variable is replaced with another. */ def replaceVariable[T](factor: Factor[Double], oldVariable: Variable[T], newVariable: Variable[T]): Factor[Double] = { if (oldVariable.range != newVariable.range) throw new IllegalArgumentException("Replacing variable with variable with different range") - val newVariables = factor.variables.map(v => if (v == oldVariable) newVariable else v) - val newFactor = - factor match { - case s: SparseFactor[Double] => new SparseFactor[Double](newVariables, List()) - case b: BasicFactor[Double] => new BasicFactor[Double](newVariables, List()) - } - for { indices <- factor.getIndices } { newFactor.set(indices, factor.get(indices)) } - newFactor + if (!factor.variables.contains(oldVariable)) factor + else { + val newVariables = factor.variables.map(v => if (v == oldVariable) newVariable else v) + val newFactor = + factor match { + case s: SparseFactor[Double] => new SparseFactor[Double](newVariables, List()) + case b: BasicFactor[Double] => new BasicFactor[Double](newVariables, List()) + } + for { indices <- factor.getIndices } { newFactor.set(indices, factor.get(indices)) } + newFactor + } } /** * The mutliplicative identity factor. */ def unit[T: TypeTag](semiring: Semiring[T]): Factor[T] = { - val result = new BasicFactor[T](List(), List()) + val result = new BasicFactor[T](List(), List(), semiring) result.set(List(), semiring.one) result } @@ -83,13 +91,13 @@ object Factory { * and create the factor mapping the inputs to their tuple. * @param inputs the variables to be formed into a tuple */ - def makeTupleVarAndFactor(cc: ComponentCollection, inputs: Variable[_]*): (Variable[List[Extended[_]]], Factor[Double]) = { + def makeTupleVarAndFactor(cc: ComponentCollection, chain: Option[Chain[_, _]], inputs: Variable[_]*): (Variable[List[Extended[_]]], Factor[Double]) = { val inputList: List[Variable[_]] = inputs.toList // Subtlety alert: In the tuple, we can't just map inputs with * to *. We need to remember which input was *. // Therefore, instead, we make the value a regular value consisting of a list of extended values. - val tupleRangeRegular: List[List[_]] = cartesianProduct(inputList.map(_.range):_*) + val tupleRangeRegular: List[List[_]] = cartesianProduct(inputList.map(_.range): _*) val tupleVS: ValueSet[List[Extended[_]]] = withoutStar(tupleRangeRegular.map(_.asInstanceOf[List[Extended[_]]]).toSet) - val tupleVar: Variable[List[Extended[_]]] = Factory.makeVariable(cc, tupleVS) + val tupleVar: Variable[List[Extended[_]]] = if (chain.nonEmpty) Factory.makeVariable(cc, tupleVS, chain.get) else Factory.makeVariable(cc, tupleVS) val tupleFactor = new SparseFactor[Double](inputList, List(tupleVar)) for { pair <- tupleVar.range.zipWithIndex } { val tupleVal: List[Extended[_]] = pair._1.value @@ -104,8 +112,8 @@ object Factory { /** * Create a BasicFactor from the supplied parent and children variables */ - def defaultFactor[T: TypeTag](parents: List[Variable[_]], children: List[Variable[_]]) = - new BasicFactor[T](parents, children) + def defaultFactor[T: TypeTag](parents: List[Variable[_]], children: List[Variable[_]], _semiring: Semiring[T] = SumProductSemiring().asInstanceOf[Semiring[T]]) = + new BasicFactor[T](parents, children, _semiring) private def makeFactors[T](cc: ComponentCollection, const: Constant[T]): List[Factor[Double]] = { val factor = new BasicFactor[Double](List(), List(getVariable(cc, const))) @@ -127,8 +135,8 @@ object Factory { * parent element, one of the result elements, and the overall chain element. */ - def makeConditionalSelector[T,U](pairVar: Variable[List[Extended[_]]], parentXVal: Extended[T], outcomeVar: Variable[U]): Factor[Double] = { - val factor = new BasicFactor[Double](List(pairVar), List(outcomeVar)) + def makeConditionalSelector[T, U](pairVar: Variable[List[Extended[_]]], parentXVal: Extended[T], outcomeVar: Variable[U], choices: Set[U])(implicit mapper: PointMapper[U]): Factor[Double] = { + val factor = new ConditionalSelector[Double](List(pairVar), List(outcomeVar)) for { (pairXVal, pairIndex) <- pairVar.range.zipWithIndex (outcomeXVal, outcomeIndex) <- outcomeVar.range.zipWithIndex @@ -139,7 +147,7 @@ object Factory { val entry = if (selectXVal.isRegular && parentXVal.isRegular) { if (selectXVal.value == parentXVal.value) { - if (overallXVal == outcomeXVal || (!overallXVal.isRegular && !outcomeXVal.isRegular)) 1.0 else 0.0 + if ((!overallXVal.isRegular && !outcomeXVal.isRegular) || (overallXVal.isRegular && outcomeXVal.isRegular && overallXVal.value == mapper.map(outcomeXVal.value, choices))) 1.0 else 0.0 } else 1.0 } else if (selectXVal.isRegular || parentXVal.isRegular) 1.0 // they are different else if (!overallXVal.isRegular) 1.0 // if parentXVal is *, the only possible outcomeXVal is * @@ -199,41 +207,49 @@ object Factory { } } + + def parameterCheck(elem: Element[_], parameterized: Boolean): Boolean = { + elem match { + case parameter: DoubleParameter => parameterized + case _ => false + } + } + /** * Invokes Factor constructors for a standard set of Elements. This method uses various * secondary factories. */ def concreteFactors[T](cc: ComponentCollection, elem: Element[T], parameterized: Boolean): List[Factor[Double]] = { - elem match { - case flip: ParameterizedFlip => DistributionFactory.makeFactors(cc, flip, parameterized) - case pSelect: ParameterizedSelect[_] => SelectFactory.makeFactors(cc, pSelect, parameterized) - case pBin: ParameterizedBinomialFixedNumTrials => DistributionFactory.makeFactors(cc, pBin, parameterized) - case parameter: DoubleParameter => makeParameterFactors(cc, parameter) - case array: ArrayParameter => makeParameterFactors(cc, array) - case constant: Constant[_] => makeFactors(cc, constant) - case f: AtomicFlip => DistributionFactory.makeFactors(cc, f) - case f: CompoundFlip => DistributionFactory.makeFactors(cc, f) - case ab: AtomicBinomial => DistributionFactory.makeFactors(cc, ab) - case s: AtomicSelect[_] => SelectFactory.makeFactors(cc, s) - case s: CompoundSelect[_] => SelectFactory.makeFactors(cc, s) - case d: AtomicDist[_] => SelectFactory.makeFactors(cc, d) - case d: CompoundDist[_] => SelectFactory.makeFactors(cc, d) - case s: IntSelector => SelectFactory.makeFactors(cc, s) - case c: Chain[_, _] => ChainFactory.makeFactors(cc, c) - case a: Apply1[_, _] => ApplyFactory.makeFactors(cc, a) - case a: Apply2[_, _, _] => ApplyFactory.makeFactors(cc, a) - case a: Apply3[_, _, _, _] => ApplyFactory.makeFactors(cc, a) - case a: Apply4[_, _, _, _, _] => ApplyFactory.makeFactors(cc, a) - case a: Apply5[_, _, _, _, _, _] => ApplyFactory.makeFactors(cc, a) - case i: Inject[_] => makeFactors(cc, i) - case r: SingleValuedReferenceElement[_] => ComplexFactory.makeFactors(cc, r) - case r: MultiValuedReferenceElement[_] => ComplexFactory.makeFactors(cc, r) - case r: Aggregate[_, _] => ComplexFactory.makeFactors(cc, r) + (elem, parameterized) match { + case (flip: ParameterizedFlip, _) => DistributionFactory.makeFactors(cc, flip, parameterized) + case (pSelect: ParameterizedSelect[_], _) => SelectFactory.makeFactors(cc, pSelect, parameterized) + case (pBin: ParameterizedBinomialFixedNumTrials, _) => DistributionFactory.makeFactors(cc, pBin, parameterized) + case (parameter: DoubleParameter, true) => makeParameterFactors(cc, parameter) + case (array: ArrayParameter, true) => makeParameterFactors(cc, array) + case (constant: Constant[_], _) => makeFactors(cc, constant) + case (f: AtomicFlip, _) => DistributionFactory.makeFactors(cc, f) + case (f: CompoundFlip, _) => DistributionFactory.makeFactors(cc, f) + case (ab: AtomicBinomial, _) => DistributionFactory.makeFactors(cc, ab) + case (s: AtomicSelect[_], _) => SelectFactory.makeFactors(cc, s) + case (s: CompoundSelect[_], _) => SelectFactory.makeFactors(cc, s) + case (d: AtomicDist[_], _) => SelectFactory.makeFactors(cc, d) + case (d: CompoundDist[_], _) => SelectFactory.makeFactors(cc, d) + case (s: IntSelector, _) => SelectFactory.makeFactors(cc, s) + case (c: Chain[_, _], _) => ChainFactory.makeFactors(cc, c) + case (a: Apply1[_, _], _) => ApplyFactory.makeFactors(cc, a) + case (a: Apply2[_, _, _], _) => ApplyFactory.makeFactors(cc, a) + case (a: Apply3[_, _, _, _], _) => ApplyFactory.makeFactors(cc, a) + case (a: Apply4[_, _, _, _, _], _) => ApplyFactory.makeFactors(cc, a) + case (a: Apply5[_, _, _, _, _, _], _) => ApplyFactory.makeFactors(cc, a) + case (i: Inject[_], _) => makeFactors(cc, i) + case (r: SingleValuedReferenceElement[_], _) => ComplexFactory.makeFactors(cc, r) + case (r: MultiValuedReferenceElement[_], _) => ComplexFactory.makeFactors(cc, r) + case (r: Aggregate[_, _], _) => ComplexFactory.makeFactors(cc, r) //case m: MakeList[_] => ComplexFactory.makeFactors(cc, m) - case m: MakeArray[_] => ComplexFactory.makeFactors(cc, m) - case f: FoldLeft[_, _] => ComplexFactory.makeFactors(cc, f) -// case f: FactorMaker[_] => f.makeFactors - case a: Atomic[_] => makeFactors(cc, a) + case (m: MakeArray[_], _) => ComplexFactory.makeFactors(cc, m) + case (f: FoldLeft[_, _], _) => ComplexFactory.makeFactors(cc, f) + case (f: FactorMaker[_], _) => f.makeFactors + case (a: Atomic[_], _) => makeFactors(cc, a) case _ => throw new UnsupportedAlgorithmException(elem) } @@ -281,4 +297,68 @@ object Factory { } } } + + /** + * Create the probabilistic factor encoding the probability of evidence in the dependent universe as a function of the + * values of variables in the parent universe. The third argument is the the function to use for computing + * probability of evidence in the dependent universe. It is assumed that the definition of this function will already contain the + * right evidence. + */ + def makeDependentFactor(cc: ComponentCollection, parentUniverse: Universe, + dependentUniverse: Universe, + probEvidenceComputer: () => Double): Factor[Double] = { + val uses = dependentUniverse.parentElements filter (_.universe == parentUniverse) + def rule(values: List[Any]) = { + for { (elem, value) <- uses zip values } { elem.value = value.asInstanceOf[Regular[elem.Value]].value } + val result = probEvidenceComputer() + result + } + val variables = uses map (cc(_).variable) + val factor = new BasicFactor[Double](variables, List()) + factor.fillByRule(rule _) + factor + } + + /** + * Make factors for a particular element. This function wraps the SFI method of creating factors using component collections + */ + def makeFactorsForElement[Value](elem: Element[_], upper: Boolean = false, parameterized: Boolean = false) = { + Variable(elem) + val comp = Variable.cc(elem) + elem match { + // If the element is a chain, we need to create subproblems for each value of the chain + // to create factors accordingly + case c: Chain[_, _] => { + val chainMap = LazyValues(elem.universe).getMap(c) + chainMap.foreach(f => { + val subproblem = new NestedProblem(Variable.cc, f._2) + Variable.cc.expansions += (c.chainFunction, f._1) -> subproblem + }) + } + // If the element is a MakeArray, we need mark that it has been expanded. Note that + // the normal Values call will expand the MakeArray, we are just setting the max expansion here + case ma: MakeArray[_] => { + val maC = Variable.cc(ma) + maC.maxExpanded = Variable.cc(ma.numItems).range.regularValues.max + } + // If the element is an apply, we need to populate the Apply map used by the factor creation + case a: Apply[Value] => { + val applyComp = comp.asInstanceOf[ApplyComponent[Value]] + val applyMap = LazyValues(elem.universe).getMap(a) + applyComp.setMap(applyMap) + } + case _ => () + } + // Make the constraint and non-constraint factors for the element by calling the + // component factor makers + val constraint = if (upper) { + comp.makeConstraintFactors(Upper) + comp.constraintFactors(Upper) + } else { + comp.makeConstraintFactors(Lower) + comp.constraintFactors(Lower) + } + comp.makeNonConstraintFactors(parameterized) + constraint ::: comp.nonConstraintFactors + } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/SelectFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/SelectFactory.scala index a19c42dc..68f0120f 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/SelectFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/SelectFactory.scala @@ -1,96 +1,113 @@ /* * SelectFactory.scala - * Description needed - * + * Methods to create factors for Select and Dist elements. + * * Created By: Glenn Takata (gtakata@cra.com) * Creation Date: Dec 15, 2014 - * + * * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ package com.cra.figaro.algorithm.factored.factors.factory -import com.cra.figaro.algorithm.lazyfactored._ import com.cra.figaro.language._ import com.cra.figaro.algorithm.factored.factors._ import com.cra.figaro.library.compound._ import com.cra.figaro.util._ -import com.cra.figaro.algorithm.factored.factors.Factory +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.structured.ComponentCollection +import com.cra.figaro.algorithm.lazyfactored.Star /** * A Sub-Factory for Select or Dist Elements */ object SelectFactory { - + /** - * Factor constructor for an AtomicDist + * Factor constructor for an AtomicDistFlip */ - def makeFactors[T](dist: AtomicDist[T]): List[Factor[Double]] = { - val (intermed, clauseFactors) = intermedAndClauseFactors(dist) + def makeFactors[T](cc: ComponentCollection, dist: AtomicDist[T]): List[Factor[Double]] = { + val (intermed, clauseFactors) = intermedAndClauseFactors(cc, dist) val intermedFactor = makeSimpleDistribution(intermed, dist.probs) intermedFactor :: clauseFactors } /** - * Factor constructor for a CompoundDist + * Factor constructor for a CompoundDist */ - def makeFactors[T](dist: CompoundDist[T]): List[Factor[Double]] = { - val (intermed, clauseFactors) = intermedAndClauseFactors(dist) - val intermedFactor = makeComplexDistribution(intermed, dist.probs) + def makeFactors[T](cc: ComponentCollection, dist: CompoundDist[T]): List[Factor[Double]] = { + val (intermed, clauseFactors) = intermedAndClauseFactors(cc, dist) + val probVars = dist.probs map (Factory.getVariable(cc, _)) + val intermedFactor = makeComplexDistribution(cc, intermed, probVars) intermedFactor :: clauseFactors } - + /** - * Factor constructor for an AtomicSelect + * Factor constructor for an AtomicSelect */ - def makeFactors[T](select: AtomicSelect[T]): List[Factor[Double]] = { - val selectVar = Variable(select) + def makeFactors[T](cc: ComponentCollection, select: AtomicSelect[T]): List[Factor[Double]] = { + val selectVar = Factory.getVariable(cc, select) if (selectVar.range.exists(!_.isRegular)) { assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} - StarFactory.makeStarFactor(select) + StarFactory.makeStarFactor(cc, select) } else { - val probs = getProbs(select) + val probs = getProbs(cc, select) List(makeSimpleDistribution(selectVar, probs)) } } /** - * Factor constructor for a CompoundSelect + * Factor constructor for a CompoundSelect */ - def makeFactors[T](select: CompoundSelect[T]): List[Factor[Double]] = { - val selectVar = Variable(select) - if (selectVar.range.exists(!_.isRegular)) { - assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} - StarFactory.makeStarFactor(select) - } else { - val probs = getProbs(select) - List(makeComplexDistribution(selectVar, probs)) - } + def makeFactors[T](cc: ComponentCollection, select: CompoundSelect[T]): List[Factor[Double]] = { + val selectVar = Factory.getVariable(cc, select) + val probs = getProbs(cc, select) + val probVars = probs map (Factory.getVariable(cc, _)) + List(makeComplexDistribution(cc, selectVar, probVars)) } /** - * Factor constructor for a ParameterizedSelect + * Factor constructor for a ParameterizedSelect */ - def makeFactors[T](select: ParameterizedSelect[T]): List[Factor[Double]] = { - val selectVar = Variable(select) - if (selectVar.range.exists(!_.isRegular)) { - assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} - StarFactory.makeStarFactor(select) - } else { - val probs = parameterizedGetProbs(select) + def makeFactors[T](cc: ComponentCollection, select: ParameterizedSelect[T], parameterized: Boolean): List[Factor[Double]] = { + if (parameterized) { + val selectVar = Factory.getVariable(cc, select) + val probs = parameterizedGetProbs(cc, select) List(makeSimpleDistribution(selectVar, probs)) + } else { + val selectVar: Variable[T] = Factory.getVariable(cc, select) + if (selectVar.range.exists(!_.isRegular)) { + // If the select has * in its range, the parameter must not have been added (because the parameter is an atomic beta) + val factor = new BasicFactor[Double](List(), List(selectVar)) + for { (selectXval, i) <- selectVar.range.zipWithIndex } { + val entry = if (selectXval.isRegular) 0.0 else 1.0 + factor.set(List(i), entry) + } + List(factor) + } else { + val paramVar: Variable[Array[Double]] = Factory.getVariable(cc, select.parameter) + val factor = new BasicFactor[Double](List(paramVar), List(selectVar)) + for { + (paramVal, paramIndex) <- paramVar.range.zipWithIndex + (selectVal, selectIndex) <- selectVar.range.zipWithIndex + } { + val entry = paramVal.value(selectIndex) + factor.set(List(paramIndex, selectIndex), entry) + } + List(factor) + } } } /** - * Factor constructor for an IntSelector + * Factor constructor for an IntSelector */ - def makeFactors[T](select: IntSelector): List[Factor[Double]] = { - val elementVar = Variable(select) - val counterVar = Variable(select.counter) + def makeFactors[T](cc: ComponentCollection, select: IntSelector): List[Factor[Double]] = { + val elementVar = Factory.getVariable(cc, select) + val counterVar = Factory.getVariable(cc, select.counter) val comb = new BasicFactor[Double](List(counterVar), List(elementVar)) comb.fillByRule((l: List[Any]) => { val counterValue :: elementValue :: _ = l.asInstanceOf[List[Extended[Int]]] @@ -102,35 +119,38 @@ object SelectFactory { List(comb) } - private def getProbs[U, T](select: Select[U, T]): List[U] = getProbs(select, select.clauses) + private def getProbs[U, T](cc: ComponentCollection, select: Select[U, T]): List[U] = getProbs(cc, select, select.clauses) /** * Get the potential (probability) for each value of an element, based on supplied rules */ - def getProbs[U, T](elem: Element[T], clauses: List[(U, T)]): List[U] = { - val selectVar = Variable(elem) + def getProbs[U, T](cc: ComponentCollection, elem: Element[T], clauses: List[(U, T)]): List[U] = { + val selectVar = Factory.getVariable(cc, elem) def getProb(xvalue: Extended[T]): U = { clauses.find(_._2 == xvalue.value).get._1 // * cannot be a value of a Select } val probs = - for { xvalue <- selectVar.range } yield getProb(xvalue) + for { xvalue <- selectVar.range.filter(_.isRegular) } yield getProb(xvalue) probs } - private def parameterizedGetProbs[T](select: ParameterizedSelect[T]): List[Double] = { + private def parameterizedGetProbs[T](cc: ComponentCollection, select: ParameterizedSelect[T]): List[Double] = { val outcomes = select.outcomes val map = select.parameter.MAPValue for { - xvalue <- Variable(select).range - index = outcomes.indexOf(xvalue.value) - } yield map(index) + xvalue <- Factory.getVariable(cc, select).range + } yield { + if (xvalue.isRegular) map(outcomes.indexOf(xvalue.value)) else 0.0 + } } - private def intermedAndClauseFactors[U, T](dist: Dist[U, T]): (Variable[Int], List[Factor[Double]]) = { - val intermed = new Variable(ValueSet.withoutStar((0 until dist.clauses.size).toSet)) + private def intermedAndClauseFactors[U, T](cc: ComponentCollection, dist: Dist[U, T]): (Variable[Int], List[Factor[Double]]) = { + val intermed = Factory.makeVariable(cc, ValueSet.withoutStar((0 until dist.clauses.size).toSet)) + val distVar = Factory.getVariable(cc, dist) + val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, None, intermed, distVar) val clauseFactors = dist.outcomes.zipWithIndex map (pair => - Factory.makeConditionalSelector(dist, intermed, pair._2, Variable(pair._1))) - (intermed, clauseFactors) + Factory.makeConditionalSelector(pairVar, Regular(pair._2), Factory.getVariable(cc, pair._1), Set())) + (intermed, pairFactor :: clauseFactors) } /** @@ -145,25 +165,80 @@ object SelectFactory { factor } - private def makeComplexDistribution[T](target: Variable[T], probElems: List[Element[Double]]): Factor[Double] = { - val probVars: List[Variable[Double]] = probElems map (Variable(_)) + /* + * When one of the probability elements includes * in its range, so will the target element. + * This is necessary because when the value of any of the probability elements is *, the normalizing factor is unknown in this case, + * so we cannot assign a specific probability to any of the regular values. Instead, we assign probability 1 to *. + * The code in the method below is designed to take into account this case correctly. + */ + private def makeComplexDistribution[T](cc: ComponentCollection, target: Variable[T], probVars: List[Variable[Double]]): Factor[Double] = { val nVars = probVars.size val factor = new BasicFactor[Double](probVars, List(target)) val probVals: List[List[Extended[Double]]] = probVars map (_.range) - for { indices <- factor.getIndices } { - // unnormalized is a list, one for each probability element, - // of the value of that element under these indices - val unnormalized = - for { (probIndex, position) <- indices.toList.take(nVars).zipWithIndex } yield { - // The probability of the particular value of the probability element in this position - val xprob = probVals(position)(probIndex) - if (xprob.isRegular) xprob.value; else 0.0 + if (target.range.forall(_.isRegular)) { + /* + * This is the easy case. For each list of indices to the factor, the first nVars indices will be indices into the range of the + * probability variables, while the last index will be the index into the range of the target variable. + * The correct probability is the value of the probability element in the given position with the appropriate index. + * Because the probabilities may vary, we need to normalize them before putting in the factor. + * + * Note that the variables in the factor are ordered with the probability variables first and the target variable last. + * But in probVals, the first index is the position into the target variable, and only then do we have the probability indices. + */ + for { indices <- factor.getIndices } { + val unnormalized = + for { (probIndex, position) <- indices.toList.take(nVars).zipWithIndex } yield { + val xprob = probVals(position)(probIndex) // The probability of the particular value of the probability element in this position + xprob.value + } + val normalized = normalize(unnormalized).toArray + factor.set(indices, normalized(indices.last)) + } + } else { + /* + * In this case, the range of the target includes *. We do not assume any particular location in the range, + * so we set targetStarIndex to the correct location. + * Now, the indices to the factor will have nVar entries for each of the probability variables, plus one for the target. + * When we get the entries for the probability variables, we get nVars. But the range of the target is nVars plus one, + * because it includes *. So we extend the indices to probPlusStarIndices, with the targetStarIndex spliced in the correct place. + * + * Next, we distinguish between two cases. In the first case, one of the probability variables is *, so we assign probability 1 + * to the target being * and probability 0 everywhere. In the other case, all probability variables are regular, so we + * assign the appropriate probability to the each regular position. We are careful to get the value of the + * probability variable from its original position in the indices, and make sure we don't get the value of *. + */ + val targetStarIndex: Int = target.range.indexWhere(!_.isRegular) + for { indices <- factor.getIndices } { + val probIndices: List[Int] = indices.toList.take(nVars) + val probPlusStarIndices: List[Int] = + probIndices.slice(0, targetStarIndex) ::: List(indices(targetStarIndex)) ::: probIndices.slice(targetStarIndex, probIndices.length) + val allProbsRegularFlags: List[Boolean] = + for { (probPlusStarIndex, position) <- probPlusStarIndices.zipWithIndex } yield { + if (position < targetStarIndex) probVals(position)(probPlusStarIndex).isRegular + else if (position == targetStarIndex) true + else probVals(position)(probPlusStarIndex - 1).isRegular + } + val allProbsRegular: Boolean = allProbsRegularFlags.forall((b: Boolean) => b) + + val unnormalized: List[Double] = + for { (probPlusStarIndex, position) <- probPlusStarIndices.zipWithIndex } yield { + val xprob: Extended[Double] = + if (position < targetStarIndex) probVals(position)(probPlusStarIndex) + else if (position == targetStarIndex) Star[Double] + else probVals(position)(probPlusStarIndex - 1) + if (allProbsRegular) { + if (position != targetStarIndex) xprob.value else 0.0 + } else { + if (position == targetStarIndex) 1.0 else 0.0 + } } - val normalized = normalize(unnormalized).toArray - // The first variable specifies the position of the remaining variables, - // so indices(last) is the correct probability - factor.set(indices, normalized(indices.last)) + + val normalized: Array[Double] = normalize(unnormalized).toArray + // The first variable specifies the position of the remaining variables, so indices(0) is the correct probability + factor.set(indices, normalized(indices.last)) + } } + factor } -} \ No newline at end of file +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/StarFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/StarFactory.scala index 767a745f..4aac45bf 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/StarFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/factory/StarFactory.scala @@ -1,13 +1,13 @@ /* * StarFactory.scala - * Description needed - * + * The default factor over an element whose only value is Star. + * * Created By: Glenn Takata (gtakata@cra.com) * Creation Date: Dec 15, 2014 - * + * * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -15,23 +15,24 @@ package com.cra.figaro.algorithm.factored.factors.factory import com.cra.figaro.language._ import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.lazyfactored.Star +import com.cra.figaro.algorithm.structured.ComponentCollection /** * A Sub-Factory to make Star Factors from arbitrary elements */ object StarFactory { - + /** * Make a StarFactor from an Element MutableMap, PriorityQueue} +import com.cra.figaro.algorithm.factored.factors._ +import scala.collection.mutable.{Map => MutableMap} + +object BlockSampler { + /** + * Caches 1000 factors and uses a VE-like procedure to compute joint factors + */ + def default = (blockInfo: Gibbs.BlockInfo) => + new SimpleBlockSampler(blockInfo) with Cached with FactorProduct + + /** + * Like BlockSampler.default, but also uses Gaussian weighting scheme when there exist Element[Double]s that are results of Chains + */ + def continuous(_variance: Double) = (blockInfo: Gibbs.BlockInfo) => + new SimpleBlockSampler(blockInfo) with Cached with GaussianWeight with FactorProduct { val variance = _variance } + + /** + * Computes the joint distribution over the block by iterating through every combination of assignments of variables + * Takes time exponential in the size of the block, and is only recommended for blocks with no determinism + */ + def joint = (blockInfo: Gibbs.BlockInfo) => + new SimpleBlockSampler(blockInfo) with Cached +} + +/** + * Class for handling sampling on a block + * @param blockInfo The pair containing the variables to sample and adjacent factors. + */ +abstract class BlockSampler(val blockInfo: Gibbs.BlockInfo) { + val (block, adjacentFactors) = blockInfo + + /** + * Get the factor from which to sample this block + * Returns a non-logarithmic factor + */ + def getSamplingFactor(currentSamples: MutableMap[Variable[_], Int]): Factor[Double] + + /** + * Sample this block once + */ + def sample(currentSamples: MutableMap[Variable[_], Int]): Unit = { + // Get the joint factor over variables in this block conditioned on the samples + val samplingFactor = getSamplingFactor(currentSamples) + // Sample from the factor + val sample = sampleFactor(samplingFactor) + // Update currentSamples + samplingFactor.variables.zip(sample).foreach(varAndSample => currentSamples(varAndSample._1) = varAndSample._2) + } + + /** + * Select a set of indices in the factor according to the weights in the factor + * Works on a non-logarithmic factor + */ + def sampleFactor(factor: Factor[Double]): List[Int] = { + // This implementation closely mirrors the sampleMultinomial method in figaro.util + val iterator = factor.getIndices.iterator + @tailrec + def helper(index: Double): List[Int] = { + val next = iterator.next + val prob = factor.get(next) + if(index < prob) next + else helper(index - prob) + } + helper(random.nextDouble) + } +} + +class SimpleBlockSampler(blockInfo: Gibbs.BlockInfo) + extends BlockSampler(blockInfo) { + + // Work in log space to prevent underflow + val semiring = LogSumProductSemiring() + + // Indices over possible assignments to variables in the block + val indices = new Indices(block) + + // Maps a variable to its index in the block + val indexMap = block.zipWithIndex.toMap + + // Override e.g. for caching + def getSamplingFactor(currentSamples: MutableMap[Variable[_], Int]): Factor[Double] = computeSamplingFactor(currentSamples) + + def computeSamplingFactor(currentSamples: MutableMap[Variable[_], Int]): Factor[Double] = { + // Sparse to make sampling faster when variables in the block are correlated + val result = new SparseFactor[Double](block, List()) + // Loop through all possible assignments of variables in this block + indices.iterator.foreach(blockIndex => { + // An index corresponds to an assignment of values to each variable + // The probability of selecting this index is the product of the probabilities of selecting this index in each adjacent factor + val prob = adjacentFactors.map(factor => { + // Have to convert block index into an index in the factor + val factorIndex = factor.variables.map(v => if(block.contains(v)) blockIndex(indexMap(v)) else currentSamples(v)) + factor.get(factorIndex) + }).foldLeft(semiring.one)(semiring.product) + if(prob != semiring.zero) result.set(blockIndex, prob) + }) + // Ensure the entries sum to 1 for sampling + normalizeFactor(result) + } + + /** + * Normalize a factor so its weights sum to 1 + * Takes a logarithmic factor and returns a non-logarithmic factor + */ + def normalizeFactor(factor: Factor[Double]): Factor[Double] = { + val normalizingConstant = semiring.sumMany(factor.contents.values) + if(normalizingConstant == semiring.zero) throw new ZeroTotalUnnormalizedProbabilityException + factor.mapTo((d: Double) => math.exp(d - normalizingConstant), SumProductSemiring()) + } +} + +/** + * A VE-like procedure that works well on large but highly sparse blocks. For each adjacent factor, this + * groups the rows of the factor into sub-factors according to possible Markov blanket assignments. Each + * of these sub-factors is accumulated into a factor over factors, which essentially maps the Markov + * blanket of the original factor to a conditional distribution over the block. This does not store any + * new information, but rather takes the information in each factor and stores it in an easier to use + * format. This trait is the reason why we currently choose ''not'' to place a Chain's parent and the + * Chain itself in the same block, since we cannot efficiently compute the product of the sub-factors of + * the adjacent ConditionalSelector factors. In other cases we can make use of a priority queue to + * compute the product in an efficient order. For ConditionalSelectors there is no way to keep the + * intermediate factors sparse, and computing this product takes exponential time. + */ +trait FactorProduct extends SimpleBlockSampler { + lazy val mbLookupFactors = adjacentFactors.map(factor => { + // Separate block and Markov blanket variables + // Markov blanket consists of variables in this factor, but not in the block + // Note how the code below allows duplicate variables in factors + val (blockVarsAndIndices, mbVarsAndIndices) = factor.variables.zipWithIndex.partition(p => block.contains(p._1)) + val (blockVars, blockVarIndices) = blockVarsAndIndices.unzip + val (mbVars, mbVarIndices) = mbVarsAndIndices.unzip + + // Collect the rows of the factor and group by Markov blanket assignments + val mbMap = factor.getIndices.map(index => { + val blockIndex = blockVarIndices.map(index(_)) + val mbIndex = mbVarIndices.map(index(_)) + (mbIndex, blockIndex, factor.get(index)) + }).groupBy(_._1) + + // The result maps a Markov blanket assignment to a sub-factor containing the rows consistent with this assignment + val result = new SparseFactor[SparseFactor[Double]](mbVars, List()) + + mbMap.foreach(pair => { + val (mbIndex, rows) = pair + val subResult = new SparseFactor[Double](blockVars, List(), semiring) + // Fill the sub-factor with the possible rows in this block + rows.foreach(row => { + val (_, blockIndex, prob) = row + // No reason to keep the zero probability rows when factors are sparse + if(prob != semiring.zero) subResult.set(blockIndex, prob) + }) + result.set(mbIndex, subResult) + }) + result + }) + + lazy val ord = new Ordering[Factor[Double]] { + def compare(x: Factor[Double], y: Factor[Double]): Int = { + // Prioritize the factor with the fewest average entries per variable, i.e. most variables per entry + val xSize = math.pow(x.contents.size, 1.0 / x.numVars.toDouble) + val ySize = math.pow(y.contents.size, 1.0 / y.numVars.toDouble) + math.signum(ySize - xSize).toInt + } + } + + override def computeSamplingFactor(currentSamples: MutableMap[Variable[_], Int]): Factor[Double] = { + // Extract the rows from each factor and take their product + val toMultiply: List[Factor[Double]] = mbLookupFactors.map(factor => factor.get(factor.variables.map(currentSamples(_)))) + /* + * This priority queue helps ensure that all intermediate products remain sparse. Even though we know + * the final product should be sparse, it's possible for intermediate factors to become exponentially + * large in the size of the block if multiplied in the wrong order. + */ + val queue = PriorityQueue(toMultiply:_*)(ord) + while(queue.length > 1) { + val product = queue.dequeue().product(queue.dequeue()) + queue.enqueue(product) + } + normalizeFactor(queue.dequeue()) + } +} + +/** + * Caches factors according to assignments of values in the Markov blanket, which avoids recomputing the same factors repeatedly + * Takes advantage of the fact in Gibbs sampling, nearby samples tend to be highly correlated + */ +trait Cached extends SimpleBlockSampler { + lazy val markovBlanket = (adjacentFactors.flatMap(_.variables).toSet -- block.toSet).toList + + // Maps an assignment of values to the Markov blanket to a distribution over variables in the block + // Implemented as a LinkedHashMap so calling cache.head returns the oldest entry + lazy val cache: MutableMap[List[Int], Factor[Double]] = LinkedHashMap() + + // Override to limit the cache size + lazy val maxSize = 1000 + + override def getSamplingFactor(currentSamples: MutableMap[Variable[_], Int]): Factor[Double] = { + val key = markovBlanket.map(currentSamples(_)) + cache.get(key) match { + // Return the cached factor + case Some(factor) => factor + // Otherwise compute the factor and store it, resizing the cache as necessary + case None => { + val factor = computeSamplingFactor(currentSamples) + if(cache.size == maxSize) cache -= cache.head._1 + cache += key -> factor + factor + } + } + } +} + +/** + * Specialized sampling for continuous (i.e. of type Double) elements in Chains. This differs from the + * default sampler in that it can override the zero probability states in ConditionalSelector factors. + * The idea here is that we allow a technically inconsistent state according to the factors when the + * value of the Chain and result element are close. This is needed for compound continuous elements + * because sampling will always produce disjoint ranges for each result element of the Chain, which + * creates determinism issues with the way we currently block variables. This solution has not been + * fully tested for accuracy of results. + */ +trait DoubleWeight extends SimpleBlockSampler { + override val adjacentFactors = blockInfo._2.map(factor => { + factor match { + case _: ConditionalSelector[_] => { + val List(selector, result) = factor.variables + + val newFactor = factor.createFactor(List(selector), List(result)) + factor.getIndices.foreach(index => { + val factorProb = factor.get(index) + if(factorProb != semiring.zero) newFactor.set(index, factorProb) + else { + val List(selectorIndex, resultIndex) = index + val selectorExtended = selector.range(selectorIndex) + val resultExtended = result.range(resultIndex) + + (selectorExtended, resultExtended) match { + // This pattern matching basically just ensures that both the Chain and result are Doubles + case (Regular(List(_, Regular(chainValue: Double))), Regular(resultValue: Double)) => + newFactor.set(index, logWeight(chainValue, resultValue)) + case _ => newFactor.set(index, semiring.zero) + } + } + }) + newFactor + } + case _ => factor + } + }) + + /** + * Function with which to assign weights in place of -Infinity. It is assumed that if + * chainValue == resultValue, the result is 0.0. Observe that setting this function to: + * {{{if(chainValue == resultValue) 0.0 else Double.NegativeInfinity}}} + * has the same effect as not using the DoubleWeight trait at all. + */ + def logWeight(chainValue: Double, resultValue: Double): Double +} + +/** + * Assigns weights to continuous variables based on a Gaussian PDF with a static variance + */ +trait GaussianWeight extends DoubleWeight { + /** + * The static variance used to compute the compatibility function + */ + val variance: Double + + def logWeight(resultValue: Double, chainValue: Double): Double = { + val diff = resultValue - chainValue + // No need to multiply by a constant at the front because we normalize later anyways + // Moreover, we want this to return 0.0 when diff == 0.0 + - (diff * diff) / (2.0 * variance) + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/gibbs/Gibbs.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/gibbs/Gibbs.scala new file mode 100644 index 00000000..c11cf278 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/gibbs/Gibbs.scala @@ -0,0 +1,297 @@ +/* + * Gibbs.scala + * A Gibbs sampler that operates on factor graphs. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: June 3, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.factored.gibbs + +import com.cra.figaro.algorithm.factored._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.{ AlgorithmException, UnsupportedAlgorithmException } +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.language._ +import com.cra.figaro.util._ +import scala.annotation.tailrec +import scala.collection.mutable.{ Map => MutableMap } +import com.cra.figaro.algorithm.factored.factors.factory.Factory + +trait Gibbs[T] extends BaseUnweightedSampler with FactoredAlgorithm[T] { + /** + * The universe in which this Gibbs sampler is to be applied. + */ + val universe: Universe + + /** + * Semiring for use in factors. + */ + val semiring: Semiring[T] + + /** + * Elements whose samples will be recorded at each iteration. + */ + val targetElements: List[Element[_]] + + /** + * List of all factors. + */ + var factors: List[Factor[T]] = _ + + /** + * Variables to sample at each time step. + */ + var variables: Set[Variable[_]] = _ + + /** + * The most recent set of samples, used for sampling variables conditioned on the values of other variables. + */ + val currentSamples: MutableMap[Variable[_], Int] = MutableMap() + + /** + * Number of samples to throw away initially. + */ + def burnIn(): Int + + /** + * Iterations thrown away between samples. + */ + def interval(): Int + + /** + * Method to create a blocking scheme given information about the model and factors. + */ + def createBlocks(): List[Gibbs.Block] +} + +/** + * The default trait for creating blocks. Works on Chain, Apply, Atomic, and Constant elements. + * Note that our choice of blocking scheme fails when the results of a Chain have disjoint ranges. + * We are forced to do this to prevent exponentially slow time performance on models with Chains. + * A more detailed explanation is available in the FactorProduct trait in BlockSampler.scala. + */ +trait ChainApplyBlockingGibbs extends Gibbs[Double] { + // Maps a variable to its deterministic parents + def variableParentMap(): Map[Variable[_], Set[Variable[_]]] = variables.map(_ match { + case ev: ElementVariable[_] => ev.element match { + // For Chain, we treat all of the result variables as deterministic parents + case c: Chain[_, _] => { + /* + * Must ensure this code is not called in SFI! + */ + val chainResults: Set[Variable[_]] = LazyValues(universe).getMap(c).values.map(Variable(_)).toSet + (ev, chainResults) + } + // For Apply, we take the deterministic parents to be its arguments + case a: Apply[_] => (ev, a.args.map(Variable(_)).toSet) + case a: Atomic[_] => (ev, Set[Variable[_]]()) + case c: Constant[_] => (ev, Set[Variable[_]]()) + case _ => throw new UnsupportedAlgorithmException(ev.element) + } + + // This handles internal variables in Chains + // We treat all of the result variables, as well as the parent variable, as deterministic parents + case icv: InternalChainVariable[_] => { + val chain = icv.chain.asInstanceOf[Chain[_, _]] + /* + * Must ensure this code is not called in SFI! + */ + val chainResults: Set[Variable[_]] = LazyValues(universe).getMap(chain).values.map(Variable(_)).toSet + (icv, chainResults + Variable(chain.parent)) + } + + // Otherwise, we assume (perhaps incorrectly) that the variable is stochastic + // It will not be blocked with any parents + case v => (v, Set[Variable[_]]()) + }).toMap + + def createBlocks(): List[Gibbs.Block] = { + val variableParents = variableParentMap() + // Maps each variable to its deterministic children, i.e. variables that should be included in a block with this variable + val variableChildren: Map[Variable[_], Set[Variable[_]]] = + variables.map(v => v -> variables.filter(variableParents(_).contains(v))).toMap + + // Start with the purely stochastic variables with no parents + val starterVariables = variables.filter(variableParents(_).isEmpty) + + @tailrec // Recursively add deterministic children to the block + def expandBlock(expand: Set[Variable[_]], block: Set[Variable[_]] = Set()): Gibbs.Block = { + if (expand.isEmpty) block.toList + else { + val expandNext = expand.flatMap(variableChildren(_)) + expandBlock(expandNext, block ++ expand) + } + } + starterVariables.map(v => expandBlock(Set(v))).toList + } +} + +trait ProbabilisticGibbs extends Gibbs[Double] { + class StarSampleException(elem: Element[_]) extends AlgorithmException + + val semiring = LogSumProductSemiring() + + protected var blockSamplers: List[BlockSampler] = _ + + def getFactors(neededElements: List[Element[_]], targetElements: List[Element[_]], upperBounds: Boolean = false): List[Factor[Double]] = { + val thisUniverseFactors = neededElements flatMap(Factory.makeFactorsForElement(_)) + val dependentUniverseFactors = + for { (dependentUniverse, evidence) <- dependentUniverses } yield Factory.makeDependentFactor(Variable.cc, universe, dependentUniverse, dependentAlgorithm(dependentUniverse, evidence)) + // Make logarithmic to prevent underflow in product computations + (dependentUniverseFactors ::: thisUniverseFactors).map(_.mapTo(Math.log, LogSumProductSemiring())) + } + + // Update the map of samples one block at a time, conditioned on intermediate samples of all the other variables + // i.e. perform a single step of blocked Gibbs sampling + def sampleAllBlocks(): Unit = { + blockSamplers.foreach(_.sample(currentSamples)) + } + + // Perform an iteration of sampling and record the results + def sample(): (Boolean, Sample) = { + sampleAllBlocks() + val result = targetElements.map(e => { + val variable = Variable(e) + val extended = variable.range(currentSamples(variable)) + // Accept the sample unless it is star + if (extended.isRegular) (e, extended.value) + else throw new StarSampleException(e) + }) + (true, MutableMap(result: _*)) + } + + protected override def doSample() = { + // This takes care of the thinning defined by interval + for (i <- 1 until interval) { + sampleAllBlocks() + } + super.doSample() + } +} + +abstract class ProbQueryGibbs(override val universe: Universe, targets: Element[_]*)( + val dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + val dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + val burnIn: Int, val interval: Int, + val blockToSampler: Gibbs.BlockSamplerCreator, upperBounds: Boolean = false) + extends BaseUnweightedSampler(universe, targets: _*) + with ProbabilisticGibbs with UnweightedSampler { + + val targetElements = targets.toList + + override def initialize() = { + super.initialize() + // Get the needed elements, factors, and blocks + val neededElements = getNeededElements(targetElements, Int.MaxValue)._1 + factors = getFactors(neededElements, targetElements, upperBounds) + variables = factors.flatMap(_.variables).toSet + val blocks = createBlocks() + // Create block samplers + blockSamplers = blocks.map(block => blockToSampler((block, factors.filter(_.variables.exists(block.contains(_)))))) + // Initialize the samples to a valid state and take the burn-in samples + val initialSample = WalkSAT(factors, variables, semiring, chainMapper) + variables.foreach(v => currentSamples(v) = initialSample(v)) + for (_ <- 1 to burnIn) sampleAllBlocks() + } + + def chainMapper(chain: Chain[_,_]): Set[Variable[_]] = LazyValues(chain.universe).getMap(chain).values.map(Variable(_)).toSet +} + +object Gibbs { + // A block is just a list of variables + type Block = List[Variable[_]] + + // Information passed to BlockSampler constructor + type BlockInfo = (Block, List[Factor[Double]]) + + // Constructor for BlockSampler + type BlockSamplerCreator = BlockInfo => BlockSampler + + /** + * Create a one-time Gibbs sampler using the given number of samples and target elements. + */ + def apply(mySamples: Int, targets: Element[_]*)(implicit universe: Universe) = + new ProbQueryGibbs(universe, targets: _*)( + List(), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + 0, 1, BlockSampler.default) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs { val numSamples = mySamples } + + /** + * Create a one-time Gibbs sampler using the given number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + new ProbQueryGibbs(universe, targets: _*)( + List(), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs { val numSamples = mySamples } + + /** + * Create a one-time Gibbs sampler using the given dependent universes and algorithm, + * the number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + new ProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs { val numSamples = mySamples } + + /** + * Create an anytime Gibbs sampler using the given target elements. + */ + def apply(targets: Element[_]*)(implicit universe: Universe) = + new ProbQueryGibbs(universe, targets: _*)( + List(), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + 0, 1, BlockSampler.default) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + + /** + * Create an anytime Gibbs sampler using the given number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + new ProbQueryGibbs(universe, targets: _*)( + List(), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + + /** + * Create an anytime Gibbs sampler using the given dependent universes and algorithm, + * the number of samples to burn in, the sampling interval, + * the BlockSampler generator, and target elements. + */ + def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + new ProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + + /** + * Use Gibbs sampling to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = Gibbs(10000, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use Gibbs sampling to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/gibbs/WalkSAT.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/gibbs/WalkSAT.scala new file mode 100644 index 00000000..760b5ac0 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/gibbs/WalkSAT.scala @@ -0,0 +1,137 @@ +/* + * WalkSAT.scala + * A WalkSAT-like procedure for finding a valid state on a set of factors. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: July 20, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.factored.gibbs + +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.util._ +import com.cra.figaro.language._ +import scala.annotation.tailrec +import scala.collection.mutable.{Map => MutableMap} +import com.cra.figaro.algorithm.factored.factors._ +import scala.collection.mutable.{Map => MutableMap} + +class StateNotFoundException extends RuntimeException + +object WalkSAT { + /** + * Produce an assignment of values to variables constrained by the given factors. + * @param factors The factors with which to constrain the variables. + * @param variables The set of variables over which to compute a sample. + * @param semiring The semiring used in the factors. + * @param prob The probability of reassigning a random variable in a factor instead of taking the greedy approach. + * @param maxIterations Maximum number of iterations to run before throwing an exception for taking too long. + */ + def apply[T](factors: List[Factor[T]], variables: Set[Variable[_]], semiring: Semiring[T], chainMapper: Chain[_,_] => Set[Variable[_]], + prob: Double = 0.1, maxIterations: Int = 100000): MutableMap[Variable[_], Int] = { + val currentSamples = MutableMap[Variable[_], Int]() + val nonConstraintFactors = factors.filterNot(_.isConstraint) + val variableParents = MutableMap[Variable[_], Set[Variable[_]]]() + + // Compute the set of parents of each variable that determine its value in generative order + // Similar but not identical to variableParentMap used for Gibbs sampling + variables.foreach(_ match { + case ev: ElementVariable[_] => ev.element match { + case a: Apply[_] => variableParents(ev) = a.args.map(Variable(_)).toSet + case _ => + } + + case icv: InternalChainVariable[_] => { + val chain = icv.chain.asInstanceOf[Chain[_, _]] + val chainResults: Set[Variable[_]] = chainMapper(chain) + //val chainResults: Set[Variable[_]] = LazyValues(chain.universe).getMap(chain).values.map(Variable(_)).toSet + variableParents(icv) = chainResults + Variable(chain.parent) + variableParents(icv.chainVar) = Set(icv) + } + case _ => + }) + + pseudoForwardSample(variables) + walkSAT(maxIterations) + + @tailrec + def pseudoForwardSample(toSample: Set[Variable[_]]): Unit = { + // Look for a variable who has no parents yet to be sampled + val variableOption = toSample.find(variable => variableParents.getOrElse(variable, Set()).intersect(toSample).isEmpty) + variableOption match { + case Some((variableToSample)) => { + // Get the adjacent factors + val adjacentFactors = nonConstraintFactors.filter(f => f.variables.contains(variableToSample)) + val varAndParents = variableParents.getOrElse(variableToSample, Set()) + variableToSample + // Marginalize to the variable and its parents + val parentFactors = adjacentFactors.map(_.marginalizeTo(varAndParents.toList:_*)) + // Produce a sample + val sampleOption = (0 until variableToSample.size).find(sample => parentFactors.forall(factor => { + factor.get(factor.variables.map(currentSamples.getOrElse(_, sample))) != semiring.zero + })) + // Update and repeat until all variables are sampled + currentSamples(variableToSample) = sampleOption.getOrElse(0) + pseudoForwardSample(toSample - variableToSample) + } + case _ => { + // In this case toSample is either empty, or we are in an invalid state + // If the latter we have to set the remaining variables to something for WalkSAT + toSample.foreach(currentSamples(_) = 0) + } + } + } + + @tailrec + def walkSAT(iterations: Int): Unit = { + // Throw an exception when the maximum allowed iterations have been performed + if(iterations == 0) throw new StateNotFoundException + // Lists of satisfied and dissatisfied factors by the current samples + val (satisfied, dissatisfied) = factors.partition(f => { + val indices = f.variables.map(currentSamples(_)) + f.get(indices) != semiring.zero + }) + + // Stop when all factors are satisfied + if(!dissatisfied.isEmpty) { + // Pick a random dissatisfied factor + val factor = dissatisfied(random.nextInt(dissatisfied.length)) + + // With some probability, pick a random variable and sample uniformly from its range + if(random.nextDouble() < prob) { + val variable = factor.variables(random.nextInt(factor.numVars)) + val sample = random.nextInt(variable.size) + // Update currentSamples + currentSamples(variable) = sample + } + // Otherwise, change the variable with the corresponding sample + // that leaves the fewest dissatisfied factors that were previously satisfied + else { + val (variable, sample, _) = factor.variables.flatMap(v => { + (0 until v.size).map(index => { + // This line prevents picking the same sample again + if(index == currentSamples(v)) (v, index, Int.MaxValue) + else { + // Count the number of previously satisfied factors that become dissatisfied + val count = satisfied.count(f => { + val indices = f.variables.map(vf => if(vf == v) index else currentSamples(vf)) + f.get(indices) == semiring.zero + }) + (v, index, count) + } + }) + }).minBy(_._3) + // Update currentSamples + currentSamples(variable) = sample + } + walkSAT(iterations - 1) + } + } + + currentSamples + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/FactoredFrontier.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/FactoredFrontier.scala index 43e32fde..5bac78fe 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/FactoredFrontier.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/FactoredFrontier.scala @@ -18,7 +18,7 @@ import com.cra.figaro.algorithm.lazyfactored.LazyValues import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler import com.cra.figaro.algorithm.{ OneTimeProbQuery, AnytimeProbQuery } import com.cra.figaro.language._ -import com.cra.figaro.algorithm.factored.factors.Factory +import com.cra.figaro.algorithm.factored.factors.Variable /** * Abstract class that runs the Factored Frontier algorithm. @@ -92,7 +92,7 @@ abstract class FactoredFrontier(static: Universe, initial: Universe, transition: * We must explicitly add all named elements from the two dummy universes, as FactoredAlgorithm cannot get them by default. * This is to ensure that they are correctly expanded and included for factor creation. */ - Factory.removeFactors + Variable.clearCache createBP(getNamedElements(currentUniverse) ::: getNamedElements(currentStatic) ::: getNamedElements(dummyUniverse), dependentUniverse, dependentAlgorithm) runBP() diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParFiltering.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParFiltering.scala new file mode 100644 index 00000000..abb3c692 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParFiltering.scala @@ -0,0 +1,78 @@ +/* + * ParFiltering.scala + * A parallel version of Filtering. + * + * Created By: Lee Kellogg (lkellogg@cra.com) + * Creation Date: Jun 2, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.filtering + +import com.cra.figaro.algorithm._ +import com.cra.figaro.language._ + +/** + * A parallel version of Filtering. Specifically a version of OneTimeFiltering, since that's the only + * target for parallelization right now. + */ +abstract class ParFiltering(transition: (Universe, Universe) => Universe) extends Algorithm with OneTime { + + /** + * Returns the probability that the element referred to by the reference + * produces the given value at the current time point. + */ + def currentProbability[T](reference: Reference[T], value: T): Double = + currentProbability(reference, (t: T) => t == value) + + /* One-time implementations */ + + /** + * Returns the distribution over the element referred to by the reference at the current time point. + */ + def currentDistribution[T](reference: Reference[T]): Stream[(Double, T)] = + computeCurrentDistribution(reference) + + /** + * Returns the expectation of the element referred to by the reference + * under the given function at the current time point. + */ + def currentExpectation[T](reference: Reference[T], function: T => Double): Double = + computeCurrentExpectation(reference, function) + + /** + * Returns the probability that the element referred to by the reference + * satisfies the given predicate at the current time point. + */ + def currentProbability[T](reference: Reference[T], predicate: T => Boolean): Double = + computeCurrentProbability(reference, predicate) + + /* The following four methods must be defined by any implementation of filtering. */ + + /** + * Advance the filtering one time step, conditioning on the given evidence at the new time point. + */ + def advanceTime(evidence: Seq[NamedEvidence[_]]): Unit + + /** + * Returns the distribution over the element referred to by the reference at the current time point. + */ + protected def computeCurrentDistribution[T](reference: Reference[T]): Stream[(Double, T)] + + /** + * Returns the expectation of the element referred to by the reference + * under the given function at the current time point. + */ + protected def computeCurrentExpectation[T](reference: Reference[T], function: T => Double): Double + + /** + * Returns the probability that the element referred to by the reference + * satisfies the given predicate at the current time point. + */ + protected def computeCurrentProbability[T](reference: Reference[T], predicate: T => Boolean): Double = + computeCurrentExpectation(reference, (t: T) => if (predicate(t)) 1.0; else 0.0) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParParticleFilter.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParParticleFilter.scala new file mode 100644 index 00000000..1bad2601 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParParticleFilter.scala @@ -0,0 +1,140 @@ +/* + * ParParticleFilter.scala + * A parallel one-time particle filter. + * + * Created By: Lee Kellogg (lkellogg@cra.com) + * Creation Date: Jun 2, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.filtering + +import com.cra.figaro.language._ +import scala.collection.parallel.ParSeq +import com.cra.figaro.algorithm.filtering.ParticleFilter.WeightedParticle +import com.cra.figaro.library.cache.PermanentCache +import com.cra.figaro.library.cache.Cache +import com.cra.figaro.algorithm.sampling.LikelihoodWeighter + +/** + * A parallel one-time particle filter. Distributes the work of generating particles at each time step over a specified + * number of threads. After generating the particles, they are recombined before re-sampling occurs. Instead of accepting + * initial and static universes as input, this method accepts functions that return universes. This is because each + * thread needs its own set of universes to work on. It is important that any elements created within those functions are + * explicitly assigned to the returned universe, not the implicit default universe. + * + * @param static A function that returns a universe of elements whose values do not change over time + * @param initial A function that returns a universe describing the distribution over the initial state of the system + * @param transition The transition model describing how the current state of the system depends on the static and previous, respectively + * @param numParticles Number of particles to use at each time step + * @param numThreads The number of threads over which to distribute the work of generating the particles at each step + */ +class ParOneTimeParticleFilter(static: () => Universe, initial: () => Universe, transition: (Universe, Universe) => Universe, val numParticles: Int, numThreads: Int) + extends ParFiltering(transition) with ParticleFilter { + + /** sequence of UniverseWindows -- one for each thread */ + private var windows: Seq[UniverseWindow] = _ + + /** (start, end) indices for each thread to divide up numParticles */ + private val indices = calculateIndices(numParticles, numThreads) + + /** generate the initial UniverseWindows */ + private def genInitialWindows(): Seq[UniverseWindow] = { + Seq.fill(numThreads)(new UniverseWindow(null, initial(), static())) + } + + /** apply the transition function to each of a sequence of UniverseWindows */ + private def advanceUniverseWindows(windows: Seq[UniverseWindow]): Seq[UniverseWindow] = { + windows map { w => advanceUniverse(w, transition) } + } + + /** + * generate particles for each thread, in parallel, then recombine and return + * + * @param windows the UniverseWindows to sample from + * @param weightedParticleCreator a function that generates a WeightedParticle, given a UniverseWindow and an index + */ + private def genParticles(windows: Seq[(UniverseWindow, LikelihoodWeighter)], weightedParticleCreator: ((UniverseWindow, LikelihoodWeighter), Int) => WeightedParticle): Seq[WeightedParticle] = { + val parWindows = windows.par + val particles = parWindows zip indices flatMap { case(window, (start, end)) => + (start to end) map { i => weightedParticleCreator(window, i) } + } + particles.seq + } + + /** compute probability of evidence for the particles, and update the belief state (after re-sampling) */ + private def doTimeStep(weightedParticles: Seq[WeightedParticle]) { + computeProbEvidence(weightedParticles) + updateBeliefState(weightedParticles) + } + + def run(): Unit = { + windows = genInitialWindows() + val windowWithCaches = windows.map(w => (w, new LikelihoodWeighter(w.current, new PermanentCache(w.current)))) + val particles = genParticles(windowWithCaches, (w, _) => initialWeightedParticle(w._1.static, w._1.current, w._2)) + doTimeStep(particles) + } + + def advanceTime(evidence: Seq[NamedEvidence[_]] = List()): Unit = { + val newWindows = advanceUniverseWindows(windows) + val newWindowsWithCaches = newWindows.map(w => (w, new LikelihoodWeighter(w.current, new PermanentCache(w.current)))) + val particles = genParticles(newWindowsWithCaches, (w, i) => addWeightedParticle(evidence, i, w._1, w._2)) + doTimeStep(particles) + windows = newWindows + } + + /** + * Calculate start and end indices for each thread dividing up the particles + */ + private def calculateIndices(numParticles: Int, numThreads: Int): Seq[(Int, Int)] = { + val indices = (1 to numThreads) map { i => + val start = (i - 1) * numParticles / numThreads + val end = i * numParticles / numThreads -1 + (start, end) + } + indices + } +} + +/** + * A parallel implementation of a OneTimeParticleFilter. + */ +object ParParticleFilter { + + /** + * A parallel one-time particle filter. Distributes the work of generating particles at each time step over a specified + * number of threads. After generating the particles, they are recombined before re-sampling occurs. Instead of accepting + * initial and static universes as input, this method accepts functions that return universes. This is because each + * thread needs its own set of universes to work on. It is important that any elements created within those functions are + * explicitly assigned to the returned universe, not the implicit default universe. + * + * + * @param static A function that returns a universe of elements whose values do not change over time + * @param initial A function that returns a universe describing the distribution over the initial state of the system + * @param transition The transition model describing how the current state of the system depends on the static and previous, respectively + * @param numParticles Number of particles to use at each time step + * @param numThreads The number of threads over which to distribute the work of generating the particles at each step + */ + def apply(static: () => Universe, initial: () => Universe, transition: (Universe, Universe) => Universe, numParticles: Int, numThreads: Int): ParOneTimeParticleFilter = + new ParOneTimeParticleFilter(static, initial, transition, numParticles, numThreads) + + /** + * A parallel one-time particle filter. Distributes the work of generating particles at each time step over a specified + * number of threads. After generating the particles, they are recombined before re-sampling occurs. Instead of accepting + * an initial universe as input, this method accepts a function that returns a universe. This is because each thread needs + * its own set of universes to work on. It is important that any elements created within that function are explicitly + * assigned to the returned universe, not the implicit default universe. + * + * @param initial A function that returns a universe describing the distribution over the initial state of the system + * @param transition The transition model describing how the current state of the system depends on the previous + * @param numParticles Number of particles to use at each time step + * @param numThreads The number of threads over which to distribute the work of generating the particles at each step + */ + def apply(initial: () => Universe, transition: Universe => Universe, numParticles: Int, numThreads: Int): ParOneTimeParticleFilter = + apply(() => new Universe(), initial, (static: Universe, previous: Universe) => transition(previous), numParticles, numThreads) + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParticleFilter.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParticleFilter.scala index 7d0a81d8..e7438d36 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParticleFilter.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/filtering/ParticleFilter.scala @@ -16,7 +16,9 @@ package com.cra.figaro.algorithm.filtering import com.cra.figaro.algorithm.sampling._ import com.cra.figaro.language._ import com.cra.figaro.util._ -import sun.swing.AccumulativeRunnable +import com.cra.figaro.library.cache.PermanentCache +import com.cra.figaro.library.cache.Cache +import com.cra.figaro.algorithm.sampling.LikelihoodWeighter /** * An abstract class of particle filters. @@ -35,15 +37,14 @@ import sun.swing.AccumulativeRunnable * @param intitial The universe describing the initial distribution of the model. * @param transition The transition function that returns a new universe from a static and previous universe, respectively. */ -abstract class ParticleFilter(static: Universe = new Universe(), initial: Universe, transition: (Universe, Universe) => Universe, numParticles: Int) - extends Filtering(static, initial, transition) { +trait ParticleFilter { + + val numParticles: Int /** The belief about the state of the system at the current point in time. */ val beliefState: ParticleFilter.BeliefState = Array.fill(numParticles)(null) protected var logProbEvidence: Double = 0.0 - protected var previousUniverse: Universe = _ - protected var currentUniverse = initial /** * Returns the expectation of the element referred to by the reference @@ -75,29 +76,21 @@ abstract class ParticleFilter(static: Universe = new Universe(), initial: Univer /* * Careful: makeWeightedParticle overwrites the previous state with the new state. That means we can't use it to generate another new particle from the same previous * state. The reason for this design is to avoid creating new snapshots and states to conserve memory. + + * TODO: previous state could be replaced by the static universe (or a universe window) */ - protected def makeWeightedParticle(previousState: State): ParticleFilter.WeightedParticle = { - Forward(currentUniverse) - // avoiding recursion - var satisfied = true - var conditionedElementsRemaining = currentUniverse.conditionedElements - while (!conditionedElementsRemaining.isEmpty) { - satisfied &= conditionedElementsRemaining.head.conditionSatisfied - conditionedElementsRemaining = conditionedElementsRemaining.tail - } - val weight = - if (satisfied) { - var w = 1.0 - var constrainedElementsRemaining = currentUniverse.constrainedElements - while (!constrainedElementsRemaining.isEmpty) { - w *= math.exp(constrainedElementsRemaining.head.constraintValue) - constrainedElementsRemaining = constrainedElementsRemaining.tail - } - w - } else 0.0 + protected def makeWeightedParticle(previousState: State, currentUniverse: Universe, lw: LikelihoodWeighter): ParticleFilter.WeightedParticle = { + + val weight = try { + math.exp(lw.computeWeight(currentUniverse.activeElements)) + } catch { + case Importance.Reject => 0.0 + } + val snapshot = new Snapshot snapshot.store(currentUniverse) val state = new State(snapshot, previousState.static) + currentUniverse.clearTemporaries (weight, state) } @@ -124,36 +117,38 @@ abstract class ParticleFilter(static: Universe = new Universe(), initial: Univer logProbEvidence = logProbEvidence + scala.math.log(sum / numParticles) } - protected def addWeightedParticle(evidence: Seq[NamedEvidence[_]], index: Int): ParticleFilter.WeightedParticle = { + protected def addWeightedParticle(evidence: Seq[NamedEvidence[_]], index: Int, universes: UniverseWindow, lw: LikelihoodWeighter): ParticleFilter.WeightedParticle = { val previousState = beliefState(index) - previousState.dynamic.restore(previousUniverse) - previousState.static.restore(static) - currentUniverse.assertEvidence(evidence) - val result = makeWeightedParticle(previousState) + previousState.dynamic.restore(universes.previous) + previousState.static.restore(universes.static) + universes.current.assertEvidence(evidence) + val result = makeWeightedParticle(previousState, universes.current, lw) result } - protected def initialWeightedParticle(): ParticleFilter.WeightedParticle = { + protected def initialWeightedParticle(static: Universe, current: Universe, lw: LikelihoodWeighter): ParticleFilter.WeightedParticle = { Forward(static) val staticSnapshot = new Snapshot staticSnapshot.store(static) val state = new State(new Snapshot, staticSnapshot) - makeWeightedParticle(state) + makeWeightedParticle(state, current, lw) } /* * Advance the universe one time step. * The previous universe becomes a copy of the current universe with all named elements replaced by constants. * This is done so we don't have to store the previous universe (and the universes previous to it), and we can release the memory. + * + * Returns a tuple: (previous, current) */ - protected def advanceUniverse() { + protected def advanceUniverse(universes: UniverseWindow, transition: (Universe, Universe) => Universe): UniverseWindow = { - previousUniverse = Universe.createNew() - for { element <- currentUniverse.activeElements.filter(!_.name.isEmpty) } { - new Settable(element.name.string, element.value, previousUniverse) + val newPrevious = Universe.createNew() + for { element <- universes.current.activeElements.filter(!_.name.isEmpty) } { + new Settable(element.name.string, element.value, newPrevious) } - currentUniverse = transition(static, previousUniverse) - + val newCurrent = transition(universes.static, newPrevious) + new UniverseWindow(newPrevious, newCurrent, universes.static) } /** @@ -163,7 +158,7 @@ abstract class ParticleFilter(static: Universe = new Universe(), initial: Univer logProbEvidence } - /** + /** * The computed probability of evidence. */ def probEvidence(): Double = { @@ -181,8 +176,12 @@ abstract class ParticleFilter(static: Universe = new Universe(), initial: Univer * @param transition The transition model describing how the current state of the system depends on the previous * @param numParticles The number of particles to use at each time step */ -class OneTimeParticleFilter(static: Universe = new Universe(), initial: Universe, transition: (Universe, Universe) => Universe, numParticles: Int) - extends ParticleFilter(static, initial, transition, numParticles) with OneTimeFiltering { +class OneTimeParticleFilter(static: Universe = new Universe(), initial: Universe, transition: (Universe, Universe) => Universe, val numParticles: Int) + extends Filtering(static, initial, transition) with ParticleFilter with OneTimeFiltering { + + var currentUniverse: Universe = initial + var previousUniverse: Universe = _ + private def doTimeStep(weightedParticleCreator: Int => ParticleFilter.WeightedParticle) { val weightedParticles = for { i <- 0 until numParticles } yield weightedParticleCreator(i) @@ -196,7 +195,9 @@ class OneTimeParticleFilter(static: Universe = new Universe(), initial: Universe * Begin the particle filter, determining the initial distribution. */ def run(): Unit = { - doTimeStep((i: Int) => initialWeightedParticle()) + val lw = new LikelihoodWeighter(currentUniverse, new PermanentCache(currentUniverse)) + doTimeStep((i: Int) => initialWeightedParticle(static, currentUniverse, lw)) + lw.deregisterDependencies() } /** @@ -204,9 +205,13 @@ class OneTimeParticleFilter(static: Universe = new Universe(), initial: Universe */ def advanceTime(evidence: Seq[NamedEvidence[_]] = List()): Unit = { - advanceUniverse() - doTimeStep((i: Int) => addWeightedParticle(evidence, i)) - + val currentWindow = new UniverseWindow(previousUniverse, currentUniverse, static) + val newWindow = advanceUniverse(currentWindow, transition) + val lw = new LikelihoodWeighter(newWindow.current, new PermanentCache(newWindow.current)) + doTimeStep((i: Int) => addWeightedParticle(evidence, i, newWindow, lw)) + previousUniverse = newWindow.previous + currentUniverse = newWindow.current + lw.deregisterDependencies() } } @@ -255,4 +260,13 @@ object ParticleFilter { /** Weighted particles, consisting of a weight and a state. */ type WeightedParticle = (Double, State) + /** Reference to parallel implementation. */ + def par = ParParticleFilter + } + +/** + * A class representing a single window in time, with a current universe, a previous universe, + * and a static universe. + */ +class UniverseWindow(val previous: Universe, val current: Universe, val static: Universe) diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/BoundedProbFactor.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/BoundedProbFactor.scala index 621320bb..9872eeac 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/BoundedProbFactor.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/BoundedProbFactor.scala @@ -19,7 +19,7 @@ import com.cra.figaro.util._ import annotation.tailrec import scala.language.existentials import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.factored.factors.Factory +import com.cra.figaro.algorithm.lazyfactored.factory.Factory /** * Methods for creating lower and upper bound probability factors. diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyValues.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyValues.scala index 4702a1e0..ecb87a9a 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyValues.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyValues.scala @@ -33,7 +33,7 @@ import com.cra.figaro.algorithm.factored.ParticleGenerator * Object for lazily computing the range of values of elements in a universe. Given a universe, you can compute the values * of elements in the universe to any desired depth. */ -class LazyValues(universe: Universe) { +class LazyValues(universe: Universe, paramaterized: Boolean = false) { private def values[T](element: Element[T], depth: Int, numArgSamples: Int, numTotalSamples: Int): ValueSet[T] = { // In some cases (e.g. CompoundFlip), we might know the value of an element without getting the values of its arguments. // However, future algorithms rely on the values of the arguments having been gotten already. @@ -42,8 +42,8 @@ class LazyValues(universe: Universe) { // Override the argument list for chains since the resultElement of the chain will be processed as the chain is processed val elementArgs = element match { - case c: Chain[_,_] => List(c.parent) - case _ => element.args + case c: Chain[_, _] => List(c.parent) + case _ => element.args } for { arg <- elementArgs } { @@ -52,22 +52,23 @@ class LazyValues(universe: Universe) { } Abstraction.fromPragmas(element.pragmas) match { - case None => concreteValues(element, depth, numArgSamples, numTotalSamples) + case None => concreteValues(element, depth, numArgSamples, numTotalSamples) case Some(abstraction) => abstractValues(element, abstraction, depth, numArgSamples, numTotalSamples) } } private def concreteValues[T](element: Element[T], depth: Int, numArgSamples: Int, numTotalSamples: Int): ValueSet[T] = - element match { - case c: Constant[_] => withoutStar(Set(c.constant)) - case f: Flip => withoutStar(Set(true, false)) - case d: Select[_, _] => withoutStar(Set(d.outcomes: _*)) - case d: Dist[_, _] => + (element, paramaterized) match { + case (p: Parameter[_], true) => ValueSet.withoutStar(Set(p.MAPValue)) + case (c: Constant[_], _) => withoutStar(Set(c.constant)) + case (f: Flip, _) => withoutStar(Set(true, false)) + case (d: Select[_, _], _) => withoutStar(Set(d.outcomes: _*)) + case (d: Dist[_, _], _) => val componentSets = d.outcomes.map(storedValues(_)) componentSets.reduce(_ ++ _) //case i: FastIf[_] => withoutStar(Set(i.thn, i.els)) - case a: Apply1[_, _] => + case (a: Apply1[_, _], _) => val applyMap = getMap(a) val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) val resultsSet = @@ -77,7 +78,7 @@ class LazyValues(universe: Universe) { getOrElseInsert(applyMap, arg1Val, a.fn(arg1Val)) } if (vs1.hasStar) withStar(resultsSet); else withoutStar(resultsSet) - case a: Apply2[_, _, _] => + case (a: Apply2[_, _, _], _) => val applyMap = getMap(a) val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) @@ -92,7 +93,7 @@ class LazyValues(universe: Universe) { getOrElseInsert(applyMap, (arg1Val, arg2Val), a.fn(arg1Val, arg2Val)) } if (vs1.hasStar || vs2.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) - case a: Apply3[_, _, _, _] => + case (a: Apply3[_, _, _, _], _) => val applyMap = getMap(a) val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) @@ -109,7 +110,7 @@ class LazyValues(universe: Universe) { getOrElseInsert(applyMap, (arg1Val, arg2Val, arg3Val), a.fn(arg1Val, arg2Val, arg3Val)) } if (vs1.hasStar || vs2.hasStar || vs3.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) - case a: Apply4[_, _, _, _, _] => + case (a: Apply4[_, _, _, _, _], _) => val applyMap = getMap(a) val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) @@ -128,7 +129,7 @@ class LazyValues(universe: Universe) { getOrElseInsert(applyMap, (arg1Val, arg2Val, arg3Val, arg4Val), a.fn(arg1Val, arg2Val, arg3Val, arg4Val)) } if (vs1.hasStar || vs2.hasStar || vs3.hasStar || vs4.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) - case a: Apply5[_, _, _, _, _, _] => + case (a: Apply5[_, _, _, _, _, _], _) => val applyMap = getMap(a) val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) @@ -149,7 +150,7 @@ class LazyValues(universe: Universe) { getOrElseInsert(applyMap, (arg1Val, arg2Val, arg3Val, arg4Val, arg5Val), a.fn(arg1Val, arg2Val, arg3Val, arg4Val, arg5Val)) } if (vs1.hasStar || vs2.hasStar || vs3.hasStar || vs4.hasStar || vs5.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) - case c: Chain[_, _] => + case (c: Chain[_, _], _) => def findChainValues[T, U](chain: Chain[T, U], cmap: Map[T, Element[U]], pVals: ValueSet[T], samples: Int): Set[ValueSet[U]] = { val chainVals = pVals.regularValues.map { parentVal => @@ -168,28 +169,25 @@ class LazyValues(universe: Universe) { val chainMap = getMap(c) val parentVS = LazyValues(c.parent.universe).storedValues(c.parent) - val samplesPerValue = math.max(1, (numTotalSamples.toDouble/parentVS.regularValues.size).toInt) + val samplesPerValue = math.max(1, (numTotalSamples.toDouble / parentVS.regularValues.size).toInt) val resultVSs = findChainValues(c, chainMap, parentVS, samplesPerValue) val startVS: ValueSet[c.Value] = if (parentVS.hasStar) withStar[c.Value](Set()); else withoutStar[c.Value](Set()) resultVSs.foldLeft(startVS)(_ ++ _) - case i: Inject[_] => + case (i: Inject[_], _) => val elementVSs = i.args.map(arg => LazyValues(arg.universe).storedValues(arg)) val incomplete = elementVSs.exists(_.hasStar) val elementValues = elementVSs.toList.map(_.regularValues.toList) val resultValues = homogeneousCartesianProduct(elementValues: _*).toSet.asInstanceOf[Set[i.Value]] if (incomplete) withStar(resultValues); else withoutStar(resultValues) - case v: ValuesMaker[_] => { + case (v: ValuesMaker[_], _) => { v.makeValues(depth) } - case a: Atomic[_] => { - if (!ParticleGenerator.exists(universe)) { - println("Warning: Sampling element " + a + " even though no sampler defined for this universe") - } + case (a: Atomic[_], _) => { val thisSampler = ParticleGenerator(universe) - val samples = thisSampler(a, numArgSamples) + val samples = thisSampler(a, numArgSamples) withoutStar(samples.unzip._2.toSet) } case _ => @@ -198,7 +196,7 @@ class LazyValues(universe: Universe) { } private def abstractValues[T](element: Element[T], abstraction: Abstraction[T], depth: Int, - numArgSamples: Int, numTotalSamples: Int): ValueSet[T] = { + numArgSamples: Int, numTotalSamples: Int): ValueSet[T] = { val (inputs, hasStar): (List[T], Boolean) = { element match { case _: Atomic[_] => @@ -241,9 +239,9 @@ class LazyValues(universe: Universe) { def apply[T](element: Element[T], depth: Int): ValueSet[T] = { val (numArgSamples, numTotalSamples) = if (ParticleGenerator.exists(universe)) { val pg = ParticleGenerator(universe) - (pg.numArgSamples, pg.numTotalSamples ) + (pg.numSamplesFromAtomics, pg.maxNumSamplesAtChain) } else { - (ParticleGenerator.defaultArgSamples, ParticleGenerator.defaultTotalSamples) + (ParticleGenerator.defaultNumSamplesFromAtomics, ParticleGenerator.defaultMaxNumSamplesAtChain) } apply(element, depth, numArgSamples, numTotalSamples) } @@ -309,7 +307,7 @@ class LazyValues(universe: Universe) { def storedValues[T](element: Element[T]): ValueSet[T] = { memoValues.get(element) match { case Some((result, _)) => result.asInstanceOf[ValueSet[T]] - case None => new ValueSet[T](Set()) + case None => new ValueSet[T](Set()) } } @@ -347,6 +345,13 @@ class LazyValues(universe: Universe) { private val applyMaps: Map[Element[_], Map[Any, Any]] = Map() universe.register(applyMaps) + /** + * Gets the mapping from apply arguments to values. + */ + def getMap[U](apply: Apply[U]): Map[Any, U] = { + getOrElseInsert(applyMaps, apply, Map[Any, Any]()).asInstanceOf[Map[Any, U]] + } + /** * Gets the mapping from apply arguments to values. */ @@ -414,11 +419,11 @@ object LazyValues { * Create an object for computing the range of values of elements in the universe. This object is only * created once for a universe. */ - def apply(universe: Universe = Universe.universe): LazyValues = { + def apply(universe: Universe = Universe.universe, parameterized: Boolean = false): LazyValues = { expansions.get(universe) match { case Some(e) => e case None => - val expansion = new LazyValues(universe) + val expansion = new LazyValues(universe, parameterized) expansions += (universe -> expansion) universe.registerUniverse(expansions) expansion diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyVariableElimination.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyVariableElimination.scala index f6d80320..2e9cef43 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyVariableElimination.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/LazyVariableElimination.scala @@ -17,7 +17,7 @@ import com.cra.figaro.algorithm.{LazyAlgorithm, ProbQueryAlgorithm} import com.cra.figaro.algorithm.factored.{FactoredAlgorithm, VEGraph} import com.cra.figaro.language.{Element, Universe} import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.factored.factors.Factory +import com.cra.figaro.algorithm.lazyfactored.factory.Factory import scala.annotation.tailrec import com.cra.figaro.util._ import scala.collection.mutable.{Map, Set} @@ -206,7 +206,7 @@ with LazyAlgorithm { var targetFactors: Map[Element[_], Factor[(Double, Double)]] = Map() private def marginalizeToTarget(factor: Factor[(Double, Double)], target: Element[_]): Unit = { - val targetFactor = factor.marginalizeTo(BoundsSumProductSemiring(), Variable(target)) + val targetFactor = factor.marginalizeToWithSum(BoundsSumProductSemiring().sum, Variable(target)) targetFactors += target -> targetFactor } @@ -354,7 +354,7 @@ with LazyAlgorithm { val best = candidates.extractMin()._1 // do not read the best variable after it has been removed, and do not add the preserved variables val touched = graph.info(best).neighbors - best -- toPreserve - val nextGraph = graph.eliminate(best) + val (nextGraph, newCost) = graph.eliminate(best) touched foreach (v => candidates += v.asInstanceOf[Variable[_]] -> graph.score(v)) eliminationOrderHelper(candidates, toPreserve, nextGraph, best :: accum) } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/ValueSet.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/ValueSet.scala index 9c72cdaa..16bf72db 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/ValueSet.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/ValueSet.scala @@ -45,18 +45,22 @@ class ValueSet[T](val xvalues: Set[Extended[T]]) { */ val regularValues = xvalues.filter(_.isRegular).map(_.value) - /* + /** + * Indicates whether there are no values, regular or star, in the value set + */ + val isEmpty = !hasStar && regularValues.isEmpty + /** * Returns the particular Star value in this value set. Throws an exception if there is no Star. */ def starValue: Star[T] = { try { - values.find(_.isInstanceOf[Star[T]]).get.asInstanceOf[Star[T]] + xvalues.find(_.isInstanceOf[Star[T]]).get.asInstanceOf[Star[T]] } catch { case _: NoSuchElementException => throw new RuntimeException("Attempt to get the Star value of a value set without a Star") } } - */ + /** * Apply a function to each value while keeping the same Star nature. diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ApplyFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ApplyFactory.scala new file mode 100644 index 00000000..f42c7165 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ApplyFactory.scala @@ -0,0 +1,203 @@ +/* + * ApplyFactory.scala + * Methods to create factors associated with Apply elements for lazy algorithms + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Dec 15, 2014 + * + * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.lazyfactored.factory + +import com.cra.figaro.algorithm.PointMapper +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.language._ + +/** + * A Sub-Factory for Apply Elements + */ +object ApplyFactory { + + /** + * Factor constructor for an Apply Element that has one input + */ + def makeFactors[T, U](apply: Apply1[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val applyMap: scala.collection.mutable.Map[T, U] = LazyValues(apply.universe).getMap(apply) + val arg1Var = Variable(apply.arg1) + val resultVar = Variable(apply) + val applyValues = LazyValues(apply.universe).storedValues(apply) + val factor = new SparseFactor[Double](List(arg1Var), List(resultVar)) + val arg1Indices = arg1Var.range.zipWithIndex + for { + (arg1Val, arg1Index) <- arg1Indices + //(resultVal, resultIndex) <- resultIndices + } { + // See logic in makeCares + if (arg1Val.isRegular) { + // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. + // By using applyMap, we can make sure that any contained elements in the result of the apply are the same + // now as they were when values were computed. + val resultVal = mapper.map(applyMap(arg1Val.value), applyValues.regularValues) + val resultIndex = resultVar.range.indexWhere(v => if (v.isRegular) v.value == resultVal else false) + factor.set(List(arg1Index, resultIndex), 1.0) + } else if (!arg1Val.isRegular && resultVar.range.exists(!_.isRegular)) { + val resultIndex = resultVar.range.indexWhere(!_.isRegular) + factor.set(List(arg1Index, resultIndex), 1.0) + } + } + List(factor) + } + + /** + * Factor constructor for an Apply Element that has two inputs + */ + def makeFactors[T1, T2, U](apply: Apply2[T1, T2, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val applyMap: scala.collection.mutable.Map[(T1, T2), U] = LazyValues(apply.universe).getMap(apply) + val arg1Var = Variable(apply.arg1) + val arg2Var = Variable(apply.arg2) + val resultVar = Variable(apply) + val applyValues = LazyValues(apply.universe).storedValues(apply) + val factor = new SparseFactor[Double](List(arg1Var, arg2Var), List(resultVar)) + val arg1Indices = arg1Var.range.zipWithIndex + val arg2Indices = arg2Var.range.zipWithIndex + val resultIndices = resultVar.range.zipWithIndex + for { + (arg1Val, arg1Index) <- arg1Indices + (arg2Val, arg2Index) <- arg2Indices + //(resultVal, resultIndex) <- resultIndices + } { + if (arg1Val.isRegular && arg2Val.isRegular) { + // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. + // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. + val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value)), applyValues.regularValues) + val resultIndex = resultVar.range.indexWhere(v => if (v.isRegular) v.value == resultVal else false) + factor.set(List(arg1Index, arg2Index, resultIndex), 1.0) + } else if ((!arg1Val.isRegular || !arg2Val.isRegular) && resultVar.range.exists(!_.isRegular)) { + val resultIndex = resultVar.range.indexWhere(!_.isRegular) + factor.set(List(arg1Index, arg2Index, resultIndex), 1.0) + } + + } + List(factor) + } + + /** + * Factor constructor for an Apply Element that has three inputs + */ + def makeFactors[T1, T2, T3, U](apply: Apply3[T1, T2, T3, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val applyMap: scala.collection.mutable.Map[(T1, T2, T3), U] = LazyValues(apply.universe).getMap(apply) + val arg1Var = Variable(apply.arg1) + val arg2Var = Variable(apply.arg2) + val arg3Var = Variable(apply.arg3) + val resultVar = Variable(apply) + val applyValues = LazyValues(apply.universe).storedValues(apply) + val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var), List(resultVar)) + val arg1Indices = arg1Var.range.zipWithIndex + val arg2Indices = arg2Var.range.zipWithIndex + val arg3Indices = arg3Var.range.zipWithIndex + //val resultIndices = resultVar.range.zipWithIndex + for { + (arg1Val, arg1Index) <- arg1Indices + (arg2Val, arg2Index) <- arg2Indices + (arg3Val, arg3Index) <- arg3Indices + //(resultVal, resultIndex) <- resultIndices + } { + if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular) { + // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. + // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. + val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value, arg3Val.value)), applyValues.regularValues) + val resultIndex = resultVar.range.indexWhere(v => if (v.isRegular) v.value == resultVal else false) + factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), 1.0) + } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular) && resultVar.range.exists(!_.isRegular)) { + val resultIndex = resultVar.range.indexWhere(!_.isRegular) + factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), 1.0) + } + } + List(factor) + } + + /** + * Factor constructor for an Apply Element that has four inputs + */ + def makeFactors[T1, T2, T3, T4, U](apply: Apply4[T1, T2, T3, T4, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val applyMap: scala.collection.mutable.Map[(T1, T2, T3, T4), U] = LazyValues(apply.universe).getMap(apply) + val arg1Var = Variable(apply.arg1) + val arg2Var = Variable(apply.arg2) + val arg3Var = Variable(apply.arg3) + val arg4Var = Variable(apply.arg4) + val resultVar = Variable(apply) + val applyValues = LazyValues(apply.universe).storedValues(apply) + val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var, arg4Var), List(resultVar)) + val arg1Indices = arg1Var.range.zipWithIndex + val arg2Indices = arg2Var.range.zipWithIndex + val arg3Indices = arg3Var.range.zipWithIndex + val arg4Indices = arg4Var.range.zipWithIndex + //val resultIndices = resultVar.range.zipWithIndex + for { + (arg1Val, arg1Index) <- arg1Indices + (arg2Val, arg2Index) <- arg2Indices + (arg3Val, arg3Index) <- arg3Indices + (arg4Val, arg4Index) <- arg4Indices + //(resultVal, resultIndex) <- resultIndices + } { + if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular) { + // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. + // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. + val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value)), applyValues.regularValues) + val resultIndex = resultVar.range.indexWhere(v => if (v.isRegular) v.value == resultVal else false) + factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), 1.0) + } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular || !arg4Val.isRegular) && resultVar.range.exists(!_.isRegular)) { + val resultIndex = resultVar.range.indexWhere(!_.isRegular) + factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), 1.0) + } + } + List(factor) + } + + /** + * Factor constructor for an Apply Element that has five inputs + */ + def makeFactors[T1, T2, T3, T4, T5, U](apply: Apply5[T1, T2, T3, T4, T5, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val applyMap: scala.collection.mutable.Map[(T1, T2, T3, T4, T5), U] = LazyValues(apply.universe).getMap(apply) + val arg1Var = Variable(apply.arg1) + val arg2Var = Variable(apply.arg2) + val arg3Var = Variable(apply.arg3) + val arg4Var = Variable(apply.arg4) + val arg5Var = Variable(apply.arg5) + val resultVar = Variable(apply) + val applyValues = LazyValues(apply.universe).storedValues(apply) + val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var, arg4Var, arg5Var), List(resultVar)) + val arg1Indices = arg1Var.range.zipWithIndex + val arg2Indices = arg2Var.range.zipWithIndex + val arg3Indices = arg3Var.range.zipWithIndex + val arg4Indices = arg4Var.range.zipWithIndex + val arg5Indices = arg5Var.range.zipWithIndex +// val resultIndices = resultVar.range.zipWithIndex + for { + (arg1Val, arg1Index) <- arg1Indices + (arg2Val, arg2Index) <- arg2Indices + (arg3Val, arg3Index) <- arg3Indices + (arg4Val, arg4Index) <- arg4Indices + (arg5Val, arg5Index) <- arg5Indices +// (resultVal, resultIndex) <- resultIndices + } { + if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular && arg5Val.isRegular) { + // arg1Val.value should have been placed in applyMap at the time the values of this apply were computed. + // By using applyMap, we can make sure that any contained elements in the result of the apply are the same now as they were when values were computed. + val resultVal = mapper.map(applyMap((arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value, arg5Val.value)), applyValues.regularValues) + val resultIndex = resultVar.range.indexWhere(v => if (v.isRegular) v.value == resultVal else false) + factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), 1.0) + } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular || !arg4Val.isRegular || !arg5Val.isRegular) && resultVar.range.exists(!_.isRegular)) { + val resultIndex = resultVar.range.indexWhere(!_.isRegular) + factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), 1.0) + } + } + List(factor) + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ChainFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ChainFactory.scala new file mode 100644 index 00000000..12cb1341 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ChainFactory.scala @@ -0,0 +1,159 @@ +/* + * ChainFactory.scala + * Create a factor from a Chain element. + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Mar 22, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.lazyfactored.factory + +import scala.collection.mutable.ListBuffer +import com.cra.figaro.algorithm.PointMapper +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.language._ +import com.cra.figaro.util._ + +/** + * @author Glenn Takata Mar 22, 2015 + * + */ +object ChainFactory { + + /** + * Factor constructor for a Chain Element + */ + def makeFactors[T, U](chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { + val chainMap: scala.collection.mutable.Map[T, Element[U]] = LazyValues(chain.universe).getMap(chain) + val outputVar = Variable(chain).asInstanceOf[ElementVariable[U]] + val parentVar = Variable(chain.parent) + + // create a selectionVar and Factor to take the place of the parent/output combination + val selectorVar = makeSelectorVariable(parentVar, outputVar) + val selector = new SparseFactor[Double](List(parentVar, selectorVar), List(outputVar)) + val parentSize = parentVar.range.size + + var tempFactors = new ListBuffer[Factor[Double]] + tempFactors.append(selector) + // selector must be passed to makeSelection as it is updated based on the resultVar/chainVar match + for { + (parent, parentIndex) <- parentVar.range.zipWithIndex + } { + // parentVal.value should have been placed in applyMap at the time the values of this apply were computed. + // By using chainMap, we can make sure that the result element is the same now as they were when values were computed. + if (parent.isRegular) { + tempFactors.append(makeSelection(chain, selectorVar, parentSize, parentIndex, Variable(chainMap(parent.value)), selector)(mapper)) + } else { + // We create a dummy variable for the outcome variable whose value is always star. + // We create a dummy factor for that variable. + // Then we use makeSelection with the dummy variable + val dummy = new Variable(ValueSet.withStar[U](Set())) + val dummyFactor = new BasicFactor[Double](List(), List(dummy)) + dummyFactor.set(List(0), 1.0) + tempFactors.append(makeSelection(chain, selectorVar, parentSize, parentIndex, dummy, selector), dummyFactor) + } + } + + tempFactors.toList + } + + /* + * Create a temporary variable representing the combination of the parent variable and the chain + * variable + */ + private def makeSelectorVariable[U](parent: Variable[_], overallVar: ElementVariable[U]): Variable[_] = { + val selectorSize = parent.size * overallVar.size + + val values: List[List[Extended[_]]] = parent.range.flatMap(p => overallVar.range.map(o => List(p, o))) + + val tupleRangeRegular: List[List[_]] = cartesianProduct(List(parent, overallVar).map(_.range): _*) + val tupleVS: ValueSet[List[Extended[_]]] = ValueSet.withoutStar(tupleRangeRegular.map(_.asInstanceOf[List[Extended[_]]]).toSet) + + new InternalChainVariable(tupleVS, overallVar.element.asInstanceOf[Chain[_,U]], overallVar.asInstanceOf[Variable[U]]) + } + + /** + * Make a selection factor used in the decomposition of chain. + * A chain defines a factor over the parent element, each of the possible result elements of the chain, + * and the overall chain element. This can produce a very large factor when there are many result elements. + * This is solved by decomposing the chain factor into a product of factors, each of which contains a + * selector variable and the result variable. + * + * The conditional selector creates a factor in which, when the selector's value is such that the result + * element is relevant to the final result, the result element and chain element must have the same + * value (handled by makeCares). Otherwise, the result element and chain element can take on any + * value (handled by makeDontCares) + * + */ + private def makeSelection[U](chain: Element[U], selectorVar: Variable[_], parentSize: Int, + parentIndex: Int, resultVar: Variable[U], selector: Factor[Double])(implicit mapper: PointMapper[U]): Factor[Double] = { + val chainVar = Variable(chain) + val chainValues = LazyValues(chain.universe).storedValues(chain) + val factor = new ConditionalSelector[Double](List(selectorVar), List(resultVar)) + + makeCares(factor, parentIndex, resultVar, chainVar, chainValues.regularValues, selector)(mapper) + for (index <- 0 until parentSize) { + if (index != parentIndex) + makeDontCares[U](factor, index, resultVar, chainVar, selector) + } + factor + } + + /* + * Values that specify the potential for the selector variable and the specified output + */ + private def makeCares[U](factor: ConditionalSelector[Double], parentIndex: Int, resultVar: Variable[U], + chainVar: Variable[U], choices: Set[U], selector: Factor[Double])(implicit mapper: PointMapper[U]): Unit = { + // We care to match up overallVar with outcomeVar + val chainSize = chainVar.size + for { + (resultVal, j) <- resultVar.range.zipWithIndex + (chainVal, k) <- chainVar.range.zipWithIndex + } { + // Star stands for "something". If outcomeVal is Star and overallVal is Star, we know something will match something, so the entry is (1,1). + // If outcomeVal is Star and overallVal is a regular value, then maybe there will be a match, so the entry is (0,1). + // If outcomeVal is regular, all the probability mass associated with that outcome should be on regular values of overallVal, so the entry is (0,0). + val entry = + if (chainVal.isRegular && resultVal.isRegular) { + if (chainVal.value == mapper.map(resultVal.value, choices)) 1.0 + else 0.0 + } else if (!chainVal.isRegular && !resultVal.isRegular) 1.0 + else 0.0 + + // calculate the relevant index of the selector Factor and update both + // the selector Factor and the current (conditional selector) Factor + val selectorIndex = parentIndex * chainSize + k + selector.set(List(parentIndex, selectorIndex, k), 1.0) + factor.set(List(selectorIndex, j), entry) + } + } + + /* + * Values that are irrelevant to the selector variable and specified output + */ + private def makeDontCares[U](factor: ConditionalSelector[Double], + parentIndex: Int, + outcomeVar: Variable[U], + chainVar: Variable[U], selector: Factor[Double]): Unit = { + + val chainSize = chainVar.size + + // If we don't care, we assign 1.0 to all combinations of the distVar and outcomeVar + for { + j <- 0 until outcomeVar.size + k <- 0 until chainSize + } { + + // calculate the relevant index of the selector Factor and update + // current (conditional selector) Factor + // The selector factor is sparse and needs no update + val selectorIndex = parentIndex * chainSize + k + factor.set(List(selectorIndex, j), 1.0) + } + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ComplexFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ComplexFactory.scala new file mode 100644 index 00000000..a2e2a1ec --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/ComplexFactory.scala @@ -0,0 +1,208 @@ +/* + * ComplexFactory.scala + * Methods to create factors associated with a variety of complex elements for lazy algorithms + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Dec 15, 2014 + * + * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.lazyfactored.factory + +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.factors._ + + + +/** + * A Sub-Factory for Complex Elements + */ +object ComplexFactory { + + /** + * Factor constructor for a SingleValuedReferenceElement + */ + def makeFactors[T](element: SingleValuedReferenceElement[T]): List[Factor[Double]] = { + val (first, rest) = element.collection.getFirst(element.reference) + rest match { + case None => + val elementVar = Variable(element) + val firstVar = Variable(first) + val factor = new BasicFactor[Double](List(firstVar), List(elementVar)) + for { + i <- 0 until firstVar.range.size + j <- 0 until elementVar.range.size + } { + factor.set(List(i, j), (if (i == j) 1.0; else 0.0)) + } + List(factor) + case Some(restRef) => + val firstVar = Variable(first) + val selectedFactors = + for { + (firstXvalue, firstIndex) <- firstVar.range.zipWithIndex + firstCollection = firstXvalue.value.asInstanceOf[ElementCollection] + restElement = element.embeddedElements(firstCollection) + } yield { + Factory.makeConditionalSelector(element, firstVar, firstIndex, Variable(restElement)) :: makeFactors(restElement) + } + selectedFactors.flatten + } + } + + /** + * Factor constructor for a MultiValuedReferenceElement + */ + def makeFactors[T](element: MultiValuedReferenceElement[T]): List[Factor[Double]] = { + val (first, rest) = element.collection.getFirst(element.reference) + val selectionFactors: List[List[Factor[Double]]] = { + rest match { + case None => + val elementVar = Variable(element) + val firstVar = Variable(first) + val factor = new BasicFactor[Double](List(firstVar), List(elementVar)) + for { + i <- 0 until firstVar.range.size + j <- 0 until elementVar.range.size + } { + factor.set(List(i, j), (if (i == j) 1.0; else 0.0)) + } + List(List(factor)) + case Some(restRef) => + val firstVar = Variable(first) + for { + (firstXvalue, firstIndex) <- firstVar.range.zipWithIndex + } yield { + if (firstXvalue.isRegular) { + firstXvalue.value match { + case firstCollection: ElementCollection => + val restElement = element.embeddedElements(firstCollection) + val result: List[Factor[Double]] = + Factory.makeConditionalSelector(element, firstVar, firstIndex, Variable(restElement)) :: Factory.make(restElement) + result + case cs: Traversable[_] => + // Create a multi-valued reference element (MVRE) for each collection in the value of the first name. + // Since the first name is multi-valued, its value is the union of the values of all these MVREs. + val collections = cs.asInstanceOf[Traversable[ElementCollection]].toList.distinct // Set semantics + val multis: List[MultiValuedReferenceElement[T]] = collections.map(element.embeddedElements(_)).toList + // Create the element that takes the union of the values of the all the MVREs. + // The combination and setMaker elements are encapsulated within this object and are created now, so we need to create factors for them. + // Finally, we create a conditional selector (see ProbFactor) to select the appropriate result value when the first + // name's value is these MVREs. + val combination = element.embeddedInject(collections) + val setMaker = element.embeddedApply(collections) + val result: List[Factor[Double]] = + Factory.makeConditionalSelector(element, firstVar, firstIndex, Variable(setMaker)) :: Factory.make(combination) ::: + Factory.make(setMaker) + result + } + } else StarFactory.makeStarFactor(element) + } + } + } + selectionFactors.flatten + } + + /** + * Factor constructor for an Aggregate Element + */ + def makeFactors[T, U](element: Aggregate[T, U]): List[Factor[Double]] = { + val elementVar = Variable(element) + val mvreVar = Variable(element.mvre) + val factor = new BasicFactor[Double](List(mvreVar), List(elementVar)) + for { + (mvreXvalue, mvreIndex) <- mvreVar.range.zipWithIndex + (elementXvalue, elementIndex) <- elementVar.range.zipWithIndex + } { + if (elementXvalue.isRegular && mvreXvalue.isRegular) factor.set(List(mvreIndex, elementIndex), if (element.aggregate(mvreXvalue.value) == elementXvalue.value) 1.0; else 0.0) + } + // The MultiValuedReferenceElement for this aggregate is generated when values is called. + // Therefore, it will be included in the expansion and have factors made for it automatically, so we do not create factors for it here. + List(factor) + } + + // adapted from Apply1 + /** + * Factor constructor for a MakeArray Element + */ + def makeFactors[T](element: com.cra.figaro.library.collection.MakeArray[T]): List[Factor[Double]] = { + val arg1Var = Variable(element.numItems) + val resultVar = Variable(element) + val factor = new BasicFactor[Double](List(arg1Var), List(resultVar)) + val arg1Indices = arg1Var.range.zipWithIndex + val resultIndices = resultVar.range.zipWithIndex + for { + (arg1Val, arg1Index) <- arg1Indices + (resultVal, resultIndex) <- resultIndices + } { + val entry = + if (arg1Val.isRegular && resultVal.isRegular) { + if (resultVal.value == element.arrays(arg1Val.value)) 1.0 + else 0.0 + } else if (!arg1Val.isRegular && !resultVal.isRegular) 1.0 + else if (!arg1Val.isRegular && resultVal.isRegular) 0.0 + else 0.0 + factor.set(List(arg1Index, resultIndex), entry) + } + List(factor) + } + + /** + * Factor constructor for a FoldLeft Element + */ + def makeFactors[T,U](fold: FoldLeft[T,U]): List[Factor[Double]] = { + def makeOneFactor(currentAccumVar: Variable[U], elemVar: Variable[T], nextAccumVar: Variable[U]): Factor[Double] = { + val result = new BasicFactor[Double](List(currentAccumVar, elemVar), List(nextAccumVar)) + val currentAccumIndices = currentAccumVar.range.zipWithIndex + val elemIndices = elemVar.range.zipWithIndex + val nextAccumIndices = nextAccumVar.range.zipWithIndex + for { + (currentAccumVal, currentAccumIndex) <- currentAccumIndices + (elemVal, elemIndex) <- elemIndices + (nextAccumVal, nextAccumIndex) <- nextAccumIndices + } { + val entry = + if (currentAccumVal.isRegular && elemVal.isRegular && nextAccumVal.isRegular) { + if (nextAccumVal.value == fold.function(currentAccumVal.value, elemVal.value)) 1.0 + else 0.0 + } else if ((!currentAccumVal.isRegular || !elemVal.isRegular) && !nextAccumVal.isRegular) 1.0 + else 0.0 + result.set(List(currentAccumIndex, elemIndex, nextAccumIndex), entry) + } + result + } + + def makeFactorSequence(currentAccumVar: Variable[U], remaining: Seq[Element[T]]): List[Factor[Double]] = { + if (remaining.isEmpty) List() + else { + val firstVar = Variable(remaining.head) + val rest = remaining.tail + val nextAccumVar = + if (rest.isEmpty) Variable(fold) + else { + val currentAccumRegular = currentAccumVar.range.filter(_.isRegular).map(_.value) + val firstRegular = firstVar.range.filter(_.isRegular).map(_.value) + val nextVals = + for { + accum <- currentAccumRegular + first <- firstRegular + } yield fold.function(accum, first) + val nextHasStar = currentAccumVar.range.exists(!_.isRegular) || firstVar.range.exists(!_.isRegular) + val nextVS = if (nextHasStar) ValueSet.withStar(nextVals.toSet) else ValueSet.withoutStar(nextVals.toSet) + new Variable(nextVS) + } + val nextFactor = makeOneFactor(currentAccumVar, firstVar, nextAccumVar) + nextFactor :: makeFactorSequence(nextAccumVar, rest) + } + } + val startVar = new Variable(ValueSet.withoutStar(Set(fold.start))) + makeFactorSequence(startVar, fold.elements) + } +} + diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/DistributionFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/DistributionFactory.scala new file mode 100644 index 00000000..2cd6b008 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/DistributionFactory.scala @@ -0,0 +1,95 @@ +/* + * DistributionFactory.scala + * Methods to create factors for simple distributions for lazy algorithms + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Dec 15, 2014 + * + * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.lazyfactored.factory + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.discrete._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored._ + +/** + * A Sub-Factory for simple probability distribution Elements + */ +object DistributionFactory { + + /** + * Factor constructor for an AtomicFlip + */ + def makeFactors(flip: AtomicFlip): List[Factor[Double]] = { + val flipVar = Variable(flip) + if (flipVar.range.exists(!_.isRegular)) { + assert(flipVar.range.size == 1) // Flip's range must either be {T,F} or {*} + StarFactory.makeStarFactor(flip) + } else { + val factor = new BasicFactor[Double](List(), List(flipVar)) + val i = flipVar.range.indexOf(Regular(true)) + factor.set(List(i), flip.prob) + factor.set(List(1 - i), 1.0 - flip.prob) + List(factor) + } + } + + /** + * Factor constructor for a CompoundFlip + */ + def makeFactors(flip: CompoundFlip): List[Factor[Double]] = { + val flipVar = Variable(flip) + if (flipVar.range.exists(!_.isRegular)) { + assert(flipVar.range.size == 1) // Flip's range must either be {T,F} or {*} + StarFactory.makeStarFactor(flip) + } else { + val probVar = Variable(flip.prob) + val factor = new BasicFactor[Double](List(probVar), List(flipVar)) + val parentVals = probVar.range + val i = flipVar.range.indexOf(Regular(true)) + for { j <- 0 until parentVals.size } { + if (parentVals(j).isRegular) { + val value = parentVals(j).value + factor.set(List(j, i), value) + factor.set(List(j, 1 - i), 1.0 - value) + } else { + factor.set(List(j, 0), 0.0) + factor.set(List(j, 1), 0.0) + } + } + List(factor) + } + } + + /** + * Factor constructor for a ParameterizedFlip + */ + def makeFactors(flip: ParameterizedFlip): List[Factor[Double]] = { + val flipVar = Variable(flip) + val factor = new BasicFactor[Double](List(),List(flipVar)) + val prob = flip.parameter.MAPValue + val i = flipVar.range.indexOf(Regular(true)) + factor.set(List(i), prob) + factor.set(List(1 - i), 1.0 - prob) + List(factor) + } + + /** + * Factor constructor for an AtomicBinomial + */ + def makeFactors(binomial: AtomicBinomial): List[Factor[Double]] = { + val binVar = Variable(binomial) + val factor = new BasicFactor[Double](List(), List(binVar)) + for { (xvalue, index) <- binVar.range.zipWithIndex } { + factor.set(List(index), binomial.density(xvalue.value)) + } + List(factor) + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/Factory.scala similarity index 98% rename from Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factory.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/Factory.scala index b22ab07c..a06ced36 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/factored/factors/Factory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/Factory.scala @@ -11,7 +11,7 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.algorithm.factored.factors +package com.cra.figaro.algorithm.lazyfactored.factory import com.cra.figaro.algorithm._ import com.cra.figaro.language._ @@ -23,14 +23,11 @@ import com.cra.figaro.algorithm.factored.factors.factory._ import com.cra.figaro.library.compound._ import com.cra.figaro.library.collection._ import com.cra.figaro.library.atomic.discrete._ +import scala.collection.mutable.HashMap +import com.cra.figaro.algorithm.factored.factors._ import scala.reflect.runtime.universe.{typeTag, TypeTag} -/** - * A trait for elements that are able to construct their own Factor. - */ -trait FactorMaker[T] { - def makeFactors[T]: List[Factor[Double]] -} + /** * Methods for creating probabilistic factors associated with elements. @@ -323,7 +320,7 @@ object Factory { case r: SingleValuedReferenceElement[_] => ComplexFactory.makeFactors(r) case r: MultiValuedReferenceElement[_] => ComplexFactory.makeFactors(r) case r: Aggregate[_, _] => ComplexFactory.makeFactors(r) - case m: MakeList[_] => ComplexFactory.makeFactors(m) + //case m: MakeList[_] => ComplexFactory.makeFactors(m) case m: MakeArray[_] => ComplexFactory.makeFactors(m) case f: FoldLeft[_, _] => ComplexFactory.makeFactors(f) case f: FactorMaker[_] => f.makeFactors @@ -432,8 +429,9 @@ object Factory { resultFactor } - private val factorCache = scala.collection.mutable.Map[Element[_], List[Factor[Double]]]() - + private val factorCache = new HashMap[Element[_], List[Factor[Double]]]() { + override def hashCode = 3 + } /** * Construct a Factor without constraints. */ @@ -498,4 +496,5 @@ object Factory { factor.fillByRule(rule _) factor } + } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/SelectFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/SelectFactory.scala new file mode 100644 index 00000000..529f080f --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/SelectFactory.scala @@ -0,0 +1,169 @@ +/* + * SelectFactory.scala + * Methods to create factors for Select and Dist elements for lazy algorithms + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Dec 15, 2014 + * + * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.lazyfactored.factory + +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.util._ + + +/** + * A Sub-Factory for Select or Dist Elements + */ +object SelectFactory { + + /** + * Factor constructor for an AtomicDist + */ + def makeFactors[T](dist: AtomicDist[T]): List[Factor[Double]] = { + val (intermed, clauseFactors) = intermedAndClauseFactors(dist) + val intermedFactor = makeSimpleDistribution(intermed, dist.probs) + intermedFactor :: clauseFactors + } + + /** + * Factor constructor for a CompoundDist + */ + def makeFactors[T](dist: CompoundDist[T]): List[Factor[Double]] = { + val (intermed, clauseFactors) = intermedAndClauseFactors(dist) + val intermedFactor = makeComplexDistribution(intermed, dist.probs) + intermedFactor :: clauseFactors + } + + /** + * Factor constructor for an AtomicSelect + */ + def makeFactors[T](select: AtomicSelect[T]): List[Factor[Double]] = { + val selectVar = Variable(select) + if (selectVar.range.exists(!_.isRegular)) { + assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} + StarFactory.makeStarFactor(select) + } else { + val probs = getProbs(select) + List(makeSimpleDistribution(selectVar, probs)) + } + } + + /** + * Factor constructor for a CompoundSelect + */ + def makeFactors[T](select: CompoundSelect[T]): List[Factor[Double]] = { + val selectVar = Variable(select) + if (selectVar.range.exists(!_.isRegular)) { + assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} + StarFactory.makeStarFactor(select) + } else { + val probs = getProbs(select) + List(makeComplexDistribution(selectVar, probs)) + } + } + + /** + * Factor constructor for a ParameterizedSelect + */ + def makeFactors[T](select: ParameterizedSelect[T]): List[Factor[Double]] = { + val selectVar = Variable(select) + if (selectVar.range.exists(!_.isRegular)) { + assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} + StarFactory.makeStarFactor(select) + } else { + val probs = parameterizedGetProbs(select) + List(makeSimpleDistribution(selectVar, probs)) + } + } + + /** + * Factor constructor for an IntSelector + */ + def makeFactors[T](select: IntSelector): List[Factor[Double]] = { + val elementVar = Variable(select) + val counterVar = Variable(select.counter) + val comb = new BasicFactor[Double](List(counterVar), List(elementVar)) + comb.fillByRule((l: List[Any]) => { + val counterValue :: elementValue :: _ = l.asInstanceOf[List[Extended[Int]]] + if (counterValue.isRegular && elementValue.isRegular) { + if (elementValue.value < counterValue.value) 1.0 / counterValue.value; else 0.0 + } else 1.0 + + }) + List(comb) + } + + private def getProbs[U, T](select: Select[U, T]): List[U] = getProbs(select, select.clauses) + + /** + * Get the potential (probability) for each value of an element, based on supplied rules + */ + def getProbs[U, T](elem: Element[T], clauses: List[(U, T)]): List[U] = { + val selectVar = Variable(elem) + def getProb(xvalue: Extended[T]): U = { + clauses.find(_._2 == xvalue.value).get._1 // * cannot be a value of a Select + } + val probs = + for { xvalue <- selectVar.range } yield getProb(xvalue) + probs + } + + private def parameterizedGetProbs[T](select: ParameterizedSelect[T]): List[Double] = { + val outcomes = select.outcomes + val map = select.parameter.MAPValue + for { + xvalue <- Variable(select).range + index = outcomes.indexOf(xvalue.value) + } yield map(index) + } + + private def intermedAndClauseFactors[U, T](dist: Dist[U, T]): (Variable[Int], List[Factor[Double]]) = { + val intermed = new Variable(ValueSet.withoutStar((0 until dist.clauses.size).toSet)) + val clauseFactors = dist.outcomes.zipWithIndex map (pair => + Factory.makeConditionalSelector(dist, intermed, pair._2, Variable(pair._1))) + (intermed, clauseFactors) + } + + /** + * Constructs a BasicFactor from a probability distribution. It assumes that the probabilities + * are assigned to the Variable in the same order as it's values. + */ + def makeSimpleDistribution[T](target: Variable[T], probs: List[Double]): Factor[Double] = { + val factor = new BasicFactor[Double](List(), List(target)) + for { (prob, index) <- probs.zipWithIndex } { + factor.set(List(index), prob) + } + factor + } + + private def makeComplexDistribution[T](target: Variable[T], probElems: List[Element[Double]]): Factor[Double] = { + val probVars: List[Variable[Double]] = probElems map (Variable(_)) + val nVars = probVars.size + val factor = new BasicFactor[Double](probVars, List(target)) + val probVals: List[List[Extended[Double]]] = probVars map (_.range) + for { indices <- factor.getIndices } { + // unnormalized is a list, one for each probability element, + // of the value of that element under these indices + val unnormalized = + for { (probIndex, position) <- indices.toList.take(nVars).zipWithIndex } yield { + // The probability of the particular value of the probability element in this position + val xprob = probVals(position)(probIndex) + if (xprob.isRegular) xprob.value; else 0.0 + } + val normalized = normalize(unnormalized).toArray + // The first variable specifies the position of the remaining variables, + // so indices(last) is the correct probability + factor.set(indices, normalized(indices.last)) + } + factor + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/StarFactory.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/StarFactory.scala similarity index 68% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/StarFactory.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/StarFactory.scala index 8824e17b..9999cdc7 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/StarFactory.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/lazyfactored/factory/StarFactory.scala @@ -1,38 +1,37 @@ /* * StarFactory.scala - * The default factor over an element whose only value is Star. - * + * The default factor over an element whose only value is Star (for lazy algorithms) + * * Created By: Glenn Takata (gtakata@cra.com) * Creation Date: Dec 15, 2014 - * + * * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured.factory +package com.cra.figaro.algorithm.lazyfactored.factory import com.cra.figaro.language._ import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.lazyfactored.Star -import com.cra.figaro.experimental.structured.ComponentCollection +import com.cra.figaro.algorithm.lazyfactored._ /** * A Sub-Factory to make Star Factors from arbitrary elements */ object StarFactory { - + /** * Make a StarFactor from an Element p -> p.zeroSufficientStatistics): _*) protected def doExpectationStep(): Map[Parameter[_], Seq[Double]] - protected def doStart(): Unit = { + protected[algorithm] def doStart(): Unit = { em() } /* * Stop the algorithm from computing. The algorithm is still ready to provide answers after it returns. */ - protected def doStop(): Unit = {} + protected[algorithm] def doStop(): Unit = {} /* * Resume the computation of the algorithm, if it has been stopped. */ - protected def doResume(): Unit = {} + protected[algorithm] def doResume(): Unit = {} /* * Kill the algorithm so that it is inactive. It will no longer be able to provide answers. */ - protected def doKill(): Unit = {} + protected[algorithm] def doKill(): Unit = {} val terminationCriteria: () => EMTerminationCriteria val targetParameters: Seq[Parameter[_]] @@ -83,21 +89,23 @@ trait ExpectationMaximization extends Algorithm with ParameterLearner { */ trait OnlineExpectationMaximization extends Online with ExpectationMaximization { + override def doStart = {} + protected var lastIterationStatistics: Map[Parameter[_], Seq[Double]] = Map(targetParameters.map(p => p -> p.zeroSufficientStatistics): _*) override val initial: Universe override val transition: Function0[Universe] protected var currentUniverse: Universe = initial - private def updateStatistics(newStatistics:Map[Parameter[_], Seq[Double]]): Map[Parameter[_], Seq[Double]] = { + private def updateStatistics(newStatistics: Map[Parameter[_], Seq[Double]]): Map[Parameter[_], Seq[Double]] = { Map((for (p <- paramMap.keys) yield { val updatedStatistics = (lastIterationStatistics(p) zip newStatistics(p)).map((pair: (Double, Double)) => pair._1 + pair._2) (p, updatedStatistics) - }).toSeq:_*) - } - - /** - * Observe new evidence and perform one expectation step and one maximization step - */ + }).toSeq: _*) + } + + /** + * Observe new evidence and perform one expectation step and one maximization step + */ def update(evidence: Seq[NamedEvidence[_]] = Seq()): Unit = { currentUniverse = transition() currentUniverse.assertEvidence(evidence) @@ -124,13 +132,12 @@ class ExpectationMaximizationWithFactors(val universe: Universe, val targetParam } - /** * An online EM algorithm which learns parameters using a factored algorithm */ class OnlineExpectationMaximizationWithFactors(override val initial: Universe, override val transition: Function0[Universe], val targetParameters: Parameter[_]*)(val terminationCriteria: () => EMTerminationCriteria) extends OnlineExpectationMaximization { - + def doExpectationStep = { val algorithm = SufficientStatisticsVariableElimination(paramMap)(currentUniverse) algorithm.start @@ -141,13 +148,13 @@ class OnlineExpectationMaximizationWithFactors(override val initial: Universe, o } } - /** * An EM algorithm which learns parameters using an inference algorithm provided as an argument */ class GeneralizedEM(inferenceAlgorithmConstructor: Seq[Element[_]] => Universe => ProbQueryAlgorithm with OneTime, val universe: Universe, val targetParameters: Parameter[_]*)(val terminationCriteria: () => EMTerminationCriteria) extends ExpectationMaximization { - protected def doExpectationStep(): Map[Parameter[_], Seq[Double]] = { + //Dependent universe doesn't work the same way. + protected def doExpectationStep(): Map[Parameter[_], Seq[Double]] = { val inferenceTargets = universe.activeElements.filter(_.isInstanceOf[Parameterized[_]]).map(_.asInstanceOf[Parameterized[_]]) @@ -161,10 +168,13 @@ class GeneralizedEM(inferenceAlgorithmConstructor: Seq[Element[_]] => Universe = for { target <- universe.directlyUsedBy(parameter) } { + val t: Parameterized[target.Value] = target.asInstanceOf[Parameterized[target.Value]] - val distribution: Stream[(Double, target.Value)] = algorithm.distribution(t) - val newStats = t.distributionToStatistics(parameter, distribution) - stats = (stats.zip(newStats)).map(pair => pair._1 + pair._2) + if (inferenceTargets.contains(t)) { + val distribution: Stream[(Double, target.Value)] = algorithm.distribution(t) + val newStats = t.distributionToStatistics(parameter, distribution) + stats = (stats.zip(newStats)).map(pair => pair._1 + pair._2) + } } result += parameter -> stats } @@ -179,25 +189,33 @@ class GeneralizedEM(inferenceAlgorithmConstructor: Seq[Element[_]] => Universe = */ class GeneralizedOnlineEM(inferenceAlgorithmConstructor: Seq[Element[_]] => Universe => ProbQueryAlgorithm with OneTime, override val initial: Universe, override val transition: Function0[Universe], val targetParameters: Parameter[_]*)(val terminationCriteria: () => EMTerminationCriteria) extends OnlineExpectationMaximization { - + protected def usesParameter(l: List[Element[_]]): Map[Parameter[_], Iterable[Parameterized[_]]] = { + (l.map { x => x match { case p: Parameterized[_] => { p -> p.parameters.head } } }).groupBy(_._2).mapValues(_.map(_._1)) + } + protected def doExpectationStep(): Map[Parameter[_], Seq[Double]] = { val inferenceTargets = currentUniverse.activeElements.filter(_.isInstanceOf[Parameterized[_]]).map(_.asInstanceOf[Parameterized[_]]) val algorithm = inferenceAlgorithmConstructor(inferenceTargets)(currentUniverse) algorithm.start() - + //println("universe: " + currentUniverse.hashCode) var result: Map[Parameter[_], Seq[Double]] = Map() + val uses = usesParameter(inferenceTargets) for { parameter <- targetParameters } { var stats = parameter.zeroSufficientStatistics - for { - target <- currentUniverse.directlyUsedBy(parameter) - } { - val t: Parameterized[target.Value] = target.asInstanceOf[Parameterized[target.Value]] - val distribution: Stream[(Double, target.Value)] = algorithm.distribution(t) - val newStats = t.distributionToStatistics(parameter, distribution) - stats = (stats.zip(newStats)).map(pair => pair._1 + pair._2) + if (uses.contains(parameter)) { + for { + target <- uses(parameter) + } { + val t: Parameterized[target.Value] = target.asInstanceOf[Parameterized[target.Value]] + if (inferenceTargets.contains(t)) { + val distribution: Stream[(Double, target.Value)] = algorithm.distribution(t) + val newStats = t.distributionToStatistics(parameter, distribution) + stats = (stats.zip(newStats)).map(pair => pair._1 + pair._2) + } + } } result += parameter -> stats } @@ -208,7 +226,7 @@ class GeneralizedOnlineEM(inferenceAlgorithmConstructor: Seq[Element[_]] => Univ } object EMWithBP { - + private val defaultBPIterations = 10 def online(transition: () => Universe, p: Parameter[_]*)(implicit universe: Universe) = { @@ -220,8 +238,11 @@ object EMWithBP { } private def makeBP(numIterations: Int, targets: Seq[Element[_]])(universe: Universe) = { - Factory.removeFactors() - BeliefPropagation(numIterations, targets: _*)(universe) + Variable.clearCache + new ProbQueryBeliefPropagation(universe, targets: _*)( + List(), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) + with OneTimeProbabilisticBeliefPropagation with OneTimeProbQuery with ParameterLearner { val iterations = numIterations } } /** * An expectation maximization algorithm using Belief Propagation sampling for inference. @@ -229,50 +250,58 @@ object EMWithBP { * @param params parameters to target with EM algorithm */ def apply(params: ModelParameters)(implicit universe: Universe) = { + println("Warning: Using BP with EM can have produce unpredictable behavior if parameterized elements are created inside a Chain.") val parameters = params.convertToParameterList new GeneralizedEM((targets: Seq[Element[_]]) => (universe: Universe) => makeBP(defaultBPIterations, targets)(universe), universe, parameters: _*)(EMTerminationCriteria.maxIterations(10)) } /** - * An expectation maximization algorithm using Belief Propagation sampling for inference. + * An expectation maximization algorithm using Belief Propagation for inference. * @param emIterations number of iterations of the EM algorithm * @param bpIterations number of iterations of the BP algorithm * @param params parameters to target with EM algorithm */ def apply(emIterations: Int, bpIterations: Int, p: ModelParameters)(implicit universe: Universe) = { + println("Warning: Using BP with EM can have produce unpredictable behavior if parameterized elements are created inside a Chain.") val parameters = p.convertToParameterList new GeneralizedEM((targets: Seq[Element[_]]) => (universe: Universe) => makeBP(bpIterations, targets)(universe), universe, parameters: _*)(EMTerminationCriteria.maxIterations(emIterations)) } /** - * An expectation maximization algorithm using Belief Propagation sampling for inference. + * An expectation maximization algorithm using Belief Propagation for inference. * @param params parameters to target with EM algorithm */ - def apply(params: Parameter[_]*)(implicit universe: Universe) = + def apply(params: Parameter[_]*)(implicit universe: Universe) = { + println("Warning: Using BP with EM can have produce unpredictable behavior if parameterized elements are created inside a Chain.") new GeneralizedEM((targets: Seq[Element[_]]) => (universe: Universe) => makeBP(defaultBPIterations, targets)(universe), universe, params: _*)(EMTerminationCriteria.maxIterations(10)) + } /** - * An expectation maximization algorithm using importance sampling for inference. + * An expectation maximization algorithm using Belief Propagation for inference. * @param emIterations number of iterations of the EM algorithm * @param bpIterations number of iterations of the BP algorithm * @param params parameters to target with EM algorithm */ - def apply(emIterations: Int, bpIterations: Int, params: Parameter[_]*)(implicit universe: Universe) = + def apply(emIterations: Int, bpIterations: Int, params: Parameter[_]*)(implicit universe: Universe) = { + println("Warning: Using BP with EM can have produce unpredictable behavior if parameterized elements are created inside a Chain.") new GeneralizedEM((targets: Seq[Element[_]]) => (universe: Universe) => makeBP(bpIterations, targets)(universe), universe, params: _*)(EMTerminationCriteria.maxIterations(emIterations)) + } /** - * An expectation maximization algorithm using importance sampling for inference. + * An expectation maximization algorithm using Belief Propagation for inference. * @param terminationCriteria criteria for stopping the EM algorithm * @param bpIterations number of iterations of the BP algorithm * @param params parameters to target with EM algorithm */ - def apply(terminationCriteria: () => EMTerminationCriteria, bpIterations: Int, params: Parameter[_]*)(implicit universe: Universe) = + def apply(terminationCriteria: () => EMTerminationCriteria, bpIterations: Int, params: Parameter[_]*)(implicit universe: Universe) = { + println("Warning: Using BP with EM can have produce unpredictable behavior if parameterized elements are created inside a Chain.") new GeneralizedEM((targets: Seq[Element[_]]) => (universe: Universe) => makeBP(bpIterations, targets)(universe), universe, params: _*)(terminationCriteria) + } } object EMWithImportance { - + private val defaultImportanceParticles = 100000 - + private def makeImportance(numParticles: Int, targets: Seq[Element[_]])(universe: Universe) = { Importance(numParticles, targets: _*)(universe) } @@ -342,7 +371,7 @@ object EMWithImportance { object EMWithMH { private val defaultMHParticles = 100000 - + private def makeImportance(numParticles: Int, targets: Seq[Element[_]])(universe: Universe) = { Importance(numParticles, targets: _*)(universe) } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/learning/SufficientStatisticsFactor.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/learning/SufficientStatisticsFactor.scala index 8111f07b..6d0bf6c5 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/learning/SufficientStatisticsFactor.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/learning/SufficientStatisticsFactor.scala @@ -16,8 +16,6 @@ package com.cra.figaro.algorithm.learning import com.cra.figaro.algorithm._ import com.cra.figaro.algorithm.sampling._ import com.cra.figaro.library.atomic.discrete._ -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.factored.factors.Factory import com.cra.figaro.algorithm.lazyfactored._ import com.cra.figaro.library.decision._ import com.cra.figaro.language._ @@ -25,9 +23,13 @@ import com.cra.figaro.library.atomic.discrete.ParameterizedBinomialFixedNumTrial import com.cra.figaro.util._ import annotation.tailrec import scala.collection._ -import scala.collection.mutable.{ Set, Map } +import scala.collection.mutable.{ Set } +import scala.collection.immutable.Map import scala.math.{ floor, pow } import JSci.maths.ExtraMath.binomial +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.structured.ComponentCollection +import com.cra.figaro.algorithm.factored.factors.factory.Factory /** * Methods for creating probabilistic factors associated with elements and their sufficient statistics. @@ -35,132 +37,74 @@ import JSci.maths.ExtraMath.binomial * @param parameterMap Map of parameters to their sufficient statistics. Expectation */ -class SufficientStatisticsFactor(parameterMap: immutable.Map[Parameter[_], Seq[Double]]) { +class SufficientStatisticsFactor(parameterMap: Map[Parameter[_], Seq[Double]]) { val semiring = new SufficientStatisticsSemiring(parameterMap) - private def makeFactors[T](const: Constant[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(Variable(const)), semiring) - val mapping = mutable.Map(parameterMap.toSeq: _*) - factor.set(List(0), (1.0, mapping)) - List(factor) + def convertFactor[T](factor: Factor[Double]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { + def doubleToParam(d: Double) = (d, Map(parameterMap.toSeq: _*)) + factor.mapTo(doubleToParam, semiring) } - private def makeFactors(flip: AtomicFlip): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val flipVar = Variable(flip) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(flipVar), semiring) - val i = flipVar.range.indexOf(Regular(true)) - val trueMapping = mutable.Map(parameterMap.toSeq: _*) - val falseMapping = mutable.Map(parameterMap.toSeq: _*) - factor.set(List(i), (flip.prob, trueMapping)) - factor.set(List(1 - i), (1 - flip.prob, falseMapping)) - List(factor) - } - - private def makeFactors(flip: CompoundFlip): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val flipVar = Variable(flip) - val probVar = Variable(flip.prob) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(probVar), List(flipVar), semiring) - val parentVals = probVar.range - val i = flipVar.range.indexOf(Regular(true)) - - for { j <- 0 until parentVals.size } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - factor.set(List(j, i), (parentVals(j).value, rowMapping)) - factor.set(List(j, 1 - i), (1.0 - parentVals(j).value, rowMapping)) - } - List(factor) - } + def partitionConstraintFactors(factors: List[Factor[Double]]) = factors.partition(_.isConstraint) private def makeFactors(flip: ParameterizedFlip): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { + val origFactor = Factory.makeFactorsForElement(flip, false, true) + val (constraints, nonconstraints) = partitionConstraintFactors(origFactor) + val flipVar = Variable(flip) val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(flipVar), semiring) val prob = flip.parameter.MAPValue val i = flipVar.range.indexOf(Regular(true)) - val falseMapping = mutable.Map(parameterMap.toSeq: _*) - val trueMapping = mutable.Map(parameterMap.toSeq: _*) + val falseMapping = Map(parameterMap.toSeq: _*) + ((flip.parameter: Parameter[_]) -> Seq(0.0, 1.0)) + val trueMapping = Map(parameterMap.toSeq: _*) + ((flip.parameter: Parameter[_]) -> Seq(1.0, 0.0)) - trueMapping.remove(flip.parameter) - falseMapping.remove(flip.parameter) - trueMapping.put(flip.parameter, Seq(1.0, 0.0)) - falseMapping.put(flip.parameter, Seq(0.0, 1.0)) factor.set(List(i), (prob, trueMapping)) factor.set(List(1 - i), (1.0 - prob, falseMapping)) - List(factor) + constraints.map(convertFactor(_)) ++ List(factor) } private def makeFactors(bin: ParameterizedBinomialFixedNumTrials): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { + val origFactor = Factory.makeFactorsForElement(bin, false, true) + val (constraints, nonconstraints) = partitionConstraintFactors(origFactor) + val binVar = Variable(bin) val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(binVar), semiring) val prob = bin.parameter.MAPValue.asInstanceOf[Double] - val mappings = binVar.range.map(i => (i, mutable.Map(parameterMap.toSeq: _*))) + val mappings = binVar.range.map(i => (i, Map(parameterMap.toSeq: _*))) for { - (ext, map) <- mappings + ext <- binVar.range if (ext.isRegular) } { val i = ext.value - map.remove(bin.parameter) - map.put(bin.parameter, Seq(i, bin.numTrials - i)) + val kv: (Parameter[_], Seq[Double]) = (bin.parameter, Seq[Double](i, bin.numTrials - i)) + val map = Map(parameterMap.toSeq: _*) + kv val density = binomial(bin.numTrials, i) * pow(prob, i) * pow(1 - prob, bin.numTrials - i) val index = binVar.range.indexOf(ext) factor.set(List(index), (density, map)) } - List(factor) + constraints.map(convertFactor(_)) ++ List(factor) } - private def makeSimpleDistribution[T](target: Variable[T], probs: List[Double]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(target), semiring) - for { (prob, index) <- probs.zipWithIndex } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - factor.set(List(index), (prob, rowMapping)) - } - factor - } + private def makeSimpleDistributionForParameterized[T](target: Variable[T], probs: List[Double], select: ParameterizedSelect[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { + val origFactor = Factory.makeFactorsForElement(select, false, true) + val (constraints, nonconstraints) = partitionConstraintFactors(origFactor) - private def makeSimpleDistributionForParameterized[T](target: Variable[T], probs: List[Double], select: ParameterizedSelect[T]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(target), semiring) //For each outcome val unzippedClauses = select.clauses.unzip for { (prob, probindex) <- probs.zipWithIndex } { - //Row is a vector of zeros for all parameters - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - //Remove the default entry for this parameter - rowMapping.remove(select.parameter) //The index in the parameter vector doesn't necessarily match the range index. //It must be retrieved from the target. val varIndex = unzippedClauses._2.indexOf(target.range(probindex).value) - val entry = select.parameter.zeroSufficientStatistics.updated(varIndex, 1.0) - - rowMapping.put(select.parameter, entry) + //Row is a vector of zeros for all parameters + val rowMapping = Map(parameterMap.toSeq: _*) + ((select.parameter: Parameter[_]) -> entry) factor.set(List(probindex), (prob, rowMapping)) } - factor - } - - private def makeComplexDistribution[T](target: Variable[T], probElems: List[Element[Double]]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val probVars = probElems map (Variable(_)) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](probVars, List(target), semiring) - val probVals = probVars map (_.range) - for { indices <- factor.getIndices } { - val probIndices = indices.take(probElems.size).zipWithIndex - val unnormalized = probIndices map (pair => probVals(pair._2)(pair._1).value) - val normalized = normalize(unnormalized).toArray - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - - factor.set(indices, (normalized(indices.last), rowMapping)) - } - factor - } - - private def selectVarAndProbs[U, T](select: Select[U, T]): (Variable[T], List[U]) = { - val selectVar = Variable(select) - - val probs = for { xvalue <- selectVar.range } yield select.clauses.find(_._2 == xvalue.value).get._1 - (selectVar, probs) + factor :: constraints.map(convertFactor(_)) } private def selectVarAndProbs[U, T](select: ParameterizedSelect[T]): (Variable[T], List[Double]) = { @@ -172,232 +116,15 @@ class SufficientStatisticsFactor(parameterMap: immutable.Map[Parameter[_], Seq[D result } - private def makeFactors[T](select: AtomicSelect[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - - val (selectVar, probs) = selectVarAndProbs(select) - List(makeSimpleDistribution(selectVar, probs)) - } - - private def makeFactors[T](select: CompoundSelect[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val (selectVar, probs) = selectVarAndProbs(select) - List(makeComplexDistribution(selectVar, probs)) - } - private def makeFactors[T](select: ParameterizedSelect[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { val (selectVar, probs) = selectVarAndProbs(select) - List(makeSimpleDistributionForParameterized(selectVar, probs, select)) - } - - private def makeDontCares[U](factor: Factor[(Double, Map[Parameter[_], Seq[Double]])], - intermedIndex: Int, - overallVar: Variable[U], - outcomeVar: Variable[U]): Unit = { - // If we don't care, we assign 1.0 to all combinations of the distVar and outcomeVar - for { - j <- 0 until overallVar.size - k <- 0 until outcomeVar.size - } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - factor.set(List(intermedIndex, j, k), (1.0, rowMapping)) - } - } - - private def makeCares[U](factor: Factor[(Double, Map[Parameter[_], Seq[Double]])], intermedIndex: Int, - overallVar: Variable[U], outcomeVar: Variable[U], choices: scala.collection.immutable.Set[U])(implicit mapper: PointMapper[U]): Unit = { - // We care to match up distVar with outcomeVar - for { - (overallVal, j) <- overallVar.range.zipWithIndex - (outcomeVal, k) <- outcomeVar.range.zipWithIndex - } { - - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val entry = if (overallVal.value == mapper.map(outcomeVal.value, choices)) (1.0, rowMapping); else (0.0, rowMapping) - factor.set(List(intermedIndex, j, k), entry) - } - } - - /** - * Make a conditional selector factor used in the decomposition of chain and other elements. - * A chain defines a factor over the parent element, each of the possible result elements of the chain, - * and the overall chain element. This can produce a very large factor when there are many result elements. - * This is solved by decomposing the chain factor into a product of factors, each of which contains the - * parent element, one of the result elements, and the overall chain element. - */ - def makeConditionalSelector[T, U](overallElem: Element[U], selector: Variable[T], - outcomeIndex: Int, resultElem: Element[U])(implicit mapper: PointMapper[U]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val overallVar = Variable(overallElem) - - val outcomeVar = Variable(resultElem) - - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(selector, outcomeVar), List(overallVar), semiring) - for { i <- 0 to selector.size } { - if (outcomeIndex == i) { - makeCares(factor, outcomeIndex, outcomeVar, overallVar, Values(overallElem.universe)(overallElem))(mapper) - } else { - makeDontCares(factor, i, outcomeVar, overallVar) - } - } - factor - } - - private def intermedAndClauseFactors[U, T](dist: Dist[U, T]): (Variable[Int], List[Factor[(Double, Map[Parameter[_], Seq[Double]])]]) = { - val intermed = new Variable(ValueSet.withoutStar((0 until dist.clauses.size).toSet)) - val clauseFactors = dist.outcomes.zipWithIndex map (pair => - makeConditionalSelector(dist, intermed, pair._2, pair._1)) - (intermed, clauseFactors) - } - - private def makeFactors[T](dist: AtomicDist[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - - val (intermed, clauseFactors) = intermedAndClauseFactors(dist) - val intermedFactor = makeSimpleDistribution(intermed, dist.probs) - intermedFactor :: clauseFactors - } - - private def makeFactors[T](dist: CompoundDist[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - - val (intermed, clauseFactors) = intermedAndClauseFactors(dist) - val intermedFactor = makeComplexDistribution(intermed, dist.probs) - intermedFactor :: clauseFactors - } - - private def makeFactors[T, U](chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - - val chainMap: scala.collection.mutable.Map[T, Element[U]] = LazyValues(chain.universe).getMap(chain) - val parentVar = Variable(chain.parent) - parentVar.range.zipWithIndex map (pair => - makeConditionalSelector(chain, parentVar, pair._2, chainMap(pair._1.value))(mapper)) - } - - private def makeFactors[T, U](apply: Apply1[T, U])(implicit mapper: PointMapper[U]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - - val arg1Var = Variable(apply.arg1) - val resultVar = Variable(apply) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(arg1Var), List(resultVar), semiring) - val arg1Indices = arg1Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - result = mapper.map(apply.fn(arg1Val.value), Values(apply.universe)(apply)) - (resultVal, resultIndex) <- resultIndices - } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val entry = if (resultVal.value == result) (1.0, rowMapping); else (0.0, rowMapping) - factor.set(List(arg1Index, resultIndex), entry) - } - List(factor) - } - - private def makeFactors[T1, T2, U](apply: Apply2[T1, T2, U])(implicit mapper: PointMapper[U]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val resultVar = Variable(apply) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(arg1Var, arg2Var), List(resultVar), semiring) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - result = mapper.map(apply.fn(arg1Val.value, arg2Val.value), Values(apply.universe)(apply)) - (resultVal, resultIndex) <- resultIndices - } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - - val entry = if (resultVal.value == result) (1.0, rowMapping); else (0.0, rowMapping) - factor.set(List(arg1Index, arg2Index, resultIndex), entry) - } - List(factor) - } - - private def makeFactors[T1, T2, T3, U](apply: Apply3[T1, T2, T3, U])(implicit mapper: PointMapper[U]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val arg3Var = Variable(apply.arg3) - val resultVar = Variable(apply) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(arg1Var, arg2Var, arg3Var), List(resultVar), semiring) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val arg3Indices = arg3Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - (arg3Val, arg3Index) <- arg3Indices - result = mapper.map(apply.fn(arg1Val.value, arg2Val.value, arg3Val.value), Values(apply.universe)(apply)) - (resultVal, resultIndex) <- resultIndices - } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val entry = if (resultVal.value == result) (1.0, rowMapping); else (0.0, rowMapping) - factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), entry) - } - List(factor) - } - - private def makeFactors[T1, T2, T3, T4, U](apply: Apply4[T1, T2, T3, T4, U])(implicit mapper: PointMapper[U]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val arg3Var = Variable(apply.arg3) - val arg4Var = Variable(apply.arg4) - val resultVar = Variable(apply) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(arg1Var, arg2Var, arg3Var, arg4Var), List(resultVar), semiring) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val arg3Indices = arg3Var.range.zipWithIndex - val arg4Indices = arg4Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - (arg3Val, arg3Index) <- arg3Indices - (arg4Val, arg4Index) <- arg4Indices - result = mapper.map(apply.fn(arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value), Values(apply.universe)(apply)) - (resultVal, resultIndex) <- resultIndices - } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val entry = if (resultVal.value == result) (1.0, rowMapping); else (0.0, rowMapping) - factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), entry) - } - List(factor) - } - - private def makeFactors[T1, T2, T3, T4, T5, U](apply: Apply5[T1, T2, T3, T4, T5, U])(implicit mapper: PointMapper[U]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val arg1Var = Variable(apply.arg1) - val arg2Var = Variable(apply.arg2) - val arg3Var = Variable(apply.arg3) - val arg4Var = Variable(apply.arg4) - val arg5Var = Variable(apply.arg5) - val resultVar = Variable(apply) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(arg1Var, arg2Var, arg3Var, arg4Var, arg5Var), List(resultVar), semiring) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val arg3Indices = arg3Var.range.zipWithIndex - val arg4Indices = arg4Var.range.zipWithIndex - val arg5Indices = arg5Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - (arg3Val, arg3Index) <- arg3Indices - (arg4Val, arg4Index) <- arg4Indices - (arg5Val, arg5Index) <- arg5Indices - result = mapper.map(apply.fn(arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value, arg5Val.value), Values(apply.universe)(apply)) - (resultVal, resultIndex) <- resultIndices - } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - - val entry = if (resultVal.value == result) (1.0, rowMapping); else (0.0, rowMapping) - factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), entry) - } - List(factor) + makeSimpleDistributionForParameterized(selectVar, probs, select) } private def makeFactors[T](inject: Inject[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { def blankRule(values: List[Any]) = { val resultValue :: inputValues = values - - val rowMapping = mutable.Map(parameterMap.toSeq: _*) + val rowMapping = Map(parameterMap.toSeq: _*) if (resultValue.asInstanceOf[List[T]].toList == inputValues) (1.0, rowMapping) else (0.0, rowMapping) } @@ -405,34 +132,36 @@ class SufficientStatisticsFactor(parameterMap: immutable.Map[Parameter[_], Seq[D def parameterRule(values: List[T], p: ParameterizedVariable[T]) = { val resultValue :: inputValues = values - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - p.element.parameters.foreach(rowMapping.remove(_)) - - if (resultValue.asInstanceOf[List[T]].toList == inputValues) { - for (pr <- p.element.parameters) { - rowMapping.put(pr, pr.sufficientStatistics(resultValue)) - } - } else { - for (pr <- p.element.parameters) { - rowMapping.put(pr, pr.sufficientStatistics(resultValue)) + val rowMapping = Map(parameterMap.toSeq: _*) ++ { + if (resultValue.asInstanceOf[List[T]].toList == inputValues) { + for (pr <- p.element.parameters) yield { + (pr, pr.sufficientStatistics(resultValue)) + } + } else { + for (pr <- p.element.parameters) yield { + (pr, pr.sufficientStatistics(resultValue)) + } } } if (resultValue.asInstanceOf[List[T]].toList == inputValues) (1.0, rowMapping) else (0.0, rowMapping) } + val origFactor = Factory.makeFactorsForElement(inject, false, true) + val (constraints, nonconstraints) = partitionConstraintFactors(origFactor) + val inputVariables = inject.args map (Variable(_)) val resultVariable = Variable(inject) - //val variables = inputresultVariable :: inputVariables val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](inputVariables, List(resultVariable), semiring) val variables = factor.variables - + val ranges: List[(List[(Any, Int)], Int)] = List() - val mapping = Map.empty[Int, Variable[_]] - for (v <- variables) { - mapping.put(v.id, v) - ranges :: List((v.range.zipWithIndex, v.id)) + val mapping = Map.empty[Int, Variable[_]] ++ { + for (v <- variables) yield { + ranges :: List((v.range.zipWithIndex, v.id)) + (v.id, v) + } } val newRanges: List[List[(Any, Int, Int)]] = List() @@ -460,174 +189,37 @@ class SufficientStatisticsFactor(parameterMap: immutable.Map[Parameter[_], Seq[D } - List(factor) - } - - private def convertProbFactor(probFactor: Factor[Double]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val result = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](probFactor.parents, probFactor.output, semiring) - for { indices <- result.getIndices } { - result.set(indices, (probFactor.get(indices), mutable.Map(parameterMap.toSeq: _*))) - } - result + factor :: constraints.map(convertFactor(_)) } - private def concreteFactors[T](elem: Element[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = + /** + * Create the probabilistic factors associated with an element. This method is memoized. + */ + def make(elem: Element[_]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { elem match { - case c: Constant[_] => makeFactors(c) - case f: AtomicFlip => makeFactors(f) - case f: CompoundFlip => makeFactors(f) case f: ParameterizedFlip => makeFactors(f) case s: ParameterizedSelect[_] => makeFactors(s) - case ab: AtomicBinomial => Factory.concreteFactors(ab).map(convertProbFactor(_)) case b: ParameterizedBinomialFixedNumTrials => makeFactors(b) - case s: AtomicSelect[_] => makeFactors(s) - case s: CompoundSelect[_] => makeFactors(s) - case d: AtomicDist[_] => makeFactors(d) - case d: CompoundDist[_] => makeFactors(d) - case c: Chain[_, _] => makeFactors(c) - case a: Apply1[_, _] => makeFactors(a) - case a: Apply2[_, _, _] => makeFactors(a) - case a: Apply3[_, _, _, _] => makeFactors(a) - case a: Apply4[_, _, _, _, _] => makeFactors(a) - case a: Apply5[_, _, _, _, _, _] => makeFactors(a) case i: Inject[_] => makeFactors(i) - // case f: ProbFactorMaker => - // Factory.concreteFactors(f).map(convertProbFactor(_)) - /*case p: Parameter[_] => makeFactors(p)*/ - case _ => throw new UnsupportedAlgorithmException(elem) - } - - private def makeAbstract[T](atomic: Atomic[T], abstraction: Abstraction[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val variable = Variable(atomic) - val values = variable.range - val densityMap = scala.collection.mutable.Map[T, Double]() - for { v <- values } { - val currentDensity = densityMap.getOrElse(v.value, 0.0) - densityMap.update(v.value, currentDensity + atomic.density(v.value)) - } - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(variable), semiring) - for { (v, i) <- values.zipWithIndex } { - val rowMapping = Map.empty[Parameter[_], Seq[Double]] ++ parameterMap - factor.set(List(i), (densityMap(v.value), rowMapping)) - } - List(factor) - } - - private def makeAbstract[T](elem: Element[T], abstraction: Abstraction[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = - elem match { - case atomic: Atomic[_] => makeAbstract(atomic, abstraction) - case apply: Apply1[_, _] => makeFactors(apply)(abstraction.scheme) - case apply: Apply2[_, _, _] => makeFactors(apply)(abstraction.scheme) - case apply: Apply3[_, _, _, _] => makeFactors(apply)(abstraction.scheme) //Applies are ok. - // In the case of a Chain, its pragmas are inherited by the expanded result elements. The abstraction will be - // taken into account when we generate factors for the result elements. - case chain: Chain[_, _] => makeFactors(chain)(abstraction.scheme) - case _ => throw new UnsupportedAlgorithmException(elem) - } - - private def makeNonConstraintFactors[T](elem: Element[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = - Abstraction.fromPragmas(elem.pragmas) match { - case None => concreteFactors(elem) - case Some(abstraction) => makeAbstract(elem, abstraction) - } - - private def makeConditionAndConstraintFactors[T](elem: Element[T]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = - elem.allConditions.map(makeConditionFactor(elem, _)) ::: elem.allConstraints.map(makeConstraintFactor(elem, _)) - - private def makeConditionFactor[T](elem: Element[T], cc: (T => Boolean, Element.Contingency)): Factor[(Double, Map[Parameter[_], Seq[Double]])] = - makeConstraintFactor(elem, (ProbConstraintType((t: T) => if (cc._1(t)) 1.0; else 0.0), cc._2)) - - private def makeConstraintFactor[T](elem: Element[T], cc: (T => Double, Element.Contingency)): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val (constraint, contingency) = cc - contingency match { - case List() => makeUncontingentConstraintFactor(elem, constraint) - case first :: rest => makeContingentConstraintFactor(elem, constraint, first, rest) - } - } - - private def makeUncontingentConstraintFactor[T](elem: Element[T], constraint: T => Double): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val elemVar = Variable(elem) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](List(), List(elemVar), semiring) - for { (elemVal, index) <- elemVar.range.zipWithIndex } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val entry = (math.exp(constraint(elemVal.value)), rowMapping) - factor.set(List(index), entry) - } - factor - } - - private def makeContingentConstraintFactor[T](elem: Element[T], constraint: T => Double, firstConting: Element.ElemVal[_], restContinges: Element.Contingency): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val restFactor = makeConstraintFactor(elem, (constraint, restContinges)) - extendConstraintFactor(restFactor, firstConting) - } - - private def extendConstraintFactor(restFactor: Factor[(Double, Map[Parameter[_], Seq[Double]])], firstConting: Element.ElemVal[_]): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - // The extended factor is obtained by getting the underlying factor and expanding each row so that the row only provides its entry if the contingent variable takes - // on the appropriate value, otherwise the entry is 1 - val Element.ElemVal(firstElem, firstValue) = firstConting - val firstVar = Variable(firstElem) - val firstValues = firstVar.range - val numFirstValues = firstValues.size - val matchingIndex: Int = firstValues.indexOf(Regular(firstValue)) - val resultFactor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](firstVar :: restFactor.parents, restFactor.output, semiring) - for { restIndices <- restFactor.getIndices } { - val restEntry = restFactor.get(restIndices)._1 - for { firstIndex <- 0 until numFirstValues } { - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val resultEntry = if (firstIndex == matchingIndex) restEntry; else 1.0 - resultFactor.set(firstIndex :: restIndices, (resultEntry, rowMapping)) + case _ => { + val origFactor = Factory.makeFactorsForElement(elem) + origFactor.map(convertFactor(_)) } } - resultFactor - } - - private val factorCache = scala.collection.mutable.Map[Element[_], List[Factor[(Double, Map[Parameter[_], Seq[Double]])]]]() - - /** - * Create the probabilistic factors associated with an element. This method is memoized. - */ - def make(elem: Element[_]): List[Factor[(Double, Map[Parameter[_], Seq[Double]])]] = { - val nonConstraintFactors = - factorCache.get(elem) match { - case Some(l) => l - case None => - val result = makeNonConstraintFactors(elem) - factorCache += elem -> result - elem.universe.register(factorCache) - result - } - makeConditionAndConstraintFactors(elem) ::: nonConstraintFactors } - /** - * Removes the factors for the specified element from the cache. - */ - def removeFactors(elem: Element[_]) { factorCache -= elem } - /** - * Removes the factors for all elements from the cache. - */ - def removeFactors() { factorCache.clear } - /** * Create the probabilistic factor encoding the probability of evidence in the dependent universe as a function of the * values of variables in the parent universe. The third argument is the the function to use for computing * probability of evidence in the dependent universe. It is assumed that the definition of this function will already contain the * right evidence. */ - def makeDependentFactor(parentUniverse: Universe, + def makeDependentFactor(cc: ComponentCollection, parentUniverse: Universe, dependentUniverse: Universe, probEvidenceComputer: () => Double): Factor[(Double, Map[Parameter[_], Seq[Double]])] = { - val uses = dependentUniverse.parentElements filter (_.universe == parentUniverse) - def rule(values: List[Any]) = { - for { (elem, value) <- uses zip values } { elem.value = value.asInstanceOf[elem.Value] } - val rowMapping = mutable.Map(parameterMap.toSeq: _*) - val result = probEvidenceComputer() - (result, rowMapping) - } - val variables = uses map (Variable(_)) - val factor = Factory.defaultFactor[(Double, Map[Parameter[_], Seq[Double]])](variables, List(), semiring) - factor.fillByRule(rule _) - factor + val factor = Factory.makeDependentFactor(cc, parentUniverse, dependentUniverse, probEvidenceComputer) + convertFactor(factor) + } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ElementSampler.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ElementSampler.scala index f330d432..234e9a0e 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ElementSampler.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ElementSampler.scala @@ -25,7 +25,7 @@ import scala.collection.mutable.Map abstract class ElementSampler(target: Element[_]) extends BaseUnweightedSampler(target.universe, target) { def sample(): (Boolean, Sample) = { - Forward(target) + Forward(target) (true, Map[Element[_], Any](target -> target.value)) } @@ -77,6 +77,7 @@ class OneTimeElementSampler(target: Element[_], myNumSamples: Int) doInitialize() super.run() update + //universe.clearTemporaries } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Forward.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Forward.scala index e1b73a16..5644c5e3 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Forward.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Forward.scala @@ -14,6 +14,13 @@ package com.cra.figaro.algorithm.sampling import com.cra.figaro.language._ +import com.cra.figaro.library.cache.Cache +import com.cra.figaro.library.cache.NoCache + +class ForwardWeighter(universe: Universe, cache: Cache) extends LikelihoodWeighter(universe, cache) { + override def rejectionAction() = () + override def setObservation(element: Element[_], obs: Option[_]) = {} +} /** * A forward sampler that generates a state by generating values for elements, making sure to generate all the @@ -21,78 +28,37 @@ import com.cra.figaro.language._ */ object Forward { /** - * Sample the universe by generating a value for each element of the universe. + * Sample the universe by generating a value for each element of the universe. Return a cache object. */ - def apply(implicit universe: Universe): Unit = apply(false)(universe) - - def apply(useObservation: Boolean)(implicit universe: Universe): Unit = { - // avoiding recursion - var state = Set[Element[_]]() - var elementsRemaining = universe.activeElements - while (!elementsRemaining.isEmpty) { - if (elementsRemaining.head.active) state = sampleInState(elementsRemaining.head, state, universe, useObservation) - elementsRemaining = elementsRemaining.tail - } - } - - def apply[T](element: Element[T], useObservation: Boolean = false) = { - sampleInState(element, Set[Element[_]](), element.universe, useObservation) + def apply(universe: Universe): Double = { + apply(universe, new NoCache(universe)) } - private type State = Set[Element[_]] + /** + * Sample the universe by generating a value for each element of the universe, and provide a cache object. Return a cache object. + */ + def apply(universe: Universe, cache: Cache): Double = { + val lw = new ForwardWeighter(universe, cache) + try { + lw.computeWeight(universe.activeElements) + } catch { + case Importance.Reject => Double.NegativeInfinity + } + } - /* - * To allow this algorithm to be used for dependent universes, we make sure elements in a different universe are not - * sampled. + /** + * Sample only part of the model originating from a single element */ - private def sampleInState[T](element: Element[T], state: State, universe: Universe, useObservation: Boolean): State = { - if (element.universe != universe || (state contains element)) state - else { - val (state1, sampledValue) = { - element match { - case d: Dist[_, _] => - val state1 = - d match { - case dc: CompoundDist[_] => - // avoiding recursion - var resultState = state - var probsRemaining = dc.probs - while (!probsRemaining.isEmpty) { - resultState = sampleInState(probsRemaining.head, resultState, universe, useObservation) - probsRemaining = probsRemaining.tail - } - resultState - case _ => state - } - val rand = d.generateRandomness() - val index = d.selectIndex(rand) - val state2 = sampleInState(d.outcomeArray(index), state1, universe, useObservation) - (state2, d.finishGeneration(index)) - case c: Chain[_, _] => - val state1 = sampleInState(c.parent, state, universe, useObservation) - val result = c.get(c.parent.value) - val state2 = sampleInState(result, state1, universe, useObservation) - (state2, result.value) - case _ => - // avoiding recursion - var state1 = state - var initialArgs = (element.args ::: element.elementsIAmContingentOn.toList).toSet - var argsRemaining = initialArgs - while (!argsRemaining.isEmpty) { - state1 = sampleInState(argsRemaining.head, state1, universe, useObservation) - val newArgs = element.args.filter(!initialArgs.contains(_)) - initialArgs = initialArgs ++ newArgs - argsRemaining = argsRemaining.tail ++ newArgs - } - element.generate - (state1, element.value) - } - } - element.value = (useObservation, element.observation) match { - case (true, Some(v)) => v - case _ => sampledValue.asInstanceOf[T] - } - state1 + element + def apply[T](element: Element[T]): Double = { + val noCache = new NoCache(element.universe) + val lw = new ForwardWeighter(element.universe, noCache) + val weight = try { + lw.computeWeight(List(element)) + } catch { + case Importance.Reject => Double.NegativeInfinity } + element.universe.deregister(noCache) + weight } + } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Importance.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Importance.scala index b70ef303..8569e144 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Importance.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/Importance.scala @@ -20,6 +20,9 @@ import scala.annotation.tailrec import scala.collection.mutable.{ Set, Map } import com.cra.figaro.experimental.particlebp.AutomaticDensityEstimator import com.cra.figaro.algorithm.factored.ParticleGenerator +import com.cra.figaro.algorithm.sampling.parallel.ParImportance +import com.cra.figaro.library.cache.PermanentCache +import com.cra.figaro.library.collection.Container /** * Importance samplers. @@ -28,55 +31,12 @@ abstract class Importance(universe: Universe, targets: Element[_]*) extends WeightedSampler(universe, targets: _*) { import Importance.State -/* - * Likelihood weighting works by propagating observations through Dists and Chains - * to the variables they depend on. If we don't make sure we sample those Dists and - * Chains first, we may end up sampling those other elements without the correct - * observations. To avoid this, we keep track of all these dependencies. - * The dependencies map contains all the elements that could propagate an - * observation to any given element. - * Note: the dependencies map is only concerned with active elements that are present - * at the beginning of sampling (even though we get a new active elements list each sample). - * Temporary elements will always be created after the element that could propagate - * an observation to them, because that propagation has to go through a permanent - * element. - * Therefore, we can generate the dependencies map once before all the samples are generated. - */ - - // Calling Values on a continuous element requires that a ParticleGenerator exist in the universe. - // For a continuous element, we don't actually want to generate any dependencies, so we have the - // ParticleGenerator return zero sample. - ParticleGenerator(universe, new AutomaticDensityEstimator, 1, 0) - private val dependencies = scala.collection.mutable.Map[Element[_], Set[Element[_]]]() - private def makeDependencies() = { - for { - element <- universe.activeElements - } { - element match { - case d: Dist[_,_] => - for { o <- d.outcomes } { dependencies += o -> (dependencies.getOrElse(o, Set()) + d) } - //In principle, we should create a dependency from the result element of a chain to - //the chain. If the result element is a permanent element, this could matter. - //However, in most relevant cases (e.g., compound versions of atomic elements), the - //outcome element will be temporary and automatically generated after the chain, - //so we don't need the dependency. On the other hand, creating the dependency requires - //a call to values which is slow and dangerous. - // - //Unfortunately, the above intuition doesn't hold. The EM with importance tests fail - //to terminate unless we add dependencies to chains. We're still searching for a better - //way to determine the possible outcome elements than to call values on the parent. - case c: CachingChain[_,_] => - val outcomes = Values(universe)(c.parent).map(c.get(_)) - for { o <- outcomes } { dependencies += o -> (dependencies.getOrElse(o, Set()) + c) } - case _ => () - } - } - } - makeDependencies() + val lw = new LikelihoodWeighter(universe, new PermanentCache(universe)) private var numRejections = 0 private var logSuccessWeight = 0.0 private var numSamples = 0 + def getSamples() = numSamples override protected def resetCounts() { super.resetCounts() @@ -85,6 +45,13 @@ abstract class Importance(universe: Universe, targets: Element[_]*) numSamples = 0 } + override def kill () { + super.kill() + lw.clearCache() + lw.deregisterDependencies() + universe.deregisterAlgorithm(this) + } + /* * Produce one weighted sample of the given element. weightedSample takes into account conditions and constraints * on all elements in the Universe, including those that depend on this element. @@ -97,15 +64,15 @@ abstract class Importance(universe: Universe, targets: Element[_]*) val activeElements = universe.activeElements val resultOpt: Option[Sample] = try { - val state = State() - activeElements.foreach(e => if (e.active) sampleOne(state, e, None)) + val weight = lw.computeWeight(activeElements) val bindings = targets map (elem => elem -> elem.value) - Some((state.weight, Map(bindings: _*))) + Some((weight, Map(bindings: _*))) } catch { case Importance.Reject => None } + universe.clearTemporaries() resultOpt match { case Some(x) => logSuccessWeight = logSum(logSuccessWeight, x._1) @@ -117,150 +84,11 @@ abstract class Importance(universe: Universe, targets: Element[_]*) } } - /* - * Sample the value of an element. If it has already been assigned in the state, the current assignment is - * used and the state is unchanged. Also, an element in a different universe is not sampled; instead its state is - * used directly - * - * This is made private[figaro] to allow easy testing - */ - private[figaro] def sampleOne[T](state: State, element: Element[T], observation: Option[T]): T = { - /* - * We have to make sure to sample any elements this element depends on first so we can get the right - * observation for this element. - */ - dependencies.getOrElse(element, Set()).filter(!state.assigned.contains(_)).foreach(sampleOne(state, _, None)) - if (element.universe != universe || (state.assigned contains element)) { - element.value - } - else { - state.assigned += element - sampleFresh(state, element, observation) - } - } - - /* - * Sample a fresh value of an element, assuming it has not yet been assigned a value in the state. This sampling - * takes into account the condition and constraint on the element. If the condition is violated, the entire sampling - * process is rejected. This function returns the state including the new assignment to the element with the weight - * of the constraint multiplied in. - */ - private def sampleFresh[T](state: State, element: Element[T], observation: Option[T]): T = { - val fullObservation = (observation, element.observation) match { - case (None, None) => None - case (Some(obs), None) => Some(obs) - case (None, Some(obs)) => Some(obs) - case (Some(obs1), Some(obs2)) if obs1 == obs2 => Some(obs1) - case _ => throw Importance.Reject // incompatible observations - } - val value: T = - if (fullObservation.isEmpty || !element.isInstanceOf[HasDensity[_]]) { - val result = sampleValue(state, element, fullObservation) - if (!element.condition(result)) throw Importance.Reject - result - } else { - // Optimize the common case of an observation on an atomic element. - // This partially implements likelihood weighting by clamping the element to its - // desired value and multiplying the weight by the density of the value. - // This can dramatically reduce the number of rejections. - val obs = fullObservation.get - - sampleArgs(element, state: State, Set[Element[_]](element.args:_*)) - - // I'm not quite sure why we have to call sampleValue here when we're about to set the value of this element to obs. - // If I remove this call, the test "should correctly resample an element's arguments when the arguments change during samples" - // fails. - sampleValue(state, element, Some(obs)) - val density = element.asInstanceOf[HasDensity[T]].density(obs) - state.weight += math.log(density) - obs - } - element.value = value - state.weight += element.constraint(value) - value - } - - /* - * Sample the value of an element according to its generative model, without considering the condition or constraint. - * Since sampling the value of this element might also involve sampling the values of related elements, the state - * must be updated and returned. - * - * Most elements can simply be handled by sampling values for the arguments and generating values for this - * element. Dist is an exception, because not all the outcomes need to be generated, but we only know which one - * after we have sampled the randomness of the Dist. For this reason, we write special code to handle Dists. - * For Chain, we also write special code to avoid calling get twice. - * - * We propagate observations on Chains and Dists to their possible outcome elements. This ensures that instead of - * sampling these elements and then checking whether the observation is satisfied, we set their values to the - * required ones. This implements likelihood weighting and leads to faster convergence of the algorithm. - */ - private def sampleValue[T](state: State, element: Element[T], observation: Option[T]): T = { - element match { - case d: Dist[_, _] => - d match { - case dc: CompoundDist[_] => dc.probs foreach (sampleOne(state, _, None)) - case _ => () - } - val rand = d.generateRandomness() - val index = d.selectIndex(rand) - sampleOne(state, d.outcomeArray(index), observation) - d.value = d.finishGeneration(index) - d.value - case c: Chain[_, _] => - val parentValue = sampleOne(state, c.parent, None) - val next = c.get(parentValue) - c.value = sampleOne(state, next, observation) - c.value - case f: CompoundFlip => - val probValue = sampleOne(state, f.prob, None) - observation match { - case Some(true) => - state.weight += math.log(probValue) - true - case Some(false) => - state.weight += math.log(1 - probValue) - false - case _ => - val result = random.nextDouble() < probValue - f.value = result - result - } - case f: ParameterizedFlip => - val probValue = sampleOne(state, f.parameter, None) - - observation match { - case Some(true) => - true - case Some(false) => - false - case _ => - val result = random.nextDouble() < probValue - f.value = result - result - } - - case _ => - val args = (element.args ::: element.elementsIAmContingentOn.toList) - sampleArgs(element, state: State, Set[Element[_]](args:_*)) - element.randomness = element.generateRandomness() - element.value = element.generateValue(element.randomness) - element.value - } - } - - @tailrec - private def sampleArgs(element: Element[_], state: State, args: Set[Element[_]]): Unit = { - args foreach (sampleOne(state, _, None)) - val newArgs = element.args.filter(!state.assigned.contains(_)) - if (newArgs.nonEmpty) - sampleArgs(element, state, Set[Element[_]](newArgs: _*)) - } - - /** + /** * The computed probability of evidence. */ def logProbEvidence: Double = { - logSuccessWeight - Math.log(numSamples + numRejections) + logSuccessWeight - Math.log(numSamples + numRejections) } } @@ -288,29 +116,29 @@ object Importance { * using the given number of samples. */ def apply(myNumSamples: Int, targets: Element[_]*)(implicit universe: Universe) = - new Importance(universe, targets: _*) with OneTimeProbQuerySampler { + new Importance(universe, targets: _*) with OneTimeProbQuerySampler with ProbEvidenceQuery { val numSamples = myNumSamples - /** - * Use one-time sampling to compute the probability of the given named evidence. - * Takes the conditions and constraints in the model as part of the model definition. - * This method takes care of creating and running the necessary algorithms. - */ - def probabilityOfEvidence(evidence: List[NamedEvidence[_]]): Double = { - val logPartition = logProbEvidence - universe.assertEvidence(evidence) - if (active) kill() - start() - Math.exp(logProbEvidence - logPartition) - } + /** + * Use one-time sampling to compute the probability of the given named evidence. + * Takes the conditions and constraints in the model as part of the model definition. + * This method takes care of creating and running the necessary algorithms. + */ + override def probabilityOfEvidence(evidence: List[NamedEvidence[_]]): Double = { + val logPartition = logProbEvidence + universe.assertEvidence(evidence) + if (active) kill() + start() + Math.exp(logProbEvidence - logPartition) + } - } + } /** * Use IS to compute the probability that the given element satisfies the given predicate. */ - def probability[T](target: Element[T], predicate: T => Boolean): Double = { - val alg = Importance(10000, target) + def probability[T](target: Element[T], predicate: T => Boolean)(implicit universe: Universe): Double = { + val alg = Importance(10000, target)(universe) alg.start() val result = alg.probability(target, predicate) alg.kill() @@ -320,7 +148,24 @@ object Importance { /** * Use IS to compute the probability that the given element has the given value. */ - def probability[T](target: Element[T], value: T): Double = - probability(target, (t: T) => t == value) + def probability[T](target: Element[T], value: T)(implicit universe: Universe): Double = + probability(target, (t: T) => t == value)(universe) + + /** + * Use IS to sample the joint posterior distribution of several variables + */ + def sampleJointPosterior(targets: Element[_]*)(implicit universe: Universe): Stream[List[Any]] = { + val jointElement = Container(targets: _*).foldLeft(List[Any]())((l: List[Any], i: Any) => l :+ i) + val alg = Importance(10000, jointElement)(universe) + alg.start() + val posterior = alg.sampleFromPosterior(jointElement) + alg.kill() + posterior + } + + /** + * The parallel implementation of IS + */ + def par = ParImportance } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/LikelihoodWeighter.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/LikelihoodWeighter.scala new file mode 100644 index 00000000..cb96b185 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/LikelihoodWeighter.scala @@ -0,0 +1,233 @@ +/* + * LikelihoodWeighter.scala + * Likelihood weighting works by propagating observations through Dists and Chains to the variables they depend on. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.sampling + +import com.cra.figaro.language._ +import scala.annotation.tailrec +import com.cra.figaro.library.cache.Cache +import scala.collection.mutable.Set + +/* + * Likelihood weighting works by propagating observations through Dists and Chains + * to the variables they depend on. If we don't make sure we sample those Dists and + * Chains first, we may end up sampling those other elements without the correct + * observations. To avoid this, we keep track of all these dependencies. + * The dependencies map contains all the elements that could propagate an + * observation to any given element. + * To avoid calling values on Chains, we dynamically build the list of dependencies. + * If we encounter the result element of a chain before it's parent, we redo the sampling + * on the result and undo any weight associated with that element. + * + */ + +/** + * A class that implements sampling via likelihood weighting on a set of elements. + */ +class LikelihoodWeighter(universe: Universe, cache: Cache) { + + /* Stores the dependencies between elements for likelihood weighting */ + private[figaro] val dependencies = scala.collection.mutable.Map[Element[_], Set[Element[_]]]() + universe.register(dependencies) + + /** + * Clear the cache + */ + def clearCache() = { + cache.clear + } + + /** + * Deregister the map of dependencies between elements + */ + def deregisterDependencies() = { + universe.deregister(dependencies) + } + + /** + * Sample each element in the list of elements and compute their likelihood weight + */ + def computeWeight(elementsToVisit: List[Element[_]]): Double = { + // Do all dependencies first, then no need to check for them in the main traversal loop + val visited: Set[Element[_]] = Set() + val dependencyWeights = traverse(List(), dependencies.values.flatten.toList, 0.0, visited) + val remaining = elementsToVisit.filterNot(visited.contains(_)) + traverse(List(), remaining, dependencyWeights, visited) + } + + /* + * Traverse the elements in generative order, and return the weight + */ + @tailrec + private[figaro] final def traverse(currentStack: List[(Element[_], Option[_], Option[Element[_]])], + elementsToVisit: List[Element[_]], currentWeight: Double, visited: Set[Element[_]]): Double = { + + // If everything is empty, just return the weight + if (elementsToVisit.isEmpty && currentStack.isEmpty) { + currentWeight + + } // If the current stack is empty, take the head of the elements to visit as the next element + else if (currentStack.isEmpty) { + traverse(List((elementsToVisit.head, getObservation(elementsToVisit.head, None), None)), elementsToVisit.tail, currentWeight, visited) + + } // If the head of the stack has already been visited, is not active, or is in another universe, we don't need to do anything, go to the next element + else if (!currentStack.head._1.active || visited.contains(currentStack.head._1) || currentStack.head._1.universe != universe) { + traverse(currentStack.tail, elementsToVisit, currentWeight, visited += currentStack.head._1) + + } // Otherwise, we need to process the top element on the stack + else { + val (currElem, currObs, currResult) = currentStack.head + + currElem match { + case dist: Dist[_, _] => + val parents = dist match { + case dc: CompoundDist[_] => dc.probs.filterNot(visited.contains(_)).map(e => (e, getObservation(e, None), None)) + case _ => List() + } + + if (parents.nonEmpty) { + traverse(parents ::: currentStack, elementsToVisit, currentWeight, visited) + } else { + val rand = dist.generateRandomness() + val index = dist.selectIndex(rand) + val resultElement = if (currResult.isEmpty) dist.outcomeArray(index) else currResult.get + val nextHead = List((resultElement, getObservation(resultElement, currObs), None), (currElem, None, Some(resultElement))) + + if (visited.contains(resultElement) && currObs.nonEmpty) { + // we did this in the wrong order, and have to repropagate the result for likelihood weighting, and add it to the dependency map so we don't do this incorrectly next time + val elementsToRedo = findDependentElements(dist, resultElement) + val newWeight = (currentWeight /: elementsToRedo)((c: Double, n: Element[_]) => undoWeight(c, n)) + traverse(nextHead ::: currentStack.tail, elementsToRedo.toList ::: elementsToVisit, newWeight, visited --= elementsToRedo) + } else if (!visited.contains(resultElement)) { + traverse(nextHead ::: currentStack.tail, elementsToVisit, currentWeight, visited) + } else { + dist.value = resultElement.value.asInstanceOf[dist.Value] + dependencies.getOrElseUpdate(resultElement, Set()) += dist + val nextWeight = computeNextWeight(currentWeight, currElem, currObs) + traverse(currentStack.tail, elementsToVisit, nextWeight, visited += currElem) + } + } + + case chain: Chain[_, _] => + if (!visited.contains(chain.parent)) { + traverse((chain.parent, getObservation(chain.parent, None), None) :: currentStack, elementsToVisit, currentWeight, visited) + } else { + val next = if (currResult.isEmpty) cache(chain).get else currResult.get + val nextHead = List((next, getObservation(next, currObs), None), (currElem, None, Some(next))) + if (visited.contains(next) && currObs.nonEmpty) { + // we did this in the wrong order, and have to repropagate the result for likelihood weighting + val elementsToRedo = findDependentElements(chain, next) + val newWeight = (currentWeight /: elementsToRedo)((c: Double, n: Element[_]) => undoWeight(c, n)) + traverse(nextHead ::: currentStack.tail, elementsToRedo.toList ::: elementsToVisit, newWeight, visited --= elementsToRedo) + } else if (!visited.contains(next)) { + traverse(nextHead ::: currentStack.tail, elementsToVisit, currentWeight, visited) + } else { + chain match { + case cc: CachingChain[_, _] => if (!next.isTemporary) dependencies.getOrElseUpdate(next, Set()) += chain + case _ => () + } + chain.value = next.value.asInstanceOf[chain.Value] + val nextWeight = computeNextWeight(currentWeight, currElem, currObs) + traverse(currentStack.tail, elementsToVisit, nextWeight, visited += currElem) + } + } + case _ => + val args = (currElem.args ::: currElem.elementsIAmContingentOn.toList) + // Find all the arguments of the element that have not been visited + val remainingArgs = args.filterNot(visited.contains(_)).map(e => (e, getObservation(e, None), None)) + // if there are args unvisited, push those args to the top of the stack + if (remainingArgs.nonEmpty) { + traverse(remainingArgs ::: currentStack, elementsToVisit, currentWeight, visited) + } else { + // else, we can now process this element and move on to the next item + currElem.randomness = currElem.generateRandomness() + currElem.value = currElem.generateValue(currElem.randomness) + val nextWeight = computeNextWeight(currentWeight, currElem, currObs) + traverse(currentStack.tail, elementsToVisit, nextWeight, visited += currElem) + } + } + } + } + + + /* + * Finds the set of elements that need to be resampled when the likelihood weighting went in the wrong order + */ + private def findDependentElements(elem: Element[_], result: Element[_]) = { + val chainUsedBy = universe.usedBy(elem) + elem + val resultUseBy = universe.usedBy(result) + result + resultUseBy -- chainUsedBy + } + + /* + * Get the observation on an element, merging with any propagated observation from likelihood weighting + */ + protected def getObservation(element: Element[_], observation: Option[_]): Option[Any] = { + (observation, element.observation) match { + case (None, None) => None + case (Some(obs), None) => Some(obs) + case (None, Some(obs)) => Some(obs) + case (Some(obs1), Some(obs2)) if obs1 == obs2 => Some(obs1) + case _ => { // incompatible observations + rejectionAction() + None + } + } + } + + /* + * Compute the current weight of the model by incorporating the weight of the current sampled element + * If there is no observation on the element, the weight is the current weight plus the constraint on the element. + * If there is a condition on the element (not an observation), then we throw a rejection if the condition is not met. + * + * If there is an observation on the element, we implement likelihood weighting. If the element has a density + * function, we add the log density to the current weight. If it doesn't have a density, we check to see if + * it satisfies the observation + */ + private[figaro] def computeNextWeight(currentWeight: Double, element: Element[_], obs: Option[_]): Double = { + val nextWeight = if (obs.isEmpty) { + if (!element.condition(element.value)) rejectionAction() + currentWeight + } else { + element match { + case f: CompoundFlip => { + setObservation(element, obs) + if (obs.get.asInstanceOf[Boolean]) currentWeight + math.log(f.prob.value) + else currentWeight + math.log(1 - f.prob.value) + } + case e: HasDensity[_] => { + setObservation(element, obs) + val density = element.asInstanceOf[HasDensity[element.Value]].density(obs.asInstanceOf[Option[element.Value]].get) + currentWeight + math.log(density) + } + case _ => { + if (!element.condition(element.value)) rejectionAction() + currentWeight + } + } + } + nextWeight + element.constraint(element.value) + } + + protected def setObservation(element: Element[_], obs: Option[_]) = element.value = obs.get.asInstanceOf[element.Value] + + /* Action to take on a rejection. By default it throws an Importance.Reject exception, but this can be overriden for another behavior */ + protected def rejectionAction(): Unit = throw Importance.Reject + + /* + * Undo the application of this elements weight if we did likelihood weighting in the wrong order + */ + private def undoWeight(weight: Double, elem: Element[_]) = weight - computeNextWeight(0.0, elem, elem.observation) + +} + + diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastings.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastings.scala index 0f5189f2..3a689edb 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastings.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastings.scala @@ -20,6 +20,8 @@ import scala.collection.mutable.Map import scala.language.existentials import scala.math.log import scala.annotation.tailrec +import com.cra.figaro.library.cache._ +import com.cra.figaro.library.collection.Container /** * Metropolis-Hastings samplers. @@ -44,7 +46,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc protected var accepts = 0 protected var rejects = 0 - private val currentConstraintValues: Map[Element[_], Double] = Map() + protected val currentConstraintValues: Map[Element[_], Double] = Map() universe.register(currentConstraintValues) /** @@ -57,9 +59,11 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc */ var debug = false - private def newState: State = State(Map(), Map(), 0.0, 0.0, scala.collection.mutable.Set()) + private def newState: State = State(Map(), Map(), 0.0, 0.0, scala.collection.mutable.Set(), List()) - private val fastTargets = targets.toSet + protected val fastTargets = targets.toSet + + protected var chainCache: Cache = new MHCache(universe) /* * We continually update the values of elements while making a proposal. In order to be able to undo it, we need to @@ -68,33 +72,24 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc * We keep track of which elements do not have their condition satisfied by the new proposal. */ private def attemptChange[T](state: State, elem: Element[T]): State = { - val newValue = { - // Don't generate a new value for an observed element because it won't agree with the observation - // For a compound element we can't do this because we have to condition the arguments by the - // probability of generating the correct value. - if (elem.observation.isEmpty || !elem.isInstanceOf[Atomic[_]]) elem.generateValue(elem.randomness) - else elem.observation.get - } + + // Don't generate a new value for an observed element because it won't agree with the observation + // For a compound element we can't do this because we have to condition the arguments by the + // probability of generating the correct value. + val newValue = if (elem.observation.isEmpty || !elem.isInstanceOf[Atomic[_]]) { + chainCache(elem) match { + case None => elem.generateValue(elem.randomness) + case Some(result) => + if (result.value == null) result.generate + result.value + } + } else elem.observation.get + // if an old value is already stored, don't overwrite it - val newOldValues = - if (state.oldValues contains elem) state.oldValues; else state.oldValues + (elem -> elem.value) - if (elem.value != newValue) { - val newDissatisfied = - if (elem.condition(newValue)) state.dissatisfied -= elem; else state.dissatisfied += elem - elem.value = newValue - State(newOldValues, state.oldRandomness, state.proposalProb, state.modelProb, newDissatisfied) - } else { - // We need to make sure to add the element to the dissatisfied set if its condition is not satisfied, - // even if the value has not changed, because we compare the dissatisfied set with the old dissatisfied set - // when deciding whether to accept the proposal. - val newDissatisfied = - if (elem.condition(newValue)) { - state.dissatisfied - elem - } else { - state.dissatisfied + elem - } - State(newOldValues, state.oldRandomness, state.proposalProb, state.modelProb, newDissatisfied) - } + val newOldValues = if (state.oldValues contains elem) state.oldValues; else state.oldValues + (elem -> elem.value) + val newDissatisfied = if (elem.condition(newValue)) state.dissatisfied -= elem; else state.dissatisfied += elem + elem.value = newValue + State(newOldValues, state.oldRandomness, state.proposalProb, state.modelProb, newDissatisfied, state.visitOrder :+ elem) } private def propose[T](state: State, elem: Element[T]): State = { @@ -112,7 +107,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc } val newProb = state.proposalProb + log(proposalProb) elem.randomness = randomness - State(state.oldValues, newOldRandomness, newProb, state.modelProb + log(modelProb), state.dissatisfied) + State(state.oldValues, newOldRandomness, newProb, state.modelProb + log(modelProb), state.dissatisfied, state.visitOrder) } val result = attemptChange(state1, elem) if (debug) println("old randomness = " + oldRandomness + @@ -143,7 +138,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc val newOldRandomness2 = if (newOldRandomness1 contains elem2) newOldRandomness1 else newOldRandomness1 + (elem2 -> oldRandomness2) - State(state.oldValues, newOldRandomness2, state.proposalProb, state.modelProb, state.dissatisfied) + State(state.oldValues, newOldRandomness2, state.proposalProb, state.modelProb, state.dissatisfied, state.visitOrder) } val state2 = attemptChange(state1, elem1) val result = attemptChange(state2, elem2) @@ -169,11 +164,11 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc case FinalScheme(elem) => propose(state, elem()) case TypedScheme(first, rest) => val firstElem = first() - val state1 = propose(state, firstElem) + val state1 = propose(state, proposeChainCheck(firstElem)) continue(state1, rest(firstElem.value)) case UntypedScheme(first, rest) => val firstElem = first() - val state1 = propose(state, firstElem) + val state1 = propose(state, proposeChainCheck(firstElem)) continue(state1, rest) case ds: DisjointScheme => val (probs, schemes) = ds.choices.toList.unzip @@ -181,10 +176,15 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc runStep(state, choice()) case SwitchScheme(first, rest) => val (elem1, elem2) = first() - val state1 = switch(state, elem1, elem2) + val state1 = switch(state, proposeChainCheck(elem1), proposeChainCheck(elem2)) continue(state1, rest) } + private def proposeChainCheck(elem: Element[_]): Element[_] = { + val e = chainCache(elem) + if (e.isEmpty) elem else e.get.asInstanceOf[Element[_]] + } + protected def runScheme(): State = runStep(newState, proposalScheme) /* @@ -217,37 +217,26 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc * is not empty, we recursively update the intersecting elements. Once those updates are completed, * we update an element and move on to the next element in the set. */ - protected def updateMany[T](state: State, toUpdate: Set[Element[_]]): State = { - var returnState = state - var updatesLeft = toUpdate - while (!updatesLeft.isEmpty) { - // Check the intersection of an element's arguments with the updates that still need to occur - var argsRemaining = universe.uses(updatesLeft.head).intersect(updatesLeft) - while (!argsRemaining.isEmpty) { - // update the element's arguments first - returnState = updateManyHelper(returnState, argsRemaining.toSet) - argsRemaining = argsRemaining.tail + @tailrec + private def updateMany(state: State, currentStack: List[Element[_]], currentArgs: Set[Element[_]], updateQ: Set[Element[_]]): State = { + + if (currentStack.isEmpty && currentArgs.isEmpty && updateQ.isEmpty) state + + else if (currentStack.isEmpty && currentArgs.isEmpty && updateQ.nonEmpty) { + val argsRemaining = universe.uses(updateQ.head).intersect(updateQ.tail) + updateMany(state, List(updateQ.head), argsRemaining.toSet, updateQ.tail -- argsRemaining) + } else if (currentStack.nonEmpty && currentArgs.isEmpty) { + val newState = updateOne(state, currentStack.head) + updateMany(newState, currentStack.tail, currentArgs, updateQ) + } else { + val argsRemaining = universe.uses(currentArgs.head).intersect(currentArgs.tail) + if (argsRemaining.isEmpty) { + val newState = updateOne(state, currentArgs.head) + updateMany(newState, currentStack, currentArgs.tail, updateQ) + } else { + updateMany(state, currentArgs.head +: currentStack, currentArgs.tail, updateQ) } - // once the args are updated, update this element - returnState = updateOne(returnState, updatesLeft.head) - updatesLeft = updatesLeft.tail } - returnState - } - - /* - * A recursive function to work in conjunction with updateMany to check the order of the element - * updates. - */ - @tailrec - private def updateManyHelper(state: State, toUpdate: Set[Element[_]]): State = { - var returnState = state - var updatesLeft = toUpdate - var argsRemaining = universe.uses(updatesLeft.head).intersect(updatesLeft) - if (argsRemaining.isEmpty) { - returnState = updateOne(returnState, updatesLeft.head) - returnState - } else { updateManyHelper(returnState, argsRemaining.toSet) } } /* @@ -257,7 +246,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc protected def proposeAndUpdate(): State = { val state1 = runScheme() val updatesNeeded = state1.oldValues.keySet flatMap (elem => universe.usedBy(elem)) - updateMany(state1, updatesNeeded.toSet) + updateMany(state1, List(), Set(), updatesNeeded.toSet) } protected var dissatisfied: Set[Element[_]] = _ @@ -294,17 +283,19 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc protected def undo(state: State): Unit = { if (debug) println("Rejecting!\n") - state.oldValues foreach (setValue(_)) - state.oldRandomness foreach (setRandomness(_)) - - /* Have to call generateValue on chains after a rejection to restore the old resulting - * element. We can't do this above because we have to ensure the value of parent is restored before we - * do this. - */ - for ((elem, value) <- state.oldValues) { + + state.visitOrder.foreach { elem => elem match { - case c: Chain[_, _] => c.generateValue - case _ => + case c: Chain[_, _] => { + chainCache(c) match { + case Some(result) => c.value = result.value.asInstanceOf[c.Value] + case None => throw new AlgorithmException + } + } + case _ => { + if (state.oldRandomness.contains(elem)) elem.randomness = state.oldRandomness(elem).asInstanceOf[elem.Randomness] + elem.value = state.oldValues(elem).asInstanceOf[elem.Value] + } } } } @@ -333,7 +324,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc protected def mhStep(): State = { val newStateUnconstrained = proposeAndUpdate() val newState = State(newStateUnconstrained.oldValues, newStateUnconstrained.oldRandomness, - newStateUnconstrained.proposalProb, newStateUnconstrained.modelProb + computeScores, newStateUnconstrained.dissatisfied) + newStateUnconstrained.proposalProb, newStateUnconstrained.modelProb + computeScores, newStateUnconstrained.dissatisfied, newStateUnconstrained.visitOrder) if (decideToAccept(newState)) { accepts += 1 accept(newState) @@ -365,7 +356,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc protected def doInitialize(): Unit = { // Need to prime the universe to make sure all elements have a generated value - Forward(true)(universe) + Forward(universe, chainCache) initConstrainedValues() dissatisfied = universe.conditionedElements.toSet filter (!_.conditionSatisfied) for { i <- 1 to burnIn } mhStep() @@ -394,7 +385,7 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc for { i <- 1 to numSamples } { val newStateUnconstrained = proposeAndUpdate() val state1 = State(newStateUnconstrained.oldValues, newStateUnconstrained.oldRandomness, - newStateUnconstrained.proposalProb, newStateUnconstrained.modelProb + computeScores, newStateUnconstrained.dissatisfied) + newStateUnconstrained.proposalProb, newStateUnconstrained.modelProb + computeScores, newStateUnconstrained.dissatisfied, newStateUnconstrained.visitOrder) if (decideToAccept(state1)) { accepts += 1 // collect results for the new state and restore the original state @@ -415,6 +406,15 @@ abstract class MetropolisHastings(universe: Universe, proposalScheme: ProposalSc proposalCounts = savedProposalCounts (acceptanceRatio, successResults, proposalResults) } + + /** + * Clean up the sampler, freeing memory. + */ + override def cleanUp(): Unit = { + super.cleanUp() + universe.clearTemporaries() + chainCache.clear() + } } /** @@ -435,14 +435,6 @@ class AnytimeMetropolisHastings(universe: Universe, super.initialize() doInitialize() } - - /** - * Clean up the sampler, freeing memory. - */ - override def cleanUp(): Unit = { - universe.clearTemporaries() - super.cleanUp() - } } /** @@ -531,9 +523,22 @@ object MetropolisHastings { def probability[T](target: Element[T], value: T): Double = probability(target, (t: T) => t == value) + /** + * Use MH to sample the joint posterior distribution of several variables + */ + def sampleJointPosterior(targets: Element[_]*)(implicit universe: Universe): Stream[List[Any]] = { + val jointElement = Container(targets: _*).foldLeft(List[Any]())((l: List[Any], i: Any) => l :+ i) + val alg = MetropolisHastings(1000000, ProposalScheme.default, jointElement)(universe) + alg.start() + val posterior = alg.sampleFromPosterior(jointElement) + alg.kill() + posterior + } + private[figaro] case class State(oldValues: Map[Element[_], Any], oldRandomness: Map[Element[_], Any], proposalProb: Double, modelProb: Double, - dissatisfied: scala.collection.mutable.Set[Element[_]]) + dissatisfied: scala.collection.mutable.Set[Element[_]], + visitOrder: List[Element[_]]) } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastingsAnnealer.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastingsAnnealer.scala index 7f45620e..0fc2d387 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastingsAnnealer.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/MetropolisHastingsAnnealer.scala @@ -73,7 +73,7 @@ abstract class MetropolisHastingsAnnealer(universe: Universe, proposalScheme: Pr override protected def mhStep(): State = { val newStateUnconstrained = proposeAndUpdate() val newState = State(newStateUnconstrained.oldValues, newStateUnconstrained.oldRandomness, - newStateUnconstrained.proposalProb, newStateUnconstrained.modelProb + computeScores, newStateUnconstrained.dissatisfied) + newStateUnconstrained.proposalProb, newStateUnconstrained.modelProb + computeScores, newStateUnconstrained.dissatisfied, newStateUnconstrained.visitOrder ) if (decideToAccept(newState)) { accepts += 1 accept(newState) @@ -87,10 +87,10 @@ abstract class MetropolisHastingsAnnealer(universe: Universe, proposalScheme: Pr private def saveState: Map[Element[_], Any] = { bestEnergy = currentEnergy - Map(universe.activeElements.map(e => (e -> e.value)): _*) + Map(universe.permanentElements.map(e => (e -> e.value)): _*) } - override protected def initUpdates() = allLastUpdates = Map(universe.activeElements.map(e => (e -> (e.value, 0))): _*) + override protected def initUpdates() = allLastUpdates = Map(universe.permanentElements.map(e => (e -> (e.value, 0))): _*) override protected def updateTimesSeenForTarget[T](elem: Element[T], newValue: T): Unit = { allLastUpdates += (elem -> (newValue, sampleCount)) @@ -104,7 +104,7 @@ abstract class MetropolisHastingsAnnealer(universe: Universe, proposalScheme: Pr if (dissatisfied.isEmpty) { sampleCount += 1 - val toUpdate = if (currentEnergy > bestEnergy) { + val toUpdate = if (currentEnergy >= bestEnergy) { saveState } else Map[Element[_], Any]() (true, toUpdate) @@ -114,7 +114,7 @@ abstract class MetropolisHastingsAnnealer(universe: Universe, proposalScheme: Pr } override def doInitialize(): Unit = { - Forward(true)(universe) + Forward(universe, chainCache) initConstrainedValues() dissatisfied = universe.conditionedElements.toSet filter (!_.conditionSatisfied) currentEnergy = universe.constrainedElements.map(_.constraintValue).sum @@ -122,6 +122,8 @@ abstract class MetropolisHastingsAnnealer(universe: Universe, proposalScheme: Pr val nextState = mhStep() currentEnergy += nextState.modelProb } + initUpdates() + if (dissatisfied.nonEmpty) bestEnergy = Double.MinValue else bestEnergy = currentEnergy } def mostLikelyValue[T](target: Element[T]): T = { diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbEvidenceSampler.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbEvidenceSampler.scala index 99b8ddb1..500e1df8 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbEvidenceSampler.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbEvidenceSampler.scala @@ -17,6 +17,7 @@ import com.cra.figaro.algorithm._ import com.cra.figaro.language._ import scala.language.existentials import com.cra.figaro.util.logSum +import com.cra.figaro.library.cache.PermanentCache /** * Algorithm that computes probability of evidence using forward sampling. @@ -25,8 +26,8 @@ import com.cra.figaro.util.logSum */ abstract class ProbEvidenceSampler(override val universe: Universe, override val evidence: List[NamedEvidence[_]] = List[NamedEvidence[_]](), partition: Double = 1.0) extends ProbEvidenceAlgorithm with Sampler { - private var successWeight: Double = _ - private var totalWeight: Double = _ + protected var successWeight: Double = _ + protected var totalWeight: Double = _ //Logarithmic versions. @@ -35,20 +36,22 @@ abstract class ProbEvidenceSampler(override val universe: Universe, override val totalWeight = 0.0 } + + val lw = new LikelihoodWeighter(universe, new PermanentCache(universe)) /* * To protect against underflow, the probabilities are computed in log-space. */ protected def doSample(): Unit = { - Forward(universe) - - //Some values in log constraints may be negative infinity. - val weight = universe.constrainedElements.map(_.constraintValue).sum - - val satisfied = universe.conditionedElements forall (_.conditionSatisfied) - + totalWeight += 1 - if (satisfied) successWeight = logSum(successWeight, weight) + + try { + val weight = lw.computeWeight(universe.activeElements) + successWeight = logSum(successWeight, weight) + } catch { + case Importance.Reject => () + } universe.clearTemporaries() // avoid memory leaks } @@ -66,6 +69,14 @@ abstract class ProbEvidenceSampler(override val universe: Universe, override val if (!active) throw new AlgorithmInactiveException logComputedResult } + + override def cleanUp(): Unit = { + // Prevent memory leaks for when we run multiple ProbEvidenceSamplers in the same universe + super.cleanUp() + lw.clearCache() + lw.deregisterDependencies() + universe.deregisterAlgorithm(this) + } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbQuerySampler.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbQuerySampler.scala new file mode 100644 index 00000000..e50a8fae --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/ProbQuerySampler.scala @@ -0,0 +1,59 @@ +/* + * ProbQuerySampler.scala + * Sampling algorithms that use projected samples to compute conditional probabilities. + * + * Created By: Lee Kellogg (lkellog@cra.com) + * Creation Date: June 2, 2015 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.sampling + +import com.cra.figaro.algorithm.ProbQueryAlgorithm +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.BaseProbQueryAlgorithm +import scala.language.higherKinds + +/** + * Sampling algorithms that compute conditional probabilities of queries on elements, + * and that use the projection of all the samples of a target variable to calculate the + * distribution of that variable or the expectation of a function on that variable. + */ +trait ProbQuerySampler extends BaseProbQuerySampler[Element] { + val universe: Universe +} + +/** + * A base trait for sampling algorithms that compute conditional probabilities of queries, + * and that use the projection of all the samples of a target variable to calculate the + * distribution of that variable or the expectation of a function on that variable. + * Generic type U is either `Element` or `Reference`. + */ +trait BaseProbQuerySampler[U[_]] extends BaseProbQueryAlgorithm[U] { + + /** + * Total weight of samples taken, in log space + */ + def getTotalWeight: Double + + /** + * Return an estimate of the expectation of the function under the marginal probability distribution + * of the target. + */ + def computeExpectation[T](target: U[T], function: T => Double) = { + val contributions = computeProjection(target) map (pair => function(pair._1) * pair._2) + (0.0 /: contributions)(_ + _) + } + + /** + * Return an estimate of the expectation of the function under the marginal probability distribution + * of the target. + */ + def computeDistribution[T](target: U[T]): Stream[(Double, T)] = + computeProjection(target) map (_.swap) toStream + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/UnweightedSampler.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/UnweightedSampler.scala index f71690c3..83e885bf 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/UnweightedSampler.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/UnweightedSampler.scala @@ -13,12 +13,14 @@ package com.cra.figaro.algorithm.sampling -import com.cra.figaro.algorithm._ -import com.cra.figaro.language._ import scala.collection.mutable.Map import scala.language.existentials import scala.language.postfixOps +import com.cra.figaro.algorithm._ +import com.cra.figaro.language._ +import com.cra.figaro.util._ + /** * Samplers that use samples without weights. */ @@ -37,7 +39,7 @@ abstract class BaseUnweightedSampler(val universe: Universe, targets: Element[_] protected var sampleCount: Int = _ /** - * Number of samples taken. + * Number of samples taken */ def getSampleCount = sampleCount @@ -76,7 +78,7 @@ abstract class BaseUnweightedSampler(val universe: Universe, targets: Element[_] val s = sample() if (sampleCount == 0 && s._1) { initUpdates - } + } if (s._1) { sampleCount += 1 s._2 foreach (t => updateTimesSeenForTarget(t._1.asInstanceOf[Element[t._1.Value]], t._2.asInstanceOf[t._1.Value])) @@ -85,33 +87,43 @@ abstract class BaseUnweightedSampler(val universe: Universe, targets: Element[_] protected def update(): Unit = { sampleCount += 1 - targets.foreach(t => updateTimesSeenForTarget(t.asInstanceOf[Element[t.Value]], t.value)) + if (allLastUpdates.nonEmpty) targets.foreach(t => updateTimesSeenForTarget(t.asInstanceOf[Element[t.Value]], t.value)) sampleCount -= 1 } } -trait UnweightedSampler extends BaseUnweightedSampler with ProbQueryAlgorithm { - - protected def projection[T](target: Element[T]): List[(T, Double)] = { - val timesSeen = allTimesSeen.find(_._1 == target).get._2.asInstanceOf[Map[T, Int]] - timesSeen.mapValues(_ / sampleCount.toDouble).toList - } +trait UnweightedSampler extends BaseUnweightedSampler with ProbQuerySampler with StreamableProbQueryAlgorithm { /** - * Return an estimate of the expectation of the function under the marginal probability distribution - * of the target. + * Total weight of samples taken, in log space */ - def computeExpectation[T](target: Element[T], function: T => Double) = { - val contributions = projection(target) map (pair => function(pair._1) * pair._2) - (0.0 /: contributions)(_ + _) + def getTotalWeight: Double = math.log(getSampleCount) + + override protected[algorithm] def computeProjection[T](target: Element[T]): List[(T, Double)] = { + if (allLastUpdates.nonEmpty) { + val timesSeen = allTimesSeen.find(_._1 == target).get._2.asInstanceOf[Map[T, Int]] + timesSeen.mapValues(_ / sampleCount.toDouble).toList + } else { + println("Error: MH sampler did not accept any samples") + List() + } } + + def sampleFromPosterior[T](element: Element[T]): Stream[T] = { + def nextSample(posterior: List[(Double, T)]): Stream[T] = sampleMultinomial(posterior) #:: nextSample(posterior) - /** - * Return an estimate of the marginal probability distribution over the target that lists each element - * with its probability. - */ - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = - projection(target) map (_.swap) toStream + if (!allTimesSeen.contains(element)) { + throw new NotATargetException(element) + } else { + val (values, weights) = allTimesSeen(element).toList.unzip + if (values.isEmpty) throw new NotATargetException(element) + + if (sampleCount == 0) throw new ZeroTotalUnnormalizedProbabilityException + + val weightedValues = weights.map(w => w.toDouble/sampleCount.toDouble).zip(values.asInstanceOf[List[T]]) + nextSample(weightedValues) + } + } } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/WeightedSampler.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/WeightedSampler.scala index 93498516..dbec27d7 100644 --- a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/WeightedSampler.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/WeightedSampler.scala @@ -17,12 +17,12 @@ import com.cra.figaro.algorithm._ import com.cra.figaro.language._ import scala.collection.mutable.Map import scala.language.postfixOps -import com.cra.figaro.util.logSum +import com.cra.figaro.util._ /** * Samplers that use weighted samples. */ -abstract class WeightedSampler(override val universe: Universe, targets: Element[_]*) extends ProbQueryAlgorithm with Sampler { +abstract class WeightedSampler(override val universe: Universe, targets: Element[_]*) extends ProbQuerySampler with Sampler with StreamableProbQueryAlgorithm { lazy val queryTargets = targets.toList /** * A sample consists of a weight and a map from elements to their values. @@ -67,8 +67,7 @@ abstract class WeightedSampler(override val universe: Universe, targets: Element } protected def doSample(): Unit = { - val s = sample() - universe.clearTemporaries() + val s = sample() totalWeight = logSum(s._1, totalWeight) allWeightsSeen foreach (updateWeightSeenForTarget(s, _)) } @@ -76,24 +75,26 @@ abstract class WeightedSampler(override val universe: Universe, targets: Element protected def update(): Unit = { } - private def projection[T](target: Element[T]): List[(T, Double)] = { + override protected[algorithm] def computeProjection[T](target: Element[T]): List[(T, Double)] = { val weightSeen = allWeightsSeen.find(_._1 == target).get._2.asInstanceOf[Map[T, Double]] weightSeen.mapValues(s => math.exp(s - getTotalWeight)).toList } - - /** - * Return an estimate of the expectation of the function under the marginal probability distribution - * of the target. - */ - def computeExpectation[T](target: Element[T], function: T => Double) = { - val contributions = projection(target) map (pair => function(pair._1) * pair._2) - (0.0 /: contributions)(_ + _) + + def sampleFromPosterior[T](element: Element[T]): Stream[T] = { + def nextSample(posterior: List[(Double, T)]): Stream[T] = sampleMultinomial(posterior) #:: nextSample(posterior) + + val elementIndex = allWeightsSeen.indexWhere(_._1 == element) + if (elementIndex < 0) { + throw new NotATargetException(element) + } else { + val (values, weights) = allWeightsSeen(elementIndex)._2.toList.unzip + if (values.isEmpty) throw new NotATargetException(element) + + val logSum = logSumMany(weights) + if (logSum == Double.NegativeInfinity) throw new ZeroTotalUnnormalizedProbabilityException + + val weightedValues = weights.map(w => math.exp(w - logSum)).zip(values.asInstanceOf[List[T]]) + nextSample(weightedValues) + } } - - /** - * Return an estimate of the expectation of the function under the marginal probability distribution - * of the target. - */ - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = - projection(target) map (_.swap) toStream } diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParAnytime.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParAnytime.scala new file mode 100644 index 00000000..7c238872 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParAnytime.scala @@ -0,0 +1,41 @@ +/* + * ParAnytime.scala + * Parallel anytime algorithms + * + * Created By: Lee Kellogg (lkellog@cra.com) + * Creation Date: May 11, 2015 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.sampling.parallel + +import com.cra.figaro.algorithm.Anytime +import scala.collection.parallel.ParSeq + +/** + * Parallel anytime sampling algorithms. These algorithms have a parallel collection of algorithm instances + * that will do their work on separate threads, over separate universes. + */ +trait ParAnytime extends ParSamplingAlgorithm { + + protected val parAlgs: ParSeq[Anytime] + + /** + * Run one step of the algorithm on each thread. + */ + def runStep() = parAlgs foreach (_.runStep()) + + /** + * Call stopUpdate on algorithms in all threads. + */ + def stopUpdate() = parAlgs foreach (_.stopUpdate()) + + /** + * Release all resources from this anytime algorithm. + */ + def shutdown = parAlgs foreach (_.shutdown) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParImportance.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParImportance.scala new file mode 100644 index 00000000..2e1caa11 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParImportance.scala @@ -0,0 +1,97 @@ +/* + * ParImportance.scala + * Parallel importance sampling. + * + * Created By: Lee Kellogg (lkellog@cra.com) + * Creation Date: May 11, 2015 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.sampling.parallel + +import scala.collection.parallel.ParSeq +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.algorithm._ +import com.cra.figaro.language._ + +object ParImportance { + + /** + * Create a parallel anytime importance sampler with the given target query references. + * + * @param generator a function that returns a universe, with any evidence applied + * @param numThreads the number of threads to spawn + * @param targets references to the target elements + */ + def apply(generator: () => Universe, numThreads: Int, targets: Reference[_]*) = { + val algs = for ( _ <- 1 to numThreads) yield { + val universe = generator() + val elements = targets.map(universe.getElementByReference(_)) + Importance(elements: _*)(universe) + } + new ParSampler(algs, targets: _*) with ParAnytime { + + override val parAlgs: ParSeq[Importance with AnytimeProbQuerySampler] = algs.par + + } + } + + /** + * Create a parallel one-time importance sampler with the given target query references + * using the given number of samples. + * + * @param generator a function that returns a universe, with any evidence applied + * @param numThreads the number of threads to spawn + * @param numSamples the number of samples to take, total, across the threads + * @param targets references to the target elements + */ + def apply(generator: () => Universe, numThreads: Int, numSamples: Int, targets: Reference[_]*) = { + val algs = for ( _ <- 1 to numThreads) yield { + val universe = generator() + val elements = targets.map(universe.getElementByReference(_)) + Importance(numSamples / numThreads, elements: _*)(universe) + } + new ParSampler(algs, targets: _*) with ParOneTime with ProbEvidenceQuery { + + override val parAlgs: ParSeq[Importance with OneTimeProbQuerySampler with ProbEvidenceQuery] = algs.par + + /** + * Compute the probability of the given named evidence. + * Takes the conditions and constraints in the model as part of the model definition. + * This method takes care of creating and running the necessary algorithms. + */ + override def probabilityOfEvidence(evidence: List[NamedEvidence[_]]): Double = { + val poes = parAlgs.toList.map { alg => + alg.probabilityOfEvidence(evidence) + } + val total = getTotalWeight + val weightedPOEs = algs zip poes map { case (alg, poe) => + // raise from log space and apply to POE + poe * math.exp(alg.getTotalWeight - total) + } + weightedPOEs.sum + } + } + } + + /** + * Use parallel IS to compute the probability that the given reference element satisfies the given predicate. + */ + def probability[T](generator: () => Universe, numThreads: Int, target: Reference[T], predicate: T => Boolean): Double = { + val alg = ParImportance(generator, numThreads, 10000, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use parallel IS to compute the probability that the given reference element has the given value. + */ + def probability[T](generator: () => Universe, numThreads: Int, target: Reference[T], value: T): Double = + probability(generator, numThreads, target, (t: T) => t == value) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParOneTime.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParOneTime.scala new file mode 100644 index 00000000..a59f6333 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParOneTime.scala @@ -0,0 +1,36 @@ +/* + * ParOneTime.scala + * Parallel one-time algorithms. + * + * Created By: Lee Kellogg (lkellog@cra.com) + * Creation Date: May 11, 2015 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.sampling.parallel + +import scala.collection.parallel.ParSeq +import com.cra.figaro.algorithm.OneTime + +/** + * Parallel one-time sampling algorithms. These algorithms have multiple instances that execute across multiple threads. + */ +trait ParOneTime extends ParSamplingAlgorithm with OneTime { + + protected val parAlgs: ParSeq[OneTime] + + /** + * Run the algorithms in parallel, performing their computations to completion. + */ + override def run() = parAlgs foreach (_.run()) + + /** Specify delegation of algorithm management to ParSamplingAlgorithm **/ + override protected[algorithm] def doStart(): Unit = super[ParSamplingAlgorithm].doStart() + override protected[algorithm] def doStop(): Unit = super[ParSamplingAlgorithm].doStop() + override protected[algorithm] def doResume(): Unit = super[ParSamplingAlgorithm].doResume() + override protected[algorithm] def doKill(): Unit = super[ParSamplingAlgorithm].doKill() +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParSampler.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParSampler.scala new file mode 100644 index 00000000..a5e89576 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParSampler.scala @@ -0,0 +1,86 @@ +/* + * ParSampler.scala + * Parallel version of a sampling algorithm. + * + * Created By: Lee Kellogg (lkellogg@cra.com) + * Creation Date: Jun 2, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.sampling.parallel + +import com.cra.figaro.language._ +import com.cra.figaro.util +import com.cra.figaro.algorithm.sampling.BaseProbQuerySampler +import com.cra.figaro.algorithm.sampling.ProbQuerySampler +import scala.collection.parallel.ParSeq + +/** + * Parallel version of a sampling algorithm. Has a parallel collection of algorithm instances + * that will do its work on separate threads, over separate universes. Uses Scala's parallel + * collections to divide up the work, so it will work best if told to run on a number of threads + * less than or equal to the number of worker threads Scala creates when operating over parallel + * collections. + * + * This creates two major differences with how a user interacts with the algorithm. First of all, + * rather than defining a model on a universe and then starting the algorithm, the user must + * define a function that generates a universe and applies any evidence, and then pass that to + * the companion object to create the algorithm. The second major difference is that elements + * must be referred to using references, since each variable will exist as multiple elements + * across the different universes. + * + * One-time sampling will be faster, since it divides the work over the different threads. + * Anytime sampling should provide more accurate results, since it can take more samples over + * the same amount of time. Both cases will most likely require more memory, at least using the + * current implementation of WeightedSampler, which keeps track of all values and weights that + * have been sampled. When querying, a weighted combination of the results of the various threads + * is returned. This last step adds some overhead, which should be negligible as long as you are + * taking a large number of samples. + */ +abstract class ParSampler(algs: Seq[ProbQuerySampler], targets: Reference[_]*) + extends BaseProbQuerySampler[Reference] with ParSamplingAlgorithm { + + /** The query targets are references in this case **/ + override val queryTargets: Seq[Reference[_]] = targets.toSeq + + /** A parallel collection of algorithms **/ + protected val parAlgs: ParSeq[ProbQuerySampler] = algs.par + + /** Get an element from a reference **/ + protected def element[T](alg: ProbQuerySampler, element: Reference[T]): Element[T] = { + alg.universe.getElementByReference(element) + } + + /** Log sum of total weights of individual algorithms **/ + def getTotalWeight: Double = util.logSumMany(parAlgs.toList map (_.getTotalWeight)) + + override protected[algorithm] def computeProjection[T](target: Reference[T]): List[(T, Double)] = { + val algs = parAlgs.toList + val combinedTotalWeight = getTotalWeight + val flatWeightedProjections: List[(T, Double)] = algs flatMap { alg => + // raise weights out of log space + val algWeight = math.exp(alg.getTotalWeight - combinedTotalWeight) + + // projection is already raised from log space + val projection: List[(T, Double)] = alg.computeProjection(element(alg, target)) + + // apply weight + projection map { case (v, w) => (v, w * algWeight) } + } + val groupedByValue = flatWeightedProjections groupBy (_._1) + val combinedWeights = groupedByValue mapValues { (group: List[(T, Double)]) => + group.map(_._2).sum + } + combinedWeights.toList + } + + /** Methods from BaseProbQueryAlgorithm **/ + protected def doDistribution[T](target: Reference[T]) = computeDistribution(target) + protected def doExpectation[T](target: Reference[T], function: T => Double) = computeExpectation(target, function) + protected def doProbability[T](target: Reference[T], predicate: T => Boolean) = computeProbability(target, predicate) + override protected def doProjection[T](target: Reference[T]) = computeProjection(target) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParSamplingAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParSamplingAlgorithm.scala new file mode 100644 index 00000000..980f31cd --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/sampling/parallel/ParSamplingAlgorithm.scala @@ -0,0 +1,46 @@ +/* + * ParSamplingAlgorithm.scala + * Parallel sampling algorithms. + * + * Created By: Lee Kellogg (lkellog@cra.com) + * Creation Date: June 2, 2015 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.sampling.parallel + +import scala.collection.parallel.ParSeq +import com.cra.figaro.algorithm.Algorithm + +/** + * Parallel sampling algorithms. These algorithms have a parallel collection of algorithm instances + * that will do their work on separate threads, over separate universes. + */ +trait ParSamplingAlgorithm extends Algorithm { + + protected val parAlgs: ParSeq[Algorithm] + + private def call(func: (Algorithm) => Unit) = parAlgs foreach func + + /** + * Calls initialize() on all algorithms. + */ + override def initialize(): Unit = call(_.initialize()) + + /** + * Calls cleanUp() on all algorithms. + */ + override def cleanUp(): Unit = call(_.cleanUp()) + + protected[algorithm] def doStart(): Unit = call(_.doStart()) + + protected[algorithm] def doStop(): Unit = call(_.doStop()) + + protected[algorithm] def doResume(): Unit = call(_.doResume()) + + protected[algorithm] def doKill(): Unit = call(_.doKill()) +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/Bounds.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Bounds.scala similarity index 90% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/Bounds.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Bounds.scala index 021aab1f..4e1dade5 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/Bounds.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Bounds.scala @@ -11,7 +11,7 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured +package com.cra.figaro.algorithm.structured sealed abstract class Bounds case object Lower extends Bounds diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/ComponentCollection.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/ComponentCollection.scala similarity index 55% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/ComponentCollection.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/structured/ComponentCollection.scala index e9eafec5..bc108ab0 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/ComponentCollection.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/ComponentCollection.scala @@ -1,109 +1,143 @@ -/* - * ComponentCollection.scala - * A data structure that holds all the problem components used in a top-level problem and its subproblems. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured - -import com.cra.figaro.language.{Element, Chain} -import com.cra.figaro.util.memo -import com.cra.figaro.library.collection.MakeArray -import com.cra.figaro.library.collection.FixedSizeArray -import scala.collection.mutable.Map -import com.cra.figaro.algorithm.factored.factors.Variable - -/** - * A collection of problem components. This data structure manages all the components being used in the solution of a top-level - * problem and its nested subproblems. - */ - -/* - * Every element exists in at most one component. - * To create a new component for an element, you need to say what problem it belongs to. - */ -class ComponentCollection { - /** All the components in the collection, each associated with an element. */ - val components: Map[Element[_], ProblemComponent[_]] = Map() - - /** - * Intermediate variables defined during the construction of factors. - * These are not associated with any element or component and are to be eliminated wherever they appear. - */ - var intermediates: Set[Variable[_]] = Set() - - /** - * A map from a function and parent value to the associated subproblem. - */ - val expansions: Map[(Function1[_, Element[_]], _), NestedProblem[_]] = Map() - - /** - * Get the nested subproblem associated with a particular function and parent value. - * Checks in the cache if an expansion exists and creates one if necessary. - */ - private[structured] def expansion[P,V](function: Function1[P, Element[V]], parentValue: P): NestedProblem[V] = { - expansions.get((function, parentValue)) match { - case Some(p) => -// println("Cache hit: function = " + function.hashCode + ", parentValue = " + parentValue.hashCode + ": " + parentValue) - p.asInstanceOf[NestedProblem[V]] - case None => -// println("Cache miss: function = " + function.hashCode + ", parentValue = " + parentValue.hashCode + ": " + parentValue) - val result = new NestedProblem(this, function(parentValue)) - expansions += (function, parentValue) -> result - result - } - } - - /** - * Returns the problem component associated with a particular variable. - * Not valid for intermediate variables. - */ - val variableToComponent: Map[Variable[_], ProblemComponent[_]] = Map() - - /** Does the element have a component in this collection? */ - def contains[T](element: Element[T]): Boolean = - components.contains(element) - - /** - * Get the component associated with this element in this collection. - * Throws an exception if the element is not associated with any component. - */ - def apply[T](element: Element[T]): ProblemComponent[T] = components(element).asInstanceOf[ProblemComponent[T]] - /** - * Get the component associated with this element in this collection. - * Throws an exception if the element is not associated with any component. - */ - def apply[P,T](chain: Chain[P,T]): ChainComponent[P,T] = components(chain).asInstanceOf[ChainComponent[P,T]] - /** - * Get the component associated with this element in this collection. - * Throws an exception if the element is not associated with any component. - */ - def apply[T](makeArray: MakeArray[T]): MakeArrayComponent[T] = components(makeArray).asInstanceOf[MakeArrayComponent[T]] - - /** - * Add a component for the given element in the given problem to the component collection and return the component. - */ - private[structured] def add[T](element: Element[T], problem: Problem): ProblemComponent[T] = { - if (contains(element)) throw new RuntimeException("Element already has a component") - val component: ProblemComponent[T] = - element match { - case chain: Chain[_,T] => new ChainComponent(problem, chain) - case makeArray: MakeArray[_] => new MakeArrayComponent(problem, makeArray).asInstanceOf[ProblemComponent[T]] - case _ => new ProblemComponent(problem, element) - } - components += element -> component - problem.components ::= component - component - } - - private[structured] def remove[T](element: Element[T]) { - components -= element - } -} +/* + * ComponentCollection.scala + * A data structure that holds all the problem components used in a top-level problem and its subproblems. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured + +import com.cra.figaro.language._ +import com.cra.figaro.library.collection.MakeArray +import scala.collection.mutable.Map +import com.cra.figaro.algorithm.factored.factors._ +import scala.collection.mutable.HashMap +/** +* To speed up factor creation time, it's necessary to override the hashcode of component collections. +*/ +object ComponentHash { + var hashCodeState = 10 + def nextCode: Int = { + hashCodeState += 1 + hashCodeState + } +} + +/** + * A collection of problem components. This data structure manages all the components being used in the solution of a top-level + * problem and its nested subproblems. + */ + +/* + * Every element exists in at most one component. + * To create a new component for an element, you need to say what problem it belongs to. + */ +class ComponentCollection { + /** Indicates whether to create chain factors by decomposing the chain into several factors or a single factor + * This defaults to false since all the existing code a decomposition + */ + var useSingleChainFactor = false + + /** + * Maps a variable to the parents needed for creating blocks using Gibbs sampling. + * TODO: test if this variable causes memory leaks. + */ + val variableParents: Map[Variable[_], Set[Variable[_]]] = Map().withDefaultValue(Set()) + + /** All the components in the collection, each associated with an element. */ + val components: Map[Element[_], ProblemComponent[_]] = new HashMap[Element[_], ProblemComponent[_]]() { + override val hashCode = ComponentHash.nextCode + } + + /** + * Intermediate variables defined during the construction of factors. + * These are not associated with any element or component and are to be eliminated wherever they appear. + */ + var intermediates: Set[Variable[_]] = Set() + + /** + * A map from a function and parent value to the associated subproblem. + */ + val expansions: Map[(Function1[_, Element[_]], _), NestedProblem[_]] = Map() + + /** + * Get the nested subproblem associated with a particular function and parent value. + * Checks in the cache if an expansion exists and creates one if necessary. + */ + private[structured] def expansion[P, V](component: ExpandableComponent[P, V], function: Function1[P, Element[V]], parentValue: P): NestedProblem[V] = { + expansions.get((function, parentValue)) match { + case Some(p) => + p.asInstanceOf[NestedProblem[V]] + case None => + val result = new NestedProblem(this, component.expandFunction(parentValue)) + expansions += (function, parentValue) -> result + result + } + } + + /** + * Returns the problem component associated with a particular variable. + * Not valid for intermediate variables. + */ + val variableToComponent: Map[Variable[_], ProblemComponent[_]] = Map() + + /** Does the element have a component in this collection? */ + def contains[T](element: Element[T]): Boolean = + components.contains(element) + + /** + * Get the component associated with this element in this collection. + * Throws an exception if the element is not associated with any component. + */ + def apply[T](element: Element[T]): ProblemComponent[T] = components(element).asInstanceOf[ProblemComponent[T]] + /** + * Get the component associated with this element in this collection. + * Throws an exception if the element is not associated with any component. + */ + def apply[P, T](chain: Chain[P, T]): ChainComponent[P, T] = components(chain).asInstanceOf[ChainComponent[P, T]] + + /** + * Get the component associated with this element in this collection. + * Throws an exception if the element is not associated with any component. + */ + def apply[T](apply: Apply[T]): ApplyComponent[T] = components(apply).asInstanceOf[ApplyComponent[T]] + + /** + * Get the component associated with this element in this collection. + * Throws an exception if the element is not associated with any component. + */ + def apply[T](makeArray: MakeArray[T]): MakeArrayComponent[T] = components(makeArray).asInstanceOf[MakeArrayComponent[T]] + + /** + * Add a component for the given element in the given problem to the component collection and return the component. + */ + private[structured] def add[T](element: Element[T], problem: Problem): ProblemComponent[T] = { + if (problem.collection.contains(element)) { + val component = problem.collection(element) + if (component.problem != problem) throw new RuntimeException("Trying to add a component to a different problem") + component + } + else { + val component: ProblemComponent[T] = + element match { + case chain: Chain[_, T] => new ChainComponent(problem, chain) + case makeArray: MakeArray[_] => new MakeArrayComponent(problem, makeArray) + case apply: Apply[_] => new ApplyComponent(problem, apply) + case _ => new ProblemComponent(problem, element) + } + components += element -> component + problem.components ::= component + component + } + } + + private[structured] def remove[T](element: Element[T]) { + components -= element + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/Problem.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Problem.scala similarity index 73% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/Problem.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Problem.scala index de6d2278..b56edc03 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/Problem.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Problem.scala @@ -11,11 +11,11 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured +package com.cra.figaro.algorithm.structured -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.language.Element -import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy /** * A Problem defines an inference problem to be solved. @@ -25,7 +25,7 @@ import com.cra.figaro.algorithm.factored.factors.Variable * The targets are elements that appear in this problem that are visible outside. * They might be newly defined in this problem or they might be defined previously, but either way, they should not be eliminated. */ -class Problem(val collection: ComponentCollection, targets: List[Element[_]] = List()) { +class Problem(val collection: ComponentCollection, val targets: List[Element[_]] = List()) { /** * Components outside of this problem that appear in the solution to this problem. */ @@ -40,11 +40,20 @@ class Problem(val collection: ComponentCollection, targets: List[Element[_]] = L * Factors over globals produced by solving the problem. */ var solution: List[Factor[Double]] = List() - + + /** + * A map for each variable that indicates the value of the variable that is maximal for each possible value of the interface + * The support of each factor is over the product of the supports of the interface variables + * + */ + var recordingFactors: Map[Variable[_], Factor[_]] = Map() + /** * A flag indicating whether the problem has been solved. */ var solved: Boolean = false + + val componentsToVisit: scala.collection.mutable.Set[ProblemComponent[_]] = scala.collection.mutable.Set() /** * Add a component for the given element to this problem. @@ -60,6 +69,15 @@ class Problem(val collection: ComponentCollection, targets: List[Element[_]] = L contains(component.problem) & !targets.contains(component.element) } } + + /** + * Determines if a variable is in scope outside of this problem + */ + def global(variable: Variable[_]): Boolean = { + !collection.intermediates.contains(variable) && + !contains(collection.variableToComponent(variable).problem) + } + /** * Determines if this problem contains the given problem. @@ -75,24 +93,24 @@ class Problem(val collection: ComponentCollection, targets: List[Element[_]] = L otherProblem == this || components.exists(componentContains(_)) } + /** * Solve the problem defined by all the components' current factors. * This will also set the globals accordingly. * All components in this problem and contained subproblems should be eliminated in the solution. */ - def solve(algorithm: solver.Solver, bounds: Bounds = Lower) { - val targetComponents = targets.map(collection(_)) + def solve( strategy: SolvingStrategy, bounds: Bounds = Lower) { + val targetComponents = targets.map(collection(_)) val allFactors = components.flatMap(c => c.nonConstraintFactors ::: c.constraintFactors(bounds)) val allVariables = (Set[Variable[_]]() /: allFactors)(_ ++ _.variables) val (toEliminate, toPreserve) = allVariables.partition(internal(_)) globals = toPreserve.map(collection.variableToComponent(_)) - solution = algorithm(this, toEliminate, toPreserve, allFactors) + val (solution, recordingFactors) = strategy.solve(this, toEliminate, toPreserve, allFactors) + this.solution = solution + this.recordingFactors = recordingFactors solved = true toEliminate.foreach((v: Variable[_]) => { if (collection.intermediates.contains(v)) collection.intermediates -= v - // It's unsafe to remove the component for the element because it might appear in a reused - // version of the nested subproblem. -// else collection.remove(collection.variableToComponent(v).element) }) } diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/ProblemComponent.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/ProblemComponent.scala similarity index 65% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/ProblemComponent.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/structured/ProblemComponent.scala index 91467dab..994c222d 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/ProblemComponent.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/ProblemComponent.scala @@ -11,16 +11,16 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured +package com.cra.figaro.algorithm.structured -import com.cra.figaro.language.{Element, Chain} +import com.cra.figaro.language._ import com.cra.figaro.algorithm.lazyfactored.ValueSet -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.factored.factors.factory._ import com.cra.figaro.library.collection.MakeArray import com.cra.figaro.library.collection.FixedSizeArray import com.cra.figaro.algorithm.factored.ParticleGenerator -import com.cra.figaro.experimental.structured.factory.{Factory, ConstraintFactory} +import com.cra.figaro.language.Constant /* /** A component of a problem, created for a specific element. */ @@ -70,7 +70,7 @@ class ProblemComponent[Value](val problem: Problem, val element: Element[Value]) */ var nonConstraintFactors: List[Factor[Double]] = List() -/* + /* // The current belief about this component, used for belief propagation. var belief: Factor[Double] = _ //StarFactory.makeStarFactor(element)(0) @@ -93,9 +93,12 @@ class ProblemComponent[Value](val problem: Problem, val element: Element[Value]) * The range will include * based on argument ranges including * or any subproblem not being expanded.\ * */ - def generateRange(numValues: Int = ParticleGenerator.defaultTotalSamples) { - range = Range(this, numValues) - setVariable(new Variable(range)) + def generateRange(numValues: Int = ParticleGenerator.defaultMaxNumSamplesAtChain) { + val newRange = Range(this, numValues) + if ((newRange.hasStar ^ range.hasStar) || (newRange.regularValues != range.regularValues)) { + range = newRange + setVariable(new Variable(range)) + } } /** @@ -132,37 +135,49 @@ class ProblemComponent[Value](val problem: Problem, val element: Element[Value]) */ } +class ApplyComponent[Value](problem: Problem, element: Element[Value]) extends ProblemComponent(problem, element) { + private var applyMap = scala.collection.mutable.Map[Any, Value]() + def getMap() = applyMap + def setMap(m: scala.collection.mutable.Map[Any, Value]) = applyMap = m +} + /** * A problem component that provides an expand method. * @param parent the element according to whose values this component should be expanded * @param element the element to which this component corresponds */ abstract class ExpandableComponent[ParentValue, Value](problem: Problem, parent: Element[ParentValue], element: Element[Value]) -extends ProblemComponent(problem, element) -{ + extends ProblemComponent(problem, element) { /** * Expand for all values of the parent, based on the current range of the parent. */ def expand() { if (problem.collection.contains(parent)) { - for { parentValue <- problem.collection(parent).range.regularValues } { expand (parentValue) } + for { parentValue <- problem.collection(parent).range.regularValues } { expand(parentValue) } } } /** Expand for a particular parent value. */ def expand(parentValue: ParentValue): Unit + + val expandFunction: (ParentValue) => Element[Value] + + /** + * The subproblems nested inside this expandable component. + * They are created for particular parent values. + */ + var subproblems: Map[ParentValue, NestedProblem[Value]] = Map() } /** * A problem component created for a chain element. */ class ChainComponent[ParentValue, Value](problem: Problem, val chain: Chain[ParentValue, Value]) -extends ExpandableComponent[ParentValue, Value](problem, chain.parent, chain) { - /** - * The subproblems represent nested problems from chains. - * They are created for particular parent values. - */ - var subproblems: Map[ParentValue, NestedProblem[Value]] = Map() + extends ExpandableComponent[ParentValue, Value](problem, chain.parent, chain) { + + val elementsCreated: scala.collection.mutable.Set[Element[_]] = scala.collection.mutable.Set() ++ chain.universe.contextContents(chain) + + val expandFunction = chain.get _ /** * The subproblems are defined in terms of formal variables. @@ -175,43 +190,84 @@ extends ExpandableComponent[ParentValue, Value](problem, chain.parent, chain) { * Memoized. */ def expand(parentValue: ParentValue) { - val subproblem = problem.collection.expansion(chain.chainFunction, parentValue) + val subproblem = problem.collection.expansion(this, chain.chainFunction, parentValue) + val chainContextElements = chain.universe.contextContents(chain) + val remainingElements = chainContextElements.filter(!elementsCreated.contains(_)) + remainingElements.foreach(subproblem.add(_)) + elementsCreated ++= remainingElements subproblems += parentValue -> subproblem } + /* + * If all the subproblems have been eliminated completely and use no globals, we can use the new chain method. + */ + private[figaro] def allSubproblemsEliminatedCompletely: Boolean = { + for { + (parentValue, subproblem) <- subproblems + } { + val factors = subproblem.solution + val vars = factors.flatMap(_.variables) + val comps = vars.map(problem.collection.variableToComponent(_)) + if (comps.exists(!subproblem.components.contains(_))) return false // the factors contain a global variable + if (problem.collection(subproblem.target).problem != subproblem) return false // the target is global + } + return true + } + /** * Make the non-constraint factors for this component by using the solutions to the subproblems. */ override def makeNonConstraintFactors(parameterized: Boolean = false) { super.makeNonConstraintFactors(parameterized) - val subproblemFactors = + if (!problem.collection.useSingleChainFactor || !allSubproblemsEliminatedCompletely) { for { (parentValue, subproblem) <- subproblems - factor <- subproblem.solution - } yield Factory.replaceVariable(factor, problem.collection(subproblem.target).variable, actualSubproblemVariables(parentValue)) - nonConstraintFactors = subproblems.values.toList.flatMap(_.solution) ::: nonConstraintFactors + } { + raiseSubproblemSolution(parentValue, subproblem) + } + } } - /* - // Raise the given subproblem into this problem - def raise(subproblem: NestedProblem[Value]) { + def raiseSubproblemSolution(parentValue: ParentValue, subproblem: NestedProblem[Value]): Unit = { + val factors = for { + factor <- subproblem.solution + } yield Factory.replaceVariable(factor, problem.collection(subproblem.target).variable, actualSubproblemVariables(parentValue)) + nonConstraintFactors = factors.toList ::: nonConstraintFactors + } + // Raise the given subproblem into this problem. Note that factors for the chain must have been created already + // This probably needs some more thought! + def raise(parentValue: ParentValue, bounds: Bounds = Lower): Unit = { + + if (subproblems.contains(parentValue) && !subproblems(parentValue).solved) { + + val subproblem = subproblems(parentValue) + val comp = subproblem.collection(subproblem.target) + val CF = subproblem.components.flatMap(c => c.constraintFactors(bounds).map(Factory.replaceVariable(_, comp.variable, actualSubproblemVariables(parentValue)))) + val NCF = subproblem.components.flatMap(c => c.nonConstraintFactors.map(Factory.replaceVariable(_, comp.variable, actualSubproblemVariables(parentValue)))) + + if (bounds == Lower) constraintLower = constraintLower ::: CF + else constraintUpper = constraintUpper ::: CF + nonConstraintFactors = nonConstraintFactors ::: NCF + } } // Raise all subproblems into this problem - def raise() { subproblems.values.foreach(raise(_)) } - * - */ + def raise(bounds: Bounds) { subproblems.foreach(sp => raise(sp._1, bounds)) } + } /** * A problem component for a MakeArray element. */ class MakeArrayComponent[Value](problem: Problem, val makeArray: MakeArray[Value]) -extends ExpandableComponent[Int, FixedSizeArray[Value]](problem, makeArray.numItems, makeArray) { + extends ExpandableComponent[Int, FixedSizeArray[Value]](problem, makeArray.numItems, makeArray) { /** The maximum number of items expanded so far. */ var maxExpanded = 0 + // This isn't really needed in MakeArray since the main expand function won't call this. + val expandFunction = (i: Int) => Constant(makeArray.makeValues(i).regularValues.head) + /** * Ensure that the given number of items is expanded. */ diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/Range.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Range.scala similarity index 64% rename from Figaro/src/main/scala/com/cra/figaro/experimental/structured/Range.scala rename to Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Range.scala index 6437c1bd..444347c1 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/Range.scala +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/Range.scala @@ -11,20 +11,20 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.experimental.structured +package com.cra.figaro.algorithm.structured -import com.cra.figaro.algorithm.lazyfactored.{ValueSet, Star} +import com.cra.figaro.algorithm.lazyfactored.ValueSet import com.cra.figaro.algorithm.lazyfactored.ValueSet._ import com.cra.figaro.language._ import com.cra.figaro.library.compound.FastIf -import com.cra.figaro.util.{MultiSet, homogeneousCartesianProduct} +import com.cra.figaro.util.{ MultiSet, homogeneousCartesianProduct } import com.cra.figaro.util.HashMultiSet import com.cra.figaro.algorithm.factored.ParticleGenerator -import com.cra.figaro.library.collection.MakeArray import com.cra.figaro.library.collection.FixedSizeArray -import com.cra.figaro.library.atomic.discrete.{AtomicBinomial, ParameterizedBinomialFixedNumTrials} +import com.cra.figaro.library.atomic.discrete.{ AtomicBinomial, ParameterizedBinomialFixedNumTrials } import com.cra.figaro.library.compound.FoldLeft import com.cra.figaro.library.compound.IntSelector +import com.cra.figaro.algorithm.ValuesMaker object Range { private def getRange[U](collection: ComponentCollection, otherElement: Element[U]): ValueSet[U] = { @@ -34,7 +34,7 @@ object Range { // Get the range of the reference by taking the union of the current ranges of all the current possible targets. // Do not add any elements or expand any other ranges in the process in the process. - private[structured] def getRangeOfSingleValuedReference[V](cc: ComponentCollection, ec: ElementCollection, ref: Reference[V]): ValueSet[V] = { + private[algorithm] def getRangeOfSingleValuedReference[V](cc: ComponentCollection, ec: ElementCollection, ref: Reference[V]): ValueSet[V] = { val (firstElem, restRefOpt) = ec.getFirst(ref) restRefOpt match { case None => getRange(cc, firstElem.asInstanceOf[Element[V]]) @@ -57,7 +57,7 @@ object Range { } } - private[structured] def getRangeOfMultiValuedReference[V](cc: ComponentCollection, ec: ElementCollection, ref: Reference[V]): ValueSet[MultiSet[V]] = { + private[algorithm] def getRangeOfMultiValuedReference[V](cc: ComponentCollection, ec: ElementCollection, ref: Reference[V]): ValueSet[MultiSet[V]] = { // This function gets all the lists of elements that could be joint targets of the given reference, starting in the given element collection. // The first return value of this function is a set of all the possible lists of targets referred to by this reference. @@ -104,7 +104,7 @@ object Range { if (hs) hasStar = true subTargetSet.toList } - val combinations: List[List[List[Element[V]]]] = homogeneousCartesianProduct(subTargetSets:_*) + val combinations: List[List[List[Element[V]]]] = homogeneousCartesianProduct(subTargetSets: _*) val targetSets = combinations.map(_.flatten).toSet (targetSets, hasStar) } @@ -112,7 +112,7 @@ object Range { def getMultiSetPossibilities(targetSet: List[Element[V]]): ValueSet[MultiSet[V]] = { val ranges: List[ValueSet[V]] = targetSet.map(getRange(cc, _)) val regularRanges: List[List[V]] = ranges.map(_.regularValues.toList) - val possibilities: List[List[V]] = homogeneousCartesianProduct(regularRanges:_*) + val possibilities: List[List[V]] = homogeneousCartesianProduct(regularRanges: _*) val multiSets: List[MultiSet[V]] = for { possibility <- possibilities @@ -134,7 +134,7 @@ object Range { multiSetPossibilities.foldLeft(starter)(_ ++ _) } - def getRangeOfFold[T,U](cc: ComponentCollection, fold: FoldLeft[T,U]): ValueSet[U] = { + def getRangeOfFold[T, U](cc: ComponentCollection, fold: FoldLeft[T, U]): ValueSet[U] = { def helper(currentAccum: ValueSet[U], remainingElements: Seq[Element[T]]): ValueSet[U] = { if (remainingElements.isEmpty) currentAccum else { @@ -155,9 +155,10 @@ object Range { def apply[V](component: ProblemComponent[V], numValues: Int): ValueSet[V] = { component match { - case cc: ChainComponent[_,V] => chainRange(cc) + case cc: ChainComponent[_, V] => chainRange(cc) case mc: MakeArrayComponent[V] => makeArrayRange(mc) - case _ => otherRange(component, numValues) + case ac: ApplyComponent[V] => applyRange(ac) + case _ => otherRange(component, numValues) } } @@ -196,55 +197,23 @@ object Range { } } - private def otherRange[V](component: ProblemComponent[V], numValues: Int): ValueSet[V] = { + private def applyRange[V](component: ApplyComponent[V]): ValueSet[V] = { val collection = component.problem.collection + val applyMap = component.getMap() component.element match { - case c: Constant[_] => withoutStar(Set(c.constant)) - - case f: AtomicFlip => withoutStar(Set(true, false)) - - case f: ParameterizedFlip => - if (getRange(collection, f.parameter).hasStar) withStar(Set(true, false)) else withoutStar(Set(true, false)) - - case f: CompoundFlip => - if (getRange(collection, f.prob).hasStar) withStar(Set(true, false)) else withoutStar(Set(true, false)) - - case s: AtomicSelect[_] => withoutStar(Set(s.outcomes: _*)) - - case s: ParameterizedSelect[_] => - val values = Set(s.outcomes: _*) - if (getRange(collection, s.parameter).hasStar) withStar(values) else withoutStar(values) - - case s: CompoundSelect[_] => - val values = Set(s.outcomes: _*) - if (s.probs.map(getRange(collection, _)).exists(_.hasStar)) withStar(values) else withoutStar(values) - - case b: AtomicBinomial => ValueSet.withoutStar((0 to b.numTrials).toSet) - - case d: AtomicDist[_] => - val componentSets = d.outcomes.map(getRange(collection, _)) - componentSets.reduce(_ ++ _) - - case d: CompoundDist[_] => - val componentSets = d.outcomes.map(getRange(collection, _)) - val values = componentSets.reduce(_ ++ _) - if (d.probs.map(getRange(collection, _)).exists(_.hasStar)) values ++ withStar(Set()) else values case i: FastIf[_] => if (getRange(collection, i.test).hasStar) withStar(Set(i.thn, i.els)) else withoutStar(Set(i.thn, i.els)) - case a: Apply1[_, _] => + case a: Apply1[_, V] => val vs1 = getRange(collection, a.arg1) - vs1.map(a.fn) -// val applyMap = getMap(a) -// val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) -// val resultsSet = -// for { -// arg1Val <- vs1.regularValues -// } yield { -// getOrElseInsert(applyMap, arg1Val, a.fn(arg1Val)) -// } -// if (vs1.hasStar) withStar(resultsSet); else withoutStar(resultsSet) + val resultsSet = + for { + arg1Val <- vs1.regularValues + } yield { + applyMap.getOrElseUpdate(arg1Val, a.fn(arg1Val)) + } + if (vs1.hasStar) withStar(resultsSet); else withoutStar(resultsSet) case a: Apply2[_, _, _] => val vs1 = getRange(collection, a.arg1) @@ -253,22 +222,10 @@ object Range { for { v1 <- vs1.regularValues v2 <- vs2.regularValues - } yield a.fn(v1, v2) + } yield { + applyMap.getOrElseUpdate((v1, v2), a.fn(v1, v2)) + } if (vs1.hasStar || vs2.hasStar) withStar(resultSet) else withoutStar(resultSet) -// val applyMap = getMap(a) -// val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) -// val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) -// val choices = cartesianProduct(vs1.xvalues.toList, vs2.xvalues.toList).asInstanceOf[List[List[Extended[_]]]] -// val resultsList = -// for { -// List(arg1, arg2) <- choices -// if (arg1.isRegular && arg2.isRegular) -// } yield { -// val arg1Val = arg1.value.asInstanceOf[a.Arg1Type] -// val arg2Val = arg2.value.asInstanceOf[a.Arg2Type] -// getOrElseInsert(applyMap, (arg1Val, arg2Val), a.fn(arg1Val, arg2Val)) -// } -// if (vs1.hasStar || vs2.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) case a: Apply3[_, _, _, _] => val vs1 = getRange(collection, a.arg1) @@ -279,24 +236,10 @@ object Range { v1 <- vs1.regularValues v2 <- vs2.regularValues v3 <- vs3.regularValues - } yield a.fn(v1, v2, v3) + } yield { + applyMap.getOrElseUpdate((v1, v2, v3), a.fn(v1, v2, v3)) + } if (vs1.hasStar || vs2.hasStar || vs3.hasStar) withStar(resultSet) else withoutStar(resultSet) -// val applyMap = getMap(a) -// val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) -// val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) -// val vs3 = LazyValues(a.arg3.universe).storedValues(a.arg3) -// val choices = cartesianProduct(vs1.xvalues.toList, vs2.xvalues.toList, vs3.xvalues.toList).asInstanceOf[List[List[Extended[_]]]] -// val resultsList = -// for { -// List(arg1, arg2, arg3) <- choices -// if (arg1.isRegular && arg2.isRegular && arg3.isRegular) -// } yield { -// val arg1Val = arg1.value.asInstanceOf[a.Arg1Type] -// val arg2Val = arg2.value.asInstanceOf[a.Arg2Type] -// val arg3Val = arg3.value.asInstanceOf[a.Arg3Type] -// getOrElseInsert(applyMap, (arg1Val, arg2Val, arg3Val), a.fn(arg1Val, arg2Val, arg3Val)) -// } -// if (vs1.hasStar || vs2.hasStar || vs3.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) case a: Apply4[_, _, _, _, _] => val vs1 = getRange(collection, a.arg1) @@ -309,26 +252,10 @@ object Range { v2 <- vs2.regularValues v3 <- vs3.regularValues v4 <- vs4.regularValues - } yield a.fn(v1, v2, v3, v4) + } yield { + applyMap.getOrElseUpdate((v1, v2, v3, v4), a.fn(v1, v2, v3, v4)) + } if (vs1.hasStar || vs2.hasStar || vs3.hasStar || vs4.hasStar) withStar(resultSet) else withoutStar(resultSet) -// val applyMap = getMap(a) -// val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) -// val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) -// val vs3 = LazyValues(a.arg3.universe).storedValues(a.arg3) -// val vs4 = LazyValues(a.arg4.universe).storedValues(a.arg4) -// val choices = cartesianProduct(vs1.xvalues.toList, vs2.xvalues.toList, vs3.xvalues.toList, vs4.xvalues.toList).asInstanceOf[List[List[Extended[_]]]] -// val resultsList = -// for { -// List(arg1, arg2, arg3, arg4) <- choices -// if (arg1.isRegular && arg2.isRegular && arg3.isRegular && arg4.isRegular) -// } yield { -// val arg1Val = arg1.value.asInstanceOf[a.Arg1Type] -// val arg2Val = arg2.value.asInstanceOf[a.Arg2Type] -// val arg3Val = arg3.value.asInstanceOf[a.Arg3Type] -// val arg4Val = arg4.value.asInstanceOf[a.Arg4Type] -// getOrElseInsert(applyMap, (arg1Val, arg2Val, arg3Val, arg4Val), a.fn(arg1Val, arg2Val, arg3Val, arg4Val)) -// } -// if (vs1.hasStar || vs2.hasStar || vs3.hasStar || vs4.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) case a: Apply5[_, _, _, _, _, _] => val vs1 = getRange(collection, a.arg1) @@ -343,93 +270,92 @@ object Range { v3 <- vs3.regularValues v4 <- vs4.regularValues v5 <- vs5.regularValues - } yield a.fn(v1, v2, v3, v4, v5) + } yield { + applyMap.getOrElseUpdate((v1, v2, v3, v4, v5), a.fn(v1, v2, v3, v4, v5)) + } if (vs1.hasStar || vs2.hasStar || vs3.hasStar || vs4.hasStar || vs5.hasStar) withStar(resultSet) else withoutStar(resultSet) -// val applyMap = getMap(a) -// val vs1 = LazyValues(a.arg1.universe).storedValues(a.arg1) -// val vs2 = LazyValues(a.arg2.universe).storedValues(a.arg2) -// val vs3 = LazyValues(a.arg3.universe).storedValues(a.arg3) -// val vs4 = LazyValues(a.arg4.universe).storedValues(a.arg4) -// val vs5 = LazyValues(a.arg5.universe).storedValues(a.arg5) -// val choices = cartesianProduct(vs1.xvalues.toList, vs2.xvalues.toList, vs3.xvalues.toList, vs4.xvalues.toList, vs5.xvalues.toList).asInstanceOf[List[List[Extended[_]]]] -// val resultsList = -// for { -// List(arg1, arg2, arg3, arg4, arg5) <- choices -// if (arg1.isRegular && arg2.isRegular && arg3.isRegular && arg4.isRegular && arg5.isRegular) -// } yield { -// val arg1Val = arg1.value.asInstanceOf[a.Arg1Type] -// val arg2Val = arg2.value.asInstanceOf[a.Arg2Type] -// val arg3Val = arg3.value.asInstanceOf[a.Arg3Type] -// val arg4Val = arg4.value.asInstanceOf[a.Arg4Type] -// val arg5Val = arg5.value.asInstanceOf[a.Arg5Type] -// getOrElseInsert(applyMap, (arg1Val, arg2Val, arg3Val, arg4Val, arg5Val), a.fn(arg1Val, arg2Val, arg3Val, arg4Val, arg5Val)) -// } -// if (vs1.hasStar || vs2.hasStar || vs3.hasStar || vs4.hasStar || vs5.hasStar) withStar(resultsList.toSet); else withoutStar(resultsList.toSet) + } + } + + private def otherRange[V](component: ProblemComponent[V], numValues: Int): ValueSet[V] = { + val collection = component.problem.collection + component.element match { + case c: Constant[_] => withoutStar(Set(c.constant)) + + case f: AtomicFlip => withoutStar(Set(true, false)) + + case f: ParameterizedFlip => + if (getRange(collection, f.parameter).hasStar) withStar(Set(true, false)) else withoutStar(Set(true, false)) + + case f: CompoundFlip => + if (getRange(collection, f.prob).hasStar) withStar(Set(true, false)) else withoutStar(Set(true, false)) + + case s: AtomicSelect[_] => withoutStar(Set(s.outcomes: _*)) + + case s: ParameterizedSelect[_] => + val values = Set(s.outcomes: _*) + if (getRange(collection, s.parameter).hasStar) withStar(values) else withoutStar(values) + + case s: CompoundSelect[_] => + val values = Set(s.outcomes: _*) + if (s.probs.map(getRange(collection, _)).exists(_.hasStar)) withStar(values) else withoutStar(values) + + case b: AtomicBinomial => ValueSet.withoutStar((0 to b.numTrials).toSet) + + case d: AtomicDist[_] => + val componentSets = d.outcomes.map(getRange(collection, _)) + componentSets.reduce(_ ++ _) + + case d: CompoundDist[_] => + val componentSets = d.outcomes.map(getRange(collection, _)) + val values = componentSets.reduce(_ ++ _) + if (d.probs.map(getRange(collection, _)).exists(_.hasStar)) values ++ withStar(Set()) else values + + //case i: FastIf[_] => + // if (getRange(collection, i.test).hasStar) withStar(Set(i.thn, i.els)) else withoutStar(Set(i.thn, i.els)) case c: Chain[_, _] => throw new RuntimeException("This shouldn't be called") // The other version of apply should always be called for a chain -// def findChainValues[T, U](chain: Chain[T, U], cmap: Map[T, Element[U]], pVals: ValueSet[T], samples: Int): Set[ValueSet[U]] = { -// val chainVals = pVals.regularValues.map { parentVal => -// val resultElem = getOrElseInsert(cmap, parentVal, chain.getUncached(parentVal)) -// val result = LazyValues(resultElem.universe)(resultElem, depth - 1, samples, samples) -// usedBy(resultElem) = usedBy.getOrElse(resultElem, Set()) + element -// result -// } -// val newParentVals = LazyValues(chain.parent.universe).storedValues(chain.parent) -// if (newParentVals == pVals) { -// chainVals -// } else { -// findChainValues(chain, cmap, newParentVals, samples) -// } -// } -// -// val chainMap = getMap(c) -// val parentVS = LazyValues(c.parent.universe).storedValues(c.parent) -// val samplesPerValue = math.max(1, (numTotalSamples.toDouble/parentVS.regularValues.size).toInt) -// -// val resultVSs = findChainValues(c, chainMap, parentVS, samplesPerValue) -// -// val startVS: ValueSet[c.Value] = -// if (parentVS.hasStar) withStar[c.Value](Set()); else withoutStar[c.Value](Set()) -// resultVSs.foldLeft(startVS)(_ ++ _) case i: Inject[_] => val argVSs = i.args.map(getRange(collection, _)) -// val elementVSs = i.args.map(arg => LazyValues(arg.universe).storedValues(arg)) + // val elementVSs = i.args.map(arg => LazyValues(arg.universe).storedValues(arg)) val incomplete = argVSs.exists(_.hasStar) val elementValues = argVSs.toList.map(_.regularValues.toList) val resultValues = homogeneousCartesianProduct(elementValues: _*).toSet.asInstanceOf[Set[i.Value]] if (incomplete) withStar(resultValues); else withoutStar(resultValues) - case a: Atomic[_] => { - if (!ParticleGenerator.exists(a.universe)) { - println("Warning: Sampling element " + a + " even though no sampler defined for this universe") - } - val thisSampler = ParticleGenerator(a.universe) - val samples = thisSampler(a, numValues) - withoutStar(samples.unzip._2.toSet) - } - case r: SingleValuedReferenceElement[_] => getRangeOfSingleValuedReference(collection, r.collection, r.reference) - case r: MultiValuedReferenceElement[_] => getRangeOfMultiValuedReference(collection, r.collection, r.reference) + case r: MultiValuedReferenceElement[_] => getRangeOfMultiValuedReference(collection, r.collection, r.reference) - case a: Aggregate[_,_] => + case a: Aggregate[_, _] => val inputs = getRange(collection, a.mvre) val resultValues = inputs.regularValues.map(a.aggregate(_)) if (inputs.hasStar) withStar(resultValues); else withoutStar(resultValues) - case f: FoldLeft[_,_] => getRangeOfFold(collection, f) + case f: FoldLeft[_, _] => getRangeOfFold(collection, f) case i: IntSelector => val counterValues = getRange(collection, i.counter) if (counterValues.regularValues.nonEmpty) { val maxCounter = counterValues.regularValues.max -// val all = List.tabulate(maxCounter)(i => i).toSet - val all = Set((0 until maxCounter):_*) + // val all = List.tabulate(maxCounter)(i => i).toSet + val all = Set((0 until maxCounter): _*) if (counterValues.hasStar) ValueSet.withStar(all); else ValueSet.withoutStar(all) } else { ValueSet.withStar(Set()) } + // Make values is hardcoded with depth. SFI should control iterative deepening, so call make values with infinite depth + case v: ValuesMaker[_] => { + v.makeValues(Int.MaxValue) + } + + case a: Atomic[_] => { + val thisSampler = ParticleGenerator(a.universe) + val samples = thisSampler(a, numValues) + withoutStar(samples.unzip._2.toSet) + } + case _ => /* A new improvement - if we can't compute the values, we just make them *, so the rest of the computation can proceed */ withStar(Set()) diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/StructuredMPEAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/StructuredMPEAlgorithm.scala new file mode 100644 index 00000000..78538806 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/StructuredMPEAlgorithm.scala @@ -0,0 +1,55 @@ +/* + * StructuredAlgorithm.scala + * Abstract class for algorithms that are structured + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: December 30, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.algorithm + +import com.cra.figaro.algorithm.Algorithm +import com.cra.figaro.language._ +import scala.collection.mutable.Map +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Semiring +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.structured.ComponentCollection +import com.cra.figaro.algorithm.OneTimeMPE +import com.cra.figaro.algorithm.AlgorithmException + +abstract class StructuredMPEAlgorithm(val universe: Universe) extends Algorithm with OneTimeMPE { + + def run(): Unit + + val semiring: Semiring[Double] + + //val targetFactors: Map[Element[_], Factor[Double]] = Map() + + val cc: ComponentCollection = new ComponentCollection + + val problem = new Problem(cc, List()) + // We have to add all active elements to the problem since these elements, if they are every used, need to have components created at the top level problem + universe.permanentElements.foreach(problem.add(_)) + val evidenceElems = universe.conditionedElements ::: universe.constrainedElements + + def initialComponents() = (universe.permanentElements ++ evidenceElems).distinct.map(cc(_)) + + /** + * Returns the most likely value for the target element. + */ + def mostLikelyValue[T](target: Element[T]): T = { + val targetVar = cc(target).variable + val factor = problem.recordingFactors(targetVar).asInstanceOf[Factor[T]] + if (factor.size != 1) throw new AlgorithmException//("Final factor for most likely value has more than one entry") + factor.get(List()) + } + + +} + + diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/StructuredProbQueryAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/StructuredProbQueryAlgorithm.scala new file mode 100644 index 00000000..ff41cee5 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/StructuredProbQueryAlgorithm.scala @@ -0,0 +1,70 @@ +/* + * StructuredAlgorithm.scala + * Abstract class for algorithms that are structured + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: December 30, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.algorithm + +import com.cra.figaro.algorithm.OneTimeProbQuery +import com.cra.figaro.algorithm.Algorithm +import com.cra.figaro.language._ +import scala.collection.mutable.Map +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Semiring +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.structured.ComponentCollection + +abstract class StructuredProbQueryAlgorithm(val universe: Universe, val queryTargets: Element[_]*) extends Algorithm with OneTimeProbQuery { + + def run(): Unit + + val semiring: Semiring[Double] + + val targetFactors: Map[Element[_], Factor[Double]] = Map() + + val cc: ComponentCollection = new ComponentCollection + + val problem = new Problem(cc, queryTargets.toList) + // We have to add all active elements to the problem since these elements, if they are every used, need to have components created at the top level problem + universe.permanentElements.foreach(problem.add(_)) + val evidenceElems = universe.conditionedElements ::: universe.constrainedElements + + def initialComponents() = (problem.targets ++ evidenceElems).distinct.map(cc(_)) + + protected def marginalizeToTarget(target: Element[_], jointFactor: Factor[Double]): Unit = { + val targetVar = cc(target).variable + val unnormalizedTargetFactor = jointFactor.marginalizeTo(targetVar) + val z = unnormalizedTargetFactor.foldLeft(0.0, _ + _) + val targetFactor = unnormalizedTargetFactor.mapTo((d: Double) => d / z) + targetFactors += target -> targetFactor + } + + /** + * Computes the normalized distribution over a single target element. + */ + def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = { + val factor = targetFactors(target) + val targetVar = cc(target).variable + val dist = factor.getIndices.filter(f => targetVar.range(f.head).isRegular).map(f => (factor.get(f), targetVar.range(f.head).value)) + // normalization is unnecessary here because it is done in marginalizeTo + dist.toStream + } + + /** + * Computes the expectation of a given function for single target element. + */ + def computeExpectation[T](target: Element[T], function: T => Double): Double = { + def get(pair: (Double, T)) = pair._1 * function(pair._2) + (0.0 /: computeDistribution(target))(_ + get(_)) + } + +} + + diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatBP.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatBP.scala new file mode 100644 index 00000000..4a13e067 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatBP.scala @@ -0,0 +1,68 @@ +/* + * FlatBP.scala + * A structured factored inference algorithm using belief propagation. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.algorithm.flat + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ + +class FlatBP(universe: Universe, iterations: Int, targets: Element[_]*) extends StructuredProbQueryAlgorithm(universe, targets:_*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveFlattenStrategy(problem, new ConstantStrategy(marginalBeliefPropagation(iterations)), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + + +} + +object FlatBP { + /** + * Create a structured belief propagation algorithm. + * @param iterations the number of iterations to use for each subproblem + * @param targets the query targets, which will all be part of the top level problem + */ + def apply(iterations: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new FlatBP(targets(0).universe, iterations, targets:_*) + } + + /** + * Use BP to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean, iterations: Int): Double = { + val alg = FlatBP(iterations, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use BP to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T, iterations: Int = 100): Double = + probability(target, (t: T) => t == value, iterations) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatGibbs.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatGibbs.scala new file mode 100644 index 00000000..1b6c6ca7 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatGibbs.scala @@ -0,0 +1,79 @@ +/* + * FlatGibbs.scala + * A flat Gibbs sampling algorithm. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 8, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.flat + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.factored.gibbs._ +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler + +class FlatGibbs(universe: Universe, numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) + extends StructuredProbQueryAlgorithm(universe, targets: _*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveFlattenStrategy(problem, new ConstantStrategy(marginalGibbs(numSamples, burnIn, interval, blockToSampler)), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + +} + +object FlatGibbs { + /** + * Create a flat Gibbs algorithm. + */ + def apply(numSamples: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new FlatGibbs(targets(0).universe, numSamples, 0, 1, BlockSampler.default, targets: _*) + } + + /** + * Create a flat Gibbs algorithm. + */ + def apply(numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new FlatGibbs(targets(0).universe, numSamples, burnIn, interval, blockToSampler, targets: _*) + } + + /** + * Use Gibbs to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = this(10000, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use Gibbs to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatVE.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatVE.scala new file mode 100644 index 00000000..ab95aceb --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/flat/FlatVE.scala @@ -0,0 +1,64 @@ +/* + * FlatVE.scala + * A flat variable elimination algorithm. + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.flat + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ + +class FlatVE(universe: Universe, targets: Element[_]*) extends StructuredProbQueryAlgorithm(universe, targets:_*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveFlattenStrategy(problem, new ConstantStrategy(marginalVariableElimination), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + +} + +object FlatVE { + /** Create a structured variable elimination algorithm with the given query targets. */ + def apply(targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new FlatVE(targets(0).universe, targets:_*) + } + + /** + * Use VE to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = FlatVE(target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use VE to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/RaisingVE.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/RaisingVE.scala new file mode 100644 index 00000000..b658933f --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/RaisingVE.scala @@ -0,0 +1,64 @@ +/* + * RaisingVE.scala + * A flat variable elimination algorithm. + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.hybrid + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ + +class RaisingVE(universe: Universe, targets: Element[_]*) extends StructuredProbQueryAlgorithm(universe, targets:_*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveRaisingStrategy(problem, new ConstantStrategy(marginalVariableElimination), RaisingStrategy.raiseIfGlobal, defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + +} + +object RaisingVE { + /** Create a structured variable elimination algorithm with the given query targets. */ + def apply(targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new RaisingVE(targets(0).universe, targets:_*) + } + + /** + * Use VE to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = RaisingVE(target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use VE to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEBPChooser.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEBPChooser.scala new file mode 100644 index 00000000..71220ca9 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEBPChooser.scala @@ -0,0 +1,71 @@ +/* + * StructuredVEBPChooser.scala + * A hybrid algorithm that chooses between variable elimination and belief propagation for each component. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.hybrid + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ + +class StructuredVEBPChooser(universe: Universe, scoreThreshold: Double, BPIterations: Int, targets: Element[_]*) + extends StructuredProbQueryAlgorithm(universe, targets: _*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new VEBPStrategy(scoreThreshold, BPIterations), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + +} + +object StructuredVEBPChooser { + /** + * Create a hybrid algorithm that chooses between variable elimination and belief propagation on each subproblem. + * @param scoreThreshold The minimum value of the increase in score caused by eliminating a variable that causes the hybrid algorithm to + * choose BP for a subproblem. + * @param bpIterations The number of iterations to use when BP is chosen for a subproblem. + * @param targets The query targets + */ + def apply(scoreThreshold: Double, BPIterations: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredVEBPChooser(targets(0).universe, scoreThreshold, BPIterations, targets: _*) + } + + /** + * Use the hybrid algorithm to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean, threshold: Double, iterations: Int): Double = { + val alg = StructuredVEBPChooser(threshold, iterations, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use the hybrid algorithm to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T, threshold: Double = 0.0, iterations: Int = 100): Double = + probability(target, (t: T) => t == value, threshold, iterations) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEBPGibbsChooser.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEBPGibbsChooser.scala new file mode 100644 index 00000000..d7067d53 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEBPGibbsChooser.scala @@ -0,0 +1,79 @@ +/* + * StructuredVEBPGibbsChooser.scala + * A hybrid algorithm that chooses between variable elimination and Gibbs sampling for each component. + * + * Created By: William kretschmer (kretsch@mit.edu) + * Creation Date: Aug 11, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.hybrid + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler + +class StructuredVEBPGibbsChooser(universe: Universe, scoreThreshold: Double, determThreshold: Double, bpIters: Int, numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) + extends StructuredProbQueryAlgorithm(universe, targets: _*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new VEBPGibbsStrategy(scoreThreshold, determThreshold, bpIters, numSamples, burnIn, interval, blockToSampler), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + +} + +object StructuredVEBPGibbsChooser { + /** + * Create a hybrid algorithm that chooses between variable elimination and Gibbs sampling on each subproblem. + */ + def apply(scoreThreshold: Double, determThreshold: Double, bpIters: Int, numSamples: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE/Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredVEBPGibbsChooser(targets(0).universe, scoreThreshold, determThreshold, bpIters, numSamples, 0, 1, BlockSampler.default, targets: _*) + } + + /** + * Create a hybrid algorithm that chooses between variable elimination and Gibbs sampling on each subproblem. + */ + def apply(scoreThreshold: Double, determThreshold: Double, bpIters: Int, numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE/Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredVEBPGibbsChooser(targets(0).universe, scoreThreshold, determThreshold, bpIters, numSamples, burnIn, interval, blockToSampler, targets: _*) + } + + /** + * Use the hybrid algorithm to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean, threshold: Double, determThreshold: Double): Double = { + val alg = this(threshold, determThreshold, 1000, 10000, 0, 1, BlockSampler.default, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use the hybrid algorithm to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T, threshold: Double = 0.0, determThreshold: Double = 0.5): Double = + probability(target, (t: T) => t == value, threshold, determThreshold) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEGibbsChooser.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEGibbsChooser.scala new file mode 100644 index 00000000..061b83a2 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/hybrid/StructuredVEGibbsChooser.scala @@ -0,0 +1,79 @@ +/* + * StructuredVEGibbsChooser.scala + * A hybrid algorithm that chooses between variable elimination and Gibbs sampling for each component. + * + * Created By: William kretschmer (kretsch@mit.edu) + * Creation Date: Aug 11, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.hybrid + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler + +class StructuredVEGibbsChooser(universe: Universe, scoreThreshold: Double, numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) + extends StructuredProbQueryAlgorithm(universe, targets: _*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new VEGibbsStrategy(scoreThreshold, numSamples, burnIn, interval, blockToSampler), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } + +} + +object StructuredVEGibbsChooser { + /** + * Create a hybrid algorithm that chooses between variable elimination and Gibbs sampling on each subproblem. + */ + def apply(scoreThreshold: Double, numSamples: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE/Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredVEGibbsChooser(targets(0).universe, scoreThreshold, numSamples, 0, 1, BlockSampler.default, targets: _*) + } + + /** + * Create a hybrid algorithm that chooses between variable elimination and Gibbs sampling on each subproblem. + */ + def apply(scoreThreshold: Double, numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE/Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredVEGibbsChooser(targets(0).universe, scoreThreshold, numSamples, burnIn, interval, blockToSampler, targets: _*) + } + + /** + * Use the hybrid algorithm to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean, threshold: Double): Double = { + val alg = this(threshold, 10000, 0, 1, BlockSampler.default, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use the hybrid algorithm to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T, threshold: Double = 0.0): Double = + probability(target, (t: T) => t == value, threshold) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredBP.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredBP.scala new file mode 100644 index 00000000..5b0af6b7 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredBP.scala @@ -0,0 +1,78 @@ +/* + * StructuredBP.scala + * A structured factored inference algorithm using belief propagation. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.algorithm.structured + + +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.algorithm.StructuredProbQueryAlgorithm +import com.cra.figaro.algorithm.structured.strategy.decompose._ + +class StructuredBP(universe: Universe, iterations: Int, targets: Element[_]*) extends StructuredProbQueryAlgorithm(universe, targets:_*) { + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new ConstantStrategy(marginalBeliefPropagation(iterations)), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } +} + +/* + * StructuredBP.scala + * A structured belief propagation algorithm. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +object StructuredBP { + /** + * Create a structured belief propagation algorithm. + * @param iterations the number of iterations to use for each subproblem + * @param targets the query targets, which will all be part of the top level problem + */ + def apply(iterations: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredBP(targets(0).universe, iterations, targets: _*) + } + + /** + * Use BP to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean, iterations: Int): Double = { + val alg = StructuredBP(iterations, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use BP to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T, iterations: Int = 100): Double = + probability(target, (t: T) => t == value, iterations) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredGibbs.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredGibbs.scala new file mode 100644 index 00000000..ac9ac81a --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredGibbs.scala @@ -0,0 +1,77 @@ +/* + * StructuredGibbs.scala + * A structured Gibbs sampling algorithm. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 7, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.algorithm.structured + + +import com.cra.figaro.algorithm._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.algorithm.StructuredProbQueryAlgorithm +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler + +class StructuredGibbs(universe: Universe, numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) + extends StructuredProbQueryAlgorithm(universe, targets: _*) { + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new ConstantStrategy(marginalGibbs(numSamples, burnIn, interval, blockToSampler)), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } +} + +object StructuredGibbs { + /** + * Create a structured Gibbs algorithm. + */ + def apply(numSamples: Int, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredGibbs(targets(0).universe, numSamples, 0, 1, BlockSampler.default, targets:_*) + } + + /** + * Create a structured Gibbs algorithm. + */ + def apply(numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator, targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run Gibbs with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredGibbs(targets(0).universe, numSamples, burnIn, interval, blockToSampler, targets:_*) + } + + /** + * Use Gibbs to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = this(10000, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use Gibbs to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredMPEBP.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredMPEBP.scala new file mode 100644 index 00000000..fa49fd44 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredMPEBP.scala @@ -0,0 +1,53 @@ +/* + * StructuredVE.scala + * A structured variable elimination algorithm. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.structured + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.factored.factors.MaxProductSemiring + + +class StructuredMPEBP(universe: Universe, iterations: Int) extends StructuredMPEAlgorithm(universe) { + + val semiring = MaxProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new ConstantStrategy(mpeBeliefPropagation(iterations)), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + } +} + +object StructuredMPEBP { + /** Create a structured variable elimination algorithm with the given query targets. */ + def apply(iterations: Int)(implicit universe: Universe) = { + new StructuredMPEBP(universe, iterations) + } + + /** + * Use VE to compute the probability that the given element satisfies the given predicate. + */ + def mostLikelyValue[T](target: Element[T], iterations: Int): T = { + val alg = new StructuredMPEBP(target.universe, iterations) + alg.start() + val result = alg.mostLikelyValue(target) + alg.kill() + result + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredMPEVE.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredMPEVE.scala new file mode 100644 index 00000000..9cf1e27f --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredMPEVE.scala @@ -0,0 +1,53 @@ +/* + * StructuredVE.scala + * A structured variable elimination algorithm. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.structured + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ +import com.cra.figaro.algorithm.factored.factors.MaxProductSemiring + + +class StructuredMPEVE(universe: Universe) extends StructuredMPEAlgorithm(universe) { + + val semiring = MaxProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new ConstantStrategy(mpeVariableElimination), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + } +} + +object StructuredMPEVE { + /** Create a structured variable elimination algorithm with the given query targets. */ + def apply()(implicit universe: Universe) = { + new StructuredMPEVE(universe) + } + + /** + * Use VE to compute the probability that the given element satisfies the given predicate. + */ + def mostLikelyValue[T](target: Element[T]): T = { + val alg = new StructuredMPEVE(target.universe) + alg.start() + val result = alg.mostLikelyValue(target) + alg.kill() + result + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredVE.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredVE.scala new file mode 100644 index 00000000..4fe00d8b --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/algorithm/structured/StructuredVE.scala @@ -0,0 +1,64 @@ +/* + * StructuredVE.scala + * A structured variable elimination algorithm. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.algorithm.structured + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.algorithm.structured.algorithm._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.factored.factors.factory._ + + +class StructuredVE(universe: Universe, targets: Element[_]*) extends StructuredProbQueryAlgorithm(universe, targets: _*) { + + val semiring = SumProductSemiring() + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new ConstantStrategy(marginalVariableElimination), defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + val joint = problem.solution.foldLeft(Factory.unit(semiring))(_.product(_)) + targets.foreach(t => marginalizeToTarget(t, joint)) + } +} + +object StructuredVE { + /** Create a structured variable elimination algorithm with the given query targets. */ + def apply(targets: Element[_]*) = { + if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") + val universes = targets.map(_.universe).toSet + if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") + new StructuredVE(targets(0).universe, targets: _*) + } + + /** + * Use VE to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = StructuredVE(target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use VE to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/BPSolver.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/BPSolver.scala new file mode 100644 index 00000000..1c84bcac --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/BPSolver.scala @@ -0,0 +1,88 @@ +/* + * BPSolver.scala + * A belief propagation solver. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.solver + +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.factored.factors.LogSumProductSemiring +import com.cra.figaro.util._ +import com.cra.figaro.algorithm.lazyfactored._ +import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.algorithm.factored.beliefpropagation.FactorNode +import com.cra.figaro.algorithm.factored.factors.factory.Factory._ +import com.cra.figaro.algorithm.factored.beliefpropagation._ +import com.cra.figaro.algorithm.structured._ + +class BPSolver(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]], val iters: Int, + val semiring: LogConvertibleSemiRing[Double]) + extends com.cra.figaro.algorithm.factored.beliefpropagation.OneTimeProbabilisticBeliefPropagation { + // We need to create a joint probability distribution over the interface to this nested subproblem. + // To achieve this, we create a new variable representing the tuple of the attributes to preserve. + // We create a factor to represent the tuple creation. + // We then run BP as usual. + // At the end, we sum the tuple variable out of this factor to obtain the solution. + + def iterations = iters + + val (tupleVar, tupleFactor): (Variable[_], Factor[Double]) = makeTupleVarAndFactor(problem.collection, None, toPreserve.toList: _*) + + def generateGraph() = { + val allFactors = tupleFactor :: factors + //factorGraph = new BasicFactorGraph(allFactors.map(makeLogarithmic(_)), logSpaceSemiring()): FactorGraph[Double] + factorGraph = new BasicFactorGraph(convertFactors(allFactors), logSpaceSemiring()): FactorGraph[Double] + } + + override def initialize() = { + if (factorGraph == null) generateGraph() + super.initialize + } + + def go(): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + initialize() + run() + val targetVars = toPreserve.toList ::: List(tupleVar) + val tupleBelief = belief(FactorNode(toPreserve + tupleVar)) + val targetBelief = tupleBelief.sumOver(tupleVar) + val targetFactors = if (semiring.isLog()) { + List(normalize(targetBelief)) + } else { + List(unmakeLogarithmic(normalize(targetBelief))) + } + + if (toPreserve.isEmpty && semiring == MaxProductSemiring()) { + val max = toEliminate.map { v => v -> makeRecordingFactor(v)} + (targetFactors, max.toMap) + } else { + (targetFactors, Map()) + } + + } + + def makeRecordingFactor[U](v: Variable[U]): Factor[U] = { + val bf = new BasicFactor[U](List(), List()) + val maxInd = beliefMap(VariableNode(v)).contents.maxBy(_._2)._1 + val maxValue = v.range(maxInd(0)) + bf.set(List(), maxValue.value.asInstanceOf[v.Value]) + bf + } + + /* Not needed for SFI */ + val dependentUniverses = null + + val dependentAlgorithm = null + + val universe = null + + val targetElements = null +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/GibbsSolver.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/GibbsSolver.scala new file mode 100644 index 00000000..a86f9e3b --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/GibbsSolver.scala @@ -0,0 +1,96 @@ +/* + * GibbsSolver.scala + * A Gibbs sampling solver. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 6, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured.solver + +import scala.annotation.tailrec +import com.cra.figaro.algorithm.OneTime +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.sampling.BaseUnweightedSampler +import com.cra.figaro.algorithm.factored._ +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.language.Chain +import com.cra.figaro.algorithm.factored.gibbs.ProbabilisticGibbs +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.factored.gibbs.WalkSAT + +class GibbsSolver(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], _factors: List[Factor[Double]], + _numSamples: Int, _burnIn: Int, _interval: Int, val blockToSampler: Gibbs.BlockSamplerCreator) + extends BaseUnweightedSampler(null) with ProbabilisticGibbs with OneTime { + + def numSamples() = _numSamples + def burnIn() = _burnIn + def interval() = _interval + + def initializeBlocks() = { + factors = _factors.map(_.mapTo(math.log, semiring)) + variables = factors.flatMap(_.variables).toSet + val blocks = createBlocks() + // Create block samplers + blockSamplers = blocks.map(block => blockToSampler((block, factors.filter(_.variables.exists(block.contains(_)))))) + } + + override def initialize() = { + super.initialize + if (blockSamplers == null) initializeBlocks() + // Initialize the samples to a valid state and take the burn-in samples + val initialSample = WalkSAT(factors, variables, semiring, chainMapper) + variables.foreach(v => currentSamples(v) = initialSample(v)) + for (_ <- 1 to burnIn) sampleAllBlocks() + } + + def chainMapper(chain: Chain[_, _]): Set[Variable[_]] = problem.collection(chain).actualSubproblemVariables.values.toSet + + def run = {} + + def go(): List[Factor[Double]] = { + initialize() + val targetVars = toPreserve.toList + val result = new SparseFactor[Double](targetVars, List()) + for (_ <- 0 until numSamples) { + for (_ <- 0 until interval) { + sampleAllBlocks() + } + val factorIndex = targetVars.map(currentSamples(_)) + result.set(factorIndex, result.get(factorIndex) + 1) + } + List(result.mapTo(_ / numSamples)) + } + + val dependentUniverses = null + + val dependentAlgorithm = null + + val targetElements = null + + def createBlocks(): List[Gibbs.Block] = { + val variables = factors.flatMap(_.variables).toSet + val variableParents = problem.collection.variableParents + // Maps each variable to its deterministic children, i.e. variables that should be included in a block with this variable + val variableChildren: Map[Variable[_], Set[Variable[_]]] = + variables.map(v => v -> variables.filter(variableParents(_).contains(v))).toMap + + // Start with the purely stochastic variables with no parents + val starterVariables = variables.filter(variableParents(_).isEmpty) + + @tailrec // Recursively add deterministic children to the block + def expandBlock(expand: Set[Variable[_]], block: Set[Variable[_]] = Set()): Gibbs.Block = { + if (expand.isEmpty) block.toList + else { + val expandNext = expand.flatMap(variableChildren(_)) + expandBlock(expandNext, block ++ expand) + } + } + starterVariables.map(v => expandBlock(Set(v))).toList + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/VESolver.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/VESolver.scala new file mode 100644 index 00000000..6e5125da --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/VESolver.scala @@ -0,0 +1,96 @@ +/* + * VESolver.scala + * A variable elimination solver. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.solver + +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.util.MultiSet +import com.cra.figaro.algorithm.factored.factors.Semiring +import com.cra.figaro.util +import com.cra.figaro.algorithm.lazyfactored._ + +class VESolver(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]], val semiring: Semiring[Double]) + extends com.cra.figaro.algorithm.factored.VariableElimination[Double] { + + debug = false + + override val comparator = semiring match { + case sum: SumProductSemiring => None + case max: MaxProductSemiring => Some((x: Double, y: Double) => x < y) + } + + def go(): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + // Convert factors to MaxProduct for MPE + val convertedFactors = semiring match { + case sum: SumProductSemiring => factors + case max: MaxProductSemiring => factors.map(_.mapTo(x => x, semiring)) + } + doElimination(convertedFactors, toPreserve.toList) + (resultFactors, recordingFactorsMap) + } + + private var resultFactors: List[Factor[Double]] = _ + // A map from each variable to a factor that maps values of the toPreserve variables to maximal values of the variable + // Note that when the toPreserve is empty, this represents the maximal value of each variable + private var recordingFactorsMap: Map[Variable[_], Factor[_]] = Map() + private def getRecordingFactor[T](variable: Variable[T]): Factor[T] = recordingFactorsMap(variable).asInstanceOf[Factor[variable.Value]] + + def finish(factorsAfterElimination: MultiSet[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = { + semiring match { + case sum: SumProductSemiring => finishSum(factorsAfterElimination, eliminationOrder) + case max: MaxProductSemiring => finishMax(factorsAfterElimination, eliminationOrder) + } + } + + /* Finish function for marginal VE */ + def finishSum(factorsAfterElimination: MultiSet[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = { + resultFactors = factorsAfterElimination.toList + } + + /* Finish function for MPE VE */ + def finishMax(factorsAfterElimination: MultiSet[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = { + resultFactors = factorsAfterElimination.toList + /* If empty, we need to know the max values for all variables in this set of factors + * Otherwise, we assume that the eliminated varaibles are internal and therefore are not queryable + * When we have non-chain decompositions, this may not hold anymore + */ + if (factorsAfterElimination.forall(f => f.size == 1 && f.numVars == 0)) { + for { (variable, factor) <- eliminationOrder.reverse.zip(recordingFactors) } { backtrackOne(factor, variable) } + } + } + + private def backtrackOne[T](factor: Factor[_], variable: Variable[T]): Unit = { + val indices = + for { variable <- factor.variables } yield util.indices(variable.range, Regular(getRecordingFactor(variable).contents.head._2)).head + recordingFactorsMap += variable -> { + val bf = factor.asInstanceOf[Factor[variable.Value]].createFactor(List(), List()) + bf.set(List(), factor.asInstanceOf[Factor[variable.Value]].get(indices)) + bf + } + } + + /* Functions not needed for SFI */ + val dependentAlgorithm: (com.cra.figaro.language.Universe, List[com.cra.figaro.language.NamedEvidence[_]]) => () => Double = null + + val dependentUniverses: List[(com.cra.figaro.language.Universe, List[com.cra.figaro.language.NamedEvidence[_]])] = null + + def getFactors(neededElements: List[com.cra.figaro.language.Element[_]], + targetElements: List[com.cra.figaro.language.Element[_]], + upperBounds: Boolean): List[com.cra.figaro.algorithm.factored.factors.Factor[Double]] = null + + val showTiming: Boolean = false + + val targetElements: List[com.cra.figaro.language.Element[_]] = null + + val universe: com.cra.figaro.language.Universe = null +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/package.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/package.scala new file mode 100644 index 00000000..2f0f5ba1 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/solver/package.scala @@ -0,0 +1,100 @@ +/* + * package.scala + * Definitions of solvers. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.algorithm.structured + +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.structured.solver.BPSolver +import com.cra.figaro.algorithm.structured.solver.GibbsSolver +import com.cra.figaro.algorithm.structured.solver.VESolver +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.factored.factors.MaxProductSemiring + +package object solver { + /** + * A Solver takes a set of variables to eliminate, a set of variables to preserve, and a list of factors. + * It returns a list of factors that mention only the preserved variables. + */ + type Solver = (Problem, Set[Variable[_]], Set[Variable[_]], List[Factor[Double]]) => (List[Factor[Double]], Map[Variable[_], Factor[_]]) + + /** + * Creates a Gibbs sampling solver. + * @param numSamples number of samples to take + * @param burnIn number of burn-in samples to throw away + * @param interval number of samples to throw away between recorded samples + * @param blockToSampler function for creating Gibbs block samplers + * @param problem the problem to solve + * @param toEliminate the variables to be eliminated + * @param toPreserve the variables to be preserved (not eliminated) + * @param factors all the factors in the problem + */ + def marginalGibbs(numSamples: Int, burnIn: Int, interval: Int, blockToSampler: Gibbs.BlockSamplerCreator)(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val gibbs = new GibbsSolver(problem, toEliminate, toPreserve, factors, numSamples, burnIn, interval, blockToSampler) + (gibbs.go(), Map()) + } + + /** + * Creates a variable elimination solver. + * @param problem the problem to solve + * @param toEliminate the variables to be eliminated + * @param toPreserve the variables to be preserved (not eliminated) + * @param factors all the factors in the problem + */ + def marginalVariableElimination(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val ve = new VESolver(problem, toEliminate, toPreserve, factors, SumProductSemiring()) + ve.go() + } + + /** + * Creates an MPE variable elimination solver. + * @param problem the problem to solve + * @param toEliminate the variables to be eliminated + * @param toPreserve the variables to be preserved (not eliminated) + * @param factors all the factors in the problem + */ + def mpeVariableElimination(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val ve = new VESolver(problem, toEliminate, toPreserve, factors, MaxProductSemiring()) + ve.go() + } + + /** + * Creates a belief propagation solver. + * @param iterations number of iterations of BP to run + * @param problem the problem to solve + * @param toEliminate the variables to be eliminated + * @param toPreserve the variables to be preserved (not eliminated) + * @param factors all the factors in the problem + */ + def marginalBeliefPropagation(iterations: Int = 100)(problem: Problem, toEliminate: Set[Variable[_]], + toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val bp = new BPSolver(problem, toEliminate, toPreserve, factors, iterations, SumProductSemiring()) + bp.go() + } + + /** + * Creates an MPE belief propagation solver. + * @param iterations number of iterations of BP to run + * @param problem the problem to solve + * @param toEliminate the variables to be eliminated + * @param toPreserve the variables to be preserved (not eliminated) + * @param factors all the factors in the problem + */ + def mpeBeliefPropagation(iterations: Int = 100)(problem: Problem, toEliminate: Set[Variable[_]], + toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val bp = new BPSolver(problem, toEliminate, toPreserve, factors, iterations, MaxProductSemiring()) + bp.go() + } + +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/BackwardChain.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/BackwardChain.scala new file mode 100644 index 00000000..e522c7fd --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/BackwardChain.scala @@ -0,0 +1,68 @@ +/* + * BackwardChain.scala + * A trait for a backward chaining algorithm for decomposition strategies + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.ProblemComponent +import com.cra.figaro.algorithm.structured.MakeArrayComponent +import com.cra.figaro.algorithm.structured.ChainComponent + +trait BackwardChain extends DecompositionStrategy { + + /* + * backwardChain takes a list of items to process. + * When the first item is taken off the list, it checks whether it has already been done. + * When an item is taken off the list, if it has not been done, all the items it depends on + * are added to the list to do. This guarantees that when an item is finally processed, + * all the items it depends on have already been processed. Also, we do not process + * any items more than once. + */ + protected[figaro] def backwardChain(toDo: List[ProblemComponent[_]], done: Set[ProblemComponent[_]]): Set[ProblemComponent[_]] = { + toDo match { + case first :: rest => + // globals should have been processed before this problem + if (done.contains(first)) backwardChain(rest, done) + else { + val argComponents = first.element.args.map(checkArg(_)) + val done1 = if (argComponents.nonEmpty) backwardChain(argComponents, done) else done + first match { + case chainComp: ChainComponent[_, _] => + processChain(first, rest, done1, chainComp) + case maComp: MakeArrayComponent[_] => + processMakeArray(first, rest, done1, maComp) + case _ => + first.element match { + //If the element is decomposable, call process on it to determine how to decompose the element + case dc: Decomposable => dc.process(first, rest, done) + case _ => + // otherwise use the default process + process(first) + backwardChain(rest, done1 + first) + } + } + } + case _ => done + } + } + + /* + * Class that defines how to process Chain components + */ + protected def processChain(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], chainComp: ChainComponent[_, _]): Set[ProblemComponent[_]] + + /* + * Class that defines how to process MakeArray components. This may not be needed since MakeArrays are deprecated. + */ + protected def processMakeArray(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], maComp: MakeArrayComponent[_]): Set[ProblemComponent[_]] + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/Decomposable.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/Decomposable.scala new file mode 100644 index 00000000..a14efb02 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/Decomposable.scala @@ -0,0 +1,23 @@ +/* + * Decomposable.scala + * A trait for indicating decomposable classes + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.algorithm.structured.ProblemComponent + +/** + * Decomposable trait. This trait can be added to elements or other decomposable classes that indicates to a + * decomposing strategy how to process this ProblemComponent + */ +trait Decomposable { + def process(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]]): Set[ProblemComponent[_]] +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/DecompositionStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/DecompositionStrategy.scala new file mode 100644 index 00000000..d0a79d77 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/DecompositionStrategy.scala @@ -0,0 +1,91 @@ +/* + * DecompositionStrategy.scala + * Base class for strategies that decompose a problem + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.language.Element +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy +import com.cra.figaro.algorithm.structured._ + + +/** + * An abstract class that defines how to decompose problems into nested problems. + * @param problem The problem to decompose + * @param solvingStrategy The solving strategy used to solve this problem + * @param rangeSizer The method to determine the range of components + * @param bounds Lower or Upper bounds + * @param parameterized Indicates if this problem parameterized + */ +private[figaro] abstract class DecompositionStrategy(val problem: Problem, val solvingStrategy: SolvingStrategy, val rangeSizer: RangeSizer, + val bounds: Bounds, val parameterized: Boolean) { + + /** + * Executes (i.e., solves) the problem + */ + def execute(): Unit = execute(problem.targets.map(problem.collection(_))) + + /** + * Executes (i.e., solves) the problem + */ + def execute(components: List[ProblemComponent[_]]): Unit + + /** + * Decomposes a nested problem + */ + def decompose(nestedProblem: Problem, done: Set[ProblemComponent[_]]): Option[DecompositionStrategy] + + protected def checkArg[T](element: Element[T]): ProblemComponent[T] = { + if (problem.collection.contains(element)) problem.collection(element) + else problem.add(element) + } + + /* + * Default process for an element. Generate the range and make the factors + */ + protected def process(comp: ProblemComponent[_]) { + comp.generateRange(rangeSizer(comp)) + val isGlobal = problem.global(comp.variable) + if (!isGlobal) { + comp.makeNonConstraintFactors(parameterized) + comp.makeConstraintFactors(bounds) + } else { + val globalsProblem = comp.problem + globalsProblem.componentsToVisit += comp + } + } + +} + +object DecompositionStrategy { + + def recursiveStructuredStrategy(problem: Problem, solvingStrategy: SolvingStrategy, rangeSizer: RangeSizer, + bounds: Bounds, parameterized: Boolean) = new RecursiveStructuredStrategy(problem, solvingStrategy, rangeSizer, bounds, parameterized) + + def recursiveFlattenStrategy(problem: Problem, solvingStrategy: SolvingStrategy, rangeSizer: RangeSizer, + bounds: Bounds, parameterized: Boolean) = new RecursiveFlattenStrategy(problem, solvingStrategy, rangeSizer, bounds, parameterized) { + override def execute(components: List[ProblemComponent[_]]) = { + backwardChain(components, Set[ProblemComponent[_]]()) + if (problem.componentsToVisit.nonEmpty) backwardChain(problem.componentsToVisit.toList, Set[ProblemComponent[_]]()) + problem.solve(solvingStrategy, bounds) + } + } + + def recursiveRaisingStrategy(problem: Problem, solvingStrategy: SolvingStrategy, raisingCriteria: (Problem, Problem) => Boolean, rangeSizer: RangeSizer, + bounds: Bounds, parameterized: Boolean) = new RecursiveRaisingStrategy(problem, solvingStrategy, raisingCriteria, rangeSizer, bounds, parameterized) { + override def execute(components: List[ProblemComponent[_]]) = { + backwardChain(components, Set[ProblemComponent[_]]()) + if (problem.componentsToVisit.nonEmpty) backwardChain(problem.componentsToVisit.toList, Set[ProblemComponent[_]]()) + problem.solve(solvingStrategy, bounds) + } + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/FlattenStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/FlattenStrategy.scala new file mode 100644 index 00000000..c9cbefdf --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/FlattenStrategy.scala @@ -0,0 +1,39 @@ +/* + * FlattenStrategy.scala + * A strategy that flattens each problem and solves the problem in a flat manner + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy +import com.cra.figaro.algorithm.structured._ + +/** + * A strategy that flattens the model and uses no structure to solve the problem + */ +private[figaro] class FlattenStrategy(problem: Problem, solvingStrategy: SolvingStrategy, recursingStrategy: Problem => DecompositionStrategy, rangeSizer: RangeSizer, + bounds: Bounds, parameterized: Boolean) + extends StructuredStrategy(problem, solvingStrategy, recursingStrategy, rangeSizer, bounds, parameterized) with BackwardChain { + + override def execute(components: List[ProblemComponent[_]]) { + backwardChain(components, Set[ProblemComponent[_]]()) + if (problem.componentsToVisit.nonEmpty) backwardChain(problem.componentsToVisit.toList, Set[ProblemComponent[_]]()) + } + + override def processChain(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], chainComp: ChainComponent[_, _]): Set[ProblemComponent[_]] = { + chainComp.expand() + val subStrategies = chainComp.subproblems.values.map(decompose(_, done)) + subStrategies.foreach(ds => if (ds.nonEmpty) ds.get.execute()) + process(chainComp) + chainComp.raise(bounds) + backwardChain(rest, done + chainComp) + } + +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/RaisingStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/RaisingStrategy.scala new file mode 100644 index 00000000..75a45dd0 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/RaisingStrategy.scala @@ -0,0 +1,77 @@ +/* + * RaisingStrategy.scala + * A strategy that chooses to raise or solve a sub-problem based on a decision function + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: Oct 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy +import com.cra.figaro.algorithm.structured._ + +/** + * A structured strategy that decomposes Chains and MakeArrays according to a given recursive strategy + * @param problem The problem to decompose + * @param solvingStrategy The solving strategy used to solve this problem + * @param recursingStrategy A function that returns a new Decomposition strategy given a nested problem + * @param rangeSizer The method to determine the range of components + * @param bounds Lower or Upper bounds + * @param parameterized Indicates if this problem parameterized + */ + +private[figaro] class RaisingStrategy(problem: Problem, solvingStrategy: SolvingStrategy, + recursingStrategy: Problem => DecompositionStrategy, raisingCriteria: (Problem, Problem) => Boolean, rangeSizer: RangeSizer, + bounds: Bounds, parameterized: Boolean) + extends StructuredStrategy(problem, solvingStrategy, recursingStrategy, rangeSizer, bounds, parameterized) with BackwardChain { + + override def execute(components: List[ProblemComponent[_]]) { + backwardChain(components, Set[ProblemComponent[_]]()) + if (problem.componentsToVisit.nonEmpty) backwardChain(problem.componentsToVisit.toList, Set[ProblemComponent[_]]()) + } + + def typedProcessChain[ParentValue, Value](first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], chainComp: ChainComponent[ParentValue, Value]): Set[ProblemComponent[_]] = { + chainComp.expand() + val subStrategies = chainComp.subproblems.map(v => (v._1, v._2, decompose(v._2, done))) + subStrategies.foreach(ds => if (ds._3.nonEmpty) ds._3.get.execute()) + process(chainComp) + subStrategies.foreach(ds => { + val (parentValue, subproblem, strategy) = ds + if (strategy.nonEmpty && raisingCriteria(problem, subproblem)) { + chainComp.raise(parentValue, bounds) + } else { + if (strategy.nonEmpty) subproblem.solve(strategy.get.solvingStrategy, strategy.get.bounds) + // if we decide to solve this subproblem, we need to bring up the subproblem solution factors since we've already + // called process (and that normally does it) + chainComp.raiseSubproblemSolution(parentValue, subproblem) + } + }) + backwardChain(rest, done + chainComp) + } + + override def processChain(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], chainComp: ChainComponent[_, _]): Set[ProblemComponent[_]] = { + typedProcessChain(first, rest, done, chainComp) + } + +} + +object RaisingStrategy { + + def raiseIfGlobal(parent: Problem, subproblem: Problem): Boolean = { + subproblem.components.exists(pc => { + val a = subproblem.internal(pc.variable) + val b = subproblem.targets.contains(pc.element) + !a && !b + }) + } + + +} + + diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/RecursiveStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/RecursiveStrategy.scala new file mode 100644 index 00000000..2ecc7255 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/RecursiveStrategy.scala @@ -0,0 +1,44 @@ +/* + * RecursiveStrategy.scala + * A strategy that fully expands a problem and solves the subproblems in a structured manner. + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy +import com.cra.figaro.algorithm.structured._ + +/** + * A recursive strategy that applies the structured strategy at every decomposition + */ +private[figaro] class RecursiveStructuredStrategy(problem: Problem, solvingStrategy: SolvingStrategy, + rangeSizer: RangeSizer, bounds: Bounds, parameterized: Boolean) + extends StructuredStrategy(problem, solvingStrategy, + (p: Problem) => new RecursiveStructuredStrategy(p, solvingStrategy, rangeSizer, bounds, parameterized), rangeSizer, bounds, parameterized) { + problem.collection.useSingleChainFactor = true +} + +/** + * A recursive strategy that applies the flatten strategy at every decomposition. + */ +private[figaro] class RecursiveFlattenStrategy(problem: Problem, solvingStrategy: SolvingStrategy, + rangeSizer: RangeSizer, bounds: Bounds, parameterized: Boolean) + extends FlattenStrategy(problem, solvingStrategy, + (p: Problem) => new RecursiveFlattenStrategy(p, solvingStrategy, rangeSizer, bounds, parameterized), rangeSizer, bounds, parameterized) { +} + +/** + * A recursive strategy that applies the Raising strategy at every decomposition. + */ +private[figaro] class RecursiveRaisingStrategy(problem: Problem, solvingStrategy: SolvingStrategy, raisingCriteria: (Problem, Problem) => Boolean, + rangeSizer: RangeSizer, bounds: Bounds, parameterized: Boolean) + extends RaisingStrategy(problem, solvingStrategy, + (p: Problem) => new RecursiveRaisingStrategy(p, solvingStrategy, raisingCriteria, rangeSizer, bounds, parameterized), raisingCriteria, rangeSizer, bounds, parameterized) { +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/StructuredStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/StructuredStrategy.scala new file mode 100644 index 00000000..11ea353f --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/StructuredStrategy.scala @@ -0,0 +1,62 @@ +/* + * StructuredStrategy.scala + * A strategy that fully expands a problem and solves the subproblems in a structured manner. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.decompose + +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy +import com.cra.figaro.algorithm.structured._ + +/** + * A structured strategy that decomposes Chains and MakeArrays according to a given recursive strategy + * @param problem The problem to decompose + * @param solvingStrategy The solving strategy used to solve this problem + * @param recursingStrategy A function that returns a new Decomposition strategy given a nested problem + * @param rangeSizer The method to determine the range of components + * @param bounds Lower or Upper bounds + * @param parameterized Indicates if this problem parameterized + */ +private[figaro] class StructuredStrategy(problem: Problem, solvingStrategy: SolvingStrategy, + recursingStrategy: Problem => DecompositionStrategy, rangeSizer: RangeSizer, + bounds: Bounds, parameterized: Boolean) + extends DecompositionStrategy(problem, solvingStrategy, rangeSizer, bounds, parameterized) with BackwardChain { + + def execute(components: List[ProblemComponent[_]]) { + backwardChain(components, Set[ProblemComponent[_]]()) + if (problem.componentsToVisit.nonEmpty) backwardChain(problem.componentsToVisit.toList, Set[ProblemComponent[_]]()) + problem.solve(solvingStrategy, bounds) + } + + /** + * For the structured strategy, if the problem is not solved the recursing strategy is invoked + */ + def decompose(nestedProblem: Problem, done: Set[ProblemComponent[_]]): Option[DecompositionStrategy] = { + // This check is important and is what enables us to perform dynamic programming + if (!nestedProblem.solved) Some(recursingStrategy(nestedProblem)) else None + } + + def processChain(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], chainComp: ChainComponent[_, _]): Set[ProblemComponent[_]] = { + chainComp.expand() + val subStrategies = chainComp.subproblems.values.map(decompose(_, done)) + subStrategies.foreach(ds => if (ds.nonEmpty) ds.get.execute()) + process(chainComp) + backwardChain(rest, done + chainComp) + } + + def processMakeArray(first: ProblemComponent[_], rest: List[ProblemComponent[_]], done: Set[ProblemComponent[_]], maComp: MakeArrayComponent[_]): Set[ProblemComponent[_]] = { + maComp.expand() + val items = maComp.makeArray.items.take(maComp.maxExpanded).toList + val itemComponents = items.map(checkArg(_)) + val done2 = backwardChain(itemComponents, done) + process(maComp) + backwardChain(rest, done2 + maComp) + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/package.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/package.scala new file mode 100644 index 00000000..58a0746e --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/decompose/package.scala @@ -0,0 +1,34 @@ +/* + * package.scala + * Definitions of expansion and solution strategies. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy + +import com.cra.figaro.algorithm.structured.strategy.decompose.StructuredStrategy +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.structured.ProblemComponent +import com.cra.figaro.algorithm.factored.ParticleGenerator + + +package object decompose { + /** + * A strategy takes a problem and does something useful, such as computing a solution. + * Type-wise, a strategy simply does something with the problem. + */ + type Strategy = Problem => Unit + + /** + * A range sizer chooses a size of range for components corresponding to atomic elements. + */ + type RangeSizer = ProblemComponent[_] => Int + + def defaultRangeSizer(pc: ProblemComponent[_]) = ParticleGenerator.defaultMaxNumSamplesAtChain +} diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/ConstantStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/ConstantStrategy.scala new file mode 100644 index 00000000..157ff344 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/ConstantStrategy.scala @@ -0,0 +1,29 @@ +/* + * ConstantStrategy.scala + * An abstract class that defines how to solve a problem + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.solve + +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.structured.solver +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Variable + +/** + * A solving strategy that applies the same solver to every problem + */ +class ConstantStrategy(solverToUse: solver.Solver) extends SolvingStrategy { + + def solve(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + solverToUse(problem, toEliminate, toPreserve, factors) + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/SolvingStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/SolvingStrategy.scala new file mode 100644 index 00000000..437842b4 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/SolvingStrategy.scala @@ -0,0 +1,30 @@ +/* + * SolvingStrategy.scala + * An abstract class that defines how to solve a problem + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.solve + +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Variable + +/** + * An abstract class that defines how to solve a problem + */ +abstract class SolvingStrategy { + + /** + * Solve the given problem with the indicated preserve and eliminate variables, and return a list of factors representing + * the joint distribution over the preserved variables. + */ + def solve(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEBPGibbsStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEBPGibbsStrategy.scala new file mode 100644 index 00000000..8c082b20 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEBPGibbsStrategy.scala @@ -0,0 +1,54 @@ +/* + * VEBPGibbsStrategy.scala + * Strategy that chooses to solve a problem with either VE or Gibbs. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: August 11, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.solve + +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.structured.solver + +/** + * A solving strategy that chooses between VE, BP, and Gibbs based on a score of the elminiation order and determinism + */ +class VEBPGibbsStrategy(val scoreThreshold: Double, val determThreshold: Double, val bpIters: Int, val numSamples: Int, val burnIn: Int, val interval: Int, val blockToSampler: Gibbs.BlockSamplerCreator) extends SolvingStrategy { + + def solve(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val (score, order) = VariableElimination.eliminationOrder(factors, toPreserve) + if (score <= scoreThreshold) { + solver.marginalVariableElimination(problem, toEliminate, toPreserve, factors) + } else { + /* + * Choose between BP and Gibbs. + * + * Look at the variables in the problem, and find the percent that are deterministic. We label a variable as deterministic if it has any variables that + * it must be blocked with for Gibbs sampling to converge properly. + * + * If the percent of variables in a model is deterministic, it is better to run BP than Gibbs. + */ + val numDeterministicVars = toEliminate.toList.map(hasDeterminism(problem, _)).count(b => b) + toPreserve.toList.map(hasDeterminism(problem, _)).count(b => b) + val totalVars = toEliminate.size + toPreserve.size + + if ((numDeterministicVars.toDouble / totalVars) > determThreshold) { + solver.marginalBeliefPropagation(bpIters)(problem, toEliminate, toPreserve, factors) + } else { + solver.marginalGibbs(numSamples, burnIn, interval, blockToSampler)(problem, toEliminate, toPreserve, factors) + } + } + } + + def hasDeterminism(problem: Problem, v: Variable[_]): Boolean = problem.collection.variableParents(v).nonEmpty + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEBPStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEBPStrategy.scala new file mode 100644 index 00000000..bf4f0508 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEBPStrategy.scala @@ -0,0 +1,35 @@ +/* + * VEBPStrategy.scala + * An abstract class that defines how to solve a problem + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.solve + +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.algorithm.structured.solver + +/** + * A solving strategy that chooses between VE and BP based on a score of the elminiation order + */ +class VEBPStrategy(val scoreThreshold: Double, val iterations: Int) extends SolvingStrategy { + + def solve(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val (score, order) = VariableElimination.eliminationOrder(factors, toPreserve) + if (score > scoreThreshold) { + solver.marginalBeliefPropagation(iterations)(problem, toEliminate, toPreserve, factors) + } else { + solver.marginalVariableElimination(problem, toEliminate, toPreserve, factors) + } + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEGibbsStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEGibbsStrategy.scala new file mode 100644 index 00000000..3d8fb9aa --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/algorithm/structured/strategy/solve/VEGibbsStrategy.scala @@ -0,0 +1,36 @@ +/* + * VEGibbsStrategy.scala + * Strategy that chooses to solve a problem with either VE or Gibbs. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: August 11, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.algorithm.structured.strategy.solve + +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.structured.solver + +/** + * A solving strategy that chooses between VE and Gibbs based on a score of the elminiation order + */ +class VEGibbsStrategy(val scoreThreshold: Double, val numSamples: Int, val burnIn: Int, val interval: Int, val blockToSampler: Gibbs.BlockSamplerCreator) extends SolvingStrategy { + + def solve(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + val (score, order) = VariableElimination.eliminationOrder(factors, toPreserve) + if (score > scoreThreshold) { + solver.marginalGibbs(numSamples, burnIn, interval, blockToSampler)(problem, toEliminate, toPreserve, factors) + } else { + solver.marginalVariableElimination(problem, toEliminate, toPreserve, factors) + } + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/CollapsedGibbs.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/CollapsedGibbs.scala new file mode 100644 index 00000000..4e4db728 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/CollapsedGibbs.scala @@ -0,0 +1,417 @@ +/* + * CollapsedGibbs.scala + * An implementation of collapsed Gibbs sampling (on factor graphs) using the algorithm in + * "Dynamic Blocking and Collapsing for Gibbs Sampling" Gogate and Venugopal 2013 + * + * Created By: Cory Scott (cscott@cra.com) + * Creation Date: June 27, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.experimental.collapsedgibbs + +import com.cra.figaro.algorithm.lazyfactored.{ LazyValues, ValueSet } +import scala.annotation.tailrec +import com.cra.figaro.language._ +import com.cra.figaro.language.Universe._ +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.util._ +import com.cra.figaro.algorithm.factored.VariableElimination +import scala.collection.mutable.{ Map => MutableMap, Set => MutableSet} +import com.cra.figaro.algorithm.factored.factors.{ Variable, ElementVariable, InternalChainVariable, Factor} +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.factored.gibbs._ +import com.cra.figaro.algorithm.factored.VEGraph +import com.cra.figaro.algorithm.{ AlgorithmException, UnsupportedAlgorithmException } +import com.cra.figaro.library.compound.^^ + + +object CollapsedGibbs { + + // A block is just a list of variables + type Block = List[Variable[_]] + + // Information passed to BlockSampler constructor + type BlockInfo = (Block, List[Factor[Double]]) + + // Constructor for BlockSampler + type BlockSamplerCreator = BlockInfo => BlockSampler + + /** + * Create a one-time collapsed Gibbs sampler using the given number of samples and target elements. + * This sampler will use the default collapsing strategy (Heuristic) and default params for that strategy. + */ + def apply(mySamples: Int, targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply("", List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + mySamples, 0, 1, BlockSampler.default, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the given strategy for collapsing, number of samples, + * and target elements. + * Uses the default params for the strategy. + */ + def apply(strategy:String, mySamples: Int, targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + mySamples, 0, 1, BlockSampler.default, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the specified strategy and parameters, and the + * given number of samples and target elements. + */ + def apply(strategy:String, collapseParameters:Seq[Int], mySamples: Int, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, collapseParameters, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + mySamples, 0, 1, BlockSampler.default, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the given number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply("", List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + mySamples, burnIn, interval, blockToSampler, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the given strategy, number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + * Uses the default params for the strategy. + */ + def apply(strategy: String, mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + mySamples, burnIn, interval, blockToSampler, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the specified strategy and parameters, + * number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(strategy: String, collapseParameters:Seq[Int], mySamples: Int, burnIn: Int, + interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + mySamples, burnIn, interval, blockToSampler, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the + * default strategy with default parameters, given dependent universes and algorithm, + * the number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply("", + dependentUniverses, + dependentAlgorithm, + mySamples, burnIn, interval, blockToSampler, targets: _*) + + /** + * Create a one-time collapsed Gibbs sampler using the given strategy, + * dependent universes and algorithm, + * the number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(strategy:String, dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + strategy match { + case "SIMPLE" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs { + val numSamples = mySamples } + case "FACTOR" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with FactorSizeCollapseStrategy { val numSamples = mySamples } + case "DETERM" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with DeterministicCollapseStrategy { val numSamples = mySamples } + case "RECURR" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with RecurringCollapseStrategy { val numSamples = mySamples } + case _ => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with HeuristicCollapseStrategy { val numSamples = mySamples } + } + /** + * Create a one-time collapsed Gibbs sampler using the given strategy and parameters, + * dependent universes and algorithm, + * the number of samples, the number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(strategy:String, collapseParameters:Seq[Int], dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + mySamples: Int, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + strategy match { + /* + * In all the constructors below: + * alpha is the maximum degree of any variable to eliminate. + * gamma is the maximum number of edges we are allowed to add while eliminating variavles + */ + case "SIMPLE" => { + val Seq(alpha, gamma) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs { + val numSamples = mySamples } + } + case "FACTOR" => { + //factorThresh is the maximum total cost of all variables eliminated. + //when we exceed this value, collapsing stops, even if there are still candidates. + val Seq(alpha, gamma, factorThresh) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with FactorSizeCollapseStrategy { + val numSamples = mySamples + override val factorThreshold = factorThresh + } + } + case "DETERM" => { + val Seq(alpha, gamma) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with DeterministicCollapseStrategy { + val numSamples = mySamples } + } + case "RECURR" => { + //sampleResetFrequency is the frequency with which we reset and re-collapse the model. + //sampleSaveFrequency is the frequency with which we store a sample to use in our marginal estimates. + val Seq(alpha, gamma, sampleResetFrequency, sampleSaveFrequency) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with RecurringCollapseStrategy { + val numSamples = mySamples + override val sampleReset = sampleResetFrequency + override val sampleRecurrence = sampleSaveFrequency + } + } + case _ => { + val Seq(alpha, gamma) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with OneTimeProbQuerySampler with ChainApplyBlockingGibbs + with HeuristicCollapseStrategy {val numSamples = mySamples } + } + } + + /** + * Create an anytime collapsed Gibbs sampler using the given target elements. + */ + def apply(targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply("", List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + 0, 1, BlockSampler.default, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the given strategy and target elements. + */ + def apply(strategy:String, targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + 0, 1, BlockSampler.default, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the given strategy, parameters, and target elements. + */ + def apply(strategy: String, collapseParameters:Seq[Int], targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, collapseParameters, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + 0, 1, BlockSampler.default, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the default strategy with default parameters, + * given number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply("", List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + burnIn, interval, blockToSampler, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the given strategy, number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(strategy:String, burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + burnIn, interval, blockToSampler, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the given number of samples to burn in, + * the sampling interval, the BlockSampler generator, and target elements. + */ + def apply(strategy:String, collapseParameters:Seq[Int], burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply(strategy, collapseParameters, List[(Universe, List[NamedEvidence[_]])](), + (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u), + burnIn, interval, blockToSampler, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the default strategy with default paramters, + * given dependent universes and algorithm, + * the number of samples to burn in, the sampling interval, + * the BlockSampler generator, and target elements. + */ + def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, + targets: Element[_]*)(implicit universe: Universe):CollapsedProbQueryGibbs = + this.apply("", + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, targets: _*) + + /** + * Create an anytime collapsed Gibbs sampler using the given strategy (with default parameters), + * given dependent universes and algorithm, + * the number of samples to burn in, the sampling interval, + * the BlockSampler generator, and target elements. + */ + def apply(strategy: String, dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + strategy match { + case "SIMPLE" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + case "FACTOR" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with FactorSizeCollapseStrategy + case "DETERM" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with DeterministicCollapseStrategy + case "RECURR" => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with RecurringCollapseStrategy + case _ => new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with HeuristicCollapseStrategy + } + + /** + * Create an anytime collapsed Gibbs sampler using the given strategy, parameters, + * dependent universes and algorithm, + * the number of samples to burn in, the sampling interval, + * the BlockSampler generator, and target elements. + */ + def apply(strategy: String, collapseParameters:Seq[Int], dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + burnIn: Int, interval: Int, blockToSampler: BlockSamplerCreator, targets: Element[_]*)(implicit universe: Universe) = + strategy match { + /* + * In all the constructors below: + * alpha is the maximum degree of any variable to eliminate. + * gamma is the maximum number of edges we are allowed to add while eliminating variavles + */ + case "SIMPLE" => { + val Seq(alpha, gamma) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + } + //factorThresh is the maximum total cost of all variables eliminated. + //when we exceed this value, collapsing stops, even if there are still candidates. + case "FACTOR" => { + val Seq(alpha, gamma, factorThresh) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with FactorSizeCollapseStrategy { + override val factorThreshold = factorThresh + } + } + case "DETERM" => { + val Seq(alpha, gamma) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with DeterministicCollapseStrategy + } + case "RECURR" => { + //sampleResetFrequency is the frequency with which we reset and re-collapse the model. + //sampleSaveFrequency is the frequency with which we store a sample to use in our marginal estimates. + val Seq(alpha, gamma, sampleResetFrequency, sampleSaveFrequency) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with RecurringCollapseStrategy { + override val sampleReset = sampleResetFrequency + override val sampleRecurrence = sampleSaveFrequency + } + } + case _ => { + val Seq(alpha, gamma, trackSamples) = collapseParameters + new CollapsedProbQueryGibbs(universe, targets: _*)( + dependentUniverses, + dependentAlgorithm, + burnIn, interval, blockToSampler, alpha, gamma) with AnytimeProbQuerySampler with ChainApplyBlockingGibbs + with HeuristicCollapseStrategy { + override val trackingSamples = trackSamples + } + } + } + + + /** + * Use Gibbs sampling to compute the probability that the given element satisfies the given predicate. + */ + def probability[T](target: Element[T], predicate: T => Boolean): Double = { + val alg = CollapsedGibbs(10000, target) + alg.start() + val result = alg.probability(target, predicate) + alg.kill() + result + } + + /** + * Use Gibbs sampling to compute the probability that the given element has the given value. + */ + def probability[T](target: Element[T], value: T): Double = + probability(target, (t: T) => t == value) +} + diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/CollapsedProbQueryGibbs.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/CollapsedProbQueryGibbs.scala new file mode 100644 index 00000000..8a7ef9a2 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/CollapsedProbQueryGibbs.scala @@ -0,0 +1,271 @@ +/* + * CollapsedProbQueryGibbs.scala + * Core class for the collapsed Gibbs sampler. All other Collapsed Gibbs samplers extend this class, + * with traits mixed in to specify strategy. + * + * Created By: Cory Scott (cscott@cra.com) + * Creation Date: July 25, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.experimental.collapsedgibbs + +import com.cra.figaro.algorithm.lazyfactored.{ LazyValues, ValueSet } +import scala.annotation.tailrec +import com.cra.figaro.language._ +import com.cra.figaro.language.Universe._ +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.util._ +import com.cra.figaro.algorithm.factored.VariableElimination +import scala.collection.mutable.{ Map => MutableMap, Set => MutableSet} +import com.cra.figaro.algorithm.factored.factors.{ Variable, ElementVariable, InternalChainVariable, Factor} +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.factored.gibbs._ +import com.cra.figaro.algorithm.factored.VEGraph +import com.cra.figaro.algorithm.{ AlgorithmException, UnsupportedAlgorithmException } +import com.cra.figaro.library.compound.^^ + +/** + * CollapsedProbQueryGibbs only uses graph information and the list of targets to collapse some variables. + * extend with HeuristicCollapser or RecurringCollapser to implement other features described in Gogate et. al. + */ +abstract class CollapsedProbQueryGibbs(override val universe: Universe, targets: Element[_]*)( +override val dependentUniverses: List[(Universe, List[NamedEvidence[_]])], +override val dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, +override val burnIn: Int, override val interval: Int, +override val blockToSampler: Gibbs.BlockSamplerCreator, alphaIn: Int = 10, gammaIn:Int = 1000, + upperBounds: Boolean = false) +extends ProbQueryGibbs(universe, targets: _*)(dependentUniverses, dependentAlgorithm, 0, interval, + blockToSampler, upperBounds) +with CollapsedProbabilisticGibbs { + override def initialize() = { + super.initialize() + //need to create the original blocks before we eliminate any variables + originalBlocks = createBlocks() + alpha = alphaIn + gamma = gammaIn + targs = targets + alphaChoose2 = math.max((alpha*(alpha-1.0))/2.0, 1.0) + varsInOrder = variables.toList + targetVariables = targets.toList.map(x => Variable(x)) + globalGraph = new VEGraph(factors) + collapseVariables() + //create the correct blocks by filtering out eliminated variables + val blocks = correctBlocks(originalBlocks) + //this is all the same as in ProbQueryGibbs + blockSamplerCreate = blockToSampler + blockSamplers = blocks.map(block => + blockToSampler((block, factors.filter(_.variables.exists(block.contains(_)))))) + val initialSample = WalkSAT(factors, variables, semiring, chainMapper) + variables.foreach(v => currentSamples(v) = initialSample(v)) + for (_ <- 1 to burnIn) sampleAllBlocks() + } +} + +trait CollapsedProbabilisticGibbs extends ProbabilisticGibbs { + /** + * We need a list of variables in order so we can access them by index. + */ + var varsInOrder: List[Variable[_]] = List() + + + var originalBlocks:List[Gibbs.Block] = List() + /** + * List of variables corresponding to target elements. + * Creating these is memoized, so we don't need to worry about duplicates. + */ + var targetVariables: List[Variable[_]] = List() + + /** + * Only variables with alpha or fewer neighbors in the primal graph are candidates for collapsing. + */ + var alpha:Int = _ + /* + gamma controls the trade-off between blocking and collapsing. See collapseVariables. + */ + var gamma:Int = _ + + + /** + * We use ( alpha C 2 ) often, may as well store it. + */ + var alphaChoose2:Double = _ + + + /** + * globalGraph lets us traverse the primal graph. + */ + var globalGraph: VEGraph = _ + + + /** + * Store which elements are our target variables so that subclasses can make use of them. + */ + var targs:Seq[Element[_]] = _ + + /** + * Store which elements are our target variables so that subclasses can make use of them. + */ + var upperB:Boolean = _ + + /* + * Keep the BlockSamplerCreator for later use. + */ + var blockSamplerCreate: Gibbs.BlockSamplerCreator = _ + + /** + * Returns how many edges would be added to the primal graph by removing var1. + * Note: this is number of edges added, NOT net edges added and removed. + * Source paper is somewhat ambiguous on whether this should be added or net. + */ + def graphTerm[T](var1: Variable[T]):Double = { + //val (updatedGraph, trash) = globalGraph.eliminate(var1) + val updatedGraph = globalGraph.eliminate(var1)._1 + val graphTerm = (variables.filter(x => x != var1).toList.map(varT => updatedGraph.info(varT).neighbors.toList.length - 1 ).sum/2 + - variables.filter(x => x != var1).toList.map(varT => (globalGraph.info(varT).neighbors.filter(x => x != var1).toList.length - 1) ).sum/2) + graphTerm + } + + + /** + * We want to alter the original blocks so that we filter out any variables which have + * been eliminated. If the original blocks overlapped a lot, then there'll be a lot of + * redundancy in the filtered blocks, so we take a further step of eliminating any block + * xs which is fully contained in another block ys. + */ + def correctBlocks(originalBlocks:List[Gibbs.Block]):List[Gibbs.Block] = { + val initial = MutableSet[Set[Variable[_]]]() + for {x <- originalBlocks.map(bl => bl.filter(y => variables.contains(y)).toSet).distinct} { + initial add x + } + for {xs <- initial} { + for {ys <- initial}{ + //println("&" + xs + " " + ys) + if ((xs subsetOf ys) && (xs != ys)) { + initial remove xs + } + } + } + initial.map(x => x.toList).toList + } + + /** + * The heuristic of a node is how many edges would be added to the primal graph by removing that variable. + * Because we make a clique over the variable's neighbors. + * Since we only eliminate variables with alpha or fewer neighbors, this is capped at (alpha C 2). + * So we return the number of edges as a percentage of (alpha C 2). + */ + def graphHeuristicFunction[T](var1: Variable[T]):Double = { + ((alphaChoose2 - graphTerm(var1))/alphaChoose2) + } + + /** + * Eliminate a variable. This follows the same approach as in VariableElimination.scala. + }*/ + def eliminate( + variable: Variable[_], + factors: MultiSet[Factor[Double]], + map: MutableMap[Variable[_], MultiSet[Factor[Double]]]): Unit = { + val varFactors = map(variable) + if (varFactors nonEmpty) { + //flatten all of varFactors into one factor + val productFactor = varFactors reduceLeft (_.product(_)) + //marginalize that factor to all variables other than variable + val resultFactor = productFactor.marginalizeTo( + productFactor.variables.filter(_ != variable): _*) + //update our multiset of factors, and our map variables ->: factors + varFactors.foreach(factors.removeOne(_)) + factors.addOne(resultFactor) + varFactors.foreach(removeFactor(_, map)) + map -= variable + addFactor(resultFactor, map) + } + } + + + /** + * Marginalize a factor to a particular variable. + */ + def marginalizeToTarget(factor: Factor[Double], target: Variable[_]) = { + val unnormalizedTargetFactor = factor.marginalizeTo(target) + val z = unnormalizedTargetFactor.foldLeft(semiring.zero, _ + _) + //val targetFactor = Factory.make[Double](unnormalizedTargetFactor.variables) + val targetFactor = unnormalizedTargetFactor.mapTo((d: Double) => d ) + targetFactor + } + + /** + * Marginalize all factors to their component variables. + */ + def marginalize(resultFactor: Factor[Double]) = + variables.map(marginalizeToTarget(resultFactor, _)).toList + + /** + * Combine all the remaining factors into one 'result factor', as in VE. + */ + def makeResultFactor(factorsAfterElimination: MultiSet[Factor[Double]]): Factor[Double] = { + // It is possible that there are no factors (this will happen if there are no queries or evidence). + // Therefore, we start with the unit factor and use foldLeft, instead of simply reducing the factorsAfterElimination. + factorsAfterElimination.foldLeft(Factory.unit(semiring))(_.product(_)) + } + + /** + * add a factor to the list + */ + def addFactor[T](factor: Factor[T], map: MutableMap[Variable[_], MultiSet[Factor[T]]]): Unit = + factor.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).addOne(factor))) + + /** + * remove a factor from the list + */ + def removeFactor[T](factor: Factor[T], map: MutableMap[Variable[_], MultiSet[Factor[T]]]): Unit = + factor.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).removeOne(factor))) + + + /** + * Sort variables by the target heuristic, if they have fewer than alpha neighbors and are not targets. + */ + def sortByHeuristic(varList:List[Variable[_]], HeuristicMap:MutableMap[Variable[_], Double]) = { + varList.filter(x => !targetVariables.contains(x) && + (globalGraph.info(x).neighbors.toList.length - 1) <= alpha).sortWith(HeuristicMap(_) > HeuristicMap(_)) + } + /** + * Perform the collapsing step. + */ + def collapseVariables() = { + //store the heuristic for every variable, so we don't have to calculate it as often. + val graphHeuristic:MutableMap[Variable[_], Double] = MutableMap() ++ variables.map(v => + v-> graphHeuristicFunction(v)).toMap + //sort the variables using stored values + var sortedVars = sortByHeuristic(variables.toList, graphHeuristic) + var edgesAdded:Int = 0 + //map and tempFactors are to help with variable elimination. + val map = MutableMap[Variable[_], MultiSet[Factor[Double]]]() + val tempFactors = HashMultiSet[Factor[Double]]() + factors foreach (x => tempFactors.addOne(x)) + for {fact <- tempFactors} { + fact.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).addOne(fact))) + } + //we collapse variables until either we are out of candidates or we've added too many edges. + while (sortedVars.length > 0 && edgesAdded < gamma) { + //eliminate the variable with highest heuristic. + var toRemove:Variable[_] = sortedVars(0) + eliminate(toRemove, tempFactors, map) + variables = variables.filter(_ != toRemove) + var oldNeighbors = globalGraph.info(toRemove).neighbors.filter(_ != toRemove) + globalGraph = new VEGraph(tempFactors) + //update all of the neighbors of the variable we just eliminated, since their scores will have changed. + for {x <- oldNeighbors} graphHeuristic(x) = graphHeuristicFunction(x) + sortedVars = sortByHeuristic(variables.toList, graphHeuristic) + } + //update the list of factors to reflect the changes we've made. + factors = tempFactors.elements + } +} + + + diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/Collapsers.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/Collapsers.scala new file mode 100644 index 00000000..f8ad0bbb --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/collapsedgibbs/Collapsers.scala @@ -0,0 +1,361 @@ +/* + * Collapsers.scala + * Three alternative collapsing strategies for collapsed Gibbs sampling. + * Each of these is a trait that must be mixed in to a CollapsedProbQueryGibbs. + * + * Created By: Cory Scott (cscott@cra.com) + * Creation Date: July 21, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.collapsedgibbs + +import com.cra.figaro.algorithm.lazyfactored.{ LazyValues, ValueSet } +import scala.annotation.tailrec +import com.cra.figaro.language._ +import com.cra.figaro.language.Universe._ +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.util._ +import com.cra.figaro.algorithm.factored.VariableElimination +import scala.collection.mutable.{ Map => MutableMap, Set => MutableSet} +import com.cra.figaro.algorithm.factored.factors.{ Variable, ElementVariable, InternalChainVariable, Factor} +import com.cra.figaro.algorithm.structured.Problem +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.factored.gibbs._ +import com.cra.figaro.algorithm.factored.VEGraph +import com.cra.figaro.algorithm.{ AlgorithmException, UnsupportedAlgorithmException } +import com.cra.figaro.library.compound.^^ + + +/** + * HeuristicCollapsedGibbs adds the Hellinger-distance-based term to the elimination heuristic. + * So the heuristic is now based on the marginal probabilities and pairwise marginals. + * These have to be estimated by some number of saved samples. + */ +trait HeuristicCollapseStrategy extends CollapsedProbabilisticGibbs { + var numSamplesSeenSoFar: Int = 0 + var totalSamples: Int = 0 + var marginals: MutableMap[Int,MutableMap[Int,Double]] = MutableMap() + var pairwiseMarignals: MutableMap[(Int, Int), MutableMap[(Int, Int), Double]] = MutableMap() + var hellingerDistances: MutableMap[(Int, Int), Double] = MutableMap() + // trackingSamples is the number of initial samples we take to estimate correlation bewteen variables. + val trackingSamples = 200 + + override def initialize() = { + super.initialize() + totalSamples = 0 + val neededElements = getNeededElements(targetElements, Int.MaxValue)._1 + factors = getFactors(neededElements, targetElements, upperB) + variables = factors.flatMap(_.variables).toSet + varsInOrder = variables.toList + targetVariables = targs.toList.map(x => Variable(x)) + globalGraph = new VEGraph(factors) + //Initialize the maps that keep track of marginals and pairwise marginals. + resetMarginals() + //Get the initial samples. + for {_ <- 1 to trackingSamples} sampleAllBlocksWithTracking + //Compute the Hellinger Distances, then use them to collapse some variables. + updateDistances() + collapseVariables() + //Perform update the original blocks to reflect eliminated variables. + val blocks = correctBlocks(originalBlocks) + blockSamplers = blocks.map(block => blockSamplerCreate((block, + factors.filter(_.variables.exists(block.contains(_)))))) + val initialSample = WalkSAT(factors, variables, semiring, + (chain: Chain[_,_]) => LazyValues(chain.universe).getMap(chain).values.map(Variable(_)).toSet) + variables.foreach(v => currentSamples(v) = initialSample(v)) + for (_ <- 1 to burnIn) sampleAllBlocks() + } + + /** + * Sample all blocks, then store that sample in the marginal and p.m. maps. + */ + def sampleAllBlocksWithTracking() = { + super.sampleAllBlocks() + numSamplesSeenSoFar += 1 + for {testVar <- 0 until varsInOrder.length} { + val samp1 = currentSamples.getOrElse(varsInOrder(testVar), -1) + marginals(testVar)(samp1) = marginals(testVar).getOrElse(samp1, 0.0) + 1.0 + // only need to go up to testVar, since we can fill upper and lower triangles of matrix simultaneously. + for {otherVar <- 0 until testVar} { + val samp2 = currentSamples.getOrElse(varsInOrder(otherVar),-1) + pairwiseMarignals((testVar, otherVar))((samp1, samp2)) = pairwiseMarignals((testVar, otherVar)).getOrElse(((samp1, samp2)) ,0.0) + 1.0 + pairwiseMarignals((otherVar, testVar))((samp2, samp1)) = pairwiseMarignals((otherVar, testVar)).getOrElse(((samp2, samp1)) ,0.0) + 1.0 + } + } + } + + /** + * Reset all the marginal and p.m. maps to empty maps. + */ + def resetMarginals() = { + numSamplesSeenSoFar = 0 + for {testVar <- 0 until varsInOrder.length} { + marginals(testVar) = MutableMap() + for {otherVar <- 0 until testVar} { + pairwiseMarignals((testVar, otherVar)) = MutableMap() + pairwiseMarignals((otherVar, testVar)) = MutableMap() + } + } + } + + /** + * Use the marginal maps to compute Hellinger maps. + */ + def updateDistances() = { + for {testVar <- 0 until varsInOrder.length} { + for {otherVar <- 0 until testVar} { + val dist = distributionDistance(varsInOrder(testVar), varsInOrder(otherVar)) + hellingerDistances((testVar, otherVar)) = dist + hellingerDistances((otherVar, testVar)) = dist + } + } + + } + + /** + * Hellinger distance is defined in the source paper (amongst other places). + * It's the sum over all values of X1 and X2 of (sqrt(P(X1,X2)) - sqrt(P(X1)*P(X2)))^2 + */ + def distributionDistance[T,U](var1: Variable[T], var2: Variable[U]) = { + var dist = 0.0 + var index1 = varsInOrder.indexOf(var1) + var index2 = varsInOrder.indexOf(var2) + for {a <- 0 to var1.range.length} { + for {b <- 0 to var2.range.length} { + dist += (math.pow(math.sqrt(pairwiseMarignals.getOrElse((index1, index2), Map[(Int, Int), Double]()).getOrElse((a, b), 0.0)/numSamplesSeenSoFar) + - math.sqrt(marginals.getOrElse(index1,Map[Int,Double]()).getOrElse(a,0.0)*marginals.getOrElse(index2,Map[Int,Double]()).getOrElse(b,0.0))/numSamplesSeenSoFar, 2.0)) + } + } + dist + } + + /** + * Compute the score of a given variable. + */ + override def graphHeuristicFunction[T](var1: Variable[T]) = { + //get number of edges left to add and distance from this variable to others + var index = varsInOrder.indexOf(var1) + var otherIndices = for {v <- 0 until varsInOrder.length if v != index} yield v + //get the graph-based score from superclass, and add the score from Hellinger Distances + super.graphHeuristicFunction(var1) + otherIndices.map(x => hellingerDistances.getOrElse((index, x), 0.0)).sum + } + + /** + * Perform the collapsing step. + */ + override def collapseVariables() = { + //store the heuristic for every variable, so we don't have to calculate it as often. + var graphHeuristic:MutableMap[Variable[_], Double] = MutableMap() ++ variables.map(v => v-> graphHeuristicFunction(v)).toMap + //sort the variables using stored values + var sortedVars = sortByHeuristic(variables.toList, graphHeuristic) + var edgesAdded:Int = 0 + //map and tempFactors are to help with variable elimination + var map = MutableMap[Variable[_], MultiSet[Factor[Double]]]() + var tempFactors = HashMultiSet[Factor[Double]]() + factors foreach (x => tempFactors.addOne(x)) + for {fact <- tempFactors} { + fact.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).addOne(fact))) + } + //we collapse variables until either we are out of candidates or we've added too many edges + while (sortedVars.length > 0 && edgesAdded < gamma) { + //eliminate the variable with highest heuristic. + var toRemove:Variable[_] = sortedVars(0) + eliminate(toRemove, tempFactors, map) + variables = variables.filter(_ != toRemove) + var oldNeighbors = globalGraph.info(toRemove).neighbors.filter(_ != toRemove) + globalGraph = new VEGraph(tempFactors) + //update all of the neighbors of the variable we just eliminated, since their scores will have changed. + for {x <- oldNeighbors} graphHeuristic(x) = graphHeuristicFunction(x) + /** In this version of collapseVariables, we also have to subtract the Hellinger Distance + * between our removed variable and all other variables from each of their scores. + */ + for {v <- variables.filter(!oldNeighbors.contains(_))} graphHeuristic(v) -= hellingerDistances.getOrElse((varsInOrder.indexOf(toRemove), varsInOrder.indexOf(v)), 0.0) + sortedVars = sortByHeuristic(variables.toList, graphHeuristic) + } + //update the list of factors to reflect the changes we've made. + //factors = marginalize(makeResultFactor(tempFactors)) + factors = tempFactors.elements + } + +} + + +/** + * This trait causes variables to collapsed until the total summed size of all of the factors collapsed + * thus far exceeds a threshold. + */ +trait FactorSizeCollapseStrategy extends CollapsedProbabilisticGibbs { + + val factorThreshold:Int = 1000 + + override def makeResultFactor(factorsAfterElimination: MultiSet[Factor[Double]]): Factor[Double] = { + //println(factorsAfterElimination.map((x:Factor[Double]) => x.size)) + super.makeResultFactor(factorsAfterElimination) + } + + /** + * Collapse variables + */ + override def collapseVariables() = { + //store the heuristic for every variable, so we don't have to calculate it as often. + var graphHeuristic:MutableMap[Variable[_], Double] = MutableMap() ++ variables.map(v => v-> graphHeuristicFunction(v)).toMap + //sort the variables using stored values + var sortedVars = sortByHeuristic(variables.toList, graphHeuristic) + var edgesAdded:Int = 0 + //map and tempFactors are to help with variable elimination + val map = MutableMap[Variable[_], MultiSet[Factor[Double]]]() + var tempFactors = HashMultiSet[Factor[Double]]() + factors foreach (x => tempFactors.addOne(x)) + for {fact <- tempFactors} { + fact.variables foreach (v => map += v -> (map.getOrElse(v, HashMultiSet()).addOne(fact))) + } + var costSum = 0 + //we collapse variables until either we are out of candidates or we've added too many edges + while (costSum < factorThreshold && sortedVars.length > 0) { + //eliminate the variable with highest heuristic. + //println(tempFactors.map((x:Factor[Double]) => x.variables.map(y => varsInOrder.indexOf(y)))) + var toRemove:Variable[_] = sortedVars(0) + val cost = map(toRemove).map((x:Factor[Double]) => x.size).product + //check to see if adding this cost will bring us above threshold. If so, move on to another variable. + if (cost + costSum < factorThreshold) { + eliminate(toRemove, tempFactors, map) + var oldNeighbors = globalGraph.info(toRemove).neighbors.filter(_ != toRemove) + globalGraph = new VEGraph(tempFactors) + variables = variables.filter(_ != toRemove) + //update all of the neighbors of the variable we just eliminated, since their scores will have changed. + for {x <- oldNeighbors} graphHeuristic(x) = graphHeuristicFunction(x) + sortedVars = sortByHeuristic(variables.toList, graphHeuristic) + costSum += cost + } + else { + sortedVars = sortByHeuristic(sortedVars.filter(_ != toRemove), graphHeuristic) + } + } + factors = tempFactors.elements + } + +} + + + +/** + * Experimental collapser that doesn't need to calculate marginal probabilities. + * The original paper doesn't distinguish between model variables, or use any meta-information + * about the variables. + * Since Figaro knows which variables are deterministic, we can use this as a proxy for the + * correlation heuristic. + */ +trait DeterministicCollapseStrategy extends CollapsedProbabilisticGibbs { + + /** + * For this strategy we need access to the distance maps as well as the blocks. + */ + var hellingerDistances: MutableMap[(Int, Int), Double] = MutableMap() + var blocks: List[Gibbs.Block] = _ + + /** + * Unlike other + */ + override def initialize() = { + super.initialize() + updateDistances() + collapseVariables() + //Perform update the original blocks to reflect eliminated variables. + val blocks = correctBlocks(originalBlocks) + blockSamplers = blocks.map(block => blockSamplerCreate((block, + factors.filter(_.variables.exists(block.contains(_)))))) + val initialSample = WalkSAT(factors, variables, semiring, + (chain: Chain[_,_]) => LazyValues(chain.universe).getMap(chain).values.map(Variable(_)).toSet) + variables.foreach(v => currentSamples(v) = initialSample(v)) + for (_ <- 1 to burnIn) sampleAllBlocks() + } + + /** + * Override the distance function. + */ + def updateDistances() = { + blocks = createBlocks() + for {testVar <- 0 until varsInOrder.length} { + for {otherVar <- 0 until testVar} { + var dist = 0.0 + /* Instead of the given definition of Hellinger Distance, use the following distance function: + * For every block that contains both variables: + * sum up the difference in index between the first variable and the second variable, + * normalized by block size. + * Since blocks are added by starting with stochastic variables and recursively adding + * children, this should be a rough measure of how much otherVar depends on testVar + */ + for {b <- blocks} { + if (b.contains(testVar) && b.contains(otherVar)) { + dist += (b.indexOf(otherVar) - b.indexOf(testVar))/b.size + } + } + hellingerDistances((testVar, otherVar)) = dist + hellingerDistances((otherVar, testVar)) = -1.0*dist + } + } + + } + + /** + * Override the heuristic function to use the new measure of distance. + */ + override def graphHeuristicFunction[T](var1: Variable[T]) = { + //get number of edges left to add and distance from this variable to others + var index = varsInOrder.indexOf(var1) + var otherIndices = for {v <- 0 until varsInOrder.length if v != index} yield v + super.graphHeuristicFunction(var1) + otherIndices.map(x => hellingerDistances.getOrElse((index, x), 0.0)).sum + } + +} + + +/** + * In the paper, the authors recommend updating the marginals every N samples and re-collapsing every few iterations. + * In practice, this is pretty slow. + * This trait will keep a running tally of samples of each of the used variables and re-collapse the factor graph + * (starting from the initial graph) periodically. + */ +trait RecurringCollapseStrategy extends HeuristicCollapseStrategy { + /** + * How often we want to re-collapse. + */ + val sampleReset:Int = 2000 + + /** + * How often we want to save one of our samples for marginal estimation. + */ + val sampleRecurrence: Int = 50 + + /** + * We override resetMarginals() to do nothing so that we don't get rid of our saved marginal data every time we re-initialize. + * If we haven't built the marginal maps yes, build them, else do nothing. + */ + override def resetMarginals() = { + if (marginals.isEmpty) { + super.resetMarginals() + } + } + + /** + * Every sampleRecurrence many samples, we alter the marginals and PairwiseMarginal Maps to record the current sample. + * Every sampleReset many samples, we re-initialize. + */ + override def sampleAllBlocks() = { + totalSamples += 1 + if ((totalSamples % sampleRecurrence) == 0) { + sampleAllBlocksWithTracking() + } + else if (totalSamples == sampleReset) { + initialize() + } + else { + super.sampleAllBlocks() + } + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/AnytimeMarginalMAP.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/AnytimeMarginalMAP.scala new file mode 100644 index 00000000..a4a8813e --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/AnytimeMarginalMAP.scala @@ -0,0 +1,50 @@ +/* + * AnytimeMarginalMAP.scala + * Anytime algorithms that compute the most likely values of some elements, and marginalize over all other elements. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Jun 27, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm._ +import com.cra.figaro.language._ +import akka.pattern.ask + +/** + * Anytime algorithms that compute most likely values of some elements, and marginalize over all other elements. + * A class that implements this trait must implement initialize, runStep, and computeMostLikelyValue methods. + */ +trait AnytimeMarginalMAP extends MarginalMAPAlgorithm with Anytime { + /** + * A message instructing the handler to compute the most likely value of the target element. + */ + case class ComputeMostLikelyValue[T](target: Element[T]) extends Service + + /** + * A message from the handler containing the most likely value of the previously requested element. + */ + case class MostLikelyValue[T](value: T) extends Response + + def handle(service: Service): Response = + service match { + case ComputeMostLikelyValue(target) => + MostLikelyValue(computeMostLikelyValue(target)) + } + + protected def doMostLikelyValue[T](target: Element[T]): T = { + awaitResponse(runner ? Handle(ComputeMostLikelyValue(target)), messageTimeout.duration) match { + case MostLikelyValue(result) => result.asInstanceOf[T] + case _ => { + println("Error: Response not recognized from algorithm") + target.value + } + } + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPAlgorithm.scala new file mode 100644 index 00000000..8eeb7fae --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPAlgorithm.scala @@ -0,0 +1,59 @@ +/* + * MarginalMAPAlgorithm.scala + * Algorithms that compute the most likely values of some elements, and marginalize over all other elements. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Jun 2, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.{Algorithm, AlgorithmException, AlgorithmInactiveException} +import com.cra.figaro.language._ + +/** + * Algorithms that compute max a posteriori (MAP) values of some elements, and marginalize over all other elements. + */ +trait MarginalMAPAlgorithm extends Algorithm { + class NotATargetException[T](target: Element[T]) extends AlgorithmException + + def universe: Universe + + /** + * Elements for which to perform MAP queries. This algorithm marginalizes over elements not in this list. + */ + def mapElements: Seq[Element[_]] + + /* + * Particular implementations of algorithm must provide the following method. + */ + def computeMostLikelyValue[T](target: Element[T]): T + + /* + * Defined in one time and anytime marginal MAP versions of this class. Does not need to be defined + * by particular algorithm implementations. + */ + protected def doMostLikelyValue[T](target: Element[T]): T + + private def check[T](target: Element[T]): Unit = { + if (!active) throw new AlgorithmInactiveException + if (!(mapElements contains target)) throw new NotATargetException(target) + } + + /** + * Returns an estimate of the max a posteriori value of the target. + * @throws NotATargetException if called on a target that is not in the list of MAP elements. + * @throws AlgorithmInactiveException if the algorithm is inactive. + */ + def mostLikelyValue[T](target: Element[T]): T = { + check(target) + doMostLikelyValue(target) + } + + universe.registerAlgorithm(this) +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPBeliefPropagation.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPBeliefPropagation.scala new file mode 100644 index 00000000..cd8e8841 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPBeliefPropagation.scala @@ -0,0 +1,177 @@ +/* + * MarginalMAPBeliefPropagation.scala + * A marginal MAP belief propagation algorithm. Based on the algorithm by Liu and Ihler (2013). + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Jun 10, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.factored.beliefpropagation._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler +import com.cra.figaro.language._ + +abstract class MarginalMAPBeliefPropagation(override val universe: Universe, targets: Element[_]*)( + val dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + val dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double) + extends MarginalMAPAlgorithm + with ProbabilisticBeliefPropagation { + + val targetElements = targets.toList + + val mapElements = targetElements + + /* + * Variables corresponding to MAP elements. This is set in the initialize() method. + */ + protected var maxVariables: Set[Variable[_]] = _ + + /** + * Value used to compute arg max messages. This could be thought of as an inverse + * "temperature", but here it is a large fixed value. + */ + val argMaxFactor = 1e5 + + override protected def getNewMessageFactorToVar(fn: FactorNode, vn: VariableNode) = { + val vnFactor = factorGraph.getLastMessage(vn, fn) + + if(maxVariables.contains(vn.variable)) { + val total = beliefMap(fn).combination(vnFactor, LogSumProductSemiring().divide) + + // Use sum-product to sum over sum variables + val sumOverSumVars = total.marginalizeTo(fn.variables.intersect(maxVariables).toSeq:_*) + // Use max-product to sum over max variables except vn.variable + val sumOverMaxVars = sumOverSumVars.marginalizeToWithSum(LogMaxProductSemiring().sum, vn.variable) + sumOverMaxVars + } + else { + // Use sum-product to sum over sum variables, note that we don't divide by the last message here + val beliefOverMaxVars = beliefMap(fn).marginalizeTo(fn.variables.intersect(maxVariables).toSeq:_*) + val maxBelief = beliefOverMaxVars.foldLeft(LogSumProductSemiring().zero, _ max _) + // Filter the indices that maximize the belief over the max variables in this factor + // This approximation of the "indicator function" is used for two reasons: + // 1) To prevent division by zero when we divide the belief map by the last message + // 2) To give nonzero weight to values close to the arg max that differ due to floating point errors + val argMaxIndicator = beliefOverMaxVars.mapTo(d => (d - maxBelief) * argMaxFactor) + + // Now the total includes the indicator function + val total = beliefMap(fn).combination(vnFactor, LogSumProductSemiring().divide).product(argMaxIndicator) + + // Use sum-product to sum over all variables except vn.variable + val sumOverAllVars = total.marginalizeTo(vn.variable) + sumOverAllVars + } + } + + /* + * We use sum product by default and switch to max product only when needed. + * Notably, we only use the sum operation through getNewMessageFactorToVar. + */ + val semiring = SumProductSemiring() + + override def initialize() = { + val (neededElements, _) = getNeededElements(universe.activeElements, Int.MaxValue) + factorGraph = new BasicFactorGraph(getFactors(neededElements, targetElements), logSpaceSemiring()) + maxVariables = factorGraph.getNodes.collect { + case VariableNode(ev: ElementVariable[_]) if mapElements.contains(ev.element) => ev + }.toSet + super.initialize() + } + + def computeMostLikelyValue[T](target: Element[T]): T = { + val beliefs = getBeliefsForElement(target) + beliefs.maxBy(_._1)._2 + } +} + +object MarginalMAPBeliefPropagation { + /** + * Creates a One Time marginal MAP belief propagation computer in the current default universe. + * @param myIterations Iterations of mixed-product BP to run. + * @param targets MAP elements, which can be queried. Elements not supplied here are summed over. + */ + def apply(myIterations: Int, targets: Element[_]*)(implicit universe: Universe) = + new MarginalMAPBeliefPropagation(universe, targets: _*)( + List(), (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) + with OneTimeProbabilisticBeliefPropagation with OneTimeMarginalMAP { val iterations = myIterations } + + /** + * Creates an Anytime marginal MAP belief propagation computer in the current default universe. + * @param targets MAP elements, which can be queried. Elements not supplied here are summed over. + */ + def apply(targets: Element[_]*)(implicit universe: Universe) = + new MarginalMAPBeliefPropagation(universe, targets: _*)( + List(), (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) + with AnytimeProbabilisticBeliefPropagation with AnytimeMarginalMAP + + /** + * Creates a One Time marginal MAP belief propagation computer in the current default universe. + * @param dependentUniverses Dependent universes for this algorithm. + * @param myIterations Iterations of mixed-product BP to run. + * @param targets MAP elements, which can be queried. Elements not supplied here are summed over. + */ + def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], myIterations: Int, targets: Element[_]*)(implicit universe: Universe) = + new MarginalMAPBeliefPropagation(universe, targets: _*)( + dependentUniverses, (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) + with OneTimeProbabilisticBeliefPropagation with OneTimeMarginalMAP { val iterations = myIterations } + + /** + * Creates an Anytime marginal MAP belief propagation computer in the current default universe. + * @param dependentUniverses Dependent universes for this algorithm. + * @param targets MAP elements, which can be queried. Elements not supplied here are summed over. + */ + def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], targets: Element[_]*)(implicit universe: Universe) = + new MarginalMAPBeliefPropagation(universe, targets: _*)( + dependentUniverses, (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) + with AnytimeProbabilisticBeliefPropagation with AnytimeMarginalMAP + + /** + * Creates a One Time marginal MAP belief propagation computer in the current default universe. + * @param dependentUniverses Dependent universes for this algorithm. + * @param dependentAlgorithm Used to determine algorithm for computing probability of evidence in dependent universes. + * @param myIterations Iterations of mixed-product BP to run. + * @param targets MAP elements, which can be queried. Elements not supplied here are summed over. + */ + def apply( + dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + myIterations: Int, targets: Element[_]*)(implicit universe: Universe) = + new MarginalMAPBeliefPropagation(universe, targets: _*)( + dependentUniverses, dependentAlgorithm) + with OneTimeProbabilisticBeliefPropagation with OneTimeMarginalMAP { val iterations = myIterations } + + /** + * Creates an Anytime marginal MAP belief propagation computer in the current default universe. + * @param dependentUniverses Dependent universes for this algorithm. + * @param dependentAlgorithm Used to determine algorithm for computing probability of evidence in dependent universes. + * @param targets MAP elements, which can be queried. Elements not supplied here are summed over. + */ + def apply( + dependentUniverses: List[(Universe, List[NamedEvidence[_]])], + dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, + targets: Element[_]*)(implicit universe: Universe) = + new MarginalMAPBeliefPropagation(universe, targets: _*)( + dependentUniverses, dependentAlgorithm) + with AnytimeProbabilisticBeliefPropagation with AnytimeMarginalMAP + + /** + * Use belief propagation to compute the most likely value of the given element. + * Runs 10 iterations of mixed-product BP. + * @param target Element for which to compute MAP value. + * @param mapElements Additional elements to MAP. Elements not in this list are summed over. + */ + def mostLikelyValue[T](target: Element[T], mapElements: Element[_]*): T = { + val alg = MarginalMAPBeliefPropagation(10, (target +: mapElements).distinct:_*) + alg.start() + val result = alg.mostLikelyValue(target) + alg.kill() + result + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPVEStrategy.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPVEStrategy.scala new file mode 100644 index 00000000..f464fd02 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/MarginalMAPVEStrategy.scala @@ -0,0 +1,44 @@ +/* + * MarginalMAPVEStrategy.scala + * A class that solves a marginal MAP problem using VE. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.factored.factors.{Factor, Variable} +import com.cra.figaro.algorithm.structured.{NestedProblem, Problem, solver} +import com.cra.figaro.algorithm.structured.strategy.solve.SolvingStrategy + +/** + * A solving strategy that uses MPE VE to solve non-nested problems, and performs the MAP step at the top level. + * It is assumed that at the top level, "toPreserve" elements are the MAP elements. + */ +class MarginalMAPVEStrategy extends SolvingStrategy { + + def solve(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): + (List[Factor[Double]], Map[Variable[_], Factor[_]]) = { + problem match { + case _: NestedProblem[_] => { + // A problem needed for the initial step of summing out the non-MAP variables + // Use marginal VE for this + solver.marginalVariableElimination(problem, toEliminate, toPreserve, factors) + } + case _ => { + // Sum over the remaining non-MAP variables (i.e. toEliminate), and MAP the rest (i.e. toPreserve) + // marginalizedFactors is a set of factors over just the MAP variables + val (marginalizedFactors, _) = solver.marginalVariableElimination(problem, toEliminate, toPreserve, factors) + // Now that we have eliminated the sum variables, we effectively just do MPE over the remaining variables + // For MPE, we eliminate all remaining variables (i.e. toPreserve), and preserve no variables (i.e. Set()) + solver.mpeVariableElimination(problem, toPreserve, Set(), marginalizedFactors) + } + } + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/OneTimeMarginalMAP.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/OneTimeMarginalMAP.scala new file mode 100644 index 00000000..a32cadcf --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/OneTimeMarginalMAP.scala @@ -0,0 +1,25 @@ +/* + * OneTimeMarginalMAP.scala + * One time algorithms that compute the most likely values of some elements, and marginalize over all other elements. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Jun 2, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.OneTime +import com.cra.figaro.language._ + +/** + * One-time algorithms that compute the most likely values of some elements, and marginalize over others. + * A class that implements this trait must implement run and computeMostLikelyValue methods. + */ +trait OneTimeMarginalMAP extends MarginalMAPAlgorithm with OneTime { + protected def doMostLikelyValue[T](target: Element[T]): T = computeMostLikelyValue(target) +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/ProbEvidenceMarginalMAP.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/ProbEvidenceMarginalMAP.scala new file mode 100644 index 00000000..7dd0a789 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/ProbEvidenceMarginalMAP.scala @@ -0,0 +1,412 @@ +/* + * ProbEvidenceMarginalMAP.scala + * A marginal MAP algorithm based on probability of evidence. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 1, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.language.Element.ElemVal +import com.cra.figaro.language._ +import com.cra.figaro.util._ + +import scala.annotation.tailrec +import scala.collection.mutable + +/** + * An algorithm for marginal MAP. This algorithm works by searching for the assignment to the MAP elements that + * maximizes the probability of evidence of observing that assignment. Uses one time probability of evidence sampling at + * each iteration for the given number of samples. Since the probability of evidence is just an estimate, this algorithm + * is allowed to repeatedly take more probability of evidence samples until it believes with high confidence that one + * state is better than another state, or it has reached the maximum number of allowed runs. The maximization is done by + * simulated annealing. + * @param universe Universe on which to run the algorithm. + * @param tolerance Confidence level used deciding to accept or reject under uncertainty. This corresponds to a maximum + * allowed p-value. Thus, setting this to 0.05 means we only accept or reject if we are at least 95% confident that + * we're making the right decision. Must between 0 and 0.5. + * @param samplesPerIteration Number of probability of evidence samples to take per run. Must be strictly greater than 1. + * A reasonable starting point is 100. + * @param maxRuns Maximum number of runs of probability of evidence sampling allowed before giving up and returning the + * proposal with higher estimated probability of evidence. Thus, at each iteration of simulated annealing, this + * algorithm can take as many as `samplesPerIteration * maxRuns` probability of evidence samples. Setting to 1 + * corresponds to using no hypothesis test at all. Setting to `Int.MaxValue` corresponds to running indefinitely until + * we are confident that the proposal should be accepted or rejected. + * @param proposalScheme Scheme for proposing new values. This can propose any element in the universe, but updates to + * non-MAP elements are only used for generating new values for MAP elements. + * @param schedule Schedule that produces an increasing temperature for simulated annealing. + * @param mapElements List of elements over which to perform marginal MAP. These elements must not have evidence on them + * that is contingent on the values of non-MAP elements. Additionally, these elements must be "observable", in the sense + * that observing values for these elements and computing the probability of evidence of those observations should not + * uniquely return zero. Typically, this is satisfiable by elements that are not both continuous and deterministic. The + * algorithm will still run if this condition is not satisfied, but it will not converge. + */ +abstract class ProbEvidenceMarginalMAP(universe: Universe, + tolerance: Double, + samplesPerIteration: Int, + maxRuns: Int, + proposalScheme: ProposalScheme, + schedule: Schedule, + val mapElements: List[Element[_]]) + // Burn-in and interval aren't needed in this context, so they are set to 0 and 1, respectively + extends MetropolisHastings(universe, proposalScheme, 0, 1, mapElements:_*) with MarginalMAPAlgorithm { + import MetropolisHastings._ + + require(samplesPerIteration >= 2, "samples per iteration must be at least 2") + require(0 < tolerance && tolerance < 0.5, "tolerance must be between 0 and 0.5") + require(maxRuns >= 1, "maximum allowed runs must be at least 1") + + // The probability of evidence sampler associated with the current state of the MAP variables. Initialized when this + // (i.e. the ProbEvidenceMarginalMAP) is started. In general, while this is active, probEvidenceSampler refers to + // an active MMAPProbEvidenceSampler that can be run for additional iterations to improve its estimate. + protected var probEvidenceSampler: MMAPProbEvidenceSampler = _ + + // Elements created by MH (and stored in chainCache) that should not be deleted while sampling probability of evidence. + // This is needed because ProbEvidenceSampler can create temporary elements while running, and they must be cleared to + // avoid memory leaks. However, we don't just call universe.clearTemporaries() because this would also clear + // chainCache, which we don't want. This is a var (as opposed to an argument to MMAPProbEvidenceSampler) because it + // may change between iterations of MH. + protected var preserve: Set[Element[_]] = _ + + // Increasing temperature used for simulated annealing. + protected var temperature = 1.0 + + /** + * Get the current temperature. Used for debugging. + */ + def getTemperature = temperature + + override protected def initConstrainedValues() = { + // We only initialize constraints for MAP elements + // After this point, the keys in the map are assumed to be fixed because the constrained MAP elements won't change + for(elem <- universe.constrainedElements.intersect(mapElements)) { + currentConstraintValues(elem) = elem.constraintValue + } + } + + override protected def computeScores(): Double = { + // Compute the log ratio of constraint values for only the MAP elements + val scores = currentConstraintValues.keys.map(elem => elem.constraintValue - currentConstraintValues(elem)) + scores.sum + } + + override protected def mhStep(): State = { + // This state is not constrained. Constraints are handled in decideToAccept because we incorporate them in the + // probability of evidence computation. + val newState = proposeAndUpdate() + // We don't care about dissatisfied elements that aren't MAP elements; remove them + newState.dissatisfied.retain(fastTargets.contains) + + if(decideToAccept(newState)) { + accepts += 1 + accept(newState) + } else { + rejects += 1 + undo(newState) + } + newState + } + + /** + Decide whether or not to accept the new (unconstrained) state, first taking into account conditions on the MAP + * elements. Does not change the state of the universe. Updates the temperature, preserved elements, and probability + * of evidence sampler accordingly. Incorporates constraints on the MAP elements. + */ + override protected def decideToAccept(newState: State): Boolean = { + // Use the same satisfied / dissatisfied rule as MH + val nothingNewDissatisfied = newState.dissatisfied subsetOf dissatisfied + val somethingOldSatisfied = dissatisfied exists (_.conditionSatisfied) + if (nothingNewDissatisfied && somethingOldSatisfied) true + else if (!nothingNewDissatisfied && !somethingOldSatisfied) false + else { + decideToAcceptSatisfied() + } + } + + /** + * Like decideToAccept, but assume all conditions on the MAP elements are satisfied. + */ + protected def decideToAcceptSatisfied(): Boolean = { + // Update the temperature + temperature = schedule.temperature(temperature, sampleCount) + + // Always accept if none of the MAP values changed. This isn't necessarily meaningless; the values of other elements + // may have changed, which could result in new proposals later. These are "observations" because they are the values + // we observe in the probability of evidence sampler. + val observations = currentMAPValues + if(observations == probEvidenceSampler.observations) return true + + // Record the universe state because probability of evidence sampling may corrupt it + val universeState = new UniverseState(universe) + // Record the elements that probability of evidence sampling shouldn't deactivate (includes those in the cache) + preserve = universeState.myActiveElements + + // Create and start a probability of evidence sampler over the current values of the MAP elements + val newProbEvidenceSampler = new MMAPProbEvidenceSampler(observations) + // This initially runs the sampler for samplesPerIteration samples + newProbEvidenceSampler.start() + + // Normally, we test if log(U[0,1]) < (log(newProbEvidence) - log(oldProbEvidence) + computeScores) * temperature. + // We want to reformat this as a hypothesis test involving the log of the mean of two sampled random variables. + // This yields log(oldProbEvidence) + (log(U[0,1]) / temperature - computeScores) < log(newProbEvidence). + // Thus, logConstant is the multiplicative constant applied to the old probability of evidence. + + // Note that in this implementation, we choose to ignore the proposal probability. In theory, this should not make a + // difference in the limiting case, but in practice choosing to incorporate or not incorporate this probability + // could affect the rate of convergence. Here, we choose to ignore it because the proposal probability may not + // correspond to a meaningful change over the MAP elements. In particular, if new values are proposed for non-MAP + // elements but none of the values of MAP elements change, it would not be sensible to weight one state as being + // more or less favorable than the other. + val logConstant = math.log(random.nextDouble) / temperature - computeScores() + // We've already run newProbEvidence sampler once by calling start(), so use maxRuns - 1 + val accepted = compareMeans(probEvidenceSampler, newProbEvidenceSampler, logConstant, maxRuns - 1) + + // Update the probability of evidence sampler and kill the one we don't keep + // Calling deregister might be unnecessary if the algorithm deregisters itself when killed + if(accepted) { + probEvidenceSampler.kill() + universe.deregisterAlgorithm(probEvidenceSampler) + probEvidenceSampler = newProbEvidenceSampler + } + else { + newProbEvidenceSampler.kill() + universe.deregisterAlgorithm(newProbEvidenceSampler) + } + + // Restore the universe state, since it may have been modified while running probability of evidence sampling. + universeState.restore() + + accepted + } + + /** + * Record the current values of all MAP elements. + */ + protected def currentMAPValues: List[ElemVal[_]] = { + // For whatever reason, the Scala compiler complains if we try to make this an anonymous function. + def makeElemVal[T](elem: Element[_]) = ElemVal[T](elem.asInstanceOf[Element[T]], elem.value.asInstanceOf[T]) + mapElements.map(makeElemVal) + } + + /** + * Decides whether or not the mean of the old sampler, multiplied by the constant given, is likely to be less than the + * mean of the new sampler. Computes in log space to avoid underflow. This may mutate the state of the universe. This + * does not take into account conditions and constraints on the MAP elements directly; these should be incorporated in + * the log constant provided. + * @param oldSampler Probability of evidence sampler for the previous state of the MAP elements. + * @param newSampler Probability of evidence sampler for the next state of the MAP elements. + * @param logConstant Log of a multiplicative constant, by which we multiply the mean of the old sampler. + * @param runs Maximum allowed additional runs of probability of evidence sampling before this method should return a + * best guess. This is a kill switch to avoid taking an absurd number of samples when the difference between the means + * is negligible. Must be >= 0. Setting this to 0 is equivalent to performing no hypothesis test at all and just + * comparing the values. + * @return A decision to accept based on a one-sided t-test of the weights sampled from the two samplers. + */ + @tailrec + protected final def compareMeans(oldSampler: MMAPProbEvidenceSampler, newSampler: MMAPProbEvidenceSampler, + logConstant: Double, runs: Int): Boolean = { + val oldLogStats = oldSampler.totalLogStatistics.multiplyByConstant(logConstant) + val newLogStats = newSampler.totalLogStatistics + + // If we aren't allowed to take more samples, our best guess is to return the comparison of the sample means. + // Otherwise, perform the t-test to see if we're confident that the means differ. We are sure that both counts are + // greater than 1 because both counts must be at least samplesPerIteration. + if(runs == 0 || LogStatistics.oneSidedTTest(oldLogStats, newLogStats) < tolerance) { + oldLogStats.logMean < newLogStats.logMean + } + // If we can't decide with the information we have, take more samples and try again. + // Run both samplers for the same number of additional iterations. + else { + oldSampler.run() + newSampler.run() + compareMeans(oldSampler, newSampler, logConstant, runs - 1) + } + } + + override def sample(): (Boolean, Sample) = { + mhStep() + if(dissatisfied.isEmpty) { + // Update as long as no MAP elements are dissatisfied + val values = mapElements.map(elem => elem -> elem.value) + (true, mutable.Map(values:_*)) + } + else { + (false, mutable.Map()) + } + + } + + override protected def updateTimesSeenForTarget[T](elem: Element[T], newValue: T): Unit = { + // Override the last update, which will later be returned as the most likely value + allLastUpdates(elem) = (newValue, sampleCount) + } + + override def computeMostLikelyValue[T](target: Element[T]): T = { + allLastUpdates(target)._1.asInstanceOf[T] + } + + override protected def doInitialize(): Unit = { + super.doInitialize() + // Only record dissatisfied MAP elements + dissatisfied = dissatisfied.intersect(fastTargets) + + // Copy the universe state, since the probability of evidence sampler may corrupt it + val universeState = new UniverseState(universe) + preserve = universeState.myActiveElements + + // Run the probability of evidence sampler for samplesPerIteration + probEvidenceSampler = new MMAPProbEvidenceSampler(currentMAPValues) + probEvidenceSampler.start() + universeState.restore() + } + + /* + * Prevent memory leaks by killing internal algorithms and clearing the cache. + */ + override def cleanUp(): Unit = { + super.cleanUp() + universe.clearTemporaries() + chainCache.clear() + probEvidenceSampler.kill() + // This is unnecessary if the algorithm deregisters itself when killed + universe.deregisterAlgorithm(probEvidenceSampler) + probEvidenceSampler = null + preserve = null + } + + /* + * We don't want to force updates at the end of sampling, since the current values of MAP elements are not necessarily + * the values that maximize the probability of evidence. We only want to make changes to allLastUpdates when a new set + * of values increases the probability of evidence. + */ + override def update(): Unit = {} + + /** + * Special probability of evidence sampler used for marginal MAP. Unlike a regular probability of evidence sampler, + * this records its own variance. It does so in an online fashion, and computes it in log space to prevent underflow. + * Additionally, this algorithm may be run multiple times. The rolling mean and variance computation incorporates the + * samples taken from all runs. + * @param observations Elements and corresponding values that should be observed each time this algorithm is run. + * Normally, this contains MAP elements and their proposed values. + */ + class MMAPProbEvidenceSampler(val observations: List[ElemVal[_]]) extends ProbEvidenceSampler(universe) + with OneTimeProbEvidenceSampler with OnlineLogStatistics { + + override val numSamples = samplesPerIteration + + /** + * Observe the necessary values of MAP elements, then run the algorithm. After this is initialized, calling this + * method again is allowed. The additional samples are accounted for when returning the total log statistics. + */ + override def run(): Unit = { + for(ElemVal(elem, value) <- observations) elem.observe(value) + super.run() + } + + /** + * Perform sampling, but additionally update the variance and clear only elements that shouldn't be preserved. + */ + override protected def doSample(): Unit = { + totalWeight += 1 + + try { + val weight = lw.computeWeight(universe.activeElements) + successWeight = logSum(successWeight, weight) + // Record the weight for the variance computation + record(weight) + } catch { + case Importance.Reject => record(Double.NegativeInfinity) + } + + // Deactivate only the temporary elements created during probability of evidence sampling + for(elem <- universe.activeElements) { + // Since an element deactivates its direct context contents when deactivated, it's possible that an element + // in the list will be deactivated before we reach it, so we have to check again that it is active + if(elem.active && !preserve.contains(elem)) elem.deactivate() + } + } + } +} + +class AnytimeProbEvidenceMarginalMAP(universe: Universe, + tolerance: Double, + samplesPerIteration: Int, + maxRuns: Int, + proposalScheme: ProposalScheme, + schedule: Schedule, + mapElements: List[Element[_]]) + extends ProbEvidenceMarginalMAP(universe, tolerance, samplesPerIteration, maxRuns, proposalScheme, schedule, mapElements) + with AnytimeSampler with AnytimeMarginalMAP { + /** + * Initialize the algorithm. + */ + override def initialize(): Unit = { + super.initialize() + doInitialize() + } +} + +class OneTimeProbEvidenceMarginalMAP(val numSamples: Int, + universe: Universe, + tolerance: Double, + samplesPerIteration: Int, + maxRuns: Int, + proposalScheme: ProposalScheme, + schedule: Schedule, + mapElements: List[Element[_]]) + extends ProbEvidenceMarginalMAP(universe, tolerance, samplesPerIteration, maxRuns, proposalScheme, schedule, mapElements) + with OneTimeSampler with OneTimeMarginalMAP { + + /** + * Run the algorithm, performing its computation to completion. + */ + override def run(): Unit = { + doInitialize() + super.run() + } +} + +object ProbEvidenceMarginalMAP { + /** + * Creates a one time marginal MAP algorithm that uses probability of evidence. + * @see [[com.cra.figaro.experimental.marginalmap.ProbEvidenceMarginalMAP]] abstract class for a complete description + * of the parameters. + */ + def apply(iterations: Int, tolerance: Double, samplesPerIteration: Int, maxRuns: Int, proposalScheme: ProposalScheme, + schedule: Schedule, mapElements: Element[_]*)(implicit universe: Universe) = + new OneTimeProbEvidenceMarginalMAP(iterations, universe, tolerance, samplesPerIteration, maxRuns, proposalScheme, schedule, mapElements.toList) + + /** + * Creates an anytime marginal MAP algorithm that uses probability of evidence. + * @see [[com.cra.figaro.experimental.marginalmap.ProbEvidenceMarginalMAP]] abstract class for a complete description + * of the parameters. + */ + def apply(tolerance: Double, samplesPerIteration: Int, maxRuns: Int, proposalScheme: ProposalScheme, + schedule: Schedule, mapElements: Element[_]*)(implicit universe: Universe) = + new AnytimeProbEvidenceMarginalMAP(universe, tolerance, samplesPerIteration, maxRuns, proposalScheme, schedule, mapElements.toList) + + /** + * Creates a one time marginal MAP algorithm that uses probability of evidence. Takes 100 samples per iteration at a + * tolerance of 0.05, up to a maximum of 100 runs. Uses the default proposal scheme and schedule. + * @see [[com.cra.figaro.experimental.marginalmap.ProbEvidenceMarginalMAP]] abstract class for a complete description + * of the parameters. + */ + def apply(iterations: Int, mapElements: Element[_]*)(implicit universe: Universe) = + new OneTimeProbEvidenceMarginalMAP(iterations, universe, 0.05, 100, 100, ProposalScheme.default(universe), Schedule.default(), mapElements.toList) + + /** + * Creates an anytime marginal MAP algorithm that uses probability of evidence. Takes 100 samples per iteration at a + * tolerance of 0.05, up to a maximum of 100 runs. Uses the default proposal scheme and schedule. + * @see [[com.cra.figaro.experimental.marginalmap.ProbEvidenceMarginalMAP]] abstract class for a complete description + * of the parameters. + */ + def apply(mapElements: Element[_]*)(implicit universe: Universe) = + new AnytimeProbEvidenceMarginalMAP(universe, 0.05, 100, 100, ProposalScheme.default(universe), Schedule.default(), mapElements.toList) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/StructuredMarginalMAPAlgorithm.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/StructuredMarginalMAPAlgorithm.scala new file mode 100644 index 00000000..df6afb5b --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/StructuredMarginalMAPAlgorithm.scala @@ -0,0 +1,51 @@ +/* + * StructuredMarginalMAPAlgorithm.scala + * Abstract class for structured marginal MAP algorithms. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Jun 3, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.{Algorithm, AlgorithmException} +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.structured.{ComponentCollection, Problem} +import com.cra.figaro.language._ + +/** + * A structured marginal MAP algorithm. + * @param universe Universe on which to perform inference. + * @param mapElements Elements for which to compute MAP queries. Elements not in this list are summed over. + */ +abstract class StructuredMarginalMAPAlgorithm(val universe: Universe, val mapElements: List[Element[_]]) + extends Algorithm with OneTimeMarginalMAP { + + def run(): Unit + + val cc: ComponentCollection = new ComponentCollection + + // Targets are our MAP elements, since the first step is to eliminate the other elements + val problem = new Problem(cc, mapElements) + + // We have to add all active elements to the problem since these elements, if they are every used, need to have components created at the top level problem + universe.permanentElements.foreach(problem.add(_)) + val evidenceElems = universe.conditionedElements ::: universe.constrainedElements + + def initialComponents() = (problem.targets ++ evidenceElems).distinct.map(cc(_)) + + def computeMostLikelyValue[T](target: Element[T]): T = { + val targetVar = cc(target).variable + val factor = problem.recordingFactors(targetVar).asInstanceOf[Factor[T]] + if (factor.size != 1) throw new AlgorithmException//("Final factor for most likely value has more than one entry") + factor.get(List()) + } + + +} + + diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/StructuredMarginalMAPVE.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/StructuredMarginalMAPVE.scala new file mode 100644 index 00000000..4c778baa --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/marginalmap/StructuredMarginalMAPVE.scala @@ -0,0 +1,56 @@ +/* + * StructuredMarginalMAPVE.scala + * A structured variable elimination algorithm to compute marginal MAP queries. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Jun 3, 2016 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.marginalmap + +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.language._ + +/** + * A structured marginal MAP algorithm that uses VE to compute MAP queries. + * @param universe Universe on which to perform inference. + * @param mapElements Elements for which to compute MAP queries. Elements not in this list are summed over. + */ +class StructuredMarginalMAPVE(universe: Universe, mapElements: List[Element[_]]) + extends StructuredMarginalMAPAlgorithm(universe, mapElements) { + + def run() { + val strategy = DecompositionStrategy.recursiveStructuredStrategy(problem, new MarginalMAPVEStrategy, defaultRangeSizer, Lower, false) + strategy.execute(initialComponents) + } +} + +object StructuredMarginalMAPVE { + /** + * Create a structured variable elimination algorithm with the given query targets. + * @param mapElements Elements for which to compute MAP queries. Elements not in this list are summed over, + * and cannot be queried. + */ + def apply(mapElements: Element[_]*)(implicit universe: Universe) = { + new StructuredMarginalMAPVE(universe, mapElements.toList) + } + + /** + * Use variable elimination to compute the most likely value of the given element. + * @param target Element for which to compute MAP value. + * @param mapElements Additional elements to MAP. Elements not in this list are summed over. + */ + def mostLikelyValue[T](target: Element[T], mapElements: Element[_]*): T = { + val alg = StructuredMarginalMAPVE((target +: mapElements).distinct:_*) + alg.start() + val result = alg.mostLikelyValue(target) + alg.kill() + result + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Beta.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Beta.scala new file mode 100644 index 00000000..757246dc --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Beta.scala @@ -0,0 +1,127 @@ +/* + * Beta.scala + * Elements representing Beta distributions. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 2, 2011 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import com.cra.figaro.language._ + +import math.{ pow, sqrt } +import JSci.maths.SpecialMath.beta +import com.cra.figaro.algorithm.ValuesMaker +import com.cra.figaro.algorithm.lazyfactored.ValueSet +import com.cra.figaro.library.atomic.continuous.Util + +/** + * A Beta distribution in which the alpha and beta parameters are provided. + * This Beta element can be used as the parameter for a ParameterizedFlip. + * + * @param a The prior alpha parameter + * @param b The prior beta parameter + */ +class AtomicBeta(name: Name[Double], a: Double, b: Double, collection: ElementCollection) + extends Element[Double](name, collection) with HasDensity[Double] with NormalProposer with DoubleParameter with com.cra.figaro.library.atomic.continuous.Beta { + // Bounds for normal proposals + override def lower = 0.0 + override def upper = 1.0 + // Proposal scale is 20% of the standard deviation + def proposalScale = 0.2 * sqrt(a * b / (a + b + 1)) / (a + b) + + override def nextRandomness(oldRandomness: Randomness): (Randomness, Double, Double) = { + // If a or b is greater than 1, the distribution is unimodal, so normal proposals are appropriate + if(a >= 1 || b >= 1) super[NormalProposer].nextRandomness(oldRandomness) + // If both a and b are less than 1, the distribution is bimodal, so we're better off proposing from the prior + else super[HasDensity].nextRandomness(oldRandomness) + } + + /** + * The learned alpha parameter + */ + var learnedAlpha = a + /** + * The learned beta parameter + */ + var learnedBeta = b + def aValue = learnedAlpha + def bValue = learnedBeta + def generateRandomness() = Util.generateBeta(a, b) + + /** + * The normalizing factor. + */ + val normalizer = 1 / beta(a, b) + + /** + * Density of a value. + */ + def density(x: Double) = pow(x, a - 1) * pow(1 - x, b - 1) * normalizer + + /** + * Returns an empty sufficient statistics vector. + */ + override def zeroSufficientStatistics(): Seq[Double] = { + Seq(0.0, 0.0) + } + + /** + * Returns an element that models the learned distribution. + * + * @deprecated + */ + def getLearnedElement: AtomicFlip = { + new AtomicFlip("", MAPValue, collection) + } + + override def sufficientStatistics[Boolean](b: Boolean): Seq[Double] = { + if (b == true) { + Seq(1.0, 0.0) + } else { + Seq(0.0, 1.0) + } + } + + private[figaro] override def sufficientStatistics[Boolean](i: Int): Seq[Double] = { + if (i == 0) { + Seq(1.0, 0.0) + } else { + Seq(0.0, 1.0) + } + } + + def expectedValue: Double = { + (learnedAlpha) / (learnedAlpha + learnedBeta) + } + + def MAPValue: Double = { + if (learnedAlpha + learnedBeta == 2) 0.5 + else (learnedAlpha - 1) / (learnedAlpha + learnedBeta - 2) + } + + def makeValues(depth: Int) = ValueSet.withoutStar(Set(MAPValue)) + + def maximize(sufficientStatistics: Seq[Double]) { + require(sufficientStatistics.size == 2) + learnedAlpha = sufficientStatistics(0) + a + learnedBeta = sufficientStatistics(1) + b + + } + + override def toString = "Beta(" + a + ", " + b + ")" +} + +object Beta { + /** + * Create a Beta distribution in which the parameters are constants. + */ + def apply(a: Double, b: Double)(implicit name: Name[Double], collection: ElementCollection) = + new AtomicBeta(name, a, b, collection) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Exponential.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Exponential.scala new file mode 100644 index 00000000..c0ab49b9 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Exponential.scala @@ -0,0 +1,47 @@ +/* + * Exponential.scala + * Elements representing exponential distributions. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 25, 2011 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Util + +import scala.math.{exp, log} + +/** + * An exponential distribution in which the parameter is a constant. + */ +class AtomicExponential(name: Name[Double], val lambda: Double, collection: ElementCollection) + extends Element[Double](name, collection) with NormalProposer { + // Lower bound for normal proposals + override def lower = 0.0 + // Proposal scale is 20% of the standard deviation + def proposalScale = 0.2 * lambda + + def generateRandomness() = Util.generateExponential(lambda) + + /** + * Density of a value. + */ + def density(d: Double) = if (d < 0.0) 0.0 else lambda * exp(-lambda * d) + + override def toString = "Exponential(" + lambda + ")" +} + +object Exponential { + /** + * Create an exponential distribution in which the parameter is a constant. + */ + def apply(lambda: Double)(implicit name: Name[Double], collection: ElementCollection) = + new AtomicExponential(name, lambda, collection) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Gamma.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Gamma.scala new file mode 100644 index 00000000..345c86b1 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Gamma.scala @@ -0,0 +1,74 @@ +/* + * Gamma.scala + * Elements representing Gamma elements. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 25, 2011 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import JSci.maths.SpecialMath.gamma +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Util + +import scala.math._ + +/** + * A Gamma distribution in which both the k and theta parameters are constants. + * Theta defaults to 1. + */ +class AtomicGamma(name: Name[Double], k: Double, theta: Double = 1.0, collection: ElementCollection) + extends Element[Double](name, collection) with HasDensity[Double] with NormalProposer { + // Lower bound for normal proposals + override def lower = 0.0 + // Proposal scale is 20% of the standard deviation of the underlying Gamma randomness + def proposalScale = 0.2 * sqrt(k) + + def generateRandomness() = Util.generateGamma(k) + + override def generateValue(rand: Randomness) = + rand * theta // due to scaling property of Gamma + + override def generateValueDerivative(rand: Randomness) = theta + + override def nextRandomness(oldRandomness: Randomness): (Randomness, Double, Double) = { + // If k is large, then the density is spread out enough for normal proposals to work reasonably well. + if(k >= 1) super[NormalProposer].nextRandomness(oldRandomness) + // If k is small, too much density is concentrated extremely close to 0 for normal proposals to be effective. + // For example, if k=0.1, the median is ~0.00059. + else super[HasDensity].nextRandomness(oldRandomness) + } + + /** + * The normalizing factor. + */ + private val normalizer = 1.0 / (gamma(k) * pow(theta, k)) + + /** + * Density of a value. + */ + def density(x: Double) = { + if (x < 0.0) 0.0 else { + val numer = pow(x, k - 1) * exp(-x / theta) + numer * normalizer + } + } + + override def toString = + if (theta == 1.0) "Gamma(" + k + ")" + else "Gamma(" + k + ", " + theta + ")" +} + +object Gamma { + /** + * Create a Gamma element in which both k and theta parameters are constants. Theta defaults to 1. + */ + def apply(k: Double, theta: Double = 1.0)(implicit name: Name[Double], collection: ElementCollection) = + new AtomicGamma(name, k, theta, collection) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/InverseGamma.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/InverseGamma.scala new file mode 100644 index 00000000..94906d85 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/InverseGamma.scala @@ -0,0 +1,75 @@ +/* + * InverseGamma.scala + * Class for a Gamma distribution in which both the k and theta parameters are constants + * + * Created By: Michael Howard (mhoward@cra.com) + * Creation Date: Dec 4, 2014 + * + * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import JSci.maths.SpecialMath.gamma +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Util + +import scala.math._ + +/** + * A Gamma distribution in which both the k and theta parameters are constants. + * Theta defaults to 1. + */ +class AtomicInverseGamma(name: Name[Double], shape: Double, scale: Double = 1.0, collection: ElementCollection) + extends Element[Double](name, collection) with HasDensity[Double] with NormalProposer { + // Lower bound for normal proposals + override def lower = 0.0 + // Proposal scale is 20% of the standard deviation of the underlying Gamma randomness + def proposalScale = 0.2 * sqrt(shape) + + def generateRandomness() = Util.generateGamma(shape) + + override def generateValue(rand: Randomness) = 1.0 / (rand * scale) // due to scaling property of Gamma + + override def generateValueDerivative(rand: Randomness) = 1.0 / (rand * rand * scale) + + override def nextRandomness(oldRandomness: Randomness): (Randomness, Double, Double) = { + // If k is large, then the density is spread out enough for normal proposals to work reasonably well. + if(shape >= 1) super[NormalProposer].nextRandomness(oldRandomness) + // If k is small, too much density is concentrated extremely close to 0 for normal proposals to be effective. + // For example, if k=0.1, the median is ~0.00059. + else super[HasDensity].nextRandomness(oldRandomness) + } + + /** + * The normalizing factor. + */ + private val normalizer = pow(scale, shape) / gamma(shape) + + /** + * Density of a value. + */ + def density(x: Double) = { + if (x < 0.0) 0.0 else { + //Convert to logarithms if this is too large. + val numer = pow(x, -1.0 * shape - 1) * exp(-1.0 * scale / x) + numer * normalizer + } + } + + override def toString = + if (scale == 1.0) "InverseGamma(" + shape + ")" + else "InverseGamma(" + shape + ", " + scale + ")" +} + +object InverseGamma { + /** + * Create an InverseGamma element. + */ + def apply(shape: Double, scale: Double)(implicit name: Name[Double], collection: ElementCollection) = + new AtomicInverseGamma(name, shape, scale, collection) + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Normal.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Normal.scala new file mode 100644 index 00000000..17112c7e --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Normal.scala @@ -0,0 +1,83 @@ +/* + * Normal.scala + * Elements representing normal distributions + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import JSci.maths.SpecialMath.error +import com.cra.figaro.language._ +import com.cra.figaro.util.random + +import scala.math._ + +/** + * A normal distribution in which the mean and variance are constants. + */ +class AtomicNormal(name: Name[Double], val mean: Double, val variance: Double, collection: ElementCollection) + extends Element[Double](name, collection) with NormalProposer { + lazy val standardDeviation = sqrt(variance) + + // Proposal scale is 20% of the standard deviation of the underlying standard Normal randomness + def proposalScale = 0.2 + + def generateRandomness() = { + val u1 = random.nextDouble() + val u2 = random.nextDouble() + val w = sqrt(-2.0 * log(u1)) + val x = 2.0 * Pi * u2 + w * sin(x) + } + + override def generateValue(rand: Randomness) = rand * standardDeviation + mean + + override def generateValueDerivative(rand: Randomness) = standardDeviation + + /** + * The normalizing factor. + */ + private val normalizer = 1.0 / sqrt(2.0 * Pi * variance) + + /** + * Density of a value. + */ + def density(d: Double) = Normal.density(mean, variance, normalizer)(d) + + override def toString = "Normal(" + mean + ", " + variance + ")" +} + +object Normal { + + def probability(mean: Double, stDev: Double)(lower: Double, upper: Double) = { + val denominator = stDev * sqrt(2.0) + val erfLower = (lower - mean) / denominator + val erfUpper = (upper - mean) / denominator + 0.5 * (error(erfUpper) - error(erfLower)) + } + + def density(mean: Double, stDev: Double)(d: Double) = { + val diff = d - mean + val exponent = -(diff * diff) / (2.0 * stDev * stDev) + exp(exponent) / (sqrt(2.0 * Pi) * stDev) + } + + def density(mean: Double, variance: Double, normalizer: Double)(d: Double) = { + val diff = d - mean + val exponent = -(diff * diff) / (2.0 * variance) + normalizer * exp(exponent) + } + + /** + * Create a normal distribution in which the mean and variance are constants. + */ + def apply(mean: Double, variance: Double)(implicit name: Name[Double], collection: ElementCollection) = + new AtomicNormal(name, mean, variance, collection) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/NormalProposer.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/NormalProposer.scala new file mode 100644 index 00000000..210a17dd --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/NormalProposer.scala @@ -0,0 +1,134 @@ +/* + * NormalProposer.scala + * Normally distributed proposals. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 17, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import com.cra.figaro.language._ +import com.cra.figaro.util._ +import org.apache.commons.math3.distribution.NormalDistribution + +import scala.annotation.tailrec + +/** + * Normally distributed proposals for univariate continuous elements. This works by proposing from a truncated normal + * distribution over the randomness of this element. This implementation assumes that the probability density of values + * associated with randomnesses in the range (`lower`, `upper`) are finite. + */ +trait NormalProposer extends Atomic[Double] { + type Randomness = Double + + /** + * Exclusive lower bound of the range of the randomness of this element. Defaults to -Infinity. Must be strictly less + * than the upper bound. + */ + def lower: Double = Double.NegativeInfinity + /** + * Exclusive upper bound of the range of the randomness of this element. Defaults to Infinity. Must be strictly + * greater than the lower bound. + */ + def upper: Double = Double.PositiveInfinity + + /** + * The scale of the normally distributed proposal. This corresponds to the standard deviation of the proposal before + * truncation. If the randomness has finite variance, this should be less than or equal to its standard deviation. + * A good place to start is e.g. 20% of the standard deviation. + */ + def proposalScale: Double + + /** + * A strictly monotone differentiable function defined on (`lower`, `upper`). Defaults to the identity function. + */ + def generateValue(rand: Randomness) = rand + + /** + * The absolute value of the derivative of `generateValue` with respect to the randomness given. This is needed to + * compute a proposal density over the transformed randomness. Defaults to 1.0, corresponding to the case where + * `generateValue` is the identity function. + */ + def generateValueDerivative(rand: Randomness): Double = 1.0 + + /** + * Generate the next randomness given the current randomness. + * Returns three values: The next randomness, the Metropolis-Hastings proposal probability + * ratio, which is: + * + * P(new -> old) / P(old -> new) + * + * and the model probability ratio, which is: + * + * P(new) / P(old) + * + * By default, this implementation proposes a normally distributed randomness from the previous randomness, truncated + * to be within the appropriate range. The probability ratios returned are associated with the values of this element + * rather than the randomness. This is for the purpose of simulated annealing, since the most likely randomness is not + * necessarily the most likely value, depending on the form of the generateValue function. + * + * One can override this to only use normal proposals in certain special cases. + */ + override def nextRandomness(oldRandomness: Randomness): (Randomness, Double, Double) = { + normalNextRandomness(oldRandomness) + } + + /** + * Computes the normal proposal for nextRandomness. This is separated from the nextRandomness so that subclasses can + * choose when to use normal proposals. + */ + @tailrec + protected final def normalNextRandomness(oldRandomness: Randomness): (Randomness, Double, Double) = { + /* + * Sample from a normal distribution centered at oldRandomness, and reject if it's outside the range. + * Rejection sampling to be in the range is justified here because of the assumptions on the scale. + * If both lower and upper bounds are finite, then the scale is less than the maximum possible standard deviation of + * (upper - lower) / 2. This yields a minimum acceptance probability of ~47.72% (cumulative density of a standard + * normal distribution from 0 to 2) when oldRandomness is one of the bounds. + * If only one of the bounds is finite, then we reject less than 50% of the time, since no more than half of the + * distribution is truncated by the bounds. If the distribution is unbounded, we never reject. + * So, the expected number of trials is never more than 1 / 0.4772 = 2.095, which is quite reasonable. + */ + val newRandomness = oldRandomness + random.nextGaussian() * proposalScale + if(lower < newRandomness && newRandomness < upper) { + val proposalRatio = proposalProb(newRandomness, oldRandomness) / proposalProb(oldRandomness, newRandomness) + val modelRatio = density(generateValue(newRandomness)) / density(generateValue(oldRandomness)) + (newRandomness, proposalRatio, modelRatio) + } + else normalNextRandomness(oldRandomness) + } + + /** + * Computes the proposal probability density in one direction. Both values should be in the interval (lower, upper). + * @param oldRandomness The previous randomness. + * @param newRandomness The newly proposed randomness. + * @return The probability density associated with the transition from `generateValue(oldRandomness)` to + * `generateValue(newRandomness)`. This is a density over the corresponding values (as opposed to randomnesses). + */ + protected def proposalProb(oldRandomness: Double, newRandomness: Double): Double = { + val stDev = proposalScale + + // The probability density of proposing the new randomness given the old randomness is the density of the new + // randomness from the normal distribution, divided by the normalizing constant of the cumulative probability + // between the upper and lower bounds. This cumulative probability corresponds to the probability of not rejecting + // when we sample from the regular normal proposal. This ensures that the PDF of the truncated distribution + // integrates to 1. + val uncorrected = + Normal.density(oldRandomness, stDev)(newRandomness) / Normal.probability(oldRandomness, stDev)(lower, upper) + + // Correct for the scaling factor associated with newRandomness. This is needed because even though the proposal + // distribution over the randomness of this element is a truncated normal distribution, the resulting proposal + // distribution over the actual values of this element may be different. Consider, for example, how an inverse Gamma + // element might use a Gamma distribution as its randomness, then invert the randomness to produce a value. Then, a + // truncated normal distribution over the randomness would have some other distribution over the values. Since we + // have the restriction that the deterministic transformation function is strictly monotone and differentiable, we + // can account for this difference; this is a standard "change of variable" computation. + uncorrected / generateValueDerivative(newRandomness) + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Uniform.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Uniform.scala new file mode 100644 index 00000000..558fe39c --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/normalproposals/Uniform.scala @@ -0,0 +1,47 @@ +/* + * Uniform.scala + * Elements representing continuous uniform distributions. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Oct 18, 2010 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.experimental.normalproposals + +import com.cra.figaro.language._ +import com.cra.figaro.util.{bound, random} + +import scala.math.log + +/** + * A continuous uniform distribution in which the parameters are constants. + */ +class AtomicUniform(name: Name[Double], override val lower: Double, override val upper: Double, collection: ElementCollection) + extends Element[Double](name, collection) with NormalProposer { + + private lazy val diff = upper - lower + + // Proposal scale is 20% of the standard deviation + def proposalScale = 0.2 * diff / math.sqrt(12) + + def generateRandomness() = random.nextDouble() * diff + lower + + private lazy val constantDensity = 1.0 / diff + + def density(d: Double) = if (d >= lower && d < upper) constantDensity; else 0.0 + + override def toString = "Uniform(" + lower + ", " + upper + ")" +} + +object Uniform { + /** + * Create a continuous uniform distribution in which the parameters are constants. + */ + def apply(lower: Double, upper: Double)(implicit name: Name[Double], collection: ElementCollection) = + new AtomicUniform(name, lower, upper, collection) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/NormalKernelDensityEstimator.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/NormalKernelDensityEstimator.scala index c74a484c..87428cf3 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/NormalKernelDensityEstimator.scala +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/NormalKernelDensityEstimator.scala @@ -1,6 +1,6 @@ /* * NormalKernelDensityEstimator.scala - * Trait to TBD + * A density estimator using a normal kernel * * Created By: Brian Ruttenberg (bruttenberg@cra.com) * Creation Date: Oct 20, 2014 diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/ParticleBeliefPropagation.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/ParticleBeliefPropagation.scala index 0938de61..004adae1 100644 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/ParticleBeliefPropagation.scala +++ b/Figaro/src/main/scala/com/cra/figaro/experimental/particlebp/ParticleBeliefPropagation.scala @@ -1,6 +1,6 @@ /* * ParticleBeliefPropagation.scala - * Trait to TBD + * A particle belief propagation algorithm * * Created By: Brian Ruttenberg (bruttenberg@cra.com) * Creation Date: Oct 20, 2014 @@ -14,27 +14,17 @@ package com.cra.figaro.experimental.particlebp import com.cra.figaro.language._ -import com.cra.figaro.algorithm.factored.FactoredAlgorithm -import com.cra.figaro.algorithm.factored.factors.{ Factory, DivideableSemiRing, Factor, LogSumProductSemiring, Variable } +import com.cra.figaro.algorithm.factored.factors.{ DivideableSemiRing, Factor, LogSumProductSemiring, Variable } import com.cra.figaro.algorithm.lazyfactored.LazyValues -import com.cra.figaro.algorithm.OneTime -import com.cra.figaro.algorithm.Anytime -import com.cra.figaro.algorithm.ProbQueryAlgorithm -import com.cra.figaro.algorithm.OneTimeProbQuery +import com.cra.figaro.algorithm._ import scala.collection.immutable.Set import scala.collection.mutable.Map -import com.cra.figaro.algorithm.factored.beliefpropagation.InnerBPHandler -import com.cra.figaro.algorithm.factored.beliefpropagation.OneTimeInnerBPHandler -import com.cra.figaro.algorithm.factored.beliefpropagation.VariableNode -import com.cra.figaro.algorithm.factored.ParticleGenerator -import com.cra.figaro.algorithm.factored.DensityEstimator -import com.cra.figaro.algorithm.AnytimeProbQuery -import com.cra.figaro.algorithm.factored.beliefpropagation.AnytimeInnerBPHandler -import com.cra.figaro.algorithm.factored.beliefpropagation.FactorNode -import com.cra.figaro.algorithm.factored.beliefpropagation.Node +import com.cra.figaro.algorithm.factored.beliefpropagation._ +import com.cra.figaro.algorithm.factored._ import breeze.linalg.normalize import com.cra.figaro.algorithm.UnsupportedAlgorithmException import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler +import com.cra.figaro.algorithm.factored.factors.factory.Factory /** * Trait for performing particle belief propagation. @@ -100,10 +90,10 @@ trait ParticleBeliefPropagation extends FactoredAlgorithm[Double] with InnerBPHa currentUniverse = universe // Remove factors on all elements that can possibly change during resampluing - dependentElems.foreach(Factory.removeFactors(_)) + //dependentElems.foreach(Factory.removeFactors(_)) // Clear the variable and values caches - Variable.clearCache + Variable.clearCache() LazyValues.clear(universe) // Create BP. @@ -163,7 +153,7 @@ trait ParticleBeliefPropagation extends FactoredAlgorithm[Double] with InnerBPHa val singleFactor = if (singleFactorIndex >= 0) lastMessages(singleFactorIndex) else throw new UnsupportedAlgorithmException(elem) // Get the original factor for this element - val originalFactor = Factory.makeNonConstraintFactors(elem) + val originalFactor = Factory.makeFactorsForElement(elem) if (originalFactor.size > 1) throw new UnsupportedAlgorithmException(elem) // Take the single factor, and divide out the original factor. We do this since the single factor in the graph // can have evidence multiplied in, so we only want to remove the original factor for it. We will use the original @@ -271,7 +261,7 @@ object ParticleBeliefPropagation { * Creates a One Time belief propagation computer in the current default universe. */ def apply(myOuterIterations: Int, myInnerIterations: Int, targets: Element[_]*)(implicit universe: Universe) = - new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultArgSamples, ParticleGenerator.defaultTotalSamples, + new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultNumSamplesFromAtomics, ParticleGenerator.defaultMaxNumSamplesAtChain, universe, targets: _*)(List(), (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) with OneTimeParticleBeliefPropagation with OneTimeProbQuery { val outerIterations = myOuterIterations @@ -283,7 +273,7 @@ object ParticleBeliefPropagation { */ def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, myOuterIterations: Int, myInnerIterations: Int, targets: Element[_]*)(implicit universe: Universe) = - new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultArgSamples, ParticleGenerator.defaultTotalSamples, + new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultNumSamplesFromAtomics, ParticleGenerator.defaultMaxNumSamplesAtChain, universe, targets: _*)(dependentUniverses, dependentAlgorithm) with OneTimeParticleBeliefPropagation with OneTimeProbQuery { val outerIterations = myOuterIterations val innerIterations = myInnerIterations @@ -316,7 +306,7 @@ object ParticleBeliefPropagation { * Creates a Anytime belief propagation computer in the current default universe. */ def apply(stepTimeMillis: Long, targets: Element[_]*)(implicit universe: Universe) = - new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultArgSamples, ParticleGenerator.defaultTotalSamples, + new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultNumSamplesFromAtomics, ParticleGenerator.defaultMaxNumSamplesAtChain, universe, targets: _*)(List(), (u: Universe, e: List[NamedEvidence[_]]) => () => ProbEvidenceSampler.computeProbEvidence(10000, e)(u)) with AnytimeParticleBeliefPropagation with AnytimeProbQuery { val myStepTimeMillis = stepTimeMillis @@ -327,7 +317,7 @@ object ParticleBeliefPropagation { */ def apply(dependentUniverses: List[(Universe, List[NamedEvidence[_]])], dependentAlgorithm: (Universe, List[NamedEvidence[_]]) => () => Double, stepTimeMillis: Long, targets: Element[_]*)(implicit universe: Universe) = - new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultArgSamples, ParticleGenerator.defaultTotalSamples, + new ProbQueryParticleBeliefPropagation(ParticleGenerator.defaultNumSamplesFromAtomics, ParticleGenerator.defaultMaxNumSamplesAtChain, universe, targets: _*)(dependentUniverses, dependentAlgorithm) with AnytimeParticleBeliefPropagation with AnytimeProbQuery { val myStepTimeMillis = stepTimeMillis } diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredBP.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredBP.scala deleted file mode 100644 index 7f3c03bc..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredBP.scala +++ /dev/null @@ -1,116 +0,0 @@ -/* - * StructuredBP.scala - * A structured factored inference algorithm using belief propagation. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ -package com.cra.figaro.experimental.structured.algorithm - -import com.cra.figaro.algorithm.OneTimeProbQuery -import com.cra.figaro.algorithm.Algorithm -import com.cra.figaro.language.Element -import com.cra.figaro.language.Universe -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.experimental.structured.Problem -import com.cra.figaro.experimental.structured.strategy.recursiveSolver -import com.cra.figaro.experimental.structured.solver.beliefPropagation -import com.cra.figaro.algorithm.factored.factors.SumProductSemiring -import com.cra.figaro.experimental.structured.factory.Factory - -class StructuredBP(val universe: Universe, iterations: Int, targets: Element[_]*) extends Algorithm with OneTimeProbQuery { - val queryTargets = targets - - var targetFactors: Map[Element[_], Factor[Double]] = _ - - var cc: ComponentCollection = _ - - def run() { - cc = new ComponentCollection - targetFactors = Map() - val problem = new Problem(cc, targets.toList) - val evidenceElems = universe.conditionedElements ::: universe.constrainedElements - evidenceElems.foreach(elem => if (!cc.contains(elem)) problem.add(elem)) - recursiveSolver(beliefPropagation(iterations))(problem) - val joint = problem.solution.foldLeft(Factory.unit(SumProductSemiring()))(_.product(_)) - - def marginalizeToTarget(target: Element[_]): Unit = { - val targetVar = cc(target).variable - val unnormalizedTargetFactor = joint.marginalizeTo(SumProductSemiring(), targetVar) - val z = unnormalizedTargetFactor.foldLeft(0.0, _ + _) - val targetFactor = unnormalizedTargetFactor.mapTo((d: Double) => d / z) - targetFactors += target -> targetFactor - } - - targets.foreach(marginalizeToTarget(_)) - } - - /** - * Computes the normalized distribution over a single target element. - */ - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = { - val factor = targetFactors(target) - val targetVar = cc(target).variable - val dist = factor.getIndices.filter(f => targetVar.range(f.head).isRegular).map(f => (factor.get(f), targetVar.range(f.head).value)) - // normalization is unnecessary here because it is done in marginalizeTo - dist.toStream - } - - /** - * Computes the expectation of a given function for single target element. - */ - def computeExpectation[T](target: Element[T], function: T => Double): Double = { - def get(pair: (Double, T)) = pair._1 * function(pair._2) - (0.0 /: computeDistribution(target))(_ + get(_)) - } -} - -/* - * StructuredBP.scala - * A structured belief propagation algorithm. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -object StructuredBP { - /** - * Create a structured belief propagation algorithm. - * @param iterations the number of iterations to use for each subproblem - * @param targets the query targets, which will all be part of the top level problem - */ - def apply(iterations: Int, targets: Element[_]*) = { - if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") - val universes = targets.map(_.universe).toSet - if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") - new StructuredBP(targets(0).universe, iterations, targets:_*) - } - - /** - * Use BP to compute the probability that the given element satisfies the given predicate. - */ - def probability[T](target: Element[T], predicate: T => Boolean, iterations: Int): Double = { - val alg = StructuredBP(iterations, target) - alg.start() - val result = alg.probability(target, predicate) - alg.kill() - result - } - - /** - * Use BP to compute the probability that the given element has the given value. - */ - def probability[T](target: Element[T], value: T, iterations: Int = 100): Double = - probability(target, (t: T) => t == value, iterations) -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredVE.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredVE.scala deleted file mode 100644 index 97695b47..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredVE.scala +++ /dev/null @@ -1,101 +0,0 @@ -/* - * StructuredVE.scala - * A structured variable elimination algorithm. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.algorithm - -import com.cra.figaro.algorithm.OneTimeProbQuery -import com.cra.figaro.algorithm.Algorithm -import com.cra.figaro.language.Element -import com.cra.figaro.language.Universe -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.experimental.structured.Problem -import com.cra.figaro.experimental.structured.strategy.recursiveSolver -import com.cra.figaro.experimental.structured.solver.variableElimination -import com.cra.figaro.algorithm.factored.factors.SumProductSemiring -import com.cra.figaro.experimental.structured.factory.Factory - -class StructuredVE(val universe: Universe, targets: Element[_]*) extends Algorithm with OneTimeProbQuery { - val queryTargets = targets - - var targetFactors: Map[Element[_], Factor[Double]] = _ - - var cc: ComponentCollection = _ - - def run() { - cc = new ComponentCollection - targetFactors = Map() - val problem = new Problem(cc, targets.toList) - val evidenceElems = universe.conditionedElements ::: universe.constrainedElements - evidenceElems.foreach(elem => if (!cc.contains(elem)) problem.add(elem)) - recursiveSolver(variableElimination)(problem) - val joint = problem.solution.foldLeft(Factory.unit(SumProductSemiring()))(_.product(_)) - - def marginalizeToTarget(target: Element[_]): Unit = { - val targetVar = cc(target).variable - val unnormalizedTargetFactor = joint.marginalizeTo(SumProductSemiring(), targetVar) - val z = unnormalizedTargetFactor.foldLeft(0.0, _ + _) - val targetFactor = unnormalizedTargetFactor.mapTo((d: Double) => d / z) - targetFactors += target -> targetFactor - } - - targets.foreach(marginalizeToTarget(_)) - } - - - /** - * Computes the normalized distribution over a single target element. - */ - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = { - val factor = targetFactors(target) - val targetVar = cc(target).variable - val dist = factor.getIndices.filter(f => targetVar.range(f.head).isRegular).map(f => (factor.get(f), targetVar.range(f.head).value)) - // normalization is unnecessary here because it is done in marginalizeTo - dist.toStream - } - - /** - * Computes the expectation of a given function for single target element. - */ - def computeExpectation[T](target: Element[T], function: T => Double): Double = { - def get(pair: (Double, T)) = pair._1 * function(pair._2) - (0.0 /: computeDistribution(target))(_ + get(_)) - } -} - -object StructuredVE { - /** Create a structured variable elimination algorithm with the given query targets. */ - def apply(targets: Element[_]*) = { - if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") - val universes = targets.map(_.universe).toSet - if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") - new StructuredVE(targets(0).universe, targets:_*) - } - - /** - * Use VE to compute the probability that the given element satisfies the given predicate. - */ - def probability[T](target: Element[T], predicate: T => Boolean): Double = { - val alg = StructuredVE(target) - alg.start() - val result = alg.probability(target, predicate) - alg.kill() - result - } - - /** - * Use VE to compute the probability that the given element has the given value. - */ - def probability[T](target: Element[T], value: T): Double = - probability(target, (t: T) => t == value) -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredVEBPChooser.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredVEBPChooser.scala deleted file mode 100644 index c1fc0f53..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/algorithm/StructuredVEBPChooser.scala +++ /dev/null @@ -1,108 +0,0 @@ -/* - * StructuredVEBPChooser.scala - * A hybrid algorithm that chooses between variable elimination and belief propagation for each component. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.algorithm - -import com.cra.figaro.algorithm.OneTimeProbQuery -import com.cra.figaro.algorithm.Algorithm -import com.cra.figaro.language.Element -import com.cra.figaro.language.Universe -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.experimental.structured.Problem -import com.cra.figaro.experimental.structured.strategy.recursiveSolver -import com.cra.figaro.experimental.structured.solver.chooseVEOrBP -import com.cra.figaro.algorithm.factored.factors.SumProductSemiring -import com.cra.figaro.experimental.structured.factory.Factory - -class StructuredVEBPChooser(val universe: Universe, scoreThreshold: Double, BPIterations: Int, targets: Element[_]*) -extends Algorithm with OneTimeProbQuery { - val queryTargets = targets - - var targetFactors: Map[Element[_], Factor[Double]] = _ - - var cc: ComponentCollection = _ - - def run() { - cc = new ComponentCollection - targetFactors = Map() - val problem = new Problem(cc, targets.toList) - val evidenceElems = universe.conditionedElements ::: universe.constrainedElements - evidenceElems.foreach(elem => if (!cc.contains(elem)) problem.add(elem)) - recursiveSolver(chooseVEOrBP(scoreThreshold, BPIterations))(problem) - val joint = problem.solution.foldLeft(Factory.unit(SumProductSemiring()))(_.product(_)) - - def marginalizeToTarget(target: Element[_]): Unit = { - val targetVar = cc(target).variable - val unnormalizedTargetFactor = joint.marginalizeTo(SumProductSemiring(), targetVar) - val z = unnormalizedTargetFactor.foldLeft(0.0, _ + _) - val targetFactor = unnormalizedTargetFactor.mapTo((d: Double) => d / z) - targetFactors += target -> targetFactor - } - - targets.foreach(marginalizeToTarget(_)) - } - - - /** - * Computes the normalized distribution over a single target element. - */ - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = { - val factor = targetFactors(target) - val targetVar = cc(target).variable - val dist = factor.getIndices.filter(f => targetVar.range(f.head).isRegular).map(f => (factor.get(f), targetVar.range(f.head).value)) - // normalization is unnecessary here because it is done in marginalizeTo - dist.toStream - } - - /** - * Computes the expectation of a given function for single target element. - */ - def computeExpectation[T](target: Element[T], function: T => Double): Double = { - def get(pair: (Double, T)) = pair._1 * function(pair._2) - (0.0 /: computeDistribution(target))(_ + get(_)) - } -} - -object StructuredVEBPChooser { - /** - * Create a hybrid algorithm that chooses between variable elimination and belief propagation on each subproblem. - * @param scoreThreshold The minimum value of the increase in score caused by eliminating a variable that causes the hybrid algorithm to - * choose BP for a subproblem. - * @param bpIterations The number of iterations to use when BP is chosen for a subproblem. - * @param targets The query targets - */ - def apply(scoreThreshold: Double, BPIterations: Int, targets: Element[_]*) = { - if (targets.isEmpty) throw new IllegalArgumentException("Cannot run VE with no targets") - val universes = targets.map(_.universe).toSet - if (universes.size > 1) throw new IllegalArgumentException("Cannot have targets in different universes") - new StructuredVEBPChooser(targets(0).universe, scoreThreshold, BPIterations, targets:_*) - } - - /** - * Use the hybrid algorithm to compute the probability that the given element satisfies the given predicate. - */ - def probability[T](target: Element[T], predicate: T => Boolean, threshold: Double, iterations: Int): Double = { - val alg = StructuredVEBPChooser(0.0, iterations, target) - alg.start() - val result = alg.probability(target, predicate) - alg.kill() - result - } - - /** - * Use the hybrid algorithm to compute the probability that the given element has the given value. - */ - def probability[T](target: Element[T], value: T, threshold: Double = 0.0, iterations: Int = 100): Double = - probability(target, (t: T) => t == value, threshold, iterations) -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ApplyFactory.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ApplyFactory.scala deleted file mode 100644 index 61e5bfe8..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ApplyFactory.scala +++ /dev/null @@ -1,181 +0,0 @@ -/* - * ApplyFactory.scala - * Methods to create factors associated with Apply elements. - * - * Created By: Glenn Takata (gtakata@cra.com) - * Creation Date: Dec 15, 2014 - * - * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.factory - -import com.cra.figaro.algorithm.PointMapper -import com.cra.figaro.algorithm.factored.factors._ -//import com.cra.figaro.algorithm.lazyfactored._ -import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured.ComponentCollection - -/** - * A Sub-Factory for Apply Elements - */ -object ApplyFactory { - - /** - * Factor constructor for an Apply Element that has one input - */ - def makeFactors[T, U](cc: ComponentCollection, apply: Apply1[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val arg1Var = Factory.getVariable(cc, apply.arg1) - val resultVar = Factory.getVariable(cc, apply) - val applyValues = cc(apply).range - val factor = new SparseFactor[Double](List(arg1Var), List(resultVar)) - val arg1Indices = arg1Var.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - } { - if (arg1Val.isRegular) { - val resultVal = mapper.map(apply.fn(arg1Val.value), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) - factor.set(List(arg1Index, resultIndex), 1.0) - } else if (!arg1Val.isRegular && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) - factor.set(List(arg1Index, resultIndex), 1.0) - } - } - List(factor) - } - - /** - * Factor constructor for an Apply Element that has two inputs - */ - def makeFactors[T1, T2, U](cc: ComponentCollection, apply: Apply2[T1, T2, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val arg1Var = Factory.getVariable(cc, apply.arg1) - val arg2Var = Factory.getVariable(cc, apply.arg2) - val resultVar = Factory.getVariable(cc, apply) - val applyValues = cc(apply).range - val factor = new SparseFactor[Double](List(arg1Var, arg2Var), List(resultVar)) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - } { - if (arg1Val.isRegular && arg2Val.isRegular) { - val resultVal = mapper.map(apply.fn(arg1Val.value, arg2Val.value), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) - factor.set(List(arg1Index, arg2Index, resultIndex), 1.0) - } else if ((!arg1Val.isRegular || !arg2Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) - factor.set(List(arg1Index, arg2Index, resultIndex), 1.0) - } - - } - List(factor) - } - - /** - * Factor constructor for an Apply Element that has three inputs - */ - def makeFactors[T1, T2, T3, U](cc: ComponentCollection, apply: Apply3[T1, T2, T3, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val arg1Var = Factory.getVariable(cc, apply.arg1) - val arg2Var = Factory.getVariable(cc, apply.arg2) - val arg3Var = Factory.getVariable(cc, apply.arg3) - val resultVar = Factory.getVariable(cc, apply) - val applyValues = cc(apply).range - val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var), List(resultVar)) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val arg3Indices = arg3Var.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - (arg3Val, arg3Index) <- arg3Indices - } { - if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular) { - val resultVal = mapper.map(apply.fn(arg1Val.value, arg2Val.value, arg3Val.value), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) - factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), 1.0) - } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) - factor.set(List(arg1Index, arg2Index, arg3Index, resultIndex), 1.0) - } - } - List(factor) - } - - /** - * Factor constructor for an Apply Element that has four inputs - */ - def makeFactors[T1, T2, T3, T4, U](cc: ComponentCollection, apply: Apply4[T1, T2, T3, T4, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val arg1Var = Factory.getVariable(cc, apply.arg1) - val arg2Var = Factory.getVariable(cc, apply.arg2) - val arg3Var = Factory.getVariable(cc, apply.arg3) - val arg4Var = Factory.getVariable(cc, apply.arg4) - val resultVar = Factory.getVariable(cc, apply) - val applyValues = cc(apply).range - val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var, arg4Var), List(resultVar)) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val arg3Indices = arg3Var.range.zipWithIndex - val arg4Indices = arg4Var.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - (arg3Val, arg3Index) <- arg3Indices - (arg4Val, arg4Index) <- arg4Indices - } { - if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular) { - val resultVal = mapper.map(apply.fn(arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) - factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), 1.0) - } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular || !arg4Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) - factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, resultIndex), 1.0) - } - } - List(factor) - } - - /** - * Factor constructor for an Apply Element that has five inputs - */ - def makeFactors[T1, T2, T3, T4, T5, U](cc: ComponentCollection, apply: Apply5[T1, T2, T3, T4, T5, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val arg1Var = Factory.getVariable(cc, apply.arg1) - val arg2Var = Factory.getVariable(cc, apply.arg2) - val arg3Var = Factory.getVariable(cc, apply.arg3) - val arg4Var = Factory.getVariable(cc, apply.arg4) - val arg5Var = Factory.getVariable(cc, apply.arg5) - val resultVar = Factory.getVariable(cc, apply) - val applyValues = cc(apply).range - val factor = new SparseFactor[Double](List(arg1Var, arg2Var, arg3Var, arg4Var, arg5Var), List(resultVar)) - val arg1Indices = arg1Var.range.zipWithIndex - val arg2Indices = arg2Var.range.zipWithIndex - val arg3Indices = arg3Var.range.zipWithIndex - val arg4Indices = arg4Var.range.zipWithIndex - val arg5Indices = arg5Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (arg2Val, arg2Index) <- arg2Indices - (arg3Val, arg3Index) <- arg3Indices - (arg4Val, arg4Index) <- arg4Indices - (arg5Val, arg5Index) <- arg5Indices - (resultVal, resultIndex) <- resultIndices - } { - if (arg1Val.isRegular && arg2Val.isRegular && arg3Val.isRegular && arg4Val.isRegular && arg5Val.isRegular) { - val resultVal = mapper.map(apply.fn(arg1Val.value, arg2Val.value, arg3Val.value, arg4Val.value, arg5Val.value), applyValues.regularValues) - val resultIndex = resultVar.range.indexWhere(xval => xval.isRegular && xval.value == resultVal) - factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), 1.0) - } else if ((!arg1Val.isRegular || !arg2Val.isRegular || !arg3Val.isRegular || !arg4Val.isRegular || !arg5Val.isRegular) && resultVar.range.exists(!_.isRegular)) { - val resultIndex = resultVar.range.indexWhere(!_.isRegular) - factor.set(List(arg1Index, arg2Index, arg3Index, arg4Index, arg5Index, resultIndex), 1.0) - } - } - List(factor) - } - -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ChainFactory.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ChainFactory.scala deleted file mode 100644 index 8750c30f..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ChainFactory.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * ChainFactory.scala - * Methods to create factors associated with Chain elements. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: Jan 1, 2009 - * - * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ -package com.cra.figaro.experimental.structured.factory - -import com.cra.figaro.algorithm.PointMapper -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.lazyfactored.ValueSet -import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured.ComponentCollection -/** - * @author Glenn Takata Feb 19, 2015 - * - */ -object ChainFactory { - /** - * Make the factors associated with a chain element. - */ - def makeFactors[T, U](cc: ComponentCollection, chain: Chain[T, U])(implicit mapper: PointMapper[U]): List[Factor[Double]] = { - val chainComp = cc(chain) - val parentVar = Factory.getVariable(cc, chain.parent) - val chainVar = Factory.getVariable(cc, chain) - val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, parentVar, chainVar) - var tempFactors = parentVar.range.zipWithIndex flatMap (pair => { - val (parentVal, parentIndex) = pair - if (parentVal.isRegular) { - // We need to create an actual variable to represent the outcome of the chain. - // This will have the same range as the formal variable associated with the expansion. - // The formal variable will be replaced with the actual variable in the solution. - // There are two possibilities. - // If the outcome element is defined inside the chain, it will actually be different in every expansion, - // even though the formal variable is the same. But if the outcome element is defined outside the chain, - // it is a global that will be the same every time, so the actual variable is the variable of this global. - val nestedProblem = cc.expansions((chain.chainFunction, parentVal.value)) - val outcomeElem = nestedProblem.target.asInstanceOf[Element[U]] - val formalVar = Factory.getVariable(cc, outcomeElem) - val actualVar = if (nestedProblem.internal(formalVar)) Factory.makeVariable(cc, formalVar.valueSet) else formalVar - chainComp.actualSubproblemVariables += parentVal.value -> actualVar - List(Factory.makeConditionalSelector(pairVar, parentVal, actualVar)) - } - else { - // We create a dummy variable for the outcome variable whose value is always star. - // We create a dummy factor for that variable. - // Then we use makeConditionalSelector with the dummy variable - val dummy = Factory.makeVariable(cc, ValueSet.withStar[U](Set())) - List(Factory.makeConditionalSelector(pairVar, parentVal, dummy)) - } - }) - pairFactor :: tempFactors - } - -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ComplexFactory.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ComplexFactory.scala deleted file mode 100644 index 91f1d25a..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/ComplexFactory.scala +++ /dev/null @@ -1,300 +0,0 @@ -/* - * ComplexFactory.scala - * Methods to create factors associated with a variety of complex elements. - * - * Created By: Glenn Takata (gtakata@cra.com) - * Creation Date: Dec 15, 2014 - * - * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.factory - -import com.cra.figaro.algorithm.lazyfactored.{ValueSet, Extended, Regular} -import ValueSet._ -import com.cra.figaro.language._ -import com.cra.figaro.library.compound._ -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.experimental.structured.{ComponentCollection, ProblemComponent, Range} -import com.cra.figaro.util.{MultiSet, HashMultiSet, homogeneousCartesianProduct} - -/** - * A Sub-Factory for Complex Elements - */ -object ComplexFactory { - - /** - * Factor constructor for a SingleValuedReferenceElement - */ - def makeFactors[T](cc: ComponentCollection, element: SingleValuedReferenceElement[T]): List[Factor[Double]] = { - // Make the list of factors starting from the given element collection with the given reference. - // startVar represents the variable whose value is the value of the reference. - def make(ec: ElementCollection, startVar: Variable[T], reference: Reference[T]): List[Factor[Double]] = { - val (firstElem, restRefOpt) = ec.getFirst(reference) - restRefOpt match { - case None => - val firstVar = Factory.getVariable(cc, firstElem) - val factor = new SparseFactor[Double](List(firstVar), List(startVar)) - for { - (firstVal, firstIndex) <- firstVar.range.zipWithIndex - } { - val startIndex = startVar.range.indexOf(firstVal) - factor.set(List(firstIndex, startIndex), 1.0) - } - List(factor) - - case Some(restRef) => - val firstVar: Variable[firstElem.Value] = Factory.getVariable(cc, firstElem).asInstanceOf[Variable[firstElem.Value]] - // For each possible first element collection, we: - // - create an element named restVar to denote the value of the reference starting from that collection - // - get the factors associated with restVar by calling make(firstCollection, restVar, restRef) - // - use a conditional selector to select thos factors appropriately - // Then we take all of the factors for all first element collections. - val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, firstVar, startVar) - val selectedFactors = - for { - (firstXValue, firstIndex) <- firstVar.range.zipWithIndex - } yield { - if (firstXValue.isRegular) { - val firstCollection = firstXValue.value.asInstanceOf[ElementCollection] - val restRange = Range.getRangeOfSingleValuedReference(cc, firstCollection, restRef) - val restVar: Variable[T] = Factory.makeVariable(cc, restRange) - val restFactors = make(firstCollection, restVar, restRef) - - Factory.makeConditionalSelector(pairVar, firstXValue, restVar) :: restFactors - } else { - val dummy = Factory.makeVariable(cc, ValueSet.withStar[T](Set())) - List(Factory.makeConditionalSelector(pairVar, firstXValue, dummy)) - } - } - pairFactor :: selectedFactors.flatten - } - } - - make(element.collection, Factory.getVariable(cc, element), element.reference) - } - - /** - * Factor constructor for a MultiValuedReferenceElement - */ - def makeFactors[T](cc: ComponentCollection, element: MultiValuedReferenceElement[T]): List[Factor[Double]] = { - def makeEmbeddedInject(inputVariables: List[Variable[MultiSet[T]]]): (Variable[List[MultiSet[T]]], Factor[Double]) = { - def rule(values: List[Extended[_]]) = { - val inputXvalues :+ resultXvalue = values - if (inputXvalues.exists(!_.isRegular)) { - if (!resultXvalue.isRegular) 1.0; else 0.0 - } else if (resultXvalue.isRegular) { - if (resultXvalue.value.asInstanceOf[List[T]] == inputXvalues.map(_.value.asInstanceOf[T])) 1.0; else 0.0 - } else 0.0 - } - val argVSs = inputVariables.map(_.valueSet) - val incomplete = argVSs.exists(_.hasStar) - val argChoices = argVSs.toList.map(_.regularValues.toList) - val resultValues: Set[List[MultiSet[T]]] = homogeneousCartesianProduct(argChoices: _*).toSet - val injectRange = if (incomplete) withStar(resultValues); else withoutStar(resultValues) - - val resultVariable = Factory.makeVariable(cc, injectRange) - val factor = new BasicFactor[Double](inputVariables, List(resultVariable)) - factor.fillByRule(rule _) - (resultVariable, factor) - } - - def makeEmbeddedApply(injectVar: Variable[List[MultiSet[T]]]): (Variable[MultiSet[T]], Factor[Double]) = { - def rule(sets: List[MultiSet[T]]): MultiSet[T] = { - val starter: MultiSet[T] = HashMultiSet[T]() - sets.foldLeft(starter)(_ union _) - } - val applyVS: ValueSet[MultiSet[T]] = injectVar.valueSet.map(rule(_)) - val applyVar = Factory.makeVariable(cc, applyVS) - val factor = new SparseFactor[Double](List(injectVar), List(applyVar)) - for { (injectVal, injectIndex) <- injectVar.range.zipWithIndex } { - if (injectVal.isRegular) { - val resultVal = rule(injectVal.value) - val resultIndex = applyVar.range.indexWhere(_.value == resultVal) - factor.set(List(injectIndex, resultIndex), 1.0) - } else if (!injectVal.isRegular && applyVar.range.exists(!_.isRegular)) { - val resultIndex = applyVar.range.indexWhere(!_.isRegular) - factor.set(List(injectIndex, resultIndex), 1.0) - } - } - (applyVar, factor) - } - - // Make the list of factors starting from the given element collection with the given reference. - // startVar represents the variable whose value is the value of the reference. - def make(ec: ElementCollection, startVar: Variable[MultiSet[T]], reference: Reference[T]): List[Factor[Double]] = { - val (firstElem, restRefOpt) = ec.getFirst(reference) - val firstVar = Factory.getVariable(cc, firstElem).asInstanceOf[Variable[firstElem.Value]] - restRefOpt match { - case None => { - // When the reference is simple, it only refers to a single element, so we just get the factor mapping - // values of that element to values of startVar - val factor = new SparseFactor[Double](List(firstVar), List(startVar)) - for { - (firstVal, firstIndex) <- firstVar.range.zipWithIndex - } { - val startIndex = - if (firstVal.isRegular) startVar.range.indexWhere(_.value == HashMultiSet(firstVal.value)) - else startVar.range.indexWhere(!_.isRegular) - factor.set(List(firstIndex, startIndex), 1.0) - } - List(factor) - } - - case Some(restRef) => { - // When the reference is indirect, we get all factors associated with all values of the first name in the reference. - // Each first value is either a single element collection, in which case we recurse simply, just like for a single-valued reference. - // Otherwise, the first value is a traversable of element collections, in which case see below. - val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, firstVar, startVar) - val selectionFactors: List[List[Factor[Double]]] = { - val selectedFactors = - for { - (firstXvalue, firstIndex) <- firstVar.range.zipWithIndex - } yield { - if (!firstXvalue.isRegular) { - val dummy = Factory.makeVariable(cc, ValueSet.withStar[MultiSet[T]](Set())) - List(Factory.makeConditionalSelector(pairVar, firstXvalue, dummy)) - } - else { - firstXvalue.value match { - case firstCollection: ElementCollection => - val restRange = Range.getRangeOfMultiValuedReference(cc, firstCollection, restRef) - val restVar: Variable[MultiSet[T]] = Factory.makeVariable(cc, restRange) - val restFactors = make(firstCollection, restVar, restRef) - Factory.makeConditionalSelector(pairVar, firstXvalue, restVar) :: restFactors - case cs: Traversable[_] => - // If the first value consists of multiple element collections, we first get a list of distinct collections. - // This is because multi-valued references use set semantics, whereby if an element is pointed to more than once, - // its value only counts once in the multiset value of the reference element. - // So, a multiset value consists of a value for each of the distinct element collections. - // For each collection, we get the factors for it using make(firstCollection, restVar, restRef) - // We add a conditional selector on the first value to determine when these factors are relevant. - // Then, we essentially create an embedded Inject on the variables representing these collections, - // and then an Apply that takes a list of values from these collections and turns them into a multiset. - // However, unlike the old implementation, we do not actually create these as elements, as that would - // break the atomicity of makeFactors required by structured factored inference. - // We just create a variable for the Inject and a variable for the Apply and create the factors directly. - val collections = cs.asInstanceOf[Traversable[ElementCollection]].toList.distinct // Set semantics - val factorsForCollections = - for { firstCollection <- collections } yield { - val restRange = Range.getRangeOfMultiValuedReference(cc, firstCollection, restRef) - val restVar: Variable[MultiSet[T]] = Factory.makeVariable(cc, restRange) - val restFactors = make(firstCollection, restVar, restRef) - (restVar, restFactors) - } - val (injectVar, injectFactor) = makeEmbeddedInject(factorsForCollections.map(_._1)) - val (applyVar, applyFactor) = makeEmbeddedApply(injectVar) - val valueFactors = applyFactor :: injectFactor :: factorsForCollections.map(_._2).flatten - Factory.makeConditionalSelector(pairVar, firstXvalue, applyVar) :: valueFactors - } - } - } - selectedFactors - } - pairFactor :: selectionFactors.flatten - } - } - } - - make(element.collection, Factory.getVariable(cc, element), element.reference) - } - - /** - * Factor constructor for an Aggregate Element - */ - def makeFactors[T, U](cc: ComponentCollection, element: Aggregate[T, U]): List[Factor[Double]] = { - val elementVar = Factory.getVariable(cc, element) - val mvreVar = Factory.getVariable(cc, element.mvre) - val factor = new SparseFactor[Double](List(mvreVar), List(elementVar)) - for { - (mvreXvalue, mvreIndex) <- mvreVar.range.zipWithIndex - } { - if (mvreXvalue.isRegular) { - val aggValue = element.aggregate(mvreXvalue.value) - val elementIndex = elementVar.range.indexOf(Regular(aggValue)) - factor.set(List(mvreIndex, elementIndex), 1.0) - } else factor.set(List(mvreIndex, elementVar.range.indexWhere(!_.isRegular)), 1.0) - } - // The MultiValuedReferenceElement for this aggregate is generated when values is called. - // Therefore, it will be included in the expansion and have factors made for it automatically, so we do not create factors for it here. - List(factor) - } - - /** - * Factor constructor for a MakeArray Element - */ - def makeFactors[T](cc: ComponentCollection, element: com.cra.figaro.library.collection.MakeArray[T]): List[Factor[Double]] = { - val arg1Var = Factory.getVariable(cc, element.numItems) - val resultVar = Factory.getVariable(cc, element) - val makeArrayComponent = cc(element) - val factor = new SparseFactor[Double](List(arg1Var), List(resultVar)) - val arg1Indices = arg1Var.range.zipWithIndex - val resultIndices = resultVar.range.zipWithIndex - for { - (arg1Val, arg1Index) <- arg1Indices - (resultVal, resultIndex) <- resultIndices - } { - if ((arg1Val.isRegular && arg1Val.value <= makeArrayComponent.maxExpanded && - resultVal.isRegular && resultVal.value == element.arrays(arg1Val.value) || - (!arg1Val.isRegular || arg1Val.value > makeArrayComponent.maxExpanded) && !resultVal.isRegular)) { - factor.set(List(arg1Index, resultIndex), 1.0) - } - } - List(factor) - } - - /** - * Factor constructor for a FoldLeft Element - */ - def makeFactors[T,U](cc: ComponentCollection, fold: FoldLeft[T,U]): List[Factor[Double]] = { - def makeOneFactor(currentAccumVar: Variable[U], elemVar: Variable[T], nextAccumVar: Variable[U]): Factor[Double] = { - val result = new SparseFactor[Double](List(currentAccumVar, elemVar), List(nextAccumVar)) - val currentAccumIndices = currentAccumVar.range.zipWithIndex - val elemIndices = elemVar.range.zipWithIndex - val nextAccumIndices = nextAccumVar.range.zipWithIndex - for { - (currentAccumVal, currentAccumIndex) <- currentAccumIndices - (elemVal, elemIndex) <- elemIndices - (nextAccumVal, nextAccumIndex) <- nextAccumIndices - } { - val entry = - if (currentAccumVal.isRegular && elemVal.isRegular && nextAccumVal.isRegular) { - if (nextAccumVal.value == fold.function(currentAccumVal.value, elemVal.value)) - result.set(List(currentAccumIndex, elemIndex, nextAccumIndex), 1.0) - } else if ((!currentAccumVal.isRegular || !elemVal.isRegular) && !nextAccumVal.isRegular) { - result.set(List(currentAccumIndex, elemIndex, nextAccumIndex), 1.0) - } - } - result - } - - def makeFactorSequence(currentAccumVar: Variable[U], remaining: Seq[Element[T]]): List[Factor[Double]] = { - if (remaining.isEmpty) List() - else { - val firstVar = Factory.getVariable(cc, remaining.head) - val rest = remaining.tail - val nextAccumVar = - if (rest.isEmpty) Factory.getVariable(cc, fold) - else { - val currentAccumRegular = currentAccumVar.range.filter(_.isRegular).map(_.value) - val firstRegular = firstVar.range.filter(_.isRegular).map(_.value) - val nextVals = - for { - accum <- currentAccumRegular - first <- firstRegular - } yield fold.function(accum, first) - val nextHasStar = currentAccumVar.range.exists(!_.isRegular) || firstVar.range.exists(!_.isRegular) - val nextVS = if (nextHasStar) ValueSet.withStar(nextVals.toSet) else ValueSet.withoutStar(nextVals.toSet) - Factory.makeVariable(cc, nextVS) - } - val nextFactor = makeOneFactor(currentAccumVar, firstVar, nextAccumVar) - nextFactor :: makeFactorSequence(nextAccumVar, rest) - } - } - val startVar = Factory.makeVariable(cc, ValueSet.withoutStar(Set(fold.start))) - makeFactorSequence(startVar, fold.elements) - } -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/DistributionFactory.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/DistributionFactory.scala deleted file mode 100644 index fa730ac8..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/DistributionFactory.scala +++ /dev/null @@ -1,159 +0,0 @@ -/* - * DistributionFactory.scala - * Methods to create factors for simple distributions. - * - * Created By: Glenn Takata (gtakata@cra.com) - * Creation Date: Dec 15, 2014 - * - * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.factory - -import com.cra.figaro.language._ -import com.cra.figaro.library.atomic.discrete._ -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.lazyfactored.{Regular, Star} -import com.cra.figaro.experimental.structured.ComponentCollection - -/** - * A Sub-Factory for simple probability distribution Elements - */ -object DistributionFactory { - - /** - * Factor constructor for an AtomicFlip - */ - def makeFactors(cc: ComponentCollection, flip: AtomicFlip): List[Factor[Double]] = { - val flipVar = Factory.getVariable(cc, flip) - if (flipVar.range.exists(!_.isRegular)) { - assert(flipVar.range.size == 1) // Flip's range must either be {T,F} or {*} - StarFactory.makeStarFactor(cc, flip) - } else { - val factor = new BasicFactor[Double](List(), List(flipVar)) - val i = flipVar.range.indexOf(Regular(true)) - factor.set(List(i), flip.prob) - factor.set(List(1 - i), 1.0 - flip.prob) - List(factor) - } - } - - /** - * Factor constructor for a CompoundFlip - */ - def makeFactors(cc: ComponentCollection, flip: CompoundFlip): List[Factor[Double]] = { - val flipVar = Factory.getVariable(cc, flip) - val probVar = Factory.getVariable(cc, flip.prob) - makeCompoundFlip(flipVar, probVar) - } - - private def makeCompoundFlip(flipVar: Variable[Boolean], probVar: Variable[Double]): List[Factor[Double]] = { - val factor = new BasicFactor[Double](List(probVar), List(flipVar)) - val parentVals = probVar.range - if (flipVar.range.exists(!_.isRegular)) { - val falseIndex = flipVar.range.indexOf(Regular(false)) - val trueIndex = flipVar.range.indexOf(Regular(true)) - val starIndex = flipVar.range.indexWhere(!_.isRegular) - for { j <- 0 until parentVals.size } { - if (parentVals(j).isRegular) { - val value = parentVals(j).value - factor.set(List(j, trueIndex), value) - factor.set(List(j, falseIndex), 1.0 - value) - factor.set(List(j, starIndex), 0.0) - } else { - factor.set(List(j, trueIndex), 0.0) - factor.set(List(j, falseIndex), 0.0) - factor.set(List(j, starIndex), 1.0) - } - } - List(factor) - } else { - val trueIndex = flipVar.range.indexOf(Regular(true)) - val falseIndex = 1 - trueIndex - for { j <- 0 until parentVals.size } { - val value = parentVals(j).value - factor.set(List(j, trueIndex), value) - factor.set(List(j, falseIndex), 1.0 - value) - } - List(factor) - } - } - - /** - * Factor constructor for a ParameterizedFlip - */ - def makeFactors(cc: ComponentCollection, flip: ParameterizedFlip, parameterized: Boolean): List[Factor[Double]] = { - if (parameterized) { - val flipVar = Factory.getVariable(cc, flip) - val factor = new BasicFactor[Double](List(),List(flipVar)) - val prob = flip.parameter.MAPValue - if (flipVar.range.forall(_.isRegular)) { - val i = flipVar.range.indexOf(Regular(true)) - factor.set(List(i), prob) - factor.set(List(1 - i), 1.0 - prob) - } else { - val trueIndex = flipVar.range.indexOf(Regular(true)) - val falseIndex = flipVar.range.indexOf(Regular(false)) - val starIndex = flipVar.range.indexWhere(!_.isRegular) - factor.set(List(trueIndex), prob) - factor.set(List(falseIndex), 1.0 - prob) - factor.set(List(starIndex), 0.0) - } - List(factor) - } else { - val flipVar = Factory.getVariable(cc, flip) - val probVar = Factory.getVariable(cc, flip.parameter) - makeCompoundFlip(flipVar, probVar) - } - } - - /** - * Factor constructor for an AtomicBinomial - */ - def makeFactors(cc: ComponentCollection, binomial: AtomicBinomial): List[Factor[Double]] = { - val binVar = Factory.getVariable(cc, binomial) - val factor = new BasicFactor[Double](List(), List(binVar)) - for { (xvalue, index) <- binVar.range.zipWithIndex } { - factor.set(List(index), binomial.density(xvalue.value)) - } - List(factor) - } - - /** - * Factor constructor for a parameterized binomial - */ - def makeFactors(cc: ComponentCollection, binomial: ParameterizedBinomialFixedNumTrials, parameterized: Boolean): List[Factor[Double]] = { - if (parameterized) { - val binVar = Factory.getVariable(cc, binomial) - val factor = new BasicFactor[Double](List(),List(binVar)) - if (binVar.range.exists(!_.isRegular)) { // parameter must not have been added since it's an atomic beta - for { (xvalue, index) <- binVar.range.zipWithIndex } { - val entry = if (xvalue.isRegular) 0.0 else 1.0 - factor.set(List(index), entry) - } - } else { - val probSuccess = binomial.parameter.MAPValue - for { (xvalue, index) <- binVar.range.zipWithIndex } { - factor.set(List(index), Util.binomialDensity(binomial.numTrials, probSuccess, xvalue.value)) - } - } - List(factor) - } else { - val binVar = Factory.getVariable(cc, binomial) - if (binVar.range.exists(!_.isRegular)) { // parameter must not have been added since it's an atomic beta - val factor = new BasicFactor[Double](List(),List(binVar)) - for { (xvalue, index) <- binVar.range.zipWithIndex } { - val entry = if (xvalue.isRegular) 0.0 else 1.0 - factor.set(List(index), entry) - } - List(factor) - } else { - ChainFactory.makeFactors(cc, binomial) - } - } - } - -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/SelectFactory.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/SelectFactory.scala deleted file mode 100644 index d12e5c9c..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/factory/SelectFactory.scala +++ /dev/null @@ -1,245 +0,0 @@ -/* - * SelectFactory.scala - * Methods to create factors for Select and Dist elements. - * - * Created By: Glenn Takata (gtakata@cra.com) - * Creation Date: Dec 15, 2014 - * - * Copyright 2014 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.factory - -import com.cra.figaro.language._ -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.library.compound._ -import com.cra.figaro.util._ -import com.cra.figaro.algorithm.lazyfactored._ -import ValueSet.withStar -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.algorithm.lazyfactored.Star - -/** - * A Sub-Factory for Select or Dist Elements - */ -object SelectFactory { - - /** - * Factor constructor for an AtomicDistFlip - */ - def makeFactors[T](cc: ComponentCollection, dist: AtomicDist[T]): List[Factor[Double]] = { - val (intermed, clauseFactors) = intermedAndClauseFactors(cc, dist) - val intermedFactor = makeSimpleDistribution(intermed, dist.probs) - intermedFactor :: clauseFactors - } - - /** - * Factor constructor for a CompoundDist - */ - def makeFactors[T](cc: ComponentCollection, dist: CompoundDist[T]): List[Factor[Double]] = { - val (intermed, clauseFactors) = intermedAndClauseFactors(cc, dist) - val probVars = dist.probs map (Factory.getVariable(cc, _)) - val intermedFactor = makeComplexDistribution(cc, intermed, probVars) - intermedFactor :: clauseFactors - } - - /** - * Factor constructor for an AtomicSelect - */ - def makeFactors[T](cc: ComponentCollection, select: AtomicSelect[T]): List[Factor[Double]] = { - val selectVar = Factory.getVariable(cc, select) - if (selectVar.range.exists(!_.isRegular)) { - assert(selectVar.range.size == 1) // Select's range must either be a list of regular values or {*} - StarFactory.makeStarFactor(cc, select) - } else { - val probs = getProbs(cc, select) - List(makeSimpleDistribution(selectVar, probs)) - } - } - - /** - * Factor constructor for a CompoundSelect - */ - def makeFactors[T](cc: ComponentCollection, select: CompoundSelect[T]): List[Factor[Double]] = { - val selectVar = Factory.getVariable(cc, select) - val probs = getProbs(cc, select) - val probVars = probs map (Factory.getVariable(cc, _)) - List(makeComplexDistribution(cc, selectVar, probVars)) - } - - /** - * Factor constructor for a ParameterizedSelect - */ - def makeFactors[T](cc: ComponentCollection, select: ParameterizedSelect[T], parameterized: Boolean): List[Factor[Double]] = { - if (parameterized) { - val selectVar = Factory.getVariable(cc, select) - val probs = parameterizedGetProbs(cc, select) - List(makeSimpleDistribution(selectVar, probs)) - } else { - val selectVar: Variable[T] = Factory.getVariable(cc, select) - if (selectVar.range.exists(!_.isRegular)) { - // If the select has * in its range, the parameter must not have been added (because the parameter is an atomic beta) - val factor = new BasicFactor[Double](List(), List(selectVar)) - for { (selectXval, i) <- selectVar.range.zipWithIndex } { - val entry = if (selectXval.isRegular) 0.0 else 1.0 - factor.set(List(i), entry) - } - List(factor) - } else { - val paramVar: Variable[Array[Double]] = Factory.getVariable(cc, select.parameter) - val factor = new BasicFactor[Double](List(paramVar), List(selectVar)) - for { - (paramVal, paramIndex) <- paramVar.range.zipWithIndex - (selectVal, selectIndex) <- selectVar.range.zipWithIndex - } { - val entry = paramVal.value(selectIndex) - factor.set(List(paramIndex, selectIndex), entry) - } - List(factor) - } - } - } - - /** - * Factor constructor for an IntSelector - */ - def makeFactors[T](cc: ComponentCollection, select: IntSelector): List[Factor[Double]] = { - val elementVar = Factory.getVariable(cc, select) - val counterVar = Factory.getVariable(cc, select.counter) - val comb = new BasicFactor[Double](List(counterVar), List(elementVar)) - comb.fillByRule((l: List[Any]) => { - val counterValue :: elementValue :: _ = l.asInstanceOf[List[Extended[Int]]] - if (counterValue.isRegular && elementValue.isRegular) { - if (elementValue.value < counterValue.value) 1.0 / counterValue.value; else 0.0 - } else 1.0 - - }) - List(comb) - } - - private def getProbs[U, T](cc: ComponentCollection, select: Select[U, T]): List[U] = getProbs(cc, select, select.clauses) - - /** - * Get the potential (probability) for each value of an element, based on supplied rules - */ - def getProbs[U, T](cc: ComponentCollection, elem: Element[T], clauses: List[(U, T)]): List[U] = { - val selectVar = Factory.getVariable(cc, elem) - def getProb(xvalue: Extended[T]): U = { - clauses.find(_._2 == xvalue.value).get._1 // * cannot be a value of a Select - } - val probs = - for { xvalue <- selectVar.range.filter(_.isRegular) } yield getProb(xvalue) - probs - } - - private def parameterizedGetProbs[T](cc: ComponentCollection, select: ParameterizedSelect[T]): List[Double] = { - val outcomes = select.outcomes - val map = select.parameter.MAPValue - for { - xvalue <- Factory.getVariable(cc, select).range - } yield { - if (xvalue.isRegular) map(outcomes.indexOf(xvalue.value)) else 0.0 - } - } - - private def intermedAndClauseFactors[U, T](cc: ComponentCollection, dist: Dist[U, T]): (Variable[Int], List[Factor[Double]]) = { - val intermed = Factory.makeVariable(cc, ValueSet.withoutStar((0 until dist.clauses.size).toSet)) - val distVar = Factory.getVariable(cc, dist) - val (pairVar, pairFactor) = Factory.makeTupleVarAndFactor(cc, intermed, distVar) - val clauseFactors = dist.outcomes.zipWithIndex map (pair => - Factory.makeConditionalSelector(pairVar, Regular(pair._2), Factory.getVariable(cc, pair._1))) - (intermed, pairFactor :: clauseFactors) - } - - /** - * Constructs a BasicFactor from a probability distribution. It assumes that the probabilities - * are assigned to the Variable in the same order as it's values. - */ - def makeSimpleDistribution[T](target: Variable[T], probs: List[Double]): Factor[Double] = { - val factor = new BasicFactor[Double](List(), List(target)) - for { (prob, index) <- probs.zipWithIndex } { - factor.set(List(index), prob) - } - factor - } - - /* - * When one of the probability elements includes * in its range, so will the target element. - * This is necessary because when the value of any of the probability elements is *, the normalizing factor is unknown in this case, - * so we cannot assign a specific probability to any of the regular values. Instead, we assign probability 1 to *. - * The code in the method below is designed to take into account this case correctly. - */ - private def makeComplexDistribution[T](cc: ComponentCollection, target: Variable[T], probVars: List[Variable[Double]]): Factor[Double] = { - val nVars = probVars.size - val factor = new BasicFactor[Double](probVars, List(target)) - val probVals: List[List[Extended[Double]]] = probVars map (_.range) - if (target.range.forall(_.isRegular)) { - /* - * This is the easy case. For each list of indices to the factor, the first nVars indices will be indices into the range of the - * probability variables, while the last index will be the index into the range of the target variable. - * The correct probability is the value of the probability element in the given position with the appropriate index. - * Because the probabilities may vary, we need to normalize them before putting in the factor. - * - * Note that the variables in the factor are ordered with the probability variables first and the target variable last. - * But in probVals, the first index is the position into the target variable, and only then do we have the probability indices. - */ - for { indices <- factor.getIndices } { - val unnormalized = - for { (probIndex, position) <- indices.toList.take(nVars).zipWithIndex } yield { - val xprob = probVals(position)(probIndex) // The probability of the particular value of the probability element in this position - xprob.value - } - val normalized = normalize(unnormalized).toArray - factor.set(indices, normalized(indices.last)) - } - } else { - /* - * In this case, the range of the target includes *. We do not assume any particular location in the range, - * so we set targetStarIndex to the correct location. - * Now, the indices to the factor will have nVar entries for each of the probability variables, plus one for the target. - * When we get the entries for the probability variables, we get nVars. But the range of the target is nVars plus one, - * because it includes *. So we extend the indices to probPlusStarIndices, with the targetStarIndex spliced in the correct place. - * - * Next, we distinguish between two cases. In the first case, one of the probability variables is *, so we assign probability 1 - * to the target being * and probability 0 everywhere. In the other case, all probability variables are regular, so we - * assign the appropriate probability to the each regular position. We are careful to get the value of the - * probability variable from its original position in the indices, and make sure we don't get the value of *. - */ - val targetStarIndex: Int = target.range.indexWhere(!_.isRegular) - for { indices <- factor.getIndices } { - val probIndices: List[Int] = indices.toList.take(nVars) - val probPlusStarIndices: List[Int] = - probIndices.slice(0, targetStarIndex) ::: List(indices(targetStarIndex)) ::: probIndices.slice(targetStarIndex, probIndices.length) - val allProbsRegularFlags: List[Boolean] = - for { (probPlusStarIndex, position) <- probPlusStarIndices.zipWithIndex } yield { - if (position < targetStarIndex) probVals(position)(probPlusStarIndex).isRegular - else if (position == targetStarIndex) true - else probVals(position)(probPlusStarIndex - 1).isRegular - } - val allProbsRegular: Boolean = allProbsRegularFlags.forall((b: Boolean) => b) - - val unnormalized: List[Double] = - for { (probPlusStarIndex, position) <- probPlusStarIndices.zipWithIndex } yield { - val xprob: Extended[Double] = - if (position < targetStarIndex) probVals(position)(probPlusStarIndex) - else if (position == targetStarIndex) Star[Double] - else probVals(position)(probPlusStarIndex - 1) - if (allProbsRegular) { - if (position != targetStarIndex) xprob.value else 0.0 - } else { - if (position == targetStarIndex) 1.0 else 0.0 - } - } - - val normalized: Array[Double] = normalize(unnormalized).toArray - // The first variable specifies the position of the remaining variables, so indices(0) is the correct probability - factor.set(indices, normalized(indices.last)) - } - } - - factor - } -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/BPSolver.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/BPSolver.scala deleted file mode 100644 index 2f1fc622..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/BPSolver.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * BPSolver.scala - * A belief propagation solver. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured.solver - -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.algorithm.factored.factors.Variable -import com.cra.figaro.algorithm.factored.beliefpropagation.FactorGraph -import com.cra.figaro.algorithm.factored.beliefpropagation.BasicFactorGraph -import com.cra.figaro.language.Element -import com.cra.figaro.algorithm.factored.factors.ElementVariable -import com.cra.figaro.algorithm.factored.beliefpropagation.VariableNode -import com.cra.figaro.algorithm.factored.beliefpropagation.Node -import com.cra.figaro.algorithm.factored.factors.LogSumProductSemiring -import com.cra.figaro.util._ -import com.cra.figaro.algorithm.lazyfactored._ -import ValueSet._ -import com.cra.figaro.algorithm.factored.factors.SparseFactor -import com.cra.figaro.algorithm.factored.beliefpropagation.FactorNode -import com.cra.figaro.experimental.structured.factory.Factory.makeTupleVarAndFactor -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.experimental.structured.Problem - -private[figaro] class BPSolver(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]], val iterations: Int) -extends com.cra.figaro.algorithm.factored.beliefpropagation.OneTimeProbabilisticBeliefPropagation { - // We need to create a joint probability distribution over the interface to this nested subproblem. - // To achieve this, we create a new variable representing the tuple of the attributes to preserve. - // We create a factor to represent the tuple creation. - // We then run BP as usual. - // At the end, we sum the tuple variable out of this factor to obtain the solution. - - val (tupleVar, tupleFactor): (Variable[_], Factor[Double]) = makeTupleVarAndFactor(problem.collection, toPreserve.toList:_*) - - def generateGraph() = { - val allFactors = tupleFactor :: factors - factorGraph = new BasicFactorGraph(allFactors.map(makeLogarithmic(_)), semiring): FactorGraph[Double] - } - - override def initialize() = { - if (factorGraph == null) generateGraph() - super.initialize - } - - def go(): List[Factor[Double]] = { - initialize() - run() - val targetVars = toPreserve.toList ::: List(tupleVar) - val tupleBelief = belief(FactorNode(toPreserve + tupleVar)) - val targetBelief = tupleBelief.sumOver(tupleVar) - List(unmakeLogarithmic(normalize(targetBelief))) - } - - def computeDistribution[T](target: Element[T]): Stream[(Double, T)] = getBeliefsForElement(target).toStream - - def computeExpectation[T](target: Element[T], function: T => Double): Double = { - computeDistribution(target).map((pair: (Double, T)) => pair._1 * function(pair._2)).sum - } - - val dependentUniverses = null - - val dependentAlgorithm = null - - val universe = null - - val semiring = LogSumProductSemiring() - - val targetElements = null -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/VEBPChooser.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/VEBPChooser.scala deleted file mode 100644 index 5acd9d47..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/VEBPChooser.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * VEBPChooser.scala - * A hybrid solver that chooses between variable elimination and belief propagation. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ -package com.cra.figaro.experimental.structured.solver - -import com.cra.figaro.algorithm.factored.VariableElimination -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.experimental.structured.Problem - -/* - * The score threshold is the threshold to determine whether VE or BP will be used. - * This score represents the maximum increase in cost of the factors through using VE, compared to the initial factors. - */ -private[figaro] class VEBPChooser(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]], - val scoreThreshold: Double, val iterations: Int) -extends VariableElimination[Double] { - def go(): List[Factor[Double]] = { - val (score, order) = VariableElimination.eliminationOrder(factors, toPreserve) -// print("Score = " + score + " - ") - if (score > scoreThreshold) { -// println("Choosing BP") - val bp = new BPSolver(problem, toEliminate, toPreserve, factors, iterations) - result = bp.go() - } else { -// println("Choosing VE") - // Since we've already computed the order, we don't call doElimination but only do the steps after computing the order - val factorsAfterElimination = eliminateInOrder(order, scala.collection.mutable.Set(factors: _*), initialFactorMap(factors)) - finish(factorsAfterElimination, order) - } - result - } - - val semiring: Semiring[Double] = SumProductSemiring() - - private var result: List[Factor[Double]] = _ - - def finish(factorsAfterElimination: scala.collection.mutable.Set[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = { - result = factorsAfterElimination.toList - } - - val dependentAlgorithm: (com.cra.figaro.language.Universe, List[com.cra.figaro.language.NamedEvidence[_]]) => () => Double = null - - val dependentUniverses: List[(com.cra.figaro.language.Universe, List[com.cra.figaro.language.NamedEvidence[_]])] = null - - def getFactors(neededElements: List[com.cra.figaro.language.Element[_]], - targetElements: List[com.cra.figaro.language.Element[_]], - upperBounds: Boolean): List[com.cra.figaro.algorithm.factored.factors.Factor[Double]] = null - - val showTiming: Boolean = false - - val targetElements: List[com.cra.figaro.language.Element[_]] = null - - val universe: com.cra.figaro.language.Universe = null - -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/VESolver.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/VESolver.scala deleted file mode 100644 index b84a7044..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/VESolver.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * VESolver.scala - * A variable elimination solver. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ -package com.cra.figaro.experimental.structured.solver - -import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.experimental.structured.Problem - -private[figaro] class VESolver(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]) -extends com.cra.figaro.algorithm.factored.VariableElimination[Double] { - def go(): List[Factor[Double]] = { - doElimination(factors, toPreserve.toList) - result - } - - debug = true - - val semiring: Semiring[Double] = SumProductSemiring() - - private var result: List[Factor[Double]] = _ - - def finish(factorsAfterElimination: scala.collection.mutable.Set[Factor[Double]], eliminationOrder: List[Variable[_]]): Unit = { - result = factorsAfterElimination.toList - } - - val dependentAlgorithm: (com.cra.figaro.language.Universe, List[com.cra.figaro.language.NamedEvidence[_]]) => () => Double = null - - val dependentUniverses: List[(com.cra.figaro.language.Universe, List[com.cra.figaro.language.NamedEvidence[_]])] = null - - def getFactors(neededElements: List[com.cra.figaro.language.Element[_]], - targetElements: List[com.cra.figaro.language.Element[_]], - upperBounds: Boolean): List[com.cra.figaro.algorithm.factored.factors.Factor[Double]] = null - - val showTiming: Boolean = false - - val targetElements: List[com.cra.figaro.language.Element[_]] = null - - val universe: com.cra.figaro.language.Universe = null -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/package.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/package.scala deleted file mode 100644 index 27a3b87e..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/solver/package.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * package.scala - * Definitions of solvers. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ - -package com.cra.figaro.experimental.structured - -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.algorithm.factored.factors.Variable - -package object solver { - /** - * A Solver takes a set of variables to eliminate, a set of variables to preserve, and a list of factors. - * It returns a list of factors that mention only the preserved variables. - */ - type Solver = (Problem, Set[Variable[_]], Set[Variable[_]], List[Factor[Double]]) => List[Factor[Double]] - - /** - * Creates a variable elimination solver. - * @param problem the problem to solve - * @param toEliminate the variables to be eliminated - * @param toPreserve the variables to be preserved (not eliminated) - * @param factors all the factors in the problem - */ - def variableElimination(problem: Problem, toEliminate: Set[Variable[_]], toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): List[Factor[Double]] = { - val ve = new VESolver(problem, toEliminate, toPreserve, factors) - ve.go() - } - - /** - * Creates a belief propagation solver. - * @param iterations number of iterations of BP to run - * @param problem the problem to solve - * @param toEliminate the variables to be eliminated - * @param toPreserve the variables to be preserved (not eliminated) - * @param factors all the factors in the problem - */ - def beliefPropagation(iterations: Int = 100)(problem: Problem, toEliminate: Set[Variable[_]], - toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): List[Factor[Double]] = { - val bp = new BPSolver(problem, toEliminate, toPreserve, factors, iterations) - bp.go() - } - - /** - * Creates a hybrid solver that chooses between variable elimination and belief propagation. - * @param threshold the minimum score increase on eliminating a variable that will cause the solver to choose belief propagation - * @param iterations number of iterations if belief propagation is chosen - * @param problem the problem to solve - * @param toEliminate the variables to be eliminated - * @param toPreserve the variables to be preserved (not eliminated) - * @param factors all the factors in the problem - */ - def chooseVEOrBP(threshold: Double, iterations: Int = 100)(problem: Problem, toEliminate: Set[Variable[_]], - toPreserve: Set[Variable[_]], factors: List[Factor[Double]]): List[Factor[Double]] = { - val solver = new VEBPChooser(problem, toEliminate, toPreserve, factors, threshold, iterations) - solver.go() - } -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/strategy/StructuredSolver.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/strategy/StructuredSolver.scala deleted file mode 100644 index b7857e6b..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/strategy/StructuredSolver.scala +++ /dev/null @@ -1,80 +0,0 @@ -/* - * StructuredSolver.scala - * A strategy that fully expands a problem and solves the subproblems in a structured manner. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ -package com.cra.figaro.experimental.structured.strategy - -import com.cra.figaro.experimental.structured._ -import com.cra.figaro.experimental.structured.solver.Solver -import com.cra.figaro.language.Element - -private[figaro] class StructuredSolver(problem: Problem, solver: Solver, - recursingStrategy: Strategy, rangeSizer: RangeSizer, - bounds: Bounds, parameterized: Boolean) { - def execute() { - backwardChain(problem.components, Set[ProblemComponent[_]]()) - problem.solve(solver) - } - - /* - * backwardChain takes a list of items to process. - * When the first item is taken off the list, it checks whether it has already been done. - * When an item is taken off the list, if it has not been done, all the items it depends on - * are added to the list to do. This guarantees that when an item is finally processed, - * all the items it depends on have already been processed. Also, we do not process - * any items more than once. - */ - private def backwardChain(toDo: List[ProblemComponent[_]], done: Set[ProblemComponent[_]]): Set[ProblemComponent[_]] = { - toDo match { - case first :: rest => - // globals should have been processed before this problem - if (done.contains(first) || !problem.contains(first.problem)) backwardChain(rest, done) - else { - val argComponents = first.element.args.map(checkArg(_)) - val done1 = if (argComponents.nonEmpty) backwardChain(argComponents, done) else done - first match { - case chainComp: ChainComponent[_,_] => - chainComp.expand() - chainComp.subproblems.values.foreach(recurse(_, done)) - process(chainComp) - backwardChain(rest, done1 + chainComp) - case maComp: MakeArrayComponent[_] => - maComp.expand() - val items = maComp.makeArray.items.take(maComp.maxExpanded).toList - val itemComponents = items.map(checkArg(_)) - val done2 = backwardChain(itemComponents ::: toDo, done1) - process(maComp) - backwardChain(rest, done2 + maComp) - case _ => - process(first) - backwardChain(rest, done1 + first) - } - } - case _ => done - } - } - - private def process(comp: ProblemComponent[_]) { - comp.generateRange(rangeSizer(comp)) - comp.makeNonConstraintFactors(parameterized) - comp.makeConstraintFactors(bounds) - } - - private def recurse(nestedProblem: NestedProblem[_], done: Set[ProblemComponent[_]]) { - // This check is important and is what enables us to perform dynamic programming - if (!nestedProblem.solved) recursingStrategy(nestedProblem) - } - - private def checkArg[T](element: Element[T]): ProblemComponent[T] = { - if (problem.collection.contains(element)) problem.collection(element) - else problem.add(element) - } -} diff --git a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/strategy/package.scala b/Figaro/src/main/scala/com/cra/figaro/experimental/structured/strategy/package.scala deleted file mode 100644 index 5187b2c7..00000000 --- a/Figaro/src/main/scala/com/cra/figaro/experimental/structured/strategy/package.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * package.scala - * Definitions of expansion and solution strategies. - * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 - * - * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. - * See http://www.cra.com or email figaro@cra.com for information. - * - * See http://www.github.com/p2t2/figaro for a copy of the software license. - */ -package com.cra.figaro.experimental.structured - -import com.cra.figaro.experimental.structured.solver.Solver - -package object strategy { - /** - * A strategy takes a problem and does something useful, such as computing a solution. - * Type-wise, a strategy simply does something with the problem. - */ - type Strategy = Problem => Unit - - /** - * Create a structured solution strategy. This strategy will recursively solve the subproblems before solving the top level problem - * with a given solver. - * @param solver the solver to use for the top level problem - * @param recursingStrategy strategy for recursively solving the subproblems - * @param rangeSizer strategy for choosing the size of the range of a component associated with an atomic element - * @param bounds lower or upper bounds - * @param parameterized flag indicating whether parameterized elements should use their MAP value or be treated as ordinary elements - */ - def structuredSolver(solver: Solver, recursingStrategy: Strategy, - rangeSizer: RangeSizer = defaultRangeSizer, bounds: Bounds = Lower, parameterized: Boolean = false): Strategy = - (problem: Problem) => { - (new StructuredSolver(problem, solver, recursingStrategy, rangeSizer, bounds, parameterized)).execute() - } - - /** - * Create a structured solution strategy in which the selfsame strategy is used to recursively solve subproblems - * @param solver the solver to use for every subproblem - * @param rangeSizer strategy for choosing the size of the range of a component associated with an atomic element - * @param bounds lower or upper bounds - * @param parameterized flag indicating whether parameterized elements should use their MAP value or be treated as ordinary elements - */ - def recursiveSolver(solver: Solver, rangeSizer: RangeSizer = defaultRangeSizer, bounds: Bounds = Lower, parameterized: Boolean = false): Strategy = - (problem: Problem) => { - def recurse(problem: Problem) = recursiveSolver(solver, rangeSizer, bounds, parameterized)(problem) - (new StructuredSolver(problem, solver, recurse, rangeSizer, bounds, parameterized)).execute() - } - - /** - * A range sizer chooses a size of range for components corresponding to atomic elements. - */ - type RangeSizer = ProblemComponent[_] => Int - - def defaultRangeSizer(pc: ProblemComponent[_]) = Int.MaxValue -} diff --git a/Figaro/src/main/scala/com/cra/figaro/language/ApplyC.scala b/Figaro/src/main/scala/com/cra/figaro/language/ApplyC.scala new file mode 100644 index 00000000..05e6095e --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/language/ApplyC.scala @@ -0,0 +1,41 @@ +package com.cra.figaro.language + +/** + * This class is a workaround for adding easier type inference to Apply class + * Since Apply object is already using apply method with different number of arguments + * it is not easy to find a workaround adding currying support to apply methods in Apply class + */ +object ApplyC { + /** + * Application of a function to one argument. + */ + def apply[T1, U](arg1: Element[T1])(fn: T1 => U)(implicit name: Name[U], collection: ElementCollection) = + new Apply1(name, arg1, fn, collection) + + /** + * Application of a function to two arguments. + */ + def apply[T1, T2, U](arg1: Element[T1], arg2: Element[T2])(fn: (T1, T2) => U)(implicit name: Name[U], collection: ElementCollection) = + new Apply2(name, arg1, arg2, fn, collection) + + /** + * Application of a function to three arguments. + */ + def apply[T1, T2, T3, U](arg1: Element[T1], arg2: Element[T2], arg3: Element[T3])(fn: (T1, T2, T3) => U)(implicit name: Name[U], collection: ElementCollection) = + new Apply3(name, arg1, arg2, arg3, fn, collection) + + /** + * Application of a function to four arguments. + */ + def apply[T1, T2, T3, T4, U](arg1: Element[T1], arg2: Element[T2], arg3: Element[T3], arg4: Element[T4]) + (fn: (T1, T2, T3, T4) => U)(implicit name: Name[U], collection: ElementCollection) = + new Apply4(name, arg1, arg2, arg3, arg4, fn, collection) + + /** + * Application of a function to five arguments. + */ + def apply[T1, T2, T3, T4, T5, U](arg1: Element[T1], arg2: Element[T2], arg3: Element[T3], arg4: Element[T4], + arg5: Element[T5]) + (fn: (T1, T2, T3, T4, T5) => U)(implicit name: Name[U], collection: ElementCollection) = + new Apply5(name, arg1, arg2, arg3, arg4, arg5, fn, collection) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/language/Chain.scala b/Figaro/src/main/scala/com/cra/figaro/language/Chain.scala index d920cc9c..a865dadc 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/Chain.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/Chain.scala @@ -22,20 +22,19 @@ import scala.collection.mutable.Set * A Chain(parent, fcn) represents the process that first generates a value for the parent, then * applies fcn to get a new Element, and finally generates a value from that new Element. * - * Chain is the common base class for caching and non-caching chains. - * All chains have a cache, whose size is specified by the cacheSize argument. - * When a parent value is encountered, first the cache is checked to see if the result element is known. - * If it is not, the resulting element is generated from scratch by calling fcn. + * Chain is the common base class for caching and non-caching chains. There is no functional difference + * between caching and non-caching chains. Algorithms can use the distinction to implement a caching procedure. * * @param parent The parent element of the chain */ -class Chain[T, U](name: Name[U], val parent: Element[T], fcn: T => Element[U], cacheSize: Int, collection: ElementCollection) +class Chain[T, U](name: Name[U], val parent: Element[T], fcn: T => Element[U], collection: ElementCollection) extends Deterministic[U](name, collection) { - def args: List[Element[_]] = if (active && resultElement != null) List(parent) ::: List(resultElement) else List(parent) + + def args: List[Element[_]] = List(parent) protected def cpd = fcn - + private[figaro] val chainFunction = fcn /** @@ -43,103 +42,23 @@ class Chain[T, U](name: Name[U], val parent: Element[T], fcn: T => Element[U], c */ type ParentType = T - /** - * The current result element that arises from the current value of the parent. - */ - var resultElement: Element[U] = _ - - /* Data structures for the Chain. The cache stores previously generated result elements. The Context data - * structures store the elements that were created in this context. We also stored newly created elements - * in a subContext, which is based on the value of the parent. - * Because Elements might be stored in sets or maps by algorithms, we need a way to allow the elements to be removed - * from the set or map so they can be garbage collected. Universe provides a way to achieve this through the use of - * contexts. When Chain gets the distribution over the child, it first pushes the context, and pops the context afterward, to mark - * any generated elements as being generated in the context of this Chain. - */ - lazy private[figaro] val cache: Map[T, Element[U]] = Map() - lazy private[figaro] val myMappedContextContents: Map[T, Set[Element[_]]] = Map() - lazy private[figaro] val elemInContext: Map[Element[_], T] = Map() - - private var lastParentValue: T = _ - - /* Must override clear temporary for Chains. We can never leave the chain in an uninitialized state. That is, - * the chain MUST ALWAYS have a valid element to return. So when clearing temporaries we clear everything - * except the current context. - */ - override def clearContext() = { - myMappedContextContents.keys.foreach(c => if (c != lastParentValue) resizeCache(c)) - } - - /* Override context control for chain data structures */ - override def directContextContents: Set[Element[_]] = if (!active) { - throw new NoSuchElementException - } else Set(myMappedContextContents.values.flatten.toList: _*) - - override private[figaro] def addContextContents(e: Element[_]) = { - myMappedContextContents(lastParentValue) += e - elemInContext += (e -> lastParentValue) - } - - override private[figaro] def removeContextContents(e: Element[_]) = { - myMappedContextContents(elemInContext(e)) -= e - elemInContext -= e - } - def generateValue() = { if (parent.value == null) parent.generate() - val resultElem = get(parent.value) - if (resultElem.value == null) resultElem.generate() - resultElem.value + val resultElement = get(parent.value) + if (resultElement.value == null) resultElement.generate() + resultElement.value } - /* Computes the new result. If the cache contains a VALID element for this parent value, then return the - * the element. Otherwise, we need to create one. First, we create an entry in the ContextContents since - * any elements created in this context will be stored in the subContext of parentValue. Then, if the cache - * is full, we resize the cache to make room for a new result element. Apply the function of the chain, and - * stored the new result in the cache. If the element created is a new element, then registers it uses - * in the Universe. - */ - /** - * Get the distribution over the result corresponding to the given parent value. Takes care of all bookkeeping including caching. + * Get the distribution over the result corresponding to the given parent value. */ def get(parentValue: T): Element[U] = { - val lruParent = lastParentValue - lastParentValue = parentValue - val cacheValue = cache.get(parentValue) - - val newResult = - if (!cacheValue.isEmpty && cacheValue.get.active) cacheValue.get - else { - myMappedContextContents += (parentValue -> Set()) - if (cache.size >= cacheSize && cacheValue.isEmpty) { - val dropValue = if (cache.last._1 != lruParent) cache.last._1 else cache.head._1 - resizeCache(dropValue) - } - val result = getResult(parentValue) - cache += (parentValue -> result) - universe.registerUses(this, result) - result - } - - resultElement = newResult - newResult + val result = getResult(parentValue) + universe.registerUses(this, result) + result } - /** - * Get the distribution over the result corresponding to the given parent value. This call is UNCACHED, - * meaning it will not be stored in the Chain's cache, and subsequent calls using the same parentValue - * could return different elements. - */ - def getUncached(parentValue: T): Element[U] = { - if (lastParentValue == null || lastParentValue != parentValue) { - myMappedContextContents.getOrElseUpdate(parentValue, Set()) - lastParentValue = parentValue - resultElement = getResult(parentValue) - universe.registerUses(this, resultElement) - } - resultElement - } + private[figaro] def getUncached(parentValue: T): Element[U] = get(parentValue) // All elements created in cpd will be created in this Chain's context with a subContext of parentValue private def getResult(parentValue: T): Element[U] = { @@ -148,19 +67,7 @@ class Chain[T, U](name: Name[U], val parent: Element[T], fcn: T => Element[U], c universe.popContext(this) result } - - /* Current replacement scheme just drops the last element in the cache. The dropped element must be deactivated, - * and removed from the context data structures. - */ - protected def resizeCache(dropValue: T) = { - cache -= dropValue - if (myMappedContextContents.contains(dropValue)) { - universe.deactivate(myMappedContextContents(dropValue)) - elemInContext --= myMappedContextContents(dropValue) - myMappedContextContents -= dropValue - } - } - + override def toString = "Chain(" + parent + ", " + cpd + ")" } @@ -168,13 +75,13 @@ class Chain[T, U](name: Name[U], val parent: Element[T], fcn: T => Element[U], c * A NonCachingChain is an implementation of Chain with a single element cache. */ class NonCachingChain[T, U](name: Name[U], parent: Element[T], cpd: T => Element[U], collection: ElementCollection) - extends Chain(name, parent, cpd, 2, collection) + extends Chain(name, parent, cpd, collection) /** * A CachingChain is an implementation of Chain with a 1000 element cache. */ class CachingChain[T, U](name: Name[U], parent: Element[T], cpd: T => Element[U], collection: ElementCollection) - extends Chain(name, parent, cpd, Int.MaxValue, collection) + extends Chain(name, parent, cpd, collection) object NonCachingChain { /** Create a NonCaching chain of 1 argument. */ diff --git a/Figaro/src/main/scala/com/cra/figaro/language/Element.scala b/Figaro/src/main/scala/com/cra/figaro/language/Element.scala index 0daed4d9..1eaef441 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/Element.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/Element.scala @@ -17,6 +17,8 @@ import com.cra.figaro.library.compound._ import scala.collection.mutable.Set import scala.language.implicitConversions +class EvidenceNotAllowedException(element: Element[_]) extends RuntimeException("Evidence not allowed on " + element.toString + " because it is a temporary element") + /** * An Element is the core component of a probabilistic model. Elements can be understood as * defining a probabilistic process. Elements are parameterized by the type of Value the process @@ -163,9 +165,9 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) throw new NoSuchElementException } else myDirectContextContents - private[language] def addContextContents(e: Element[_]): Unit = myDirectContextContents += e + private[figaro] def addContextContents(e: Element[_]): Unit = myDirectContextContents += e - private[language] def removeContextContents(e: Element[_]): Unit = myDirectContextContents -= e + private[figaro] def removeContextContents(e: Element[_]): Unit = myDirectContextContents -= e /** * Returns true if this element is temporary, that is, was created in the context of another element. @@ -222,7 +224,9 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) universe.registerUses(this, elem) } - private var myConditions: List[(Condition, Contingency)] = List() + private def checkIfEvidenceAllowed = if (isTemporary) throw new EvidenceNotAllowedException(this) + + private[language] var myConditions: List[(Condition, Contingency)] = List() /** All the conditions defined on this element.*/ def allConditions = myConditions @@ -237,7 +241,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * If an element has any other condition besides this observation, we cannot use the * observation. However, it can have a constraint. */ - private[figaro] var observation: Option[T] = None + private[figaro] var observation: Option[Value] = None /* * Testing whether a condition is satisfied can use any type of value. The condition can only be satisfied if the value has the right type and the condition returns true. @@ -268,6 +272,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) /** Add the given condition to the existing conditions of the element. By default, the contingency is empty. */ def addCondition(condition: Condition, contingency: Contingency = List()): Unit = { + checkIfEvidenceAllowed universe.makeConditioned(this) contingency.foreach(ev => ensureContingency(ev.elem)) observation = None @@ -287,11 +292,12 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * Set the condition associated with the contingency. Removes previous conditions associated with the contingency. By default, the contingency is empty. */ def setCondition(newCondition: Condition, contingency: Contingency = List()): Unit = { + checkIfEvidenceAllowed removeConditions(contingency) addCondition(newCondition, contingency) } - private var myConstraints: List[(Constraint, Contingency)] = List() + private[language] var myConstraints: List[(Constraint, Contingency)] = List() /** * The current soft constraints on the element. @@ -349,6 +355,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * Add a contingent constraint to the element. By default, the contingency is empty. */ def addConstraint(constraint: Constraint, contingency: Contingency = List()): Unit = { + checkIfEvidenceAllowed universe.makeConstrained(this) contingency.foreach(ev => ensureContingency(ev.elem)) myConstraints ::= (ProbConstraintType(constraint), contingency) @@ -358,6 +365,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * Add a log contingent constraint to the element. By default, the contingency is empty. */ def addLogConstraint(constraint: Constraint, contingency: Contingency = List()): Unit = { + checkIfEvidenceAllowed universe.makeConstrained(this) contingency.foreach(ev => ensureContingency(ev.elem)) myConstraints ::= (LogConstraintType(constraint), contingency) @@ -381,6 +389,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * Set the constraint associated with the contingency. Removes previous constraints associated with the contingency. By default, the contingency is empty. */ def setConstraint(newConstraint: Constraint, contingency: Contingency = List()): Unit = { + checkIfEvidenceAllowed removeConstraints(contingency) addConstraint(newConstraint, contingency) } @@ -389,6 +398,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * Set the log constraint associated with the contingency. Removes previous constraints associated with the contingency. By default, the contingency is empty. */ def setLogConstraint(newConstraint: Constraint, contingency: Contingency = List()): Unit = { + checkIfEvidenceAllowed removeConstraints(contingency) addLogConstraint(newConstraint, contingency) } @@ -397,6 +407,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) * Condition the element by observing a particular value. Propagates the effect to dependent elements and ensures that no other value for the element can be generated. */ def observe(observation: Value): Unit = { + checkIfEvidenceAllowed removeConditions() set(observation) universe.makeConditioned(this) @@ -412,7 +423,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) removeConditions() } - private var setFlag: Boolean = false + private[language] var setFlag: Boolean = false /** * Allows different values of the element to be generated. @@ -448,7 +459,7 @@ abstract class Element[T](val name: Name[T], val collection: ElementCollection) set(generateValue(randomness)) } - private var myPragmas: List[Pragma[Value]] = List() + private[language] var myPragmas: List[Pragma[Value]] = List() /** * The pragmas attached to the element. diff --git a/Figaro/src/main/scala/com/cra/figaro/language/Evidence.scala b/Figaro/src/main/scala/com/cra/figaro/language/Evidence.scala index 18d2729c..0c0439d4 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/Evidence.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/Evidence.scala @@ -60,7 +60,10 @@ case class LogConstraint[T](function: T => Double) extends Evidence[T] { } /** - * Evidence representing observing a particular value for the element. + * Evidence representing observing a particular value for the element. Note that using an Observation + * on a reference is not the same as calling observe on the element directly. This version applies + * a condition to the element, thus bypassing some specialized operations that can be accomplished + * by calling the observe on the element directly (such as Likelihood weighting). * * @param value The observed value of the element. */ diff --git a/Figaro/src/main/scala/com/cra/figaro/language/HasDensity.scala b/Figaro/src/main/scala/com/cra/figaro/language/HasDensity.scala index 65a7f4bd..26d99d7e 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/HasDensity.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/HasDensity.scala @@ -1,6 +1,6 @@ /* * HasDensity.scala - * Trait for TBD + * Trait elements that have a density defined. * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jun 10, 2014 @@ -19,4 +19,35 @@ package com.cra.figaro.language trait HasDensity[T] extends Element[T] { /** The probability density of a value. */ def density(t: T): Double + + /** + * Generate the next randomness given the current randomness. + * Returns three values: The next randomness, the Metropolis-Hastings proposal probability + * ratio, which is: + * + * P(new -> old) / P(old -> new) + * + * and the model probability ratio, which is: + * + * P(new) / P(old) + * + * This implementation produces a sample using generateRandomness, which means that: + * + * P(new -> old) / P(old -> new) = P(old) / P(new) + * + * We use the fact that this element can compute densities for values to compute P(new) and + * P(old) explicitly. Note that the two returned ratios will still multiply to 1. This does + * not affect normal Metropolis-Hastings, but helps the Metropolis-Hastings annealer find + * maxima. + */ + override def nextRandomness(oldRandomness: Randomness): (Randomness, Double, Double) = { + val newRandomness = generateRandomness() + val pOld = density(generateValue(oldRandomness)) + val pNew = density(generateValue(newRandomness)) + // Note that these density ratios could overflow/underflow, particularly if there is nonzero probability of + // generating a value that has infinite density. For example, this may occur when generating from a Gamma + // distribution with sufficiently small shape parameter such that the value 0 may be produced, which has infinite + // density. We don't account for this explicitly because MH is unlikely to work well for these models to begin with. + (newRandomness, pOld / pNew, pNew / pOld) + } } diff --git a/Figaro/src/main/scala/com/cra/figaro/language/Inject.scala b/Figaro/src/main/scala/com/cra/figaro/language/Inject.scala index 597ad201..a1b00e9b 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/Inject.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/Inject.scala @@ -30,7 +30,10 @@ class Inject[T](name: Name[List[T]], val xs: Seq[Element[T]], collection: Elemen def args = xs.toList - def generateValue() = (xs map (_.value)).toList + def generateValue() = { + xs foreach (x => if (x.value == null) x.generate()) + (xs map (_.value)).toList + } override def toString = "Inject(" + xs.mkString(", ") + ")" } diff --git a/Figaro/src/main/scala/com/cra/figaro/language/ReferenceElement.scala b/Figaro/src/main/scala/com/cra/figaro/language/ReferenceElement.scala index 47352975..923ef89e 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/ReferenceElement.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/ReferenceElement.scala @@ -17,7 +17,7 @@ import com.cra.figaro.algorithm.ValuesMaker import com.cra.figaro.algorithm.lazyfactored._ import ValueSet._ import com.cra.figaro.algorithm.factored.factors._ -import com.cra.figaro.algorithm.factored.factors.Factory +import com.cra.figaro.algorithm.factored.factors.factory.Factory import com.cra.figaro.util._ import scala.collection.mutable.Map import scala.language.existentials diff --git a/Figaro/src/main/scala/com/cra/figaro/language/Universe.scala b/Figaro/src/main/scala/com/cra/figaro/language/Universe.scala index f86cec35..d3cda666 100644 --- a/Figaro/src/main/scala/com/cra/figaro/language/Universe.scala +++ b/Figaro/src/main/scala/com/cra/figaro/language/Universe.scala @@ -36,7 +36,7 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol */ override val universe = this - private val myActiveElements: Set[Element[_]] = Set() + private[language] val myActiveElements: Set[Element[_]] = Set() /** * The active elements in the universe. @@ -46,7 +46,7 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol /** Elements in the universe that are not defined in the context of another element. */ def permanentElements: List[Element[_]] = myActiveElements.toList filterNot (_.isTemporary) - private val myConditionedElements: Set[Element[_]] = Set() + private[language] val myConditionedElements: Set[Element[_]] = Set() /** Elements in the universe that have had a condition applied to them. */ def conditionedElements: List[Element[_]] = myConditionedElements.toList @@ -55,7 +55,7 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol private[language] def makeUnconditioned(elem: Element[_]) { myConditionedElements -= elem } - private val myConstrainedElements: Set[Element[_]] = Set() + private[language] val myConstrainedElements: Set[Element[_]] = Set() /** Elements in the universe that have had a constraint applied to them. */ def constrainedElements: List[Element[_]] = myConstrainedElements.toList @@ -64,7 +64,7 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol private[language] def makeUnconstrained(elem: Element[_]) { myConstrainedElements -= elem } - private val myStochasticElements = new HashSelectableSet[Element[_]] + private[language] val myStochasticElements = new HashSelectableSet[Element[_]] /** * The active non-deterministic elements in the universe. @@ -89,7 +89,7 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol * any Chain. Conversely, if it is not empty, it must have been created within a chain, so it is temporary. It is * possible to remove all temporary Elements from a Map. */ - private var myContextStack: List[Element[_]] = List() + private[language] var myContextStack: List[Element[_]] = List() private[figaro] def contextStack = myContextStack @@ -118,12 +118,12 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol myContextStack = myContextStack dropWhile (_ != element) drop 1 } - private val myUses: Map[Element[_], Set[Element[_]]] = Map() + private[language] val myUses: Map[Element[_], Set[Element[_]]] = Map() - private val myUsedBy: Map[Element[_], Set[Element[_]]] = Map() + private[language] val myUsedBy: Map[Element[_], Set[Element[_]]] = Map() - private val myRecursiveUsedBy: Map[Element[_], Set[Element[_]]] = Map() - private val myRecursiveUses: Map[Element[_], Set[Element[_]]] = Map() + private[language] val myRecursiveUsedBy: Map[Element[_], Set[Element[_]]] = Map() + private[language] val myRecursiveUses: Map[Element[_], Set[Element[_]]] = Map() /** * Returns the set of elements that the given element uses in its generation, either directly or recursively. @@ -143,7 +143,7 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol } /** - * Returns the set of elements that are directly used by the given element, without recursing. + * Returns the set of elements that use the given element in their generation, without recursing. */ def directlyUsedBy(elem: Element[_]): Set[Element[_]] = myUsedBy.getOrElse(elem, Set()) @@ -170,8 +170,6 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol private[language] def activate(element: Element[_]): Unit = { if (element.active) throw new IllegalArgumentException("Activating active element") -// if (element.args exists (!_.active)) -// throw new IllegalArgumentException("Attempting to activate element with inactive argument") element.args.filter(!_.active).foreach(activate(_)) myActiveElements.add(element) if (!element.isInstanceOf[Deterministic[_]]) myStochasticElements.add(element) @@ -182,8 +180,6 @@ class Universe(val parentElements: List[Element[_]] = List()) extends ElementCol } element.args foreach (registerUses(element, _)) element.active = true -// myRecursiveUsedBy.clear -// myRecursiveUses.clear } private[language] def deactivate(element: Element[_]): Unit = { diff --git a/Figaro/src/main/scala/com/cra/figaro/language/UniverseState.scala b/Figaro/src/main/scala/com/cra/figaro/language/UniverseState.scala new file mode 100644 index 00000000..67ffd078 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/language/UniverseState.scala @@ -0,0 +1,166 @@ +/* + * UniverseState.scala + * Saving and restoring universe state. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 3, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.language + +import scala.collection.mutable + +/** + * Saves the mutable state of a universe. This is useful for algorithms that need to maintain a state over elements, but + * involve a sub-procedure that mutates the state of the universe. This class provides the functionality for restoring + * the previous state. Immutable. + * + * @param universe Universe to save. Information about the current state of this universe is copied in the constructor. + */ +class UniverseState(universe: Universe) { + // Immutable types + val myContextStack = universe.myContextStack + + // Mutable types, stored as immutable types + val myActiveElements = universe.myActiveElements.toSet + val myStochasticElements = universe.myStochasticElements.toSet + + val myConditionedElements = universe.myConditionedElements.toSet + val myConstrainedElements = universe.myConstrainedElements.toSet + + val myUses = makeImmutable(universe.myUses) + val myUsedBy = makeImmutable(universe.myUsedBy) + val myRecursiveUses = makeImmutable(universe.myRecursiveUses) + val myRecursiveUsedBy = makeImmutable(universe.myRecursiveUsedBy) + + // States over elements + val elementStates: Map[Element[_], ElementState] = myActiveElements.map(e => (e, new ElementState(e))).toMap + + /** + * Restores the universe to its state at the time of construction of this class. In general, this means that any calls + * to the public API in `Universe` and `Element` will behave as if they were called when this class was instantiated, + * regardless of what calls to the API were made since instantiation. + * + * There are subtle exceptions to this rule. In particular, registered universe maps, element maps, and universe maps + * will not be changed. Additionally, `myElementMap` from `ElementCollection` will not be changed. + * + * Note that classes that extend functionality of `Element` and `Universe` with additional mutability may have + * undefined behavior with respect to this mutable information. For example, parameters that store learned values will + * not have their learned values copied. + * + * Since this class is immutable, this method can be called multiple times to repeatedly restore the same state after + * subsequent changes to the universe are made. + */ + def restore(): Unit = { + // Deactivate any newly created elements + for(newElement <- universe.myActiveElements -- myActiveElements) newElement.deactivate() + + // For immutable types, we can just replace the references + universe.myContextStack = myContextStack + + // For mutable types, we get the reference to the set and replace its contents + replace(universe.myActiveElements, myActiveElements) + replace(universe.myStochasticElements, myStochasticElements) + replace(universe.myConditionedElements, myConditionedElements) + replace(universe.myConstrainedElements, myConstrainedElements) + + replace(universe.myUses, myUses) + replace(universe.myUsedBy, myUsedBy) + replace(universe.myRecursiveUses, myRecursiveUses) + replace(universe.myRecursiveUsedBy, myRecursiveUsedBy) + + // Element states update on their own + elementStates.values.foreach(_.restore()) + } + + /** + * Replace the contents of a mutable set with the contents of an immutable set. + * @param toReplace Set whose elements should be replaced. + * @param replaceWith Set of elements to replace with. + */ + private def replace[T](toReplace: mutable.Set[T], replaceWith: TraversableOnce[T]): Unit = { + toReplace.clear() + toReplace ++= replaceWith + } + + /** + * Replace the contents of a mutable map to mutable sets with the contents of an immutable map to immutable sets. + * @param toReplace Map whose elements should be replaced. + * @param replaceWith Map of elements to replace with. + */ + private def replace[T](toReplace: mutable.Map[T, mutable.Set[T]], replaceWith: Map[T, Set[T]]): Unit = { + toReplace.clear() + for((key, value) <- replaceWith) { + toReplace(key) = mutable.Set() ++ value + } + } + + private def makeImmutable[T](map: mutable.Map[T, mutable.Set[T]]): Map[T, Set[T]] = { + // map.toMap.mapValues(_.toSet) + // Wait! Calling mapValues returns a VIEW into the original map, which we don't want because the underlying sets are + // mutable. See Scala issues SI-4776. + map.map{ case (key, value) => (key, value.toSet) }.toMap + } +} + +/** + * Saves the mutable state of a single element. This only saves and restores the mutable information found in the + * `Element` class; it does not propagate this information to the universe level. Immutable. + * + * @param element Element to save. Information about the current state of this element is copied in the constructor. + */ +class ElementState(val element: Element[_]) { + // Immutable types + val active = element.active + val setFlag = element.setFlag + + val value: element.Value = element.value + val randomness: element.Randomness = element.randomness + + val myPragmas: List[Pragma[element.Value]] = element.myPragmas + + val myConditions = element.myConditions + val myConstraints = element.myConstraints + val observation: Option[element.Value] = element.observation + + val myContext = element.myContext + + // Mutable types, stored as immutable types + val myDirectContextContents = element.directContextContents.toSet + + /** + * Restores the element to its state at the time of construction of this class. + * + * This only restores mutable information found in the `Element` class; any mutable information added by subclasses + * will be ignored. Furthermore, information about this element is not propagated to the universe level. + * + * Since this class is immutable, this method can be called multiple times to repeatedly restore the same state after + * subsequent changes to the element are made. + */ + def restore(): Unit = { + // For immutable types, we can just replace the references + element.active = active + element.setFlag = setFlag + + element.value = value + element.randomness = randomness + + element.myPragmas = myPragmas + + element.myConditions = myConditions + element.myConstraints = myConstraints + element.observation = observation + + element.myContext = myContext + + // For mutable types, we get the reference to the set and replace its contents + val referenceToMyDirectContextContents = element.directContextContents + referenceToMyDirectContextContents.clear() + referenceToMyDirectContextContents ++= myDirectContextContents + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Beta.scala b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Beta.scala index 430a1d92..4d804ef9 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Beta.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Beta.scala @@ -29,7 +29,7 @@ import argonaut._, Argonaut._ * @param b The prior beta parameter */ class AtomicBeta(name: Name[Double], a: Double, b: Double, collection: ElementCollection) - extends Element[Double](name, collection) with Atomic[Double] with DoubleParameter with ValuesMaker[Double] with Beta{ + extends Element[Double](name, collection) with Atomic[Double] with DoubleParameter with Beta{ type Randomness = Double /** @@ -97,6 +97,8 @@ class AtomicBeta(name: Name[Double], a: Double, b: Double, collection: ElementCo else (learnedAlpha - 1) / (learnedAlpha + learnedBeta - 2) } + // Values for Beta parameters now handled directly in the algorithms + @deprecated("Values for Beta parameters are now handled directly in the algorithms", "4.1.0") def makeValues(depth: Int) = ValueSet.withoutStar(Set(MAPValue)) def maximize(sufficientStatistics: Seq[Double]) { diff --git a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Dirichlet.scala b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Dirichlet.scala index 555a2a30..37937b8f 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Dirichlet.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Dirichlet.scala @@ -32,7 +32,7 @@ import argonaut.Argonaut._ * @param alphas the prior concentration parameters */ class AtomicDirichlet(name: Name[Array[Double]], val alphas: Array[Double], collection: ElementCollection) - extends Element[Array[Double]](name, collection) with Atomic[Array[Double]] with ArrayParameter with ValuesMaker[Array[Double]] with Dirichlet{ + extends Element[Array[Double]](name, collection) with Atomic[Array[Double]] with ArrayParameter with Dirichlet { /** * The number of concentration parameters in the Dirichlet distribution. @@ -134,6 +134,8 @@ class AtomicDirichlet(name: Name[Array[Double]], val alphas: Array[Double], coll result } + // Values for Beta parameters now handled directly in the algorithms + @deprecated("Values for Beta parameters are now handled directly in the algorithms", "4.1.0") def makeValues(depth: Int) = ValueSet.withoutStar(Set(MAPValue)) override def toString = "Dirichlet(" + alphas.mkString(", ") + ")" @@ -143,12 +145,12 @@ class AtomicDirichlet(name: Name[Array[Double]], val alphas: Array[Double], coll * Dirichlet distributions in which the parameters are elements. */ class CompoundDirichlet(name: Name[Array[Double]], alphas: Array[Element[Double]], collection: ElementCollection) - extends NonCachingChain[List[Double], Array[Double]]( - name, - new Inject("", alphas, collection), - (aa: Seq[Double]) => new AtomicDirichlet("", aa.toArray, collection), - collection) - with Dirichlet { + extends NonCachingChain[List[Double], Array[Double]]( + name, + new Inject("", alphas, collection), + (aa: Seq[Double]) => new AtomicDirichlet("", aa.toArray, collection), + collection) + with Dirichlet { def alphaValues = alphas.map(_.value) @@ -184,7 +186,7 @@ object Dirichlet extends Creatable { //Needs to be a nested field or a jEmptyArray implicit def DirichletEncodeJson: EncodeJson[Dirichlet] = EncodeJson((d: Dirichlet) => - ("name" := d.name.string) ->: ("alphaValues" := jArray((for (a <- d.alphaValues) yield { jNumber(a) }).toList)) ->: jEmptyObject) + ("name" := d.name.string) ->: ("alphaValues" := jArray((for (a <- d.alphaValues) yield { jNumber(a) }).toList)) ->: jEmptyObject) implicit def DirichletDecodeJson(implicit collection: ElementCollection): DecodeJson[AtomicDirichlet] = DecodeJson(c => for { @@ -197,17 +199,15 @@ object Dirichlet extends Creatable { */ def apply(alphas: Double*)(implicit name: Name[Array[Double]], collection: ElementCollection) = new AtomicDirichlet(name, alphas.toArray, collection) - + def apply(alphas: Array[Double])(implicit name: Name[Array[Double]], collection: ElementCollection) = new AtomicDirichlet(name, alphas, collection) - + /** * Create a Dirichlet distribution in which the parameters are elements. */ def apply(alphas: Element[Double]*)(implicit name: Name[Array[Double]], collection: ElementCollection) = new CompoundDirichlet(name, alphas.toArray, collection) - - type ResultType = Array[Double] diff --git a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Exponential.scala b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Exponential.scala index 553b73a2..dac59eb5 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Exponential.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/Exponential.scala @@ -20,7 +20,7 @@ import scala.math.{ log, exp } /** * An exponential distribution in which the parameter is a constant. */ -class AtomicExponential(name: Name[Double], lambda: Double, collection: ElementCollection) +class AtomicExponential(name: Name[Double], val lambda: Double, collection: ElementCollection) extends Element[Double](name, collection) with Atomic[Double] { type Randomness = Double diff --git a/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/KernelDensity.scala b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/KernelDensity.scala new file mode 100644 index 00000000..abad1283 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/library/atomic/continuous/KernelDensity.scala @@ -0,0 +1,63 @@ +/* + * KernelDensity.scala + * Element representing a kernel density estimate + * + * Created By: Dan Garant (dgarant@cra.com) + * Creation Date: May 27, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.library.atomic.continuous + +import com.cra.figaro.language._ + +/** + * Performs density estimation using a Gaussian kernel to smooth density estimates around observed inputs. + * @param samples Observed points + * @param bandwidth Parameter of the Gaussian kernel + */ +class KernelDensity(name: Name[Double], val samples: Seq[Double], val bandwidth: Double, collection: ElementCollection) + extends Element[Double](name, collection) with Atomic[Double] { + + // this represents the Gaussian kernel centered at one of the input points + val normalElt = Normal(0, bandwidth) + + // randomness is a random index along with a random element from normalElt + type Randomness = (Int, normalElt.Randomness) + + /** Generates a random sample index and offset */ + def generateRandomness:Randomness = { + val idx = com.cra.figaro.util.random.nextInt(samples.length) + val rand = normalElt.generateRandomness() + (idx, rand) + } + + /** Generates a random value from the KD distribution */ + def generateValue(rand:Randomness):Double = { + val shift = normalElt.generateValue(rand._2) + return samples(rand._1) + shift + } + + /** Computes the density of a new point */ + def density(point:Double):Double = { + val densities = samples.map(s => { + normalElt.density(s - point) + }) + + densities.sum / densities.length + } + + override def toString = "KernelDensity(bandwidth=" + this.bandwidth + ")" +} + +object KernelDensity { + /** + * Create a kernel density estimator with specified bandwidth + */ + def apply(samples: Seq[Double], bandwidth: Double)(implicit name: Name[Double], collection: ElementCollection) = + new KernelDensity(name, samples, bandwidth, collection) +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/library/cache/Cache.scala b/Figaro/src/main/scala/com/cra/figaro/library/cache/Cache.scala new file mode 100644 index 00000000..e48b8728 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/library/cache/Cache.scala @@ -0,0 +1,52 @@ +/* + * Cache.scala + * Abstract class to manage caching of element generation for a universe. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.library.cache + +import com.cra.figaro.language._ +import scala.collection.generic.Shrinkable + +/** + * Abstract class to manage caching of element generation for a universe. This class can be used + * by algorithms to manage caching of chains. + */ +abstract class Cache(universe: Universe) extends Shrinkable[Element[_]] { + + /** + * Return the next element from the generative process defined by element. If no process + * is found, return None + */ + def apply[T](element: Element[T]): Option[Element[T]] + + universe.register(this) + + /** + * Clear any caching + */ + def clear(): Unit + +} + +/** A Cache class which performs no caching */ +class NoCache(universe: Universe) extends Cache(universe) { + def apply[T](element: Element[T]): Option[Element[T]] = { + element match { + case c: Chain[_,T] => Some(c.get(c.parent.value)) + case _ => None + + } + } + def clear() = {} + def -=(element: Element[_]) = this +} + + diff --git a/Figaro/src/main/scala/com/cra/figaro/library/cache/MHCache.scala b/Figaro/src/main/scala/com/cra/figaro/library/cache/MHCache.scala new file mode 100644 index 00000000..16889b2b --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/library/cache/MHCache.scala @@ -0,0 +1,152 @@ +/* + * MHCache.scala + * Implements caching for caching and non-caching chains, specifically designed for MH. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.library.cache + +import com.cra.figaro.language._ +import scala.collection.mutable.Map +import scala.collection.mutable.Set + +/** + * A class which implements caching for caching and non-caching chains, specifically designed for MH + * + * For caching chains, the result of the Chain's function is cached for each value of the parent element + * that is queried. This cache is infinitely large. + * + * For non-caching chains, we only "cache" two resulting elements of the chain. The cache is actually + * a 2-element stack, where the top of the stack represents the most recent element for the chain, and the + * bottom of the stack represents the last element (and parent value) used. This is primarily to benefit + * MH; if a proposal is rejected, we want to switch a chain back to where it was without much overhead. + * + */ +class MHCache(universe: Universe) extends Cache(universe) { + + /* Caching chain cache that maps from an element to a map of parent values and resulting elements */ + private[figaro] val ccCache: Map[Element[_], Map[Any, Element[_]]] = Map() + + /* The inverted cache. This maps from result elements back to the chain that uses them. This is needed + * to properly clean up deactivated elements + */ + private[figaro] val ccInvertedCache: Map[Element[_], Map[Element[_], Any]] = Map() + + /* + * The non-caching chain "cache". This is a map from elements to a list of: + * (parent value, result element, Set of elements created in the context of the parent value) + * The Set is needed since once a parent value falls off the stack, we have to clear all the elements + * created in the context of that parent value or else we will have memory leaks + */ + private[figaro] val nccCache: Map[Element[_], List[(Any, Element[_], Set[Element[_]])]] = Map() + + /** + * Retrieve any cached element generated from the current value of the supplied element. Returns None if + * the element does not generate another element. + * + */ + def apply[T](element: Element[T]): Option[Element[T]] = { + element match { + case c: CachingChain[_, T] => { + doCachingChain(c) + } + case c: NonCachingChain[_, T] => { + doNonCachingChain(c) + } + case _ => None + } + } + + /* + * Retrieves an element from the caching chain cache, or inserts a new one if none is found for + * the value of this element + */ + private def doCachingChain[U, T](c: CachingChain[U, T]): Option[Element[T]] = { + + val cachedElems = ccCache.getOrElseUpdate(c, Map()) + val cachedValue = cachedElems.get(c.parent.value) + if (!cachedValue.isEmpty) cachedValue.asInstanceOf[Option[Element[T]]] + else { + // If the value of the element is not found in the cache, generate a new element by calling the chain, + // add it to the cache and the inverted value, and return + val result = c.get(c.parent.value) + cachedElems += (c.parent.value -> result) + val invertedElems = ccInvertedCache.getOrElseUpdate(result, Map()) + invertedElems += (c -> c.parent.value) + Some(result) + } + } + + /* + * Retrieves an element for a non-caching chain. This is not really a cache but rather, + * for each element, a 2-deep stack is maintained that has the current result element of the chain + * at the top of the stack, and the last result element at the bottom of the stack. This is for use in + * MH. When a proposal is made, the chain may change its value. In such a case, we don't want to lose + * the current result element in case the proposal is rejected, so it is moved to the back of the stack. + * If the proposal is reject, the chain is regenerated and the old element is restored to the top of the stack. + * + */ + private def doNonCachingChain[U, T](c: NonCachingChain[U, T]): Option[Element[T]] = { + val nccElems = nccCache.getOrElse(c, List()) + + if (nccElems.isEmpty) { + // If no element has been stored for this chain, generate a value and store it in the stack for this element + val result = c.get(c.parent.value) + nccCache += (c -> List((c.parent.value, result, Set()))) + Some(result) + } else if (c.parent.value == nccElems.head._1) { + // If the current value of the parent matches the value at the top of the stack, return the element at the top of the stack + Some(nccElems.head._2.asInstanceOf[Element[T]]) + } else if (nccElems.size > 1 && c.parent.value == nccElems.last._1) { + // If the current value matches the value at the bottom of the stack, then we need to do a swap; move the back to the front + // and the front to the back + + // Store the elements in the context of the top of the stack. This is the current context of the chain minus the context + // of the value at the back of the stack + val oldContext = c.directContextContents.clone -- nccElems.last._3 + // swap the head and last positions + val head = (nccElems.last._1, nccElems.last._2, Set[Element[_]]()) + val last = (nccElems.head._1, nccElems.head._2, oldContext) + nccCache += (c -> List(head, last)) + Some(nccElems.last._2.asInstanceOf[Element[T]]) + } else { + // Otherwise, we have a new parent value. In this case, we drop the bottom of the stack, deactivate the element + // and the context of the value we are dropping. + if (nccElems.size == 2) universe.deactivate(nccElems.last._3) + val oldContext = c.directContextContents.clone + val result = c.get(c.parent.value) + val head = (c.parent.value, result, Set[Element[_]]()) + val last = (nccElems.head._1, nccElems.head._2, oldContext) + nccCache += (c -> List(head, last)) + Some(result) + } + } + + /** + * Removes an element from the cache. This is needed to properly clean up elements as they are deactivated. + */ + def -=(element: Element[_]) = { + ccCache -= element + nccCache -= element + val invertValue = ccInvertedCache.get(element) + if (invertValue.nonEmpty) invertValue.get.foreach(e => if (ccCache.contains(e._1)) ccCache(e._1) -= e._2) + ccInvertedCache -= element + this + } + + /** + * Clears the cache of all stored elements. + */ + def clear() = { + ccCache.clear() + ccInvertedCache.clear() + nccCache.clear() + universe.deregister(this) + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/library/cache/PermanentCache.scala b/Figaro/src/main/scala/com/cra/figaro/library/cache/PermanentCache.scala new file mode 100644 index 00000000..a0592ae3 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/library/cache/PermanentCache.scala @@ -0,0 +1,93 @@ +/* + * PermanentCache.scala + * Only caches permanent result elements in chain. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.library.cache + +import com.cra.figaro.language._ +import scala.collection.mutable.Map + +/** + * A class which only caches permanent result elements in chain. This class does not cache any non-caching + * chain result elements. Since this class does not implement any element cleanup operations, it is + * best used in an algorithm that clears temporary elements periodically. + * + * + */ +class PermanentCache(universe: Universe) extends Cache(universe) { + + /* Caching chain cache that maps from an element to a map of parent values and resulting elements */ + private[figaro] val ccCache: Map[Element[_], Map[Any, Element[_]]] = Map() + + /* The inverted cache. This maps from result elements back to the chain that uses them. This is needed + * to properly clean up deactivated elements + */ + private[figaro] val ccInvertedCache: Map[Element[_], Map[Element[_], Any]] = Map() + + /** + * Retrieve any cached element generated from the current value of the supplied element. Returns None if + * the element does not generate another element. + * + */ + def apply[T](element: Element[T]): Option[Element[T]] = { + element match { + case c: CachingChain[_, T] => { + doCachingChain(c) + } + case c: NonCachingChain[_, T] => { + Some(c.get(c.parent.value)) + } + case _ => None + } + } + + /* + * Retrieves an element from the caching chain cache, or inserts a new one if none is found for + * the value of this element + */ + private def doCachingChain[U, T](c: Chain[U, T]): Option[Element[T]] = { + + val cachedElems = ccCache.getOrElseUpdate(c, Map()) + val cachedValue = cachedElems.get(c.parent.value) + if (!cachedValue.isEmpty) cachedValue.asInstanceOf[Option[Element[T]]] + else { + // If the value of the element is not found in the cache, generate a new element by calling the chain, + // add it to the cache -only if the result is a permanent element- + val result = c.get(c.parent.value) + if (!result.isTemporary) { + cachedElems += (c.parent.value -> result) + val invertedElems = ccInvertedCache.getOrElseUpdate(result, Map()) + invertedElems += (c -> c.parent.value) + } + Some(result) + } + } + + /** + * Removes an element from the cache. This is needed to properly clean up elements as they are deactivated. + */ + def -=(element: Element[_]) = { + ccCache -= element + val invertValue = ccInvertedCache.get(element) + if (invertValue.nonEmpty) invertValue.get.foreach(e => if (ccCache.contains(e._1)) ccCache(e._1) -= e._2) + ccInvertedCache -= element + this + } + + /** + * Clears the cache of all stored elements. + */ + def clear() = { + ccCache.clear() + ccInvertedCache.clear() + universe.deregister(this) + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/library/collection/Container.scala b/Figaro/src/main/scala/com/cra/figaro/library/collection/Container.scala index 26cd27a7..619653f7 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/collection/Container.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/collection/Container.scala @@ -11,17 +11,13 @@ * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.library.collection - -import com.cra.figaro.language._ +package com.cra.figaro.library.collection + import scala.language.implicitConversions -import com.cra.figaro.library.compound.{FoldLeft, FoldRight, Reduce} -import com.cra.figaro.algorithm.{Values, ValuesMaker} -import com.cra.figaro.algorithm.factored.factors.FactorMaker -import com.cra.figaro.algorithm.factored.factors.Variable -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.algorithm.lazyfactored.ValueSet -import com.cra.figaro.library.atomic.discrete.FromRange + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.discrete.FromRange +import com.cra.figaro.library.compound._ /** * A Container is a Process with a defined sequence of indices. diff --git a/Figaro/src/main/scala/com/cra/figaro/library/collection/VariableSizeArray.scala b/Figaro/src/main/scala/com/cra/figaro/library/collection/VariableSizeArray.scala index 1aebab8e..9619ca1a 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/collection/VariableSizeArray.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/collection/VariableSizeArray.scala @@ -21,7 +21,7 @@ import com.cra.figaro.algorithm.factored._ /** * Holder for an element whose value is a fixed size array. */ -class FixedSizeArrayElement[Value](private val fsa: Element[FixedSizeArray[Value]]) +class FixedSizeArrayElement[Value](protected[figaro] val fsa: Element[FixedSizeArray[Value]]) extends ContainerElement[Int, Value](fsa.asInstanceOf[Element[Container[Int, Value]]]) { /** diff --git a/Figaro/src/main/scala/com/cra/figaro/library/compound/Fold.scala b/Figaro/src/main/scala/com/cra/figaro/library/compound/Fold.scala index b2ddd449..f8b49c21 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/compound/Fold.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/compound/Fold.scala @@ -1,6 +1,6 @@ /* * Fold.scala - * Class for TBD + * An element that folds an operation through a sequence of elements * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Nov 27, 2014 diff --git a/Figaro/src/main/scala/com/cra/figaro/library/compound/MakeList.scala b/Figaro/src/main/scala/com/cra/figaro/library/compound/MakeList.scala index ab2ae5c0..6299ddb9 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/compound/MakeList.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/compound/MakeList.scala @@ -14,11 +14,15 @@ package com.cra.figaro.library.compound import com.cra.figaro.algorithm.ValuesMaker -import com.cra.figaro.algorithm.lazyfactored.{ValueSet, LazyValues, Regular} +import com.cra.figaro.algorithm.lazyfactored.{ ValueSet, LazyValues, Regular } import com.cra.figaro.algorithm.factored.factors._ import com.cra.figaro.language._ import com.cra.figaro.util._ import scala.collection.mutable.Map +import com.cra.figaro.library.collection.VariableSizeArray +import com.cra.figaro.library.collection.FixedSizeArrayElement +import com.cra.figaro.library.collection.FixedSizeArray +import com.cra.figaro.library.collection.MakeArray /** * An element representing making a list of a random number of random items. @@ -28,100 +32,22 @@ import scala.collection.mutable.Map * * @param numItems The element representing the number of items in the list * @param itemMaker A function that creates an element representing a single item in the list + * @deprecated("MakeList is deprecated. Please use the collections library for future support of MakeList capabilities", "3.2.1") */ -class MakeList[T]( - name: Name[List[T]], - val numItems: Element[Int], - val itemMaker: () => Element[T], - collection: ElementCollection) - extends Deterministic[List[T]](name, collection) - with ValuesMaker[List[T]] with IfArgsCacheable[List[T]] { - /** - * An infinite stream of items in the list. At any point in time, the value of this element - * is the prefix of items specified by the value of numItems. - */ - lazy val items = Stream.continually({ - val item = itemMaker() - universe.registerUses(this, item) - item - }) - - override def args = numItems :: (items take numItems.value).toList - - override def generateValue = (items take numItems.value map (_.value)).toList - - /** - * Return the i-th item in the list. Throws IllegalArgumentException if i is greater - * than the current value of numItems - */ - def apply(i: Int) = i < numItems.value match { - case true => items(i) +@deprecated("MakeList is deprecated. Please use the collections library for future support of MakeList capabilities", "3.2.1") +class MakeList[T](name: Name[List[T]], vsa: FixedSizeArrayElement[T], collection: ElementCollection) + extends Apply1[List[T], List[T]](name, vsa.foldLeft(List[T]())((c: List[T], n: T) => c :+ n), (l: List[T]) => l, collection) { + + val numItems = vsa.fsa.asInstanceOf[MakeArray[T]].numItems + + def items = vsa.fsa.value.generate(vsa.fsa.value.indices.toList).map(_._2).toStream + + def apply(i: Int) = i < vsa.fsa.value.size match { + case true => vsa.fsa.value(i) case _ => throw new IllegalArgumentException("Invalid indices to MakeList") } - private def values = LazyValues(universe) - - /* We need to make sure that values are computed on the embedded Injects. Therefore, we create them in makeValues, store them, and use them in makeFactors. - */ - val embeddedInject: Map[Int, Element[List[T]]] = Map() - - def makeValues(depth: Int): ValueSet[List[T]] = { - // This code is subtle. - // If we used itemMaker here, it would create bugs, as the items that appeared in the values would be different from the ones actually used by the Makelist. - // On the other hand, if we used Values(items(0)) as a template for the values of all items, it would create other bugs, as different indices would have the same values, - // even when the values include "new C", so they should all be different. - // Therefore, we use Values()(items(i)) to get the possible value for each item in the stream. - def possibleItemLists(length: Int): ValueSet[List[T]] = { - val inject = Inject(items.take(length):_*) - embeddedInject += length -> inject - values(inject, depth - 1) - } - val possibleLengthValues = values(numItems, depth - 1) - val possibleLengths = possibleLengthValues.regularValues - val itemListsForLengths = possibleLengths.map(possibleItemLists(_)) - val resultValues = - for { - itemLists <- itemListsForLengths - list <- itemLists.regularValues - } yield list - val incomplete = itemListsForLengths.exists(_.hasStar) || possibleLengthValues.hasStar - if (incomplete) ValueSet.withStar(resultValues); else ValueSet.withoutStar(resultValues) - } - -// def makeFactors: List[Factor[Double]] = { -// val parentVar = Variable(numItems) -// // We need to create factors for the items and the lists themselves, which are encapsulated in this MakeList -// val regularParents = parentVar.range.filter(_.isRegular).map(_.value) -// val maxItem = regularParents.reduce(_ max _) -// val itemFactors = List.tabulate(maxItem)((i: Int) => Factory.make(items(i))) -// val indexedResultElemsAndFactors = -// for { i <- regularParents } yield { -// val elem = embeddedInject(i) -// val factors = Factory.make(elem) -// (Regular(i), elem, factors) -// } -// val conditionalFactors = -// parentVar.range.zipWithIndex map (pair => -// Factory.makeConditionalSelector(this, parentVar, pair._2, Variable(indexedResultElemsAndFactors.find(_._1 == pair._1).get._2))) -// conditionalFactors ::: itemFactors.flatten ::: indexedResultElemsAndFactors.flatMap(_._3) -// } - - override def isCachable = { - - if (itemMaker().isCachable == false) { - false - } - - for (arg <- args) yield { - if (arg.isCachable == false) { - false - } - } - - true - } - } object MakeList { @@ -129,7 +55,12 @@ object MakeList { * Create a MakeList element using numItems to determine the number of items * and itemMaker to create each item in the list. */ + @deprecated("MakeList is deprecated. Please use the collections library for future support of MakeList capabilities", "3.2.1") def apply[T](numItems: Element[Int], itemMaker: () => Element[T])(implicit name: Name[List[T]], collection: ElementCollection) = { - new MakeList(name, numItems, itemMaker, collection) + val vsa = VariableSizeArray(numItems, (i: Int) => itemMaker())("", collection) + new MakeList(name, vsa, collection) } + } + + diff --git a/Figaro/src/main/scala/com/cra/figaro/library/compound/RichCPD.scala b/Figaro/src/main/scala/com/cra/figaro/library/compound/RichCPD.scala index f7e58232..331e1e58 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/compound/RichCPD.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/compound/RichCPD.scala @@ -38,7 +38,7 @@ class RichCPD2[T1, T2, U]( clauses: Seq[((CPDCase[T1], CPDCase[T2]), Element[U])], collection: ElementCollection) extends CachingChain[(T1, T2), U]( name, - ^^(arg1, arg2), + ^^(arg1, arg2)("", collection), (p: (T1, T2)) => RichCPD.getMatch(clauses, p._1, p._2), collection) @@ -53,7 +53,7 @@ class RichCPD3[T1, T2, T3, U]( clauses: Seq[((CPDCase[T1], CPDCase[T2], CPDCase[T3]), Element[U])], collection: ElementCollection) extends CachingChain[(T1, T2, T3), U]( name, - ^^(arg1, arg2, arg3), + ^^(arg1, arg2, arg3)("", collection), (p: (T1, T2, T3)) => RichCPD.getMatch(clauses, p._1, p._2, p._3), collection) @@ -69,7 +69,7 @@ class RichCPD4[T1, T2, T3, T4, U]( clauses: Seq[((CPDCase[T1], CPDCase[T2], CPDCase[T3], CPDCase[T4]), Element[U])], collection: ElementCollection) extends CachingChain[(T1, T2, T3, T4), U]( name, - ^^(arg1, arg2, arg3, arg4), + ^^(arg1, arg2, arg3, arg4)("", collection), (p: (T1, T2, T3, T4)) => RichCPD.getMatch(clauses, p._1, p._2, p._3, p._4), collection) @@ -86,7 +86,7 @@ class RichCPD5[T1, T2, T3, T4, T5, U]( clauses: Seq[((CPDCase[T1], CPDCase[T2], CPDCase[T3], CPDCase[T4], CPDCase[T5]), Element[U])], collection: ElementCollection) extends CachingChain[(T1, T2, T3, T4, T5), U]( name, - ^^(arg1, arg2, arg3, arg4, arg5), + ^^(arg1, arg2, arg3, arg4, arg5)("", collection), (p: (T1, T2, T3, T4, T5)) => RichCPD.getMatch(clauses, p._1, p._2, p._3, p._4, p._5), collection) diff --git a/Figaro/src/main/scala/com/cra/figaro/library/decision/Decision.scala b/Figaro/src/main/scala/com/cra/figaro/library/decision/Decision.scala index 83f5361c..9467aef6 100644 --- a/Figaro/src/main/scala/com/cra/figaro/library/decision/Decision.scala +++ b/Figaro/src/main/scala/com/cra/figaro/library/decision/Decision.scala @@ -44,8 +44,7 @@ import scala.collection.mutable.Set * noncaching uses an approximate policy algorithm (based on kNN) and should be used for continuous * parents or discrete parents with a very large range. */ -abstract class Decision[T, U](name: Name[U], parent: Element[T], private var fcn: T => Element[U], cacheSize: Int, collection: ElementCollection) - extends Chain(name, parent, fcn, cacheSize, collection) with PolicyMaker[T, U] { +trait Decision[T, U] extends Chain[T, U] with PolicyMaker[T, U] { /** * The parent type. @@ -57,6 +56,8 @@ abstract class Decision[T, U](name: Name[U], parent: Element[T], private var fcn */ type DValue = this.Value + var fcn: T => Element[U] + /** * The decision function. fcn is declared as a var and can change depending on the policy. */ @@ -100,24 +101,24 @@ abstract class Decision[T, U](name: Name[U], parent: Element[T], private var fcn fcn = new_fcn // Remove all factors since the old factors are now out of date - Factory.removeFactors + Variable.clearCache // Have to nullify the last result even if parents the same since the function changed clearContext // Have to clear the last element in the cache since clearTempory always leaves an element in the cache - if (cache.nonEmpty) resizeCache(cache.last._1) + //if (cache.nonEmpty) resizeCache(cache.last._1) // Have to remove the expansion of the universe since it is out of data LazyValues.clear(universe) // Must regenerate a new value since the cache should never be empty generateValue() } - } + /** * Abstract class for a NonCachingDecision. It is abstract because makePolicy has not been defined yet. */ -abstract class NonCachingDecision[T, U](name: Name[U], parent: Element[T], fcn: T => Element[U], collection: ElementCollection) - extends Decision(name, parent, fcn, 1, collection) { +abstract class NonCachingDecision[T, U](name: Name[U], parent: Element[T], var fcn: T => Element[U], collection: ElementCollection) + extends NonCachingChain(name, parent, fcn, collection) with Decision[T, U] { override def toString = "NonCachingDecision(" + parent + ", " + this.cpd + ")" } @@ -125,8 +126,8 @@ abstract class NonCachingDecision[T, U](name: Name[U], parent: Element[T], fcn: /** * Abstract class for a CachingDecision. It is abstract because makePolicy has not been defined yet. */ -abstract class CachingDecision[T, U](name: Name[U], parent: Element[T], fcn: T => Element[U], collection: ElementCollection) - extends Decision(name, parent, fcn, 1000, collection) { +abstract class CachingDecision[T, U](name: Name[U], parent: Element[T], var fcn: T => Element[U], collection: ElementCollection) + extends CachingChain(name, parent, fcn, collection) with Decision[T, U] { override def toString = "CachingDecision(" + parent + ", " + this.cpd + ")" } diff --git a/Figaro/src/main/scala/com/cra/figaro/util/ColorGradient.scala b/Figaro/src/main/scala/com/cra/figaro/util/ColorGradient.scala new file mode 100644 index 00000000..af6aabd9 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/ColorGradient.scala @@ -0,0 +1,124 @@ +/* + * ColorGradient.scala + * A Factory for color schemes to be used in tables, charts, histograms, etc + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Apr 9, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util + +import scala.collection.mutable.ListBuffer + +/** + * @author Glenn Takata + * + */ +case class ColorPoint(red: Float, green: Float, blue: Float, value: Float) + +class ColorGradient { + var selectedGradient = ColorGradient.MONOCHROME + var gradients: Map[String, List[ColorPoint]] = Map() + + buildGradientMap + + def setGradient(gradient: String) { + selectedGradient = gradient + } + + def buildGradientMap { + gradients += (ColorGradient.HEATMAP -> heatMap, + ColorGradient.MONOCHROME -> monochromeMap, + ColorGradient.TWOCOLOR -> twoColorMap, + ColorGradient.SINGLECOLOR -> singleColor + ) + } + + // A buffer of ColorPoints in ascending order of cumulative value (probability) + def currentGradient: List[ColorPoint] = gradients(selectedGradient) + +// // Inserts a new color point into its correct position in the heatmap: +// def addColorPoint(red: Float, green: Float, blue: Float, value: Float) { +// heatmap.find(_.value > value) match { +// case Some(c) => +// val index = heatmap.indexOf(c) +// heatmap.insert(index - 1, ColorPoint(red, green, blue, value)) +// case _ => +// heatmap += ColorPoint(red, green, blue, value); +// } +// } +// +// // Clears colormap +// def clearGradient() { heatmap.clear(); } + + //-- Inputs a (value) between 0 and 1 and outputs the (red), (green) and (blue) + //-- values representing that position in the gradient. + def getColorAtValue(value: Float): Option[ColorPoint] = { + if (currentGradient.isEmpty) { + None + } else { + var previousPoint = currentGradient.head + var result = currentGradient.last + currentGradient.find(value < _.value) match { + case Some(point) => + val index = Math.max(0, currentGradient.indexOf(point)) + previousPoint = currentGradient(index - 1) + val distance = point.value - previousPoint.value + val distanceRatio = if (distance == 0) 0 else (value - previousPoint.value) / distance + val red = (point.red - previousPoint.red) * distanceRatio + previousPoint.red + val green = (point.green - previousPoint.green) * distanceRatio + previousPoint.green + val blue = (point.blue - previousPoint.blue) * distanceRatio + previousPoint.blue + Some(ColorPoint(red, green, blue, value)) + case _ => Some(result) + } + } + } + + // A variety of color maps + def heatMap: List[ColorPoint] = { + var newColors = new ListBuffer[ColorPoint] + newColors.append(ColorPoint(0, 0, 1, 0.0f)) // Blue. + newColors.append(ColorPoint(0, 1, 1, 0.25f)) // Cyan. + newColors.append(ColorPoint(0, 1, 0, 0.5f)) // Green. + newColors.append(ColorPoint(1, 1, 0, 0.75f)) // Yellow. + newColors.append(ColorPoint(1, 0, 0, 1.0f)) // Red. + + newColors.toList + } + + def monochromeMap: List[ColorPoint] = { + var newColors = new ListBuffer[ColorPoint] + newColors.append(ColorPoint(1, 1, 1, 0.0f)) // White + newColors.append(ColorPoint(1, 0, 0, 1.0f)) // Red + + newColors.toList + } + + def twoColorMap: List[ColorPoint] = { + var newColors = new ListBuffer[ColorPoint] + newColors.append(ColorPoint(0, 0, 1, 0.0f)) // Blue + newColors.append(ColorPoint(1, 1, 0, 1.0f)) // Yellow + + newColors.toList + } + + def singleColor: List[ColorPoint] = { + var newColors = new ListBuffer[ColorPoint] + newColors.append(ColorPoint(0, 0, 1, 0.0f)) // Blue + newColors.append(ColorPoint(0, 0, 1, 1.0f)) // Blue + + newColors.toList + } +} + +object ColorGradient { + val HEATMAP = "heatMap" + val MONOCHROME = "monochrome" + val TWOCOLOR = "twoColor" + val SINGLECOLOR = "singleColor" +} + diff --git a/Figaro/src/main/scala/com/cra/figaro/util/LogStatistics.scala b/Figaro/src/main/scala/com/cra/figaro/util/LogStatistics.scala new file mode 100644 index 00000000..244df7d0 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/LogStatistics.scala @@ -0,0 +1,117 @@ +/* + * LogStatistics.scala + * Utilities for statistics in log space. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 16, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.util + +import org.apache.commons.math3.distribution.TDistribution + +import scala.math._ + +/** + * Represents basic statistics of a sample from a nonnegative-valued univariate distribution. All computations on these + * statistics are done in log space to prevent underflow. + * @param logMean The log of the mean of the samples. + * @param logVariance The log of the variance of the samples. + * @param count The number of samples taken. + */ +case class LogStatistics(logMean: Double, logVariance: Double, count: Int) { + /** + * Returns the statistics corresponding to the distribution wherein each sample is multiplied by the given constant. + * @param logConstant The log of the constant to multiply by. + * @return Statistics such that the mean and variance are updated, and the count is unchanged. + */ + def multiplyByConstant(logConstant: Double) = { + LogStatistics(logMean + logConstant, logVariance + logConstant * 2, count) + } +} + +object LogStatistics { + /** + * Performs a one-sided t-test for the comparison of v1.logMean and v2.logMean. This compares the smaller mean to the + * larger mean, i.e. it computes a p-value for min(v1.logMean, v2.logMean) < max(v1.logMean, v2.logMean). + * @param v1 Mean, variance, and sample count from first distribution. Requires v1.count > 1. + * @param v2 Mean, variance, and sample count from second distribution. Requires v2.count > 1. + * @return A p-value for the hypothesis. A small p-value indicates high confidence that the population mean of the + * sample with lesser mean is less than the population mean of the other sample. + */ + def oneSidedTTest(v1: LogStatistics, v2: LogStatistics): Double = { + require(v1.count > 1 && v2.count > 1, "t-test requires counts > 1") + // If both variances are 0, we return here to avoid NaNs. + if(v1.logVariance == Double.NegativeInfinity && v2.logVariance == Double.NegativeInfinity) { + if(v1.logMean == v2.logMean) 0.5 + else 0.0 + } + // Otherwise, at least one of the variances is positive, so everything below is well-defined. + else { + // log(v1.variance / v1.count), respectively v2. These quantities are used multiple times in computing the degrees + // of freedom and the t-score, so it's helpful not having to recompute them. + val logVar1OverCount1 = v1.logVariance - log(v1.count) + val logVar2OverCount2 = v2.logVariance - log(v2.count) + + // Compute log numerator and denominator of the degrees of freedom using the Welch-Satterthwaite equation. + val logDoFNum = 2 * logSum(logVar1OverCount1, logVar2OverCount2) + val logDoFDenom = logSum(2 * logVar1OverCount1 - log(v1.count - 1), 2 * logVar2OverCount2 - log(v2.count - 1)) + val degreesOfFreedom = exp(logDoFNum - logDoFDenom) + + // To get a negative t-score, take the positive difference in log space, then invert the sign after exponentiation. + val logNegativeTScore = logDiff(v1.logMean max v2.logMean, v1.logMean min v2.logMean) - + 0.5 * logSum(logVar1OverCount1, logVar2OverCount2) + val tScore = -exp(logNegativeTScore) + new TDistribution(degreesOfFreedom).cumulativeProbability(tScore) + } + } +} + +/** + * Trait for computing mean in variance in log space in an online fashion. + */ +trait OnlineLogStatistics { + + // Fields for recording the mean and variance across runs. See Wikipedia, "Algorithms for calculating variance". + // Modified to work in log space to prevent underflow. + // Number of samples taken + protected var count = 0 + // Log of mean of samples taken so far + protected var logMean = Double.NegativeInfinity + // Log of variance of samples taken so far, multiplied by (count - 1) + protected var logM2 = Double.NegativeInfinity + + /** + * Record the weight in the rolling mean and variance computation. + * @param logWeight Log of the weight to record. + */ + def record(logWeight: Double) = { + // Online variance algorithm from Wikipedia + count += 1 + if(logWeight >= logMean) { + val logDelta = logDiff(logWeight, logMean) + logMean = logSum(logMean, logDelta - math.log(count)) + // assert(logMean >= logWeight) + logM2 = logSum(logM2, logDelta + logDiff(logWeight, logMean)) + } + else { + val logNegativeDelta = logDiff(logMean, logWeight) + logMean = logDiff(logMean, logNegativeDelta - math.log(count)) + // assert(logMean < logWeight) + // The negatives cancel in the right hand size + logM2 = logSum(logM2, logNegativeDelta + logDiff(logMean, logWeight)) + } + } + + /** + * Return the combined statistics for the log probability of evidence over all runs of this sampler. + * If the number of observations is 0, the returned log mean is -Infinity. + * If the number of observations is 0 or 1, the returned log variance is NaN. + */ + def totalLogStatistics = LogStatistics(logMean, logM2 - math.log(count - 1), count) +} diff --git a/Figaro/src/main/scala/com/cra/figaro/util/package.scala b/Figaro/src/main/scala/com/cra/figaro/util/package.scala index 4306f2a9..9e808096 100644 --- a/Figaro/src/main/scala/com/cra/figaro/util/package.scala +++ b/Figaro/src/main/scala/com/cra/figaro/util/package.scala @@ -249,7 +249,17 @@ package object util { if (seq.isEmpty) throw new IllegalArgumentException("Empty list") else helper(seq.tail.toList, 1, 0, seq.head) } - + + /** + * Computes the difference of two probabilities in log space. + * Returns NaN if p2>p1, -Infinity if p2=p1 + */ + def logDiff(p1: Double, p2: Double): Double = { + // Also covers the case where p1 = p2 = -Infinity + if(p2 == Double.NegativeInfinity) p1 + // log(exp(p1) - exp(p2)) = log(exp(p1) * (1 - exp(p2) / exp(p1))) = p1 + log(1 - exp(p2 - p1)) + else p1 + Math.log1p(-Math.exp(p2 - p1)) + } /** * Sums two probabilities in log space. diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/DataView.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/DataView.scala new file mode 100644 index 00000000..e21dc505 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/DataView.scala @@ -0,0 +1,36 @@ +/* + * DataView.scala + * Internal data repository/method for use by tables, charts, histograms, etc + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Apr 9, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization + +import prefuse.data.Table +import prefuse.data.query.NumberRangeModel +import prefuse.util.ui.ValuedRangeModel + +/** + * @author Glenn Takata (gtakata@cra.com) + * + * Mar 17, 2015 + */ +trait DataView { + def name: String + def title: String + def range: ValuedRangeModel + + def nValues: Int + + def getTable: Table + + def yMax: Double + def dataType: Int + def yRangeModel: NumberRangeModel +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/ResultsGUI.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/ResultsGUI.scala new file mode 100644 index 00000000..f9982a8c --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/ResultsGUI.scala @@ -0,0 +1,142 @@ +/* + * ResultsGUI.scala + * The main controller for visualizations. Coordinates data input and display as well as user interaction with displays. + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Mar 16, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization + +import scala.swing._ +import com.cra.figaro.language.{Element} +import com.cra.figaro.util.visualization.results.{ ContinuousData, DiscreteData, ResultsData, ResultsTable, ResultsView } +import com.cra.figaro.util.visualization.histogram.{ Histogram } +import com.cra.figaro.util.visualization.distribution.{Distribution} +import com.cra.figaro.util.visualization.reduction.DataReduction +import com.cra.figaro.util.ColorGradient +import scala.swing.event.Event +import scala.swing.event.TableRowsSelected + +/** + * @author Glenn Takata (gtakata@cra.com) + */ + +case class NewResult(result: ResultsData) extends Event + +class ResultHandler extends Publisher { + def newResult(result: ResultsData) { + Swing.onEDT( + publish(NewResult(result)) + ) + } +} + +class EmptyTab extends BoxPanel(Orientation.Vertical) { + import ResultsGUI._ + preferredSize = new Dimension(TAB_WIDTH, TAB_HEIGHT) +} + +object ResultsGUI extends SimpleSwingApplication { + val TAB_WIDTH = 600 + val TAB_HEIGHT = 300 + + val TABLE_WIDTH = 600 + val TABLE_HEIGHT = 250 + + val results = new ResultHandler +// def addResult(result: ResultsData) { + def addResult(name: String, dist: Any) { + val result = dist match { + case l: List[(Double, Double)] => DiscreteData(name, DataReduction.binToDistribution(l)) + case e: Element[_] => ContinuousData(name, e) + } + results.newResult(result) + } + + def top = new MainFrame { + title = "Figaro Results" + + preferredSize = new Dimension(TAB_WIDTH, 600) + + var currentColor: String = ColorGradient.SINGLECOLOR + + def setColor(color: String) { + currentColor = color + table.revalidate + } + + // table + val table = new ResultsTable + + val graphs = new TabbedPane() { + preferredSize = new Dimension(TAB_WIDTH, TAB_HEIGHT) + resizable = true + + pages += new TabbedPane.Page("", new EmptyTab) + } + + menuBar = new MenuBar { + contents += new Menu("File") { + contents += new MenuItem(Action("Exit") { + sys.exit(0) + }) + } + } + + val mainPanel = new BorderPanel { + import BorderPanel.Position._ + resizable = true + add(graphs, Center) + add(table, South) + } + + contents = mainPanel + + listenTo(results, table.getSelection) + reactions += { + case NewResult(result) => { + table.add(result) + // histogram + updateHistogram(result) + mainPanel.revalidate() + mainPanel.repaint + } + case TableRowsSelected(source, range, false) => { + val row = table.getSelectedRow +// println(row) + updateHistogram(row) + mainPanel.revalidate() + mainPanel.repaint + } + } + + private def updateHistogram(result: ResultsData) { + graphs.pages.clear() + + result match { + case DiscreteData(name, dist) => { + val color = currentColor + val histogramTab = new Histogram(new ResultsView(result), color) + graphs.pages += new TabbedPane.Page(result.name + " Distribution", histogramTab) + } + case ContinuousData(name, dist) => { + val color = ColorGradient.HEATMAP + val distributionTab = new Distribution(new ResultsView(result), color) + graphs.pages += new TabbedPane.Page(result.name + " Density", distributionTab) + + } + case _ => + } + + graphs.revalidate + graphs.repaint + } + + visible = true + } +} diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/distribution/Distribution.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/distribution/Distribution.scala new file mode 100644 index 00000000..85cf5691 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/distribution/Distribution.scala @@ -0,0 +1,237 @@ +/* + * Distribution.scala + * Setup and display distributions based on continuous element data + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jul 6, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.distribution + +import java.awt.Color +import java.awt.event._ +import java.awt.geom.Rectangle2D +import java.text.NumberFormat +import javax.swing.BorderFactory +import javax.swing.Box +import scala.collection.JavaConversions +import scala.swing._ +import scala.swing.BorderPanel.Position._ +import prefuse.Constants +import prefuse.Display +import prefuse.Visualization +import prefuse.action.ActionList +import prefuse.action.RepaintAction +import prefuse.action.assignment.ColorAction +import prefuse.action.filter.VisibilityFilter +import prefuse.action.layout.AxisLayout +import prefuse.action.layout.AxisLabelLayout +import prefuse.controls.ControlAdapter +import prefuse.data.Table +import prefuse.data.expression.AndPredicate +import prefuse.data.expression.Expression +import prefuse.data.expression.Predicate +import prefuse.data.expression.parser.ExpressionParser +import prefuse.data.io.CSVTableReader +import prefuse.data.query.NumberRangeModel +import prefuse.data.query.RangeQueryBinding +import prefuse.render.RendererFactory +import prefuse.render.AxisRenderer +import prefuse.util.ColorLib +import prefuse.util.FontLib +import prefuse.util.UpdateListener +import prefuse.util.ui.JFastLabel +import prefuse.util.ui.UILib +import prefuse.visual.VisualItem +import prefuse.visual.expression.VisiblePredicate +import com.cra.figaro.util.visualization.ResultsGUI._ +import com.cra.figaro.util.visualization.DataView + +/** + * @author Glenn Takata (gtakata@cra.com) + */ +class Distribution(val dataview: DataView, var color: String) extends BorderPanel { + // fonts, colours, etc. + UILib.setColor(peer, ColorLib.getColor(0, 0, 0), Color.BLACK); + val itemRenderer = new DistributionRenderer(color, dataview) + + // title + val title = new Label(dataview.title) + title.preferredSize = new Dimension(200, 20) + title.verticalAlignment = Alignment.Top + title.font = FontLib.getFont("Tahoma", 18) + + // visualization (main container) + val vis: Visualization = new Visualization() + val visualTable = vis.addTable(dataview.name, dataview.getTable) + + // dynamic query based on name + val valueQ: RangeQueryBinding = new RangeQueryBinding(visualTable, "Value"); + val filter: AndPredicate = new AndPredicate(valueQ.getPredicate()); + val nf = NumberFormat.getIntegerInstance(); + nf.setMaximumFractionDigits(2); + + // X-axis + val xaxis: AxisLayout = new AxisLayout(dataview.name, "Value", Constants.X_AXIS, VisiblePredicate.TRUE); + + // add the labels to the x-axis + val xlabels: AxisLabelLayout = new AxisLabelLayout("xlab", xaxis) + xlabels.setNumberFormat(nf) + vis.putAction("xlabels", xlabels) + + // Y-axis + val yaxis: AxisLayout = new AxisLayout(dataview.name, "Probability", Constants.Y_AXIS, VisiblePredicate.TRUE); + + // ensure the y-axis spans the height of the data container + yaxis.setRangeModel(dataview.yRangeModel) + // add the labels to the y-axis + val ylabels: AxisLabelLayout = new AxisLabelLayout("ylab", yaxis); + ylabels.setNumberFormat(nf); + + // drawing actions + // specify the fill (interior) as a static colour (white) + val fill: ColorAction = new ColorAction(dataview.name, VisualItem.FILLCOLOR, 0); + + val draw: ActionList = new ActionList() + draw.add(fill) + draw.add(xaxis) + draw.add(yaxis) + draw.add(ylabels) + draw.add(new RepaintAction()) + vis.putAction("draw", draw) + + // update actions + val update: ActionList = new ActionList() + update.add(new VisibilityFilter(dataview.name, filter)); // filter performs the size/name filtering + update.add(xaxis) + update.add(yaxis) + update.add(ylabels) + update.add(new RepaintAction()) + vis.putAction("update", update) + + // create an update listener that will update the visualization when fired + val lstnr: UpdateListener = new UpdateListener() { + def update(src: Object) { + vis.run("update"); + } + }; + + // add this update listener to the filter, so that when the filter changes (i.e., + // the user adjusts the axis parameters, or enters a name for filtering), the + // visualization is updated + filter.addExpressionListener(lstnr); + + // add the listener to this component + peer.addComponentListener(lstnr); + + + val display = setupDisplay(vis) + + vis.setRendererFactory(new RendererFactory() { + + val yAxisRenderer = new AxisRenderer(Constants.FAR_LEFT, Constants.CENTER) + val xAxisRenderer = new AxisRenderer(Constants.CENTER, Constants.BOTTOM) + + def getRenderer(item: VisualItem) = { + if (item.isInGroup("ylab")) { + yAxisRenderer + } else if (item.isInGroup("xlab")) { + xAxisRenderer + } else { + itemRenderer + } + } + }) + + // container for elements at the top of the screen + val topContainer = new BoxPanel(Orientation.Horizontal) { + peer.add(Box.createHorizontalStrut(5)); + contents += title + peer.add(Box.createHorizontalGlue()); + peer.add(Box.createHorizontalStrut(5)); + } + + // add the containers to the JPanel + layout(topContainer) = North + layout(Component.wrap(display)) = Center + // add(slider, BorderLayout.SOUTH); + + vis.run("draw"); + vis.run("xlabels"); + + def setupDisplay(visualization: Visualization) = { + val disp = new Display(visualization); + + // set the display properties + disp.setBorder(BorderFactory.createEmptyBorder(5, 20, 5, 10)); + disp.setSize(new Dimension(TAB_WIDTH, TAB_HEIGHT)); + disp.setHighQuality(true); + + // call the function that sets the sizes of the containers that contain + // the data and the axes + displayLayout(disp, dataview); + + // whenever the window is re-sized, update the layout of the axes + disp.addComponentListener(new ComponentAdapter() { + + override def componentResized(e: ComponentEvent) { + displayLayout(disp, dataview); + } + }); + + disp.addControlListener(new ControlAdapter() { + override def itemClicked(item: VisualItem, event: MouseEvent) { + val table = dataview.getTable + val filter = ExpressionParser.predicate("Name = " + item.getSourceTuple.get("Name")) + val rows = table.rows(filter) + for (item <- JavaConversions.asScalaIterator(table.tuples(rows))) { + println(item) + } + } + }); + + disp + } + + /* + * calculate the sizes of the data and axes containers based on the + * display size, and then tell the visualization to update itself and + * re-draw the x-axis labels + */ + def displayLayout(display: Display, data: DataView) { + val insets = display.getInsets(); + val width = display.getWidth(); + val height = display.getHeight(); + val insetWidth = insets.left + insets.right; + val insetHeight = insets.top + insets.bottom; + + val viewXOffset = 20 + val viewYOffset = 15 + val yAxisWidth = 5; + val xAxisHeight = 10; + val displayHeight = height - xAxisHeight - insetHeight - 2 * viewYOffset + val maxDisplayWidth = width - yAxisWidth - insetWidth - 2 * viewXOffset + + val displayWidth = math.min(data.range.getExtent * 60, maxDisplayWidth) + + val dataView: Rectangle2D = new Rectangle2D.Double(insets.left + yAxisWidth + viewXOffset, insets.top, displayWidth, displayHeight) + val xView: Rectangle2D = new Rectangle2D.Double(insets.left + yAxisWidth + viewXOffset, insets.top + displayHeight + viewYOffset , displayWidth, xAxisHeight) + val yView: Rectangle2D = new Rectangle2D.Double(insets.left, insets.top, yAxisWidth, displayHeight) + + // reset all the bounds + itemRenderer.setBounds(dataView) + + xaxis.setLayoutBounds(dataView); + xlabels.setLayoutBounds(xView) + + yaxis.setLayoutBounds(dataView); + ylabels.setLayoutBounds(yView) + + vis.run("update"); + vis.run("xlabels"); + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/distribution/DistributionRenderer.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/distribution/DistributionRenderer.scala new file mode 100644 index 00000000..72d1841a --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/distribution/DistributionRenderer.scala @@ -0,0 +1,98 @@ +/* + * DistributionRenderer.scala + * Display distribution elements based on position, value, color gradient + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jul 6, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.distribution + +import java.awt.Graphics2D +import java.awt.Shape +import java.awt.geom.Rectangle2D + +import prefuse.Constants +import prefuse.render.AbstractShapeRenderer +import prefuse.util.ColorLib +import prefuse.util.GraphicsLib +import prefuse.visual.VisualItem + +import com.cra.figaro.util.ColorGradient +import com.cra.figaro.util.visualization.DataView + +/** + * @author Glenn Takata + * + */ +class DistributionRenderer(color: String, dataview: DataView) extends AbstractShapeRenderer { + var bounds: Rectangle2D = _ + var isVertical: Boolean = true + var orientation = Constants.ORIENT_BOTTOM_TOP; + var barWidth: Int = 5 + var nBars: Int = dataview.nValues + var pMax: Double = dataview.yMax + var rect = new Rectangle2D.Double(); + + val gradient = new ColorGradient + gradient.setGradient(color) + + def setBounds(newBounds: Rectangle2D) { + bounds = newBounds; + barWidth = (bounds.getWidth / nBars).toInt + } + + def setOrientation(orient: Int) { + + if (orient != Constants.ORIENT_LEFT_RIGHT && + orient != Constants.ORIENT_RIGHT_LEFT) + { + throw new IllegalArgumentException( + "Invalid orientation value: " + orient); + } + orientation = orient; + isVertical = (orientation == Constants.ORIENT_TOP_BOTTOM || + orientation == Constants.ORIENT_BOTTOM_TOP); + } + + override def getRawShape(item: VisualItem): Shape = { + var width: Double = 0 + var height: Double = 0 + + var x = item.getX() + var y = item.getY() + + width = math.min(bounds.getWidth / nBars, 30) + height = bounds.getHeight - y + + // Center the bar around the x-location + if (width > 1) { + x = x - width / 2; + + } + + rect.setFrame(x, y, width, height); + return rect; + + } + + override def render(g: Graphics2D, item: VisualItem) { + val shape = getShape(item); + + val probability = item.getSourceTuple.getFloat("Probability") + val value = probability / pMax + + gradient.getColorAtValue(value.floatValue()) match { + case Some(color) => + item.setFillColor(ColorLib.rgba(color.red, color.green, color.blue, 1)) + GraphicsLib.paint(g, item, shape, null, AbstractShapeRenderer.RENDER_TYPE_FILL); + case _ => + } + + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/histogram/Histogram.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/histogram/Histogram.scala new file mode 100644 index 00000000..404f5f15 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/histogram/Histogram.scala @@ -0,0 +1,241 @@ +/* + * Histogram.scala + * Setup and display histograms based on distribution (prob, value) data + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Apr 9, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.histogram + +import java.awt.Color +import java.awt.event._ +import java.awt.geom.Rectangle2D +import java.text.NumberFormat +import javax.swing.BorderFactory +import javax.swing.Box +import scala.collection.JavaConversions +import scala.swing._ +import scala.swing.BorderPanel.Position._ +import prefuse.Constants +import prefuse.Display +import prefuse.Visualization +import prefuse.action.ActionList +import prefuse.action.RepaintAction +import prefuse.action.assignment.ColorAction +import prefuse.action.filter.VisibilityFilter +import prefuse.action.layout.AxisLayout +import prefuse.action.layout.AxisLabelLayout +import prefuse.controls.ControlAdapter +import prefuse.data.Table +import prefuse.data.expression.AndPredicate +import prefuse.data.expression.Expression +import prefuse.data.expression.Predicate +import prefuse.data.expression.parser.ExpressionParser +import prefuse.data.io.CSVTableReader +import prefuse.data.query.NumberRangeModel +import prefuse.data.query.RangeQueryBinding +import prefuse.render.RendererFactory +import prefuse.render.AxisRenderer +import prefuse.util.ColorLib +import prefuse.util.FontLib +import prefuse.util.UpdateListener +import prefuse.util.ui.JFastLabel +import prefuse.util.ui.UILib +import prefuse.visual.VisualItem +import prefuse.visual.expression.VisiblePredicate +import com.cra.figaro.util.visualization.ResultsGUI._ +import com.cra.figaro.util.visualization.DataView + +/** + * @author Glenn Takata (gtakata@cra.com) + */ +class Histogram(val dataview: DataView, var color: String) extends BorderPanel { + // fonts, colours, etc. + UILib.setColor(peer, ColorLib.getColor(0, 0, 0), Color.BLACK); + val itemRenderer = new HistogramRenderer(color, dataview.nValues) + + // title + val title = new Label(dataview.title) + title.preferredSize = new Dimension(200, 20) + title.verticalAlignment = Alignment.Top + title.font = FontLib.getFont("Tahoma", 18) + + // visualization (main container) + val vis: Visualization = new Visualization() + val visualTable = vis.addTable(dataview.name, dataview.getTable) + + // dynamic query based on name + val valueQ: RangeQueryBinding = new RangeQueryBinding(visualTable, "Value"); + val filter: AndPredicate = new AndPredicate(valueQ.getPredicate()); + val nf = NumberFormat.getIntegerInstance(); + nf.setMaximumFractionDigits(2); + + // X-axis + val xaxis: AxisLayout = new AxisLayout(dataview.name, "Value", Constants.X_AXIS, VisiblePredicate.TRUE); + + // ensure the axis spans the width of the data container + xaxis.setDataType(Constants.NOMINAL) + xaxis.setRangeModel(dataview.range) + + // add the labels to the x-axis + val xlabels: AxisLabelLayout = new AxisLabelLayout("xlab", xaxis); + xlabels.setNumberFormat(nf) + vis.putAction("xlabels", xlabels) + + // Y-axis + val yaxis: AxisLayout = new AxisLayout(dataview.name, "Probability", Constants.Y_AXIS, VisiblePredicate.TRUE); + + // set the y-axis range + yaxis.setRangeModel(dataview.yRangeModel) + // add the labels to the y-axis + val ylabels: AxisLabelLayout = new AxisLabelLayout("ylab", yaxis); + ylabels.setNumberFormat(nf); + + // drawing actions + // specify the fill (interior) as a static colour (white) + val fill: ColorAction = new ColorAction(dataview.name, VisualItem.FILLCOLOR, 0); + + val draw: ActionList = new ActionList() + draw.add(fill) + draw.add(xaxis) + draw.add(yaxis) + draw.add(ylabels) + draw.add(new RepaintAction()) + vis.putAction("draw", draw) + + // update actions + val update: ActionList = new ActionList() + update.add(new VisibilityFilter(dataview.name, filter)); // filter performs the size/name filtering + update.add(xaxis) + update.add(yaxis) + update.add(ylabels) + update.add(new RepaintAction()) + vis.putAction("update", update) + + // create an update listener that will update the visualization when fired + val lstnr: UpdateListener = new UpdateListener() { + def update(src: Object) { + vis.run("update"); + } + }; + + // add this update listener to the filter, so that when the filter changes (i.e., + // the user adjusts the axis parameters, or enters a name for filtering), the + // visualization is updated + filter.addExpressionListener(lstnr); + + // add the listener to this component + peer.addComponentListener(lstnr); + + + val display = setupDisplay(vis) + + vis.setRendererFactory(new RendererFactory() { + + val yAxisRenderer = new AxisRenderer(Constants.FAR_LEFT, Constants.CENTER) + val xAxisRenderer = new AxisRenderer(Constants.CENTER, Constants.BOTTOM) + + def getRenderer(item: VisualItem) = { + if (item.isInGroup("ylab")) { + yAxisRenderer + } else if (item.isInGroup("xlab")) { + xAxisRenderer + } else { + itemRenderer + } + } + }) + + // container for elements at the top of the screen + val topContainer = new BoxPanel(Orientation.Horizontal) { + peer.add(Box.createHorizontalStrut(5)); + contents += title + peer.add(Box.createHorizontalGlue()); + peer.add(Box.createHorizontalStrut(5)); + } + + // add the containers to the JPanel + layout(topContainer) = North + layout(Component.wrap(display)) = Center + // add(slider, BorderLayout.SOUTH); + + vis.run("draw"); + vis.run("xlabels"); + + def setupDisplay(visualization: Visualization) = { + val disp = new Display(visualization); + + // set the display properties + disp.setBorder(BorderFactory.createEmptyBorder(5, 20, 5, 10)); + disp.setSize(new Dimension(TAB_WIDTH, TAB_HEIGHT)); + disp.setHighQuality(true); + + // call the function that sets the sizes of the containers that contain + // the data and the axes + displayLayout(disp, dataview); + + // whenever the window is re-sized, update the layout of the axes + disp.addComponentListener(new ComponentAdapter() { + + override def componentResized(e: ComponentEvent) { + displayLayout(disp, dataview); + } + }); + + disp.addControlListener(new ControlAdapter() { + override def itemClicked(item: VisualItem, event: MouseEvent) { + val table = dataview.getTable + val filter = ExpressionParser.predicate("Name = " + item.getSourceTuple.get("Name")) + val rows = table.rows(filter) + for (item <- JavaConversions.asScalaIterator(table.tuples(rows))) { + println(item) + } + } + }); + + disp + } + + /* + * calculate the sizes of the data and axes containers based on the + * display size, and then tell the visualization to update itself and + * re-draw the x-axis labels + */ + def displayLayout(display: Display, data: DataView) { + val insets = display.getInsets(); + val width = display.getWidth(); + val height = display.getHeight(); + val insetWidth = insets.left + insets.right; + val insetHeight = insets.top + insets.bottom; + + val viewXOffset = 20 + val viewYOffset = 10 + val yAxisWidth = 5; + val xAxisHeight = 10; + val displayHeight = height - xAxisHeight - insetHeight - 2 * viewYOffset + val maxDisplayWidth = width - yAxisWidth - insetWidth - 2 * viewXOffset + + val displayWidth = math.min(data.range.getExtent * 60, maxDisplayWidth) + + val dataView: Rectangle2D = new Rectangle2D.Double(insets.left + yAxisWidth + viewXOffset, insets.top + viewYOffset, displayWidth, displayHeight) + val xView: Rectangle2D = new Rectangle2D.Double(insets.left + yAxisWidth + viewXOffset, insets.top + displayHeight + viewYOffset , displayWidth, xAxisHeight) + val yView: Rectangle2D = new Rectangle2D.Double(insets.left, insets.top + viewYOffset, yAxisWidth, displayHeight) + + // reset all the bounds + itemRenderer.setBounds(dataView) + + xaxis.setLayoutBounds(dataView); + xlabels.setLayoutBounds(xView) + + yaxis.setLayoutBounds(dataView); + ylabels.setLayoutBounds(yView) + + vis.run("update"); + vis.run("xlabels"); + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/histogram/HistogramRenderer.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/histogram/HistogramRenderer.scala new file mode 100644 index 00000000..ac89da91 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/histogram/HistogramRenderer.scala @@ -0,0 +1,94 @@ +/* + * HistogramRenderer.scala + * Renders individual bars for a histogram, taking into account color and position + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Apr 9, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.histogram + +import java.awt.Graphics2D +import java.awt.Shape +import java.awt.geom.Rectangle2D + +import prefuse.Constants +import prefuse.render.AbstractShapeRenderer +import prefuse.util.ColorLib +import prefuse.util.GraphicsLib +import prefuse.visual.VisualItem + +import com.cra.figaro.util.ColorGradient + +/** + * @author Glenn Takata + * + */ +class HistogramRenderer(color: String, nBars: Int = 5) extends AbstractShapeRenderer { + var bounds: Rectangle2D = _ + var isVertical: Boolean = true + var orientation = Constants.ORIENT_BOTTOM_TOP; + var barWidth: Int = 5 + var rect = new Rectangle2D.Double(); + + val gradient = new ColorGradient + gradient.setGradient(color) + + def setBounds(newBounds: Rectangle2D) { + bounds = newBounds; + barWidth = (bounds.getWidth / nBars).toInt + } + + def setOrientation(orient: Int) { + + if (orient != Constants.ORIENT_LEFT_RIGHT && + orient != Constants.ORIENT_RIGHT_LEFT) + { + throw new IllegalArgumentException( + "Invalid orientation value: " + orient); + } + orientation = orient; + isVertical = (orientation == Constants.ORIENT_TOP_BOTTOM || + orientation == Constants.ORIENT_BOTTOM_TOP); + } + + override def getRawShape(item: VisualItem): Shape = { + var width: Double = 0 + var height: Double = 0 + + var x = item.getX() + var y = item.getY() + + width = math.min(bounds.getWidth / nBars, 30) + height = bounds.getHeight - y + + // Center the bar around the x-location + if (width > 1) { + x = x - width / 2; + + } + + rect.setFrame(x, y, width, height); + return rect; + + } + + override def render(g: Graphics2D, item: VisualItem) { + val shape = getShape(item); + + val values = item.getSourceTuple + + gradient.getColorAtValue(values.getFloat("Probability")) match { + case Some(color) => + item.setFillColor(ColorLib.rgba(color.red, color.green, color.blue, 1)) + GraphicsLib.paint(g, item, shape, null, AbstractShapeRenderer.RENDER_TYPE_FILL); + case _ => + } + + } + +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/reduction/DataReduction.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/reduction/DataReduction.scala new file mode 100644 index 00000000..2e6fe72c --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/reduction/DataReduction.scala @@ -0,0 +1,62 @@ +/* + * DataReduction.scala + * Setup and display distributions based on continuous element data + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jul 6, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.reduction + +import scala.collection._ + +object DataReduction { + def binToDistribution(data: List[(Double, Double)]): List[(Double, Double)] = { + if (data.size > 50) { + var mean = 0.0 + var totalProb = 0.0 + var count = 0 + var min = Double.MaxValue + var max = Double.MinValue + var ss = 0.0 + + for ((prob, value) <- data) { + totalProb += prob + mean += prob * value + min = math.min(min, value) + max = math.max(max, value) + ss += prob * value * value + count += 1 + } + + val variance = ss - mean * mean + val sd = math.sqrt(variance) + + val distMax = mean + 3 * sd + val distMin = mean - 3 * sd + + val nInterval = math.min(count, 300) + val interval = (distMax - distMin) / nInterval + var dist = Array.fill[Double](nInterval)(0) + + for ((prob, value) <- data) { + val pos = math.max(math.min(math.floor((value - distMin) / interval).toInt, nInterval - 1), 0) + val posProb = dist(pos) + dist(pos) = posProb + prob + } + + val probDist = for (i <- 1 to nInterval) yield { + val value = distMin + i * interval + (dist(i - 1), value) + } + + probDist.toList + } else { + data + } + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsData.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsData.scala new file mode 100644 index 00000000..4f11c5a4 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsData.scala @@ -0,0 +1,97 @@ +/* + * ResultsData.scala + * Trait and classes representing data input by the user. Includes discrete (distribution List) and continuous (element) + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Apr 9, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.results + +import org.apache.commons.math3.distribution._ +import scala.collection.mutable.{ ListBuffer } +import com.cra.figaro.language.{ Atomic, Element, HasDensity } +import com.cra.figaro.library.atomic.continuous.{ AtomicExponential, AtomicNormal, AtomicUniform } + +/** + * @author Glenn Takata (gtakata@comcast.net) + */ +trait ResultsData { + val name: String + def resultString: String + def distribution: List[(Double, _)] +} + +case class DiscreteData[T](override val name: String, override val distribution: List[(Double, T)]) extends ResultsData { + override def resultString: String = { + val buffer = new StringBuffer("{") + for ((prob, value) <- distribution) { + buffer.append("(").append("%06.4f".format(prob)).append("->").append(value).append(")") + } + buffer.append("}") + return buffer.toString() + } +} + +case class ContinuousData[T](override val name: String, val element: Element[_]) extends ResultsData { + val intervals = 200 + + override def resultString: String = { + if (element.isInstanceOf[AtomicNormal]) { + val normal = element.asInstanceOf[AtomicNormal] + val mean = normal.mean + val variance = normal.variance + s"Normal($mean, $variance)" + } else if (element.isInstanceOf[AtomicUniform]) { + val uniform = element.asInstanceOf[AtomicUniform] + val lower = uniform.lower + val upper = uniform.upper + s"Uniform($lower, $upper)" + } else if (element.isInstanceOf[AtomicExponential]) { + val exponential = element.asInstanceOf[AtomicExponential] + val lambda = exponential.lambda + s"Exponential($lambda)" + } else { + "" + } + } + + override def distribution: List[(Double, Double)] = { + if (element.isInstanceOf[AtomicNormal]) { + val normal = element.asInstanceOf[AtomicNormal] + val min = normal.mean - 3 * normal.standardDeviation + val max = normal.mean + 3 * normal.standardDeviation + constructDistribution(new NormalDistribution(normal.mean, normal.standardDeviation), min, max) + } else if (element.isInstanceOf[AtomicUniform]) { + val uniform = element.asInstanceOf[AtomicUniform] + val min = uniform.lower + val max = uniform.upper + constructDistribution(new UniformRealDistribution(min, max), min, max) + } else if (element.isInstanceOf[AtomicExponential]) { + val exponential = element.asInstanceOf[AtomicExponential] + val min = 0 + val max = 3.0 / exponential.lambda + constructDistribution(new ExponentialDistribution(1 / exponential.lambda), min, max) + } else { + val buffer = new ListBuffer[(Double, Double)]() + buffer.toList + } + } + + private def constructDistribution(function: AbstractRealDistribution, min: Double, max: Double): List[(Double, Double)] = { + val buffer = new ListBuffer[(Double, Double)]() + val spacing = (max - min) / intervals + var previous = min + for (x <- 1 until intervals) { + val value = min + x * spacing + val density = function.density(value) + buffer.append((density, value)) + previous = value + } + buffer.toList + } +} \ No newline at end of file diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsTable.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsTable.scala new file mode 100644 index 00000000..735924b3 --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsTable.scala @@ -0,0 +1,88 @@ +/* + * ResultsTable.scala + * Visual element for a table to display user inputs. Includes discrete (distribution List) and continuous (element) + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Apr 9, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.results + +import java.awt.Dimension +import javax.swing.table.{ AbstractTableModel } +import scala.collection._ +import scala.swing.{ BoxPanel, Orientation, ScrollPane, Table } +import com.cra.figaro.util.visualization.{ResultsGUI} + +/** + * @author gtakata + */ +class ResultsTable extends BoxPanel(Orientation.Vertical) { + import ResultsGUI._ + + preferredSize = new Dimension(TAB_WIDTH, TABLE_HEIGHT) + + def add(result: ResultsData) { + tableModel.addResult(result) + + table.revalidate + table.repaint + } + + val tableModel = new ResultsTableModel(Array[Array[Any]](), List("Element", "Distribution")) + val table = new Table() { + model = tableModel + } + + def getSelectedRow = { + val row = table.selection.rows.head + tableModel.getRow(row) + } + + def getSelection = { + table.selection + } + + contents += new ScrollPane(table) +} + +class ResultsTableModel(var rowData: Array[Array[Any]], val columnNames: Seq[String]) extends AbstractTableModel { + override def getColumnName(column: Int) = columnNames(column).toString() + + var results = new mutable.ListBuffer[ResultsData]() + + def getRowCount() = rowData.length + def getColumnCount() = columnNames.length + + def getValueAt(row: Int, col: Int): AnyRef = { + val result = results(row) + col match { + case 0 => result.name + case 1 => result.resultString + case _ => "" + } + } + + override def isCellEditable(row: Int, column: Int) = false + override def setValueAt(value: Any, row: Int, col: Int) { + rowData(row)(col) = value + } + + def addRow( data: Array[AnyRef]) { + rowData ++= Array(data.asInstanceOf[Array[Any]]) + } + + def addResult(result: ResultsData) { + results.append(result) + addRow(Array[AnyRef](result.name, result.distribution)) + } + + def getRow(row: Int) = { + results(row) + } +} + diff --git a/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsView.scala b/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsView.scala new file mode 100644 index 00000000..fce1ef4b --- /dev/null +++ b/Figaro/src/main/scala/com/cra/figaro/util/visualization/results/ResultsView.scala @@ -0,0 +1,87 @@ +/* + * ResultsView.scala + * A visual component to display a table of user data. Includes discrete (distribution List) and continuous (element) + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Mar 16, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.util.visualization.results + +import scala.collection.JavaConversions._ +import prefuse.Constants +import prefuse.data.Table +import prefuse.data.Tuple +import prefuse.data.io.CSVTableReader +import prefuse.data.query.{NumberRangeModel, ObjectRangeModel} +import prefuse.util.ui.ValuedRangeModel +import scala.collection.JavaConversions + +import com.cra.figaro.util.visualization.DataView + +/** + * @author Glenn Takata + * + * Mar 16, 2015 + */ +class ResultsView[T](data: ResultsData) extends DataView { + val name = data.name + val title = name + + def nValues = data.distribution.size + + def range: ValuedRangeModel = { + val values = JavaConversions.asJavaCollection(data.distribution.map(_._2)).toArray() + new ObjectRangeModel(values.asInstanceOf[Array[Object]]) + } + + def getTable = readTable + + def readTable: Table = { + val resultsTable = new Table() + resultsTable.addColumn("Name", classOf[String]) + resultsTable.addColumn("Value", classOf[Object]) + resultsTable.addColumn("Probability", classOf[Float]) + + resultsTable.addRows(nValues) + + var row = 0 + val name = data.name + for (value <- data.distribution) { + + resultsTable.set(row, "Name", name) + resultsTable.set(row, "Value", value._2) + resultsTable.set(row, "Probability", value._1) + + row += 1 + } + resultsTable + } + + def dataType = { + data match { + case ContinuousData(_, _) => Constants.NUMERICAL + case _ => Constants.NOMINAL + } + } + + def yMax = math.min(1.1 * data.distribution.reduceLeft((a, b) => if (a._1 > b._1) a else b)._1, 1.0) + def yRangeModel = { + data match { + case ContinuousData(_, _) => + new NumberRangeModel(0, yMax, 0, yMax ) + case _ => { + if (yMax < 0.5) + new NumberRangeModel(0, yMax, 0, yMax) + else + new NumberRangeModel(0, 1.0, 0, 1.0) + } + } + } +} + +class NoResult() extends ResultsView(DiscreteData("No Data", List((1.0, true)))) \ No newline at end of file diff --git a/Figaro/src/test/resources/BookData/Labels.txt b/Figaro/src/test/resources/BookData/Labels.txt new file mode 100644 index 00000000..28e56f57 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Labels.txt @@ -0,0 +1,200 @@ +0 NormalTrainingEmail_0.txt +0 NormalTrainingEmail_1.txt +0 NormalTrainingEmail_2.txt +0 NormalTrainingEmail_3.txt +0 NormalTrainingEmail_4.txt +0 NormalTrainingEmail_5.txt +0 NormalTrainingEmail_6.txt +0 NormalTrainingEmail_7.txt +0 NormalTrainingEmail_8.txt +0 NormalTrainingEmail_9.txt +0 NormalTrainingEmail_10.txt +0 NormalTrainingEmail_11.txt +0 NormalTrainingEmail_12.txt +0 NormalTrainingEmail_13.txt +0 NormalTrainingEmail_14.txt +0 NormalTrainingEmail_15.txt +0 NormalTrainingEmail_16.txt +0 NormalTrainingEmail_17.txt +0 NormalTrainingEmail_18.txt +0 NormalTrainingEmail_19.txt +0 NormalTrainingEmail_20.txt +0 NormalTrainingEmail_21.txt +0 NormalTrainingEmail_22.txt +0 NormalTrainingEmail_23.txt +0 NormalTrainingEmail_24.txt +0 NormalTrainingEmail_25.txt +0 NormalTrainingEmail_26.txt +0 NormalTrainingEmail_27.txt +0 NormalTrainingEmail_28.txt +0 NormalTrainingEmail_29.txt +0 NormalTrainingEmail_30.txt +0 NormalTrainingEmail_31.txt +0 NormalTrainingEmail_32.txt +0 NormalTrainingEmail_33.txt +0 NormalTrainingEmail_34.txt +0 NormalTrainingEmail_35.txt +0 NormalTrainingEmail_36.txt +0 NormalTrainingEmail_37.txt +0 NormalTrainingEmail_38.txt +0 NormalTrainingEmail_39.txt +0 NormalTrainingEmail_40.txt +0 NormalTrainingEmail_41.txt +0 NormalTrainingEmail_42.txt +0 NormalTrainingEmail_43.txt +0 NormalTrainingEmail_44.txt +0 NormalTrainingEmail_45.txt +0 NormalTrainingEmail_46.txt +0 NormalTrainingEmail_47.txt +0 NormalTrainingEmail_48.txt +0 NormalTrainingEmail_49.txt +0 NormalTrainingEmail_50.txt +0 NormalTrainingEmail_51.txt +0 NormalTrainingEmail_52.txt +0 NormalTrainingEmail_53.txt +0 NormalTrainingEmail_54.txt +0 NormalTrainingEmail_55.txt +0 NormalTrainingEmail_56.txt +0 NormalTrainingEmail_57.txt +0 NormalTrainingEmail_58.txt +0 NormalTrainingEmail_59.txt +1 SpamTrainingEmail_0.txt +1 SpamTrainingEmail_1.txt +1 SpamTrainingEmail_2.txt +1 SpamTrainingEmail_3.txt +1 SpamTrainingEmail_4.txt +1 SpamTrainingEmail_5.txt +1 SpamTrainingEmail_6.txt +1 SpamTrainingEmail_7.txt +1 SpamTrainingEmail_8.txt +1 SpamTrainingEmail_9.txt +1 SpamTrainingEmail_10.txt +1 SpamTrainingEmail_11.txt +1 SpamTrainingEmail_12.txt +1 SpamTrainingEmail_13.txt +1 SpamTrainingEmail_14.txt +1 SpamTrainingEmail_15.txt +1 SpamTrainingEmail_16.txt +1 SpamTrainingEmail_17.txt +1 SpamTrainingEmail_18.txt +1 SpamTrainingEmail_19.txt +1 SpamTrainingEmail_20.txt +1 SpamTrainingEmail_21.txt +1 SpamTrainingEmail_22.txt +1 SpamTrainingEmail_23.txt +1 SpamTrainingEmail_24.txt +1 SpamTrainingEmail_25.txt +1 SpamTrainingEmail_26.txt +1 SpamTrainingEmail_27.txt +1 SpamTrainingEmail_28.txt +1 SpamTrainingEmail_29.txt +1 SpamTrainingEmail_30.txt +1 SpamTrainingEmail_31.txt +1 SpamTrainingEmail_32.txt +1 SpamTrainingEmail_33.txt +1 SpamTrainingEmail_34.txt +1 SpamTrainingEmail_35.txt +1 SpamTrainingEmail_36.txt +1 SpamTrainingEmail_37.txt +1 SpamTrainingEmail_38.txt +1 SpamTrainingEmail_39.txt +0 TestEmail_0.txt +1 TestEmail_1.txt +0 TestEmail_2.txt +0 TestEmail_3.txt +0 TestEmail_4.txt +0 TestEmail_5.txt +1 TestEmail_6.txt +0 TestEmail_7.txt +1 TestEmail_8.txt +1 TestEmail_9.txt +1 TestEmail_10.txt +0 TestEmail_11.txt +1 TestEmail_12.txt +1 TestEmail_13.txt +0 TestEmail_14.txt +0 TestEmail_15.txt +1 TestEmail_16.txt +1 TestEmail_17.txt +0 TestEmail_18.txt +0 TestEmail_19.txt +1 TestEmail_20.txt +0 TestEmail_21.txt +0 TestEmail_22.txt +1 TestEmail_23.txt +0 TestEmail_24.txt +0 TestEmail_25.txt +0 TestEmail_26.txt +1 TestEmail_27.txt +1 TestEmail_28.txt +0 TestEmail_29.txt +1 TestEmail_30.txt +0 TestEmail_31.txt +1 TestEmail_32.txt +1 TestEmail_33.txt +1 TestEmail_34.txt +1 TestEmail_35.txt +1 TestEmail_36.txt +1 TestEmail_37.txt +0 TestEmail_38.txt +0 TestEmail_39.txt +1 TestEmail_40.txt +1 TestEmail_41.txt +0 TestEmail_42.txt +1 TestEmail_43.txt +1 TestEmail_44.txt +1 TestEmail_45.txt +1 TestEmail_46.txt +0 TestEmail_47.txt +0 TestEmail_48.txt +0 TestEmail_49.txt +0 TestEmail_50.txt +0 TestEmail_51.txt +0 TestEmail_52.txt +0 TestEmail_53.txt +0 TestEmail_54.txt +0 TestEmail_55.txt +0 TestEmail_56.txt +0 TestEmail_57.txt +0 TestEmail_58.txt +1 TestEmail_59.txt +0 TestEmail_60.txt +1 TestEmail_61.txt +1 TestEmail_62.txt +0 TestEmail_63.txt +0 TestEmail_64.txt +1 TestEmail_65.txt +0 TestEmail_66.txt +0 TestEmail_67.txt +1 TestEmail_68.txt +0 TestEmail_69.txt +0 TestEmail_70.txt +1 TestEmail_71.txt +0 TestEmail_72.txt +0 TestEmail_73.txt +0 TestEmail_74.txt +1 TestEmail_75.txt +0 TestEmail_76.txt +1 TestEmail_77.txt +1 TestEmail_78.txt +0 TestEmail_79.txt +0 TestEmail_80.txt +0 TestEmail_81.txt +0 TestEmail_82.txt +1 TestEmail_83.txt +0 TestEmail_84.txt +0 TestEmail_85.txt +0 TestEmail_86.txt +0 TestEmail_87.txt +0 TestEmail_88.txt +1 TestEmail_89.txt +0 TestEmail_90.txt +1 TestEmail_91.txt +0 TestEmail_92.txt +0 TestEmail_93.txt +1 TestEmail_94.txt +0 TestEmail_95.txt +0 TestEmail_96.txt +0 TestEmail_97.txt +0 TestEmail_98.txt +0 TestEmail_99.txt diff --git a/Figaro/src/test/resources/BookData/LearnedModel.txt b/Figaro/src/test/resources/BookData/LearnedModel.txt new file mode 100644 index 00000000..92fd0c10 --- /dev/null +++ b/Figaro/src/test/resources/BookData/LearnedModel.txt @@ -0,0 +1,26386 @@ +100 +0.39805825242718446 +0.7511511859608347 +0.022563096312814418 +0.35502396915627843 +0.2671072909553941 +13039 +100 +date +100 +from +100 +to +100 +with +100 +messageid +100 +subject +100 +received +100 +by +99 +id +99 +returnpath +98 +esmtp +97 +for +96 +contenttype +94 +deliveredto +92 +of +91 +127001 +88 +localhost +86 +the +86 +a +85 +10 +83 +is +77 +mimeversion +76 +and +76 +on +76 +in +76 +postfix +73 +at +73 +textplain +69 +this +69 +contenttransferencoding +67 +smtp +67 +no +65 +as +65 +be +64 +sender +62 +it +61 +not +59 +you +59 +list +57 +xuidl +57 +your +55 +0100 +54 +that +54 +or +53 +i +53 +are +52 +email +52 +listunsubscribe +51 +listsubscribe +51 +2002 +51 +precedence +50 +81168116 +49 +xaccountkey +48 +re +48 +xoriginalto +48 +listhelp +48 +listid +48 +xmozillakeys +47 +xmozillastatus +45 +an +45 +xmozillastatus2 +45 +using +44 +0700 +44 +00000000 +43 +body +43 +fetchmail590 +42 +singledrop +42 +pdt +42 +dogmaslashnullorg +42 +may +42 +use +41 +if +41 +0000 +41 +but +40 +all +40 +0400 +39 +listpost +39 +mon +39 +unsubscribe +39 +xbeenthere +38 +out +38 +mail +37 +can +37 +have +37 +charsetusascii +37 +listarchive +36 +only +36 +references +35 +our +35 +ist +35 +its +35 +message +35 +charsetiso88591 +35 +has +35 +inreplyto +35 +2010 +34 +get +34 +edt +34 +more +34 +7bit +34 +imap +34 +xvirusscanned +33 +was +33 +xmailer +33 +html +33 +mxgooglecom +32 +just +32 +which +32 +domain +31 +bulk +31 +jmlocalhost +31 +record +31 +texthtml +31 +mlsubscribertechcsminingorg +30 +system +30 +3 +29 +they +28 +here +28 +required40 +28 +below +28 +authenticationresults +28 +fallback +28 +about +28 +like +28 +up +28 +so +28 +tue +28 +version +28 +googlecom +27 +one +27 +do +27 +included +27 +will +27 +receivedspf +27 +dont +27 +wrote +27 +other +27 +spfpass +27 +xmailmanversion +26 +tr +26 +would +26 +1 +26 +we +26 +spamassassin +26 +table +26 +breakdown +26 +im +26 +xloop +25 +then +25 +contact +25 +now +25 +debianuserlistsdebianorg +25 +manual +25 +designates +25 +thu +25 +2525 +25 +pass +25 +permitted +25 +listsdebianuserlisztdebianorg +25 +network +25 +jalapeno +25 +port +25 +2 +24 +ldowhitelist5 +24 +yes +24 +resentdate +24 +sep +24 +utc +24 +trouble +24 +errorsto +24 +charsetutf8 +24 +first +24 +taggedabove10000 +24 +account5 +23 +resentmessageid +23 +when +23 +name +23 +public +23 +clientip8219575100 +23 +please +23 +head +23 +wed +22 +mailtodebianuserrequestlistsdebianorgsubjecthelp +22 +see +22 +quotedprintable +22 +some +22 +any +22 +p +22 +8219575100 +22 +xasgtag +22 +new +22 +know +22 +8bit +22 +xbarracudaconnect +21 +taglevel10000 +21 +been +21 +rule +21 +me +21 +xstatus +21 +those +21 +how +21 +hibodycsminingorg +21 +325 +21 +these +21 +resentsender +21 +archive +21 +2007091301 +21 +2008110401 +21 +required53 +21 +there +21 +normal +20 +go +20 +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +20 +24 +20 +microsoft +20 +xbarracudaspamstatus +20 +yyyylocalhostnetnoteinccom +20 +xrcvirus +20 +listmasterlistsdebianorg +20 +mailing +20 +what +20 +td +20 +19216818120 +19 +xbarracudaspamscore +19 +xlevel +19 +did +19 +back +19 +oldreturnpath +19 +debian +19 +httpspamgwcsminingorg8000cgibinmarkcgi +19 +0001 +19 +them +19 +0900 +19 +information +19 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +19 +code +19 +20080610 +19 +replyto +19 +xamavisstatus +19 +because +19 +unknown +19 +qmqp +19 +br +19 +also +19 +0200 +19 +their +18 +outlook +18 +pts +18 +why +18 +still +18 +xasgdebugid +18 +xcheckerversion +18 +found +18 +fri +18 +who +18 +useragent +18 +than +18 +meta +18 +cc +18 +resentfrom +18 +apr +18 +xmailinglist +18 +doctype +18 +good +18 +xrcspam +18 +exim +17 +account2 +17 +think +17 +most +17 +does +17 +5 +17 +350 +17 +bank +17 +sent +17 +address +17 +xasgorigsubj +17 +version325 +17 +need +17 +xpriority +17 +url +17 +enht +17 +font +17 +2009 +17 +host +17 +free +17 +into +17 +make +17 +testsbayes002 +17 +quarantinelevel10000 +17 +had +17 +were +17 +end +17 +trusted +16 +could +16 +jmjmasonorg +16 +used +16 +xbarracudavirusscanned +16 +both +16 +mime +16 +csminingorg +16 +amavisdnew +16 +barracuda +16 +reputation +16 +description +16 +years +16 +spam +16 +rules +16 +spamgwcsminingorg +16 +010 +16 +htmlmessage +16 +jmasonorg +16 +policy +16 +time +16 +rate +16 +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +15 +am +15 +relay +15 +through +15 +being +15 +aligncenter +15 +after +15 +v1 +15 +23 +15 +border0 +15 +envelopefrom +15 +my +15 +xbarracudastarttime +15 +parts +15 +lisztdebianorg +15 +0 +15 +2011 +15 +low +15 +style +15 +mailtodebianuserlistsdebianorg +15 +transitionalen +14 +forkxentcom +14 +many +14 +result +14 +w3cdtd +14 +custom +14 +sun +14 +phobos +14 +doesnt +14 +people +14 +11 +14 +xmsmailpriority +14 +20 +14 +since +14 +click +14 +before +14 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +14 +well +14 +100 +14 +help +14 +linux +14 +000 +14 +build +14 +best +14 +0500 +13 +jst +13 +mailtoforkrequestxentcomsubjecthelp +13 +rights +13 +httpxentcommailmanlistinfofork +13 +around +13 +state +13 +killlevel10000 +13 +work +13 +should +13 +us +13 +view +13 +peruser +13 +aug +13 +service +13 +brl +13 +xbarracudaurl +13 +autolearnfailed +13 +web +13 +produced +13 +online +13 +lairxentcom +13 +two +13 +phoboslabsnetnoteinccom +13 +image +13 +same +13 +invoked +13 +phone +13 +div +12 +look +12 +users +12 +firewall +12 +again +12 +available +12 +rohit +12 +security +12 +xbarracudaspamreport +12 +without +12 +autolearnham +12 +old +12 +form +12 +19216818251 +12 +mimeole +12 +encodingutf8 +12 +26 +12 +want +12 +over +12 +bgcolorffffff +12 +helvetica +12 +sat +12 +contentdisposition +12 +way +12 +send +12 +m0620212mailcsminingorg +12 +mimehtmlonly +12 +change +12 +contains +12 +read +12 +listsdebianorg +12 +antispam +12 +u +12 +version250cvs +12 +off +12 +9 +12 +even +12 +keep +12 +img +12 +problem +12 +zzzzlocalhost +11 +client +11 +httpequivcontenttype +11 +set +11 +software +11 +httpxentcompipermailfork +11 +better +11 +file +11 +contenttexthtml +11 +ive +11 +done +11 +group +11 +7 +11 +try +11 +4 +11 +added +11 +thanks +11 +area +11 +great +11 +signature +11 +cached +11 +actually +11 +debianuserrequestlistsdebianorg +11 +support +11 +xspamstatus +11 +his +11 +long +11 +31 +11 +via +11 +own +11 +day +11 +ns1csminingorg +11 +verify +10 +wish +10 +including +10 +above +10 +rdnsnone +10 +b +10 +fontsize +10 +helo +10 +believe +10 +internet +10 +total +10 +made +10 +color +10 +life +10 +words +10 +source +10 +trtd +10 +multipartalternative +10 +19216818250 +10 +thing +10 +very +10 +never +10 +inline +10 +mailtoforkrequestxentcomsubjectsubscribe +10 +150 +10 +6 +10 +hi +10 +sgamma +10 +f +10 +show +10 +xbarracudaspamflag +10 +format +10 +such +10 +much +10 +local +10 +barracudaheaderfp20 +10 +didnt +10 +19216831 +10 +check +10 +etc +9 +directory +9 +enus +9 +httpequiv3dcontenttype +9 +give +9 +based +9 +mailtoforkrequestxentcomsubjectunsubscribe +9 +050 +9 +dcsminingorg +9 +500 +9 +include +9 +dkimsignature +9 +every +9 +another +9 +special +9 +images +9 +06 +9 +certificate +9 +data +9 +s +9 +sure +9 +link +9 +893893 +9 +services +9 +rdns +9 +17 +9 +having +9 +size +9 +arial +9 +line +9 +receiving +9 +0px +9 +files +9 +going +9 +year +9 +week +9 +future +9 +trying +9 +anyone +9 +cant +9 +where +9 +r +9 +c +9 +too +9 +really +9 +find +9 +22 +9 +os +9 +required50 +9 +delivered +9 +add +9 +say +9 +anything +9 +inc +9 +khare +8 +jul +8 +yet +8 +within +8 +sponsored +8 +manager +8 +headericsminingorg +8 +company +8 +cnofws +8 +needed +8 +12 +8 +input +8 +yyyylocalhostspamassassintaintorg +8 +xoriginaldate +8 +problems +8 +xpolicydweight +8 +theres +8 +forkadminxentcom +8 +put +8 +windows +8 +dynamiclooking +8 +world +8 +almost +8 +sfnet +8 +250 +8 +united +8 +center +8 +might +8 +experience +8 +multipart +8 +post +8 +styletextalign +8 +importance +8 +cellpadding0 +8 +allow +8 +said +8 +already +8 +scores +8 +mailwebnotenet +8 +bits +8 +complete +8 +install +8 +cellspacing0 +8 +receive +8 +among +8 +sansserif +8 +rpmlistadminfreshrpmsnet +8 +forged +8 +times +8 +10143348 +8 +score7 +8 +copyright +8 +august +8 +left +8 +gw1csminingorg +8 +case +8 +testsldosubscriberldowhitelist +8 +wont +8 +alignleft +8 +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +8 +19 +8 +month +8 +create +8 +size2 +8 +organization +8 +start +8 +different +8 +fontfamily +8 +little +8 +subscription +7 +1000 +7 +unsubscription +7 +herea +7 +he +7 +government +7 +getting +7 +09 +7 +401 +7 +cannot +7 +gw1csminingorg19216818250 +7 +gnulinux +7 +mail1csminingorg +7 +newsletter +7 +though +7 +running +7 +once +7 +oct +7 +account4 +7 +rbl +7 +secure +7 +6416122236 +7 +helod +7 +june +7 +xacceptlanguage +7 +httpwwwlinuxiemailmanlistinfoilug +7 +technical +7 +width100 +7 +today +7 +sale +7 +pm +7 +however +7 +none +7 +xhtml +7 +50 +7 +recently +7 +least +7 +removed +7 +point +7 +emails +7 +department +7 +takes +7 +de +7 +469 +7 +border3d0 +7 +few +7 +lowest +7 +between +7 +forkexamplecom +7 +top +7 +thought +7 +autolearnno +7 +happened +7 +far +7 +clean +7 +friends +7 +page +7 +17128113151 +7 +x +7 +application +7 +each +7 +site +7 +ns2csminingorg +7 +formatflowed +7 +textdecoration +7 +appears +7 +28 +7 +61 +7 +01 +7 +something +7 +jmnetnoteinccom +7 +program +7 +several +7 +rdnsdynamic +7 +original +7 +means +7 +nor +7 +width1 +7 +0800 +7 +style3dcolor +7 +office +7 +person +7 +contain +7 +currently +7 +thats +7 +print +7 +next +6 +matches +6 +either +6 +ip +6 +script +6 +price +6 +mandarklabsnetnoteinccom +6 +30 +6 +versiontlsv1sslv3 +6 +yyyylocalhostexamplecom +6 +uswsflist1bsourceforgenet +6 +offers +6 +xsender +6 +19412514545 +6 +similar +6 +until +6 +809122912 +6 +type +6 +book +6 +notinblnjabl15 +6 +easy +6 +matter +6 +xentcom +6 +along +6 +size1 +6 +order +6 +idea +6 +question +6 +less +6 +span +6 +error +6 +xbarracudarblip +6 +personal +6 +35 +6 +face3darial +6 +radio +6 +april +6 +working +6 +listed +6 +return +6 +come +6 +forkspamassassintaintorg +6 +stylecolor +6 +setup +6 +youre +6 +212173515 +6 +name3dgenerator +6 +apple +6 +cest +6 +hand +6 +auth02nlegwnnet +6 +strong +6 +lots +6 +content3dtexthtml +6 +solid +6 +groups +6 +fourla01 +6 +iluglinuxie +6 +widely +6 +216136171252 +6 +simply +6 +05 +6 +important +6 +0300 +6 +19216832 +6 +400 +6 +seconds +6 +testsbsfsc0sa148a +6 +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +6 +job +6 +title +6 +multipartsigned +6 +run +6 +selling +6 +background +6 +control +6 +power +6 +makes +6 +process +6 +padding +6 +media +6 +regards +6 +subscribed +6 +score110 +6 +pgp +6 +helouswsflist1sourceforgenet +6 +single +6 +stephen +6 +hope +6 +reply +6 +hundreds +6 +stylefontsize +6 +exchange +6 +connect +6 +charset3diso8859 +6 +starting +6 +express +6 +xkeywords +6 +place +6 +maybe +6 +history +6 +wants +6 +called +6 +right +6 +25 +6 +full +6 +bsfsc0satofromaddrmatch +6 +search +6 +htmlfontbig +6 +planning +6 +userid +6 +helpunsubscribeupdate +6 +involved +6 +namesignatureasc +6 +test +6 +business +6 +19317254 +6 +under +6 +created +5 +development +5 +e +5 +assistance +5 +agent +5 +must +5 +n +5 +w +5 +directly +5 +late +5 +s0832016219710167 +5 +ratehard +5 +text +5 +bsfsc0sa392hl +5 +simple +5 +high +5 +xmlnshttpwwww3org1999xhtml +5 +related +5 +yyyyexamplecom +5 +213105180140 +5 +seeing +5 +whether +5 +xbarracudabblip +5 +standard +5 +checks +5 +probably +5 +pfont +5 +she +5 +computer +5 +city +5 +operate +5 +call +5 +share +5 +issues +5 +nothing +5 +professional +5 +reading +5 +mailtoforkspamassassintaintorg +5 +underline +5 +xcomplaintsto +5 +stop +5 +lists +5 +t +5 +intent +5 +existsxmailer +5 +second +5 +michael +5 +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +5 +take +5 +decided +5 +dkimneutral +5 +15 +5 +james +5 +21 +5 +able +5 +en +5 +rssfeedsjmasonorg +5 +warning +5 +interest +5 +avisited +5 +18 +5 +c1 +5 +dwl +5 +bsmtpd +5 +htmlhead +5 +tdtd +5 +tdfont +5 +account +5 +base +5 +locally +5 +due +5 +satofromaddrmatchhl +5 +cases +5 +bug +5 +prices +5 +buy +5 +default +5 +070 +5 +deliverydate +5 +provided +5 +bsfsc0sa392f +5 +trade +5 +external +5 +xbitdefenderwksspam +5 +begin +5 +various +5 +digital +5 +d +5 +m +5 +seems +5 +valigntop +5 +built +5 +user +5 +latest +5 +border +5 +hostname +5 +increase +5 +roman +5 +memory +5 +number +5 +brbr +5 +hit +5 +details +5 +her +5 +collection +5 +unfortunately +5 +visit +5 +bit +5 +copy +5 +money +5 +entire +5 +121 +5 +notinsblxblspamhaus15 +5 +story +5 +gets +5 +xspamlevel +5 +let +5 +bold +5 +legal +5 +issue +5 +days +5 +comments +5 +pgpsignature5 +5 +politics +5 +letters +5 +stylefontfamily +5 +cellspacing3d0 +5 +got +5 +health +5 +latin +5 +accept +5 +align3dcenter +5 +02 +5 +margin +5 +cut +5 +fall +5 +states +5 +market +5 +protocolapplicationpgpsignature +5 +offer +5 +cellpadding3 +5 +minutes +5 +companies +5 +released +5 +close +5 +ilug +5 +height +5 +facearial +5 +pc +5 +ahover +5 +ie +5 +primary +5 +40 +5 +mail2csminingorg +5 +styleborderwidth +5 +choose +5 +esmtps +5 +insurance +5 +recipient +5 +team +5 +targetblank +5 +aligncentera +5 +serif +5 +moved +5 +width +5 +big +5 +someone +5 +talk +5 +remember +5 +guarantee +5 +fonttd +5 +width3d100 +5 +ns4csminingorg +5 +writes +5 +looking +5 +provide +5 +5px +5 +technology +5 +sa392hl +5 +law +5 +news +5 +open +5 +perl +5 +home +5 +turn +5 +access +5 +minute +5 +hello +5 +cdt +5 +2008 +5 +qmail +5 +marked +5 +started +5 +pages +5 +gldudebianuser2mgmaneorg +5 +msgidfrommtaheader2 +5 +align3dcenterbfont +5 +reason +5 +mx +5 +popular +5 +suite +4 +trick +4 +ad +4 +hard +4 +war +4 +large +4 +logmaneorg +4 +launched +4 +move +4 +nations +4 +080 +4 +during +4 +ul +4 +root +4 +pretty +4 +suggest +4 +dog +4 +moving +4 +wrong +4 +rpmlistfreshrpmsnet +4 +startup +4 +greetings +4 +mean +4 +school +4 +europe +4 +required +4 +match +4 +tlsv1 +4 +cost +4 +platform +4 +divfont +4 +following +4 +color666666 +4 +driver +4 +gnupg +4 +viagra +4 +surbl +4 +david +4 +against +4 +avoid +4 +usually +4 +refer +4 +potential +4 +certain +4 +arguments +4 +103113 +4 +60026000000 +4 +ranging +4 +confidential +4 +uk +4 +mailings +4 +8 +4 +instead +4 +score69 +4 +points +4 +height6 +4 +ago +4 +went +4 +sites +4 +terms +4 +width75 +4 +folder +4 +19216833 +4 +official +4 +calls +4 +private +4 +deleted +4 +committee +4 +jun +4 +native +4 +gecko20100317 +4 +2007 +4 +highly +4 +vs +4 +discussion +4 +removal +4 +un +4 +unstable +4 +position +4 +a0 +4 +especially +4 +hear +4 +size3d1a +4 +packages +4 +ignored +4 +past +4 +whitelisted +4 +store +4 +framework +4 +uri +4 +stuff +4 +j +4 +2001 +4 +miss +4 +kde +4 +million +4 +rather +4 +zzzzlocalhostnetnoteinccom +4 +mail3csminingorg +4 +xoriginatingip +4 +invalid +4 +pick +4 +fairly +4 +border0atd +4 +bsfsc0tg035a +4 +v +4 +comes +4 +discuss +4 +esmtpsa +4 +yourself +4 +model +4 +soon +4 +uid +4 +male +4 +sort +4 +intel +4 +property +4 +suspicious +4 +according +4 +film +4 +reasons +4 +mailtodebiankdelistsdebianorg +4 +course +4 +crelaxedrelaxed +4 +adrian +4 +lucamerciadristudentulgacbe +4 +1200 +4 +tda +4 +washington +4 +faster +4 +valignbottom +4 +fine +4 +cellpadding1 +4 +older +4 +cellpadding3d0 +4 +textalign +4 +card +4 +questions +4 +love +4 +quote +4 +jeremy +4 +installation +4 +freshrpms +4 +value +4 +yyyyspamassassintaintorg +4 +sa154 +4 +saving +4 +sign +4 +neutral +4 +happens +4 +32 +4 +google +4 +usabr +4 +international +4 +members +4 +08 +4 +building +4 +updating +4 +months +4 +write +4 +live +4 +14 +4 +report +4 +quite +4 +kept +4 +tdtr +4 +doing +4 +gw2csminingorg +4 +mozilla50 +4 +29 +4 +helped +4 +sylpheed +4 +arsasha256 +4 +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +4 +september +4 +applicationpgpsignature +4 +species +4 +tool +4 +newslettertitle +4 +disk +4 +heaven +4 +while +4 +cipher +4 +th +4 +ma +4 +feb +4 +10px +4 +sell +4 +priority +4 +tools +4 +text000000 +4 +john +4 +color000000 +4 +domainkeysignature +4 +systems +4 +tag +4 +typesubmit +4 +early +4 +ready +4 +him +4 +bodyhtml +4 +engineer +4 +worldwide +4 +true +4 +council +4 +mailtorpmzzzlistfreshrpmsnet +4 +red +4 +rpm +4 +height8td +4 +became +4 +text3d000000 +4 +project +4 +mailtorazorusersexamplesourceforgenet +4 +bgcolor3dffffff +4 +ones +4 +quick +4 +disaster +4 +particular +4 +gmexim +4 +chris +4 +certainly +4 +l +4 +inbox +4 +common +4 +type3dsubmit +4 +modern +4 +limited +4 +across +4 +partition +4 +shows +4 +0pt +4 +tdimg +4 +unless +4 +exactly +4 +learn +4 +char +4 +thank +4 +o +4 +listmasterlinuxie +4 +fill +4 +209sfnet +4 +xmimeole +4 +021 +4 +102313469 +4 +example +4 +scientific +4 +content3dmshtml +4 +mailtoexmhworkersrequestspamassassintaintorgsubjecthelp +4 +possible +4 +advertising +4 +rpmzzzlistadminfreshrpmsnet +4 +machine +4 +throughout +4 +employment +4 +street +4 +331vamm2 +4 +linked +4 +1010010911 +4 +theyre +4 +jabber +4 +further +4 +rpmlist +4 +death +4 +associated +4 +box +4 +assuming +4 +release +4 +190200 +4 +guess +4 +mj1963 +4 +interested +4 +75 +4 +xmailerpresent +4 +reported +4 +blue +4 +201 +4 +option +4 +dhersaaes256sha +4 +language +4 +300 +4 +expect +4 +black +4 +enough +4 +given +4 +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +4 +english +4 +ratio +4 +07 +4 +gecko20100411 +4 +johnson +4 +sister +3 +blocklist +3 +rant +3 +pay +3 +pan0132 +3 +marginheight0 +3 +wednesday +3 +member +3 +basic +3 +addr +3 +forces +3 +sales +3 +alink +3 +fontfont +3 +6521715966 +3 +mentioned +3 +stylecolor5f800dclick +3 +192168110 +3 +brbfont +3 +comment +3 +105 +3 +feet +3 +determined +3 +books +3 +dollars +3 +xbarracudabrltag +3 +quicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +3 +stopped +3 +perfect +3 +become +3 +regular +3 +htmlimageratio02 +3 +location +3 +keeping +3 +ideal +3 +willing +3 +sa275bhl +3 +httpslistssourceforgenetlistslistinforazorusers +3 +japanese +3 +alexander +3 +51 +3 +uses +3 +latter +3 +stylestyle +3 +width160 +3 +wasnt +3 +british +3 +rates +3 +mozilla +3 +discount +3 +items +3 +martin +3 +actions +3 +impact +3 +the20 +3 +update +3 +note +3 +69 +3 +shared +3 +plan +3 +xbarracudaenvelopefrom +3 +earlier +3 +1px +3 +fgout1718googlecom +3 +worry +3 +real +3 +dvds +3 +mta +3 +hours +3 +understand +3 +sa148a +3 +events +3 +supplied +3 +ave +3 +session +3 +configure +3 +al +3 +youll +3 +always +3 +longer +3 +log +3 +required70 +3 +color3dffffff +3 +gmt +3 +met +3 +down +3 +jack +3 +reports +3 +comic +3 +chips +3 +century +3 +size3d2 +3 +webmasterefiie +3 +operations +3 +wholesale +3 +colorffffff +3 +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +3 +policyabr +3 +proper +3 +requests +3 +women +3 +concluded +3 +micalgpgpsha1 +3 +mailfollowupto +3 +arsasha1 +3 +abuse +3 +leave +3 +03 +3 +names +3 +last +3 +sometimes +3 +ii +3 +bssiguanasuicidenet +3 +captured +3 +key +3 +towards +3 +pnbspp +3 +dear +3 +men +3 +hoping +3 +spamassassintalkadminexamplesourceforgenet +3 +shop +3 +dhcp +3 +17230098 +3 +performance +3 +three +3 +cable +3 +pdf +3 +general +3 +showing +3 +140 +3 +mac +3 +mark +3 +pcs +3 +unwanted +3 +join +3 +cool +3 +cause +3 +engineering +3 +football +3 +remote +3 +front +3 +secrets +3 +degrees +3 +remained +3 +away +3 +215 +3 +often +3 +offering +3 +eventually +3 +living +3 +positions +3 +spain +3 +thunderbird304 +3 +addresses +3 +height10br +3 +instructions +3 +cvfnqqhiggoxky895ni2p06ys7q26btysl77b0nrg5xa +3 +bigger +3 +alternative +3 +localhostlocaldomain +3 +dvd +3 +road +3 +asked +3 +helodellbuntulocalnet +3 +singapore +3 +interface +3 +york +3 +struggle +3 +ignore +3 +borderleft +3 +600600016788 +3 +3px +3 +played +3 +inner +3 +xbarracudabwlip +3 +166 +3 +known +3 +shopping +3 +brand +3 +six +3 +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +3 +solution +3 +short +3 +light +3 +told +3 +66187233211 +3 +mating +3 +ultimate +3 +eest +3 +method +3 +side +3 +specific +3 +browser +3 +board +3 +checking +3 +xinjectedviagmane +3 +air +3 +v1410 +3 +mix +3 +worked +3 +nice +3 +commercial +3 +usb +3 +small +3 +energy +3 +ok +3 +ot +3 +section +3 +havent +3 +seen +3 +g +3 +pop3 +3 +present +3 +features +3 +murphy +3 +mailtoexmhusersspamassassintaintorg +3 +064 +3 +kmail1132 +3 +concerned +3 +mailtodebiankderequestlistsdebianorgsubjectsubscribe +3 +considered +3 +worlds +3 +contacts +3 +clients +3 +warm +3 +youd +3 +xmailscanner +3 +115 +3 +zzzzexamplecom +3 +river +3 +processes +3 +admin +3 +handle +3 +maintain +3 +remove +3 +java +3 +areas +3 +confident +3 +gif +3 +listmanspamassassintaintorg +3 +ever +3 +mailer +3 +focus +3 +depends +3 +ill +3 +essential +3 +population +3 +nov +3 +environment +3 +size3d1 +3 +81168112 +3 +cult +3 +a20 +3 +messaging +3 +filed +3 +choice +3 +five +3 +width5 +3 +whole +3 +msgidfrommtaid +3 +command +3 +mount +3 +topmargin0 +3 +broken +3 +uswsflist2sourceforgenet +3 +cdo +3 +friend +3 +forward +3 +penned +3 +httpslistmanspamassassintaintorgmailmanprivateexmhusers +3 +definition +3 +america +3 +legs +3 +appointed +3 +consider +3 +watch +3 +sequence +3 +ryan +3 +guy +3 +upgrade +3 +101428120 +3 +posted +3 +shell +3 +1023115130 +3 +ideas +3 +turned +3 +articles +3 +national +3 +opportunities +3 +previously +3 +word +3 +music +3 +apply +3 +style3dborderright +3 +msgidfrommtaheader +3 +interactive +3 +centera +3 +research +3 +fix +3 +ipad +3 +class +3 +video +3 +server +3 +razor +3 +presented +3 +sa154hl +3 +52 +3 +north +3 +accredited +3 +intmx1corpspamassassintaintorg +3 +282 +3 +1alnuq0007hv00 +3 +feel +3 +privacy +3 +nolayer +3 +whats +3 +score101 +3 +others +3 +player +3 +feature +3 +thousands +3 +width3d95 +3 +pbayes +3 +airkvbgf3v46fcoh1nyafmw17 +3 +wood +3 +2px +3 +actual +3 +status +3 +computing +3 +bouncedebiankdemlsubscribertechcsminingorglistsdebianorg +3 +aircraft +3 +agree +3 +nt +3 +safety +3 +gespr111chen +3 +lerleramilerctrorg +3 +obviously +3 +thursday +3 +101431608 +3 +providing +3 +later +3 +creation +3 +curious +3 +managed +3 +generated +3 +httpspamgwcsminingorg8000cgimodmarkcgi +3 +namedescription +3 +stan +3 +maingmaneorg +3 +campaign +3 +interesting +3 +boost +3 +231 +3 +exmhusers +3 +package +3 +division +3 +coming +3 +aligncenterfont +3 +iso88591qptbs5oo9toumhbdke23otci0a093cbil8dtogzk0bf2yq35wsewvyet2e +3 +3d +3 +except +3 +geneva +3 +debianuser +3 +discussions +3 +developers +3 +wanted +3 +seem +3 +16 +3 +gw2csminingorg19216818251 +3 +sven +3 +england +3 +rob +3 +ffffff +3 +ireland +3 +adam +3 +allows +3 +led +3 +mr +3 +speak +3 +rfc +3 +looks +3 +x11 +3 +closed +3 +faceverdana +3 +lot +3 +hardware +3 +face3dtimes +3 +credit +3 +httplistsapplecommailmanlistinfojavadev +3 +picture +3 +0530 +3 +federal +3 +ilugadminlinuxie +3 +says +3 +american +3 +press +3 +policya +3 +helodomain +3 +fork +3 +realtime +3 +huge +3 +reduction +3 +inside +3 +processing +3 +cellpadding3d2 +3 +saremsgidlong450893 +3 +unsubscribea +3 +xcmscore +3 +conditions +3 +score +3 +notice +3 +suggestions +3 +13 +3 +con +3 +httpthinkgeekcomsf +3 +likely +3 +viewing +3 +enabled +3 +financial +3 +lugh +3 +60029002180 +3 +things +3 +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +3 +ctspresumed +3 +tv +3 +gmail1 +3 +supports +3 +popescu +3 +transmission +3 +ns2egwnnet +3 +loss +3 +jmrpmjmasonorg +3 +cheers +3 +passing +3 +community +3 +81258125submit +3 +cellspacing2 +3 +demand +2 +height50 +2 +requires +2 +host66insuranceiqcom +2 +1900 +2 +detected +2 +famous +2 +raw +2 +mailtoexmhusersrequestredhatcomsubjectunsubscribe +2 +saa22715 +2 +wouldnt +2 +vessels +2 +medium +2 +easier +2 +rolex +2 +therefore +2 +readingsstrongap +2 +gnus +2 +presently +2 +tr20 +2 +abr +2 +valuable +2 +jonathan +2 +allanktocsminingorg +2 +splitsabbr +2 +took +2 +role +2 +width100img +2 +whose +2 +xpyzor +2 +staggering +2 +height38 +2 +99 +2 +aspect +2 +bgcolor000000 +2 +intended +2 +modem +2 +sa392f +2 +nobody +2 +afraid +2 +connected +2 +81258125 +2 +recommended +2 +targetblankhot +2 +httpslistmanspamassassintaintorgmailmanlistinfoexmhusers +2 +cellspacing3d2 +2 +en118ir111nment +2 +approach +2 +xbownrlfmyomadaucn8k0wwacnofetlpuiujtrc55rts7lholwgznrghtwbkesselso5wxosms +2 +zzzzasonorg +2 +lie +2 +501 +2 +truly +2 +remain +2 +score49 +2 +margintop +2 +heath +2 +sa148ahl +2 +management +2 +fontsize20px +2 +titlenewslettertitle +2 +st +2 +alone +2 +industrial +2 +third +2 +value3d304704 +2 +writing +2 +benchmarks +2 +tahoma +2 +ads +2 +boot +2 +quarter +2 +officers +2 +sty +2 +saisissez +2 +jmuseperljmasonorg +2 +buildings +2 +ig +2 +sources +2 +ubuntu +2 +nnfmp +2 +hmimeversioninreplytoreferencesdatemessageidsubjectfromto +2 +amd64 +2 +h +2 +purchase +2 +organ +2 +39 +2 +responsibility +2 +nupep +2 +avbidderfortytwoch +2 +debate +2 +score109 +2 +looked +2 +se +2 +countries +2 +deals +2 +diagram +2 +beautiful +2 +players +2 +specifically +2 +incident +2 +prove +2 +osocyla +2 +chat +2 +jump +2 +arising +2 +adding +2 +handled +2 +hindi +2 +usage +2 +kingdom +2 +bits168 +2 +mike +2 +stands +2 +beneficiary +2 +sol +2 +else +2 +s0ft +2 +seth +2 +multiple +2 +described +2 +gui +2 +uncle +2 +posting +2 +strange +2 +staff +2 +fifty +2 +came +2 +size3d1img +2 +young +2 +associates +2 +quality +2 +downtown +2 +denied +2 +mandrake +2 +claiming +2 +excess +2 +testsalltrusted +2 +alsofontfontdiv +2 +alarms +2 +sorry +2 +congress +2 +96000hz +2 +designs +2 +religion +2 +facts +2 +tough +2 +leadership +2 +conference +2 +cpunkslocalhost +2 +elections +2 +october +2 +false +2 +httplistsapplecommailmanoptionsjavadevmlsubscribertech40csminingorg +2 +investors +2 +wi +2 +servicesfonta +2 +button +2 +significantly +2 +mass +2 +communication +2 +authenticated +2 +71o +2 +rzbyisgmwxqcytabagafp3aexnczu5apzxobjlagjainnaakckburkcdtamvemvnaihqafjqbt +2 +sony +2 +urncontentclassesmessage +2 +hrefhttpclickthruonlinecomclickqc2oxxlquedqhem4sgvnxxfzciuinr +2 +sequences +2 +alink3d663366 +2 +tabletd +2 +phy115ic115 +2 +shouldnt +2 +name3dstate +2 +paper +2 +tired +2 +explorer +2 +tbody +2 +namekeywords +2 +bsfsc2sa031 +2 +territory +2 +unavailable +2 +sunday +2 +forgotten +2 +trust +2 +relatively +2 +border3d1 +2 +helodynamicipaddr +2 +httplistsfreshrpmsnetpipermailrpmzzzlist +2 +trtdimg +2 +premier +2 +1022364205 +2 +crew +2 +clicks +2 +radiation +2 +href3dhttpwwwcomicscomwebmail +2 +1995 +2 +happen +2 +statistics +2 +objects +2 +earn +2 +et +2 +perhaps +2 +deserve +2 +4be972907010109comcastnet +2 +pr0nsubject +2 +bz3applecom +2 +ratesfontbtd +2 +solved +2 +contrary +2 +bsfsc5mj3761 +2 +achieve +2 +forwarded +2 +engines +2 +height3 +2 +intentionally +2 +antarctic +2 +align3dleft +2 +listmanager4084623190753220020820121844mothlightfastmailfmls1sendoutmailcom +2 +size3d50 +2 +pig +2 +laptop +2 +drug +2 +whereupon +2 +httpwwww3orgtrhtml4loosedtd +2 +lsmtp +2 +worth +2 +lies +2 +k +2 +ca +2 +guaranteed +2 +thinking +2 +fontsize11px +2 +possibly +2 +styleheight +2 +benefits +2 +written +2 +ac +2 +bsfsc1tg070 +2 +mailtorpmzzzlistrequestfreshrpmsnetsubjecthelp +2 +king +2 +542 +2 +located +2 +visiting +2 +nbsp +2 +links +2 +ask +2 +hurry +2 +config +2 +afkkriqjqggea10 +2 +00 +2 +valign3dtop +2 +partitions +2 +teens +2 +ctl00ctl00ctl00ctl00rcrbsblnkcategoriesctl01linksctl01link +2 +wmv +2 +jobs +2 +enemy +2 +towns +2 +movie +2 +bsfsc2sa154 +2 +neither +2 +round +2 +httpusclickyahoocompt6ybbnxieaamvfiaa7gsolbtm +2 +rel97ti118ely +2 +lake +2 +125 +2 +attended +2 +f117n100ing +2 +degree +2 +edge +2 +0xbb +2 +zdnet +2 +died +2 +stronga +2 +httpsourceforgenetmailarchivesforumphpforumspamassassindevel +2 +42 +2 +brief +2 +caribbean +2 +posts +2 +target3dblank +2 +permanent +2 +average +2 +easytouse +2 +egwnnet +2 +giants +2 +deltacsmuozau +2 +supply +2 +steve +2 +won +2 +paid +2 +certification +2 +imre +2 +htmlwithoutplaintext +2 +boingboing +2 +repair +2 +taken +2 +httpslistssourceforgenetlistslistinfospamassassinsightings +2 +practices +2 +wonder +2 +razorusers +2 +generations +2 +taking +2 +bsfsc2sa154a +2 +bgcolorff0000 +2 +testsawlemailattributioninreptoknownmailinglist +2 +president +2 +largest +2 +entry +2 +autoprotectionextensions +2 +kent +2 +kmail199 +2 +fact +2 +block +2 +ns3csminingorg +2 +shui +2 +integrate +2 +bgcolor3d660000font +2 +accessible +2 +fear +2 +020 +2 +madame +2 +referred +2 +dmzfirewall +2 +videos +2 +giorno +2 +size3d2fontnbspdiv +2 +onbr +2 +modified +2 +15pt +2 +farquhar +2 +bsfsc0sa290rn +2 +mt +2 +color3d +2 +ceo +2 +goal +2 +me9vbiyuafmb3fwipmjymnvlgecahkspmon4ihqlqbelxsodskupaodczkgjygo3u571njasrgn +2 +pcie +2 +representative +2 +size3d2emailfontbtd +2 +ray +2 +railway +2 +task +2 +sending +2 +flawed +2 +width430 +2 +textdecorationnone +2 +levels +2 +pharmacy +2 +exmh +2 +bfontp +2 +11px +2 +xgmanenntppostinghost +2 +hotels +2 +name3dhdnrecipienttxt +2 +footer +2 +includes +2 +beginning +2 +tuesday +2 +256256 +2 +leader +2 +m10grpsnvyahoocom +2 +2418 +2 +dates +2 +bvlgarirolexdior +2 +kyzeqoz8g1ojrmwopejnjelbhzgkmwopdebqwz5u4ic3d1c4cfnvmrqqpjzdgzyvv +2 +carolingian +2 +minor +2 +field +2 +claimed +2 +je +2 +findings +2 +continental +2 +a40abr +2 +hmimeversiondatemessageidsubjectfromtocontenttype +2 +winter +2 +marginheight3d8 +2 +21618927135 +2 +quotedemailtextreferencesspamphrase0001 +2 +marker +2 +shares +2 +releasing +2 +assist +2 +attorney +2 +ends +2 +collecting +2 +moines +2 +french +2 +httpslistssourceforgenetlistslistinfospamassassintalk +2 +helodynamicipaddr2 +2 +icq +2 +theories +2 +saturday +2 +applications +2 +bfont +2 +harris +2 +cambridge +2 +continues +2 +frozen +2 +snip +2 +rugby +2 +producing +2 +cypherpunksoutgoing +2 +irish +2 +ana +2 +xhabeasswe7 +2 +htmlmessage1 +2 +weight +2 +camp +2 +police +2 +continued +2 +ability +2 +28003200 +2 +razorusersexamplesourceforgenet +2 +viruses +2 +strategy +2 +kn111wn +2 +hat +2 +liberal +2 +rich +2 +200 +2 +treviso +2 +situation +2 +1002627 +2 +mailtorazorusersrequestexamplesourceforgenetsubjecthelp +2 +spamassassintalkadminlistssourceforgenet +2 +20020918 +2 +culture +2 +mailtorazorusersrequestlistssourceforgenetsubjectsubscribe +2 +and20 +2 +bsfsc0sa119a +2 +military +2 +messnum +2 +router +2 +neunet +2 +rootlughtuathaorg +2 +window +2 +maximum +2 +gafioc4f0gmdscaqvgfmp5zviyg0uaaidqrzgidcfdzgwgguycgkqebgkfyutqpn2lfvtpdjda +2 +80 +2 +sp +2 +475 +2 +products +2 +progress +2 +winner +2 +smoke +2 +printers +2 +dated +2 +width375 +2 +fats +2 +kind +2 +pto +2 +hrefhttp254118gsd423150mcomadulttoyhtmlclick +2 +mailin11applecom +2 +peace +2 +bouncedebiansecuritymlsubscribertechcsminingorglistsdebianorg +2 +introduce +2 +exclusive +2 +bsfsc5sa161f +2 +ftp +2 +product +2 +shareholders +2 +useless +2 +fwd +2 +replytypeoriginal +2 +463 +2 +insist +2 +occurs +2 +languages +2 +pulling +2 +rootlocalhost +2 +bracht +2 +occurredpine +2 +play +2 +desktop +2 +evolution +2 +srchttpc2fbpoxokuwcnspacergif +2 +issuing +2 +approved +2 +accucast +2 +codes +2 +mar +2 +served +2 +16th +2 +dotancohencsminingorg +2 +hash +2 +sheet +2 +artists +2 +relevant +2 +paddingright +2 +screen +2 +mistake +2 +hibody +2 +anas +2 +2009title +2 +hype +2 +firm +2 +daniel +2 +286 +2 +100estr117ya +2 +xbarracudaheaderalert +2 +delivery +2 +diamond +2 +104 +2 +bird +2 +characteribus +2 +justin +2 +christmas +2 +empire +2 +aggressive +2 +messagesfontfont +2 +metal +2 +spamassassindevellistssourceforgenet +2 +moneyback +2 +value65000000650000option +2 +e9crit +2 +200910251104n9pb4osg025054ns2csminingorg +2 +july +2 +whilst +2 +rssfeedsexamplecom +2 +versions +2 +mo6541216221staembarqhsdnet +2 +display +2 +arranged +2 +club +2 +colspan3d5 +2 +unsolicited +2 +spamtrap +2 +srchttphomecnetcombgif +2 +divdiv +2 +addressed +2 +needs +2 +missing +2 +carpeta +2 +thomas +2 +augdlistsapplecom +2 +execute +2 +relaydubt31nwcgroupcom +2 +persons +2 +formerly +2 +10024 +2 +width3d200 +2 +west +2 +fixed +2 +replicawatch +2 +httpiguanasuicidenet +2 +defaults +2 +weekend +2 +pickup +2 +232 +2 +studios +2 +widespread +2 +eg +2 +allums +2 +assume +2 +module +2 +claims +2 +eastern +2 +selfrule +2 +agreements +2 +quotes +2 +summary +2 +sells +2 +rh +2 +decay +2 +straight +2 +lee +2 +polish +2 +tall +2 +wait +2 +re102lect97nce +2 +artist +2 +weather +2 +co +2 +backgroundcolor +2 +grokked +2 +height3d337br +2 +mirror +2 +zzzzilugjmasonorg +2 +firewalls +2 +httpnewscomaucommonstorypage040575037762255e1376200html +2 +borderright +2 +lines +2 +height3d1 +2 +knew +2 +emphasis +2 +jpg +2 +spnews +2 +blog +2 +class3d +2 +owners +2 +land +2 +internal +2 +20100521152315ga5433googlecom +2 +transactions +2 +impossible +2 +robertgtaylorcsminingorg +2 +highlight +2 +78701 +2 +properly +2 +055 +2 +safe +2 +mailtospamassassintalkrequestlistssourceforgenetsubjectsubscribe +2 +console +2 +grow +2 +asking +2 +identify +2 +scheme +2 +wh +2 +sound +2 +2000 +2 +81128112 +2 +ranch +2 +rdnsdynamic01 +2 +familiar +2 +hrefhttpwwwecobuildercomlockergnomecfmecobuilderabr +2 +x8664pclinuxgnu +2 +hqpronsnet +2 +additional +2 +httpwwwgeocrawlercomredirsfphp3listrazorusers +2 +palo +2 +themselves +2 +gain +2 +main +2 +monday +2 +stylepaddingright +2 +rssfeedsspamassassintaintorg +2 +shown +2 +clamav +2 +placing +2 +term +2 +pa +2 +device +2 +typetextcss +2 +nmh104 +2 +srchttpwwwzdnetcombgif +2 +instantly +2 +fee +2 +9px +2 +originally +2 +edit +2 +width50 +2 +figures +2 +hosting +2 +heat +2 +split +2 +behind +2 +i386redhatlinux +2 +thinks +2 +editor +2 +isnt +2 +define +2 +chief +2 +bsfsc0sa148a +2 +half +2 +tour +2 +belong +2 +minded +2 +214 +2 +delayed +2 +isoiec +2 +colspan3 +2 +tom +2 +ps +2 +sounds +2 +undervalued +2 +imho +2 +felt +2 +quicktime +2 +ff +2 +profitability +2 +translation +2 +exmhusersadminredhatcom +2 +es +2 +appeal +2 +egp +2 +paddi +2 +revolutionary +2 +225551 +2 +forever +2 +316 +2 +grand +2 +goes +2 +registered +2 +flag +2 +foreign +2 +myself +2 +reference +2 +attention +2 +officials +2 +tls +2 +x8664 +2 +discounts +2 +court +2 +meet +2 +select +2 +committed +2 +andstrong +2 +excessive +2 +width3d60 +2 +iso88591qyjru4abtypxviyxqnqlrxwtcsj0a096gudsnwt0nxblkvuwjblkwvjr +2 +heavy +2 +aaaaaqaaaua +2 +splitting +2 +xgreylist +2 +hotel +2 +tells +2 +craigdeersoftcom +2 +space +2 +xmras +2 +10223513 +2 +v11 +2 +slrnhsjcc4bu7liampotooledipsyselfiporg +2 +plate +2 +merely +2 +10000 +2 +12440203452da100a60000w4twrl +2 +potent +2 +bespanbspan +2 +nehru +2 +lost +2 +alignrightfont +2 +study +2 +052 +2 +southwest +2 +spamassassintalk +2 +printing +2 +economy +2 +50abr +2 +marry +2 +sm1 +2 +hold +2 +recycling +2 +vancouver +2 +qyk5 +2 +datwinkdaddy +2 +pan +2 +019 +2 +homes +2 +16px +2 +part +2 +notification +2 +widt +2 +industry +2 +cypherpunksforwarddspronsnet +2 +mode +2 +0102 +2 +2482 +2 +romance +2 +authors +2 +gary +2 +table20 +2 +attach +2 +properties +2 +age +2 +advanced +2 +000044124742000066f100007dd5jerrysdodgecom +2 +extremely +2 +operating +2 +outside +2 +plane +2 +server2 +2 +189 +2 +muscle +2 +discover +2 +reliable +2 +accepted +2 +bytes +2 +advertisement +2 +align +2 +killing +2 +final +2 +subscribe +2 +width599 +2 +engine +2 +anymore +2 +220233140121 +2 +wounded +2 +suicide +2 +httpwwwshagmailcomunsubhistoryhtml +2 +california +2 +pageafont +2 +charsetwindows1252 +2 +gamasutra +2 +bush +2 +bs +2 +smtp4cybereccom +2 +100elights +2 +friday +2 +2001fonthr +2 +canadian +2 +membership +2 +band +2 +dghlbmvkdqoncnjlchjlc3npbmcncg0kzgltaw5pc2hlcw0kdqpucmvhdg1lbnqncg0kz2vuzxrp +2 +aikctkhd0fzma10 +2 +color3d000000 +2 +messageidreferencesmimeversioncontenttypecontentdisposition +2 +download +2 +tdtrtable +2 +audio +2 +ba +2 +texas +2 +symbol +2 +clumsy +2 +mx1csminingorg +2 +inhabitants +2 +harri +2 +aes256sha256 +2 +cars +2 +payment +2 +sides +2 +australian +2 +structure +2 +mapping +2 +1712811332 +2 +bad +2 +193120211219 +2 +completely +2 +jan +2 +jmdeadbeefjmasonorg +2 +tasks +2 +norte +2 +sc +2 +10pxpowered +2 +supported +2 +50000 +2 +types +2 +san +2 +ehmadscientistcom +2 +24th +2 +communities +2 +satalk +2 +masterlistsapplecom +2 +uswsffw2sourceforgenet +2 +apollo +2 +013 +2 +alignright +2 +advice +2 +tried +2 +celebrate +2 +workaround +2 +provider +2 +parliament +2 +tree +2 +historians +2 +itself +2 +gocomimagesbtp2pgifccode3d1d74c2eapcode3def39755c +2 +rv11 +2 +javadevbouncesmlsubscribertechcsminingorglistsapplecom +2 +enable +2 +nametopa +2 +arab +2 +turk182chipwarenet +2 +nbspa +2 +inventory +2 +collapse +2 +nnggttffexpiryformalsiemensoldliabilityminimumsitiosbenefitbackedsmoothiescientificfollowswoodspointsphysiciansagainjewelledbedhasamhalloholdensureattractedwitnessed +2 +celejar +2 +abbian111 +2 +man +2 +pressure +2 +knows +2 +typemultipartalternative +2 +keyboards +2 +stylecolor003c78 +2 +superior +2 +growing +2 +mutual +2 +photography +2 +impinging +2 +existing +2 +aptget +2 +style3dfont +2 +cypherpunkseinsteinsszcom +2 +expand +2 +81168116egwn +2 +pin +2 +width85 +2 +v2012 +2 +save +2 +transport +2 +overview +2 +1000725 +2 +whatever +2 +northern +2 +mgbdebianyosemitenet +2 +1023139131 +2 +fight +2 +v106 +2 +exact +2 +reboot +2 +base64 +2 +mouse +2 +73 +2 +popmart +2 +22240 +2 +750 +2 +centernumbers +2 +sleep +2 +brother +2 +backup +2 +himself +2 +altsuperior +2 +founded +2 +fraction +2 +185717 +2 +dinter +2 +immediately +2 +37d60f6e688115b15863026nmzodjeteh6411613266 +2 +availablebr +2 +separate +2 +paddingtop +2 +steep +2 +bonus +2 +bsfsc0sa275bhl +2 +bsfsc0satofromaddrmatchhl +2 +mov +2 +readersbfontbr +2 +hibodycsminingorgbr +2 +mailtorazorusersrequestlistssourceforgenetsubjectunsubscribe +2 +score119 +2 +disclose +2 +contentclass +2 +beyond +2 +inquiries +2 +commitments +2 +middle +2 +happy +2 +smtpmailaugdbouncesmlsubscribertechcsminingorglistsapplecom +2 +tocww6lmavwpx3xnrwsxipjrspeotjp2ntzrzjatk7m4wk4pxcie3zddm8mrrm0sni4pwkpwocbk +2 +concealed +2 +obliged +2 +tim +2 +transmit +2 +merciadri +2 +sentence +2 +shot +2 +zzzzlocalhostspamassassintaintorg +2 +reads +2 +deliver +2 +rd +2 +changing +2 +smtpsvc5021952966 +2 +output +2 +testsawlknownmailinglistnospamincreferences +2 +influence +2 +opportunity +2 +color3d0000ff +2 +marginwidth0 +2 +fun +2 +training +2 +athletic +2 +info +2 +colorff6600bgratitudes +2 +authority +2 +couple +2 +archives +2 +plant +2 +standards +2 +snow +2 +stations +2 +concern +2 +major +2 +stated +2 +cipherrc4md5 +2 +chairman +2 +investorbr30day +2 +marketing +2 +shbig505413body104 +2 +aware +2 +fax +2 +avenue +2 +numerous +2 +vga +2 +works +2 +2098516147 +2 +writers +2 +4152010 +2 +detailed +2 +2440 +2 +reconnect +2 +ratwaregeckobuild +2 +webnotenet +2 +white +2 +quinlanpathnamecom +2 +specials +2 +door +2 +promises +2 +sees +2 +esmtpa +2 +disabled +2 +size2a +2 +haringey +2 +rv1919 +2 +dark +2 +bz1applecom +2 +settings +2 +razorusersadminexamplesourceforgenet +2 +ordinary +2 +education +2 +zzzzteana +2 +rwrr +2 +titlecnet +2 +answers +2 +ttype +2 +debts +2 +366 +2 +purpose +2 +seminar +2 +2500 +2 +improve +2 +confiscated +2 +speed +2 +testsdnsfromrfcwhois +2 +forgot +2 +ptdtd +2 +per +2 +switch +2 +fonta20 +2 +reach +2 +predators +2 +night +2 +southeastern +2 +iii +2 +5638e5ddae35font +2 +god +2 +6219710167 +2 +scrutiny +2 +cif +2 +prix +2 +c2b9 +2 +peoplesofta +2 +color3dff0000 +2 +gtk +2 +archived +2 +installed +2 +bst +2 +1984 +2 +frommxmatchesnothelodomain0 +2 +reg +2 +aanlktilqtvstnerew1lmnlsyto13zpxsygphtejgivmailcsminingorg +2 +graphics +2 +kst +2 +sansserif20 +2 +difficult +2 +stable +2 +1978 +2 +cell +2 +tested +2 +mail1insuranceiqcom +2 +tapes +2 +leading +2 +isnbspqqqqqqqqqqzdnetspamassassintaintorgfontp +2 +r2g81c921f31004260456z3c6f41ddg86e45cdae1257104mailcsminingorg +2 +shorter +2 +machines +2 +removing +2 +current +2 +fontweightbold +2 +sex +2 +crashes +2 +10br +2 +tolerance +2 +charges +2 +fontweightboldsaleatd +2 +purchased +2 +ten +2 +asia +2 +size4tell +2 +httpslistmanexamplecommailmanlistinfoexmhusers +2 +capture +2 +exciting +2 +bgcoloreeeeee +2 +catholic +2 +examples +2 +track +2 +sune +2 +albany +2 +1721652254 +2 +po +2 +administration +2 +peter +2 +unverified +2 +fully +2 +programs +2 +walkiewicz +2 +900 +2 +garden +2 +charged +2 +dario +2 +tax +2 +figure +2 +estate +2 +alink0033cc +2 +bebusinessbbr +2 +alimony +2 +xauthorityanalysis +2 +italy +2 +ways +2 +ghnrg +2 +060026 +2 +evidence +2 +mechanism +2 +usenetdoughgmaneorg +2 +fed +2 +making +2 +response +2 +canada +2 +color3dff0000models +2 +johnstebbingcsminingorg +2 +fort +2 +helorovervipulnet +2 +mnenhy076666 +2 +answer +2 +255 +2 +fontweight +2 +reservation +2 +cash +2 +recent +2 +xhabeasswe5 +2 +servers +2 +waste +2 +reporting +2 +comfortable +2 +name3dnoknokr2c2atdtrtbodytabletdtrtbodytable +2 +spamassassintalklistssourceforgenet +2 +testsgmailhtmlmessage +2 +12th +2 +ensure +2 +mbr +2 +knowledge +2 +experimental +2 +1eitlxvlay2zpas6htg4dak4yzfjbfmo7vfi66uufdmsf9u18yxwvtgzkqlbonpl7kxvblx42trh +2 +magnitude +2 +background3dhttpwwwcomi +2 +115pecul97rly +2 +isbn +2 +undertake +2 +constantly +2 +transaction +2 +16pt +2 +opera +2 +shame +2 +postgrey131 +2 +airline +2 +weekly +2 +percei118ing +2 +optin +2 +modules +2 +papers +2 +divspan +2 +1921681100 +2 +designed +2 +336 +2 +defaultspcmsurround51device +2 +stylepadding +2 +rest +2 +exmhworkerslistmanredhatcom +2 +specified +2 +finally +2 +garymcanadacom +2 +111p111sici +2 +transmitted +2 +selected +2 +588432 +2 +sid +2 +zdnetbfontbr +2 +east +2 +81 +2 +submit +2 +fffsecuritya +2 +name3dcity +2 +plays +2 +anderson +2 +wastemindernet +2 +social +2 +mailqy0f175googlecom +2 +flags +2 +ezine +2 +buried +2 +somehow +2 +debiankdelistsdebianorg +2 +humans +2 +alignrighta +2 +96 +2 +uswsflist1sourceforgenet +2 +height3d30 +2 +mailinglist +2 +leerlingen +2 +exmhworkers +2 +scientists +2 +popup +2 +entrepreneurs +2 +dangerous +2 +clearance +2 +act +2 +104904 +2 +organizing +2 +youve +2 +width4 +2 +replica +2 +charset3dusascii +2 +held +2 +powered +2 +invaliddate +2 +hen +2 +byte +2 +17189895746 +2 +affects +2 +stylewidth +2 +ve +2 +bsfsc5mj1840 +2 +size1laws +2 +p111117rrez +2 +zero +2 +1267126689 +2 +httpnewmarksdoorblogspotcom +2 +xapparentlyto +2 +receipt +2 +regions +2 +spamassassintaintorg +2 +european +2 +cellpadding5 +2 +mine +2 +urgent +2 +mdocabletv305 +2 +height3d5td +2 +6b9e313a4747 +1 +looms +1 +usrsrclinux +1 +diseases +1 +worried +1 +src3dhttpiiqusimagestba0729200207293gif +1 +ksdiz6cwz29mu09zczhsodn8b8gl4cpslc5wndy39dg77cvslc0kcoshssf9wkmdj +1 +hrefhttp72cfnodoruccndudoky3c7351c62216edb9eaf5efcejjhoql274231831104886586939978zmhvbgojprkjo +1 +former +1 +meab +1 +64286735 +1 +daylight +1 +drunken +1 +localhostcsminingorg +1 +ht +1 +ewy6 +1 +growth +1 +seiner +1 +facehelvetica +1 +weightbr +1 +rigorous +1 +2de4213a5b3e +1 +founding +1 +humm +1 +huntsville +1 +alt3dmore +1 +hrefhttpwwwgnomedexcomimg +1 +webmasterelectrowebcom +1 +myzme65geyo +1 +412222002733161846763r2d2 +1 +afontfont +1 +132624 +1 +finalist +1 +datapower +1 +frankly +1 +size2flinstonesfonttd +1 +172642 +1 +ace +1 +e11si4648635fga820100426084457 +1 +103719 +1 +menu +1 +tub +1 +b36mr626059muo561271387385372 +1 +980 +1 +newsphere +1 +malden +1 +lua +1 +w20mr1219671ebn491272147491918 +1 +satisfy +1 +bizsmtp +1 +2165419436 +1 +spent +1 +brwhy +1 +hrefhttpclickthruonlinecomclickq18whdzi987phkswbgtimeyr1n675ir +1 +hrefhttp69f32brownchooseru +1 +seko +1 +1258025765 +1 +frustrating +1 +referencesspamphrase0001tolocalparteqreal +1 +65553480 +1 +20020829 +1 +welcome +1 +thosep +1 +bwhyb +1 +10204625 +1 +noneuropean +1 +slacknet +1 +204709 +1 +titled +1 +gap +1 +164137 +1 +12742049512021118camelfamilypaciferacom +1 +width3d141 +1 +fhoflnpfdhnl0u3noskroxgs1r4165spnxszqalhuuxkwfwgyrpoakpm9codumwmy +1 +adjusts +1 +carry +1 +opening +1 +alinkcc0000 +1 +mobo +1 +11fe0d0bb7cc2ae000000813744bc4c8fe301d +1 +1014118715 +1 +3141 +1 +141842 +1 +costbenefit +1 +adler +1 +btwwpo9f291xkktzssqtutlxf9t3aheqqlttyyvbznq2rmdhgsqlxtt99xnsf3tapm +1 +testsawlinreptoknownmailinglistplingquery +1 +080156 +1 +strongmiss +1 +anytimeb +1 +disable +1 +083317 +1 +moderate +1 +cccccc +1 +ã +1 +electoral +1 +cannonschart +1 +revoke +1 +httpbugsdebianorgcgibinbugreportcgibug3d366797 +1 +115111 +1 +iusrsrcredhatbuildalsadriver090rc3include +1 +noelamaccsminingorg +1 +advance +1 +httpwwwbarracudacomreputationip1872387149 +1 +operationthe +1 +score96 +1 +6ebzto5ezlavfbfm5ulbliszt +1 +zzvjznhtirfrna2ujyp503cayznofw0smolng9qwbbrsnps5llybbc5safkt4wcfxsck1pqa1rc +1 +valuemgmadagascaroption +1 +emdynasties +1 +testsawl +1 +hrefhttpclickthruonlinecomclickq9dkkcwqv0hoccbmdnuklvd75wkzzpr +1 +timbila +1 +thanhnaclightingcouk +1 +bsfsc0tg074k +1 +reminiscent +1 +import +1 +ij6mbqpcpm2w2ie8l2zvbnqpc9wpg0kicagicagphagywxpz249imnlbnrlciipgzvbnqgzmfj +1 +china +1 +hk7xso4vm1lmmuqscmwoj9tebbkzbgf4hen4rzmpr8vkvne98hppelri0cwhrf1qmu4vuw5csyeawyqlcj7acbcwmjhsawvildbirbqobzexiov2f8nfq7yatbdwrfifc4s5sbcrogsmku0eoereovahdfhhgiykv0ku9qxkqomywa5wjqqkfpimr2bbegc70wyo2ymzipjhhsvhhat99ezdefepy3hs5bklsk3z1rkjydngtcbbgh2 +1 +formula +1 +b1fgx1awjrvxvthsufnh4e2wvhhdqewv1jat3dgj2vz2pkcmvfnvj5aiwbxrndqayuqou5g +1 +0667713a56dc +1 +gmailid1284e4b5df252bb7 +1 +width478 +1 +183740 +1 +href3dhttpb93d7csorowxfcn +1 +10300286476462tmdadeepeddyvirciocom +1 +t14mr402106fan851273255343636 +1 +4fontfontbblockquote +1 +151150 +1 +17hwir00006r00 +1 +limit +1 +existence +1 +1015 +1 +17o2rz0005ts00 +1 +httpftpczdebianorgdebianpoolmaineeglibclibc621026amd64deb +1 +htmltagexisttbody +1 +href3dhttp6767ydeceyercnyenaqaop3d6767b54ba85639caff829z +1 +sa392f2 +1 +fontcenter +1 +1921683267 +1 +sgo +1 +aligncentersomep +1 +alink3d006600 +1 +backgroundcolorffffff +1 +describes +1 +canbsp +1 +user2injgi2dslmindspringcom +1 +pursuing +1 +sourceloopaes +1 +114625 +1 +ifontbfontpblockquoteblockquoteblockquotetdt +1 +subscriptions +1 +indiv +1 +httpwwwdigitalmediatreecomjimweblog +1 +sponsor +1 +6pwrvo6d6qer +1 +sourcefontbr +1 +octoberthe +1 +prevent +1 +talking +1 +vm7090206 +1 +hrefhttpclickthruonlinecomclickqf2ozvtquwwqrqaoa0pvvdi5z7ednr +1 +llcbr +1 +conspiracy +1 +gmailid1289c3a90444d06f +1 +mother +1 +emaem +1 +kde443 +1 +0lvebg1nddvg2ykg010var +1 +m0620212mail1csminingorg +1 +distinction +1 +4th +1 +felicityklugenet +1 +link3dffffff +1 +2009strong +1 +dictatorship +1 +color3dyellowbcenter +1 +suspicions +1 +204 +1 +walt111n +1 +i6illcsxhi0zxzz775m8b82mbss036v1cr6f7gfqetwpo1fqfx0wbjpopw4fuk6l29g8q62hxt +1 +pwvk4icljsbiwvujidyeaojne5xvnfehsiy5aozidj86s1syrlj2gjigmyrap5cuf4bx30pof +1 +dslbased +1 +nil +1 +macro +1 +df8696c +1 +tulsa +1 +ignorant +1 +frank +1 +nonvoting +1 +millionusd +1 +recession +1 +105824 +1 +xessionerrors +1 +pregnantlooking +1 +nknwdz0v8awzantlwcsf2dexclj4qr6bzm8b4ud1mnkn8az0oahuy8akn9nxsj +1 +suuurreeee +1 +tackle +1 +changes +1 +production +1 +reward +1 +fs +1 +catches +1 +borders +1 +fit +1 +agentpm +1 +feelings +1 +le97118ing +1 +1618 +1 +superuseronly +1 +border0abrtd +1 +tvpredictionscomb +1 +1264975042 +1 +href3dhttpwwwa +1 +noelamac +1 +brdivdiv +1 +000459 +1 +151627 +1 +nospamvuoreladk +1 +decorated +1 +nevertheless +1 +knowingly +1 +90 +1 +elsewhere +1 +vormkrijgers +1 +mailmsnjabber +1 +namelist0104 +1 +aad +1 +httpnewsbbccouk1hi2210091stm +1 +resources +1 +1011153 +1 +frommxmatchesnotunvrhelodomain16 +1 +24e1616f03 +1 +httpwwwlifetimetvcommoviesinfomove3002html +1 +enlt +1 +tha +1 +17ysox00048e00 +1 +tactezlaa20 +1 +65007 +1 +distruption +1 +18774518336 +1 +blsit2yfnt3ixzgw5ipzxampu1v81gpd9ls26afanhb1breugpt2xqczczsvqzqcwzxv +1 +titlecheck +1 +110241 +1 +midfi +1 +acn +1 +morepatrioticthanthou +1 +salon +1 +thecenter +1 +formulations +1 +alpinedeb20010042715341008360localhost +1 +dominant +1 +062215 +1 +rmosunnmorenet +1 +smtpmailjacquelinsteffenhagen8hlhotmailcom +1 +hoeppner +1 +n9sefysq021224 +1 +behaving +1 +nzf89gompeohy56iiivasa3ctu2xdo6d2pakeaxk3bqbtkiudtkf8ajxrswsep0yjmzoydaz +1 +isikybu3d53130353067unsubscribeanbspnbsp +1 +20020719162521805035 +1 +043701 +1 +oldschoolcompressionalgorithms +1 +gltranslatef00f +1 +churches +1 +113225 +1 +httpwwwfragmentizedcomjaykul +1 +277725 +1 +href3dmailtomaddu +1 +tellc +1 +httpemaservertrafficmagnetnettrafficmagnetwwwr100002304448423bpoq88r31zff +1 +207202171254 +1 +ia0kicagicagicagicagicagicagicagicagicagicagag91cnmupc9gt05u +1 +bottom +1 +httpmembersaolcominossencekenkeyhtml +1 +aptrpmtuxfamilyorg +1 +32836194125148311030699168squirrelspamassassintaintorg +1 +3331e13a6314 +1 +participation +1 +metaphors +1 +lus +1 +size1who +1 +fc695557d42cccad92de8ff27964ade9 +1 +pr0n +1 +fred +1 +3000 +1 +noone +1 +btomeraider +1 +offline +1 +127226628578f6cc820001w4twrl +1 +brightly +1 +diego +1 +backing +1 +firmopinionholding +1 +operation +1 +r2989529896 +1 +quickly +1 +scrolls +1 +093650 +1 +town +1 +5dd6f13a564f +1 +3015 +1 +hmimeversioninreplytoreferencesfromdatemessageidsubjectto +1 +hunderten +1 +210 +1 +yuo +1 +manuals +1 +busy +1 +keating +1 +bhb8rm7q0doza3sljid4fvbjeqe1kdpsrj0jepxe7kum +1 +titleget +1 +spacenbspnbsp +1 +hrefbuzzimg +1 +cdsjbhdaeirha3gbwzizew5hfe7kha4hjxhq2ishgq5ha3ddgq3agfyyqn8wbabqaa0arfy4au4 +1 +patternlength +1 +psyche +1 +65194124178 +1 +aycw2a8e4ojqiurts7uurgs5ojxuiot1ojj4yafibkg5p27nv92d1pxkom0crublmlzuhmz74p2 +1 +aaaaarpwkeg +1 +basil +1 +hrefhttpc2fbpoxokuwcnemqxaku2743e57decfd4aef63302dudyalnhibodycsminingorgyxjxipi1422834888982689422018 +1 +adaware7 +1 +zo4eh1mohzjthtbxldjl21s4vptelft6bbsxikpomlckkasohrukpxlflocdhso4m0ezs3dntz +1 +titlealbany +1 +giov +1 +20315833245dyniinetnetau +1 +highratio +1 +fonttdtrtr +1 +lipresentationsli +1 +a87d +1 +20020930t0033140600 +1 +770px +1 +intmx1corpredhatcom +1 +linuxx86 +1 +overhead +1 +urlhttpwwwenglishupenneduafilreis50sschleslibhtml +1 +sareadult2 +1 +a01 +1 +beta +1 +gmailid12801d634d937683 +1 +zzzzlocalhostjmasonorg +1 +hegemony +1 +sserif +1 +dateinpast12240992 +1 +classcopyimg +1 +saying +1 +1931228222 +1 +ghosts +1 +friendship +1 +ssksbpyahoocom +1 +krmv2vy3zvvxtd7hguwn0kcfaomkc0jx0yszzmqojqd7mjpwmq6qujgwna9xgzf9y6j0p23 +1 +httplistsapplecommailmanlistinfox11users +1 +invoiced +1 +serviceb +1 +demos +1 +ro +1 +analogous +1 +modestasvainiuseu +1 +fire +1 +stylecolor0000ffagournayaldebaranroboticscomadiv +1 +bgcolor3d000000 +1 +c117ri111sit +1 +fruit +1 +174402 +1 +13fonttd +1 +gw01es3egwnnet +1 +energizers +1 +20100504063239783tchateletfreefr +1 +signalservicethezscom +1 +lights +1 +typeimagexicon +1 +2czdonc9zoedud4dtj7dik3acol6myfqalzn16pg1v45lgfptesltvjm9f2kt1nedavkmve +1 +herebbr +1 +dysmeromorph +1 +ghostscriptdivdivi +1 +orirob +1 +billions +1 +poslednjih +1 +nameq +1 +caunt +1 +121001 +1 +hrefhttpclickthruonlinecomclickqa9ilbqqpuxjlucjrqz2jqzv91zvmlr +1 +360ptspan +1 +difference +1 +friendsrelatives +1 +033628 +1 +of0d +1 +psychoblogger +1 +180241 +1 +smtpmailbouncedebiannewsmlsubscribertechcsminingorglistsdebianorg +1 +strongspan +1 +titleonline +1 +050106 +1 +111126 +1 +hereptdtrtablebr +1 +subscribers +1 +ameless2gif +1 +delineatedsaalvincentdeniervergangenheitdesconfianzavikingaircraftwsxvict2edishonorabledestinedrecruitmentgiuliogoogleplexvinjettvisbygirouxvitaminavolckerdisfranchisevolviendovoorheesrepopulatecarmadinrepositorieswdgacceptedwcfarvatowhmacerbategreenvillecarnotrimbaudaquitaniaanthropometricsricordi +1 +uc +1 +blocked +1 +9psofyd3p9ld7fgar7vt7kh2hd6fzqxyak70yh9h3en2upswacvu9psofyd3p9ld7fgar7vt7 +1 +cat +1 +1382150 +1 +windows20071031 +1 +132312 +1 +ij2qav8veoemlhhbenqlarqqcxl8inygifuhqtwgteq5ewoq4icnmoeoc9led47wp0gaeaphomep +1 +947dd47c67 +1 +historya +1 +cards +1 +mautobgu200906181519197 +1 +unticking +1 +href3dhttp31ada4fdekemoscnejm3d51990ef85a6ab65c10b3oyj3d3 +1 +200208280042154bca2588matthiasrpmforgenet +1 +conflict +1 +8c44e4289 +1 +toepassing +1 +editionbr +1 +mailin12applecom +1 +glycosides +1 +210512 +1 +subglacial +1 +121406 +1 +httpeinsteinsszcomcdr +1 +hrefhttpclickthruonlinecomclickqa9ilbqqpuxnpyb8eoz2jqzv91zvmlr +1 +testsforgedrcvdtrailforfreeinvalidmsgid +1 +srchttps005radikalrui20910020aa59a15823ee5jpg +1 +cab +1 +pwuekpersonalro +1 +7xd8urcbazpaopcvhu8lbliszt +1 +numberb +1 +blcmp +1 +4bc6fab06080406studentulgacbe +1 +square +1 +hrefhttpbac70nihosslcnyxaavyaq36895as587hr8z052078190etyluiixag7060351295602543767160uuhedekypc51u0jt2z60nw0a692640einamolas27426626062224502832 +1 +editionfont +1 +powers +1 +alu1lnth3yahur64x95cp9eobvgdrcxunped84df3awrrvz7oblvvkih3vvqzktoy2sst1jjqu +1 +financi +1 +moodenhancing +1 +ba8w1if4fi94v1d +1 +brleads +1 +oceanfree +1 +pse +1 +anastos +1 +183635 +1 +flurry +1 +challenging +1 +ear +1 +xgmailreceived +1 +intranets +1 +httpgo2emicrosoft2ecom3flinkid3d9729707 +1 +webmastereiscatuitno +1 +1952419461 +1 +g99jpsl22137 +1 +routed +1 +191129 +1 +yusmallb +1 +gnome +1 +httpuseperlorgarticleplsid020913162209 +1 +052430 +1 +undeciphered +1 +bfontabr +1 +flow +1 +laughing +1 +lesser +1 +cordelia +1 +17216196 +1 +collapsed +1 +reflecting +1 +ddx9ygsej5amfx1uz63g58lvqjqvn +1 +blended +1 +adaptor +1 +newscom +1 +gray +1 +xswfyntfy6l4lkdb06kefekrfmcqohqbz8x0j4gskb2l4exycyfmtusgtuqfj80lwmrxnbopc +1 +jednom +1 +2si9362550fks1220100503102310 +1 +inspected +1 +6c9353ed4c +1 +5000 +1 +iso88591qxd86p3bwyaj25f726z5fbw7ddkm2el2423h25dudk7thshc +1 +vector +1 +percentages +1 +asks +1 +mail8101 +1 +bedford +1 +mid +1 +hate +1 +configured +1 +uonlinecomclickq3d05gdulig8pctun3efnodtxi8fiiur +1 +highrise +1 +11597me +1 +integration +1 +jeffgpronet +1 +20020917172627a1dbdc44dargotech +1 +unsubscribeoracleeblastcoma +1 +212906450 +1 +tank +1 +1921680101 +1 +223619 +1 +germany +1 +href3dhttpasayjjcqopxabuvcnkesqteaq3d0f56545f43dbf237 +1 +srchttpyl60futurexclusiveinfoimages608jpgabr +1 +k2cs40471iba +1 +6a7d743d45 +1 +vjimosz +1 +609 +1 +bxcbwpxvtlhqrhai5i4oky764kodisgbweavpqutrbgwgocd1nolbyyjf5vtj5fp3qry58we8zpt +1 +systemerroremit +1 +boundaryboundaryid57z4opjiwztvbinvv279zq +1 +argotech +1 +nate +1 +comercio +1 +color3dcc3300span +1 +ty +1 +prescription +1 +indices +1 +divrelative +1 +klabsbe +1 +hereafontbr +1 +r10cs66423wfa +1 +b5ec019dc13d626f7c3199db2d74attcanadanet +1 +size2br +1 +21313209237 +1 +httpwwwnettempscomcareerdev +1 +bgcolor3d3399cc +1 +httpseekerdicecomseekereplrelcode31 +1 +seid1zxst3evhytxkzu4gw49cv4jquptr2st6lithm2hc9qzm7izle3e99l9oluglkplmz5bqgf +1 +115318 +1 +yellow +1 +0023122 +1 +httpslistmanredhatcommailmanlistinfoexmhusers +1 +hrefhttpclickthruonlinecomclickq145mraifzahzjyfu8zvqiztktr1yr +1 +500000 +1 +me20 +1 +httplistsdebianorg4bd9ce8690009hardwarefreakcom +1 +acceptable +1 +voice +1 +95nbspadobe +1 +lineheight150 +1 +2084010766 +1 +071313 +1 +p0olqpgtmbgo9ifnp8ay08doxkrgyo0we2kolker0oxgubikaeok0o4x0fnf7uooaizlgssm +1 +htmllinkpushhere +1 +server1 +1 +tzkhf0dphzcabyhrdrepbwh0ewqhgsqkcxqgsmfnbebqhuhadrcbuwj1nabirtyjgwfabeslsazg +1 +alton +1 +centerblack +1 +22pxfont +1 +092607 +1 +discuss2 +1 +gates +1 +beautifullienet +1 +executives +1 +boundaryenigf8bc083208413a0c634acbf8 +1 +9si6249810fks5620100411013312 +1 +suc +1 +towerboard +1 +enhanced +1 +vanwoerkom +1 +ccc +1 +height368 +1 +clarifications +1 +fundamentalsbrbyahoo +1 +slip +1 +041311 +1 +m8cs29630wfo +1 +webstuff +1 +namelist0203 +1 +jeri +1 +bordertop +1 +voted +1 +width1tdtr +1 +histoptionsnbuckets +1 +convincing +1 +622469193 +1 +10300283774901tmdadeepeddyvirciocom +1 +pd0vpvt7n5dndpbzxvkxyvf7n0u2bwuxsl2yl1fovuvvb6j66vgf6dp0zlomoynasilucn +1 +gmailid12884051554c7c34 +1 +av +1 +pgpsignature +1 +115umm97rize +1 +050517 +1 +mia50hotmailcom +1 +ztmyx0aeqaaaup9beplqfqfxovhkzl0sab63maaaaaav329ju54zjwmi8p5f4f7lkttem8bcuc +1 +tiresome +1 +size3d3 +1 +srchttpwwwtheserversidecomimagesborlandgif +1 +effie +1 +classbluelinkbold2 +1 +titlem +1 +wrapping +1 +libperl4camlocamldev +1 +21078 +1 +n6gm2ryx024578 +1 +105335 +1 +124911 +1 +scholarly +1 +20968120 +1 +dissent +1 +brto +1 +143134 +1 +d13si8606656fka220100412080008 +1 +bbh3ramc6ffk0mxe1qee6h123fotm6bqrek6tyegrvep2ik9n2klfltnypyeq0q +1 +2446007860y80qlkqqujwsopjxhwp5daa9luciano +1 +slobodanu +1 +2ba312940e2 +1 +value6363option +1 +lenape +1 +width96 +1 +freeatd +1 +penguins +1 +reflex +1 +gmbh +1 +linuximage26323686 +1 +egroupsew082 +1 +classifierspam +1 +mailtodivxlistsdivxcom +1 +src3dhttpsccommunitiesmsncomimgcgif +1 +oaa07112 +1 +assistanttitle +1 +v1078 +1 +vriicuor46g1kzhd7dsbzlkq0eger0aqpbngmikboxide6im6gg6zeamkcnozkxj0sn7wuz6jkn +1 +114250 +1 +runnersup +1 +114910 +1 +httpenwikipediaorgwikismartstandardsandimplementation +1 +mckeownb +1 +lsb36 +1 +n9sdvpg0029958 +1 +kosovohowever +1 +25456 +1 +149 +1 +04232010 +1 +5021020020726133547028d0a10dogmaslashnullorg +1 +smalltalk +1 +mason +1 +battalion +1 +112529 +1 +blockquotedivbrdivbrdivdivdivi +1 +factoring +1 +125534 +1 +markup +1 +nytimescom +1 +sucked +1 +9b51d13a649c +1 +optionally +1 +gross +1 +heres +1 +d8qbf0okxekpyxufknyt6v5uxfysbantsbantqyke2pk4qu3bthwakug4h1gasitqhzpazs5oa +1 +m8cs105855wfo +1 +211327 +1 +hellenistic +1 +jesti +1 +1022324133 +1 +vile +1 +truffles +1 +backgroundhttptechupdatezdnetcomtechupdateibg232850gif +1 +7262002 +1 +ysbizxr0zxigaw52zxn0bwvudcb3axroigegzmfzdgvyihjldhvybibhbmqg +1 +grande +1 +judgementschild +1 +o2d82tw3019774 +1 +151708 +1 +a0a8 +1 +usable +1 +mincount +1 +yuan20 +1 +autofontspanp +1 +pdepriver2420521191rnocrescharterpipelinenet +1 +618px +1 +thailan100e +1 +titlecreate +1 +ase +1 +width550 +1 +level +1 +200209261527g8qfreg24218dogmaslashnullorg +1 +repair2e22 +1 +compatibility +1 +6pg5mf7ehe4idbtwoevilsmailcom +1 +979be8feccf611d6817e000393a46deaalumnicaltechedu +1 +02brz9re2cymif3f5v50wczwaq4nndk99gqezhzi7leeiurmtsoa5zfnwdwnhpkexqdy4 +1 +0l0t00i73pl2h150amtaout23012netil +1 +cease +1 +httpwwwnewsisfreecomclick48723998215 +1 +iabmkv7c2dlm8y78yzcg5xnfdbe4dgegd0rev7dnt7mrebab1zohtujxqzq2izkz6zkd3rq7qa +1 +fbemdawmdberdawmdawmeqwmdawmdawmdawmdawmdawmdawmdawmdawmdazwaarcabwafad +1 +updated +1 +ex97mple115 +1 +hrefhttpdrbluehornetcomphase2survey1surveyhtmcidjtveeqactionupdateeemailhibodycsminingorgmh4e73c3d93c4d7a41990b4238b47fedd3unsubscribe +1 +185716 +1 +dialling +1 +1222 +1 +centerplanning +1 +122431 +1 +kazka1 +1 +anoneundhover +1 +082051 +1 +filter +1 +pumpanddump +1 +062457 +1 +children +1 +g12cf1323284 +1 +ciphers +1 +11fe0d07b7cf7ae000003b23ae4bdb81520458 +1 +httpusclickyahoocomsrpzmcktmeaamvfiaa7gsolbtm +1 +ctoryquot +1 +nextpart000000701ca5279acd2ff70 +1 +size3d2phonefonttd +1 +ch20 +1 +identified +1 +scott +1 +width388 +1 +vp8 +1 +increasing +1 +verdana +1 +cellpadding3d1 +1 +illinois +1 +disinterested +1 +unpissup +1 +rfe +1 +hrefhttpwwworaclecomebusinessnetworkshowiseminarhtml1293857watcha +1 +fm +1 +inbound +1 +exaggerated +1 +hrefhttpclickthruonlinecomclickq5b5lowibkdutf1y12xcjjvdswizzcr +1 +filters +1 +piles +1 +genericphar +1 +doodamanswbellnet +1 +1999brswansea +1 +frames +1 +wail +1 +size3d2computer +1 +sw +1 +pinelnx433020909155345032400100000hydrogenleitlorg +1 +qpyhfreereportsflashmailcom +1 +petiti111n +1 +206160226 +1 +xrnaw +1 +texteaebec +1 +ez +1 +1300 +1 +1881416f03 +1 +cameras +1 +forh2 +1 +httpwwwimakenewscom +1 +xacceptlang +1 +height115px +1 +hrefhttpclickthruonlinecomclickqf2ozvtquwprrj3gv0pvvdi5z7ednr +1 +commentatd +1 +athelinakarimuecplazanet +1 +9222458232 +1 +minos +1 +noah +1 +6498b8nbsptd +1 +investements +1 +audibility +1 +safelyubr +1 +influential +1 +hanuman +1 +5megapixel +1 +colm +1 +zprwzpbcadptcee9cojd2qzddwindzfn34pdjdaxtgqlfqfsqtzzfg3rdknetbzfkakmckdxjn +1 +width3d13bfont +1 +th111115e +1 +offensive +1 +empires +1 +height9 +1 +124384838211ce00120000w4twrl +1 +moralesrlyceumcom +1 +gmailid128aa41a82d2c15c +1 +alagny10115237abowanadoofr +1 +345b25ffb04e47b89d07a99a9e33c0d4applecom +1 +ffff +1 +enig46ffd16c11dad5966d847622 +1 +manila +1 +ministere +1 +httpblogsanityofsamcom +1 +e7a6a2d0d79 +1 +rebuilt +1 +162608 +1 +height329 +1 +kohparitnvawawvwi +1 +toptionsgt +1 +6o6dsmti048gzqeks4xmgr59laj5shxfnjt8suvk5wvycoxllhppij0zu1ysvko5klg5lt1tmpuk +1 +color3366cc +1 +newsx +1 +207171180101amazoncom +1 +lawyers +1 +cried +1 +recognized +1 +conteplativeness +1 +color000000bfont +1 +emaillifetimetvcom +1 +treatmelikeanobject +1 +013425 +1 +g4s9x9e32410 +1 +132804 +1 +manhattan +1 +raid1 +1 +ou +1 +speech +1 +align3dcenter20 +1 +374 +1 +width1td +1 +anuradha +1 +depending +1 +color3dcc3366b199bfontfontdiv +1 +8d5f943f99 +1 +occasionally +1 +unload +1 +brresidency +1 +practically +1 +dbrxoye608622f3a1c3222237190179 +1 +painless +1 +simulated +1 +zhao +1 +value3dsubmit +1 +20100508090848771mgbdebianyosemitenet +1 +cheap +1 +linuximage26325br +1 +contact20 +1 +sta +1 +width565 +1 +httpwwwalertnetorgthenewsnewsdeskn07347915 +1 +biased +1 +abstracting +1 +judge +1 +presidnt +1 +g8pkxec12677 +1 +nowafontfontcenter +1 +ph +1 +psi0iibjb2xvcj0iizawmdbgriis8yxtbdqrq5zraku7kybf6pc9mb250pjwvcd4nciagicag +1 +encourage +1 +presentday +1 +bgcolorffcc00 +1 +4bd1e83c3010603pupmunivfcomtefr +1 +133928 +1 +published +1 +keumri +1 +alerttitle +1 +membermount +1 +sean +1 +cvgarkv6wxulwczxlyffz6yngldvhtzyaxtudctjej4sc +1 +b111llettin111 +1 +height314 +1 +welltraveled +1 +employer +1 +filetime71fd20e001c23159 +1 +aaa19167 +1 +httpinfolinkhrcxjmhtml4ynulwkhpp +1 +g8dekol07782 +1 +002909 +1 +pacific +1 +contentmshtml +1 +fulltime +1 +herebap +1 +iesubericnet +1 +subjects +1 +canem +1 +l20mr2017097qak671271734562360 +1 +fleet +1 +httpconferencesoreillynetcomcsmacosx2002viewesess3281 +1 +analysts +1 +notify +1 +suse +1 +stylefontsize16pxfontweightboldcolor000000padding9px +1 +replay +1 +pydyebzgbpvvqpb1bkm3vbaxg321ig4ayjghebwm4rnxplfxi2u1ej1bnlaqvmocltmtjeqkcvr +1 +dealing +1 +height3d21 +1 +width3d85 +1 +dpoboxcom +1 +extra +1 +2862 +1 +edition +1 +stylemargin +1 +bsfsc2sa154d +1 +margintop10px +1 +001e680f15c08ca11a0485ba5202 +1 +width592 +1 +24002800 +1 +nurst +1 +daa17258 +1 +nosend3d1td +1 +hits121 +1 +pump +1 +kernel +1 +webbed +1 +pantsfontali +1 +5087726408156400551715001108381 +1 +contentand +1 +beziehung +1 +085955 +1 +pgijp2cejugaaeezjzlbliszt +1 +abtstndynamic253145164122airtelbroadbandin +1 +intense +1 +hrefhttp0afpillsorenruepeqoladf01cdd12fd4f0click +1 +c111ncerne +1 +bgcolor3d5492c3 +1 +ecological +1 +urania +1 +sansseriffontsize13pxcolor4ddb5e +1 +aw5npsiwiibjzwxsugfkzgluzz0imcigym9yzgvypsiwij4ncjxuqk9ewt4ncjxuuj4ncjxu +1 +vient3dgap +1 +du20 +1 +trademarkets +1 +strongperiod +1 +650 +1 +type3dhidden +1 +lth +1 +spfneutral +1 +construction +1 +monstercouk +1 +maker +1 +har +1 +publicly +1 +z +1 +filetxt +1 +pet +1 +hrefhttpclickthruonlinecomclickq88ec5pqqkxb5kg4uyru358atsbbir +1 +searcher3c2ffont3e3c2fb3e +1 +hazardousi +1 +651d213a43ea +1 +chekitb5876com +1 +supplies +1 +030059 +1 +mangled +1 +lmzuwxio3atb0nktd8h1uahdblikdtkreftu +1 +homelessness +1 +trusting +1 +15000 +1 +jfoster81747verizonnet +1 +hackers +1 +reservedp +1 +du +1 +rtorvivsnlnet +1 +ny +1 +revolutionb +1 +suomenlinna +1 +style3dfontsize10px +1 +uid121431201487268 +1 +operas +1 +game +1 +maeda +1 +2009br +1 +122815 +1 +20412720261 +1 +421101c4b +1 +race +1 +valuepgpapua +1 +promotions +1 +enig91847f0ca91f3bcc9384f202 +1 +guarantees +1 +105203 +1 +blockdevice +1 +ms4ymi4xmdevaw50zxjlc3qvaw5kzxg1mju0nc5odg0ipjxgt05uia0kicag +1 +1mzszoafupm03njmgbatnnzszoafmkztc0maah5pm03njmgb2am03ngaafzszpm0ualmkzsuual +1 +na3dk3lx021747 +1 +width3d561 +1 +g7nm2tz09890 +1 +popularbatd +1 +supposed +1 +texfq1rjt05tdqonckltu1vfuw0kdqpgzwrlcmf0aw9udqoncm1lzglhc3rpbmfsdqoncmfudgli +1 +g976wqk21300 +1 +crude +1 +assortment +1 +funds +1 +conventions +1 +hrefhttpwwwtuduwrjcnazoon68ebeb7fc8d97f4d437e +1 +cans +1 +profilename +1 +downloaded +1 +nmhxlvfapflvecuutfacyopamuajrrrqauuuuafjs0uaiaarmn0hoaztqnli6vpseucmm03vk2 +1 +score406 +1 +hrefhttpclickthruonlinecomclickq3doi9nibz4yt04jzvwlgwutnantpr +1 +michel +1 +seesaw +1 +masking +1 +similarly +1 +satisfaction +1 +pinelnx433020905160120022237100000watchermithralcom +1 +earliest +1 +valueoh +1 +altitudes +1 +zv70f5naaprp0vdkm4u7rulxsu6wr6abz88n4avotpbxjjuh3y0oadlzmxxssn5fjq5fp +1 +sheds +1 +threadindex +1 +0pxls +1 +noninfringing +1 +arises +1 +regarding +1 +archivelatest573453 +1 +013518 +1 +htmlfontlowcontrast +1 +gmailid12898ad365e23d18 +1 +dialog +1 +davis +1 +triad +1 +151621 +1 +everythingshakespeare +1 +naj93ium021972 +1 +yyyyuseperlexamplecom +1 +d4faf8c83e1 +1 +forced +1 +color3dff0000199fontfontdivtd +1 +clicking +1 +094622 +1 +160641 +1 +g7djqr416060 +1 +cbyi +1 +rokovodstvu +1 +hrefhttpclickthruonlinecomclickq65k82zidptigiqcgwd70vof0m8p9zr +1 +bgcolor3d2200000022 +1 +chanting +1 +m9v4k8os +1 +windowsreg +1 +rootdevhda1 +1 +200210060800g9680ak15248dogmaslashnullorg +1 +gallaudet +1 +titlefree +1 +anand +1 +newslettersa +1 +creole +1 +contrib +1 +outoftown +1 +000021 +1 +argument +1 +clipeqheloip2 +1 +20100507 +1 +taa26129 +1 +147they146re +1 +indirectly +1 +1022922178 +1 +height3d2 +1 +102047200 +1 +hits6828 +1 +174900 +1 +n1n6ogen005513 +1 +subscriber +1 +icedove +1 +bii +1 +smtpmailtimesharecarriedtrd2finestscorcom +1 +cannes +1 +arialhelveticasansserif +1 +coliseum +1 +mad +1 +vienna +1 +m8cs138636wfo +1 +025201 +1 +ji1xwakuxhddaoodynafugzhysqxgmbwavdaqsedlsrmagmejtbcqgvorahlrlnnmgdjfecaqc32 +1 +plus +1 +2139621475 +1 +113uiz +1 +width3d534 +1 +valuelatvia +1 +serious +1 +contended +1 +fuller +1 +jmnemonetcom +1 +irs +1 +hrefhttpku12thestrongstoreinfo13820250f3c745790671htm +1 +guts +1 +cjjggeefmaajbkinrquwaarfewrbrhxptuhfh728vyfbvjhrxxxwr1qqimbpghegwe2eaccl75y +1 +distinct +1 +⺠+1 +056 +1 +frewicessucsbedu +1 +burdened +1 +102232669 +1 +promise +1 +33aaf29409a +1 +higher +1 +largemorespanafontbp +1 +manuscripts +1 +m8cs29094wfj +1 +explicitly +1 +copybrtdtrtbodytablebrbrbrbrtdtrtablebr +1 +blending +1 +congruence +1 +13220349890520020925163453magnesiumnet +1 +66187233200 +1 +5img +1 +c4194a +1 +hrefhttpupdateslockergnomecomlatest +1 +asianetcoth +1 +forb +1 +quebec +1 +tnonsensefrom9293tnonsensefrom9394 +1 +32227031 +1 +srchttpgservzdnetcomclearoutboundgifappid2emid25136482nle539issue20020716 +1 +httptopicacomub1dcmwb2yxqn +1 +g82h3bl27025 +1 +shooting +1 +hardenp +1 +155609 +1 +submarines +1 +align3dcenter3e3ccenter3e +1 +manufacturer +1 +target3d5fblank3e3cfont +1 +260 +1 +locked +1 +facilities +1 +aqeaf4m2bepsv6bpknrikyccfkpmjbtqdqpgraj9djil2w9m8yquzraielgrbniblrmsvffyb +1 +genes +1 +httpdnbjackierxru58931d5277aa5b481dd1abe600a4010d +1 +vbanvfakvoagpualteaivfai5h0jmuaiyxsjklajkiagfvaivvakj4amj7akvoadahajv3aj11al +1 +uid134271201487268 +1 +borb +1 +father +1 +eldred +1 +20100504024346ga7799chunchems20nix +1 +dalnet +1 +isiloxc +1 +emailed +1 +ywh40 +1 +n20grpscdyahoocom +1 +chu +1 +fgdksxsi51egdznt9qsbua11lbvpn93qv1rwqzgxflrs1pvssn1vl5drcy44hs97f0le3z +1 +size3d2fo +1 +ptsxufu8z8xdlvtlfxwlkoonh8ewgzmrtrnrftwxcfex7pspuopbmsnml8nvomvhcedofg4npfm +1 +0d1cc15d864e2 +1 +1000s +1 +encryption +1 +qaa07539 +1 +width3d150 +1 +dispute +1 +bgcolor999999 +1 +tablets +1 +recreational +1 +hrefhttpclickthruonlinecomclickq87izdoq8n4w11bhleplrq8kasp4er +1 +c00112d0c59 +1 +6hx1xjapiaqemec9lwd8gxabnfffka4opezgp7r8grabnqp4s1uqpe57zjhrrrz +1 +103203 +1 +fffff +1 +clothing +1 +fat +1 +tami +1 +twoway +1 +spamassassinsightingslistssourceforgenet +1 +httpwwwlinuxiepipermaililug +1 +designing +1 +attractive +1 +clientsservers +1 +axzwajkqayugb2rgalmgqjblbd25dgacdwbeb7gt2slbflaagvwc1ejfaxozzzzzaurrayleacbk7 +1 +p111rtaf111li111 +1 +anyway +1 +rwxrxrx +1 +quartiere +1 +divmuchdiv +1 +akron +1 +alt3drendezvous +1 +133517 +1 +men39s +1 +width670 +1 +o13si7770220fah4120100525101217 +1 +part121076313585444161028690797657 +1 +1022324148 +1 +httpslistmanspamassassintaintorgmailmanlistinfoexmhworkers +1 +ielnn4tjfextmw5akqtwyeskbzebjqap9emrqwiegaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +1 +070737 +1 +contacted +1 +ote +1 +473 +1 +monitored +1 +mcispanspan +1 +inanamayacaksiniz2e3cbr3eherseye +1 +china20 +1 +httpwwwpixiechildcomjournalindexshtml +1 +regimental +1 +testing +1 +q +1 +120140 +1 +tear +1 +ieyearecaayfakvmyosacgkqvsrxyk4409xcwcgixo8aswa8db8m2sp4kv3c2l +1 +bhdhor6pysudjmg9dsm7oamqnh7dczpk9mdcndrdrs6g8 +1 +mello +1 +tires +1 +z8o3a4rjwmuttp4zjftr22rnttrgvkby4fxtwoigbioimztcmjorzsyht5ip4u35wo1o1wjf6rw +1 +distros +1 +annonarian +1 +ps2 +1 +dfdfdf +1 +width3d435 +1 +tatankastortekcom +1 +115ur10297ce +1 +1711 +1 +charset3dwindows +1 +xwebtvsignature +1 +antipiracybabr +1 +ronan +1 +valuelockergnomedigitinput +1 +1237 +1 +mp3oggs +1 +spidmax3d1026 +1 +quvfl3od6drxlv5kvd9ozji5xucxpya3e6ny7burqextdqap8ajlpyuqlirxy6e7 +1 +127341333778f43e780001w4twrl +1 +owned +1 +ckloiber +1 +rowspan3d3 +1 +udell4 +1 +dreamed +1 +failure +1 +sacramento +1 +island +1 +whrkisaix8jwiekli0gk0glbzhjhaqrqtgj18 +1 +morganatic +1 +g7se9iq22132 +1 +17cvgp00014l00 +1 +le9vas6ankazxzixzaybjkj2yhyknkzncxfmwmmzcsewif2mmrcnplzwkq0xk8grhhnsxrooeow +1 +selfesteem +1 +084030 +1 +exmhworkersadminredhatcom +1 +poetry +1 +principle +1 +ol +1 +g4db54e30128 +1 +jox1pshytan0pa0vwnalofbrx9jlab1psp8a2ff90p8awnpanhfddamcxjqsxop9 +1 +facearialshell +1 +paddingbottom5px +1 +invasion +1 +href3d22http3a2f2f2022e1012e1632e343a812fultimatehgh5frun2f22 +1 +dac7f1635c +1 +pid +1 +uu3hilnuoae +1 +11mbpsbrwireless +1 +902416 +1 +lan +1 +000000 +1 +72 +1 +e580 +1 +finish +1 +10115398 +1 +zezujaep +1 +023030 +1 +csl +1 +103926 +1 +uizygfxb9khus3e3y6xpfmbciw93efuskn3161mthfr7og6ocyncqjj1k3sogpb6vp2ujiek +1 +prevention +1 +h2mr17697830ann751270621908541 +1 +236ibbrfontfont +1 +wrotebrblockquote +1 +kdbmkiqni7yvikeedxe0ppupgwxk7putvdcqljiitovr8lsqxxgoaxrah51bhzcmwpk0bjic +1 +width3d2230022 +1 +finds +1 +fails +1 +g8okn0c20652 +1 +211185156157 +1 +uaa28017 +1 +whitehead +1 +mia +1 +smtpsvc +1 +reached +1 +chaste +1 +lcomemailsz3d468x60ord3d +1 +111102 +1 +inconvenient +1 +worldwritable +1 +romanstrongl +1 +bordercolor32336b +1 +wartime +1 +gwb17 +1 +alignleftthesep +1 +boring +1 +replying +1 +sheets +1 +xuserid +1 +800758220309 +1 +070341 +1 +031744 +1 +dri +1 +ywnvldo6kvwufoc7ptkbaut7d8vzv7v0rx0xx90nuqoipaysro4f3f1rqoqfoplfwm580nd6n +1 +emacs222 +1 +lvzzyvs9rccyyzzzzy5rnjtv8auzwx22lffo113tmv5prgzfy1rdgr3vbjjy113026feosx +1 +styleborderbottom +1 +size3d4spa +1 +titlesoftware +1 +pounce +1 +hdds +1 +bantalcsminingorg +1 +imaginary +1 +urbaniak +1 +1006 +1 +httpadmanmailcomsubscriptionaspemjmnetnoteinccomlsgo +1 +113242 +1 +instrumental +1 +suitefontfont +1 +5eb1e3ed6a +1 +operationsparadigm +1 +doquotbr +1 +gnomiesabr +1 +debiankde +1 +ptdtr +1 +forastero +1 +archivelatest576487 +1 +filesystems +1 +2036916370 +1 +httpscriptingnewsuserlandcombackissues20020923when35654pm +1 +uid57441201487268 +1 +itifont +1 +waddler +1 +rootdownload +1 +samad +1 +realism +1 +181845 +1 +torment +1 +knownmailinglistrcvdinmultihopdsbl +1 +zwwmg1kds60mpbhqzjuufeivz4rhlniutb1oy4mg65lab7ty8ttmstzaxw1ktfbgq6v +1 +toward +1 +085355 +1 +aanlktin9u3dmww3x0et8i9crpfwphp7lefe9ltoptljmailcsminingorg +1 +namesitename +1 +abroad +1 +iiiii +1 +constant +1 +swap +1 +transform +1 +underpowered +1 +spot +1 +oversight +1 +48 +1 +exist +1 +4bd596d85050002heardname +1 +photograph +1 +hell +1 +pittsburg +1 +mp7e1h92d969a6qtpeog86of49fry6or +1 +baskinrobbins +1 +size3d1v +1 +90e5e13a4f72 +1 +ifnerr +1 +12pximg +1 +cascade +1 +httplistsdebianorg4bbeb3a17050609coxnet +1 +2dda329418e +1 +m8cs73574wfj +1 +1600 +1 +tdbfont +1 +ljye5dbxrginsih5nrhmwdycogownpuz4iw4s6tjosa2nskykaybqigajxoukwsgpeakwi +1 +5joay7kykazfvg5k5pfomss0nmtjaqym2nnrhaiysekotzqkq8e4pntixygyxumz4gopxp2sg9tv +1 +decompression +1 +0099ffabout +1 +spamproxyd +1 +ilugsocial +1 +rlyvhihzwwgahibc3c0lbliszt +1 +ordi +1 +dsun358adbvdesignanalogcom +1 +lowlevel +1 +valuegoodgood +1 +letter +1 +150904 +1 +album +1 +httpwwwiamerickacom +1 +doubleact +1 +derived +1 +heu5hzlhsaiffvr0mmmwql8k5mwv4hanssjz8wzwmz3682yi2inknfn5i0o8y3agq3mc2gjv9rfn +1 +song +1 +permit +1 +confluence +1 +chf +1 +hercenter +1 +httpwwwiscorgservicespubliclistsfirewallshtml +1 +seven +1 +nmo4wogfc02abk4huhfqfniqsjaofuvkrzekngti9keymh96rvcx0ilgvudc1wht5xm80qgom8 +1 +ofb +1 +mailtoforkexamplecom +1 +storyba +1 +orwant +1 +httpwwwbarracudacomreputationip2127520558 +1 +639388152lsandialmoonglobalcom +1 +total20 +1 +cup +1 +3d6538748010204barreraorg +1 +drugstores +1 +valuetntunisiaoption +1 +hopoallergenic +1 +095905 +1 +showcases +1 +align3dmiddlea +1 +ceremonia +1 +sharing +1 +neg +1 +sa191f1 +1 +205638 +1 +feeder +1 +westrong +1 +hulled +1 +grew +1 +rgb103 +1 +srchttphomecnetcominlshrhcmostpopgif +1 +brazilian +1 +halfhearted +1 +style3dcolor666666 +1 +altsoftware +1 +1860 +1 +hemoglobin +1 +apt +1 +score2636 +1 +kcvpixelbufferwidthkey +1 +g2v55kt24035 +1 +httpdevelopersslashdotorgarticleplsid021112161219 +1 +panama +1 +ba7b73ee17 +1 +20020201150022b11472cshelsinkifi +1 +contradict +1 +gb1hcl808717 +1 +href3dhttpwwwthezscom0003 +1 +idx0000t138 +1 +only20 +1 +hide +1 +10229220133 +1 +burtonp +1 +79ee316f16 +1 +hrefhttpclickthruonlinecomclickq37emlqiqzfsldtpruq93kjz6ejcner +1 +200607 +1 +munnariozau +1 +fe734aa0f08144ebad86a8676e5cdd8aapplecom +1 +awdodg1hcmdpbj0imcigbwfyz2luagvpz2h0psiwiibtyxjnaw53awr0ad0i +1 +hrefhttpwwwmilkapilkaruuiqxjs9b98bbd62aefa +1 +87192220226 +1 +cerificates +1 +httplistsdebianorg2010051021595631e1142babydosstargateorguk +1 +bags +1 +hspace5 +1 +mwsfud +1 +smtpq4gnmailissas9143net +1 +wolf +1 +listuserneopittstateedu +1 +w0u2xtvitqgz8mtxmoelsuyrkb6qrfe6xyapnscopqd5ndbwpohakkkrqv5n8ef8akslpsjj +1 +titlenews +1 +2010p +1 +atd +1 +lbiejxrayqe3ebdaawwxrmarfb4rukmdhdbchs6yufhmfzygaamccebairte4ayhudmauggywogf +1 +methods +1 +hed +1 +184351 +1 +65925578 +1 +schedule +1 +cs +1 +p9si1842418fkb320100429152508 +1 +dies +1 +ontime +1 +10112 +1 +pappygiyiucefqroeyvlso0erk6qzzncogjluykxu6mk0y6macoadxecaqothnryq +1 +aligncentermiltonp +1 +buanqavo +1 +6ookxih7dsyh4fguk5ams763nz5t0zys1yzeze9opjcxncb7rfig +1 +cazin +1 +dramatically +1 +185605 +1 +193906 +1 +oxijionuk +1 +rhinestones +1 +hosted +1 +2136 +1 +craft +1 +111752 +1 +unpeaceably +1 +23b4c27550a9a +1 +lnf8apofn3lnf8aautzozuxmvwa8n5rjr5r8apvja40bcloqlzxwcfet81waaxzg +1 +closer +1 +realization +1 +hrefhttpclickthruonlinecomclickqa4hgtsq6p4mtc1olg463lvvi2rgydr +1 +robbing +1 +httpwwwfurnarinet +1 +schiltz +1 +intminline1msgidminmsgmaxlineminline2maxmsgminms +1 +reasonable +1 +itifontp +1 +gwendolynmay +1 +minoltas +1 +msncouk +1 +icagicagica8ynidqogicagicagicagicagicagicagidwvzm9udd48l2i +1 +a82uruejr2cjwd8na11hwrf5gst7ku3nzb3eglnfw4x3hlhhvtuurjkd00fgy4sifuz7ifzp8a +1 +wodaxugu +1 +bsome +1 +forgive +1 +qp +1 +nonstandard +1 +processed3ddestinationdconso +1 +color0000ffnbspclick +1 +152529 +1 +orh5 +1 +conducting +1 +sought +1 +ghetto +1 +punter +1 +raymond +1 +20100428191714982mgbdebianyosemitenet +1 +20020828100430378c3856matthiasrpmforgenet +1 +agreement +1 +tester +1 +apologies +1 +analystsbrbintel +1 +320 +1 +mailtoexmhworkersspamassassintaintorg +1 +jim +1 +7149194185 +1 +netdiscover +1 +httpkotiwelhocompmatilai +1 +worries +1 +093629 +1 +31104244 +1 +httpenigmailmozdevorg +1 +reducedifonttd +1 +164 +1 +hrefhttp4b1asteadquietruvybilozaucdab4e98736c +1 +nonehttpwwwfacebookcomninboxreadmessagephpt189327550468mid307afe1fdf8ad7a7f1c6c0617661bf8bnmfacebook +1 +syntax +1 +distance +1 +flaw +1 +protests +1 +por +1 +n17grpscdyahoocom +1 +expanded +1 +cb4ed98bd07747faac7c3740bb63b29fapplecom +1 +colts +1 +4bbb0a8a9010407coxnet +1 +060658 +1 +114133 +1 +vanderkolk +1 +positionabsolute +1 +262621lenny4 +1 +141317 +1 +divtdtr +1 +molecular +1 +src +1 +174102 +1 +114829 +1 +parrot +1 +facearialstreetnbspaddressfonttdcentercentertd +1 +luba +1 +ptsize3d14 +1 +redox +1 +sbin +1 +neighbour +1 +a9a1bae2a2b7a3a0e5e9a5b8b9b7fce1e2fff4f4e3f2e8fff +1 +body8bits15 +1 +18yrs +1 +voiceless +1 +taste +1 +ltmodem +1 +nn1matxuah2 +1 +17xfeo0002td00 +1 +httpwwwgeocrawlercomredirsfphp3listwebmaketalk +1 +bantaboobycsminingorg +1 +aaadaabptflnufvtierjr0luquwgq0fnrvjbicagicagicagae9mwu1qvvmgt1busunbtcbdty4s +1 +3198870 +1 +ibuthousands +1 +020838 +1 +stabbed +1 +width3d600 +1 +54 +1 +brbigger +1 +194235 +1 +sweeping +1 +size3d3corporate +1 +reservedptd +1 +beneficial +1 +isnbspqqqqqqqqqqzdnetspamassassintaintorgp +1 +weinthal +1 +172164831 +1 +8d549279 +1 +101501320 +1 +sigjflqfragj4pcpl2ep06lgs +1 +emsp +1 +einzelnen +1 +geege +1 +friendsfonttd +1 +taa03135 +1 +williams +1 +phoboslabsspamassassintaintorg +1 +051929 +1 +robin +1 +midiplug +1 +securities +1 +1111 +1 +colspan3d3 +1 +modifications +1 +valuekhcambodiaoption +1 +211612 +1 +dealloc +1 +independence +1 +074320 +1 +selfassured +1 +1287 +1 +vextedit +1 +filetimee7a87e5001c26e55 +1 +versionsslv3 +1 +zs48l3nwyw4pc9mb250pjwvyj48l3adqo8ccbhbglnbj0ibgvmdcipgi +1 +prso +1 +httpwwwmarciafeldmancomhtmlnewsshtml +1 +4dec029409f +1 +backtd +1 +laserfax +1 +entity +1 +equipment +1 +rgb0 +1 +boundarynextpart000000701ca5881378b1e10 +1 +illegal +1 +size3for +1 +htmlimageonly24 +1 +5a7c82c39c +1 +5ewqfbn6k0j8 +1 +y2u9irzqt6lf6sigc2l6zt0incigy29sb3i9iimwmdawrkyiprncpmg8l2zvbnqpgzvbnqgzmfj +1 +200911110851nab8p9hr027976ns1csminingorg +1 +linuxmodulesafipxver +1 +fastgrowing +1 +anywhere +1 +webmail +1 +afford +1 +tellus +1 +srchttpi42tinypiccomatlrg3jpg +1 +billwstoddardcom +1 +arise +1 +iibmsu5lpsijrkywmdawiibwtelosz0ii0zgmdawmcigquxjtks9iingrjaw +1 +15071 +1 +webmasteregayzzncom +1 +100045 +1 +testamento +1 +005601 +1 +targetblankprivacy +1 +bragging +1 +m86ed5m86me7we75ew0k3 +1 +idcontent +1 +335 +1 +inning +1 +size3d5font +1 +someones +1 +hilt111n +1 +season +1 +colorredcitizen +1 +1210 +1 +proposals +1 +wabfontbrbrf +1 +bbeing +1 +reviewed +1 +7d9162d0db1 +1 +1204 +1 +httpwwwugnncom201004macusenet +1 +wwwcomicscomcomicsdilbertdailydilbertimagescomicscomlogogif +1 +quickest +1 +ahiithemailcom +1 +59tmkulcnitzqske9ilqdfm0wvgds4yy6nikyklgoifkcnaqjmyucvniaiwarhgpwkajebksa +1 +zeroed +1 +20px +1 +104435 +1 +595af2c2d5 +1 +score1277 +1 +picnic +1 +size1andfont +1 +pmyimage +1 +murdered +1 +hasnt +1 +insatiable +1 +d6f9213a50f6 +1 +align3dcenterfont20 +1 +squareshaped +1 +gig +1 +127280762829087camelpcstevenlan +1 +moc +1 +114013 +1 +divareas +1 +gamallomanuel1csminingorg +1 +7cqfxtxt2tqkcsohsuxmylkcgj1wk39pbrwtnbpbqmwohjk8xfba6uybpuedfl57vbxpwl1arv +1 +httplistsdebianorg1252913082894381272137255091javamailrootmd01wowsynacorcom +1 +recorded +1 +g6n8qop12861 +1 +color3d22ff0000223e3cb3ehediyeli2e2e3c2fb3e3c2ffont3e3c2fu3e3c2ffont3e3c2ftd3e +1 +execution +1 +101036720 +1 +replacement +1 +color3de0e0e0 +1 +tci +1 +awq249neoplusadsltpnetpl +1 +regulation +1 +herby +1 +vertigo +1 +k2biztld +1 +bhw9q1vzvgyjjgrlz1vxxiksmzd72gixi4pbzfwtt1bu +1 +readi +1 +045343 +1 +shania +1 +pan20100511164828csminingorg +1 +alignmiddleba +1 +999999 +1 +ride +1 +swr +1 +lpr +1 +2si4205263fxm3520100409151515 +1 +arbitrarily +1 +paddingleft4 +1 +clasica +1 +contained +1 +pairings +1 +usual +1 +xsaslcyrusserverfirst +1 +acid +1 +listssecurityfocuscom +1 +rat117re +1 +assured +1 +ubuntuites +1 +ecke +1 +8226nbspduration +1 +wikipedia +1 +18si798529fks520100419185917 +1 +074527 +1 +2adid +1 +concentration +1 +shrivels +1 +bsfsc5mj2022 +1 +192241 +1 +httpclickthruonlinecomclickqe72ygqmvxons0rb0vqs8cmb0qvser +1 +235101 +1 +arsafont +1 +wind +1 +vol +1 +architect +1 +cdale +1 +ce89e27c8bbf3 +1 +yearsprecisely +1 +vigor +1 +alt3dplease +1 +29995 +1 +valuerussia +1 +metapackage +1 +httpwwwmailcomesandgoescomvirilityaffiliate +1 +10011245 +1 +ddingalt +1 +shootout +1 +dav39lk9vdnw3yqogam00002a9ahotmailcom +1 +hutchinson +1 +classtthis +1 +rubidium +1 +ddos +1 +17str30000sj00 +1 +hrefhttp5967fkvotufidcnun010d2e6735b0b012c975vqw01081934237070017543 +1 +marital +1 +nonremovable +1 +spiritual +1 +5501 +1 +wasdiv +1 +1144124187dynamichinetnet +1 +ground +1 +107181 +1 +syntactical +1 +institutions +1 +allygatorrcsminingorg +1 +duba +1 +36b2 +1 +podium +1 +size1emailb +1 +resultsfontfontblockquote +1 +ggerdesyahoocom +1 +substantial +1 +score6 +1 +size3d22622 +1 +alignrightanzusp +1 +didn +1 +eclipticcomparison +1 +152931 +1 +000b11a65aad8587c2b33bb78ec3ubrjup +1 +altm +1 +pounds +1 +gmailid1287f4e5de11edd8 +1 +internets +1 +095854 +1 +fewer +1 +corrinneanalisamsbordersgroupinccom +1 +border3d0font +1 +httpwwwteledyncom +1 +cadeau +1 +jaa25881 +1 +workers +1 +pppppppppp +1 +bought +1 +164233 +1 +namesubmittdform +1 +width3d2270022 +1 +pro +1 +advertiseabrfont +1 +webcam +1 +rfsrzuftkuysiydf6zjkuokdsue6vdmpbnru +1 +annals +1 +bgcolor +1 +jnj75oyotr76axpajstbsdkaxqeyui75dtqen2p2yd2ep7k3mrirxhb096fb2gdum7fw1ewa +1 +profile +1 +cs78128057pphtvfi +1 +mjvc2067wvqk +1 +size4a +1 +wraps +1 +025105 +1 +differently +1 +mega +1 +guys +1 +hdready +1 +formbr +1 +guns +1 +ton +1 +013506 +1 +u23mr11814098ybh1501273682315890 +1 +18si6022688far020100524143041 +1 +n3d0 +1 +historical +1 +11pt3b +1 +gaa23883 +1 +tend +1 +trtable +1 +182d7168a1 +1 +httpsourceforgenetdocmandisplaydocphpdocid9026groupid13487 +1 +digits +1 +helodynamicdialin +1 +rapidly +1 +verh111117100ing +1 +045244 +1 +axis +1 +yosyxworldnetattnet +1 +voyage +1 +ymmv +1 +programming +1 +sunited +1 +fremont +1 +60 +1 +transportation +1 +apprentice +1 +14195 +1 +httpbumptiouslyblogspotcom +1 +romanstrongc +1 +c5c8b16f16 +1 +115944 +1 +ake +1 +moreover +1 +director +1 +tablep +1 +082813 +1 +caninitwithfilepath +1 +englishspeaking +1 +ziyaret +1 +dirttrack +1 +src3dulgif +1 +ancestral +1 +wplayingbuttonmoreogifhttpwwwtributecanewsletter48imagesn +1 +fontimg +1 +participate +1 +classsidebar +1 +ientertainment +1 +chesnels +1 +hereafontbfontp +1 +b24ncg0ky29uc2lzdgvuda0kdqplcglkawr5bwfsdqoncmv4yw1pbmf0aw9udqoncnbob3nwagf0 +1 +moutngkundenserverde +1 +xvrscore +1 +211006 +1 +produ +1 +numbers +1 +sendmailcf +1 +addicted +1 +dsa1842 +1 +sanction +1 +mutuelle +1 +br20 +1 +classtitlenbspgnomecoreptdtrtablep +1 +wc +1 +regulated +1 +600600016825 +1 +assertion +1 +engelbertstr +1 +20021007103831642c1bbbhostingj2solutionsnet +1 +115827 +1 +bsuckstarts +1 +upon +1 +httpmaddogweblogscom +1 +tomas +1 +200756 +1 +g6pgtwc05929 +1 +battles +1 +nonecenterspan +1 +recalibrate +1 +httpwwwcan4linuxorg +1 +qb1in +1 +brby +1 +debianlaptoprequestlistsdebianorg +1 +8025 +1 +026 +1 +20100503134740005bssiguanasuicidenet +1 +belpic +1 +c111n111zcan +1 +shortduration +1 +iso88591qivborw0kggoaaaansuheugaaadaaaaawbamaaacllos0aaaamfbmveusk +1 +3a8ef13a553e +1 +snicket +1 +replayed +1 +lack +1 +asians +1 +concreteeye +1 +democratic +1 +townsp +1 +nv +1 +prior +1 +h2disruptive +1 +cjqqtutw2mmqfgkik4bieaeqgq4emrhvtaafopbpybo60bjuhydyiaaqrbzuw5jvkdg5uu1bcd +1 +p14mr1394629fah471271513762089 +1 +function +1 +enjoyed +1 +17k9nm0005yw00 +1 +commuting +1 +8pt +1 +loan +1 +inderjitsinghsainicsminingorg +1 +4175fontp +1 +collect +1 +score54 +1 +decaying +1 +size2recipients +1 +4400 +1 +cipheredhdssdescbc3sha +1 +butterflies +1 +qing +1 +7188806 +1 +replied +1 +2071911661 +1 +odd +1 +m8cs106909wfj +1 +da29d2c632 +1 +acquisitionsfontbrbrbdivlidiv +1 +mx2csminingorg +1 +shocked +1 +href3d22httpnewsmoneycentralmsnco +1 +i486pclinuxgnu +1 +bogus +1 +xmailerinfo +1 +iulpbfont +1 +monitors +1 +184858 +1 +flavors +1 +suburban +1 +490a93ed5c +1 +4ljj2ndu9p16ms9wtwog4r8mnkjwsq8whj9gs3dxnpdsfz61ll42ev8azbxmazvwbn +1 +v17cs9479wfj +1 +dilute +1 +022520 +1 +00breplica20 +1 +addition +1 +eyes +1 +2009td +1 +alsalib +1 +girls +1 +hyam +1 +size3d22222bfont +1 +yzbhpipm8umpkwaam230ptrzojhdjtiimuck7ukzhqqjeqmdscp5totptqgvhwq3ekly2eraahbp +1 +posuere +1 +color3dff0000hostingvccomputersiefontafont +1 +atdtr +1 +marks +1 +quotefree +1 +rowspan2 +1 +mailtodnsswapdigestlistsironcladnetau +1 +color3d0000ff3eeasy +1 +portion +1 +width3d40 +1 +legend +1 +concentrate +1 +pgegahjlzj0iahr0cdovl3d3dy5tb3j0z2fnzxbvd2vymy5jb20vcmvtb3zl +1 +windows1251bq2hhbmvsifdhdgnozxm +1 +11fe0d0ab7c42ae000006518e54bcb20d5ca85 +1 +scottsdale +1 +ut +1 +stylebackgroundcolorffffffbordertop0px +1 +width10nbsptd +1 +procedures +1 +wrapwhat +1 +spawn +1 +alfa +1 +reichsgaue +1 +hrefhttp2f78cokecoccnjzomibuwix1a0c18717b09911579aafbrpjhlwnlnhibodycsminingorgiykijile0607054147055103466697previous +1 +azeojxyrrgmea10 +1 +rodents +1 +c111111kie +1 +2436b21aa2203eee039e2cbb458 +1 +10143320 +1 +laws +1 +overwhelming +1 +lineheight13px +1 +h5unit +1 +l111111115ely +1 +verdanaif +1 +substations +1 +h2ah2 +1 +confusing +1 +defenses +1 +cliff +1 +dsa1863 +1 +decimus +1 +commander +1 +pyknic +1 +valueluluxembourgoption +1 +valuefiji +1 +085946 +1 +m8cs60639wfo +1 +definedconfigsndusbmidimodule +1 +125717 +1 +width3d4 +1 +somewhere +1 +upgraded +1 +hrefhttpwwwlockergnomecomrecommendhtmlrecommend +1 +umuioxoywa +1 +michlmayr +1 +wb1wozrf3e79rhrzuwcyprs7gdrw99liuj37xaaoyxf3u79rhrzu5gtlsyn +1 +archivelatest576724 +1 +rivers +1 +petersburg +1 +bones3cbr3e +1 +approximately +1 +arialfont20 +1 +005427 +1 +rabbit +1 +february +1 +lite +1 +mailtoexmhworkersexamplecom +1 +e3si1197745fga2920100505233753 +1 +savviest +1 +6zjcdvgbjfrdvwmvocub06rz2tr4eq5u6hundapu0q2ecmaqqs1twpbqnbvblqooyircl +1 +glen +1 +hibodycsminingorgfont +1 +145856 +1 +100irigir +1 +163225 +1 +classifieds +1 +discoveries +1 +izawodawmcigym9yzgvyy29sb3i9iimxmtexmteipg0kicagidx0cj4nciagicagidx0zcb2qwxp +1 +0031 +1 +famvirb +1 +88135 +1 +zclient5011ga2696rhel4 +1 +harmony +1 +0025 +1 +freebr +1 +etccrontab +1 +convert +1 +h97ven39t +1 +societyp +1 +th3d98 +1 +confidence +1 +treats +1 +namerestrict +1 +stylefontsize15pxpadding3px +1 +g2t5ans15641 +1 +ratify +1 +margin10in +1 +laude +1 +101143am +1 +ceremony +1 +re102lect115 +1 +httplistsdebianorgdebianuser200909msg00721html +1 +091425 +1 +marvelously +1 +euro +1 +global +1 +93000 +1 +062526 +1 +hrefhttpcooa14jnuaixagcnybuoymuhep1890426e73a9fb81yevqbufjdy1550214841 +1 +oxx5jmwebs4dwecxxs5nan0rvib5m8cbehjtvkru3f4ks9xzb7jtoa0m1iaa4hdtcclicb4nn +1 +6826 +1 +135019 +1 +3dph1 +1 +testsawlknownmailinglistquotedemailtextreferences +1 +villa +1 +matsudaira +1 +recorderabr +1 +331b31c6b5fab +1 +hrefhttp239dforlupccnaifoyerife93401t5k487g6088n3f58viatuxiy80059289025789459473460 +1 +representing +1 +bjtxqup282nazlewsfzlbliszt +1 +349ifontbrfont +1 +selfappraisal +1 +testsbsfsc0sa218 +1 +blow +1 +xfelkmailscannerto +1 +srchttpwwwzdnetcomincludeadsjsrgroup2766 +1 +sons +1 +displays +1 +allowing +1 +documented +1 +jeffersonfullscale +1 +101850 +1 +tnpitchingcom +1 +lightweight +1 +mentions +1 +udev +1 +vowels +1 +12100 +1 +ukrainian +1 +10s +1 +manpower +1 +plugging +1 +height3d200td +1 +non +1 +ricklinuxmafiacom +1 +boosting +1 +bdear +1 +g533nl726103 +1 +mailtodebianlaptoprequestlistsdebianorgsubjecthelp +1 +132920 +1 +104834 +1 +entitled +1 +tiputil1 +1 +alive +1 +0quot +1 +folders +1 +aligncenterthis +1 +asiassa +1 +realize +1 +ll117via +1 +lifetime +1 +bsamplesb +1 +dmmpjsrc3dai0d0d0dfunction +1 +mimwangrahs0ahghacwgaazaaf0abv3aagcqad9wbns5c8akehy5el7wawkwctsglgrqaitqnik5 +1 +cols11 +1 +httppatrickwebcomweblogcategoriesmyhobbies +1 +eliminates +1 +sciences +1 +httplistsdebianorgdebianuser +1 +ieyearecaayfakvjzfsacgkq7ro5m7lpzdj3oqceopwxwspcvusohxxnmdhdau +1 +aanlktiloguohoz10mtb5iggjhseh0ohpvihesak7gcmailcsminingorg +1 +xhabeasswe1 +1 +ngbrmtfg5v7isqalzwdij7hpjpyxajhwtwvbdoyv76d7mjf9wh5clvdw3okr6dqistbzhex6h1 +1 +clymerb +1 +felonious +1 +bgcolor3dff0000 +1 +happening +1 +other39s +1 +issued +1 +maxlength15 +1 +5fb78440f0 +1 +h4mr788869fas561273277074209 +1 +g7mbihl25223 +1 +unsubscribeatd +1 +virtools +1 +htddw840abr +1 +24eecdfb60 +1 +ha2gz61qrmddzdjhthpvihmpxfciqvjjvtmlgooxryauuvdrgdqynqvfthi6sk0535wgcz4c9cvh +1 +fake +1 +operationfontbspan20 +1 +hrefhttpclickthruonlinecomclickq20poifirzzenfabiotrpp7us2qf9r +1 +tobias +1 +valuewebmasterefiie +1 +postfixlughtuathaorg +1 +boundary595337d60f6e68b8115b +1 +shoddy +1 +ignores +1 +001619 +1 +pdv5 +1 +avail +1 +52523 +1 +uxv5vrxporvvuvgz9mthptp3fnwluoev7fwaxq9vsu5pjjbycuzsxoqaqaqaqaqaqa5pryq +1 +f28mr13294898fgk701270638043781 +1 +size2bfind +1 +alignrighttend +1 +height146img +1 +darbar +1 +reserved +1 +receiver +1 +investigations +1 +smtpwowsynacorcom +1 +4464cbcf14d57c0ea0525985629 +1 +634554 +1 +repeat +1 +131130 +1 +spinetail +1 +src3dhttpiiqusimages4s200201223agif +1 +temple +1 +1b43616845 +1 +jkqqfrbxkuoqwymdzjglxkasksdjntiajjgneaiuvdhadvaadujesyaatjaarugboaej1jnxbbr +1 +hmuprzd +1 +mundo +1 +anddiv +1 +moutians +1 +ac90 +1 +commerce +1 +contractual +1 +raa04566 +1 +gmailid127d4404807ca52b +1 +agecroft +1 +merchant +1 +064802 +1 +nextpart000042601c26ebd8df911e0 +1 +height3d104 +1 +heads +1 +125135 +1 +exhrlsvrgemsbarbadoscom +1 +prayers +1 +205902 +1 +8542716f03 +1 +colspan7 +1 +tiger +1 +hrefhttpwwwimakenewscomeletraupdatecfmxmediaunspun2czzzunspunspamassassintaintorgclick +1 +a8f2fd042a9a +1 +testsemailattributioninreptoknownmailinglist +1 +equivalence +1 +gmailid128bb599a95e11cd +1 +blast +1 +httpuberbinnet +1 +mitsubishi +1 +scenes +1 +h5as +1 +valuebdbangladeshoption +1 +nameindexr3c3 +1 +override +1 +245 +1 +xml +1 +032035 +1 +110 +1 +xkxlbfqbm7hg +1 +width3d618 +1 +httpwwwchimpswithkeyboardscomblog +1 +mailtobitbitchmagnesiumnet +1 +bzzzzzzz +1 +nonmed +1 +adderallbr +1 +deceased +1 +dãvaluation +1 +mailtonogold4me666excitecomsubjectremove +1 +20020930203145570e6717matthiasegwnnet +1 +verwendung +1 +quotupdatequotbr +1 +adapting +1 +ccg2sjwadlbr +1 +depiction +1 +aawhnoaaaaaaaaagqaokcxlams9bksist4xva6ammsp5ryulfzbwveb0hhoksbkkn4hyuvltjdqo +1 +hrefhttplockergnomepricegrabbercomsearchgetprodphpmasterid568577pioneer +1 +challenges +1 +obtain +1 +uid131321201487268 +1 +12696442904c369b800001w4twrl +1 +dave +1 +values +1 +bhmsg0z8rtvqetjll7zlzc1voaioj4jtbncf0amfj9u +1 +4px +1 +twenty +1 +sing +1 +abdullah +1 +deb +1 +httppackagesdebianorgunstablemainpythonflask +1 +olor +1 +18004014948 +1 +score081 +1 +procurement +1 +seals +1 +didunfortunately +1 +benefit +1 +725524d7d8601394183f59bd344bee82squirrelwebmailapplecom +1 +k3b +1 +n8aav48kjjkfymwdmyfbz58vsunf8zzvsmefprxtjef4sur7d2f8addwnszzq19p7f5 +1 +todays +1 +001043 +1 +styletextdecorationnone +1 +spamassassintalkexamplesourceforgenet +1 +0019 +1 +212745 +1 +021009 +1 +cpmnbsp +1 +ape +1 +hack +1 +archivelatest1295 +1 +alias +1 +natalensis +1 +easily +1 +80347440cc +1 +colorcc0000font +1 +96c805179a3143a8be1b1de74217c166csminingorg +1 +generate +1 +155548 +1 +periodically +1 +centerwascenter +1 +wellknown +1 +width3d76 +1 +hv77a40ickgq +1 +36 +1 +ertzinger +1 +baxter +1 +10pxand +1 +belowfontdiv +1 +dtable +1 +etcaptsourceslist +1 +maccarthy +1 +1374 +1 +hrefhttpclickthruonlinecomclickq35d6jtibizyije2zzknxltsypwc3ur +1 +dropped +1 +blu0smtp7 +1 +pbbenefitsbp +1 +nextpart0000023823983c1f9dfb9df87 +1 +4f98c13a46a1 +1 +des34newsahotmailcom +1 +transfer +1 +literature +1 +neue +1 +125744 +1 +calculate +1 +rzmta +1 +recognition +1 +msergeantstartechgroupcouk +1 +hot +1 +bernie +1 +src3dhttpiiqusimagesindy120020610101ajpg +1 +cd +1 +gmailid1280f785a65bb1c2 +1 +phillips +1 +n12mr206682fad351271790911462 +1 +fontispanspan +1 +possess +1 +c111n115equence +1 +nowatd +1 +023c33c46c3c5555b5a10ab57ad8alpasn +1 +pearls +1 +nosave +1 +conservation +1 +leoma +1 +fingers +1 +testscapinithtmlmessage +1 +approval +1 +142612 +1 +kre +1 +romanstrongviagra +1 +cancer +1 +bolceri +1 +tracker +1 +cmmsysaolcom +1 +immutable +1 +9645519711031163664828javamailrootmonkey +1 +homecbipccom +1 +hrefhttpclickthruonlinecomclickq96pjnwqtdhdsjmy1ujuf1uaoeg0srr +1 +pepper +1 +sentto2242572601671038790646jmjmasonorgreturnsgroupsyahoocom +1 +designate +1 +0pxacentertd +1 +industrybr +1 +behalf +1 +excl117si111ns +1 +orwig +1 +glettersummarylocationhover +1 +requested +1 +colspa +1 +kaa82784 +1 +deloptesyahoocom +1 +publishes +1 +84c7 +1 +302 +1 +f64law4hotmailcom +1 +portions +1 +xspam +1 +ssvqukvbh7h7ffem3cyf48nu4or874loeknl8zgakbkkwprcqa7t85zt39dc9vd8f0h +1 +hughppatcommediacom +1 +035150 +1 +060256 +1 +variety +1 +clubs +1 +friction +1 +linuxaccesible +1 +serie +1 +hrefhttpclickthruonlinecomclickq42rf7cipkskxmmrb8pz3f74emirdr +1 +96b8a2c313 +1 +width214td +1 +grubgfxmode640x480 +1 +9a0a4440a8 +1 +di102102erenti97lly +1 +godo +1 +175109 +1 +skt4dbr5ztcs92wvrlahpcyhd6ubvtazx0qzyj2mtylwylaczwhcortchzq +1 +switched +1 +centerbr +1 +compulsory +1 +ids +1 +nvidia +1 +vprx +1 +started20 +1 +supermicro +1 +lenny +1 +d4cdcd +1 +freddie +1 +hicsanchordeskfrontpagebgif +1 +class3dtitle149tdtd +1 +camaleãn +1 +height3d8720 +1 +alyon153176230w86202abowanadoofr +1 +lagos +1 +162605 +1 +admitted +1 +classstyle2a +1 +pack +1 +r16mr854383ebj391273426686767 +1 +115656 +1 +hughescr +1 +grads +1 +brmy +1 +singh +1 +ll +1 +n1iejasc014383 +1 +7ac06fc132 +1 +192733 +1 +loud +1 +n6s95v91017981 +1 +reimaged +1 +economical +1 +2461113164 +1 +hjzpo95gxteabsc46izlbliszt +1 +hijackers +1 +butterworth +1 +001937 +1 +p0cgsr1rbjkanlhhy3lbliszt +1 +srchttpviewatdmtcomaveviewaws05000012avedirectwi160hi60001 +1 +dsa20511 +1 +burning +1 +downstairs +1 +uid121371201487268 +1 +ttnetnettr +1 +17th +1 +butt +1 +aligncenteralso +1 +breaks +1 +advancedspanbp +1 +191108 +1 +vlink3dc0c0c0 +1 +n7hllnc4009662 +1 +230 +1 +t5cejhoavnwojfoxhvs653llidasxhhti8p0pl3xwk7pt2rillv7ladaqbid4fjy24 +1 +httpedtechdevorgrights +1 +isnbspqqqqqqqqqqcnetnewslettersexamplecomp +1 +bluea0 +1 +communications +1 +plug +1 +hrefhttpa03vkunrorfcnucydodyh94hg50z49913x91v375i8qiamaa900799114670527935932 +1 +fm9krnmbjknpthnrmukdzjjvdqalnm4pyzk3b61rvychhgj0pcd1oa06kjgcyqkx6kvjqaucue +1 +102134077 +1 +1434a441cc +1 +verdanawhen +1 +wisdom +1 +itbr +1 +cerro +1 +fffwebsitesatd +1 +5173916f16 +1 +importing +1 +cpu +1 +recognizes +1 +mailtoaugdlistsapplecom +1 +regulatory +1 +champion +1 +47453e +1 +zfqm06mneuxjnnrtyjelltbhsiwxg5zuqdkmpambr3fsospgau9gqifuxt8uqyzav6u7fz2gmxr +1 +dateinpast96xx +1 +ngdbrxm0msqhfbwii7pnhllplhmcmthxamedfieklbimam2wik0ectyh2qvpz4qwbicggjbne +1 +size3d26 +1 +nextserver +1 +6521443159 +1 +be67e6c8431 +1 +searsheatsystems +1 +blood +1 +di +1 +greater +1 +specialistabr +1 +boyffriend +1 +virginia +1 +regardless +1 +totoro +1 +smiling +1 +respectfully +1 +motherboard +1 +brbrmy +1 +w8it4ejlkqswliuj61vq81znhuxhuqfda5ii9s10nttzgdbvcjcgvzsoo5b1fjk56e9kricszq +1 +align3dcenterb +1 +200210010801g9181ak15561dogmaslashnullorg +1 +dk828c65f4fvat6c1vkmf1ckj746v2mmlnk1m6akfzdkgwi5oc0rcpgy0gmmajkyioxc9cauyr +1 +score060 +1 +debt +1 +separated +1 +sivaram +1 +teaching +1 +smh +1 +235733 +1 +dr +1 +hrefhttp0b8b9xbapobazcnqb7d50344902d57383a04684dq4830232601834252602452 +1 +obscures +1 +yorks +1 +xmimeautoconverted +1 +cryp +1 +lughtuathaorg +1 +drill +1 +uid69781201487268 +1 +sufficient +1 +lrwxrwxrwx +1 +unsigned +1 +colord76f3a +1 +color3d22ff0000223esanayi +1 +listing +1 +consecutive +1 +privs +1 +palm +1 +stopvipeprojectwebcom +1 +4096 +1 +penny +1 +p4cs140617wxp +1 +dadada +1 +qnxmdrostcom +1 +085616 +1 +recommendationustronguem +1 +ckcakkkks2stdrlyp2afj0gqx6grfetkhbqofdhh5pli8trf6dp2uddiji2hbyxb2ifsusjf2h +1 +o126x7ic008323 +1 +vidal +1 +httplistsdebianorg87zl0ccgerfsfturtlegmxde +1 +dicecomb +1 +886882 +1 +e15si5258490fai2320100514020855 +1 +xfce4notesplugin +1 +arialbrowse +1 +nonvowel +1 +style3dbackgrou +1 +311 +1 +fraudulent +1 +033621 +1 +thrush +1 +httpradioweblogscom0100945 +1 +17hyuf0000ob00 +1 +1244f270b0654 +1 +wwwdatalocalhost +1 +occurred +1 +tests +1 +aol +1 +holidays +1 +282178175 +1 +104411 +1 +195554 +1 +o11lr7nz000092 +1 +ushered +1 +specializing +1 +quarries +1 +telling +1 +thisb +1 +amirah +1 +015841 +1 +psychological +1 +infoznioncom +1 +jfq3p2ffwopipvimfbdjcxaaoopishyejdyt7t4r2zxclpibwt1vt37xciigyncta +1 +040947 +1 +verticalalign +1 +excerpt +1 +g6hlklj24842 +1 +freebies +1 +dunno +1 +wf +1 +aaaaaaaaavqaaahdaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadhcsu0efbdm +1 +asmtp +1 +elaborate +1 +1030135594011692caramailcom +1 +9b1243ca2c5 +1 +programi +1 +235829 +1 +hrefhttpfdthedrickerectruywafodabif05fecf0efa1cb3 +1 +width36 +1 +trustworthyfontdiv +1 +18441 +1 +forecast +1 +dolly +1 +christ +1 +123003 +1 +airkvbgf3v46fcoh1nyafmw117 +1 +titleterms +1 +diocese +1 +ust +1 +src3dhttpwwwciscocomaucomvpsgtccdadisplayimage0img24700 +1 +icagicagicagicagicanciagicagicagicagicagicagidxmb250ignvbg9y +1 +97dd +1 +chiefs +1 +nonepark +1 +avittaa +1 +ew91cibzcgvha2vyczwvyj4sighpz2hsawdodgluzyb0agugdgv4dcbhcybpdcbyzwfkcyeg +1 +country +1 +solitaireabr +1 +9b4c44bdc +1 +fel +1 +024820 +1 +thinganonymously +1 +intuitive +1 +6621867201 +1 +llengua +1 +0976c13a4ed2 +1 +brwebsite +1 +walking +1 +width3d50font +1 +50000000 +1 +214223 +1 +newly +1 +thep +1 +alien +1 +indisecouk +1 +410 +1 +lefebvre +1 +rmgbam0zngaahk0mabmjnads0zpmam0apzszpuam0aozszpm0maahzoztc0zoaxngatnjmgb2at +1 +verifyno +1 +lespaulbhotmailcom +1 +162358 +1 +1253735491 +1 +roundup +1 +ancestorb +1 +71 +1 +bsfsc0satofromdomainmatchhl +1 +bjkgnbxng1vdahiniqc0rawgbheojhu1c83xmommxytwaydjsstiqzyugrqyrlavatu5ys5oab +1 +233 +1 +aidshiv +1 +200209070351g873pc613144pcp02138704pcsreston01vacomcastnet +1 +alternat +1 +profiled +1 +revenues +1 +excuses +1 +o1mmtksn028028 +1 +quotnquot +1 +phillipm +1 +height3d217td +1 +daytitle +1 +betas +1 +atom +1 +standardizing +1 +http13624411618000cgibinmarkcgi +1 +121633 +1 +121819 +1 +stylebackgroundcolorfbb034 +1 +54jwo3xf0fnl8cjfai3vxuvg54q0rs7vuz9s05orsb53ctyliqqwomooccvowsbxjyjou9 +1 +residential +1 +hulda +1 +width550a +1 +formating +1 +derivadas +1 +285326 +1 +091632 +1 +b250igzhy2u9irzqt6lf6sigy29sb3i9iingrjawrkyiihnpemu9ijyipqazm6ztxmehpc9mb250 +1 +220617 +1 +cloud +1 +teach +1 +ofh5 +1 +unionleader20 +1 +width304 +1 +55344fontpbrbra +1 +741433120 +1 +74ef813a5136 +1 +partner +1 +48995 +1 +o3q8nxjs010416 +1 +carpenter +1 +alt3dfeatured +1 +divcontent +1 +ogxkqpzfz3auda8fnlimnpzuip8awn73eudesfq7vzhzmti3gfsuhgacjetu3b5gdvxkfitcdsh +1 +worldnbspnbspnbsp +1 +devselfast +1 +emotion +1 +jmwebdevjmasonorg +1 +wholesa +1 +httpwwwmemoriafotonet +1 +marketplace +1 +gmailid128031e1627510ba +1 +opakkzzaiw0wtvdlwsbd1xbeuqgcsmcgef8k8tkdo3bgjgjud6h3r3u4xwbrph7sdsg6sixt8k +1 +submittdform +1 +bbsayencsminingorg +1 +paddingleft +1 +adjusting +1 +dsimaccocoaviewref +1 +dictionary +1 +faqfontfontafonttd +1 +4jj8xc5jj6ihxa9bvcg7g8scbrkipfu5yahdwzxleqy1qdey6kcsm4chqb1ru0usog0u7redyd5q +1 +practiced +1 +boundarynextpart00000234301ca9e59c243bc5a +1 +332 +1 +httpdocsyahoocominfoterms +1 +sediments +1 +326 +1 +puslingcom +1 +firmware +1 +httpwwwbarracudacomreputationip63218169154 +1 +pennies +1 +rgb255 +1 +legala +1 +citizens +1 +valuehttpwwwsearchcomsearchtagexelectronicsdispatchsrchwebthe +1 +combine +1 +width3d517trtdpfont +1 +wyf22 +1 +razorcheck +1 +bsfsc0sa081a +1 +determining +1 +shipping +1 +focused +1 +berserk +1 +yv7vdjsur +1 +20100506140413721bssiguanasuicidenet +1 +decline +1 +administrative +1 +colorff3333bexclusive +1 +banner +1 +abandonment +1 +4996 +1 +hasb +1 +brcenter +1 +settler +1 +professionals +1 +1388464 +1 +diploma +1 +libslinet +1 +magnifying +1 +signatureshortdensespamphrase0102useragentthebat +1 +c0bfc1bf7c068 +1 +yill01hotmailcom +1 +yield +1 +ikifi +1 +q11mr2461275ago901274680109300 +1 +etch +1 +axjlywwdqo8aw1nihnyyz1odhrwoi8vz3jvdxbzlnlhag9vlmnvbs9ncm91cc96agvuc2hpmdev +1 +fingertips +1 +thoughts +1 +value4444option +1 +h97s +1 +evan +1 +offices +1 +100000 +1 +173118 +1 +softwarefontbtd +1 +srchttpmta33annfilesfourscombp5reloowuanicecxiriaeltfwaevxnadeeakbtoeineiioceeoqxeaaaiqceanakwie +1 +g8tihhkl005444 +1 +r5e8c72pf0ycxet6sqc +1 +tongue +1 +v15 +1 +complaints +1 +networks20 +1 +66b6e16f16 +1 +navigator +1 +width3d43 +1 +cqfehwwyeciarzatjaw4ahehiq61sboocbfllqlashcmmpaggifdjhpzabtagdoudqqwooacaba +1 +smtpmailbouncedebianlaptopmlsubscribertechcsminingorglistsdebianorg +1 +interaction +1 +mãtier +1 +aligncenternbspp +1 +frances +1 +retpoc59dto +1 +apathy +1 +bifont +1 +priceliulp +1 +045703 +1 +httpradioweblogscom0105617 +1 +size3d20 +1 +httpwwwrolandtanglaocomcategoriessoapnxmlrpc +1 +win +1 +c13mr4723586rvi861270530782703 +1 +xhost +1 +linuxmidrangecom +1 +newspaper +1 +mozborderradius5pxwebkitborderradius5px +1 +styleborder0px +1 +possibility +1 +featureprods +1 +mutt1251i +1 +m09 +1 +065659 +1 +n8f7tgt9010451 +1 +aligncenterhurricane +1 +alguien +1 +width445 +1 +height454 +1 +3cdiv +1 +o2g2fc5f091004111716oe078137dse05f0b82f4e519ccmailcsminingorg +1 +idl7569s971t24u0 +1 +listsdebiankdelisztdebianorg +1 +hrefhttp8642fsponevohcnjp11d2ff2653f3bb19d8a9aqk008199133291771685656 +1 +faceverdanabr +1 +114258 +1 +activizes +1 +0padding +1 +facetimes +1 +203338 +1 +jmart63573attnet +1 +661784619 +1 +tape +1 +27a +1 +jmcheetahnet +1 +2c4af43f99 +1 +090147 +1 +implement3dwiels +1 +ikozi1950lexiconpartnerscouk +1 +fazz2lxxjfbdlqmfvgi4ldy09h1vvu06l91vrlp5jllcxtdbsceofdqbduesshgfczxw9vsken +1 +poorest +1 +malheurs +1 +282177138 +1 +proof +1 +controlling +1 +bulkscore0 +1 +color999999balso +1 +aligncenterlondonp +1 +aptcache +1 +005217 +1 +styletextdecorationnonespan +1 +hopefully +1 +18b4429409a +1 +125in +1 +aekawacsminingorg +1 +httprecquyfmyjcom +1 +3189319 +1 +color0000ffclick +1 +thereafterthe +1 +boundarynextpart2rfkindysadvnqw3nerasdfcharsetgb2312 +1 +16410024360 +1 +classmsobodytext2 +1 +2827262524232221201f1e1d1c1b1a19181716151413121110 +1 +fscking +1 +uw +1 +demgegenuber +1 +alignabsmiddle +1 +e20 +1 +alt31509jpg +1 +ãbrigens +1 +moosomin +1 +tomalformed117 +1 +color000000worldwide +1 +accountant +1 +studying +1 +smalltext +1 +labr +1 +4h70akibujkhenxgzotimsmgjokldq7ocgc1hparxkylhpbfobx0ol4702ktcy5djdzd5bzxvzo +1 +n4gnsmkx016065 +1 +343332c58d +1 +bordercolor3dffffff +1 +provides +1 +mebr +1 +20769200243 +1 +cds +1 +204942 +1 +tagged +1 +acuoyloagiol +1 +3energy +1 +trjsuma07fhjlsn6vuuatjqrbmxtjinjmcnlfn21kevsluzpsptycijk4aqxakkchsiiecmkgx +1 +redmond +1 +harshly +1 +2112483279 +1 +cellspacing15 +1 +xig4oeoqow1co7xarbx9dwlgzicemorfqxtzisq2kdngqsbx3pxwmssemokqcussxfp2nokm3wk +1 +packging +1 +attained +1 +hrefhttpclickthruonlinecomclickq1ay6zkilxxica1bbd5q0hzb17wnr +1 +expenses +1 +disc +1 +cluster +1 +banta +1 +sessionin +1 +lowered +1 +celebrity +1 +styletextdecorationunderline +1 +43094764257 +1 +archduke +1 +shellsubscriptionslockergnomecom +1 +153536 +1 +color880000please +1 +b4zgvdnr0dadzfzxw6lbliszt +1 +width112 +1 +size1a +1 +talkabout +1 +size3d22322 +1 +sticks +1 +stylemargin0px +1 +uid71111201487268 +1 +195143 +1 +gotten +1 +0l1a00c85x0u4bd6vms173015mailsrvcsnet +1 +mattei +1 +httpwwwugnncom201004macmalwarealert +1 +border0td +1 +reporter +1 +booka +1 +retail +1 +began +1 +qgjqqifkxpdcyhbcwpdoe0xeky5k3g7pcb4ycu8xiakcqifsxxibabowhkbpx3igxndegl08ajb4 +1 +los +1 +size3d6span +1 +xarrde7yy5px61oseod2oaan3fllmp5uapzmk5pc5aortldrgjrrsz5pxawuf8ahf8ayj19wbs +1 +green +1 +backgr +1 +1950 +1 +060259 +1 +krammer +1 +hlduejgyhqbe3ldumrqjxmda9tb28rqhlduejgyhqbe3ldumrqjxmda9tb28rqhlduejgyhqbe3l +1 +verifyok +1 +wwkarboswmlfbkugywt2oyaxgwaqzogjw2yfmcgacbdqxoqxnqkrbekxsn8ie8zqkdw37ji6hs +1 +2010042717180153fff689celejarcsminingorg +1 +65000 +1 +zenith +1 +115134 +1 +nosend3d1 +1 +bindustry +1 +5143 +1 +f111rmatting +1 +2771816f1b +1 +referencing +1 +sitenamewhatever +1 +bunsubscribeb +1 +room +1 +gasp +1 +boundaryc04e5c24b3ae4ba8a82d2aa0500e2d84 +1 +columns +1 +route3f +1 +22sm2126438iwn020100405221301 +1 +94304 +1 +shuffle20 +1 +rvbpulrtifrprefzieforcbhrvqgu1rbulrfra0kt04gwu9vuibst0feifrp +1 +200210050800g9580wk02532dogmaslashnullorg +1 +virgin +1 +httplistsdebianorg20100511055911ga17757europaoffice +1 +elias +1 +tonsure +1 +kerdylio +1 +popularity +1 +glpopmatrix +1 +par +1 +k2q308d4701004140621ibf159f8dycc2bb534b1aeb7bdmailcsminingorg +1 +008 +1 +httpsecuritytrackerdebianorgtracker +1 +hugodebianfvwm +1 +flashbased +1 +1beae43f9b +1 +anda +1 +fast +1 +convenient +1 +defjakxfyceavqjrf4tmkkb83hecqli5kq4wcat5ndub62gzt4y6ecsfwqn6riwantzs +1 +f563 +1 +size3d43enbsp3b +1 +twitt +1 +cents3dmejor +1 +utilizing +1 +courtroyal +1 +172204166 +1 +titleuwouonovaqiv +1 +newer +1 +081502 +1 +valuejpjapanoption +1 +89448p +1 +5a1ebbfa9aaa9ada6bbeca9beeca3a7a1adf6e389f6e2e882 +1 +exchangeeliteadvcom +1 +2d20 +1 +width0 +1 +height3d181 +1 +rectangles +1 +eat +1 +pcetlsbzyxzlzcbmcm9tihvybd0omdaymilodhrwoi8vaw50zxjuzxquzs1t +1 +3d1e5c99 +1 +01aab78f9c3 +1 +instinct +1 +lbnyepl2uuqszsm4omqfpznhfas0uswcehrqlla1tzhedouwkuzc849x0ufy8d32b1wuqchzppot +1 +outofprocess +1 +h17mr2281000fab451273377014844 +1 +123507 +1 +width715 +1 +sw9 +1 +pricy +1 +musicians +1 +hrefhttpnlcomcomservleturlloginemailqqqqqqqqqqzdnetexamplecombrandzdnetmanage +1 +restricting +1 +gulf +1 +vietnamera +1 +weld +1 +sweepstakes +1 +125117 +1 +zzzzspamassassintaintorg +1 +144006 +1 +shout +1 +minimum +1 +ettled +1 +altthe +1 +replace +1 +doit +1 +value3dinsuranceiq +1 +skeptic +1 +alignrightsent +1 +width3d15nbsptd +1 +214046 +1 +titlefor +1 +041 +1 +hrefhttp8ad48d3d6ef183pointdoesru +1 +noticeafont20 +1 +j2se +1 +andreimpopescucsminingorg +1 +r16mr3596565fam621273464972509 +1 +3189699 +1 +87r5m4d0w8fsfmerciadrilucaeeeworkgroup +1 +northwest +1 +enter +1 +102809235184 +1 +072701 +1 +hes +1 +3d6e09845030708startechgroupcouk +1 +041718 +1 +lnv106sbdfkide +1 +hrefhttpclickthruonlinecomclickq11l8cii2ap8sfpx5xa0eixbhh7rrr +1 +httpwwwtheregistercoukcontent426170html +1 +20020910020728ofei9751sccrmhc01attbicomintellistation +1 +215956 +1 +4c2 +1 +fontp +1 +893893112 +1 +31103662 +1 +z32qt8o5phnv5oflgvmzyomtnnnvjyofkbdvepyghsmun2qjaodwk1qe15qmsthdbdhwkivxx +1 +color0000ffbthe +1 +nyu +1 +alttech +1 +neil +1 +15inch +1 +6118820325 +1 +size2table +1 +risc +1 +paul +1 +targetblanknewsletterssubscriptionshtrdwbrcomastrong +1 +httpternetblahcom8080 +1 +boundaries +1 +hrefhttpwwwlockergnomecomsubmithtmlsuggest +1 +cs144080pphtvfi +1 +kelly +1 +effect +1 +bob +1 +accounts +1 +motion +1 +095320 +1 +contributions +1 +muycjbpw7sqq7arjkdar1agv6uejdjhwlu7u465pd0ohsgvyptwdyl0mtlh9xusgg5akqjbf +1 +near +1 +width600font +1 +hampshirenew +1 +e102102ect +1 +girl +1 +hendrick +1 +minuteifont +1 +align3dcenterall +1 +administratively +1 +mlgaytrdxk5kjd84qb0mgwwa7ix5pxxgfrobczg1zdpgo2n9xobaaxmxg6zdpfylqaolfzhigd +1 +800 +1 +relay2applecom +1 +s117cces +1 +coworkers +1 +dancers +1 +imprononcable2ldosubscriberldowhitelistmurphywrongword1 +1 +102809231923 +1 +polyketide +1 +testimony +1 +n9e9rdga028827 +1 +spamd +1 +facilement +1 +1d1f02049c5d68fb4 +1 +rsync +1 +pjwvcd4ncjxwigfsawdupsjszwz0ij48yj48zm9udcbzaxplpsi0ij48c3bh +1 +takeover +1 +larger20 +1 +hydroxyl +1 +name3dhowsoon +1 +align3dcenterfont +1 +removeeuropeaninternetcom +1 +unique +1 +randomly +1 +mothers +1 +m95r97u97tsp82qc +1 +imperious +1 +sbfontbr +1 +hammer +1 +uid46431201487268 +1 +hyderabad +1 +mplayer +1 +mailservice4imakenewscom +1 +titleyour +1 +0pxoff +1 +solely +1 +hrefhttpwwwdmbodycollectorguashabby22html +1 +lobbying +1 +896px +1 +xenical +1 +garst +1 +sir +1 +blogspot3 +1 +5cab713a4f5a +1 +attemps +1 +repository +1 +stars +1 +cathedrals +1 +aimsh2 +1 +srchttpwww3videoprofessorcomimagesjohncdjpgaptdtrtablebr +1 +wxg2fumgb7gpmoufq1ruwvs5vpyyq3plvacdofvvvusmxly9tnckbo9lbvwq7machip6002 +1 +075648 +1 +3193991 +1 +181908 +1 +vlc +1 +o18nishx010565 +1 +194913 +1 +ac5b713a526b +1 +sametime +1 +zzzzlocalhostexamplecom +1 +exceed +1 +younger +1 +devnewdevice +1 +httpwwwauracomcomteledyn +1 +e1ecb2c737 +1 +l30mr1080942waf1341273840106932 +1 +wjqccl8tyur4kqjghsvsuhlxegzskxhwakjc1okjeiul0 +1 +limewire +1 +windsor +1 +httpsvcsafferonetrmphprjkeating +1 +solve +1 +xegroupsreturn +1 +xygmxtrusted +1 +epicenter +1 +charset3diso88591 +1 +titlezdnet +1 +height9a +1 +titlebold11px +1 +bailey +1 +confirmationfontstrongdivdivbodyhtml +1 +r13mr2301595wak111271447853982 +1 +inches +1 +dir3dltrlt +1 +bulb +1 +097 +1 +m8cs47341wfo +1 +2iudjoyinlotgv24tssq8awkxegagjiywnsfwiyz7vu9tiklaapkffuqalyx4qadi4hgqvaf2dr +1 +1010112520 +1 +prosper +1 +cdd980289cb141329f52b72c6d375dddattnet +1 +1layqdjehgkkpalg6gvza +1 +zealand +1 +userfriendly +1 +c111ntext +1 +ily +1 +hrefnews42htmltopback +1 +hrefhttpclickthruonlinecomclickq3doi9nibz4iutiayrvwlgwutnantpr +1 +ccdsgensetscom +1 +wvyhysgn9xx5ex7gsvdwtnotmd1zwf9ug5nutw8d6ukdwmcw5lpac1iwglcnctyzstdts +1 +135211 +1 +pf22hxseyrvfdg6iqrtxfu8pxestskwrkhd3i +1 +davelocalhost +1 +invariably +1 +undertaking +1 +synaptics +1 +trtdp +1 +il +1 +052810 +1 +aeoliccenter +1 +fbi +1 +lord +1 +164034 +1 +fam +1 +color404040contactfontanbspnbspnbsp +1 +k2n1f5d398f1004061040t82b62056s7b582b73b234f060mailcsminingorg +1 +uname +1 +guwaazyu1fnriwopaszgdocroqeefgikjostpu8jatrmrdnapeecbdzibpayzlieemayuhbgg9o7 +1 +100ivers111 +1 +emthatem +1 +height3d349 +1 +hrefhttpd5d01qutuhixcnsod47524c20ffbc695a1ac0c8iz2431271099774536068075 +1 +aaaaaa +1 +11fe0d07b7c36ae000006674e24bd201b2e0b4 +1 +brjmnetnoteinccom +1 +ãããããããºãããã +1 +tyle3dwidth +1 +romanstrongcialis +1 +132734 +1 +ol203154fibertelcomar +1 +thmfginactivetext +1 +185844 +1 +6hshjppdzpojntn2qjac1gaccbkc15pxpiphjfimqbevmb3encsoafifpjxpmbivuzo4ueg +1 +chef +1 +132119 +1 +6yh2htacfeet1sq9qvf0umsoqrfdzyexafexdz6a5l63ihvdwakmpscpzzdijlody +1 +tradition +1 +sideeffects +1 +5si910059fxm3220100408104230 +1 +chaos +1 +catsmx2ucscedu +1 +jury +1 +deeps +1 +max +1 +switching +1 +180438 +1 +soldadura +1 +name3dsubmit +1 +blc4kclstjndy1wiiz1qp2r3blwzqnlwz63zdwd1vuxb2ipq4lsxqvdqrrlghxbx54m +1 +electric +1 +wilmitdekalbnet +1 +score39 +1 +httphgsaqeduxurobcomaae494e46ec68b71f45b720e723294525d4 +1 +135453 +1 +gmailid128d2a74ede5b55e +1 +diversifieds +1 +readonly +1 +li +1 +valuehawaii +1 +size4 +1 +hrefhttpwwwgnomedexcomour +1 +651page +1 +daly +1 +bluecat +1 +03beta +1 +acaangauadaaaaabadhcsu0ebgxkuevhiff1ywxpdhkaaaaabwaaaaaaaqea4adkfkb2jlagsa +1 +httpwwwlifetimetvcomreallifehealthfeaturesdrugfreehtml +1 +helenaoption +1 +lesserpowered +1 +color3d666600afontfonttdtrtbodyta +1 +torquent +1 +icmgmi4gswygew91igrpzcbub3qsignvbnrpbnvligfkdmvydglzaw5nig9y +1 +namegeneratorhead +1 +decades +1 +alignleftp +1 +win32 +1 +binsh +1 +3c2ffont3e3cfont +1 +extranet +1 +icompare +1 +highresolution +1 +swang +1 +chksnsiubhxgsr9ewp8mmxiefxvyaro28hartmfcxccdbfhaapwmpfbeaulbbgeqemajfj1qbloe +1 +parting +1 +smtpserver1cflrrcom +1 +121303 +1 +10874936 +1 +floating +1 +wantap +1 +style3dtextalign +1 +3ak1ky3ehpyphfrzvmimvrfj1hnic4g9oagcjkvfkgahigumd6lapmu0zxmmjpz1pu5dzvfdl6 +1 +molt23aolcom +1 +sendmail +1 +grave +1 +m3grpscdyahoocom +1 +instrument +1 +versioned +1 +hermanos +1 +novak +1 +morissette +1 +gbgm +1 +valigntoptd +1 +objections +1 +haataja +1 +hrefhttpdhue39parkchanceinfo106313155381d2114410248htmif +1 +pbxf1ex1vmz2hpamtsbw5vynn0dxz3ehl6e3xaaawdaqaceqmrad8a7p7dzolf2zel9us0 +1 +guaranteeing +1 +g7fkd8717169 +1 +rolandapiggee3zfhotmailcom +1 +signs +1 +wblrbl7f3udxrsutzoskkkkacumlvacthv8atj6osuzrjpi7ythv2p8a6osgd5oooopl +1 +g2icdeec9051005080532z810046edveb2a8ea4c51fa0d9mailcsminingorg +1 +mind +1 +101 +1 +020dd40c52 +1 +peaceful +1 +204045 +1 +g8girhc05769 +1 +selfnextserver +1 +imo +1 +133mubabstnbstnmacodslattnet +1 +proclaimed +1 +4924161298765444333 +1 +hin +1 +bgcolor000000img +1 +1257841461244d004b0000w4twrl +1 +border3d0b +1 +centerjoannes +1 +ocean +1 +nowrap +1 +k111nk117rrenz +1 +debianid +1 +intervals +1 +venture +1 +suppressant +1 +1ofaib0006mdol +1 +2495 +1 +dynamics +1 +chico +1 +thurman +1 +c911f16f6a +1 +gcolor3de40e18 +1 +jocks +1 +200420 +1 +tremendous +1 +122916 +1 +testshtmlmessageimprononcable1 +1 +color3dcc3366b164bfontfontdiv +1 +named +1 +rebel +1 +picked +1 +vspace1 +1 +size3d5bguaranteedb +1 +i3ookxilckivwcrlavp0qtjerwvfcapj4tm6tz2gnka5wn1uobehqk0fti6pdhnbabugxcxofsc4 +1 +contentdescription +1 +avoided +1 +gmailid1288ae8d351ea5ac +1 +3d2223cccc9922table +1 +lis +1 +132635 +1 +3cp3e3cb3e3cfont +1 +walk +1 +jody +1 +hits56 +1 +httpwwwfacebookcomeditaccountphp +1 +dorm +1 +71ce443f9b +1 +preaching +1 +pointing +1 +archivelatest573746 +1 +implied +1 +strigi +1 +riverp +1 +1931201713 +1 +booksfontfontfontblockquote +1 +align3dleftfont +1 +hrefhttp994emedgaspard24aruuaiwiaaq1c3c22a5e86 +1 +prestaties +1 +returns +1 +serves +1 +heraldic +1 +brbrit +1 +2fnsvxbs3dfasgh7slbliszt +1 +polygalae +1 +repositories +1 +valueguyana +1 +nbspnbsp20 +1 +javaspecific +1 +zdctwnctraifibajemsgsgjggbrkgegkhheeafjsmrghghzxcaijxpuip3ek4uqfdsaby60ed +1 +marginbottom +1 +phoboslabsexamplecom +1 +srchttpimg188imageshackusimg188518jivezezyialeekjpg +1 +boggs +1 +timeline +1 +irrational +1 +ttyx +1 +colorcc00003650fonttd +1 +width3d403font +1 +onlybfont +1 +zzzzexmhexamplecom +1 +marginwidth3d0 +1 +value3dinfofairlaneannuitycom +1 +arialb +1 +nbebfve3023203 +1 +180642 +1 +abvsfo1acmta5cnetcom +1 +ij607wfeygk +1 +oral +1 +fonts +1 +201518 +1 +width3td +1 +href3dhttpwwwjhstuckeycom1080jpeghttp +1 +t2si11383950faa7820100511070717 +1 +promotional +1 +16r5pmvbfkw80dztjz6g8bnf5iazvtjfgq42t7f9ogjt0qgi1vyhkrgwrwr5rkkt8mv4gzerocel +1 +boundaryenigbb12e0cc4da2db4f9cf5ef72 +1 +gamix +1 +couplings +1 +d97367fab23 +1 +compliant +1 +httpwwwwomanchildcombloghtml +1 +nbspwe +1 +81238123us4srs +1 +color3d000000save +1 +settling +1 +face3darialhelveticasansserifguinness +1 +barent +1 +19216810053 +1 +asset +1 +49d10a687b034multipartmimeboundary +1 +volunteers +1 +unrealistic +1 +permanently +1 +bordercollapse +1 +halfhessian +1 +9126mm +1 +postulate +1 +managing +1 +sinyal +1 +skills +1 +172122 +1 +17021013354 +1 +width3d590 +1 +subverted +1 +dp6h4t9mu9xxvruoyzekynp57impiz2hlpumcuwnnnmpa5edmmfuba4gz8gpm21xgtxfo +1 +appreciate +1 +54e7d16858 +1 +emwriterem +1 +reposed +1 +tcpip +1 +href3dhttpwwwfabulousmailcomhigh +1 +vivekkhera +1 +092036 +1 +200910211139n9lbdlav015616ns2csminingorg +1 +3height +1 +nvspqehnbdc6tbsdyfrb8rzanj1ztuk3ncylneu0guuaqu9chixrl5faaaayl4jk6oadlrntkj +1 +vorfeld +1 +vspace10 +1 +quotedemailtextreferencesreplywithquotes +1 +27af0440f0 +1 +ldowhitelist +1 +122 +1 +trtda +1 +aligncenterbillionaire +1 +223948 +1 +bsfsc0sa271x3 +1 +vizepekey +1 +outweighs +1 +3skglkm3qpsh1wbg21otxbxtx91wqx4y91kon0w2epuaz6dap5ot0uwcrptqyawortymt +1 +gmailid127f8745bb10e852 +1 +alink66cc66 +1 +airborne +1 +int +1 +anthropogenic +1 +httpswwwinphoniccomrasprsourceforge1refcode1vs3390 +1 +educationin +1 +1986839 +1 +1146 +1 +aframelink +1 +mailgw0f47googlecom +1 +receivefontb +1 +cellspacing1 +1 +hits9733 +1 +size2you +1 +basisi +1 +text3d444444 +1 +align3dcentertable +1 +tray +1 +tune +1 +httpwiorynivhexliqsnet +1 +httpfreshmeatnetprojectswebcppgui +1 +151149 +1 +vcmfc +1 +nuevamente +1 +10142633 +1 +herestrongp +1 +170725 +1 +cannibis +1 +httpwwwlifetimetvcommoviesindexhtml +1 +nbhib5gu010161 +1 +debianspecific +1 +20020204210703ijfy1068fep02appkolumbusfipihlajakotilo +1 +141832 +1 +k117nstmarkt +1 +hazardous +1 +17231331 +1 +emwac +1 +f97cebook +1 +5232c1c26e994 +1 +000356 +1 +regulates +1 +we0ffer +1 +archivelatest575647 +1 +145847 +1 +1087506 +1 +7667403577451363125 +1 +c4 +1 +aye072 +1 +185850 +1 +creeps +1 +johofalohanet +1 +53 +1 +smtpmailysauadyco3604eurotelcz +1 +karesava +1 +unconscious +1 +20010803114020704271a6051rovdb001rovingcom +1 +101426910 +1 +umgs +1 +19972002 +1 +tablesidebar +1 +resolution +1 +howland +1 +ttpwwwwannaberichnetmortgageclick +1 +werent +1 +0063c6 +1 +href3dhttpdiscountshackcomehtmclick +1 +partnershipfontfontfontblockquote +1 +strongtechnology +1 +1246002203 +1 +httpwwwpaulkohlernet +1 +vzfejtuxyheekhcti1bxnbyo17iunt7pskfxk +1 +xsessionerrors +1 +v149 +1 +0909tr09090909090909td +1 +weblogs +1 +68 +1 +hubert +1 +link1 +1 +120730 +1 +nangabadan +1 +bord +1 +cooperating +1 +023515 +1 +1110 +1 +breplica20 +1 +025239 +1 +202555 +1 +apachelocalhost +1 +1091516 +1 +kioslave +1 +runtime +1 +n4py43roqji9hti9qpddhfdahencxzt78muujyyrnyp9ugiioapszaknfk0w6rdofteaijwnfl +1 +kings +1 +resell +1 +gmsorlyuko5xvaj2qpve9t2nvjrh6aytmad755j9kxafdpluwrpyq2njfjp5pxi4ra2pxcdfr +1 +miles +1 +trusteddealer +1 +excerpts +1 +072859 +1 +073615 +1 +epsons +1 +dxb +1 +addres +1 +l5f903kp6lzwh8eneootn9yfwq1b1pm7op5fqpdpj5zm9hfxatscqtjzoag7nbvzvbd1 +1 +mohsin +1 +eeheoxaba +1 +width121 +1 +10223132210 +1 +fu3d +1 +3dffffef +1 +ua20 +1 +n17mr6108986fab231273577581701 +1 +orgdebianuserrequestlistsdebianorgabr +1 +62dvklb4ecgdvl6l9kbxl7zyuyfrgulq7prcxjyfuktowyfrmjqp96hw6noxfqx2q75f8arze +1 +airnav +1 +creating +1 +redhats +1 +12807050632050161270780727932javamailrootmd01wowsynacorcom +1 +transmissions +1 +testsblogspotldosubscriber +1 +hp +1 +httpwwwlinuxiemailmanlistinfosocial +1 +barreraorg +1 +aaaaaaaaaaaaaaaaaaaaaaaaaabyulglkswtn3vpi0wpyqklzapo7pdj9znthtcy3t6o9dnv +1 +n4i7qerg014308 +1 +244b5106fe204cc6b65abb558954c536latencyzerocom +1 +gmailid12803d3209d148ae +1 +c3ktdg8tcmvtzw1izxigzg9tywluig5hbwugzm9yihrozsbzyw1lihbyawnl +1 +mailtodebiantestingsecurityannouncelistsdebianorg +1 +hrefhttpa02fxqufojehcnibenumki63n8g2722955g455044li3wropozaoz61102176139027379476click +1 +ink +1 +0innbspopoppp +1 +imagine +1 +definitely +1 +g6d6kg5c001416 +1 +brings +1 +slocum +1 +ussr +1 +112834 +1 +hrefhttp7a50tabjamie39aruyrasyye1d8f0a6daac4 +1 +pulitzera +1 +needp +1 +glgl3v1hwcr38ahv8ayvqlmtmx1cg7q2f9vstcwhplvc7bg4fnbg2sybpuer9ptpkrvdgwmh +1 +37441033577684munnariozau +1 +1o8op70005igcl +1 +120328 +1 +nnetwvvuvfu +1 +m8cs84211wfo +1 +ict +1 +hrefhttpwwwmedcenterexpectcom9747hibody +1 +hess +1 +emssrv0spooldir +1 +qqqqqqqqqqzdnetspamassassintaintorg +1 +consolidation +1 +acs +1 +tainted +1 +200168162123 +1 +cybercare +1 +230402 +1 +height463 +1 +20750143226 +1 +1032 +1 +pragmatic +1 +reed +1 +055658 +1 +lta +1 +interact +1 +127272240578f661e50001w4twrl +1 +lfctfen +1 +href3dhttpwwwwidxixidcnqvazifiaeqpq3d +1 +peltonen +1 +pleased +1 +200208281704g7sh48z01928dogmaslashnullorg +1 +backups +1 +enbankment +1 +height3d361 +1 +ontsize +1 +barr +1 +varies +1 +coldfusion +1 +firefox +1 +002824 +1 +cmes +1 +method3dpost +1 +wash5 +1 +face3dverdana +1 +tg9hbnmsigfuzcbnb3jlisa8l0irxzlbib3axroigxlc3mgdghhbibwzxjm +1 +width3d250img +1 +206248157232 +1 +alt3dpmg +1 +prevailed +1 +3d9ccf545070909freefr +1 +cache +1 +suppor +1 +xegroupsfrom +1 +highway +1 +com +1 +astonishment +1 +classbodya +1 +pmunoznetnowcl +1 +gmailpem +1 +httpoutoftouchblogspotcom +1 +httpwheniridemybikeunitedsituationde +1 +alerts +1 +helomd01wowsynacorcom +1 +nonindustrialised +1 +maincontent +1 +sa290hl +1 +cellpadding4 +1 +mailtospamassassinsightingsexamplesourceforgenet +1 +arent +1 +hrefhttpdigitalmedialockergnomecomlatest +1 +407 +1 +etcnetworkinterfaces +1 +cultural +1 +height82 +1 +styletextdecoration +1 +individual +1 +mobutu +1 +titledoes +1 +kgi09istrwytjbikcwuxwaaxl1wziiualalxoj30gmlhldq8amubqtvlxqogo3h9lwtyugqckefb +1 +color000000bpoor +1 +totalitarians +1 +surveys +1 +gmailid12830badb3615247 +1 +140510 +1 +archivelatest576226 +1 +httpslistssourceforgenetlistslistinfospamassassindevel +1 +shortened +1 +facility +1 +243e22c414 +1 +cnetcombgif +1 +prevails +1 +hoonting +1 +zimbra +1 +color3d990033malefontbtd +1 +hrefhttpwwwhandangocomplatformproductdetailjspplatformid1ampproducttype2ampsectionid0ampproductid30551ampcatalog1ampsiteid159 +1 +border0fontbfont +1 +width588 +1 +brbrfont +1 +132905 +1 +090249 +1 +clears +1 +goofy +1 +811105 +1 +af6decorecpp350 +1 +abandoned +1 +214424 +1 +etcinitdclamavdaemon +1 +zrmjks9x5kvtv39pbwtwf3pr0ciw00ffpd48fec4rtvhvu7g8pdcx8myiakeiysrlzj8w0agun +1 +160835 +1 +175403 +1 +pv9f9f9f9f9f0vl4fc93h +1 +phases +1 +tep9vjqhfynfv4or874loic45kkgsfat7w2fqj7t9lc5vd8fs47rwem6kubyanpy +1 +n9sjpdum010186 +1 +klicken +1 +a225813917473867footerarchivecolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightboldfontstylenormaltextdecorationnone +1 +impression +1 +hits00 +1 +bfonta +1 +gt +1 +coins +1 +teachertube +1 +believes +1 +1br +1 +insults +1 +care +1 +fungi +1 +differencethe +1 +o2r6d4219cc1004091025s676528f9s81b1926f2acdd799mailcsminingorg +1 +103700 +1 +hrefhttp935aa0sqopacincn568358x3y06i26j143229qjelixumefexyphibody +1 +whip +1 +cage +1 +exhaustion +1 +httptechupdatezdnetcomtechupdatestoriesmain014179286454000html +1 +classmates +1 +e9vywlwkrwv3mlwsvlq0jrs4oxssmilzfrbbsbqm1vhjtc7b6ubaktdruospmy7aowkzmjdryo +1 +073033 +1 +scratch +1 +016d67e76d4c5225e4b57ab08de4jeramx +1 +democraciesh2the +1 +j05pnmmba8nmyjhgoilh40km6a3uy7lphrng7kq2sigbpsp91dts6lvyql81wg142uu3win +1 +width3d1 +1 +103709 +1 +xyuw2b2gxmiph0h2amwirldskjfnge1vknfb18ye1ewnxweuyj95sa0gwfgmfszd8 +1 +signhtmltagsupperandlowercasenmd08100101 +1 +rolled +1 +archivelatest573554 +1 +undisclosedrecipientscometsmeccosimonet +1 +waterslide +1 +cladists +1 +classtitlenbspgnomeaudioptdtrtablep +1 +seg117ente +1 +nonechristiana +1 +snap +1 +msklbajfiyrgr9pcxz7lvwau87ozczvnws +1 +celpulsomercedeslinenet +1 +lihome +1 +c76104234217hsd1wacomcastnet +1 +202pm +1 +114147 +1 +0inspan +1 +koi8rbig1hdhrlcib3agvyzsb5b3ugyxjllia +1 +grubbr +1 +experiences +1 +includin +1 +gachallan +1 +2051586244 +1 +arrang +1 +rbthomaspoboxcom +1 +insphere +1 +cheating +1 +psa +1 +brabout +1 +b4aa82 +1 +campaigning +1 +w6radvfxp5fzvbqv83x8f4jspmdb53ahjmwcpwdpwkvterbanvxpelkmwvlnmgepv9 +1 +db09716f1a +1 +highyield +1 +valentine +1 +21012 +1 +attached +1 +scarcity +1 +pilgrims +1 +communicationp +1 +technological +1 +postings +1 +schampeolocalhost +1 +kde4 +1 +16002000 +1 +satellite +1 +december +1 +allowed +1 +tkerror +1 +hrefhttpiwaydrugru954ad68b35a101bb70a22f6012923422 +1 +202051 +1 +boxes +1 +o5oaghf1mev7xlkwcmuahy2eu4xzenlwbcvzcoh4bby3d1xlbgjwdtuw37y9v78a +1 +hoop +1 +javatm +1 +compete +1 +target3dblankhttpvcspkgorgabr +1 +p111tete +1 +guinness +1 +width3d91font +1 +centerfontp +1 +loop +1 +engineers +1 +cts +1 +reserves +1 +region +1 +powerpc +1 +155431 +1 +6001 +1 +hacking +1 +d3rmtr1kwzoxhb9fxrxiuy7hagi3hqkri4tmaygjj1qvevkucoqdyvh1qnrf6tojbxbccbfotu +1 +coastline +1 +preferably +1 +8f12a2c322 +1 +month20 +1 +messenger +1 +alignleftthree +1 +27fa5341dd +1 +gcd +1 +67ze67768ae7mjqbunk5zbgrnojsp7bligc4ukbiaac3ufneugdxogtdogdce7dx7tvq7duu +1 +caffeinecsclubuwaterlooca +1 +5so1340930qyk3 +1 +herself +1 +transferred +1 +117tilizzare +1 +194610 +1 +introduction +1 +nows +1 +ffffffweb +1 +publishers +1 +ip2416321087adsl2staticversatelnl +1 +manner +1 +persistent +1 +abstraction +1 +111r +1 +refund +1 +yk0reyyfickojejevmaoyvjeskxonoiv6yzqopmkzkcjypmwasa2hmuong40icq6iggmjjrfgno +1 +15da916f6b +1 +600px +1 +thefonta +1 +stanhardwarefreakcom +1 +uid35281201487268 +1 +9515533151 +1 +vlink333399 +1 +60744zrtrot7406616waahoooqot1725317ebxeicxxtozrtrd2finestscorcom +1 +size3d3grown +1 +4kwmgtvudeggouephewojfksyuijbecazlkq5caqbcs0pfnt5k2coxxutpmsz0affomehdqw6emk +1 +deep +1 +republic +1 +c565dabb5c +1 +daily +1 +brdivdivbr +1 +gn +1 +href3dhttprdyahoocomdirhttp611 +1 +skip +1 +alignleftchouth +1 +frommxmatcheshelodomain2 +1 +15so316802fxm6 +1 +173843 +1 +uefopjwvrk9ovd48rk9ovcbmywnlpsjcb29rbwfuie9szcbtdhlszsigc2l6zt0inyipfnq +1 +193a30 +1 +scrollbarhighlightcolor +1 +ludendorff +1 +12111175112 +1 +20100524 +1 +helogrunt2ihugconz +1 +rememberance +1 +bukkake +1 +aligncentergobi +1 +tx +1 +finale +1 +titlenewsletter +1 +1b95d13a48c0 +1 +unnerving +1 +valuevirginiavirginiaoption +1 +m8cs75626wfj +1 +bsmithnetnvnet +1 +3dbea4305f7 +1 +strongsite +1 +41 +1 +clientip172541338 +1 +knighthood +1 +party2e +1 +hibodycsminingorgp +1 +score12 +1 +decree +1 +499 +1 +interestednbspin +1 +programmable +1 +grace +1 +e6859de083 +1 +httpwwwjabbercomosdnxim +1 +consistently +1 +rulenotspam +1 +broadening +1 +styletextdecorationnonemore +1 +andp +1 +pd2ml3sossvcprodshawca +1 +association +1 +mailtoleavecustomers949326kmailryanairmailcom +1 +122442 +1 +20019 +1 +1227 +1 +alignmiddle +1 +yestastyru +1 +zzzzexmhspamassassintaintorg +1 +emailp +1 +kde442 +1 +personaly +1 +200208210251g7l2pqkb001805turingpoliceccvtedu +1 +telecomitaliait +1 +lottery +1 +httpclickthruonlinecomclickq89h2jbrpjirahzk5rhpcqnpvwaeyl9srr +1 +addressbook +1 +defining +1 +httplistsapplecommailmanoptionsaugdmlsubscribertech40csminingorg +1 +imposes +1 +owlet +1 +permits +1 +hrefhttprealmedtechcomimg +1 +nict +1 +lliq4okelextvwvss5egz1yjcij9uvvzl69lirwkdopnboozxbicf8qzc1nj5azknjwl3dmcqv +1 +attacked +1 +archivelatest576879 +1 +170146 +1 +wednesdayi +1 +164121 +1 +menna +1 +155803 +1 +zlcpbxht4o6emumizufoa5kfkh4kyfdm17qz03qdbtyscvkjttdktofmf0yhrbvusvt4zwcqpho +1 +width399 +1 +jlennon909aolcom +1 +powerbr +1 +srchttps2redeyeemailcom80storestoregraph54509597042gif +1 +3196794 +1 +81228122 +1 +colorffffffpublish +1 +bordercolorlightffffcc +1 +colore4e4e4 +1 +equity +1 +differences +1 +theserversidecom +1 +10pxdukefonta +1 +siltation +1 +grantsfontfontfontbh3center +1 +tests2 +1 +identifier +1 +sportb +1 +sndhdaintel +1 +width3d28 +1 +uppertitle +1 +nmh104dev +1 +arrive +1 +g8n58oh19980 +1 +17wx8n00031d00 +1 +thath5 +1 +11fe0d0bb7b68ae0000042ab8c4bd2f195dbdd +1 +dslfqazrtltqzcmt0fmf5cfushikwa5mmft91z6skv0defrvlws9xx7cpidngispnnzj0fciugh +1 +marvin +1 +htsac6hguhptrip68u48cjyaygfokg7h0pm5jj0tlh29vrtc48kdvztwkcqwx9pek2y6 +1 +destroyed +1 +grokster +1 +172740 +1 +cropped +1 +115104 +1 +color3dcc3366b350bfontfontdiv +1 +104557 +1 +pattern +1 +nights +1 +n19grpscdyahoocom +1 +contentuniversity +1 +barrichello +1 +hrefhttp422tabjamie39aruuopicebyemcf968d35a7cd9f +1 +151335 +1 +family3dsansserif +1 +f3gawjbe6papgcz7fzlbliszt +1 +giving +1 +categories +1 +abvsfo1acagent7 +1 +133159 +1 +twofinger +1 +height3d7 +1 +213620 +1 +slight +1 +alignmiddlea +1 +wireless +1 +widen +1 +mai +1 +io +1 +223639 +1 +nosend1tdtrtbodytable +1 +nanotechnology +1 +worthy +1 +uid35541201487268 +1 +coral +1 +o3ji1oav002872 +1 +gmail +1 +185343 +1 +courage +1 +ftpchdebianorg +1 +174614 +1 +size3in +1 +protein +1 +iiypqralqvqqibs7vylilormpzn7dil5g4klu4czoboi5tcookiiephagrlhslufudec7ucel6r +1 +logged +1 +hypochondria +1 +namepresident +1 +size3d4to +1 +durãe +1 +hereabr +1 +href3dmailtobbyaletesiaolcom +1 +wkjw5mhkdk4a755zwkvyczjhk6havparcpltej2gcuadkxda3pbhpikktbj61kr26v4whovrck +1 +003234 +1 +husband +1 +alloc +1 +bafter +1 +testsldowhitelist +1 +tuigif +1 +german +1 +cognizance +1 +pgnrcomrhtmlc3d1113074r3d1112136t3d1225499824l3d1d3d89042084u +1 +rogers +1 +errorsbrdiv +1 +forms +1 +worsening +1 +vic +1 +width561 +1 +criticized +1 +afomaafomaclubinternetfr +1 +plataforma +1 +hrefhttpb8gjxdjfharuyzaqaqih532d4ae155c +1 +installing +1 +a3hfawjxjkbmiazv8rdjmtcvobwkaczotu5cmyidj2btp8zafqyzqbygeig1dw7xxeaczqqaha +1 +protectionsownwership +1 +traps +1 +210016 +1 +15274145157 +1 +20020720184341e23917iesubericnet +1 +list20 +1 +1027711160103586camelavalon +1 +wisconsin +1 +75604018 +1 +f97109ily +1 +oezex +1 +d3nhtvdvaz9ywcfr9jeymezgudtaahsmgdirncelbivxsfm1usxpiwmmea4tt32nz5w +1 +authentication +1 +161126 +1 +protected +1 +httplistsdebianorgpan20100511215248csminingorg +1 +timid +1 +meters +1 +iso88591qvdjlbzrnbhqxeiu9smsqcwukmnmjitkdnaaccbw2sh1fxqk3ijrtknzsol +1 +cams +1 +030253 +1 +conducted +1 +spliced +1 +smtpd32606 +1 +valid +1 +contentdefined +1 +parc +1 +gsurveyresultslinkhover +1 +belloutput +1 +drop +1 +srchttpwwwselebrettyru10gif +1 +silly +1 +drivers +1 +inline20 +1 +criminal +1 +absolute +1 +hrefhttpdrbluehornetcomct27833172825285769m1133771572ae07dadf1297fbb0452048e91119d515img +1 +4bbca70f5090401pixelturecom +1 +originated +1 +occur +1 +orion +1 +1vd9ng3fpaztbsvyf2f2ibdsttd92aelkpifjbrwyua7k3q3funinrznaphf +1 +publisher +1 +laura +1 +sa154a +1 +caves +1 +120442 +1 +worse +1 +oath +1 +ldowhitelistmurphysexl2plingquery +1 +055943 +1 +bonuses +1 +laa29642 +1 +labyrinthine +1 +3457158148 +1 +a653e58a4d +1 +httpadrantsrantworkscom +1 +cvsrootspamassassinspamassassinmanifestv +1 +moode +1 +countrycode +1 +httpwwwfingerfarmcomblog +1 +likewise +1 +taciti +1 +invoicing +1 +themws +1 +8bff916f20 +1 +084410 +1 +sr71 +1 +identity +1 +83 +1 +width3d650 +1 +outfielders +1 +width483 +1 +4ucphlhghapz0f8kbhqyjwdwds69v0h1pftnuhmzyjhqxuyfsfwbns8acgvpl8 +1 +archbishop +1 +hemispheres +1 +5206 +1 +mailnbspat +1 +village +1 +vaa27638 +1 +152735 +1 +013046 +1 +hrefhttpae7d6ef0f427cd5fellgatherruunsubscribea +1 +gelernt +1 +102809238414 +1 +eastrmmtao107coxnet +1 +ageinhibit +1 +spielend +1 +imprononcable21 +1 +urlnumericsubdomain +1 +uid69301201487268 +1 +oqoaenjl8quavghuhkp8awpxrtuifubd4x64watgctzkeusvxefmu5b9r3rrdi8qku7h0opx3r +1 +12205770404ebfe5ae4bbb9a4709599320491 +1 +alignleftmassachusettsp +1 +azfxj001pgpzmkjsundscbwb5o6ka4 +1 +18662377397font +1 +height3d +1 +q2z951d34ac1004191247oa5db050v24437077ee2cbf9fmailcsminingorg +1 +brdefeat +1 +ff80ff +1 +200600 +1 +valueoregonoregonoption +1 +034856 +1 +sa335 +1 +nextpart00000b788d77d7bc7065a48 +1 +12021 +1 +httpwwwaccucastcom +1 +stability +1 +yle3dmargin +1 +agenda +1 +utf8qdwpreheun7x1kjxybf0zhlwsnsyjru4abtypxviyxqnqlrxwtcsj0a096gud +1 +httpradioweblogscom0106884 +1 +httpwwwcrazytracycomblog +1 +clallam +1 +120400 +1 +001517447a467934870485859c9f +1 +muller +1 +married +1 +102344 +1 +greece +1 +340976951424841270571770256javamailrootmd01wowsynacorcom +1 +src3dhttpwwwzdnetcomgraphicscleargif +1 +g8o80cc26642 +1 +priority3d67 +1 +lnh4eg1hdgnolm5ldc9ozwfydc5nawyiihdpzhropsixnsigagvpz2h0psix +1 +bears +1 +bnextgeneration +1 +a8bfa0f1eafbfcfbedeee8a8a3b9aca9baf7fcf5f4988a8588 +1 +5404 +1 +omaha +1 +oz +1 +presses +1 +coastnbspbrand +1 +bugs +1 +rick +1 +nonsilly +1 +alignlefta +1 +divarchive +1 +oaa11786 +1 +profit +1 +155413 +1 +libraryin +1 +width582 +1 +203 +1 +20120190204 +1 +191128 +1 +millions +1 +thisbr +1 +2si2934000fks1220100423144735 +1 +s113117eeze +1 +oulook +1 +201002041038o14ac1vg001318ns1csminingorg +1 +glvertex3f00f +1 +valuehttpwwwlockergnomecomimageslgtoppergifinput +1 +brandy +1 +sunlight +1 +incurred +1 +bgcolor3dwhite +1 +httpwwwdormwarsnet +1 +lightsensitive +1 +chaque +1 +theft +1 +6416883170 +1 +alt3dsylvia +1 +leonewest +1 +forwarding +1 +lsj1fixjalw1gab5jntlqsturk41pxbxhwjaduvcbewz9rfc2gflpj2ruqhuotm8f48vxya526b +1 +090051 +1 +sblxblspamhaus +1 +110228 +1 +harvesting +1 +abandon +1 +difficulty +1 +6b30816f03 +1 +nahjerqx018166 +1 +mailtoilugrequestlinuxiesubjectunsubscribe +1 +winnick +1 +estimatesbr +1 +forgedyahoorcvd +1 +jeaatiry +1 +tooted +1 +httpgeigersundayblogspotcom +1 +nugent +1 +archivelatest574208 +1 +size5trckatomicdotmpqhwqrwhlqffrp8font +1 +appeared +1 +bcc +1 +6135221168 +1 +vlgt1utrxrxvry8zki0fegknhw7azsjksya4k1ixlwdouwpzdzg7pij8wvywsslqnri8wg5ftbq +1 +departure +1 +students +1 +httppatrickwebcomweblogcategoriesinternettechnology +1 +gmailid128b92888512dbdf +1 +barracudaheaderfp3043 +1 +tercera +1 +consultant +1 +g93h04f17315 +1 +cent +1 +8938112 +1 +biological +1 +ptanjung +1 +disks +1 +centeramtrak +1 +johnhallevergonet +1 +potency +1 +shops +1 +a5093c44db8a7241ad5f67dfa8e +1 +m8cs38513wfj +1 +alto +1 +hobby +1 +simultaneous +1 +intranet +1 +9175 +1 +reject +1 +unlist4meyahoocom +1 +tributaries +1 +simpson +1 +soapelecohclick +1 +hbpnul563pib6gguwaaaaaaaaaaaaaaz9ze6ir27d0njiz7lypyjxlfhao8xk0xad3aiepydcfb9 +1 +srchttpzdnetshoppercnetcomiceps12096258061201gif +1 +071240 +1 +schampeoheskethcom +1 +srchttpwwwzdnetcomproductsgraphicspid561958jpg +1 +httpwwwcaterwaulingcomblog +1 +101937 +1 +plugin +1 +unlikely +1 +colourful3dreduction +1 +jeeg +1 +kccf9dvqk4dsuqe81llwmn3wb6yqulntxfjj2qozgwlp1mblggjkynvvmihzaj8acgmrcohb1nis +1 +20020720051950ge28205linuxmafiacom +1 +softlink +1 +webjunkies +1 +96de40a2655bb0e854e962782aa +1 +ass +1 +currency +1 +informationnbspnbspnbsp +1 +murmured +1 +threads +1 +132821 +1 +httpyl60futurexclusiveinfo6091312843e5e114410248htm +1 +color3d333333quot +1 +boundarynextpart0000023e47e62046639db9805 +1 +ed1fa16f03 +1 +archivelatest574307 +1 +fontfontfont +1 +c078416f03 +1 +narrative +1 +gmailid12825cb607dd63da +1 +archivelatest575026 +1 +accordingly +1 +3c2ftd3e3c2ftr3e3ctr3e3ctd +1 +leroy +1 +plingquery139 +1 +berkeley +1 +moyennes +1 +mgraphicsanchordeskfrontpageheader2yourgif +1 +floppy +1 +nirmala +1 +phd +1 +httpwwwmejlgaarddkblog +1 +ywlsaw5nlca8l2zvbnqpc9mb250pjxmb250igzhy2u9imfyawfsiibzaxpl +1 +62 +1 +statisticsp +1 +productsservices +1 +224754 +1 +colspan6 +1 +strongsave +1 +family +1 +enforces +1 +g8 +1 +filedeb +1 +score698 +1 +homeabbr +1 +eskso0ztqm4e +1 +truruspamsubj +1 +d13si2423542fka220100430010606 +1 +fep02appkolumbusfi +1 +yep +1 +mourning +1 +targettop +1 +visitante +1 +freegonefee +1 +size3d2emailfonttd +1 +ikip +1 +malaysia +1 +bat +1 +1031314884 +1 +pushing31csminingorg +1 +htmlimageonly32 +1 +marginleft +1 +impulse +1 +regardsmrs +1 +088 +1 +hereafontfont +1 +n6kd5hpg016109 +1 +vps +1 +kamal +1 +14523923587 +1 +21doctype +1 +color0063c6click +1 +httpsexamplesourceforgenetlistslistinforazorusers +1 +servicea +1 +httpwwwblasercocomblogs +1 +reservedtd +1 +repeaters +1 +kicked +1 +4dfa2c44e +1 +width780 +1 +09a +1 +5503 +1 +reforming +1 +174713 +1 +themnbspfont +1 +face3darialverdana +1 +elitedish +1 +14b +1 +plasmids +1 +c28si5988633fka4420100501192704 +1 +tfoq4ifn70qefughsd5x6vf0ijxfxkrnbwkoydfwrx8nqht51pqtvx7apoakfxodu0swhyd1pij +1 +svog +1 +noreply +1 +pint +1 +20100503 +1 +32224988 +1 +httpwwwcandygeniuscom +1 +ss +1 +passed +1 +annually +1 +size2statenbsp +1 +mailtodebianlaptoprequestlistsdebianorgsubjectunsubscribe +1 +213251 +1 +hrefhttpclickthruonlinecomclickq6b8g5zihecwprago280ywq4d0p6cr +1 +134nbspnbspnbsp +1 +102233010 +1 +71141641 +1 +jesus +1 +align3dcenterbrbr +1 +src3dhttp365sunshinehuicomcreditfixemaillinegifbr +1 +declared +1 +1086639 +1 +farmer +1 +0h35003jgmyx9gmta7pltn13pbinet +1 +ascended +1 +netscape +1 +httpwwwbarracudacomreputationip12412014391 +1 +leavecustomers949326kmailryanairmailcom +1 +varied +1 +policiesp +1 +againnbsp +1 +labels +1 +mainefontbr +1 +12398877622da0000b0000w4twrl +1 +settlements +1 +res117lta100111s +1 +tls10rsaaes256cbcsha132 +1 +cas +1 +frame +1 +tywkrkontaznlegxtfqypbsczkeae4zn8iqal4dcguuakit8 +1 +boldpaddingtop +1 +kh9ngakjqem +1 +4bc634fe1020700studentulgacbe +1 +proposal +1 +vander +1 +hatched +1 +testsfourlaldosubscriber +1 +bythinkgeek +1 +highligh +1 +163501 +1 +110327 +1 +altapproved +1 +zyb0agugz2ftymxpbmcgdhlwzswgaxqgdg9vaybtzsbzzxzlcmfsihdlzwtz +1 +klgdfjhbv9amqgcquxygrngdghnqgokvgehmimzeoitj2obcbyqghpjkafooj06xpgloqivjcq +1 +advertise +1 +0w7ozluyrv7g5ll9gs5m0ldlmljwljcypklbqabamwstt1vvczomsn7kbrgsuhgt0lsapyeok0ly +1 +t3mr3646835fau161272924148788 +1 +3b +1 +becoming +1 +article +1 +oh +1 +passe +1 +href3dhttprubilzycnfont +1 +uptotheminute +1 +bsfsc0sa392 +1 +profundities +1 +x1120081209 +1 +4odoeusrgdlsepbpxrwata3zbobmokqdqtgn7urawydjy2auz61cbltus2ptmprclcwayma94fp +1 +attracting +1 +width264 +1 +omission +1 +ughhh +1 +083805 +1 +taxonomists +1 +hrefhttpclickthruonlinecomclickq59y53oienthmcognglpwbkmqztfr +1 +g8nlixc03143 +1 +decorappropriate +1 +185910 +1 +0825070004 +1 +bannerfonttd +1 +165656 +1 +9cfb827e4120b +1 +sharp +1 +neglecting +1 +isle +1 +color80000 +1 +powder +1 +8252002 +1 +uggs +1 +jq25nyahoocom +1 +spanked +1 +77 +1 +krb5 +1 +httpsciscientificdirectnetcasp83999041248fb1d7ff79fc36 +1 +101220 +1 +c111mpelling +1 +ranged +1 +fairings +1 +v41vm1qyqlb5zyz7ibldrjprrvvzq8mejtmopcw8hujc6isy0wbextbf1rh9ntve9qd2qesen +1 +mailtosocialrequestlinuxiesubjectsubscribe +1 +theater +1 +stating +1 +bgcolor3deee8f615td +1 +onyour +1 +4bbfba3b20009webde +1 +ldowhitelistnorealnamepgpsignature +1 +sa +1 +awarded +1 +iplanet +1 +ha +1 +di102102erence115 +1 +virtue +1 +solidst +1 +whi +1 +drugserectileobfu +1 +jmnetcomukcouk +1 +215638 +1 +bgcolorfffffftd +1 +name3dbtnsubmit +1 +dollar +1 +cam20 +1 +servicescnauthsmtppsuedu +1 +automatically +1 +roofingspecialistsabaloneelr2grassillcom +1 +msgkeys +1 +href3dhttp60f667tewetemcnekoigetaci3d9478u2 +1 +205628 +1 +aroun +1 +courts +1 +height166ap +1 +c0arpqdrad2fogilnrjkqdpptzhkrrugczviqsfyhk0fmoh3lqm44pm2j11t8e0t9gy +1 +vvkbjjjbarbwz0gh5bayh0rayvf0fgpbmugog3ddtrjbcrnpltssyj5zoooplfjupqldf8axz5 +1 +114243 +1 +href3dmailtooneincomelivingremovegroupsmsncom +1 +022048 +1 +whatsoever +1 +colspan2font +1 +size3d3onef +1 +devalue +1 +zhdkgyxh2tua2gihruwwm3yta6iblmabzwq0cycpnxf0kxbqmeeeyjgsiuwqkrbugcnjk9dovha +1 +leisure +1 +activate +1 +returned +1 +postal +1 +width3d2210022 +1 +missives +1 +versatile +1 +di102102erent +1 +turkishresident +1 +duncan +1 +fontbp +1 +fontsuper20 +1 +inhouse +1 +prematureejaculation +1 +a34bc16f03 +1 +srchttpartvcomcnet1dinlftgif +1 +score791 +1 +hrefhttpclickthruonlinecomclickq16tcji4cfvq9diqgdblruabdxru4r +1 +score000 +1 +comparing +1 +a4rjf0nbhntpt6i8vq1c9cw17 +1 +titlethe +1 +suspect +1 +iso88591bt2zmawnpbmugugfuzxjhasbxyxrjagvz +1 +rose +1 +richard +1 +e8211mailstrong +1 +682169198 +1 +plugin2 +1 +israelis +1 +iglmihlvdsbzzw5kigfuigvtywlsihrvihjtdmhqannzczc3xzlaewfob28u +1 +kinda +1 +enormous +1 +visuvondfyserieskathevolontariapparatchiksvoltagegammaltvotidc2vrijwilligerswatersportsganasarchipelagowc1xwdgtvuelingganginggastroair2waterwvaluewhujs2fwashappetitwarriorswallstwarshipwidelyusedwatermanwcsstorewatersolublewatertable2fmarketinggbvbmjwaymakerwebsupportweavinggearbeitet +1 +o8avpi0baknr6oqyv3jpayhkhrf7b2x7v3ol72feupuw5gunl2tcwhcwspjmdlviepye9ihpvh +1 +width3d160font +1 +compensate +1 +17s8t20003io00 +1 +txqueuelen0 +1 +sa011 +1 +pan3d22222 +1 +brbrthe +1 +httplistsdebianorg4bec15064090508studentulgacbe +1 +wininsider +1 +cypherpunkssszcom +1 +17cvti0007rs00 +1 +size2july +1 +colorffcc33 +1 +59 +1 +110507 +1 +nzdqpcaaapxcsaixk94aqccddfcgot0lanwefakuzcq6aegcahdal4mv7gziq4iubpurmzhawwaw +1 +gateway1messagingenginecom +1 +165155 +1 +mjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjlwaarcadraqgdasia +1 +vol1 +1 +width728px +1 +commission +1 +torch +1 +church +1 +20624818377dslteksavvycom +1 +gddrescue +1 +painted +1 +gmailid1288723aca21d4db +1 +marcletde +1 +lirefinance +1 +bordercolor3d111111 +1 +68142198106 +1 +leptosomic +1 +rustic +1 +narbkrmw006300 +1 +misconduct +1 +aeussern +1 +songs +1 +spec +1 +3e +1 +dispatchtitle +1 +merit +1 +160925 +1 +20000922 +1 +7xsvh7fikzyanuaavswd6sagf3tlrfl1jlqnqgnhs96xq9b8h1l9otlknpjktxeiiinposiddi +1 +trackedbr +1 +settlement +1 +hrefhttpdrbluehornetcomphase2survey1surveyhtmcidjtveeqactionupdateeemailhibodycsminingorgmh83e256ca7e16ed7a9d39f54c68e290a9unsubscribeafontp +1 +height100 +1 +tvdspaceratio2219 +1 +104836 +1 +concentrations +1 +18si1915898fkq420100421110532 +1 +661520186dedbtitelecomnet +1 +xuid +1 +438363 +1 +whopping +1 +dsa1787 +1 +hood +1 +tribes +1 +archivelatest574186 +1 +20100508104207ga2878heimagjkdk +1 +unread +1 +aka +1 +edward +1 +n51jiin9008982 +1 +geology +1 +tumor +1 +typesetting +1 +155208254214 +1 +composer +1 +5teuz38dsvikvdikzathw5eqwoaiviyueerjhew1t9yun7rrjsqtd6dzzreowvqlr +1 +facetony +1 +gasoline +1 +8f8ba13a4a34 +1 +blair +1 +familiarity +1 +10th +1 +chemistry +1 +183538 +1 +bolder +1 +width601img +1 +c18mr2917281fat481273916694875 +1 +volatile +1 +restructuring +1 +gmailid128afc0045de029b +1 +desktops +1 +firmness +1 +some20 +1 +anaconda +1 +358c566c0d3 +1 +dialects +1 +icagicagicagie1vcnrnywdlpc9gt05upjwvqj48l0rjvj48l1repjwvvfi +1 +namegcblnqjpg +1 +pan20100507181232csminingorg +1 +tomwhoreslacknet +1 +g8qfwjg25145 +1 +n28grpscdyahoocom +1 +o0re1cax006312 +1 +savage +1 +msofareastfontfamilytimes +1 +komeis +1 +hd +1 +g9181kk15695 +1 +casey +1 +faa +1 +8d96985e292dfbd97625453689c +1 +grover +1 +092402 +1 +055322 +1 +housing +1 +1865 +1 +httpwwwqvescomtrimsociallinuxie7c297c134077 +1 +080122 +1 +bonkers +1 +ywwgywr2zxj0axnlbwvudhmgd2hpy2ggyxjlihzpzxdlzcbiesb0ag91c2fu +1 +g96805k15046 +1 +face3darialbviagrabfontp +1 +width180 +1 +paying +1 +aligncenternbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +1 +constantinople +1 +cmvvbndlyi5jb20ncg0kww91igfyzsb3zwxjb21licencg0kotc1n0fxrgoz +1 +wide +1 +gmailid128742ff92172087 +1 +assumptions +1 +sansseriftotal +1 +12557011680b2a006f0000w4twrl +1 +vine +1 +78174104140 +1 +stronger +1 +gaqgrsw11496 +1 +supplier +1 +v35takv2xnqubzfeuyuy0khknt45wk3drcqsm56oomelcjwzcmtcjbfc5h1wsluwjldi8vz4 +1 +102br +1 +xurl +1 +size1just +1 +slashdot +1 +possession +1 +duck +1 +44 +1 +alumni +1 +leveling +1 +spend +1 +dio +1 +tel16503272600 +1 +abrtshirt +1 +cctld +1 +ypsb32lsgpzrgcqb19twa0ir3mi2n0aqlubpvumt0chcgjr39khksjvy0ylxltbgn2el +1 +goto +1 +developments +1 +finn +1 +href3dhttpnyavekepcnimg +1 +hospitals +1 +poems +1 +httpwwwcaextremeorg +1 +015538 +1 +x21q7sczk2kwehgnhbnxxgy56uxrkihtt5fm14cmsbbuysr0yxzqo8sxwdslgpkonyxiynqlhdbn +1 +rolexb +1 +httpwwwozzienetblog20020925htmla65 +1 +tournament +1 +cddb +1 +db +1 +llars +1 +exercised +1 +freebie +1 +careful +1 +scots +1 +32bit +1 +november +1 +size2wsiwyg +1 +200209302251315fe5aa07matthiasegwnnet +1 +36284000 +1 +width252 +1 +mcrypt +1 +stylecolor000000view +1 +styleverticalaligntop +1 +hrefhttpclickthruonlinecomclickqffsvovqms4myisimnzxrhbm0whther +1 +httpwwwforbescomfinancemktguideappscompinfocompanytearsheetjhtml +1 +prefigure +1 +201005050521o455lxor011731gw2csminingorg +1 +earthquake +1 +improvements +1 +lennybrbrdefault +1 +href3dhttplykoqljunpujuwcnajk3d +1 +statement +1 +boundarynextpart000000001c24a394307a140 +1 +cybersecurity +1 +1426113a6887 +1 +contenttypemultipartalternative +1 +interrupted +1 +remnant +1 +dynamicadsl84223220162clientitiscaliit +1 +v2y8f23146b1004061031i5e410c8dj686a5625f24e0f43mailcsminingorg +1 +roberts +1 +fu +1 +respal100111 +1 +esr +1 +highest +1 +armies +1 +poojafernscsminingorg +1 +tlo2txg1x5yahza9lzp5s921knveopcxovdj8fcjessd5e7ey4cmee8q4yy5cprl83ppue +1 +marchette +1 +6v2ctpn3d5i9qvmos8sy3tbdvkej4wupvkq5aimhpz2udysty6po36v2csdm6vpsuzhzgjbov +1 +tanzania +1 +dsl206191151102evergonet +1 +uy5svrozctmu +1 +fierce +1 +trial +1 +options +1 +1991 +1 +bhdjzp4nl5m6c50bwczf7m1hnnfpbq2o5kky2lycksl4 +1 +v7mr3307366qaq1871271949435099 +1 +increasedecrease +1 +080548 +1 +0099ffunsubscribea +1 +appp701106td +1 +flist +1 +2zmuipfgzpwuphhjtn4pksy3zup8oassiitq2vo4mx71pfkwghiuhlcqs2rnokkwbbzjimcmxff +1 +httpwwwgeekstylecouk +1 +bgcolor666666 +1 +style3dmargi +1 +owing +1 +responding +1 +hsubjectfromtoinreplytoreferencescontenttypedatemessageidmimeversioncontenttransferencoding +1 +dsa1871 +1 +insertion +1 +games +1 +namens +1 +gmailid12821dc56c4f5fb4 +1 +performances +1 +maintaining +1 +gafontbr +1 +altctrlf1 +1 +dramatica +1 +sector +1 +expensive +1 +overcome +1 +onmouseoverwindowstatusbigger +1 +jefferson +1 +1999fontbtdtr +1 +httpwwwstudentmontefioreulgacbemerciadri +1 +httpwwwfrogstonenet +1 +gibson +1 +matt +1 +3839lko100z326ibjmmoaxyso25306d0c91pcguymuleya +1 +e6f3016f03 +1 +194351 +1 +signally +1 +hrefhttpclickthruonlinecomclickq08ifoqirjfbeeakm5g3fercpgy9ir +1 +oef6zy1fvpa +1 +agreed +1 +shaneryaneducationgovie +1 +mexican +1 +c2lnbi5jb20vcnbhmasga1uddwqeawifodadbgnvhsuefjaubggrbgefbqcdbayikwybbquh +1 +ms +1 +billing +1 +too2e +1 +373 +1 +imposed +1 +erl561mailspamassassindnspmline374did001 +1 +n111rm97lly +1 +dispatches +1 +fast91gipac20westernbargecom +1 +p2g880dece01004110349x335b1dcdx88bdf3472c07fb1cmailcsminingorg +1 +shippingfontibtd +1 +midrighttopmsg +1 +fees +1 +ebay +1 +httpradioweblogscom0108992 +1 +push +1 +debian3f +1 +debug +1 +rhythms +1 +697px +1 +counterpressure +1 +1921685425 +1 +filesystem +1 +cmdline +1 +draft +1 +span24 +1 +102914 +1 +patrick +1 +rocks +1 +charsetkoi8r +1 +width570 +1 +cellspacing2trtd +1 +1030507320321439camelheralddragonsdawnnet +1 +rh8 +1 +rgb207 +1 +valhalla +1 +ludjmpizoykalcfyq05lbliszt +1 +fornbsp +1 +clave +1 +applicantsbr +1 +diskarbitration +1 +mest +1 +z9sjbf21f6isnf8pi8j1mn5f29agftkutzkn0145jfyq22azru9ylsbos8cfe1mpvtatt5a0pp +1 +s40abr +1 +easytoread +1 +eduardokalinowskicombr +1 +boipelobeldencsminingorg +1 +consumer +1 +k28mr299794mui251271352434848 +1 +20020913030633dlcl13768gelincikcorreoruraltourcom +1 +nowp +1 +z10si5121319fah7420100508144024 +1 +080407 +1 +2100000 +1 +listsdebiantestingsecurityannouncelisztdebianorg +1 +width579 +1 +directorys +1 +height282 +1 +checksum +1 +gffpwcx71hy0l3xjv38lcragwkzgfppkv1ljcjdwmnwn3zebfc94xutebdrhqexaozcrnguytd +1 +porstacshelsinkifi +1 +baa19534 +1 +ptsize10 +1 +dslb088074028166poolsarcoripnet +1 +bhl1wigmdthc6srkmsnehxql58zchewddyz3yw4nnava +1 +zm7q6wegxrhqp1ec1j7tef4c6d3o7lqjrroxxnvsur7kz40ckh7hvjwywq4uw27psqhmeu5qqcaw +1 +junk +1 +arrested +1 +gmailid127e290c76a7952f +1 +pshows +1 +nsmeilleursitescom +1 +httpwwwbarracudacomreputationip8121497254 +1 +archivelatest575097 +1 +variant +1 +11a6613a58d2 +1 +earth +1 +8725d16f03 +1 +servicebr +1 +hrefhttpclickthruonlinecomclickqdadt8jqupqnel9djbjsmvkpv3dxsr +1 +score108 +1 +httpclickthruonlinecomclickq9etwqoqtmqjgrfwghhscn0u7nqud4r +1 +013346 +1 +corporation +1 +inci100entes +1 +suggestionaspan +1 +kabini +1 +httpwww1tcnetnejpfmuratalinuxdown +1 +westkacuedu +1 +militant +1 +2dfuvxrfyy0u +1 +foeman +1 +showed +1 +neugensliberoit +1 +sansserif5250fontdivtdtd +1 +color3d000000downloadsfonta +1 +nt20 +1 +gfx +1 +hrefhttpurqpotentfigurecom +1 +mp3s +1 +frolovsigmaisprasru +1 +bgcolor3d2223ebedcb22 +1 +f160law15hotmailcom +1 +peoples +1 +westwall +1 +iron +1 +class6jm +1 +height8 +1 +erected +1 +height3d76 +1 +valueukraine +1 +depict +1 +argud +1 +delay +1 +i0uiywe9xdhemunohecfkun4cmxfkokxt5txdk2mfvr6osgdjgewzjihrcv3cdorejjvxb2fb +1 +bordercolor111111 +1 +laccompagnement +1 +improvem +1 +accession +1 +crashing +1 +twemlow +1 +emjmnetnoteinccomem +1 +hrefhttpzdnetshoppercnetcomshoppingresellers0708514119946796htmltagstcrmpdppr9946796 +1 +midfield +1 +alt3dnews +1 +listsdebianvolatilelisztdebianorg +1 +squeaks +1 +productive +1 +o0a1jnjk029348 +1 +httpwwwnewsisfreecomclick28418828215 +1 +dilemma +1 +obsequious +1 +michaelashcsminingorg +1 +askedfontfontblockquote +1 +shuffling +1 +stylefontfamilyarial +1 +sashimi +1 +invite +1 +544 +1 +acl146neoplusadsltpnetpl +1 +httpwwwpycsnetworkbenchcategoriessports +1 +refugees +1 +qzuveohejkkargromusouko5ukesminkcfkglc9cnuiw9bajugaskigacqag0hlict6hmgkenncb +1 +tdnbsptd +1 +s8a7tr2svg2gvu6d9zwd2nqcpgakkkzqu6p8a1iuu2nrwcsx6igd65x7qqutnxo1fokxms +1 +jaski +1 +1719 +1 +bsfsc5mj3865 +1 +121950 +1 +stories +1 +constitutionally +1 +trate +1 +url0022httpinternetemail +1 +size3d2july +1 +rlfranksender6a38a9paradigmomegacom +1 +c1kpxzmh1irrs9603jephocjfpnnlcgs3vydpukf7nngpabu5ry6ztxyzy28nll2ynntmvvj2xgw +1 +e6cda440c8 +1 +emailnbsp3b +1 +encrypt +1 +opposing +1 +archivelatest573790 +1 +strolled +1 +genom +1 +boundarynextpart00000237401caa0914eb1732f +1 +anam +1 +myuqf0mcocy +1 +8exborderleft1p +1 +cisco +1 +classmsonormal +1 +fluid +1 +issuesbr +1 +mariners +1 +wealth +1 +tm7mr0zkeqs2 +1 +hostyle +1 +href +1 +height19 +1 +de915934cbf911d6a1d9003065f93d3atopsailorg +1 +width800a +1 +6c2ncovqgq03x4wi +1 +expected +1 +9286142199 +1 +c11197ted +1 +terrible +1 +bailiff +1 +arabia +1 +disrupt +1 +wwwn9gl0xwiz1p0rl8qiirrhp9szpo3lfevxncfdx8vvkgcfvg4aqmmrq2fj0utuldyncvsvbl +1 +foml +1 +213671963 +1 +policies +1 +initialization +1 +online31983206muudgykaughfdrr1bnewsletteronlinecom +1 +hereditary +1 +ayour +1 +9029110 +1 +xc1m3hwapl2h3jnm3x8d3rcyzwq9s5kqbstgsryo5fdeywnrfmdlkgkkbl3kmo6bpqn1awq3b +1 +altblocked +1 +001429 +1 +likubos6311brasiltelecomnetbr +1 +lang3d0nbspnbspnbsp +1 +specializes +1 +c36f +1 +code7 +1 +greeting +1 +barbaracom +1 +altmainhead +1 +dpusanet +1 +size2bfloyd +1 +spring +1 +aada8aba9bbb9eca4bdbee8b9a5bfadeee8a4ada6adf18a9f +1 +instant +1 +160832 +1 +105613 +1 +atlantahosieryfusenet +1 +marketingsubject +1 +rarely +1 +priority3d34 +1 +mountain +1 +17217132188 +1 +httpwwwglenbrookpartnerscomweblogsjl +1 +dgggbgl0dgxlignvc3qgdg8gew91libjigzvbgxvd2vkdqp0agugc2ltcgxl +1 +203541 +1 +y20si5570588fah10220100524081209 +1 +idilbertemailcomemailsz3d468x60ptile3d1ord3d +1 +vb3ygfwnugvasb3k084wy0g5jxnsk0ban7dev93iu1smxz57y4gk1tpih2ekb3opq6cgn7rn +1 +11fe0d0bb7b68ae0000042abf14bcce44036ee +1 +mailin +1 +171930 +1 +83d5813a51c7 +1 +snowman +1 +uid49221201487268 +1 +133210 +1 +repeatedly +1 +confined +1 +food +1 +fontsizelarge +1 +url3d0022httpinternetemail +1 +awhile +1 +107040100 +1 +59hu +1 +opensourceeeethzch +1 +steal +1 +size1125fontfonttdtrtrtd +1 +r92aoxjjpeoz4bdxnbuuvkzpsavwtuclm6najxcg5bhxfvzini9kv39tjyxjglbrtvykdgiofgq +1 +160 +1 +laa31421 +1 +lm8li7zgn8f9rxtivpvds2fetsqgaepnh6a9bhevnympkeh97o7hti4knttqltjqwiydqqafshp +1 +privately +1 +ethnic +1 +morefonttdtr +1 +name3dtitle +1 +windows1251bsgvybwvzifdhdgnozxm +1 +030355 +1 +1884823157 +1 +esrs +1 +20020906134417ga16820cthulhugergca +1 +href3dhttpwwwinteractivemailingcomb012merchfont +1 +smtpsendmyrealboxcom +1 +lyrics +1 +3721f13a567b +1 +60ptmsopa +1 +exceeds +1 +721 +1 +amazon +1 +coinstallable20 +1 +zukeoomoh +1 +frequently +1 +fringilla +1 +20020719052006b12fa2940a1xentcom +1 +grossi +1 +concurrent +1 +effort +1 +zumblog +1 +majority +1 +angle +1 +000f01cabb9238c3dd20949e1d7d897b7b084a8844evcpwb +1 +063530 +1 +magnetophon +1 +24325 +1 +rowspan3d2 +1 +aaaaaaaaaaaaaaaaaaaaaaaafcvwyfk4lcuxrmplf8aacxrmplf8fhxl1mhyv8agsaaaaaaaaaa +1 +vm4010327 +1 +xorg0log +1 +yuan +1 +4813b144 +1 +sa218 +1 +healthy +1 +acter20 +1 +primarily +1 +briefly +1 +portland +1 +increased +1 +11fe0d0bb7c81ae0000050dc3b4bbf816b5daa +1 +aabvvqeo2iixjjet04joamroak5fwoliqfbtpmx2n3h2awfrk8acdddpqcuaseibdtahroax +1 +href3dhttppantmagcomfont +1 +tlcb +1 +mansueto +1 +1301494218 +1 +moss +1 +ava +1 +socialists +1 +harmlessly +1 +ppformcentertdtrtbodytablediv +1 +donnerstag +1 +nameofficially +1 +4069 +1 +tex +1 +size1shipping +1 +crash +1 +predicts +1 +alllifecom +1 +85px +1 +size3d2fontstrongbr +1 +mymac0 +1 +brands +1 +lbertarchiveimagesdilbert2003482820710gif +1 +231403 +1 +81fontbspanfont +1 +eaa21173 +1 +httptiltedwisdomcomgashtml +1 +0108 +1 +jumpstartbr +1 +g8jgs8ft008051 +1 +pria +1 +78gb +1 +u18 +1 +thailand +1 +nelson +1 +channelonline +1 +1973 +1 +size3d2bstatebfonttd +1 +valuevavatican +1 +heed +1 +17hywh0006nv00 +1 +17ui1g00011d00 +1 +ofastrongfonttd +1 +outtheyll +1 +wings +1 +d1dee3ec9e +1 +version240cvs +1 +121707 +1 +height28 +1 +rtrxgoqk6c2kthvah1gditoowzrn6nbsxdvn4xubyz0wgvkquqew9vecpeellkv7fw9i +1 +aw9ucw0kdqplbw90aw9uywxseq0kdqpzb2npzxr5j3mncg0kzxhwzwn0yxrpb25zdqoncnjlywxp +1 +etbssiguanasuicidenetagtspan +1 +aabdb3b5cmlnahqgkgmpide5otggsgv3bgv0dc1qywnryxjkienvbxbhbnkaagrlc2maaaaaaaaa +1 +httpdiveintomarkorgarchives20021002htmlrdfredux +1 +keanuvisionyou +1 +ktkxi1yg6oppqguq0xmqhh92uembfwam4rpnk1qlczhenc0gzsqyuu4kjgdsgz6fypquayzya81 +1 +online320224c4ddw5l6ay9s9ksrr1newsletteronlinecom +1 +65lso +1 +qqq +1 +photo +1 +091706 +1 +eprint +1 +lw15fdlaw15hotmailmsncom +1 +ah +1 +sakiko +1 +ecomclickq3d78d5asidukhx8ilqr5vnmun5xsporr +1 +linuxdebianuser +1 +1area +1 +examine +1 +populations +1 +waterloo +1 +recognised +1 +200208221811laa21283maltesecat +1 +rust +1 +fullupgrade +1 +10873520 +1 +apartment +1 +traction +1 +hmessageiddatefromuseragentmimeversiontosubject +1 +responsibilities +1 +accused +1 +acquire +1 +0l0k003z6uku1090asmtp026maccom +1 +495pxborder +1 +braccess +1 +c3 +1 +acroread +1 +tits +1 +chouer +1 +1255468845 +1 +pluckersitescooper +1 +dffbqidlud2dsxi4uytqs9z3udrrimg +1 +c2vudgf0aw9udqoncm1lzglhc3rpbmfsdqoncm1lcgvyawrpbmuncg0kyxnzb2npyxrlza0kdqph +1 +causing +1 +110636 +1 +clarity +1 +412 +1 +glossy +1 +vietnam +1 +sharply +1 +mdt +1 +burgunderstrasse +1 +spiritually +1 +telecasts +1 +rolexdaytona +1 +reunion +1 +nbb8xpg3u9tgktzuzv7bgfbpfvbx5zn7mrcu0fyjbmb61zppeihyurcumbdid0hcuoudpvqv5z +1 +piazza +1 +width35font +1 +092114 +1 +jainas +1 +ywnlpufyawfsihnpemu9mj4ncjxesvysgvsbg8mbmjzcdtiyw50yw8spc9esvydqo8relwpizu +1 +logic +1 +p26mr3985135wfa671271882229739 +1 +uasbr +1 +httpwwwaskbjoernhansencomarchives20020924html +1 +balance +1 +140149 +1 +b2b +1 +4bc02e709020904homenl +1 +testimonies +1 +nhc +1 +cline +1 +perfectly +1 +score668 +1 +seaside +1 +climate +1 +6129 +1 +julissa +1 +bgcolor0c7a65nbsptd +1 +47 +1 +hrefhttpclickthruonlinecomclickqe72ygqmvxo3akrdfvqs8cmb0qvser +1 +slsfyasny3fwbfgrbhgoi6eacunmkxnjszozqatfnzrmgbsam0mam0agam0lfac5qdqvlz9 +1 +hrefhttp58ebb9dbupotxpcn +1 +asp +1 +aanlktil3iybhkej2xkvaoomq9wlcp0phw8zy2xm8lmailcsminingorg +1 +groundbfontbr +1 +reservedfontfontptd +1 +detriment +1 +nameh1b +1 +84h97nks +1 +connectivityfonttd +1 +tokenizertokenize +1 +164319 +1 +wondersharediv +1 +szczecin +1 +anxiety +1 +10tb +1 +pressing +1 +epcsvqlscamunqlsmnw1ca1s8a2pwaae8ayhfwl3zwugfqntlqw9uanssameaazkamx4aticamx +1 +enraged +1 +breakup +1 +browse +1 +stoney +1 +wwii +1 +altavista +1 +174553 +1 +kennedy +1 +corrections +1 +productsabr +1 +libno +1 +g7jhl5z02133 +1 +qualitybfontbr +1 +religions +1 +b98dd16f6c +1 +10px98ment2k2003br +1 +522px +1 +74xaappit6nfcwcfh9ltmqfwl2meitkayjczjg1vqddwcqvjb4i61xv8nuoo0jefsm1ojrr +1 +antics +1 +092213 +1 +topquality +1 +242321665 +1 +upda +1 +populationa +1 +2213 +1 +type3dtext +1 +capital +1 +httpradioweblogscom0110837 +1 +deed +1 +height13 +1 +string +1 +hrefhttp3255tsajalircniciud22of9awer648b56e5z460f54ampsieehokefy56643156563099499332159industry +1 +064147 +1 +svom +1 +fullscreen +1 +nameimage002jpg +1 +exmhworkersspamassassintaintorg +1 +drwxrr +1 +21757110137 +1 +222427 +1 +ambience +1 +165236 +1 +4327 +1 +181838 +1 +abstenerse +1 +194651 +1 +pratice +1 +5446d13a4752 +1 +01ca0953edcc5f900ac2995dxpyeeueb +1 +colspan2img +1 +httpwwwhumorisdeadcomblog +1 +3urao8yglvqu1ezivkmprxuuzaftgygvsgczdxdv6zggzbgqreqzyypznxhl0csgivdtk4hsuu +1 +prize +1 +size3 +1 +titledo +1 +ext3 +1 +nazis +1 +htmlfontcolorgreenhtmlfontcolorred +1 +bpobapbektaxisbyy7kpjvacbiblfbb1eaautmjrsaafqecb7vv8ayamsaajaacdkfkpegchjdy +1 +adjust0 +1 +moravianlyceumcom +1 +multipartmixed +1 +52week +1 +fryiuagygfii2pay +1 +200208020404faa00665webnotenet +1 +dancing +1 +conclusions +1 +5so913933qyk3 +1 +associate +1 +4bec15064090508studentulgacbe +1 +123812 +1 +cardinalate +1 +g81afvg18761 +1 +encylopedia +1 +corps +1 +090015 +1 +feed +1 +downline +1 +consequences +1 +scategorytype +1 +hoy +1 +102221 +1 +sole +1 +muslims +1 +static +1 +o35ifnt8003983 +1 +vmware +1 +emacs23250 +1 +tgcv6amperagebarrieutredningpanamanianreinsurernewtbijeenkomstminorityknechtbeliebigsxadentrevorchromesangliachikumammographylgrebovinemaillenwagensittartemiaafricanamentointegralepostkarteabakinyibiloxiltrequenchforfaitsdorinebeispiel8thbillingsfuseactionbowdler +1 +n8e7t7hq015760 +1 +pre +1 +senior +1 +a78twmb4j55oefz9klutsnpactdcgufbibb2p19ktxdt9pwp59jrycrtjiypwoj7wad4tzq +1 +stylefontweight +1 +sown +1 +httpequiv3d22contenttype22 +1 +rlyyk04aolmdcom +1 +bsfsc5mj3661 +1 +134145 +1 +gothic +1 +color0099333529fontfonttdtr +1 +f13mr1383825fai981273092457506 +1 +214054 +1 +emrs +1 +hackedbox +1 +principeoption +1 +183334 +1 +iguana +1 +httplistsdebianorgv2mb36d43c31005010313gc968713chc61e587759958e7dmailcsminingorg +1 +216163192220 +1 +1ocfnr0007x6uv +1 +loc +1 +dryers +1 +archivelatest577506 +1 +175105 +1 +dvdrfontbfontli +1 +betterbr +1 +inconsistent +1 +002c01c22f674216c280f264a8c0sabeoie +1 +1022366204 +1 +234344 +1 +dust +1 +covering +1 +194759 +1 +manuallybr +1 +maintainer +1 +bgcolor3de0e0e0nbsptd +1 +direct20 +1 +enrolled +1 +four +1 +brfont +1 +explain +1 +color000000downloadsfonta +1 +25110 +1 +danso +1 +unter +1 +agencies +1 +0801 +1 +rita +1 +hereatd +1 +secproghelpsecurityfocuscom +1 +hjr2jpawvou42juniamhie3kauyqorcjlebl83ho1kyy8vmyi8v0dcf8ojyy9eunnjyu1lcv +1 +stylecolorce0000contact +1 +50029196900 +1 +size18px +1 +79d9413a5693 +1 +rescanline +1 +titlehttp6623113363indexhtmlida12b +1 +ufkky6fwrxmbx8gp1npxihpws3jcuxodehifqpaxjjjx6ds1zzjpqzkodjn1quzwc5zd34q0pujf +1 +gnomedex +1 +spamassassindevel +1 +rqtjfptgc06oqxc5ifo35fozmsqyaj0qfro7c52jtnjjpssnlxzdyfpqvsvlper38xvzvzk9bsls +1 +carrbfonttd +1 +1931042218 +1 +0xe580b363 +1 +bearer +1 +centerof +1 +bsfsc5sa011 +1 +093613 +1 +pricesfontip +1 +611px +1 +titleinstitute +1 +13px +1 +1960s +1 +distro +1 +house +1 +size16 +1 +alt3d8003271460br +1 +contends +1 +bsfsc5tg242d +1 +12564618246d7b00f60000w4twrl +1 +081c +1 +startling +1 +saa15363 +1 +09747lenny2 +1 +color3d0000ffbigbigstrongclick +1 +21021494112 +1 +solicit +1 +offensives +1 +congo +1 +httpstreamrippersourceforgenet +1 +devices +1 +apostolic +1 +httpwwwheldincontemptcom +1 +donfelipenetnowcl +1 +mailgx0f216googlecom +1 +stylecolor3974b4newsletterijuypocomspan +1 +200102 +1 +d97te +1 +testsgmailldosubscriber +1 +stylepadding0 +1 +fired +1 +sentto2242572559141033992044zzzzspamassassintaintorgreturnsgroupsyahoocom +1 +titlewuhuqoqaozuk +1 +countryifont +1 +lerlerctrorg +1 +xdebian +1 +amepkebldjjccdejhamiieajfiaaejwcseucscedu +1 +73jaadx8lqmqt8otrrsomsjjzk5kwghjsdu4rxsettgfrypticzurwvj1dsxmxpy5o1c +1 +1310sansserif +1 +smokejo +1 +jmsajmasonorg +1 +philippines +1 +asusmode6 +1 +color3dff0000how +1 +accra +1 +ele11897ted +1 +006a83e56e0c5852c1e85bb85ae3humcbb +1 +46rom +1 +justifynon +1 +heathernetnvnet +1 +lected +1 +091415 +1 +size3d7 +1 +joe +1 +void +1 +crackmicecrackmicecom +1 +6822 +1 +ohrt90feqflw3i5ikdmlphzp4wbljuqs0bjksaf54qeuwsepoa3awysrzzwvo1qybc3lnlst +1 +hock +1 +novel +1 +ruffled +1 +kho +1 +us25000 +1 +httpwirelesskernelorgenusersdriversb43supportedchiptypes +1 +regdwfcom +1 +dir +1 +ac0261616a +1 +jon +1 +cash09 +1 +alertsbr +1 +welchpanasascom +1 +alignrightcontainedp +1 +2002fontbr +1 +192546 +1 +pmitra +1 +005708 +1 +31108086 +1 +operator +1 +ceaseanddesist +1 +darwin +1 +enhancements +1 +performers +1 +v7zqqky28rffel9lj6lurxsmqmlwzfkmr2ov1octuxshzodlvwgmqrztgdsxqk8qk6e3kiipmfax +1 +size3d63enbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3b3c2ffont3e +1 +courierimap +1 +133801 +1 +hrefhttp04bd9axagonoqcnoyvyw1634943328913ug79n7g83uubofih79579335184697095905104 +1 +systematicsh5 +1 +211626 +1 +colorffffff09090909reg5 +1 +1967 +1 +140551 +1 +0b32f43f9b +1 +libamrnb +1 +valuesmsan +1 +height3d433 +1 +issuer +1 +solutions +1 +8xfq7v3mvciqog10eajna70kcewlgrnykllfbaeedjdtarc9 +1 +b49d9ce08c39 +1 +href3dhttpwwwtributecacontestfeardotcom080202detailsaspreferr +1 +joelonsoftware +1 +powell +1 +tpwwwtwelvehorsescomwrapperdocviewjhtmld3dtescoiarea +1 +g86hvgc18460 +1 +lyllandcsminingorg +1 +alignleftoleo +1 +190453 +1 +terroristes +1 +153940 +1 +mexicooption +1 +460250211 +1 +dich +1 +g8ohbni01193 +1 +dsblorgskip0 +1 +xpe +1 +antivirus +1 +hrefh2 +1 +successfontfontbblockquote +1 +httpwwwhabeascomreport +1 +182848 +1 +searched +1 +80236232177 +1 +awes111me +1 +mms +1 +x8664s +1 +replica20 +1 +pcibiosbiosirq +1 +mailpz0f178googlecom +1 +095238 +1 +httpwwwnewsisfreecomclick18381140215 +1 +xen +1 +russian +1 +assure +1 +size1grants +1 +combosabr +1 +ppm +1 +en1argerpenis +1 +monitor +1 +hour +1 +mailtonunomagalhaeseuipppt +1 +nameprogid +1 +autowhitelist +1 +zlinuxmanwowwaycom +1 +stephenson +1 +rãgion +1 +idbb7c321c +1 +hrefhttp192pillsorenruvotixic361e8f5ca602 +1 +7molnv5sp5eyhbcqajuzagvmdghi6cgczazjkuy2ddv7vyygkp0pkusrofqyudkwtsa1jsiblmp +1 +s1753845 +1 +kahlessaddixnet +1 +style3ddisplay +1 +1193 +1 +cou +1 +theastrongfonttd +1 +fiveyear +1 +typewriterbspan +1 +formally +1 +value4848option +1 +finland +1 +ultraflat +1 +occasions +1 +401k +1 +kiev +1 +hrefhttpc9bbthedrickerectruatiawyokieb7a50ff27 +1 +wifis +1 +httpfreshmeatnetusersdougedoug +1 +van +1 +124208 +1 +sackville +1 +eate +1 +en8xgptnf8zctrmneaqmmgvcgq7yga3qqgewbngmc4ykxfmxsma9wbcuxovuls0hgzygrgwasrp +1 +adenophora +1 +computerprinterbinding +1 +thingsbr +1 +45004550 +1 +size4human +1 +al7waiqbsjqy252zjsmlqarmmkxn3ixo6crapljngk2injkz2ggnhiazyzcgdqfah6yqbxc4j +1 +muchlittle +1 +commitment +1 +cellpadding3d3 +1 +161653 +1 +bsfsc2sa154hl +1 +n0l3mfzobjttdvgojg4x482zvglnpvfc3hbtfjlqxxvjf9tudpfvkoijlbracy20xp0jusobfmt +1 +pinelnx444020906212058013901100000isolnetsuxtechmonkeysnet +1 +relate +1 +marginleft3a +1 +hoses +1 +7uwkiqypnebag9c2mc7lbliszt +1 +e20cs407967wfb +1 +bgcolor10517fterrys +1 +recommendation +1 +5101402002080109374100addec0jidmailnmcourtscom +1 +size1bcomplete +1 +058 +1 +4bed4f8d2080007cyberspaceroadcom +1 +nonof +1 +213527 +1 +d3997utres +1 +cgi +1 +mailfx0f47googlecom +1 +erttd +1 +hrefhttpabsexyhumbleruojujip19d78fcd9fe1 +1 +testsawlknownmailinglistsignatureshortsparse +1 +1241854 +1 +src3dhttpemmansell +1 +17eeb156a1e3b +1 +rjuaqhot0jfnridvjh3j3faerdzrhyyez5naopcnkdr3p8by4pjvzer3qwkkkqpccomqxoch3z7e +1 +173501 +1 +brfrom +1 +size3d5lendersfontbnbspfontspan +1 +101005639034 +1 +speciesb +1 +destructive +1 +df6bf16223 +1 +initial +1 +subjugating +1 +wilt +1 +width3d45 +1 +forteanayahoogroupscom +1 +kaa20920 +1 +2822 +1 +steady +1 +xmlendifif +1 +dot +1 +1022465221 +1 +kioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkio8qli +1 +hrefhttpclickthruonlinecomclickq8bsbgbqy0yjeu6yseyieeldh4b1rr +1 +renova +1 +spamassassindevelexamplesourceforgenet +1 +colorred +1 +empcorreoonolabcom +1 +rg1ct58hiaswa4fsxsw71nkcxvxzbsypnwbinu6swkzlnameu9ck4qzrhctyxqygrjiqak4aylw +1 +reflexive +1 +c2a0swapbr +1 +bsfsc0sa148lf3 +1 +glpzkxvwmbxgbsigmv6waztpkxvwij0pbpdpbwjamznpmvypnbxamzzgzrkwz7kizqlqqdqkqpc +1 +unusual +1 +q2w880dece01004180920m4f3f9906u46872de28d1d722fmailcsminingorg +1 +091826 +1 +1988 +1 +header +1 +nav4 +1 +surgery +1 +d982b16f16 +1 +vorstellte +1 +jaa22837 +1 +myfileselect +1 +width468 +1 +dinh +1 +ciopcom +1 +contrast +1 +gmailid128cadef90bf1112 +1 +httpwwwbarracudacomreputationip8511596156 +1 +loaded +1 +chargefontfontblockquote +1 +preading +1 +color3dff0000yoursfonti +1 +hang +1 +mm +1 +hmessageiddatefromuseragentmimeversiontosubjectreferences +1 +possibilities +1 +presidentbachelor +1 +rojinsky +1 +slow +1 +bhow +1 +sorj9ngao0pkmwygpzccytwsmkmis0je42mo3ku7lnve24rnjbo7zznxyhv5k5ktnltqmi9w +1 +scoped +1 +compliancebr +1 +compressability +1 +name3dnptrail2111121 +1 +touch +1 +perhpas +1 +074730 +1 +33000 +1 +hits +1 +17b3076f0ae +1 +expert +1 +deepens +1 +size3d1balso +1 +wh53afxjx3hdvi8r9hevnxt1vpvi8sncs6jpuszijk09xrnblb0kqm3rekcacacae9vqo7xv1g +1 +textunderline +1 +henry +1 +spread +1 +mjnk +1 +desktopcopernicusdemoncouk +1 +consolebr +1 +akonadi +1 +testsfourlagmailhtmlmessage +1 +g8tiw2f01446 +1 +facecomic +1 +spanish +1 +nrolled +1 +ze +1 +a0mixer +1 +alignleftprinting +1 +7197122240 +1 +ffa +1 +h2ofh2 +1 +socalled +1 +054044 +1 +wwwdbsinfocom +1 +48br +1 +stylemargin0paddingbottom6px +1 +arialcolor8c3256textdecorationnone +1 +alltrusted +1 +temp +1 +russians +1 +1259 +1 +hrbthk0ickg3 +1 +nonehomburg +1 +swissrep1ica +1 +102239978 +1 +pjwvrk9ovd48rk9ovcbzdhlszt0irk9ovc1tsvpfoia2chqiia0kicagicag +1 +idautonumber3 +1 +pinelnx433020903132104011949100000watchermithralcom +1 +e1955f2acc3814ceesjajebtac6d4c3wwwfacebookcom +1 +egypt +1 +title123turnkeycomtitle +1 +144332 +1 +125629 +1 +163548 +1 +libdvdcss +1 +unusable +1 +surface +1 +pppa28resalekansascity14r7102dialinxnet +1 +115qu97re115 +1 +httpcariadunixwiznet +1 +pnbspptable +1 +debating +1 +donahue +1 +bsfsc0sa335 +1 +chatbrbr +1 +colspan4img +1 +625 +1 +indian +1 +gbbfr7ir9eoa20yy +1 +d7mr9176665fgi751272854165513 +1 +cg0kdqpnyw5hz2vtzw50dqoncnzpbmnyaxn0aw5ldqoncmlmb3nmyw1pzgunpc9tuefopgoncmfs +1 +colorfffffftable +1 +score71 +1 +adhere +1 +rise +1 +smtpmailgnxouaolcom +1 +hrefhttpclickthruonlinecomclickqcfebmqlw2hw8nmhvjaylio7r3fxer +1 +175411 +1 +valign3dmiddle +1 +tel +1 +excellent +1 +younbsp +1 +081538pm +1 +colspan3d4a +1 +guerin +1 +8220stiffers8221abr +1 +dishh2 +1 +hrefhttpwwwmilkapilkaruqdoofay9b98bbd62aefa +1 +95nbspspancontribute +1 +efontfont +1 +o1cihrlq011620 +1 +174556 +1 +clamsmtp +1 +lãgaux +1 +defeated +1 +heralded +1 +rvp +1 +calling +1 +cover +1 +src3dspacergif +1 +033626 +1 +vspace54 +1 +h5countriesh5 +1 +gmailid1285fa5e445783b4 +1 +g6vhzd203399 +1 +height3d52 +1 +hospital +1 +emacs +1 +omega +1 +hazaribagh +1 +httprdyahoocomhttp80717188asianaid818932 +1 +mitchell +1 +a274101923387038footerpdfcolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightboldfontstylenormaltextdecorationnone +1 +hrefhttpswwwsandhillscomsecuresmartcomputingcpufreetrial3aspsourcecp491bonuscomputer +1 +moneybli +1 +fatal +1 +18012980122 +1 +2125442165 +1 +theirp +1 +racked +1 +href3dhttpclickthruonlinecomclickq3d1frvwqi58w5nhc5a01x +1 +breath +1 +puffynbspfontfontb +1 +thediv +1 +sa011c +1 +valuemauritius +1 +q8a3myp80fkuhy8sl9p2awxrpp7u5rs4pfvfhiw7s2petj9hfvfubkf5owdscx7c +1 +imprononcable1ldosubscriberldowhitelistmurphywrongword1 +1 +delete +1 +park +1 +technology22 +1 +1049 +1 +successesquotbrbr +1 +family3d +1 +investigated +1 +clientip8316119359 +1 +1260277451 +1 +l3300 +1 +8365980d93b +1 +professionsl +1 +ldowhitelistnorealname +1 +score152 +1 +height1nbsptd +1 +processed +1 +flames +1 +interviewed +1 +020030 +1 +federation +1 +wdd +1 +g715ra232199 +1 +timbwcvuzbyaunlbqewqvs1siks22flsqro2fbj9bvukwjgzwx7cm2lujk5vvwc4uen2q3f +1 +sullivan +1 +1550 +1 +mostly +1 +httplistsapplecommailmanoptionsx11usersmlsubscribertech40csminingorg +1 +revolution +1 +marshallese +1 +012425 +1 +415 +1 +41113104222 +1 +srchttpwwwhandangocomimgsthechampion2002featuredsoftwarebargif +1 +87db7 +1 +dhumjgyndmam2qzzdq0njg1advonda2odaknww3edfgoew4udkkoza5dpsotg7sdu4pcq8ld0ep +1 +014035 +1 +size1btop +1 +bitbitchmagnesiumnet +1 +ed7cc1c3b5 +1 +thread +1 +bbb6a27456102 +1 +w39mr393328mue201271360368122 +1 +champions +1 +238 +1 +1265059627 +1 +he7vt1pojbdfrvgbr9sh9fkd9dajdd94sd7mmedi33d16807pcscp5bbsyip0penhp1rul0y +1 +nonfreely +1 +indivisible +1 +2003 +1 +32229347 +1 +customize +1 +btreatment +1 +luis +1 +1955 +1 +groom +1 +boy +1 +sporting +1 +te +1 +plain +1 +innovative +1 +a05200a01b9c80b70e2c220910320317 +1 +imagined +1 +hits188 +1 +1b619212d30 +1 +nonefederala +1 +temperatures +1 +supplemental +1 +132934 +1 +brbrbrbrbrbrbrbr +1 +20985221194 +1 +201005120653562352qmailwwwbeamtelecom +1 +paints +1 +tourism +1 +schizophrenic +1 +householder +1 +investing +1 +sept +1 +panamanian +1 +height153 +1 +rkyipg0kicagicagicagia0kicagicagicagidxkaxygywxpz249injpz2h0 +1 +htmlshortlinkimg3 +1 +x6falds46gdqals8vf3uwwy0evqe6nduxjpamhr6viuccni7go3hwgrhlvmi4nubueon8skavqp +1 +4bf2e76b9070702studentulgacbe +1 +forth +1 +tenderness +1 +link242424 +1 +hrefhttpclickthruonlinecomclickqf7n0qnqlgzfyw3rqhlrzigir6imoer +1 +holder +1 +width3d670 +1 +valuelockergnomeinput +1 +xmessageflag +1 +ruled +1 +865cd7a196d +1 +sobom +1 +forum +1 +ptfont +1 +2222 +1 +formspanbptdtrtbodytable +1 +inbox3448 +1 +trtdfont +1 +181811 +1 +kiener +1 +tibetan +1 +stylecolorffffff +1 +1862 +1 +featureabr +1 +orchestras +1 +burgeoning +1 +size3onefontspan +1 +overregulation +1 +bugtraq +1 +copyfight +1 +grublegacy +1 +aukqymbshrj39kzzuwzule6n1bgzrhhzd2m8h3qqnmuktdqulwfy1kyjecoppfdvyx8vadgf3u +1 +461ca2c54a +1 +egovernment +1 +hotline +1 +connection +1 +solitaire +1 +chanted +1 +sciscientificdirectnet +1 +vouchersfontatd +1 +nullmailer +1 +shafer +1 +valuegwguineabissauoption +1 +andy567487yahoocom +1 +enjoyingbr +1 +dcc4e84cb8fc01ca8f8654c982ec8526 +1 +clock +1 +005120 +1 +57937 +1 +n7uja4av011200 +1 +faced +1 +6541216221 +1 +14these +1 +hrefhttpqysn80jnuaixagcnxoqufoluxy1890426e73a9fb81qmusijh1550214841 +1 +squeaky +1 +005306 +1 +howtos +1 +foreknowledge +1 +httpwwwbarracudacomreputationip8913422138 +1 +032304 +1 +suburbs +1 +colorbluevolunteersfont +1 +debians +1 +antvilletreffen +1 +bootvmlinuz2632trunk686 +1 +belphegorehughesfamilyorg +1 +ywr91642908075547052401100531077373 +1 +82 +1 +hex +1 +hail +1 +networknbsp +1 +22bf513a5a7b +1 +unregistered +1 +inquiry +1 +xjqxsr3mamwbl8rsditvbvsrj8mvfp8uid0sjmsat0kdzfioaaach5bafkagcall0bogctaf0a +1 +value3djerryliversaigannuitycom +1 +titus +1 +nobodylocalhost +1 +zip +1 +colspan9nbsptd +1 +smtpmaillkelliwqnpdnrwnet +1 +9dd154ff43d8dd67cbc9e92a91156362 +1 +landing +1 +g59bthg14256 +1 +cpe653144139wohresrrcom +1 +httpwwwscreenitcommovies2002thesumofallfearshtml +1 +hrefhttp0314bgeyozzncncuzubegiz812661sl3w5jt762p05h6rqwenasocite32476721001939642710161logusobe9849580118z859765k98l136kizyqu423651015299591373164click +1 +97 +1 +target3dblankun +1 +size2lock +1 +vaa13010 +1 +xyahooprofile +1 +bsfsc0satfamsbl +1 +architects +1 +microsofts +1 +devfb0 +1 +rgif +1 +210234 +1 +282166630 +1 +srchttpwwwahahealthcomassocimagese6e6dsamtote14gif +1 +42e0 +1 +owns +1 +auszug +1 +152318 +1 +1439 +1 +color000080 +1 +httpuseperlorgarticleplsid0209191443213 +1 +design +1 +efforts +1 +a25si4724118faa1620100523143748 +1 +rosedale +1 +190747 +1 +publishing +1 +seek +1 +v60027412600 +1 +yisr2rhyetl3bkk23x8ax4ib3x6v44w89c768jjyya8oyux6ezxztm37m5xcasyaseyr9a +1 +ukemoxavu +1 +3d6e73479090304sricom +1 +duis +1 +greg +1 +russell +1 +clickthruonlinecomclickq3da7fr2q8bwvdmi5xhlf5vlk7kcgfer +1 +10223581 +1 +alleviated +1 +23064401c23a7883301c506b01a8c0insuranceiqcom +1 +0fw +1 +59m +1 +455px +1 +cnetbfontbr +1 +102231867 +1 +href3dhttpc87hucojojcnwyxypj3deb101e14f0be4a1ffdf44791 +1 +drydio5hnhpxljletldqgp3kxyufyakqzaekda05q2qwgu4jxvj6irtv4o0xv7oacmxysrlvc +1 +cleveland +1 +somep +1 +charset3dutf8 +1 +together +1 +o4olgknc004323 +1 +lennyvolatile +1 +e53a816f72 +1 +nbspbsome +1 +afterthought +1 +helodarwinctymecom +1 +name3demail +1 +n8lakf68011484 +1 +grep +1 +062723 +1 +1932521969 +1 +todtaclightingcouk +1 +nike +1 +httpbroadscapeventurescomweblogdfme +1 +compiled +1 +3br +1 +123929 +1 +hrefhttpnlcomcomservleturlloginemailqqqqqqqqqqzdnetexamplecombrandzdnetfree +1 +mailtospamassassintalkrequestexamplesourceforgenetsubjecthelp +1 +hrefhttpwwwcokodageruysygexyfymekjceb5d662abcb7 +1 +methodpost +1 +des +1 +blockquotedivbrdoing +1 +profits +1 +arbitration +1 +n9p0mjjd008936 +1 +21font +1 +weightl0ss +1 +xqqajuz1sfsjazabecqweejvwqcoawqahqaa49vb8jawahoq8iuyeqiekhohjwgiya6ika0cwml +1 +200932 +1 +australia +1 +bgcolorccccccimg +1 +pdeciding +1 +httpfortytwochgpg92082481 +1 +comoros +1 +g984ch428341 +1 +browns +1 +none7dah +1 +pocket +1 +prompt +1 +bprevious +1 +unnecessarily +1 +src3dhttpiiqusimagesceugo7jpg +1 +policyap +1 +inventor +1 +richmond +1 +200209092354g89ns8g04304lin12triumfca +1 +httpwwwtheregistercoukcontent3926249html +1 +webmasteredgecpacom +1 +emc +1 +samuel +1 +helo1921681241 +1 +6e5bb6f +1 +notifysuspend +1 +17a14c0002ww00 +1 +gmailid1280567e9242a03d +1 +tightening +1 +g8p28hc04819 +1 +nl +1 +httplistsdebianorg4bed2a894000902allumscom +1 +class89016491926062002font +1 +tmsgidgoodexchangeturicount01 +1 +piteously +1 +rgpujrqayobny449klerjxipxqcdamdptwuecigaqg9dsk4qojqswcdb5q0wboe1adn5j4ogna +1 +digest +1 +0x015a4e35 +1 +gmailid128ba18fd504d3c8 +1 +load +1 +outlive +1 +httpsearchbetacpanorg +1 +cnet +1 +cheapest +1 +172541337 +1 +xmailscannerclean +1 +afaik +1 +puta +1 +heavily +1 +91184323 +1 +sf8azjlsf0fif0v9o72ekbxjbdud987n30ulqn1b6r026xlllraagc7iqy81ajdouy36cq1 +1 +200905171242n4hcgb7n015410ns2csminingorg +1 +g7mijaz19382 +1 +oscar +1 +mps +1 +cd3sf3f +1 +score239 +1 +20100512194713123modestasvainiuseu +1 +style3dtextdecoratio +1 +jokerunibech +1 +g93llbu05318 +1 +dgltzsb0byb0ywtligfkdmfudgfnzsbvzibmywxsaw5nigludgvyzxn0ihjh +1 +ownercypherpunkseinsteinsszcom +1 +assigned +1 +pecondonmesanetworksnet +1 +icsnewslettershdanchorgif +1 +mda3mjq4lc0xmzq3nze5nzesmtg2mzkzodi1mswtnzcxnjm4otmylc0zodew +1 +bpmqyedj826jrrcstyj2sras73fjjtioe2qdgkdqvsxw9zlforyzfacxesvmfwliyd3qb0osc2u +1 +postcards +1 +ldapdb +1 +llyovalyaolcom +1 +filled +1 +copies +1 +eugen +1 +h95f875ldeafyifzeo2d60ft +1 +282193981 +1 +confidentiality +1 +insights +1 +054926 +1 +increasingly +1 +seqset +1 +ja +1 +8734icum5941fzrj1099dvnh4450zrld0601vnia4052smew9113qiau3562mewq9923ibvn21775 +1 +chpkjshnkjsgn0fugtweuqlyv3mot4fcbi0ighe0imhe0iigeuejgkyejqgaetqiaptmoqga +1 +reportbug +1 +parties +1 +hrefhttpwwwlockergnomecomsubmithtmlsubmit +1 +190123 +1 +cuoco +1 +alchemy +1 +130 +1 +091622txtdogmaslashnullorg +1 +y20mr5015654fau781273047034832 +1 +csbookswileycom20 +1 +222103 +1 +jj +1 +224508 +1 +texts +1 +lib32ncurses5 +1 +g14eg5a26662 +1 +recycled +1 +640 +1 +h111pef117lly +1 +signal +1 +1253202349 +1 +roper +1 +treatments +1 +dav23pvtyw4zlvuv6xp00009131hotmailcom +1 +g98gf1l25092 +1 +julnbsp2009fontstrongppnbspp +1 +20090614 +1 +selectedmmoption +1 +intelligently +1 +npr +1 +1932110245 +1 +percent +1 +buildercoma +1 +restricted +1 +uybf0qfq9xfsbhbwxlwjjcg9erevdk6tyolwyt1mk5wtypmp1rc5mpowl6g0ntmgmezhpwiyz +1 +copying +1 +furchterlichste +1 +facearialhelveticafont +1 +741258247 +1 +muddled +1 +villain +1 +paddingleft0px +1 +dpkg +1 +size2font +1 +j35j31gicdbpuyb4lh9wzmvrovnr2xdjcvoqu3meqbcxjehrvu1olvv390t9m +1 +valign3dtoptd +1 +fruitful +1 +dick +1 +rqzxzlkirhhqsikvo6msxolvan0lf1x0uqqwarrgldcekrqargrzoowcxwskt58yg2nlfndltru +1 +180452 +1 +width3d65nbsptdtrtr +1 +cols3d1 +1 +111529 +1 +43ac144155 +1 +301793 +1 +rewardsfontb +1 +trustworthy +1 +albino +1 +hrefhttp8afdb0nzucocadcnyiqezy85nv28m876397hbg468typypi654021682483656937023 +1 +stream +1 +165251 +1 +hda +1 +fairchild +1 +hrefhttpcodeweaverscomproductscrossover +1 +a25si15961158faa6820100518084353 +1 +threatening +1 +friendafonttd +1 +154027 +1 +dimkhh0xuaguxufqf3flzw37dc6xownquow7h7e0ddlnwnmn09rcnkhhpgxlzisfdr7xa3ewern +1 +211217214 +1 +flap +1 +r2yb36d43c31004300127q8adf79dan4640c1148027b04emailcsminingorg +1 +c113d13a51be +1 +size1free +1 +srchttpa02fcvkunrorfcnlogogif +1 +235342 +1 +chronicles +1 +converters +1 +b6a0b2d0c78 +1 +smime +1 +9xo1xkkklngljyu42aayodyrtzfg6cvpgnhcchfdrv9jxpwfapyb5ul1v1l8yzlldme35tww +1 +remv0615cbkru +1 +pt09pt09pt09dqpcrsbbie1jtexjt05bsvjfiexjs0ugt1rirvjtifdjvehj +1 +kevin +1 +cwaaaaaqaaaaaqaaakaaaajaaaaaadhcsu0ehg1vukwgb3zlcnjpzgvzaaaabaaaaaa4qklnbbog +1 +href3dhttp2005811533rl000306jgnmzmhtml +1 +stature +1 +043912 +1 +political +1 +046 +1 +towering +1 +lender +1 +asthma +1 +hems +1 +mindful +1 +ee38913a461f +1 +criteria +1 +drugs +1 +reminders +1 +alignrightofp +1 +10297ct111r115 +1 +828fb2940eb +1 +testsfvgtmmultiodd +1 +srchttp65112138121mailblockimagesmblogotransgif +1 +waa23907 +1 +municipality +1 +lbhrkgkkufgv6gmkxqryblgt1wpkownywfiju9mdquoobggrvj3m2h1lilxrvcexrs0ukaeopcu +1 +dcom +1 +gillmor +1 +skqaxylrpdga6wbyxgvlbliszt +1 +743282 +1 +interactions +1 +222400 +1 +rates10guaranteed +1 +7412420024 +1 +6170 +1 +register +1 +hrefhttpclickthruonlinecomclickqad4iaqzebsic0nbiqhaihsszky0pr +1 +ind +1 +62016 +1 +escribic3b3 +1 +tolerable +1 +unbounded +1 +href3dhttpqrdspellyourcomzwz +1 +size3d15brbrbrpto +1 +clkmp2sqwsop0qftstcawchtjmmwrwk33vt61oqdlg0a8w0gip7zyyvdpnzdftgkrrxnofs0q +1 +k29si3575023fkk1520100420151340 +1 +diversity +1 +brthat +1 +81118111 +1 +eda37003 +1 +y0ilbyele0idlcaraa3hb8qod32eqr6jncu3wfaavhpzwasemfchuspdnslscxikqi5zwgbgvfo +1 +fish +1 +garymteledyncom +1 +sends +1 +sterling +1 +163500 +1 +surprising +1 +153745 +1 +src3dhttpwwwcnetcominlshsubfeatgif +1 +libgnomekbdcommon +1 +225545 +1 +krslwwezfwrfzbpucmndhmbabexeygodp0t4 +1 +3ef8b7b +1 +smarthost02mailzennetuk +1 +score458 +1 +prevents +1 +arcane +1 +width3d518 +1 +mondayfridayopopspanstrongp +1 +geek +1 +hxcqkbrhsqcpai8wmqaxetbkeapvgab0xf8iqovqany1gf3oaxjwitmuazmtvqvogvfka4lkaiz +1 +netherlandsin +1 +133810 +1 +relay1applecom +1 +strongturn +1 +iran +1 +pbs +1 +pfp1j1kxdgegin0yj2xgw9uiarhqldlgfvjxsgggbkhsca8liaigm1dkfairawhamwuweubcdnew +1 +32055 +1 +zzzzjmasonorg +1 +100224 +1 +anders +1 +20985218210 +1 +choke +1 +baxnjmjnads0uazrmgbzawxioqjzqbnlmgcpy1oezp2axnaaqadadswkzs7qaemypsmmm0nlmgc +1 +throw +1 +omniweb +1 +width150 +1 +g7b9brj25638 +1 +episodes +1 +colorwhite +1 +covered +1 +wojciechowski +1 +photoelectric +1 +setting +1 +deedsmisaculinknet +1 +033023 +1 +meddling +1 +dictate +1 +v1408089726 +1 +gmailid12801d148efba944 +1 +busta +1 +6img +1 +href3d22http3a2f2fwww2ewldinfo2ecom2fdownload2femail2fnewees2ezip223edownload +1 +aspx +1 +hrefhttpclickthruonlinecomclickq1ejquli4jwhrb77cpqesb8le27qdrr +1 +1oerkd0004w6ag +1 +distributor +1 +ontpageauthors442gif +1 +153448 +1 +5245868 +1 +v30m +1 +221108 +1 +a22481297394458footerpublishercolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightnormalfontstylenormaltextdecorationnone +1 +resigned +1 +popfriendly +1 +roi +1 +defined +1 +consciousness +1 +194222 +1 +href3dhttpbjpoidfecnfont +1 +2day +1 +241a +1 +color3dfff +1 +6416612219 +1 +commonly +1 +spiess +1 +iso88591qq3b2h0a096265 +1 +wfzky1fgmdhlxfrh +1 +244 +1 +findemubr +1 +stiffen +1 +itsuaopdsumaafnspas0g05gr1frzqzgp3asplsncrpsqlnoo9op2kmvixtfoxrqbkkshlm5g +1 +103 +1 +otx50sedyumiqcnyviwuf3d398eb616cbbca9ce86pjigjja3d26597304777 +1 +661451366 +1 +messageidreferencestoxmailer +1 +n51jifp4022489 +1 +399 +1 +decision +1 +eputqgejnj +1 +attempted +1 +marketing09 +1 +explode +1 +1934 +1 +strongthis +1 +targetblankothersa +1 +dprasworldnetattnet +1 +winnt +1 +phicsanchordeskfrontpagebgif +1 +neki +1 +e15si17739102fai7520100519034950 +1 +httpwwwjayredingcom +1 +131201 +1 +gens +1 +2xive +1 +testshtmlmessageldosubscriber +1 +homerperfectpresencecom +1 +203001 +1 +announced +1 +bnisch +1 +visible +1 +hollywoods +1 +millionfont +1 +color3dcc3366b187bfontfontdiv +1 +rsescommmclickthrough233170622702155823333143ubn6sdjvj8dmr08myuluac +1 +ancestor +1 +71ypdaowdqvdn9pettingzoonet +1 +nonecompaq +1 +iq +1 +journalism +1 +ahhh +1 +iny +1 +lori +1 +aligncenteragep +1 +engaged +1 +width3d349 +1 +rowspan4 +1 +h2septemberh2 +1 +period +1 +rasa +1 +mozillas +1 +exmhuserslistmanredhatcom +1 +width132 +1 +1268570875079de6ff0001w4twrl +1 +searchnetworkingliststechtargetcom +1 +54pt +1 +score3813 +1 +href3dhttp981e9evqificancnjql3df06301bc2f23f8cc117c885j +1 +3d1tdtr +1 +alarm +1 +hrefhttp6bdrefillmarten81aruiolijivotud057d9ca250d2 +1 +hrefhttpwwwcompagniehumaineorgautumn91html +1 +knaana2xq8lpi6rtgd06yimcbrme45ih +1 +uid119331201487268 +1 +demandsbr +1 +173705 +1 +bgcolorccccccfont +1 +codec +1 +tholins +1 +blepcenterbodyhtml +1 +websitefontdiv +1 +pci +1 +144119 +1 +authfail +1 +tdemailbody +1 +boiling +1 +eaa +1 +1906517166 +1 +httpwwwhawaiistoriescom +1 +height3d40 +1 +xdreksuhlkuytkyspnjct +1 +9lmss2jtrki56yojfgh9spksqezai3l08b8v4sq6k8jrwwtpdk2wwliulmwrfqyymimqbpsjmt5 +1 +waiter +1 +esophageal +1 +mission +1 +700 +1 +jyh5ax3y3jmclflse6hvwr1dzzduhwrzdlm9vrwat8y1egzg7tq6wbgxegmnhhcbchm +1 +choicemail +1 +kindly +1 +ce3darialverdana +1 +stir +1 +eirikur +1 +impervious +1 +galilei +1 +httplistsdebianorghscb3af9f1doughgmaneorg +1 +smokestack +1 +mccormac +1 +bgcolore5e5e5img +1 +freedexter +1 +a8cc113a5297 +1 +1242671534 +1 +character +1 +250ms +1 +8899nbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +1 +b3xmqqrkciye0sgvv0l5qqnkxsuveigkrvbu6rdtbxwva3xplqu04qjzxn0htriacede354 +1 +20020812fz464416600wwwdudexnet +1 +size3d4 +1 +size3br +1 +208971325 +1 +trigger +1 +reso +1 +hrefhttpwwwnytimescom20020812businessmedia12bookhtmlhttpwwwnytimescom20020812businessmedia12bookhtmla +1 +xx286646anonpenetfi +1 +eliascseucscedu +1 +bye +1 +width600 +1 +income +1 +offutt +1 +pan20100429194506csminingorg +1 +b83a03f6e9 +1 +uid106971201487268 +1 +gmailid127e948f3756dccb +1 +visite100 +1 +hdomainkeysignaturemimeversionreceivedreceivedinreplyto +1 +subtracted +1 +brbrbr +1 +capactior +1 +1a04813a457c +1 +hrefhttp77hardsupplyruevifaawi13418c9c4eca38e +1 +deb3dsetopt +1 +desperately +1 +neediness +1 +uncommented +1 +800dff5b3 +1 +wim +1 +oecd +1 +inp +1 +e5fbfd +1 +carbonate +1 +islamism +1 +dutch +1 +nameliberalism +1 +ofstrong +1 +prophecy +1 +1ehttcwkeaecjklonnghpe4uc2xhumt3u2ta0q0qe6sgi48va2pjjjfapjjjjtapz +1 +saintes +1 +agency +1 +facetahoma +1 +classnormal +1 +screens +1 +therapist +1 +bookbr +1 +cellspacing3d3 +1 +94304tdtrtabletdtrtablebodyhtml +1 +cultures +1 +1250665900 +1 +unpacking +1 +007 +1 +hinged +1 +size3d1nbspsans +1 +g8deioh01540 +1 +4bbff50c8010308homenl +1 +ditch +1 +19jy5mhbq04l3a3u9acm9epf8nforkspamassassintaintorg +1 +january +1 +001517475c8a468ca20483bd259e +1 +contentcollege +1 +14pt +1 +e15mr2610440fgi231272296699829 +1 +platinum +1 +castle +1 +tls10dhersaaes256cbcsha132 +1 +purposebuilt +1 +um +1 +tube +1 +12730920611953974camelfamilypaciferacom +1 +mailtodebiankderequestlistsdebianorgsubjecthelp +1 +ia0ku2f2zsb0aglzig9uigegzglzaybhcyb3zwxsigp1c3qgaw4gy2fzzsbp +1 +focusclick +1 +4bed20469000006coxnet +1 +hrhotmail +1 +ccfl +1 +vtbt5zfkb6kasptbmwgxgt1bo0rufewpy7zcadzqpuxafptomilx2fm0pjljqnnk8xqzxjr9pw +1 +turmoilcenter +1 +msright +1 +fedex +1 +conventional +1 +levitra +1 +internetall +1 +083105 +1 +dc68713a538f +1 +13380006681398081270566209388javamailrootmd01wowsynacorcom +1 +9p +1 +countrys +1 +fetchmail +1 +sa275hl +1 +hunter +1 +a52block +1 +noblot +1 +174559 +1 +ccsun37ccntuedutw +1 +unix +1 +3several +1 +transitioning +1 +utf8q3vr9szxa5tvz2dbl2dyyrfv6k76a93l6g0a098gbpej2tkfikpap8d93f6v +1 +quo +1 +88d404986 +1 +kaddressbook +1 +nowrapfont +1 +rafael +1 +cybozxcgum9tyw47ig1zby1myxjlyxn0lwzvbnqtzmftawx5oibuaw1lcybo +1 +050020 +1 +newsh2 +1 +app +1 +anybr +1 +xmk31sivc4gvaqys57vq3yzpvo31gdkuzpr2c5jvpymseo1ru6nigwo9q9nwbk6qxbvque1bnhmc +1 +bhnua2m5uhyg9hexlryyoknpchy7xzf4o3qpooc7nyqq +1 +adsl +1 +frankenstein +1 +stylecolor336699homefonta +1 +httpwwwportigalcom +1 +mx1spamassassintaintorg +1 +jackson +1 +depart +1 +intro +1 +1859 +1 +ihn0ewxlpsjtyxjnaw4tdg9woiawchg7ig1hcmdpbi1ib3r0b206idbwecigywxpz249imnlbnrl +1 +methodmancanadacom +1 +756j0fk913z +1 +074304 +1 +llc +1 +impress +1 +xk4nv7cqsidb1bh0mrvefur1mnxk4t6q9qnoq7cefzdlmetwop7bv3fli6n5jthwzmf0trfz +1 +height375 +1 +moneybr +1 +hrefhttpwwweasywebeditorcomcreatewebpage000004htmeasy +1 +stock +1 +3d9b62478090205endeavorscom +1 +bgcoloreeeeeeimg +1 +mtainterface +1 +pool +1 +vobcopy +1 +welles +1 +customizable +1 +f09c52c51e +1 +evolves +1 +href3dhttpwwwinsuranceiqcomoptout20 +1 +runs +1 +body9px +1 +psybill +1 +70 +1 +officer +1 +1241621903 +1 +333a +1 +vncstartup +1 +newsbfontbr +1 +height129 +1 +faceverdanaarialhelveticacant +1 +httplistsdebianorgj2nd0bf7b0b1004301943o2dd0f30fwb811b71e1ef8682amailcsminingorg +1 +weber +1 +holders +1 +hrefhttpclickthruonlinecomclickq36wg7iii4fxkcfotmxvh3qainxq4r +1 +tqcl8jfofrhhusr +1 +8024132206ucnombresttdes +1 +municipalities +1 +httplistsapplecompipermailx11users +1 +cyclical +1 +bordercolorlight3dffffff +1 +thirdparty +1 +ripoff +1 +091121 +1 +2k7jwtavhdec2lrbpr +1 +fujiwara +1 +102113 +1 +153853 +1 +hmessageid +1 +mini +1 +unreliability +1 +doubt +1 +traumatize +1 +hits182 +1 +127 +1 +10pxcomfonta +1 +schemes +1 +pumped +1 +priceperformance +1 +httpwwwdigitalrivercom +1 +envex +1 +288 +1 +mach +1 +qualitã +1 +xproofpointdetails +1 +know20 +1 +fontsize16pxadobe +1 +stylefontsize11pxlineheight16px +1 +checker +1 +140536 +1 +73w8l +1 +vlink0033cc +1 +doublecapsword +1 +062918 +1 +antelivefreenet +1 +a3qpwe13e5nma10 +1 +secondary +1 +ty5iuks5jqy6y86srt9tnom1ixhmpslt7tbexsctltmroof7uvxgg0vhsuf5zczjelrh2 +1 +d55072c1c0 +1 +smtpmaillikubos6311brasiltelecomnetbr +1 +sutilbrbr +1 +style3dborderwidth +1 +17kdru0001sk00 +1 +yjcs2i8b8eyfqvnvgndjaiw4oik8picaahj1vvavgepkacrdcwnbktoyjqhiapaalpeanyugsl +1 +141141 +1 +francebrfonttd +1 +stabilization +1 +180611 +1 +quinn +1 +streams +1 +saulcisupennedu +1 +i5m9o9vr2wtp6tfwn0war9ouzsueli4jtwbs9virlz7bo7ess79kwzgyfdsrb6e7v61oqt +1 +rmb +1 +bmaking +1 +metronom +1 +althea +1 +targetblankmacon +1 +departments +1 +copymydvd +1 +appearances +1 +2556491791925261270748527351javamailrootmd01wowsynacorcom +1 +0pxdiv +1 +dacia +1 +woman +1 +bgu1aqaceqmhmrieqvfhcsitbtkbkrshsuijwvlr8dmkyufygpjduxvjcztxjqyworkdbyy1 +1 +lose +1 +hall +1 +g918s1f18842 +1 +m8grpscdyahoocom +1 +repr +1 +hrefhttp48cmilealsoruuyhecaa29f5e14f84d5ba +1 +dynnjablskip0 +1 +20021002t0934000800 +1 +xxsmallif +1 +consistent +1 +ipv620014b7820002 +1 +score149 +1 +printed +1 +clarify +1 +qtj +1 +records +1 +counterparts +1 +b4000b +1 +swiss +1 +c2sbx +1 +200207271701saa23172lughtuathaorg +1 +fox +1 +64 +1 +eas +1 +cwgdated102779151394b180deepeddycom +1 +sattriyah5 +1 +dcbe9294206 +1 +re2 +1 +tartiest +1 +1075a29409c +1 +fdisk +1 +716901cae8b38f6a75ebda4de317qeb12pi +1 +01302003 +1 +103726 +1 +graph +1 +attributed +1 +eq1rjtentjp8ackpinwjamvnkjp8kkklivkifeshl88aisf4uf8actawrj7zx1ztci +1 +frederik +1 +styletextdecorationnonehereafontbrp +1 +65722 +1 +reminder +1 +whirligigs +1 +130stop +1 +debianlivelistsdebianorg +1 +xc5a +1 +131656 +1 +uninstall +1 +5e16016f03 +1 +dnsfromrfcwhoisfourlafvgtmmultioddimprononcable1imprononcable2 +1 +mjulianivcedu +1 +extras +1 +hrefhttp46franklinedrunujya715914adf8click +1 +colspan3d7 +1 +httpwwwgeocitiescomkindlyrat +1 +abraham +1 +annual +1 +titlesecured +1 +sections +1 +water +1 +strongall +1 +kdestandard +1 +programme +1 +yahoo +1 +oaipuses +1 +3r3109uznbspnbspif +1 +roundabout +1 +deletedyou +1 +6123027136 +1 +score908 +1 +excitecom +1 +backgroundcolorb00836 +1 +li4uli4uli4umtegbw9udghzihbhc3nlzcb0agvuigl0igx1y2tpbhkgy2ft +1 +kencoargoluxcom +1 +branislav +1 +systemoutprintlnlist +1 +ickes +1 +183657 +1 +nas +1 +idheaderfooterp +1 +listboxes +1 +113700 +1 +strongly +1 +20100512112158209howlandprisscom +1 +extraction +1 +boundarynextpart00000d237d32c7ae3536a81 +1 +023043 +1 +chipset +1 +editing +1 +24720 +1 +appear +1 +rare +1 +surroundingsh2 +1 +saw +1 +105549 +1 +letterspacing +1 +u1rssunuifbssvzbq1k8l2zvbnqpgjypg0kicagicagicagifnwzwnpywwg +1 +train +1 +risk +1 +ftpftpximiancompubximiangnomeredhat71i386gaimapplet05911ximian2i386rpm +1 +responsive +1 +1142 +1 +beating +1 +lives +1 +140047 +1 +disapproval +1 +easierdiv +1 +conservative +1 +4ovttcjuy8hauk8vsuc2dwztziwixuuznyoa3sbnopztjlbmkycrsmbwocvzjw76bl9ssig1vu +1 +ebenezer +1 +84h97nk +1 +25si1493039fxm4920100408164221 +1 +144728 +1 +retained +1 +j5skhbsvxntk9kw1xdxl9vzmdoawprbg1ub2n0dxz3ehl6e3x9fn9xeaagibagqeawqfbgch +1 +spacer +1 +colspan +1 +vi +1 +yugoslav +1 +testsbigfontctypejusthtmldearsomebodyforgedrcvdtrail +1 +potentially +1 +yy4xhzadbgnvbastflzlcmltawduifrydxn0ie5ldhdvcmsxoza5bgnvbastmlrlcm1zig9m +1 +value990000input +1 +nfrwalso +1 +ha13sm1786896ibb1520100510132526 +1 +webmasterekuhnlede +1 +naeojlmpjmjdfplhiojogehbdbaabmordiconnicholsoncom +1 +src3dhttpwwwzdnetcomgraphicsanchordeskf +1 +utf8qcicycxrcoxwmurhckrvhcedaur7own20a099cqewexrexa7tjlyn9gscdhb +1 +1341476414 +1 +mailtoexmhusersrequestspamassassintaintorgsubjecthelp +1 +slacklnecom +1 +hmimeversionmessageiddatetofromsubjectcontenttype +1 +hsgmw3ctxukbenz0otazavjtabxsnid5bmpg8vuw2anlkjgaawwtqadlovw3sq0m1jkda7mrgnww +1 +hrefhttpwwwtuduwrjcnxu68ebeb7fc8d97f4d437e +1 +columbia +1 +g8ukr2f11316 +1 +instructionfontp +1 +bypass +1 +replenishment +1 +bledisloe +1 +indicating +1 +theo +1 +kbqcc2ztau6niyu61cexdq2dmexybjnkewuhg2ewxzanisgspasoakhjqu8vqcznufjmnihgmyls +1 +20082010 +1 +sirius3dsmuggling +1 +scandinavia +1 +uvifbluoftqo7cbd2janw6akqeb4gifxbgubgarprieqlgik8gjbqbkr1grmfgihocnlxalkjim +1 +href3dhttpxnet123alias +1 +strength +1 +falseexprossr1html +1 +markallumscom +1 +4f5d32d0c6b +1 +leather +1 +i686 +1 +darren +1 +lennybackports +1 +20020474168 +1 +af2sx6b92j091lhfioa9 +1 +blogabr +1 +tony +1 +thousand +1 +colonies +1 +55ckeq8vqefzwebzv8hkxzs3km6vwvqkw0acxle656xnuyx7iakgolrapaagicawl07aaqygsf +1 +leboutillier +1 +kxvdcoajql2hwyr2qbeewkatccbemzolayrqoaiyyztzmpnw1iioqiaxwmknoeti7ga7ugbbtga +1 +se7enorg +1 +hrefhttpclickthruonlinecomclickq0eeotiiqzharlshv4r8djx98lbk4r +1 +4242010 +1 +dealsa +1 +holocaust +1 +emacand +1 +7646 +1 +usa +1 +accelerator +1 +xbarracudafingerprintfound +1 +3lxox7qdget +1 +mckelvey +1 +robinrkrahlde +1 +suggests +1 +hei +1 +101421203 +1 +134625 +1 +configuration +1 +hrefmailtobestoffersdishnetdslnetsubjectremovebestoffersdishnetdslneta +1 +publication +1 +touristoriented +1 +buttons +1 +swe +1 +fontfamilytahoma +1 +lcd +1 +127469173478f46c370001w4twrl +1 +sslnbspshopping +1 +nonemaking +1 +gletterpost +1 +candidate +1 +4946oo139026e76697508048btshbpfnwltxrgj327921145049 +1 +conveniently +1 +808080br +1 +7twy +1 +111601 +1 +sketch +1 +launch +1 +leramilerctrorg +1 +mtn +1 +listmanredhatcom +1 +023241 +1 +multisearch +1 +size3d224223e3cb3eor +1 +lshaped +1 +471 +1 +v55045221200 +1 +iahuqexite9829chellonl +1 +0600 +1 +leaving +1 +href3dhttp2cb3dpimihzvcnjvo3d6f46e8ca584f42a01557 +1 +farbmagazin +1 +gettext +1 +driving +1 +trimg +1 +esc110329738836411025989487446468inrovingcom +1 +649118141 +1 +srchttpcnetcombgif +1 +httpsciscocomauaucomciscologin11516use100htmlfont20 +1 +o0qf1zho018748 +1 +083834 +1 +height284a +1 +face3d +1 +xmbmessagetype +1 +hopes +1 +symbolism +1 +guinea +1 +hrefhttpd3979f118a6fzyhuhscn +1 +roxus +1 +mrhealthbtamailnetcn +1 +fictional +1 +e20cs37224wfb +1 +accented +1 +spamassassindeveladminexamplesourceforgenet +1 +src3dhttpgservzdnetcomclearoutboundgifappid3d2emid3d25136482 +1 +qmailldap103 +1 +videobased +1 +uncensored +1 +163008 +1 +manufacture +1 +lwv26awu +1 +05042010 +1 +68144 +1 +aolcom +1 +5star +1 +706px +1 +filenameecp10ccgif +1 +065559 +1 +hidden3dtrue20 +1 +paperweightdarwinnasagov +1 +lower +1 +001414 +1 +bordercolor000080 +1 +00020 +1 +012023 +1 +silent +1 +pratique +1 +6318616221 +1 +y3tfxrvaiqiiiyhze3aampz497tal4bwckpw8daopxe074baokooosgshfe81tn3t +1 +rlcglj6m4qii +1 +o11cjord022711 +1 +nbspnbsp +1 +size3d2statefontbtd +1 +cu88l76nlcarhvwe6lbliszt +1 +everything +1 +greatest +1 +233423 +1 +facearialnbspnbspnbspnbsp +1 +tokenizertokenizertokenizeheaders +1 +028 +1 +declarations +1 +ssl +1 +bpbsaslquonix +1 +naturalization +1 +o1gdkwiu010677 +1 +chriss +1 +width29 +1 +pharmacyfontfont +1 +hrefhttponlinedesignradixwebnetdapper91html +1 +tkukwdcwbvb4faqbismlprfejkmw8snnlyfmrbsehojkxcqv0cim2oeaa8wnrmuccbf17vougeuz +1 +sa031 +1 +textdecorationnoneupdate +1 +pplayerabilityp +1 +approve +1 +score41 +1 +donate +1 +ticket +1 +1o32xq0003cgnu +1 +rhdjprwg2hbydhttaridywcvofarumakak4ppt0jbwsktjdkuarg2hhzz9quqn2jyppvpifspla +1 +7797g +1 +3ware +1 +consisting +1 +homepage +1 +uttered +1 +bootbr +1 +meantime +1 +istanbul +1 +5ebn6onq8vp09fb3pn69oadambaairaxeapwdcooopgjs9s0ds00nuaymxsaqtio2eomkp +1 +bring +1 +downbfontbfontli +1 +albertwhiteirelandsuncom +1 +envisager +1 +virtusertable +1 +validate +1 +height552 +1 +raa09761 +1 +3f4da348c53 +1 +10997p +1 +8739yjfge7fsfmerciadrilucastationmerciadriluca +1 +anwarulhaq +1 +bmake +1 +httplistsdebianorg20100414102012gf6054wastelandhomelinuxnet +1 +ghai +1 +howls +1 +yunnan +1 +pause +1 +fori +1 +psuedocode +1 +anthony +1 +124126 +1 +5aemjcuo4ycinbto21legc5oblqed61tuzs3ef5ruzmkjikbb4pfvcvhv8alk5pvykuxconqrll +1 +namegenerator +1 +asap +1 +hrefhttpfplumprockruea53e26f5bac7630d7042ce7825f04ef77 +1 +tiwanaku +1 +infringements +1 +mailtosecproghelpsecurityfocuscom +1 +preferences +1 +reimburse +1 +topic +1 +content +1 +policyfontafonttd +1 +barred +1 +hrefhttpa62hardsupplyruriseyhyo8efdae71f7eee +1 +rue +1 +destac +1 +copied +1 +feltokru +1 +glassner +1 +2006 +1 +winds +1 +220430 +1 +friedman +1 +096 +1 +225ptpadding0inmsopaddingalt +1 +testsforgedrcvdtrailinreptoknownmailinglist +1 +012a12a22e6b4242c4a36cb82ad2nsnnbl +1 +theh2 +1 +taxing +1 +nbspfontp +1 +102236577 +1 +31106356 +1 +89551f9 +1 +stylefontsize13px +1 +knptpv5fmyyk01jewwz61sf0nbvqa3brkeczfuxo2q8qgzm2qip6dbqj19oxqszzpwkmr +1 +somebody +1 +height3d39font +1 +actodc +1 +mgbyosemitenet +1 +181453 +1 +carolina +1 +explanation +1 +195343 +1 +httpwwwgeocrawlercomredirsfphp3listspamassassintalk +1 +20100507070848362mspdebianorg +1 +60foot +1 +m4l8s +1 +rankings +1 +blackberryaebr +1 +targetmail +1 +eh75bw9ategtt2k9815zcrsqkugtnn6v0gbums5k0urgizzfn170jm7hrtae1bobmmfvquwhd +1 +lived +1 +srchttpwwwebonylust4freecommailerimagesebonylust01jpg +1 +maddest +1 +101735 +1 +trades +1 +usrlibx11xorgconfd10synapticsconf +1 +human +1 +senate +1 +experimenting +1 +situations +1 +textdecorationnonemobilea +1 +utilities +1 +httpninonanospeedblogspotcom +1 +formation +1 +bhorzi2mwoxznc8me1exltlnnoy6cnddgwbyhyty5s +1 +boundary22773175alt +1 +12hours +1 +360 +1 +zoo +1 +w6ua7cov2qraa80u4juaztc2ryyxyhvjh61e42tzw2hgppjq5bqnirj73bnllblxbqpcxslxq7 +1 +033014 +1 +href3dhttpclickthruonlin +1 +remains +1 +043902pm +1 +elder +1 +equivalent +1 +tenbagger +1 +content02 +1 +extraastrongfonttd +1 +problembr +1 +gecko20090706 +1 +025223 +1 +theyll +1 +exmh2113882517p +1 +jmcarpeprodigynet +1 +loved +1 +nonepayment +1 +l6ffpaiyvyrm3jmq205hkscasrbhsrca4k989tvaupyg0toqwbivzop2zinhlejydnkv2ajni +1 +130238 +1 +predict +1 +1931201712 +1 +dreamweaver +1 +1904 +1 +pstrongto +1 +felindapop2athenetnet +1 +hrefhttpwwwwebcredit2002comcid2020ampmid3300img +1 +msdouglasnetokcom +1 +jayerkvtwcpochdfluxpdpxdcgqnld5kjtn1dpaynlpjfe5vtgoqfmjcola8sqyc6tprx0j03x +1 +trump +1 +null +1 +sociallinuxie +1 +color666666span +1 +pinebso444020822152438028231100000crankslacknet +1 +href3dhttp24e67ydeceyercniuq +1 +height414 +1 +20020824 +1 +intercourse +1 +yw1pbmf0aw9udqoncktvbglvcg91bg9zdqoncnjhzglvz3jhcggncg0kmthgluzeryyjodixmq0k +1 +preg117ntar +1 +httpradiontwizardsnetcategoriespda +1 +brthe +1 +httpwwwntknet20020927jesuslargejpg +1 +valueflorida +1 +refuses +1 +uvowegisud +1 +clncgggzeasfjkdvgvakp1jbzzl8kqc9zvckocpjbydwspyujigw0r0j6dp50kvgvv0omuoydg7 +1 +ediyoruz2e2e3c2ffont3e3c2ftd3e +1 +fair +1 +mccarthy +1 +simplify +1 +rodbegbie +1 +200208071828045d5e4dcbkilroykamakiriadcom +1 +titlefirststop +1 +26302686 +1 +idaho +1 +39b3016f22 +1 +uals +1 +illustrated +1 +stone +1 +heart +1 +daves +1 +ebusiness +1 +windowmaker +1 +enhancing +1 +broadcasts +1 +p5qmvtoxhkuhjsbigvdffkyfqjppsao0wiam +1 +fw +1 +comicscomcomicsdilbertdailydilbertimagesdailydilbertheadermidbot +1 +est +1 +200341 +1 +height1brtd +1 +hrefhttpec4d1diwohevcn53j515k08687iw1sejfp90r5ywoazihanhibodyhyxoyso +1 +64105237170 +1 +155430 +1 +gaining +1 +targetblankwine +1 +063235 +1 +colorff6600gtfontbfonttd +1 +n5hmps1s016489 +1 +distribute +1 +friendfontab +1 +centerfont +1 +ga +1 +size3d2thank +1 +party +1 +ore +1 +cfcfcfpadding20px +1 +37077 +1 +1264068587 +1 +federate +1 +principles +1 +miniplantas +1 +213922 +1 +m2rjk0iue6ejttgq2ggawh0c7hjte7t0yrce6jusatgce5ggcdpt3pongsrbuerhgtgkesgawxa +1 +fontabba +1 +advancefee2 +1 +now2cnbsp3b +1 +patelpp +1 +shootingb +1 +perform +1 +iqecbaebcaagbqjl7qcaaaojehnws3jeoi3pv5shwfc9d44t7bpqvde7wzivacm +1 +judgmentsbr +1 +021946 +1 +8369116f6d +1 +planbfont +1 +manufacturing +1 +twells +1 +alt3dimage +1 +190009 +1 +damnificados +1 +1980816894 +1 +abolitionism +1 +manipulating +1 +subsidiary +1 +morning +1 +longitudinal +1 +srchttpecuyuxbilozcnspacergif +1 +regnetdichotomizedbolsasdicamentsrepairedefficacerellenadoremarkableremixespiosresiduesvorschriftenrendererrendirrenzyvriendschapreprepitiendoweigertreprocessedgracchusresolveudirectrixresentmentsanrioreservadadirigidadisallowsubroutinesdiscardingrestrictiverevenusretrasowacooldiscussrfcdjustwalgreenrindisheveler +1 +markers +1 +cdptpaomtalbmailrrcom +1 +accelerating +1 +jdk14x +1 +quotedemailtextrcvdinmultihopdsbl +1 +testsawlinreptoknownmailinglistpgpsignaturereferences +1 +clickherelink +1 +ayblock +1 +modulesp +1 +altnewsletter +1 +mulberry220b1 +1 +pgh0bwwdqo8ym9ket4ncjxmb250ignvbg9ypsjmzmzmzmyipnnretwvzm9u +1 +eights +1 +postcard +1 +l3nwyw4t1g2aaraqekqvrlepncm86zirmg7tmnqpc669lj0wcikar6lkvppc9mb250pjxmb250 +1 +border3d10 +1 +mailtoobjclanguagerequestlistsapplecomsubjectunsubscribe +1 +offscreen +1 +200434 +1 +bolympus +1 +valuesasaudi +1 +171303 +1 +ldosubscriberldowhitelistrere +1 +helonausicaa +1 +httpfreshmeatnetprojectscfg +1 +hrefhttpcbbdrugsiain81aruubaqytoh730b8a03d7a2c9 +1 +colors +1 +200449 +1 +lions +1 +boundaryexmh1581861840p +1 +124122183196 +1 +beat +1 +rackspace +1 +8px +1 +ds5izsbszwz0ihrozsbzdw0gb2ygjdgwig1pbgxpb24gihvzigrvbgxhcnmgaw4gy2fzacwg +1 +predatorfree +1 +reminds +1 +cpunkseinsteinsszcom +1 +psrykkemmnd46e2xrw2yg2o11bpztww5caekw2gzhovvz6svmp3sqngzhez0krs8h +1 +w3c +1 +cellsp +1 +cellpadding3d8trtdfont +1 +safetyfont +1 +grasp +1 +herebfontap +1 +qu111t115pecul97rlyqu111t +1 +87d3xrd8hufsfsmailinffhbrsde +1 +charsetiso885915 +1 +c0wrg06dkmudwwp0qtisrsag0lsrnmbtokuxpkwx61tabymwywq4fxlntb7eqpixvkh1u1yiica +1 +mitten +1 +slash +1 +homeowner +1 +031128 +1 +i102 +1 +000060c62ff6000068bc00001cd1alabamacojp +1 +xenvelopeto +1 +border0tdtr +1 +090041 +1 +middlefont +1 +bay +1 +favour +1 +pornstrongfontabrtable +1 +20020920104055201c2986matthiasrpmforgenet +1 +pharmaceutical +1 +jvmplatformversion +1 +httplistsfreshrpmsnetmailmanlistinforpmlist +1 +hrefhttpclickthruonlinecomclickq75whwtiku3zcagrokrjups7llz5zr +1 +size2ifont +1 +hrefhttp2bmedeven33bruedinaec9d601f22763c7 +1 +cupa1b0ee0oa04b8yrwlbliszt +1 +m8cs67090wfj +1 +amacatergalacticdemoncouk +1 +etcautomaster +1 +100rag111ns +1 +mileage +1 +94 +1 +classstyle14font +1 +multipartmixedboundary +1 +reliance +1 +c9825d053119 +1 +families +1 +probability +1 +cuirassiers +1 +listening +1 +experts +1 +friendatdtrtablebody +1 +e11652d0db9 +1 +facearialbia +1 +cvs +1 +sophisticated +1 +hrefhttpclicktopspecialofferscomcgibintplid1075060763006 +1 +001 +1 +suit +1 +pushbr +1 +yours +1 +rons +1 +1a01d3ef12 +1 +value75000000750000option +1 +gallowglass +1 +width3d478 +1 +mailin13applecom +1 +thisisagreatfreepornmovieframesetupcom +1 +111136 +1 +httplistsdebianorg2010041516570240bbaa5babydosstargateorguk +1 +httpwwwoperacommail +1 +boys +1 +altclick +1 +moment +1 +centercnncenter +1 +barrmumford +1 +hypotheses +1 +081135 +1 +daa21056 +1 +tenth +1 +srchttpdou2findflorcomfg56foiluzenoqiuceobnawirzexklqsrrexqooeehnuooeczbelfoc +1 +20100417073348ga2861kelgar0x539de +1 +lxnlcmlmiibzaxplptigy29sb3i9izawmdawmd5bdxrvlwluc3rhbgxhdglvbia8l2zvbnq +1 +re97cti111n115 +1 +hence +1 +recreate +1 +64157321 +1 +g86k17q07388 +1 +name3dcontactname +1 +19415019111 +1 +phpmailersourceforgenet +1 +113115 +1 +value7070option +1 +trace +1 +aoki +1 +200209051326422938347386mailmanlairxentcom +1 +serial +1 +220901 +1 +factory +1 +height60 +1 +9281171174 +1 +correspondencebr +1 +entrenched +1 +doubtless +1 +altbranded +1 +xghtmtggc +1 +okar +1 +overpriced +1 +405388 +1 +bethlehem +1 +advantage +1 +ignoring +1 +repos +1 +javascript +1 +e20cs468587wfb +1 +64422195 +1 +094353 +1 +184323 +1 +nicole +1 +expresscnets +1 +102993570829981camelshiva +1 +color3dffffffagefonttd +1 +krunner +1 +todayaspanfontp +1 +233603 +1 +162849 +1 +submitted +1 +waldner +1 +proc +1 +billion +1 +c9b11d289c04ce70215ea0435da +1 +pmatilai +1 +1311805735 +1 +aanlktiluy9zeq3g6jozedqwcynpau7hxoewytbt1kchmailcsminingorg +1 +11fe0d07b7cf7ae000003b237a4beab92b4b78 +1 +decide +1 +commoditizing +1 +jcjreemsncom +1 +1052 +1 +jordan +1 +618011096fontp +1 +child +1 +trtr +1 +hrefhttptsproductinfocomcasp679736c65a20642aaeb0a112 +1 +httpwwwtheregistercoukcontent426135html +1 +href3dhttpclickthruonlinecomclickq3d703vnjidp +1 +81388138 +1 +versionabr +1 +att +1 +afrikaner +1 +fastest +1 +abbe +1 +quizzes +1 +xfuhafi +1 +0h3700ivfjmhs3mta7pltn13pbinet +1 +robert +1 +altabstract +1 +124744 +1 +advise +1 +105033 +1 +g6n1qwqv025043 +1 +contenttypecontenttransferencoding +1 +grab +1 +fontsize18px +1 +4bbae46a6060104wanadoofr +1 +affect +1 +demon +1 +solutionsthezscom +1 +b3vnacbvcmrlcnmgdg8gyxqgbgvhc3qgz2v0ig15ig1vbmv5igjhy2snjy4n +1 +mdodating201 +1 +untilb +1 +californias +1 +habit +1 +altshop +1 +neprestano +1 +value7979option +1 +harisena +1 +nested +1 +httphnvzatsehsyrcom +1 +utility +1 +testscapinitldosubscriber +1 +typically +1 +filetimeb59cf9f001c260c6 +1 +placed +1 +goiubntfkszcurfbnwn9qz01xmfft2kidgplm +1 +width3d30 +1 +anger +1 +gibsona +1 +stays +1 +dapplications +1 +structures +1 +exmhworkersadminspamassassintaintorg +1 +undefinedundefined +1 +114629 +1 +rudinkin +1 +displaced +1 +diagrams +1 +64b +1 +iso88591qitzpxyncnmw1x62g9y2gufkejxo0a09niylkgpxnrhhecjmyuaujmnd +1 +helary +1 +221728 +1 +e1955f2acc3814ceepimac6d4c3wwwfacebookcom +1 +subdivided +1 +nbspbr +1 +rolling +1 +providers +1 +10218114 +1 +categorized +1 +garticlefullstoryhover +1 +224552 +1 +soft +1 +width3d535 +1 +centerfo +1 +lexicon +1 +1993 +1 +bts +1 +debianlenny +1 +correct +1 +attachment +1 +class3dtextbthere +1 +ventilation +1 +ww3orgtrxhtml1dtdxhtml1transitionaldtd +1 +113800pm +1 +itspantd +1 +sa331 +1 +19fca41723 +1 +o4pbpe6s012000 +1 +resolvedipisnothelo05 +1 +xev +1 +refinancing +1 +mikes +1 +120837 +1 +w12si12208346fah8720100517015948 +1 +202200 +1 +xxxxxxxxjp +1 +cartier +1 +tipspointersetc +1 +smokestacks +1 +jimdouglasmaccom +1 +inf111rmiert +1 +fcrfcnler2c +1 +hrefhttpwwwfirststopwebsearchcomlockergnomeofferhtmlfirststop +1 +4bcdd3f0a +1 +005146 +1 +oxidational +1 +sadev +1 +zulu +1 +0935270005 +1 +nusrat +1 +scappaticci +1 +serverside +1 +bi111graphy +1 +loading +1 +149nbsp1998 +1 +attacker +1 +technique +1 +enjoy +1 +ring +1 +utrecht +1 +sack +1 +huckleberry +1 +172901 +1 +055927 +1 +dd45613a54bf +1 +v117lnerable +1 +calculating +1 +917c7169ff +1 +165314 +1 +375 +1 +pmwuqffx60lkn31tahvyjba57u5voettahwppshzaebwfmtddjog4pq1uyqdmcabv05czbxue6 +1 +sansserifb1995br +1 +turns +1 +185744 +1 +1910 +1 +boundarynextpart0000023d201c9e57a88ad7350 +1 +194108 +1 +skaterswyahoocom +1 +vocalese +1 +hits2877 +1 +9fde3fefef5e1e2ede99eada0b7aaabeae2edebb9a8a9a2af +1 +hrefhttp81154geniusliftru +1 +netnuevonetnuevocom +1 +freshrpmsnet +1 +selfmotivated +1 +sa424 +1 +blacknbspnbsp +1 +estimates +1 +189015170212xddynamicctbcnetsupercombr +1 +chose +1 +c6ac866d5f5 +1 +bphfcskucivcaxpav8obvwo75rvkd6yvgwomafa2srb97hee33qicgmwpzuyl4uhp +1 +1124 +1 +httpgoldrockerscom +1 +winning +1 +133935 +1 +theology +1 +classaltnormalbr +1 +separately +1 +532ed16a39 +1 +ours +1 +u7mr11500141wee511272631068543 +1 +highwater +1 +euoaiwu3ruoyazlgwuuhoaz5ouniaoqfir1nxs7keohamyxq43ocn6ukstaxtokednplh4qcqwh +1 +displaying +1 +065214 +1 +pinp +1 +nyah +1 +computers +1 +tri +1 +style3dfontsize11px +1 +132118 +1 +hrefhttplockergnomepricegrabbercomsearchgetprodphpisbn0966103254get +1 +issuesanbspnbspnbspnbspa +1 +tastes +1 +fnm4ala6acj4uul1g6k7n7vzmhrgk0iyhm5ovyfmxr4ja3egcqzwkpkfoqzcmwswyyzqmw9gxq5q +1 +cooker +1 +buddhism +1 +obliges +1 +1471454022 +1 +20100506091306917dididebiancknoworg +1 +18si7794034wfa1220100516172542 +1 +employeescelebrations +1 +xtlssubjectcgb25417s9 +1 +vent +1 +obvious +1 +meanwhile +1 +10pxprincefonta +1 +20080517 +1 +bordercolor3dcc3333 +1 +hair +1 +cardinal +1 +ordinators +1 +gb23ilrr471946 +1 +193457 +1 +hrefhttpbc3f6c59brownchooseruabout +1 +insurancetitle +1 +width653abr +1 +crfontfontfontcentertd +1 +gener97ted +1 +sincerest +1 +foundation +1 +34df813a55d3 +1 +socialadminlinuxie +1 +8cida7kgiivng2diwyzapfkmm44tiqnzngfy9aso0cswusg0mvhzrjusutwws4lpfqjvpiakgu5 +1 +egtan +1 +monique +1 +ohio +1 +langenusopopspandiv +1 +103543 +1 +999 +1 +paddingbottom +1 +xhabeasswe6 +1 +lemony +1 +href3dhttpgroupsmsncomoneincomelivingone +1 +87zlo9y88ahtyaaaaaaaaaaaj9lzix2irmj0pb9q2yz7zh5fjcnaak28srp0sw7kz9evsbiyu +1 +deloptes +1 +stockbr +1 +28639 +1 +unplug +1 +617399 +1 +benzodiazepine +1 +titlelibrary +1 +m8cs51580wfo +1 +170136 +1 +webmastereditincom +1 +replete +1 +172334 +1 +spamassassinpmv +1 +desktopxab +1 +qualifybrbr +1 +dqp2b2x1bwv0cmljdqoncmrpc2nvdmvyzwqncg0kcgfydgljaxbhbnrzdqoncm5vbmnhbgnpzmll +1 +bair +1 +harley +1 +size3d2phonefontbtd +1 +14ytxrwst2agavnupbxjlogntdmwati4kboamkjdpstumvvq9tqlak24zetbmlk6jtxj3ob +1 +army +1 +2163418188 +1 +17pxstrongbr +1 +1087687 +1 +0xdeadbeefrequestpettingzoonet +1 +abiwurcnqofeb3dffe0c9534c34a11fac231627 +1 +plugins +1 +colspan3d2font +1 +wylieblog +1 +dartmouth +1 +unzaecsmjfv646svxeqthz2mesv9toyqjkvzjnx97aqplatpdeivzehepnpvnirtxvtfj82byx +1 +ebooksabr +1 +target3dblankhttpsdeveloperskypecomjirabrowsescl510a +1 +hrefhttpwwwglobalcybercollectivecomwsbfreetrialofferhtmclick +1 +011858 +1 +varsity +1 +src3dhttpwwwzdnetcomgraphicsanchordeskfrontpagebgif +1 +lurking +1 +d116716018 +1 +marketers +1 +holding +1 +planet +1 +93fileid +1 +aggregator +1 +nts7antzgbftq9b8qrp86rbasxcrlnvwlpccwikzs1satgasigb2aksigb1fjrqatfffac0ulf +1 +920c815e5f +1 +g89jafm01952 +1 +swsyxkrrjvmo9wd3r9escl3x6mio3qyx1jpfdx6mldlmmumuhkhlzloecy56ubwhgjw7fj8yrme +1 +everythingbr +1 +cxvaeu7fis1yaxonhtkjw6y5cvijka2qvw5vteu3x2podknqcqwbtzmiihdnjdiop6p8aw4tp +1 +logging +1 +ebook +1 +052632 +1 +261 +1 +07aa329413e +1 +111214 +1 +simplified +1 +colonizationem +1 +141045 +1 +1265981829 +1 +duman2kvrkwjgsiimqzswe5qnwptuvv +1 +gmailid1285056bffc4e4c5 +1 +xaa29507 +1 +escribif3 +1 +self +1 +httpwwwsammydmancom +1 +7ex95b1ifjda0pe9rwylbliszt +1 +5925 +1 +ratifying +1 +conflation +1 +kellyclowerscsminingorg +1 +063229 +1 +romanband20 +1 +143825 +1 +hassles +1 +501px +1 +11323631212 +1 +usrsbinntpdate +1 +srchttpwwwcnetcominlinvhdgif +1 +httpwwwnewsisfreecomclick183997281440 +1 +11400 +1 +participatte +1 +g7mjsyz21645 +1 +093 +1 +bottle +1 +partit111 +1 +sj1slb03gw2 +1 +12422172182by +1 +althot +1 +4em +1 +agugt2zmawnlig9miefzc29jawf0zq0krglyzwn0b3igzm9yie1hcmtldglu +1 +d4si1809556fac9420100521074142 +1 +1022350193 +1 +19si5248675fkr920100416193600 +1 +m56wwhdwchq4kjrct6jjdewmvntpfjvvfaa6p1rm7y8g7mfvx7f5p6zw4ob6gilhnfeq +1 +smtpmailquicktimeusersbouncesmlsubscribertechcsminingorglistsapplecom +1 +romanstrongv +1 +127227827278f7e0870001w4twrl +1 +demostrate +1 +topoftheline +1 +pantry +1 +humanargonhu +1 +smtpmailquicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +1 +pweve +1 +10pxfewfonta +1 +pcmsurround71 +1 +distribution +1 +unparseable +1 +hdakecmtpl8rj7n45mxmu8qyzirsiabhro4dm +1 +20pxfontsize13px +1 +1021349204 +1 +organizations +1 +1804060375 +1 +spearhead +1 +score11 +1 +wenbspwill +1 +151703 +1 +valuebebelgiumoption +1 +15pxpadding22px +1 +glassmolding +1 +resubmit +1 +dumb +1 +installationfontp +1 +utnhsljx3rvs0cwvekpnropbppmcq2etzylzlwgzgsyjyvplbhvk2pbfz0ekcnbzurxj56ynvr +1 +isa +1 +typetext +1 +7412582175 +1 +reservedbra +1 +jalotat +1 +bv8adfbkd9z8kyzu38mdj05obktieln8wf3xaogwakqyd68a3w3fadx0ohquwsx +1 +174727 +1 +162942 +1 +king100111ms +1 +valuecyprus +1 +renderings +1 +282165498 +1 +nak54cfi001394 +1 +042142 +1 +r111ugh +1 +debtors +1 +imprononcable1imprononcable2ldosubscriberldowhitelist +1 +width3d550 +1 +fobif +1 +purcheses +1 +tablepbr +1 +rightmargin10 +1 +signed +1 +afcc513a4ef4 +1 +kklilbwkgqf8xajdxcnf8jhq3p3wcq1woopjepilvt1uvia4uf8jdqvp15dxaaoo +1 +stuck +1 +partialrfc2369 +1 +nature +1 +uncomment +1 +httpcaffeinediaryblogspotcom +1 +cpunkshqpronsnet +1 +tnonsensefrom4050useragentuseragentmutt +1 +owbxsjhx0cpz3xyv41tmm4rzifc8aaqudvdxvfj3u2zlfdtzbdkpxjtdem47osxlbj68mluva +1 +srchttpwwwtheserversidecomimagesj2eelogobluegif +1 +cibwzw5pcybpbibhihrvdgfsbhkgbmf0dxjhbcb3yxkuidxwpldlihjlywxp +1 +flowu +1 +guide +1 +20100415153853f46e5f69celejarcsminingorg +1 +bas +1 +bfonttd +1 +mysteri111117s +1 +optout +1 +facing +1 +reparable +1 +fashioned +1 +pond +1 +smtpmailpaulthrivementoringcom +1 +subhellouseragentuseragentkmail +1 +qdzxwcfex81waapnfaj95fzxabr0crk9fqecwdz7ymvnhnp8a88v5rjqfyeiopofn3 +1 +ldowhitelistratwaregeckobuild +1 +4302010 +1 +openprintingorg +1 +n111t +1 +partern +1 +151916 +1 +wasb +1 +eshop +1 +scriptures +1 +97re +1 +exile +1 +circulation +1 +v511 +1 +revision +1 +lifepreserving +1 +height105 +1 +internationally +1 +razorusersadminlistssourceforgenet +1 +olpcpowerd +1 +1265981829761db09a0001w4twrl +1 +xxsmall +1 +definitionsmain1005180171 +1 +stylecolor747474nbspcopy +1 +brnbspnbspnbsp +1 +score104 +1 +td6ru1bzk1ae +1 +berra +1 +melrto6wanadoofr +1 +n51ldpv5007728 +1 +hr +1 +eppsnetnomadcom +1 +200908301535n7ufziua001946ns1csminingorg +1 +eu +1 +gya1 +1 +fitnessatd +1 +definitional +1 +treyrefillsru +1 +ensembles +1 +valuemt +1 +arbitrary +1 +ehjmcg6aklicgiiuim5wonqirr7eztdofjri8udi8ljkzsb2wbdasikjdaqmf40nf7ghhce +1 +worthwhile +1 +urlhttpwwwfeinsteinsnetharlan +1 +aviation +1 +prague +1 +churchespsychoanalysis +1 +d0b4d7d6c2a5a1a2bed3cad2bacdb9a4b3ccd5d5c3f7a1a3p +1 +120924 +1 +yyhstsb0j1gyzu7cjty8544zeelplnaaukfpxjptvdcx2bax7zn5ubwv7jihd7bd83scvuu4g +1 +0xb7eb7ec8avparameters +1 +clause +1 +apueid +1 +hits6745 +1 +c1c3a440cc +1 +hrefhttpopertunitiestoday2comindexphpimg +1 +landshark +1 +fileshare +1 +contactthe +1 +score1893 +1 +harding +1 +bascialy +1 +distributed +1 +93139103196 +1 +successfully20 +1 +67c5716f03 +1 +mechanics +1 +si +1 +mug +1 +lgpkqmttguyzvtx9c1ftivcls8lghejep8nzq3eep51csfcviqwarlbawcdnaej5wd0e16f +1 +524px +1 +fontsize12px +1 +ctrentcom +1 +fontweightbolda +1 +shower +1 +msgiddollars +1 +sourceforge +1 +brenough +1 +z2saae88f271004301657m1cbc6552nf528f7446c7572ebmailcsminingorg +1 +alignrightfreedom +1 +haze +1 +auctionfontbfont +1 +eridanus +1 +wrongdoers +1 +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c07jpg +1 +200203290529g2t5tsd11847host11websitesourcecom +1 +b248l3rkpg0kicagicagicagicagphrkihdpzhropsi0miins4yntalpc90 +1 +gmailid128939d4a3e158e7 +1 +httpwebpagesmarlboroedudebockweblogindexhtml +1 +bayes002 +1 +hgqqyulyeclisn2y4eqfeyeexa3ok3qothfm8asjhgwo3njjcyiicnaecmbhjaihcsoo9ldjjo6u +1 +r10cs25765wfa +1 +incompatible +1 +clientip172541337 +1 +implementation +1 +bliss +1 +nonmarxist +1 +captain +1 +cypherpunksdspronsnet +1 +z14mr636586fag971274398034507 +1 +wukanj8t2lp8iafk2dqraqjsf8o9dz9u +1 +chandler +1 +httpwwwfsfginccom +1 +trampoline +1 +erlaubt +1 +colspan3d22222 +1 +lockergnomecom +1 +55 +1 +spandivtdtrtbodytabletdtrtbodytabletable +1 +x1b427zpeiyo2dyrng6zwndx9wq9wzk4url3slx2224vuydrn1mfhenuxln6hs5b9onl +1 +cousin +1 +mailtoquicktimeapirequestlistsapplecomsubjecthelp +1 +lauraswansondogmaslashnullorg +1 +hdomainkeysignaturemimeversionreceiveddatereceivedmessageid +1 +theresas +1 +threaten +1 +vanessasoarecsminingorg +1 +xoriginalarrivaltime +1 +despair +1 +caspian +1 +color000000tech +1 +143933 +1 +anyonecanedit +1 +uninhabited +1 +durnik +1 +fffhomea +1 +echoed +1 +ricochetusrlocaletcricochet +1 +eonauthrelay2 +1 +opp +1 +182410 +1 +element +1 +rev +1 +alignleftchange +1 +bouncedebianlaptopmlsubscribertechcsminingorglistsdebianorg +1 +serv108segiulgacbe +1 +112th +1 +wwwdoteuokodotcombr +1 +classasotvlinkimg +1 +gaming +1 +9ydntazxsknlelol +1 +kqpxkedqhoyfgcm6ghxcajvjseucfsda8hai5ifmtm6psipsmkagjaguzvpcmoxhpvstrgly0ta +1 +hrefhttpwwwbufunoceruulutoa69602a8b9c +1 +luma56csminingorg +1 +1022331136 +1 +3dcenterfont +1 +usd176m +1 +height15td +1 +exposes +1 +proud +1 +31108047 +1 +045557 +1 +55265313 +1 +rsa +1 +height3d435 +1 +smtpdsaslsecurityoptions +1 +standing +1 +nagve +1 +healthstrongfontbr +1 +practice +1 +request +1 +nowrapnbsptd +1 +feeding +1 +hrefhttpactionidahjorlsalsatrackjspv2ctnm1sflqjjv2b4t2fdzkhhsdt2f2yu05hzqchange +1 +vba26jejbtngef7xfjcwkuwyi8pfseych6hr3z0g0uoveeikrjdoo3ucurs8jg0suu6r5mk5xk +1 +g8ufio223184 +1 +mta21bigpongcom +1 +binder +1 +problemsaccessed +1 +emledem +1 +size2save +1 +dynnjablerr0 +1 +infjpbqfhyap +1 +h2intramural +1 +216136204125 +1 +g7h70q622253 +1 +41140153155 +1 +xeec25lndts +1 +zbvkacssdgj5mkhcwsy17atgl1jyiaoyuewaj8vvfhnkypfvf7todr6d7fen7gtphnyt4lz +1 +httpwwwbarracudacomreputationip66118651 +1 +4e0705992da50cfb4 +1 +computerfont +1 +2adrhss8cww8rvqkil6h63wmgzf7ha6zr8ufrnboupmaczl8atusjmbkl0hbtwmpwxgo +1 +educations +1 +remaining +1 +width3d74 +1 +hrefhttpwwworaclecomgosrc1393043ampact9check +1 +486432 +1 +brlost +1 +alb +1 +nonwicked +1 +literacystrong +1 +name3dphone +1 +124810 +1 +3928288 +1 +postscript +1 +g6pefgi5066072 +1 +3fb0e0profile +1 +104421 +1 +coreabr +1 +width3d538 +1 +bsfsc0tg035 +1 +deadly +1 +traded +1 +unifying +1 +5712a16f8b +1 +035e67c20d4a4732e4a02ca23db0txpuyp +1 +width115nbsptd +1 +testsfourlahtmlmessage +1 +frontpage +1 +ca131503e1 +1 +1195 +1 +bgjk1cb0wi5xrq9qav4wngqzcy0vasajbluwxlqabdnwvo2pbkdlabwqogtsxs5fazggcbvaasmq +1 +20d5e13a4ff5 +1 +bibliography +1 +bankruptcy +1 +a4fd116f70 +1 +charity +1 +zcvsnddetj3hwqj24t3oupcadnu7fnubpj0vljqzl4g7nuhqxoyvpekdyybjiub +1 +samu60csminingorg +1 +555555lineheight +1 +gmailid1280ab09584f6020 +1 +fontfamilyarial +1 +02eb78a7e3f +1 +nlix7mqwl1r2lmfopudnzkgufs37nxsf9lotzrer6zkxsqbvdukvbqxndmsb3pqoa6gja +1 +planovima +1 +182404 +1 +oztnotoifalfiheqikaeqbiakzimta12automated00onlinecom +1 +eyeballing +1 +width548 +1 +sec +1 +archivelatest574275 +1 +western +1 +2e +1 +okidata +1 +e4aizrshv0iajegeaoacx8ybowqla2r7dkiddnhvmyczvqikgxiafbaen3adskicqcacpjhtjdgi +1 +ross +1 +speaks +1 +saa05985 +1 +finding +1 +onmouseover +1 +raised +1 +2131863320 +1 +binge +1 +asserted +1 +rollout +1 +width3d247 +1 +691ae2020 +1 +complained +1 +ibkbskwscdxxhgpgpb4fa0tyruuzffqfx9e2eljbxfivbzimgfme4bvhhwso4jegiy1w0yolpkn +1 +color808000inbspnbspnbspifontfont +1 +denali99aolcom +1 +silva +1 +61218821026951304845javamailrootabvsfo1acagent4 +1 +6445128110 +1 +parkway +1 +5d6f01ca57db22b9f3005288ba27aclightingcouk +1 +0m1180s +1 +srchttphomecnetcominedsitngif +1 +ribe +1 +yf265whg1aifsfprotonpathnamecom +1 +f117ente +1 +repaynbsp +1 +39efa13a46d6 +1 +435368043088 +1 +httpmaybeiotoeuropaorg +1 +debianuserspanishlistsdebianorg +1 +tower +1 +e67fe3ea26 +1 +t6bulletmudyahoocom +1 +contracts +1 +n7j7bcyb009184 +1 +newsletters +1 +xravantivirus +1 +width575 +1 +sandbanks +1 +violating +1 +height3d32920 +1 +nightmare +1 +roll +1 +19212399 +1 +impressive +1 +1969 +1 +aligncenterdouglashome +1 +migration +1 +measured +1 +thirty +1 +strip +1 +hrefhttpclickthruonlinecomclickq89h28bqj1qvniv5fkumbxb6w5wwcr +1 +httpwwwjibberjabbacom +1 +skynet +1 +3980f4416e +1 +aligncenterp +1 +89c9613a55cc +1 +ximn +1 +imbrwqkyopigawec0sszt6u32fh1niqwlzn2k +1 +resizes +1 +leaders +1 +hungarians +1 +width94 +1 +0aaf1440cc +1 +color404040 +1 +residence +1 +flpeie0trabxljq6spit5iphgib7pgqt2wcptgvhl2ajcspnalw5cdsrdvtty93t10n +1 +devoted +1 +m2efrjy5m2gatxciokvlbliszt +1 +sign20 +1 +textdecorationunderline +1 +jmhall +1 +absolutely +1 +httpwwwinsurancemailnet +1 +100919am +1 +initiates +1 +080055 +1 +color404040subscribefontafonttd +1 +shfileutils +1 +nashua +1 +telephone +1 +477594 +1 +yourselfa +1 +virus +1 +stroking +1 +replybot +1 +hanchemathntnuno +1 +sores +1 +1115682 +1 +202011 +1 +size1your +1 +icemannetnvnet +1 +leopard +1 +iwujbynyc4knng1drgg6prrcwesc1ltod1oizswe4mgbkahcqntng71rqscjfadidkyem6osphz +1 +p05111a5bb9ae46820c6e66149496 +1 +tb0gzvvivaw1rozqpsu9gc +1 +listsdebiansecuritylisztdebianorg +1 +src3dhttpimg245imageshackusimg24510487176jpg +1 +jenandtonic +1 +hu6b7u0ickg1 +1 +iervie5vdcbjagfuz2ugaxqgaw4gyw55ihdhes4gsxqgd29ya3mgzxhjzwvk +1 +width674 +1 +andcenter +1 +fond +1 +sworn +1 +g8igudc07204 +1 +hrefhttpwwwzdnetcomtechupdatefiltersmrc014175602044300html +1 +commandline +1 +d74e726e94477 +1 +companybr +1 +heaviest +1 +v60029002180 +1 +rv100 +1 +092605 +1 +route +1 +gcc44 +1 +msdtc +1 +car +1 +irc5v7m4rud5jdlvv7h012jw2dmfvmxrp23klmienynaq8yxksvqozidcs5fw51m1j3zwcx5 +1 +width5tdtd +1 +uid44501201487268 +1 +movement +1 +lands +1 +pamphlet +1 +digeststyle +1 +200209011956g81ju6te002619oriondwfcom +1 +186 +1 +215415 +1 +hero20 +1 +storagefonttdtr +1 +cccccctrtd +1 +httplistsdebianorg1710565045925434770heavisideinvalid +1 +digit +1 +19412517218 +1 +192959 +1 +gmailid127d42d7d9120bf0 +1 +jimnetnogginscom +1 +force +1 +name3dwebsite +1 +pnbsp +1 +hatred +1 +gonzaga +1 +555 +1 +rottnest +1 +shape +1 +065925 +1 +m8cs44860wfj +1 +skeptical +1 +incl117i100as +1 +colordfdfc1 +1 +harder +1 +g79il0b19672 +1 +hints +1 +bb30f13a4a5b +1 +aqebaqaaaaaaaaabaaidbaugbwgjcgsqaaeeaqmcbaifbwyibqmmmweaahedbcesmqvbuwet +1 +0725 +1 +152443 +1 +12811455196 +1 +spanbp +1 +testsawltnonsensefrom4050 +1 +srchttpgservcnetzdnetcomclearoutboundgifappid2emid25136487nle440issue20020710 +1 +streamlining +1 +826930110 +1 +ztyaoidgsvpxvd9pk3rv3k6ylm30a0e +1 +hiding +1 +class3dgmailquoteon +1 +indeed +1 +72b2313a537a +1 +amounts +1 +width3d40bfont +1 +21550 +1 +exim4 +1 +isn39t +1 +6621866216 +1 +zurich +1 +6c4d016f21 +1 +placep +1 +luciano +1 +petitbourgeois +1 +briefcase +1 +marque +1 +topmargin3d6 +1 +pudge +1 +boundarynextpart000000501c264127545c1d0 +1 +hrefhttpclickthruonlinecomclickqb1yjxjq0mipripyur9ifb0nlnlr +1 +ehances +1 +741258351 +1 +cssb +1 +marking +1 +width18nbsptd +1 +abnegatebr +1 +ciagicagicagicagicagicagidquli4uliakntasmdawicsnciagicagicag +1 +subrange +1 +width5td +1 +peaks +1 +aa7fncl8tommwdbt2whfq3oeooopm4ua4oakkan3hnfepbucmf9mkwakephfiremph6gjp8a +1 +cccc +1 +berlusconi +1 +httpwwwisilocominfobetaisiloppchtm +1 +7128494401050921271557302610javamailrootmd01wowsynacorcom +1 +aenrup4estpardfz5g9lbliszt +1 +ant3z3swpiwvlesdxqbtanfx8jjxhyxf9jlawswzqrfeaxzfytyfbp3rjjp3h3xrprmuaxpsmn +1 +141111 +1 +sports +1 +paired +1 +090803 +1 +httpquantumtunnelscom +1 +website +1 +citythe +1 +slightly +1 +120627 +1 +10877330 +1 +resortfontp +1 +advertisements +1 +width100font +1 +undisclosed +1 +wristwatches +1 +smaller +1 +ga9dbgsi3mfygvzzkneo8ppzjjgt5duge75hfpjvy18wkpungcslwglpcfl0wo9ueqqmqkf1 +1 +httplistsdebianorg4bc79b4f2080500hardwarefreakcom +1 +color0000cchow +1 +ehlo +1 +avatar +1 +ensuing +1 +storybafont +1 +hspace6 +1 +interestingspanbbspan +1 +1024x768 +1 +d0a5acd2c8bd +1 +httplistsdebianorg87fx1p7wwjfsfmerciadrilucastationmerciadriluca +1 +171941 +1 +subscriptionslockergnomecom +1 +010602 +1 +squirm +1 +defaming +1 +browsera +1 +httpadvogatoorgpersonraphdiaryhtmlstart252 +1 +hrefhttp7deskonibutcnazosifimci576h4o93gt36976o763698a +1 +060000 +1 +georgenetnitronet +1 +g8k3lhc19280 +1 +husbands +1 +diet +1 +ffffffreviewsabr +1 +hrefhttpb2136wekuvkdcnprivacy +1 +repaybrb +1 +standardized +1 +httpwwwcixcoukjimhweblogbloggerhtml +1 +moonwatcher +1 +oops2dat +1 +49 +1 +alliance +1 +scored +1 +height3d15f +1 +refute +1 +104931 +1 +stylemarginbottom16pxhello +1 +colorff0000namebr +1 +entertainment +1 +2829313a5536 +1 +httpwwwtimesonlinecoukprintfriendly014335108300html +1 +exit +1 +1014118827 +1 +virtualincludebannertxt +1 +c3mr11346361wfj171273035702604 +1 +subscriptionsstrong +1 +cheapcializ +1 +090023 +1 +hfromtosubjectdateuseragentreferencesinreplytomimeversion +1 +discounttime +1 +resultsbr +1 +5434655660web7602mailinyahoocom +1 +limbonull +1 +xrandr +1 +comedystrong +1 +99pool8557195dynamicorangees +1 +section1 +1 +baptist +1 +value3ddrogersinsbuyercom +1 +alleged +1 +202911 +1 +123856 +1 +technically +1 +1245743696 +1 +wishful +1 +name3dzip +1 +kilograms +1 +lifenbspbr +1 +sideeffect +1 +smd9ypi0yswud8txk2tqxpvazlcxtyv724oprtii5r1m91a2wlnclvug5rgutxulhilqlrnr +1 +prolific +1 +n39grpscdyahoocom +1 +detect +1 +26si9745315fxm7620100407005145 +1 +asuna +1 +consultation +1 +mimelite +1 +205234 +1 +optionsabr +1 +6621867196 +1 +municipal +1 +193403 +1 +silver +1 +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c09jpg +1 +hrefhttpwwwwowthisiscoolnet81removalbocaclick +1 +align3dmiddle +1 +established +1 +pollack +1 +awfully +1 +partners +1 +llyralocalhost +1 +value5151option +1 +otherland +1 +olive +1 +charlton +1 +121157 +1 +045935 +1 +hedotuot +1 +networkp +1 +14720000 +1 +gecko20020802 +1 +paclitaxel +1 +nononsense +1 +pgrkhkampq9a6qkqoycxdeuce55lf6jc2jharedseornvijdwrxss9pdoujebcua1bwrbkc8ss +1 +mediaabr +1 +softwareb +1 +scottish +1 +36864 +1 +171731 +1 +styletextaligncenterspan +1 +r10cs60290wfa +1 +bsfsc5mj1963 +1 +189735581mganm703dslbrasiltelecomnetbr +1 +httpwwwnewmarscomfirstwords +1 +confided +1 +touchscreens +1 +g7r4gez25436 +1 +places +1 +wsysi62820e90e66c599723319 +1 +18668654548 +1 +operates +1 +fictitious +1 +102298012 +1 +sse2 +1 +0909090909noscript +1 +akashanetpl +1 +talkboards +1 +architecture +1 +singer +1 +divmsonormal +1 +e17pfih0005yh01cpu59osdncom +1 +200208250144caa18619webnotenet +1 +require +1 +accusing +1 +passive +1 +television +1 +assistantbr +1 +mustknow +1 +sense +1 +color3dff0000freefontbfontp +1 +size3d2namefontbtd +1 +net +1 +srchttp33dbe1b0bzubohqpcn5c7ce73719ce261f5gif +1 +131245 +1 +fingerprint +1 +clickq3d42rf7cipkgjavfr3ppz3f74emirdr +1 +compaq +1 +asgard +1 +size1no +1 +nbspptd +1 +coast +1 +082901 +1 +customer +1 +polo +1 +minus +1 +3156 +1 +desert +1 +chetnik +1 +height3d219 +1 +shamed +1 +mithlondarda +1 +alignmiddlefont +1 +disease +1 +kernel42926070322 +1 +125115 +1 +width3d300trtd +1 +brwere +1 +rumor +1 +le115115 +1 +mbfontfont +1 +width563 +1 +hrnlocp881doughgmaneorg +1 +qibaryhe2476superkabelde +1 +series +1 +godly +1 +moscow +1 +00007f066f3a000058d000001428webnamecom +1 +wondered +1 +color3d0000cc +1 +informationb +1 +combo +1 +uprising +1 +incidents +1 +mimeversioncontenttypecontenttransferencoding +1 +72bca1c38a +1 +converted +1 +poker +1 +trimming +1 +virtuosostrigi +1 +nanticoke +1 +yaol +1 +44yesonlineisbestcom +1 +gonzowaiderie +1 +mesa +1 +jacobs +1 +20100414173940ga17726acampbellorguk +1 +uvk1fla210u5ca3krrnxgrornzsiblddnptjbojzuugn7ycfn60ltyzipbxvod0zlogdce +1 +bx2gnexpracrab8ss5alaawqeoptsicuwumfvtwbi9ke1suxp1cxex7k8pvvxnzfk +1 +g93bupk26666 +1 +new3e +1 +httpwwwbarracudacomreputationip1156921310 +1 +ops +1 +onesu +1 +ota1qlo52ffffabszfnj5pm80wekfpsgjpfielaoyhpeabxszyag7cfdscogoakkqi4rfu4dw7p +1 +hill +1 +185438 +1 +trademarks +1 +v20mr325089fab431271372650326 +1 +sizablebr +1 +slaves +1 +htmlimageonly28 +1 +20010805 +1 +size1binteractive +1 +24px +1 +jazz +1 +vm8000100 +1 +requesting +1 +restrictions +1 +nigeria +1 +200210010801g9181ek15588dogmaslashnullorg +1 +fifa +1 +httpfgvgiovannablogspotcom +1 +tiputil1corptipreleasecom +1 +sbr +1 +ucoa +1 +20100523102039gb2494kinakutalocal +1 +44d1516f03 +1 +sa269hl +1 +commons +1 +extracts +1 +ifpoliteru +1 +shock +1 +spending +1 +todaybr +1 +eizldtfkopobhhstrvobgep771vpmhcc0ksxstlheqqr1kuptqnvhx0iummdsoyagoq +1 +promised +1 +ft +1 +8500 +1 +xbrightmailtracker +1 +httpwwwbozombocomhugetitsindexphpaffid1534 +1 +ht3d10 +1 +nautilus +1 +classen +1 +interrogated +1 +xsieve +1 +ung0ciqjb53pppoepx6mgwkm5j4wlxgm5nzs5pyrdblhhewiudrgulgslbvluxmhrlmvuo2xhh +1 +unclosedbracket2206 +1 +minutesfont +1 +bottles +1 +273 +1 +searcher +1 +scanning +1 +150638 +1 +alnvj5flnt9xapudxpajflhl2knt9ov9xv3bl063gcfmf86fq8a9zqx8aipf86fq +1 +loader +1 +width3d22717223e +1 +gw2panasascom +1 +drumming +1 +183417 +1 +incorrectly +1 +p05111a4eb9869cb2eb2a66149496 +1 +valign3dbottom20 +1 +panerai +1 +lc0yntc5mju4odqsltewotaxmti3mtmslte4mtq4otmxmjksmza0otewmtg0 +1 +travelstrongfont +1 +harvest +1 +bsfsc0sa424 +1 +60028001106 +1 +3197816 +1 +personalthe +1 +argentsrv2com +1 +emacs211 +1 +re115ult +1 +090125 +1 +100000000 +1 +ctible +1 +commissioning +1 +cpunkssszcom +1 +113933 +1 +0011 +1 +wrap +1 +digestive +1 +consisted +1 +charts +1 +1252059779 +1 +2010042018310663af750dabydosstargateorguk +1 +iroijlaplap +1 +webmasteredgarmoskoppde +1 +unreliable +1 +oregon +1 +06232000 +1 +iwvmjrwzhjlvtgdcwg8pxqhggs0cq7qtcjhd8 +1 +aforementioned +1 +reportabuse +1 +c452 +1 +mailtoforkadminxentcom +1 +003341 +1 +135146 +1 +tech +1 +shortly +1 +a601116f56 +1 +jsamsnetcentcom +1 +g6ncmjxu073580 +1 +dominate +1 +speedstream +1 +256962d1ef3 +1 +mhv9vyikzyo00ii4j4p440ceaha7j9xakkcaa0ipbnqsmewh3lywsxgholo8lpymwbnetpkshmh +1 +forwardedby +1 +width620 +1 +specially +1 +b32si3112301ana20071101095540 +1 +181246 +1 +5a9a02941ce +1 +ijlqle +1 +ih7cq0lbpdn8ngzcptio8uhiephd9dkmmslbwmieqzuag6co8ksl2ol7tcnvlxtoh1q +1 +httpwwwnewsisfreecomclick38688976215 +1 +averages +1 +legislature +1 +20pre1 +1 +translate +1 +asl +1 +petrol +1 +3564443f9e +1 +remind +1 +shes +1 +pyoli4ml2acigqpajnrd13xtaox4jlzv1y2w13uvlczsgalg1361dx5fqici2qwiwzqn2yrn2dky +1 +recall +1 +hrefhttpclickthruonlinecomclickq019oxdi2jhs5wf8rmvurmpgpxz6rlr +1 +flexibility +1 +ithe0d +1 +divblockquote +1 +ts01050sligoindigoie +1 +homeastrong +1 +solaris +1 +src3dhttpiiqusimagesaigsetrate20020912101jpg +1 +proved +1 +channel +1 +qb +1 +sagfsgfsgcom +1 +marginbottom0pxmargintop0px +1 +trucks +1 +eclipsed +1 +httpgroupsgooglecomgroupsq22whitewaterfarting22 +1 +103396471712638camelamd1800 +1 +connects +1 +class3dthmfgmisctext +1 +themesinfo +1 +062044 +1 +210568221 +1 +license +1 +westminster +1 +mipsel +1 +mohr +1 +cuckoomany +1 +15120997190 +1 +colspan3a +1 +mexico +1 +colore77d2d +1 +sandra +1 +necrotic +1 +recordbreaking +1 +450 +1 +audrey +1 +10877021 +1 +sufficiently +1 +145129 +1 +nzst +1 +border3d5 +1 +residents +1 +5gay1e0094dcmzy02gaz5b +1 +business09 +1 +citizensb +1 +httpjhmphebcyxehtanet +1 +va3wzsdrzuabs29artsvcferkdzbn9dlfldydwogoo65yzdqk3j5fm4dtp60o7mul5hbksdbd +1 +dofontfontblockquote +1 +quietly +1 +value4040option +1 +12921512853 +1 +hrefhttpwwwhandangocomplatformproductdetailjspplatformid1ampproducttype2ampsectionid0ampproductid43645ampcatalog1ampsiteid159 +1 +scripts +1 +unintentionally +1 +100049 +1 +husatoxiet +1 +li115t +1 +153 +1 +appliances +1 +345 +1 +gojomousanet +1 +clipeqfrommx31 +1 +hrefhttptss3applycallcome25mlfeacxeblcniwaiaztiezcwllasidaaedneoxeiaweooeiimg +1 +libboostdatetimedev +1 +disperse +1 +3b1a52d0dc5 +1 +catchy +1 +mid2003 +1 +16200 +1 +thoma +1 +234307 +1 +height3d330 +1 +10227137143 +1 +jnlp +1 +bnetworkingbbr +1 +10pxthisfonta +1 +promotion +1 +parker +1 +8510226115 +1 +hometheater +1 +1de832ce54 +1 +mi +1 +bi +1 +shoe +1 +135131 +1 +resistivity +1 +santa +1 +partija +1 +bgcolor3dccccccimg +1 +hrefhttplockergnomepricegrabbercomsearchgetprodphpmasterid533869harman +1 +virtually +1 +httpwwwbondedsendercom +1 +plugin1 +1 +hrefhttpwwwlewdozedcnjkiruqxaciojcosyexifqmicaben0691431 +1 +234142 +1 +192146 +1 +nets +1 +quam +1 +ac0tnzrkuaz10264ptxqpu068ueyrdv1qjwxms3ubx2gocuk1k2dtmd0fiomvqkvx61v49bsjb6 +1 +otonysucal +1 +eats +1 +reinhart +1 +switche100 +1 +7ed4016f03 +1 +caches +1 +2850979285781970422 +1 +helouser +1 +recipe +1 +nova +1 +fromlocalnovowel +1 +trailer +1 +netil +1 +bgcolor3dfffffffont +1 +ttbomk +1 +58 +1 +zwlyihnlcxvlbmnlig9uihrozsbsaxn0lcbpbibhbnkgd2f5ig90agvyihro +1 +ichat +1 +hacksawhacksaworg +1 +50meta +1 +150857 +1 +inspiring +1 +ronljohnsoncoxnet +1 +mord +1 +1100 +1 +iciafontfonti20 +1 +httpleesadevfarmcom +1 +behaviour +1 +ben +1 +facebook +1 +visibly +1 +class3dstyle2font +1 +unsubscribep +1 +light3dffffff +1 +orsonhacksawmachinecom +1 +110236 +1 +biographical +1 +favor +1 +090110 +1 +33c3213a4737 +1 +archivelatest575913 +1 +pcebbd3dab9e3b6abd6d0c9bdb9c5d5f2b5c4c5b7cdfed5d5 +1 +athletics +1 +profitable +1 +idt +1 +iso88591qcolin20nevin +1 +size2926td +1 +blvd +1 +till +1 +751189522 +1 +stopping +1 +7113422550 +1 +valuedetailquote +1 +1274527006 +1 +75one +1 +volume +1 +proceeded +1 +n8okgjrn013740 +1 +size5a +1 +fasooption +1 +width152 +1 +width3d447nbsp +1 +valentuathaorg +1 +coastoption +1 +pipeline +1 +spamassassinsightings +1 +qaehulah +1 +n0nbn0nbus +1 +archtop +1 +indepth +1 +entitlement +1 +terrific +1 +httpboingboingnet85528842 +1 +tracks +1 +154149 +1 +030510 +1 +hospitales +1 +opinions +1 +jones +1 +xsaeximscanned +1 +sonjamaajonl +1 +storeyed +1 +nextpartvxklypzzzdwwhofz5e1rz +1 +infrastructure +1 +516 +1 +width3d77 +1 +1505 +1 +size1b1800br +1 +alyssalt08yourip21com +1 +bomb +1 +saou +1 +iezjtkfoq0lbtcbgukvfre9nicencj09pt09pt09pt09pt09pt09pt09pt09 +1 +transcaucasian +1 +0c5022c65d +1 +describing +1 +implement +1 +dedication +1 +gwynedd +1 +color336699br +1 +surcharges +1 +updatesfonta +1 +ne +1 +pitch +1 +spelt +1 +18px +1 +color838285br +1 +debeers +1 +width371 +1 +eradicated +1 +fools +1 +nameicq +1 +15000000000 +1 +incbrall +1 +wife +1 +201004271524o3rfoole017373gw1csminingorg +1 +netblocks +1 +score160 +1 +canopener +1 +srchttpshoppercnetcomiceps12086539401201gifa +1 +005401c1e7060ecc13c0147ba8c0xg395local +1 +xanaxlbr +1 +blackcombpanasascom +1 +encycl111p +1 +parish +1 +jealous +1 +e0 +1 +racing +1 +hrefhttplockergnomepricegrabbercomsearchgetprodphpmasterid555517musicmatch +1 +questionsbr +1 +loftware +1 +effects +1 +preventing +1 +linkb +1 +095618 +1 +hong +1 +spouse +1 +pr111pertie115 +1 +morefontfontblockquote +1 +smtp1admanmailcom +1 +d6f0hfueot84xtcxzdcczchosx2v3aukamx7p0xpv7vfv8a09ncpsy8rijih2g38mwzs +1 +cellpad +1 +fco1ncktcbdr70x2y5pelijcy9qhizua4igau9iaqfl1j5pwnbacgbueuu2shjntfp20uarmy0z +1 +bonetitle +1 +nw +1 +rnqjrmlh0n9ddgx9f8at1x0ycmtngpeb9nbgpvho53xv52xf8acwv7yjyf1hy8zibj +1 +2page +1 +hypertension +1 +recovery +1 +aimed +1 +trash +1 +formatted +1 +17i7oe00023r00 +1 +fergal +1 +walked +1 +consensus +1 +e2mr1813852fac1011274469166544 +1 +partei +1 +thunderbird +1 +e118en +1 +colorff0000strongurevenues +1 +4581516f03 +1 +maria +1 +estore +1 +2db642c72a +1 +sdatatype3dinteger +1 +suites +1 +46bff43f99 +1 +unretained +1 +mx1examplecom +1 +ofeglpgpchpacfljpailoeicecaamacarthyiolie +1 +heartbeat1messagingenginecom +1 +barbary +1 +wheres +1 +monthly +1 +allrecursive +1 +lets +1 +xenigmailversion +1 +lawrence +1 +bgcolorffffffdiv +1 +partnership +1 +134552 +1 +civilian +1 +narrow +1 +shakespear +1 +representation +1 +1022329156 +1 +listened +1 +enriching +1 +in100ische +1 +a117100it111r +1 +s111me +1 +httpwwwdebianorg +1 +valign3dbottom +1 +incworldcom +1 +uhhhhhhh +1 +9156an52269c79186883587xipysominjowaej59network9437e03 +1 +playhouse +1 +esq +1 +face3d22comic +1 +hancock +1 +razoradmin +1 +portability +1 +spasic +1 +4cc202d0c28 +1 +regiments +1 +spanp +1 +uhyaqddw4cxycvr42xwb95qvqemldwmmekgvseygnzb30zupnnupgore5um5msopsjmfvbynx0m2 +1 +width34a +1 +opened +1 +subdivisions +1 +webmailspamcopnet +1 +cotãs +1 +classtitlenbspgnomestuffptdtrtablep +1 +47ft8k0lz924ec8mmf6xnph14h8qldux0oykffjx5z4d2qc8jb71gkukrhcjbtgtjd8j1rp +1 +5simx +1 +37 +1 +ancient +1 +jerry +1 +msoborderbottomalt +1 +leenxcoza +1 +biggest +1 +friday20 +1 +skylands +1 +ease +1 +particularly +1 +intellectuals +1 +wayside +1 +125604 +1 +ejthmqeruikawyazfigfeewhjpjziebsigqkagsficbacgznceyyaqs0ggfz0hw1abibafw0sc +1 +20020829205404exzw15998imf13bisbellsouthnetadsl1572310msybellsouthnet +1 +efocu +1 +varlogxorg0log +1 +srchttpwwwzdnetcomincludeadsjsrgroup2560 +1 +fff +1 +addressa +1 +errors +1 +belly +1 +httpwwwquicktopiccomboinghk9nshvkkrrxi +1 +mcdonalds +1 +patch +1 +dir3dltrlta +1 +shortcomings +1 +owenpermafrostnet +1 +masks +1 +jya0hherjwnuugyecyogsceobdegdyinwyqayvx7cltsbeilnvxyjo0sick83aleubrbaxt +1 +085728 +1 +946a726a41419 +1 +youfontfontdivbodyhtml +1 +notes +1 +precisely +1 +abelard +1 +size2homebrpagefonta +1 +textdecorationnoneforward +1 +hrefhttpgvrlogduckrud80c870d65fview +1 +lam5xylwexfhsnzv1jsuxxjrzvx7ih0ujkwq66ldjyu2ppds42hs7vjlqogeyeavuyjkposas9tv +1 +bgcolor3dffffcc +1 +eek +1 +steadily +1 +gmailid128739f44994a175 +1 +httpbidocom +1 +viewap +1 +slower +1 +william +1 +axymu26kzwibababafupmhdtux1mhsqbbssop06nt26gk2uxo50umfvuz80vx00n4tzfv3uj3hu +1 +004459 +1 +444 +1 +virtualincludesubscribesidetxt +1 +howells +1 +iven +1 +auth +1 +woodworking +1 +style3dtextaligncenterif +1 +noel8rcygromjygpkju2nzg5okneruzhselku1rvvldywvpjzgvmz2hpann0dxz3ehl6gooe +1 +width3d3020 +1 +score76 +1 +halt +1 +gone +1 +distinguishable +1 +hrefhttpwww3dpageturningebookcom +1 +020602 +1 +194305 +1 +ide +1 +b160f16f1e +1 +satfamsbl +1 +transit +1 +lucky +1 +04 +1 +4101213a6407 +1 +runaway +1 +union +1 +spinosae +1 +113723 +1 +dos +1 +newbie +1 +2020 +1 +11fe0d0bb7b68ae0000042ab554bd44e6cdd60 +1 +mccane +1 +68116126 +1 +g34si5943649wej920100516113019 +1 +divnewsletter +1 +central +1 +chapter +1 +deflexed +1 +contentious +1 +eating +1 +022908 +1 +africawhere +1 +noshade +1 +brought +1 +specification +1 +bindinggenerator +1 +ieyeabecaayfakvpfwsacgkqmrvqrkwzhmekqwcgpypb8xu7tcwsj0drznn7goe +1 +kb +1 +cutthroat +1 +59nbspnbspnbspnbsp +1 +wyf23 +1 +berating +1 +concede +1 +nowgettime +1 +compiler +1 +codebases +1 +payperview +1 +1867 +1 +hing +1 +bsfsc5sa269hl +1 +c28si5457288fka1420100501112250 +1 +headertopbackgroundcolord3d3d3bordertop3px +1 +163334 +1 +ag +1 +sottolinea +1 +size3how +1 +alignleftsome +1 +headers +1 +pan20100425105933csminingorg +1 +displ +1 +gtfrom +1 +accumulate +1 +museum +1 +d651043963eda6fb4 +1 +stt008linuxsite +1 +assert +1 +journaltitle +1 +mpscph +1 +tao +1 +bgcolorwhite +1 +aligncentertrouble +1 +mosque +1 +4bce303150404coxnet +1 +bannerflextop +1 +srchttpmta4rle2rainroleszcomkfcblaqatlifteiceaaaooa +1 +gauntlets +1 +wollen +1 +110515 +1 +servicemen +1 +lm7dlyvv +1 +hrefhttpct2consumertodaynetcgibin14floyhe10cczme0d40er0acchevy +1 +20100508085900025lisireiszcsminingorg +1 +mattersnbsp +1 +theme +1 +county +1 +buxzle5gombdjtzr1mqcfvae2bz8cnapccntsbio4geur1lxhp4un2i0kzxbtom7ntldzqwutwk +1 +0a86fa +1 +resqlplperl8383110lenny1mipsdeb +1 +le3ddisplayblock +1 +c4192a +1 +usbrbrclick20 +1 +e2e1616f7d +1 +connections +1 +syb0mpfjrd8ipu1nu4ozdqbs5xa9p0myz6bbitksg5q7wd4sufp0dupwilk3a03nivqwikparquu +1 +boragurecsminingorg +1 +forfontp +1 +pcp01978751pcsaubrnh01micomcastnet +1 +ricathe +1 +101012089 +1 +12px +1 +140715 +1 +mj3762 +1 +142253 +1 +src3dhttpiiqusimagesfairlane200207182a03gif +1 +spokeswoman +1 +size3d15 +1 +g8ohvjf28951 +1 +moonbasezanshincom +1 +etcsysconfignetworkscripts +1 +downloadsa +1 +indicate +1 +ranunculaceae +1 +m0089lla +1 +ikotoik +1 +futureproofing +1 +hrefhttpclickthruonlinecomclickqfdbxdmqgor1rekypqoiti64q9t8pr +1 +fundamental +1 +xoperatingsystem +1 +03xui5yi5loebl1ast4plfmoottvlewbw8ppluvllunekrhjfltr9ivl6dkyqfjxamsk0mk2kwnh +1 +intimate +1 +07132001 +1 +bards +1 +leg +1 +pixelbuffer +1 +sweet +1 +ieyeabecaayfakvpw3qacgkqdnbfk86fc0rdwcghxga5lciwc9vpymzpe5jiqjk +1 +yrp3o8zjo +1 +een +1 +htmlimageonly16 +1 +surrounding +1 +workflow +1 +igfsd2f5cybmcmvlisa8ysbocmvmpsjodhrwoi8vbxlmcmvlcgf5c2l0zs41 +1 +successful +1 +004348 +1 +murphywrongword101 +1 +resale +1 +niue +1 +312 +1 +lawn +1 +embassy +1 +readers +1 +divisions +1 +053402 +1 +thingy +1 +depth +1 +height1 +1 +millarem +1 +133311 +1 +1941711618customerteliacom +1 +114608 +1 +landowners +1 +127331923178f706100001w4twrl +1 +troopersmarry +1 +bebayb +1 +sa290rn +1 +poets +1 +levelbfont +1 +target3d +1 +fkli8p4mztx9ho3lbuzpgcnxcnvp6dyhvain3mmdlkdaqtyjxwk88rlslvsllgmvjef0bbkn +1 +brsuch +1 +mortgage +1 +stalag +1 +crossfirewall +1 +125733 +1 +libxfcegui44 +1 +enfant +1 +qne6u037n3cab8cppz8lbliszt +1 +archivelatest573879 +1 +hrefhttpwwwfurbidizcnautepqzi58182qfaxqumuutomigek58182unsubscribehibodycsminingorgunsubscribe +1 +haymarketedacuk +1 +stdoutfilenomsgbufamountread +1 +vanished +1 +113426 +1 +spana +1 +landp +1 +wired +1 +000e0cd5d0120026320486684691 +1 +fontspanfont +1 +tdhr +1 +182250 +1 +gb241w808214 +1 +nnhtmlwithouttp +1 +desperate +1 +constitutional +1 +af6decorecpp291 +1 +httpradioweblogscom0100463 +1 +recipes +1 +timesp +1 +radioed +1 +corrosion +1 +causes +1 +prison +1 +tropical +1 +hu9vf3fap5amymvthj9krgle712kebhenodkdkj8k6w1010u633hmvnng6bg0fst4k6azdg8mz +1 +546 +1 +cd7d2d63c894 +1 +missingheaders +1 +411 +1 +border3d +1 +malay +1 +stylefontsize12pxcolor000000fontfamilyarial +1 +nominated +1 +rootkits +1 +o1oj3v5b000913 +1 +24232154203 +1 +pleasespan +1 +passwords +1 +indigenous +1 +n46ewmod025893 +1 +1441 +1 +presentation +1 +emssrv0 +1 +winfrey +1 +res +1 +greeted +1 +uhqhzcdc4ze4xpcylejuqpze1afy9escjsnwozqy2naibembmdldjvwjwxxvzj7ttdlpurbiypg +1 +eeeeee +1 +pinelnx44402100712315604199100000urgentrugacbe +1 +deceive +1 +3d6b46ab162571ed4e92clocalhost +1 +noted +1 +083815 +1 +srpskecenter +1 +hrefhttpwwwhandybackupcomimg +1 +10216186138 +1 +202745 +1 +useragentoe +1 +001635 +1 +aton +1 +lasernetpueblavirtualcommx +1 +websites +1 +worrying +1 +pursuit +1 +redistribution +1 +unconventional +1 +includesysasoundh +1 +networks +1 +forests +1 +5708835877795spanspan +1 +color003366 +1 +192332 +1 +a1a2cdb6c9e4b5c6a1a2b9a4b3ccd3c3b5c6b5c4bfaab7a2ba +1 +brbeing +1 +facegeorgiaunsubscribefont +1 +000801c26287f29604600200a8c0jmhall +1 +compatible +1 +vacate +1 +score568 +1 +uae +1 +deathtospamdeathtospamdeathtospam +1 +productivity +1 +h4y5qrrnsuguyq33kj2clin8behbrcuwc3zv4 +1 +scotfree +1 +eecsberkeleyedu +1 +yf27ki6cco2fsfprotonpathnamecom +1 +002249 +1 +samson +1 +hdatefromtosubjectmessageidreferencesmimeversion +1 +ashersky +1 +patchunifieddiffreferencesspamphrase0001xloop +1 +624 +1 +qqqqqqqqqqregexamplecom +1 +234217 +1 +2bturns +1 +httpsecuritydebianorgpoolupdatesmainppostgresql83libecpgcompat383110lenny1i386deb +1 +readmetxt +1 +testshtmlimageonly16 +1 +irection +1 +to423xfit32ulhcezzzrrhvq6sjdr7bslvxwksejee6kagxsccv2xt7s6jy5gyuvuq1ukw5lwruf +1 +norbr +1 +himalayan +1 +fashion +1 +redesigned +1 +11897rying +1 +leipzig +1 +governing +1 +denis +1 +lang3d0bstaff +1 +bccd +1 +kick +1 +beastiemcwaolcom +1 +bsagicagicagp3n1ymply3q9bw9yzwluzm8ipmhlcmu8l2epc9wpg0kpc9k +1 +li71 +1 +k89jglpfeh66xxc6fdlgetayicojf7bewmmg9exjgclagznbehfvkxfcfee2b13voma22xar2x +1 +correspondence +1 +incfontuspanfont +1 +parodies +1 +raa26647 +1 +montanaro +1 +alternatives +1 +rhitta +1 +wsuhjlsanzi3v0yzdkcginqxvgur0pxiq4errirspozszozxqyi5opkm0alrszooam0zpkm0al +1 +mxsqv8aoco5wbh25dgzirdsbl9wjhgoqdad0az2noqc48gzycs14om1vvoqce59nnkfonj6 +1 +113618 +1 +3d8d5c3b2090202barreraorg +1 +httpskobblogspotcom +1 +serverlayout +1 +bent +1 +trunk +1 +p17si2535543fka4620100506103749 +1 +194842 +1 +cznxuykhqdg8msorzb67cpf5zf5tbbohk98ce9daamgwl1oazxtnxpods5nmabgm1gqg6inmemz +1 +offending +1 +sans +1 +ls9xjyovhjzjz7mms3lhv5546umqqp5u4rqq9qm93ooxeszb95mastsg7udcj9mum4cqv4pmrk +1 +ppl +1 +corp +1 +93129105218 +1 +competition +1 +yearfontfont +1 +k2gb59a0661005052317jf0fa9ec4n2399a8ed8d64e1damailcsminingorg +1 +loripatelhotmailcom +1 +3ccenter3e3ctable +1 +nametitleimage +1 +university +1 +messed +1 +o126x70f008326 +1 +moneybfontbr +1 +valign3dtop3e3c2ftd3e3c2ftr3e3c2ftbody3e3c2ftable3e3cbr3e +1 +eloquence +1 +puerto +1 +282179628 +1 +documents +1 +size1b +1 +nonlocal +1 +036 +1 +909 +1 +targetnew +1 +17inch +1 +psiwiibjzwxsc3bhy2luzz0imyigy2vsbhbhzgrpbmc9ijeipg0kicagicag +1 +leftwing +1 +threatens +1 +websitea +1 +clamdscan +1 +raytitle +1 +piii +1 +sa081a +1 +0c5deu9p8koxhhtw25wqaakqxyq8cr08tzsgvdlwjd7v3skrraxbcft8u0cing14pwnf7msop +1 +httpwwwinteralianetindexphp +1 +hrefhttpnewscomcom12001120933954htmltagstnlhorizontalxprotxtnewsvisioncnet +1 +dmail200212 +1 +httpsecuritydebianorgpoolupdatesmainppostgresql83libpq583110lenny1armeldeb +1 +restore +1 +path +1 +114005 +1 +face3dariosoixxxx +1 +132030 +1 +1087 +1 +schaefer +1 +reverse +1 +pieces +1 +directories +1 +arialbflowchart20 +1 +125244 +1 +outgoingsecurityfocuscom +1 +22e +1 +postgresql +1 +195679309258527jyadrzdfoydwsvo7816220961 +1 +colspan2 +1 +elite +1 +mx1freebsdorg +1 +c2131715213bjarenet +1 +colordark3dffff99a +1 +germana +1 +size2owed +1 +goodman +1 +g8o80bc26569 +1 +forteanaunsubscribeegroupscom +1 +094052am +1 +estaciãn +1 +herbal +1 +cellspacing3d +1 +bfar +1 +aes128sha128 +1 +height3d129 +1 +52ft +1 +g990ahk08455 +1 +fieldengineer76228205248 +1 +n672brv2024954 +1 +bouncedebiannewsmlsubscribertechcsminingorglistsdebianorg +1 +georgetown +1 +bilbao +1 +teacher +1 +maya +1 +lettres +1 +goldilocks +1 +heck +1 +115816 +1 +delspyes +1 +itemheader +1 +previous +1 +aligncenterbidp +1 +barbara +1 +lunchboxes +1 +yyyydogmaslashnullorg +1 +rites +1 +135 +1 +inland +1 +15in +1 +overheard +1 +mlm +1 +052334 +1 +xmuorgmfr24d04o0lmo8h6brvclnvjca6ksdlixqgkhwhhqbp2vjwobm2cd6bi4lptfbny58wmn +1 +bgcolor3ddddddd +1 +k9si18235928fad2920100519074529 +1 +mail1001 +1 +camerab +1 +enig31caaed0fb049d5bd7604594 +1 +nbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +1 +ptwqs6fjpjna0nbbxe1lbliszt +1 +102234217 +1 +errata +1 +083353 +1 +veronica +1 +qtimydot +1 +12si3898778fks2020100424042847 +1 +hrefhttpwww185wiildaccesscomdownload +1 +windows1251bw1nqqu1dia +1 +size1city +1 +testsalltrusted18 +1 +20125312211811520011108 +1 +underwent +1 +chambers +1 +banking +1 +bia +1 +2425 +1 +classtipnewstitle +1 +burn +1 +color0000ffinfocomprosyscomfontabfontpblockquotetd +1 +height80 +1 +dashed +1 +tabs +1 +031c16b24b3e3426a3b12dd05ae2hfbrnu +1 +virtualincludesubscription2txt +1 +181021 +1 +icagicagica8relwigfsawdupwnlbnrlcj48qj48rk9ovcanciagicagicag +1 +backgroundhttpwwwlockergnomecomimagesissuetopright2gif +1 +c650fc3c555b11df8dd3d033ee7ef46b02150157apbsaslquonixpoboxcom +1 +171338 +1 +changed +1 +stylebdiv +1 +a22481297391075contentsarticletitlecolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightnormalfontstylenormaltextdecorationunderline +1 +parentage +1 +wwwlindowscommyths +1 +31108178 +1 +234 +1 +simultaneously +1 +pop +1 +890 +1 +ypjk +1 +onl +1 +intervene +1 +bhwwap0brjmd4vhiueq8togmtcsztbrntweskbi1plk +1 +tracked +1 +deepening +1 +size3d2your +1 +1255176789 +1 +oldest +1 +bells +1 +z6ssftzyozdl71ntxep79dt0mddz7c4odj4pd48lq7y8wp7czv6cuz8dcwmny8tuyprv +1 +executing +1 +c0e8813a5529 +1 +cellpadding10 +1 +src3dhttpwwwtributecanewsletter48imagesvindieselgif +1 +103443 +1 +selfregard +1 +104307 +1 +duplicated +1 +httpwwwcompletereviewcomsaloonindexhtm +1 +url0050httpwwwwannawatchcomorgysexfc01cgitappers +1 +maoidceqkc9thdazjjs1nismyyigyxhd3d3ajqwhwt1e1ntjaqdcpntgwdyvffw9gjjfaugx +1 +bkk8ochrhed7okz122ztw3tybmfv9hhhqgj8tolgoxj49tf0fo0mdt4dv76kbewab0 +1 +sold +1 +132732 +1 +economics +1 +212947 +1 +glance +1 +av24166comexru +1 +mengjiang +1 +extremes +1 +height3d120 +1 +c7b3d1671b +1 +debianmultimedia +1 +aislesorted +1 +width134 +1 +fashionnbspthe +1 +xbulkmail +1 +suggestion +1 +qy98 +1 +httpwwwadvbizcncontactus +1 +o3r2ygtn006857 +1 +brbri +1 +giveaway +1 +mayfair +1 +165526 +1 +srchttp20912615962spacergif +1 +elzyx2xe5gatedatbofhit +1 +042106 +1 +6e78aa7beaaf01773dc69b0033c9f7d8 +1 +hotmailcom +1 +celmer +1 +size1nameinput +1 +namewarezcdshtml +1 +1592265943 +1 +valuehttpwwwlockergnomecominput +1 +notified +1 +sheffield +1 +povão +1 +qsobtapjqci3hjrxapuukkjkkxeffku0piuvuvpdrlkxkg49kegmt2spkepjjp9lg6vzv +1 +iqeu3pat8lwp6xnz2kjf8kcaetacrg3ad2f8lwqljg32uumatka9f9qpstjaqzf61bgkb +1 +bizarre +1 +broadly +1 +ion +1 +httplistsdebianorg2010050707432040ecb684mihira +1 +boundarynextpart00000233be1e640405ba1e005 +1 +width3d59 +1 +fraud +1 +type3dtextcss +1 +width145font +1 +106160185195 +1 +pardon +1 +gods +1 +1921681183 +1 +httpwwwadclickwspcfmo283spk007 +1 +ls0tls0tls0tls0tls0tdqpzb3ugyxjlihjly2vpdmluzyb0aglzig1lc3nhz2ugymvjyxvz +1 +prevail +1 +sonys +1 +saxon +1 +matched +1 +legitimately +1 +flemmis +1 +buyers +1 +colorccccccfontnbsp +1 +messy +1 +directmarkeudoramailcom +1 +mailtoiluglinuxie +1 +super +1 +divides +1 +162609 +1 +kdepim +1 +httpwwwbarracudacomreputationip19066150124 +1 +222624 +1 +190353 +1 +dgljdqoncmfzc29jawf0zwqncg0kcmvjb21tzw5kyxrpb25zdqoncm9jy2fzaw9uywxseq0kdqo4 +1 +n2 +1 +h2big +1 +ap +1 +urihex +1 +184649 +1 +b059a4412f +1 +diligent +1 +333 +1 +width3d +1 +overa +1 +ron +1 +650px +1 +scale +1 +bfontlitdtrtbody +1 +20000 +1 +sec117re100 +1 +stevep +1 +height3d63 +1 +2002081111113628aab804matthiashaasebennewitzcom +1 +1266000236 +1 +qmymnjzwsbaigi2fvsjuniayrsacsalcagbiuutvakotidnhzowmbvrcmfgjippwabb6kisg9v +1 +bcut +1 +102043 +1 +befa62c495 +1 +successfully +1 +nominee +1 +enables +1 +wto +1 +unions +1 +108 +1 +0l2x00j8pn7x00h0amtaout20012netil +1 +practical +1 +mailmiestosk +1 +httpcvemitreorgcgibincvenamecginamecve20093876 +1 +burst +1 +grandchildren +1 +compixelturesystemhookssystemhooks +1 +color3d000000shipping +1 +failing +1 +apfont +1 +200912111649nbbgn4oq019048ns1csminingorg +1 +mailman +1 +winerseems +1 +rea +1 +mint +1 +nosend1br +1 +applying +1 +eleven +1 +00005e83686900001971000014a2mailcareernetnl +1 +approximate +1 +repaid +1 +1648 +1 +lovingly +1 +i2laee79c571004070434pf2f6d34cq57be4fc100f60309mailcsminingorg +1 +lexgatd +1 +trees +1 +clamp +1 +hrefhttpwwwdivlcomindexaspl2cid78img +1 +origin +1 +library +1 +q0t7tu2p2sabbyzbbh6iyxn0fphrns8ytxxqcrsisesnmjc3972xopocqbxhgyxr +1 +mail63opentransfercom +1 +t97ke +1 +argumentativeness +1 +68116120 +1 +200119151175 +1 +caa12216 +1 +size3d2mean +1 +b2tjzhjvbs8pgltzybzcmm9ahr0cdovl3d3dy5nyxrlmtiwlmnvbs9ib29ry2ryb20vy2qz +1 +9338145197ip71fastwebnetit +1 +httpwww12hscomimagestesco1020offmincepiesgif +1 +src3dhttpwwwcnetcombgif +1 +style3dpaddingright +1 +commentary +1 +212035 +1 +hrefhttp4d4cbfxqufojehcnquaoxejybq7427c64373r88411234g4uxadon911817498129208230405632 +1 +noekaaqbpdwktrqjirknqs45nnk7auijsklpsvx5xjljswn5ckl3wx4jhmmlizjjmqmfyaajrv +1 +o4gnvud2024304 +1 +style3dwi +1 +pmiles +1 +critical +1 +hardly +1 +beverley +1 +e2fsck +1 +lor3dff0000bvigoral +1 +travel +1 +124335 +1 +wall +1 +color3d333333bdear +1 +kiffened +1 +moreplicates +1 +archivelatest577066 +1 +nates20 +1 +20020805041438j42527100000mooncampusluthse +1 +class3dbold4panda +1 +teemu +1 +httpfreshmeatnetprojectswolfpack +1 +camera +1 +ulfontb +1 +helps +1 +parks +1 +efc0a12c8ba2 +1 +style3dfontfamilyarial +1 +084459 +1 +scared +1 +injections +1 +kuznetsovs +1 +401en +1 +srchttpwwwcnetcominlinvrhclosegif +1 +jmilugjmasonorg +1 +href3dmailtofirewalruscsminingorgfirewalruscsminingorgagt +1 +hcm +1 +jydrbmv3jycgaxqgd291bgruj3qgd29yay4gsm9kesb0b3rhbgx5iglnbm9y +1 +dbrayfordcsminingorg +1 +para +1 +fmsmtp06dlancineticde +1 +spare +1 +width31 +1 +svenjoacgmxde +1 +stare +1 +printer +1 +panels +1 +2410200153 +1 +225pt +1 +m2h308d4701004251507y5ab5e093ye814f86e7e1d8c11mailcsminingorg +1 +michigan +1 +heading +1 +xqhkorhq7tr +1 +debiansid +1 +throwing +1 +pda +1 +nncnsubdomainadn +1 +worksbra +1 +stylemsobidifontweightnormalspan +1 +k5cs246737ibd +1 +cpunksmindernet +1 +ci8vls0dqo8l3njcmlwdd4ncg0kpc9ib2r5pg0kpc9odg1spg0k +1 +manynbspcompanies20 +1 +ubohunyw +1 +144328 +1 +estima +1 +hits78 +1 +hrefhttpwwwimakenewscomeletrachangecfmxmediaunspun2czzzunspunspamassassintaintorg2ctxtclick +1 +snug +1 +natural +1 +leitl +1 +unmount +1 +170648 +1 +laugh +1 +complex +1 +lead +1 +configure1353 +1 +hrefhttpclickthruonlinecomclickqe6qg8rqwyqdk1pmprxgugh6nqyvu4r +1 +httplistsdebianorg4bedaed34020803cyberspaceroadcom +1 +fontb +1 +upleaseu +1 +085428 +1 +xgpgkeyserver +1 +supporter +1 +bfonttdtrtbodytable +1 +bhmwcfwzcjcosnbymvu4qxusb8rat28wpzn6vncotsy +1 +submitting +1 +httpjeremyzawodnycomblogarchives000203html +1 +hrefhttpa2drugselliot90aruuzosuehid2b77876fd +1 +relationship +1 +anadebianorg +1 +ilk +1 +httphomefilternetnljanwolsheimer2xalthtml +1 +archivelatest32193 +1 +andor +1 +everybody +1 +c6e9a13a62d9 +1 +us62 +1 +microsoftr +1 +918 +1 +jeffrey +1 +6e51c7bc36a345c2ad7533b49394124acsminingorg +1 +123 +1 +india +1 +f20cs84061wfi +1 +brown +1 +iso88591qz4ltpu6qolya6g9aw0krjlajcsgcicycxrcoxwmurhckrvhcedaur7own2 +1 +officialsin +1 +n91mgltr022871 +1 +tonersnbspfontp +1 +122436267 +1 +cve20101170 +1 +styletd +1 +worthless +1 +stracona +1 +1002264587 +1 +naacpcrf013812 +1 +202952 +1 +o2d0t531004925 +1 +800mo +1 +quantum +1 +hey8aiang5aitb4qwakbeaijlfc0ohcb8x65dsbiiff6awf5rwkam1smgagilurxvudohkccnpgk +1 +width1br +1 +strongthe +1 +lxa648hf +1 +joint +1 +hrefhttpsupportmicrosoftcomdefaultaspxscidkbenusq321126httpsupportmicrosoftcomdefaultaspxscidkbenusq321126a +1 +obligated +1 +6217143253 +1 +telnet +1 +ui9caa6gaaaykaaguaaaygaaaahizjjtrkm1z48gogzqaaaaaaaaaazxvdk6oabniaadkaaaabka +1 +anb +1 +hyvezj9h2fxs4yab2fxs4ydhv7op2aadhv7op2hyvezj9gahyvezj9h2fxs4yab2fxs4ydhv +1 +212716 +1 +htmlshortlinkimg2 +1 +n6oc9lh5017607 +1 +shall +1 +m14zltsfh1trwlcm3zkcloy6gcaimqehucrtxjr4drcf2e0ioahc6uobeo27vcyflultrq3tgbbj +1 +underscores +1 +tips +1 +7140abr +1 +stoddard +1 +generation +1 +disclaimer +1 +andnbsp +1 +ties +1 +10223589 +1 +17tvuk0001cs00 +1 +hrefhttp62b7gnaburencnowahyulat8p3xy3i14699blo8o79057qowelauadyhy80972204357334287373 +1 +bgcolor3dd +1 +factor +1 +shots +1 +necessity +1 +capturing +1 +divided +1 +bilv2ltkkobhygowmmokyfedl6wjqdjqvi7wkymhvspy5qi3u107guteiylypkonpmzuffyyn9c +1 +dinosaurs +1 +20100523232855gb26914penguincodegnomeorg +1 +nomore +1 +gmailid1284c780487ba08a +1 +105941 +1 +otdpxu7xbxrbhqlacvf6bwfvndwtw3uilt5ukjprlorumk8ptn1lsbelubeywxyuuubvvkdtt7iu +1 +cellspacing10 +1 +1246729415 +1 +titlezapp +1 +thinggoesherecomremoveaspclick +1 +sanctions +1 +color000000if +1 +hrefhttpxui67bixyeqawcndaicqzugovyburjetoimyyij1da342067njogozqreoix +1 +la +1 +2611 +1 +0748213a4ff9 +1 +retirement +1 +4bdb3f212cd6d47dtradalnet +1 +519015741132491272296881284javamailrootmd01wowsynacorcom +1 +discriminationthe +1 +devel +1 +beaches +1 +suburb +1 +bar +1 +institute +1 +22adv22 +1 +variable +1 +dyahoocouk +1 +fulgencio +1 +theb +1 +optimistic +1 +2096118386 +1 +3cu3e +1 +aedes +1 +eyesstrong +1 +ff0000brfontfont +1 +triptpolers +1 +p9si4124506fkb320100430192113 +1 +20910320317dialin1oshathenetnet +1 +ielovevstkvudqogdqo9pt09pt09pt09pt09pt09pt09pt09pt09pt09pt09 +1 +132229216135 +1 +vivendi +1 +15px +1 +mo +1 +pfc6bgnmolbcrghkjcvky0nerdjxmruahjgqnjkoow85fszkdzk1mk4qkhka0brwoklcmtvcqj +1 +adiv +1 +alberta +1 +electro +1 +m111re +1 +wwwmailshellcom +1 +8220here +1 +camnaughtoncsminingorg +1 +silence +1 +vision +1 +replyb +1 +face3darialthese +1 +hired +1 +ct +1 +symantec +1 +20771218155 +1 +s6txp0gmbiymxaywthi0pkrgrsm9zwvzm9udd48l3adqogicagica8ccbhbglnbj0iy2vudgvy +1 +doctors +1 +21br +1 +8223136115 +1 +nzetodc1dqonckzyzwvgdwxsvgv4da0kdqpqyw5lba0kdqpzdwjjyxjpbmfsdqoncmx5bxboywrl +1 +invited +1 +20897132119 +1 +south +1 +initially +1 +284 +1 +resorts +1 +predicting +1 +iso88591qbgskvlvm9zsejlgaaaac3rstlmzs96sgzles95lgbtq30a096qaaai0surb +1 +102919 +1 +a96suajsyqg8vibg1e4njea4hayssqab7kkd8aoya7hapnjf2d3zxqxgvpjfv12ggh7jn696anta +1 +nonbmp +1 +hits193 +1 +1889625141 +1 +0909tr0909090909090909tr090909090909090909td +1 +joystickcom +1 +jdecfhkdx4bdsodeq3q6zeckfyglxof5y6x6rzuyp6jfyqgwzozsfzywokfk7o6xlhqr +1 +exmyeqwmdawmdbemdawmdawmdawmdawmdawmdawmdawmdawmdawmaq0lcw0odraodhaudg4ofbqo +1 +m7mr3342517qag3801274726689910 +1 +122548 +1 +whenever +1 +surrounded +1 +anytime +1 +gnus513 +1 +formfont +1 +q8ybghgosa1ee1b0jx7seywab6i4di8tqdo0dr54samangdagglfdrhgwwaw3ojvrqaiocubghlo +1 +slackmoehrlemecom +1 +nextpart000000701ca57d5af6060a0 +1 +clickable +1 +kgattinetechtargetcom +1 +tic +1 +bartholomieu +1 +lightning +1 +6f1df414f4329ee27ada8e9b63a0c56dsquirrel1921681100 +1 +sec111n100a +1 +missed +1 +tm +1 +reply20 +1 +011102 +1 +0a1ba26a93978 +1 +size3d2sick +1 +2002072010543231116qmailmailshellcom +1 +29th +1 +adet +1 +214638 +1 +plans +1 +ulcontent +1 +isolnetsuxtechmonkeysnet +1 +125490006564b3003a0000w4twrl +1 +intelligent +1 +divas +1 +pomerania +1 +d983d16f03 +1 +sequencing +1 +17w6ex0003ng00 +1 +164857 +1 +colornavy +1 +immune +1 +apc +1 +consolidated +1 +experiencing +1 +fragmentation +1 +httpastonvillablogfootballcomdrkeene +1 +rtl8168d8111d +1 +andojapancom +1 +srchttpwwwcnetcomidpsmrbgif +1 +g6nhhphy090509 +1 +businessfontp +1 +3dhttpviviacomuamhtml2nvilk4ix8moreap +1 +unit +1 +size5149149149149149click +1 +pthe +1 +msif +1 +style3dpositionabsolute +1 +colorredcitizenfont +1 +animal +1 +administrations +1 +forteanaowneryahoogroupscom +1 +valuehtdiginput +1 +regime +1 +esecurity +1 +lighter +1 +acceleration +1 +xmimetrack +1 +width445td +1 +rq2d +1 +align3d22center22table +1 +centerin +1 +171800 +1 +1rvjg1e8prnyzsrj5uqaxf6vvd84ztt4a681i22upygnmkmqbgceelrfykjkncj8hwr0zvjldhi1 +1 +kids +1 +valignbottomimg +1 +fbiwinter +1 +g8uku6k15698 +1 +srchttpetne104attendbuzzinfoimages3395jpgabr +1 +belongs +1 +1256211881 +1 +chirac +1 +preinstalled +1 +style3dtextal +1 +accomplish +1 +moderator +1 +clearly +1 +pst +1 +heard +1 +4965513a510c +1 +204145 +1 +hrefhttpwwwgnometomescomtome001828htmlinternet +1 +preserved +1 +boundaryboundaryofdocument +1 +1126 +1 +london +1 +httpdandaniellehomemindspringcom +1 +donna +1 +d97n +1 +johannesburg +1 +rv099 +1 +fraught +1 +drive +1 +margo +1 +9fc7response +1 +arrangements +1 +help0d +1 +002732 +1 +errorreceived +1 +9qkjheaszucmktjhf9s966rzeuajiirqsmrio4nugfyn8arwyuaqowynkkkygwmuwamykqmotxo +1 +192818 +1 +fromexcessbase642 +1 +h111llan100 +1 +dddddd +1 +blocks +1 +596 +1 +readersdigest +1 +parallels +1 +bronze +1 +49a9816f03 +1 +e2wcmajqrdgdvym1wdwob8mjatcbyxtsasi8mvduvbrhhzqitqmcspqzafhjsdrq1lz +1 +administrable +1 +041224 +1 +1255176789033900390000w4twrl +1 +independent +1 +padding5px +1 +hz +1 +s111llten +1 +hits99 +1 +livnptntp5bxhoed5i9jwurtrjqo2i3fkxjlxm1dk0rqrejsewijbdtnnduynkbclww03gjxr +1 +inpmrfci01 +1 +3b9de4bf1 +1 +tt7pmjansrtp0lhpw4ybsfwmjvvs9sy5z8yaspukdb9psgbzd0d9avyfluaxiqmzzqykhgtqrag +1 +pvb32 +1 +youfontbr +1 +needed2 +1 +qqthgnphlsf8asnygztlqixzrqtjkmoxy5nddgxqbiqftjrvhhf9nztgtgqchjwlghsfvoca +1 +smx +1 +alignleftstatesp +1 +pc9cpnrozsbyawdodcbwzxjzb24goikncjxhighyzwy9imh0dha6ly93d3cuag90zgf0ztjuaxrl +1 +disclosures +1 +altspecial +1 +073949 +1 +c111n115t97ncy +1 +transition +1 +subjectremove +1 +12245142206clientattbicom +1 +20020909193637590bbb1whatexitorg +1 +1022321204 +1 +boosters +1 +hrefhttpwwwgfx2swfcomgnomespecialboomer +1 +083548 +1 +awygew91igrpzcbub3qgyxnrigzvcib0aglzlcbvcib3axnoihrvigjligv4 +1 +suomen +1 +191543 +1 +evros +1 +111ther +1 +gnomecredits +1 +pgh0bwwdqoncjxozwfkpg0kpg1ldgegahr0cc1lcxvpdj0iq29udgvudc1uexbliibjb250zw50 +1 +115c97ttering +1 +fatherand +1 +lifont +1 +dhmgmmdjvb47 +1 +mutt1520 +1 +unquoted +1 +1258102535 +1 +alignright20thcenturyp +1 +cmd +1 +sitemesh +1 +sa392c +1 +incorrect +1 +rescue +1 +inevitable +1 +urlhttpwwwstudentmontefioreulgacbemerciadri +1 +width209 +1 +msogenericfontfamilyroman3b +1 +bgcolorcccccc +1 +ws +1 +hrefhttpe43pharmraimund20arunymotuhiuu7feb9cabc4 +1 +14si22934720muo220100413060027 +1 +bccnkad3nwobfijmq729o1onnvojzsk9ramu75bhbtapqyha9kd7dapk6oqutguniwikflv +1 +creates +1 +forest +1 +classic +1 +map +1 +style3dtextdecorationnoneto +1 +namerobots +1 +downs +1 +184517 +1 +lithium +1 +281726 +1 +hrefhttp515franklinedruypuoybc35ef5c8cb6a3d +1 +n111117vel +1 +jennip +1 +absent +1 +gmailid128a8b054ff1dede +1 +melkkicshelsinkifi +1 +laaljusticegcca +1 +material +1 +xoriginalnewsgroups +1 +berlin +1 +src3dhttpwwwzdnetcomgraphicsancho +1 +8403zmsx211012 +1 +vorbis +1 +p3plsmtpa0105prodphx3secureserver +1 +although +1 +1571119187 +1 +074606 +1 +10pt +1 +127214956143802camellocalhost +1 +10872235 +1 +httpwwwineedmorecowbellcomblog +1 +mutt14i +1 +h3font +1 +httpwwwtheshiftedlibrariancomcategoriespdas +1 +gmailid1287febd66e45370 +1 +4less +1 +cios +1 +thyroid +1 +locales +1 +124917 +1 +pgpsignaturercvdindnswlmed +1 +12231148143 +1 +viewpoint +1 +descbc3sha +1 +32a6813a4a14 +1 +v60026000000 +1 +altpress +1 +entering +1 +naf +1 +sh +1 +20020823094833abdf4c44eargotech +1 +c3azwxsuk1jcwxghvjglnnf2nqswjgcbatjjrv5sae81sd7ipbalwd6844xu2j5gzubqhekt7d +1 +unravel +1 +062140 +1 +satelle +1 +004513 +1 +101041 +1 +influenced +1 +200fontfamily +1 +400pxbackgroundcolor +1 +n9hatorb022732 +1 +rewriteengine +1 +colorcc0000freefontb +1 +reaping +1 +0h35ebw1tf +1 +1269644290 +1 +kunt +1 +50005 +1 +v546 +1 +stadium +1 +8122812220020902revision +1 +uj2znt5wei5t +1 +fx4 +1 +challenge +1 +rendered +1 +award +1 +r111aste100 +1 +lonely +1 +hassle +1 +valigntopibfont +1 +mikemichaelmoorecom +1 +p97st +1 +pgdev83110lenny1mipseldeb +1 +filetimef7b6512001c1f874 +1 +galactic +1 +backwardscompatibility +1 +debiankderequestlistsdebianorg +1 +prototyping +1 +shipment +1 +besides +1 +795eb3 +1 +rising +1 +7600 +1 +href3dhttpwwwtwelvehorsescommmclickthrough2 +1 +norfolk +1 +caro +1 +201129 +1 +jiggetsbars +1 +height56 +1 +144604 +1 +rated +1 +ke +1 +failed +1 +mmc +1 +evening +1 +troops +1 +keywords +1 +metrop +1 +boycott +1 +zzzzilugspamassassintaintorg +1 +uci +1 +boldwellnessspanatdtr +1 +b2b352942bb +1 +auto000000434609apperetocom +1 +manifested +1 +172541336 +1 +httpwwwbarracudacomreputationip20117225294 +1 +href3dhttpwwwblurgreatcomfont +1 +replyby +1 +d557ed124559 +1 +0nbsp +1 +10pxunsubscribefonta +1 +midlands +1 +bgcolortrtdtable +1 +cpunkswastemindernet +1 +permission +1 +g8a8l1i24928 +1 +50769 +1 +eastrmmtao105coxnet +1 +poster +1 +ionel +1 +notebooklaptop +1 +1980s +1 +4bc714707040501studentulgacbe +1 +o2d2h3qe004345 +1 +2283 +1 +span40 +1 +supportedengines +1 +dreams +1 +xpgpkey2 +1 +dead +1 +align3dmiddlebfont +1 +h5portuguese +1 +castileapplecom +1 +quiet +1 +mailtojavadevrequestlistsapplecomsubjectsubscribe +1 +velit +1 +href3dhttpsmootheithercomimg +1 +193351 +1 +qdns +1 +patients +1 +partialpressure +1 +rhughes +1 +httpwwwzimbracomproductsdesktopfeatureshtml +1 +vanish +1 +duep +1 +dãsseldorf +1 +dsa1813 +1 +tion20 +1 +verrall +1 +href3dhttpwwwdiskeepercomdiskeeperhomet +1 +1093808 +1 +intmx1corpexamplecom +1 +lodging +1 +barebones +1 +destination +1 +storage +1 +linkabr +1 +width3d465 +1 +srchttpartvcomcnet1dipg021502pointcamerajpg +1 +riskthis +1 +bet +1 +constructed +1 +mailnetnoteinccom +1 +pop4nxwyzy6waa9bmaaaex5t9yj8p6hj1mknfaprassxxny3vs9plbvq8nsvidm1x8umxxaoinm +1 +valuecvcape +1 +als +1 +3351 +1 +09bf69fe8b +1 +1phrmd0hktytoo4e5fautdctafqat36dwnmj2iozz3sqavnkdaggxzlafw04eaxyeeupcyxpkmp +1 +virtual +1 +y4mr1555525faf511274452903404 +1 +height3d114br +1 +150911 +1 +2qgaldjtzwmmxxiruqixnvvamnfx4x8ookunviscinzfbgd0oaolrgqdtft3p79dxtw6cgqo4op +1 +m8cs15448wfj +1 +transitionalen +0.3347387608659775 +0.03236066388387246 +forkxentcom +0.0750425279410844 +0.20974110172102423 +many +0.14586877328423709 +0.16136461867452812 +result +0.051433779493366895 +0.2258665960698562 +w3cdtd +0.35834750931369547 +0.016235169535040454 +custom +0.33473876086597754 +0.03236066388387245 +sun +0.12226002483651939 +0.1774901130233604 +phobos +0.07504252794108443 +0.20974110172102423 +doesnt +0.05143377949336692 +0.2258665960698563 +people +0.1930862701796722 +0.1291136299768644 +11 +0.09865127638880207 +0.19361560737219222 +xmsmailpriority +0.35834750931369536 +0.016235169535040454 +20 +0.16947752173195457 +0.14523912432569636 +since +0.1930862701796722 +0.1291136299768644 +click +0.2403037670751071 +0.09686264127920029 +before +0.12226002483651945 +0.17749011302336035 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +0.02782503104564942 +0.24199209041868816 +well +0.12226002483651946 +0.1774901130233604 +100 +0.33473876086597754 +0.03236066388387247 +help +0.16947752173195446 +0.14523912432569633 +linux +0.07504252794108444 +0.20974110172102423 +000 +0.3583475093136953 +0.016235169535040447 +build +0.2403037670751071 +0.09686264127920034 +best +0.26391251552282446 +0.08073714693036846 +0500 +0.1694775217319544 +0.1291136299768644 +jst +0.334738760865977 +0.016235169535040457 +mailtoforkrequestxentcomsubjecthelp +0.02782503104564941 +0.22586659606985626 +rights +0.2639125155228245 +0.06461165258153645 +httpxentcommailmanlistinfofork +0.051433779493366895 +0.20974110172102423 +around +0.24030376707510714 +0.08073714693036853 +state +0.1930862701796722 +0.11298813562803232 +killlevel10000 +0.3347387608659767 +0.016235169535040454 +work +0.14586877328423709 +0.14523912432569633 +should +0.09865127638880217 +0.17749011302336046 +us +0.19308627017967225 +0.11298813562803234 +view +0.2639125155228245 +0.06461165258153641 +peruser +0.3347387608659776 +0.016235169535040454 +aug +0.12226002483651953 +0.16136461867452825 +service +0.2639125155228246 +0.06461165258153644 +brl +0.33473876086597754 +0.01623516953504045 +xbarracudaurl +0.3347387608659775 +0.016235169535040454 +autolearnfailed +0.02782503104564942 +0.22586659606985618 +web +0.16947752173195446 +0.12911362997686435 +produced +0.21669501862738988 +0.09686264127920025 +online +0.24030376707510714 +0.08073714693036847 +lairxentcom +0.02782503104564944 +0.2258665960698562 +two +0.12226002483651953 +0.16136461867452814 +phoboslabsnetnoteinccom +0.12226002483651956 +0.16136461867452812 +image +0.28752126397054206 +0.04848615823270436 +same +0.09865127638880213 +0.17749011302336035 +invoked +0.051433779493366895 +0.20974110172102423 +phone +0.24030376707510678 +0.08073714693036854 +div +0.28752126397054206 +0.03236066388387246 +look +0.09865127638880204 +0.1613646186745282 +users +0.051433779493366874 +0.19361560737219233 +firewall +0.287521263970542 +0.03236066388387246 +again +0.09865127638880186 +0.16136461867452803 +available +0.1930862701796722 +0.09686264127920034 +rohit +0.02782503104564941 +0.20974110172102423 +security +0.1458687732842371 +0.12911362997686443 +xbarracudaspamreport +0.31113001241825977 +0.016235169535040443 +without +0.14586877328423709 +0.12911362997686446 +autolearnham +0.027825031045649408 +0.20974110172102423 +old +0.12226002483651947 +0.1452391243256963 +form +0.24030376707510678 +0.06461165258153641 +19216818251 +0.31113001241825977 +0.016235169535040454 +mimeole +0.21669501862738985 +0.08073714693036844 +encodingutf8 +0.02782503104564941 +0.20974110172102423 +26 +0.2403037670751072 +0.06461165258153644 +want +0.14586877328423714 +0.12911362997686435 +over +0.21669501862738996 +0.08073714693036854 +bgcolorffffff +0.28752126397054206 +0.03236066388387245 +helvetica +0.26391251552282463 +0.04848615823270433 +sat +0.09865127638880207 +0.16136461867452812 +contentdisposition +0.05143377949336689 +0.19361560737219227 +way +0.14586877328423709 +0.1291136299768644 +send +0.19308627017967225 +0.0968626412792003 +m0620212mailcsminingorg +0.3111300124182597 +0.016235169535040457 +mimehtmlonly +0.31113001241825977 +0.016235169535040443 +change +0.16947752173195446 +0.1129881356280323 +contains +0.21669501862738985 +0.08073714693036843 +read +0.16947752173195446 +0.11298813562803231 +listsdebianorg +0.02782503104564941 +0.20974110172102423 +antispam +0.3111300124182597 +0.016235169535040454 +u +0.1222600248365194 +0.14523912432569636 +version250cvs +0.027825031045649415 +0.20974110172102428 +off +0.16947752173195446 +0.11298813562803228 +9 +0.16947752173195443 +0.1129881356280323 +even +0.09865127638880185 +0.16136461867452803 +keep +0.12226002483651947 +0.14523912432569636 +img +0.28752126397054195 +0.03236066388387247 +problem +0.12226002483651952 +0.14523912432569633 +zzzzlocalhost +0.1694775217319544 +0.0968626412792003 +client +0.051433779493366916 +0.1774901130233603 +httpequivcontenttype +0.28752126397054206 +0.016235169535040447 +set +0.07504252794108442 +0.1613646186745281 +software +0.14586877328423714 +0.11298813562803232 +httpxentcompipermailfork +0.05143377949336687 +0.17749011302336035 +better +0.07504252794108444 +0.16136461867452817 +file +0.05143377949336693 +0.17749011302336032 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_0.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_0.txt new file mode 100644 index 00000000..87e48e71 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_0.txt @@ -0,0 +1,301 @@ +httpmemberscoxnetnickwareblogmyjournaljournalhtml +franklin +down +target3dblankhttpwwwamazoncommartinfkraffteb001k8 +copyright +reidacbellatlanticnet +headline +for +postfix +database +2139621475 +smtp +2020 +monitor +involved +raid +window +this +in +certificate +parameter +have +returnpath +mailtodebianuserlistsdebianorg +are +is +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +101431608 +testsbayes002 +0l0v00900a6t7900amtaout20012netil +00000000 +085859 +subject +hard +pgp +224823 +relay +qmqp +40 +50887440f0 +union +httpwwwfreefallblogit +2002 +offspring +want +seemingly +forecast +email +listunsubscribe +yes +httplistsdebianorg20100517211206ga5127spinnennetzorg +15 +but +0000 +9 +jmlocalhost +security +correctly +what +would +reporter +spamphrase0001useragentxacceptlang +33 +spoilsports +ist +fourla01 +good +apr +012034 +big +fallback +our +333dd348e71 +simutrans +thu +normal +tue +domain +se2uq2oey45p +had +entry +look +music +audio +3350c2daef586c1130b21108ade5bd88 +types383110lenny1sparcdeb +idea +undue +just +else +t5cfd28840dac1d1004129mailgatewaycursorsystemcom +it +sun +xoriginaldate +esmtp +bhandspring +listsdebianuserlisztdebianorg +blackcombpanasascom +local +a +intense +movie +newer +deliveredto +office +resentsender +213446 +wrote +96c6e43f99 +localhost +msquadratnospampleasegmxnet +as +dogmaslashnullorg +inline +modelname +spfpass +2200 +has +messageid +matt +pay +10 +wwg30 +phoboslabsnetnoteinccom +0700 +lickthruonlinecomclickq3da26m7qxzhfzgbk9r0cgkhtsonr +103726 +020023 +config +derisively +httpxentcommailmanlistinfofork +or +64286773 +resentfrom +uswsflist2sourceforgenet +nonmember +2007091301 +resulted +contenttransferencoding +required50 +way +last +building +1o9rgx00068j7w +gardner +textplain +127001 +whatever +eff +muppet +that +approachable +listsdebianorg +gmt +to +score100 +6278128237 +cve20100842 +took +singledrop +xlevel +now +20080610 +child +returned +employed +most +help +resentdate +inreplyto +believe +references +there +found +6 +mysqlservercore51 +at +1 +id +destar +reuters +cardspcmsurround51 +modules +listhelp +on +same +black +yyyylocalhostexamplecom +92d352feab +complete +my +051037 +xmsmailpriority +dates +date +deltacoepsuacth +lack +utf8qxcmcwbrvs0a09py2lnmmz2fmiourzprez4ltpu6qolya6g9aw0krjlajcsg +googlecom +mail +by +105217 +en +think +lisztdebianorg +new +005706 +xbeenthere +specific +not +listsubscribe +with +modems +from +enht +brighton +090008 +express +httpwwwfsmetrocom +media +3rr +thunderbird20022 +listid +fetchmail590 +available +lucamerciadristudentulgacbe +permitted +precedence +message +didnt +errorsto +7srvewx1xhoztflmkfva60cbohhxkahleizg3idtvkmmvv4wlqwvy97ssxs +mailto0xdeadbeefpettingzoonet +contentdescription +7bit +archivelatest577436 +box +be +listpost +shape +xpolicydweight +videos +static10410bluelinemg +nf7 +exmhuserslistmanredhatcom +jogging +gfooterpublisherhover +a01 +1712811333 +install +7 +bgcolor999999img +type +catchup +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +s30abr +ridiculous +mirror +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +discount +ytrl1hexycwi +archivelatest576492 +example +200210070800g9780ek23245dogmaslashnullorg +3 +productivity +stable +no +m8cs44524wfo +sfnet +requests +router +games +score84 +210012 +mydomain +policy +version325 +jhstuckeycsminingorg +contenttype +possible +of +settings +ive +01 +and +received +one +desire +wed +posted +list +nt +pgpsignature +213164145162 +s0832016219710167 +commycompanymyproductx +search +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_1.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_1.txt new file mode 100644 index 00000000..40165f72 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_1.txt @@ -0,0 +1,374 @@ +11pxtips +ffffffh3 +please +read +5px +rcvdnumerichelo +for +postfix +httpequivlastmodified +d97n +business +lunchboxes +smtp +xbarracudaspamstatus +carriers10by +test +worms +feel +hwy +xmlnshttpwwww3org1999xhtml +body +years +name +0staackkgxxadijewa9sxfbqvzu4hhjl6b1e2ladbafraccpfrkvos1w71wpqdbrssv99sxaclj +colspan2img +this +in +italicisi +2435021 +your +returnpath +imap +0001 +arthur +are +mon +monobr +learn +why +rd +xasgdebugid +bgcolorffffff +19 +too +manage +president +account2 +12b9a4251 +213105180140 +aids +said +fam +subject +christianity +hostname +using +produced +system +flooding +included +2002 +three +fontsize +email +yes +bsfsc5mj1963 +contenttexthtml +724603 +color3d0 +poorer +police +webmasteregwcom +if +project +clinical +deliver +description +bsfsc0sa392f +big +tag +process +mailtospamassassintalkrequestlistssourceforgenetsubjecthelp +differencethe +color00b050bfont +notification +our +xbarracudaconnect +250 +normal +keep +p51ds +friend +all +head +unnecessarily +had +bec97u115e +height3d39font +06i9307j9e5bsczrgmzemaunqjt9551e8b5131bobe +cellspacing0 +0400 +aerospace +before +replaced +acc111117nt +track +immediate +left +vgxfizoxp2nxkosyakryj6oxj6ow6cqxkotx6wyw8zd4m7g4dxo5enl8ufj8obi7s9u7r9e3q +score +site +united +dollar +esmtp +24 +a +10pxtrtd +dont +deliveredto +office +aboutu +xmozillastatus2 +hibodycsminingorg +020441 +xbitdefenderwksspamstamp +barracuda +localhost +serif +qnhbmujpg603801fa38184244 +size3d2b8212 +as +internets +build +div +rule +score1120 +cexcept +vebin +messageid +500 +ska +list20 +cellpadding0 +mime +monasteries +cellspacing3d1 +or +name3dchoicefont +allanktocsminingorg +color +contact +010 +qld +xbarracudaurl +contenttransferencoding +mmpl +href3dhttpmkkzacapalolabizpo +section +building +mimeole +i +127001 +outlook +bsmtpd +helvetica +communications +bulgaria +value60000000600000option +concern +to +sep +miss +charsetiso88591 +singledrop +industrialized +now +9xml +browsera +code +october +you +gigt +httpspamgwcsminingorg8000cgibinmarkcgi +td +jun +someones +maximum +bgcolor3dffffff +mathews +font +most +cult +internet +was +version +brother +httppgexqumloafcom +change +îõñóã +4033 +htmlfontbig +msgidfrommtaheader2 +nonencoded +1 +replyto +300 +added +low +titlehealthmediatitle +id +meta +largest +rdns +25 +mar +nextpart00000236d01c9d96ead688cd2 +click +individual +sa424 +xmsmailpriority +g8jmtyc06196 +date +viagra +centerit +xbarracudastarttime +invalid +12px +font20 +xbarracudaspamreport +international +erie +me +forgot +newly +mail +house +by +unsubscribe +initiates +antispam +actually +height360 +valigncenter +0100 +p +static7197122240dllstxdslwverizonnet +alignrighta +trusted +replays +with +from +texfq1rjt05tdqonckltu1vfuw0kdqpgzwrlcmf0aw9udqoncm1lzglhc3rpbmfsdqoncmfudgli +fixed +0 +unlike +100agger +mimeversion +scores +sara +welc111ming +bccccdukclhqblgbxmcoqh2s9arwqfu9a48wgapowmadfvastoi6sgpuk2o9aarfewms5xl2to0 +gw2csminingorg +drugserectile +103057 +bucks +include +validity +media +killlevel10000 +width652 +aligncenter +den +xmozillastatus +copy +valuebwbotswanaoption +victory +exclusivity +xaccountkey +sender +junk +its +which +mitten +02title +say +place +zimbabwe +7bit +a66d516b53 +pr0nsubject +thesestrong +ansserif +be +eoe +respect +poems +large +goes +numeric +xauthenticationwarning +hrefhttpa62hardsupplyruriseyhyo8efdae71f7eee +utilize +texthtml +rules +breastsreturn +latest +xvirusscanned +microsoft +passionspantd +a2c +annually +resend +type +html +81168116 +matches +activity +fri +zie3pwsbiluv6xjrdh3xfcv3y81hndenptnyssspyuzpfduffcgkdg57io5gbj5izt81g4zvnihi +adventure +promise +many +013546 +class3dbold4presidential +width100 +3 +no +we +having +anavlinkonlink +geneva +corporation +designated +webmillion +jst +sty +contenttype +xmozillakeys +185743 +bsfsc2sa154 +volume +practice +crossing +centertdtr +listed +feb +of +dynamiclooking +csminingorg +and +width89 +fontfamilytahoma +19216834 +received +one +30b2143f99 +peast +niederhaslich +posted +name3dphone +h2khomeinih2 +table +sufferers +lang3d0s +leave +www +addresses +exgkylbgqkrrhqeafbbkgqnw1ghqtk0eogzqucbjbsgkambq8ikikg1ugdexgrcqswajechqw +sell +the +176 +size1if +xpriority +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_10.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_10.txt new file mode 100644 index 00000000..c2493f78 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_10.txt @@ -0,0 +1,376 @@ +deprivation +composes +penal +e +xasgorigsubj +please +trouble +lookup +number +xmailer +able +reserved +for +width100trtd +program +smtp +xbarracudaspamstatus +span +butter3dgrocer +colorblack +payments +body +residing +years +name +this +environmentalism20 +in +myself +your +elizabeth +returnpath +suspicious +point +0001 +breakdown +is +nights +xasgdebugid +mais111ns +0l017o6f504755z32vosiza3d34826415097206834511 +among +cmvtb3zpbmcgc3lzdgvtigf0ihdlynnpdguudqo8cd4ncjwvahrtbd4ncg0k +100 +account2 +00000000 +militia +mailtoadsub68btamailnetcnsubjectremove +waiti +houseis +bodies +subject +border3d1 +xbarracudaspamflag +given +using +1700 +routes +listarchive +system +leftmargin0 +included +want +avenue +size3d2bphonebfonttd +msgidfrommtaid +quotedprintable +shbig505413body104 +http61145116186user0201indexaspafftqm10 +size3d221223ebu +11 +israel +below +best +unable +similar +another +ns2csminingorg +re115t +xsmall +s3wul4rjqofkdbzdhdtzxxnb005aaaaaacz4rk5scnzbyevfjxom1zuweobxsgansgaaeoxln4i +weeks +description +saying +rdnsnone +stylecolor666666 +delivered +10pxunsubscribefonta +statementabr +thenlongest +dreams +our +quarterly +per +xbarracudaconnect +do +interrupted +yugoslav +tue +empires +waterstruckcom +jbwvz8h86ujhbkiwxthwuaxhibgvggp9yskaoawbx8ggsvtgcau3cb2hkopfiavxscqokoig0kkf +pyesnbspnbsp +size3d2fontnbspdiv +cd +size3d3estee +oursbut +journalconstitution +size3d2helps +img +xmailerpresent +nor +cellspacing0 +bspace +it +site +hidden +wealthy +esmtp +quoth3 +colorado +preferiti +u3si2654471fan8020100513023416 +d512e47cc2 +a +syase +width50 +omega +deliveredto +decades +xmozillastatus2 +8226 +timepiece +hibodycsminingorg +336699 +described +arrest +therapeuticp +vojvodina +as +dogmaslashnullorg +rule +height15font +n3dcenter +experienced +has +messageid +350 +10 +atd +selected +6xygpgge1vcktlrsblopphvwt3pxhwddylmewwlxppfakyulspkycqbasmxqeaqo6acacacaca +risk +addressspanfontptd +itsdzafcyvuug7de3w0tv6crgqnc7jf5v8v3wav7fqu0qyrwlbgsaa6dqrhyqqtznisdscmctz +linked +xbarracudaspamscore +titlewhat +fjo171internetdsltpnetpl +size3d2bdiagnose +letters +residents +6521715966 +greece +doctype +hrefhttpb7d4medjodie41arupiuecyajy57e58291bff4741unsubscribeap +colorfffffftextdecorationnonelistenspanatd +southernoption +such +medium +peruser +secondary +contenttransferencoding +ordered +193120211219 +ssr +xbitdefenderwksspam +napoleonic +size1complete +service +outlook +helvetica +19412514545 +that +wzwjt +styletextalign +quarantinelevel10000 +201004161345o3gdjgso014964gw2csminingorg +to +targetblankabout +exact +innerworkings +charsetiso88591 +national +97b115111rb115 +2pxfont +technology +1246729415419f002d0000w4twrl +pills +32 +khgh8az161uznhjpnzag449atbdmz5nwocf8aqydamas6xhq4wchkhgh8az16wgjgq +carbonate +keeping +share +name3dprogidhead +recently +most +tornadoes +these +was +version +44yesonlineisbestcom +doing +at +replyto +profitable +been +id +meta +mandarklabsnetnoteinccom +wrap +over +uwhfqjxjor4gp2vah2kskkseodgkoff5npjhxlvxv1m01yal9l8391527zimtrs9evsvyv8ra +height3d28aspanp +on +123947 +25 +xoriginalarrivaltime +htmlmessage +bail +ulasabilirsiniz +government +complete +click +appears +after +usf2mtouqsc0mtdchtmhx0monvrkeh2lnzqfin8o7jxutjvqcwrp9rl72xefwxkoz9rst2hspt +zuckmail +date +spamgwcsminingorg +viagra +espialbbr +border0td +invalid +96 +boiling +marketing +2010 +sales +zo +valuable +naturalization +xuidl +holistically +261138529 +circuit +astrongbr +mail +4fb0caa2c3294c20b331229e681acee3 +by +status +httpspamgwcsminingorg8000cgimodmarkcgi +htmlheadtitlemilfhuntertitle +default +will +link +width700 +style3dheight60pt +while +criticized +park +15319656114 +not +2693 +with +from +approvalsbefore +0cecf2c3e1 +width1 +incidentes +subscribed +mimeversion +scores +0099ffto +size3d2moodys +easily +target3dblankuxey +210 +airport +000 +2 +true +pc9odg1spg +hrefhttp0c7epharmwoodman44aruiaabiaefd3aec1955a60 +aligncenter +deterrence +xmozillastatus +carfinancing +httpviviacomuam +size3d2bemailbfonttd +philosophical +message +diduniversity +face3dtimes +american +xkeywords +020 +282175050 +be +meet +dialing +shows +considered +account +fallout +httpequivcontenttype +church +nb5a6u6nywa6z6nfetv8sfekp3gwcauorf1nbfl8t7nysupirjjgxa4zwftwhly75gob5c49ti +more +microsoft +an20 +130 +busy +html +billing +nnggttffexceptionoccasionallyi1planningi2accordingi3joi4teami5endedt1gonnat2aprt3analysingstuffshareholderspaintackthingsinterviewstownleadercodesproductionscountorganizermovedpetsdismissal +tr20 +none +nw +save +xoriginalto +aaaahdaqa3osbs6ghu6buy0xlgt48vza9egoorsjuk6bzt8crboonoyg6cnj6zasak9q3gixp4a +width234 +0900 +uppercase +no +very +ladies +depend115 +confrontation +introduced +great +jst +contenttype +ouspe1cfb2ch6tpbgf0rjdlztrofvyfuvam1xi6idhfnbcctx5oolfu2xbnysnf0bq5cazmiadl +cellpadding0trtd +input +sold +may +listed +of +sum +csminingorg +cassie +and +fa +received +cpunkslocalhost +manoharan +cellpadding3d2 +list +phone +dotteddecimal +table +namethe +drugs +073026 +a100resse +sa154a +madam +sell +the +stylefontfamilyverdanafontsize13pxcolorffffffwhitespacenowrapbnbspnbsphealth +25294cablelink4intercablenet +religious +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_11.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_11.txt new file mode 100644 index 00000000..eeb04ee4 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_11.txt @@ -0,0 +1,261 @@ +legacy +trouble +for +postfix +drive +mailtodebianuserrequestlistsdebianorgsubjecthelp +httpsecuritydebianorgpoolupdatesmainppostgresql83postgresqlpltcl8383110lenny1sparcdeb +find +coords +works +20100408100612gc20324wastelandhomelinuxnet +181255 +g8he0kc12569 +name +account5 +instead +in +cached +have +returnpath +imap +touchscreen +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +score140 +course +somewhat +23 +085419 +server +subject +path +8ccd216f73 +provider +filemaker +listarchive +system +xenigmailversion +2002 +imo +110841 +bulk +micalgpgpsha1 +listunsubscribe +yes +but +obligated +jmlocalhost +thompson +what +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +event +forkxentcom +generation +if +prescription +browser +versions +big +59de413a57f2 +fallback +unix +tue +domain +book +previous +50 +folks +dv6rlxqjrlvw +innoventive +run +enter +0400 +clean +required40 +hearing +it +sun +esmtp +httpxentcompipermailfork +listsdebianuserlisztdebianorg +local +a +dont +deliveredto +xmozillastatus2 +g8nkuut20028 +2011 +dogmaslashnullorg +has +messageid +signature +10 +trade +ilug +jeremytech4learningcom +unknown +linux2628grml +merciadriluca +143155 +httpradioaiesecwsusers0000023 +width348img +doable +interested +unecessarily +contact +garden +command +null +they +ganz +provide +far +contenttransferencoding +required50 +way +forkadminxentcom +031352 +open +textplain +i +127001 +outlook +that +ted +listsdebianorg +gmt +to +singledrop +utc +meyerhans +20100512104812843bssiguanasuicidenet +edit +38bf813a53eb +continuing +8d74813a624f +6 +20002010 +personal +at +1 +debianuserlistsdebianorg +id +211531 +someone +httpfreshmeatnetprojectskemerge +also +port +listhelp +on +mailtoexmhworkersspamassassintaintorg +languages +25 +happy +1mb +relate +14 +worth +date +v149 +i2hbeee84cb1004291703h52e06954nfc99eaeabf3ad471mailcsminingorg +69 +httpsexamplesourceforgenetlistslistinforazorusers +8121811324 +2010 +43094764353 +exmhworkers +razoruserslistssourceforgenet +l +envelopefrom +by +en +unsubscribe +81018101 +listmasterlistsdebianorg +curious +2525 +xbeenthere +much +0100 +m8cs30035wfj +not +listsubscribe +with +from +0 +santa +mimeversion +0500 +enht +gets +cb01 +give +whe +2 +media +afternoon +height1 +pass +xmozillastatus +mode +make +xaccountkey +regards +sender +permitted +precedence +its +errorsto +an +160 +ill +taggedabove10000 +aug +be +listpost +instability +into +longer +matter +texthtml +151335 +anything +kept +hello +behavior +7 +205512 +type3dapplicationxshockwaveflash +083327 +really +imao +sgamma +except +iso88591qivborw0kggoaaaansuheugaaadaaaaawbamaaacllos0aaaamfbmveusk +debianuserrequestlistsdebianorg +add +writes +let +major +intermail +harder +contenttype +xmozillakeys +helpunsubscribeupdate +months +xrcvirus +may +of +13913683 +01 +and +record +received +without +version241cvs +later +dsa1992 +the +notificationsgroupsmsncom +172164831 +ugly diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_12.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_12.txt new file mode 100644 index 00000000..0349b55e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_12.txt @@ -0,0 +1,333 @@ +california +tbody +xasgorigsubj +del +style3dcolor00125ehome +host13christianwebhostcom +xmailer +for +postfix +214644 +srchttpwwwsalealertscomdotgif +width3d +lots +smtp +bsfsc0satofromaddrmatchhl +centerwritten +vigilancia +begin +series +website +semifinals +city +alt3dgenerate +content3dtexthtml +women +firewall +have +your +align3dcenter20 +returnpath +1598 +mon +backup +moon +11px +show +aw5lcw0kdqpjb250cmlidxrlza0kdqpjb2xszwn0aw9udqoncnb1ymxpy2f0aw9udqoncm1hbnvz +bestsellers +subject +hard +try +nobody +relay +using +40 +system +hrefmailtonothanks3321btamailnetcn20nothanks3321btamailnetcnunsubscribea +2002 +030224 +quotedprintable +rural +shbig505413body104 +jmjmasonorg +srchttp62232161231freeinfoimagesrightsidestuffjpgp +substantial +135009 +below +future +dhrwoi8vz3jvdxbzlnlhag9vlmnvbs9ncm91cc94aw54aw5yzw4vzmlszxmvd2hfc21hbgxfyi5n +incredible +fusce +what +different +would +style +recharge +ist +world +replacement +description +big +httpwwwadclickwspcfmo383spk1 +thep +xbarracudaconnect +normal +urncontentclassesmessage +head +rbl +size3d2fontnbspdiv +maa21231 +host +img +bethea +xmailerpresent +receivedspf +enter +msgidfrommtaheader +cellspacing0 +full +xmimeole +222623 +edt +wrestlers +stylefontsize15pxpadding3px +esmtp +cable +styleborderright +893893 +ein +mx +a +nametopspan +deliveredto +g6nakg410709 +office +hibodycsminingorg +35 +barracuda +dtl +localhost +hrefhttpbgaveanswerrujyqux05529f21cefoofa282390033f60 +slim +clientip6412025122 +div +mimehtmlonly +has +messageid +cqkrfxwrzjsfrknxgkllbqfoh1bqjxka38ladm0shgtjg8jsjcqvbiezviwps8citbqmikhcji4 +goal +industry +grabs +10 +montreux +mime +tao20 +yours +text +verdana +phoboslabsnetnoteinccom +0700 +men +account4 +holdover +divnbspdivbodyhtml +doctype +main +ased +come +2391 +010 +peruser +taglevel10000 +anb +193120211219 +multipartalternative +reply +size3d2td +service +divided +1023139131 +textplain +i +marques74csminingorg +127001 +radius +color3dcc3300span +accession +uwiri +vizopprusexcitecom +quarantinelevel10000 +scheduled +to +jews +115ur10297ce115 +singledrop +19216832 +19018241104 +hrefhttp6623113368finalindexremoovhtmlid1512clicking +you +know +kimonos +sachiko +presented +around +minutes +niet +bgcolor3dffffff +enables +indicatesdiv +share +consultation +font +most +fat +trtbodytable +mxgooglecom +founding +6 +block +cyberlink +color3dff0000293fontfontdivtdtr +n9j3h2ij019246 +id +meta +dollars +surrendered +pts +rdns +pickup +also +inclination +texttransform +clientip1894866120 +updates +click +how +bclaim +183453 +xmsmailpriority +after +date +dqo8uefsqu0gtkfnrt0itw92awuiifzbtfvfpsjodhrwoi8vd3d3lm1sbxjlcg9ydgvylmnv +xbarracudastarttime +23c2fa3enbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3b +abr +134714 +ehopqmjk +8zsjxsipreuwjajoca4uunhissjdbuenzwgjaho2nbsk3l8klk520ekv9lasbfwrde9pzczycdsd +xuidl +trailer +house +1909 +scene +by +hrefhttpdiscountrxscombfont +httpspamgwcsminingorg8000cgimodmarkcgi +suffit +abolished +antispam +002629 +stateprovince +new +httpwwwjabbercomosdnxim +few +wonder +p +att +not +c111ncerne +bordertop +enormous +with +from +team +person +significant +16px +000 +ordering +killlevel10000 +value3dreset +xmozillastatus +httpwwwbarracudacomreputationip6224045211 +could +charsetutf8 +words +xaccountkey +sender +message +phoboslabsspamassassintaintorg +178141255exhjdznb3dxldcmxtqrrktjy +which +cellpadding20 +060 +altthe +spanbspa +herea +ill +55265555 +rated +discussion +be +shows +gw2csminingorg19216818251 +extensively +9cvgojwuosxvlwgxxcp7t9j3s8oie63k6ua5rzksa2osipxczlkzvtvhwjkmllzlgbvgqupkvrt +texthtml +newsafonttd +changing +charsetwindows1252 +srchttpc2fbpoxokuwcnspacergif +wwe +1000s +lewiskenspluscom +bsfsc2sa154a +httpequiv3dcontenttype +sp +kill +iiuadminzzzzasonorg +current +fri +federation +preparation +nbspfontfont +xoriginalto +royal +lines +value3dboth +3 +imre +we +let +top +content3dmshtml +past +k +white +does +jst +contenttype +xmozillakeys +practice +like +of +received +affiliated +cpunkslocalhost +today +roofing +ac4amaaaaaeaoejjtqqgdepqrucguxvhbgl0eqaaaaahaaaaaaabaqd7gaoqwrvymuaziaaaaab +ever +wires +accounts +list +powerfulnbsp3b +holding +booking +shipping +rowspan1 +21doctype +the +parts +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_13.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_13.txt new file mode 100644 index 00000000..072ac166 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_13.txt @@ -0,0 +1,323 @@ +diabetics +recipient +xasgorigsubj +lineheight150nbsp +outsize +contentworld +imprisonment +bordered +please +number +for +postfix +smtp +xbarracudaspamstatus +bsfsc5sa269 +method +regulators +series +this +paul +400 +airpurifying +in +your +bayes +returnpath +jul +style3dfontsize100ptfontfamilyarialcolorblackdefend +0001 +centerbr +why +xasgdebugid +companies +19 +basarili +president +andh5 +account2 +11px +said +c0jyphqyxnaisufafwbntf8ahqwypklhp8a05agdetszziiob0iqvniykerkehpoxozyz +subject +besiel +cipheredhdssdescbc3sha +fontbfont +produced +1028311679886057142 +namegeneratorbase +disadvantage +2002 +31108178 +state +0800 +best +zgqamgdijq8hogagoazieaaobg50kinaqa6tdgoeixnctklwiqasgpmjwdcrmwokuoacwiqjgwq +konnen +004155 +asy100as122solsuperonlinecom +162002 +another +seibert +reconstruction +ist +victor +description +xraymedia +european +indulged +14px +so +250 +must +normal +coast +play +13d4oz6tvw7k6s8fhzkf57psaxus6qnzvwje7h2omwvhxony2hkuvd9kfvqdtb6e7vz9 +a2fc34fc1 +all +him +qualified +br2006 +sa161f +aggression +traveling +music +else +esmtp +neues +fix +24 +zhejiang +boundarypart557010a3c21c38008345dc768 +technology22 +k2biztld +featured +threatens +information +provides +accepted +super +became +httpwwwlotsonetcomopportunity +10016 +principle +prince +forward +5501 +dogmaslashnullorg +build +inline +div +score1120 +store +instantly +messageid +year +zzzzlocalhostspamassassintaintorg +iplaneti +10 +atd +trade +continued +preservative +xbarracudaspamscore +account4 +residents +greece +doctype +color3d0000ff +peruser +they +81128112 +contenttransferencoding +thi115 +way +im +brand +127001 +pt0gt25jzsb5b3ugagf2zsbyzwnlaxzlzcaxmdagb3igbw9yzsbvcmrlcnmg +outlook +8bit +width535 +that +out +border3d0 +quarantinelevel10000 +nad9tdkn024045 +188498731 +reprehensible +to +rif +iberian +094917 +charsetiso88591 +singledrop +code +you +hrefhttp2ffree3hostnetsite5takemeoffhtmlclick +td +32 +watches +commentators +company +titles +consequence +height83 +acquired +eentta1bxs7o0imfsiscsgxlhj5q3tnia20baadb7ubaadbrtoatbsffagbhulmzdqbe0zpam +these +week +publications +altview +addition +at +size3d2 +been +cover +often +added +boldheadlinereader +id +meta +north +harte +mandarklabsnetnoteinccom +rdns +92275a +jegogyuunu +need +g3cs42694ibe +external +tome +how +href3dhttp4 +after +rankings +date +width548 +trading +bgcolor2fa6fdtd +ship +xbarracudastarttime +31 +alignleft +neiman +marketing +value42500000425000option +noreply +third +paid +bereitete +duduzy8njrvfzywoq1ksyoxck8umbydejmkzuqczqrtxsad1qf0jbpzng5boykjnfwk6kocdbriw +by +kiucehum +color3d23ffffff +assigned +goods +032 +will +february +050 +zvzsduiynsvr8jhwrobk1mtrtbwcqxifwu3fpsxchdy1m6woc2sacf1f6ywpmcbjefasm +0100 +border3d0abr +src3ddottedlinegif +with +from +target3dblanksupport +mimeversion +jmiqmdhaiqwwpaiewm4asakkrgcoagbzmd5n3tec1tfg2dhclr8inaqxmib8ardlfhaztrwf5hbi +fromlocalnovowel +programmable +173328 +210 +colorffcc33 +000 +media +knowledge +paginated +faceverdana +sa275bhl +xaccountkey +xrovingqueued +extra +sender +position +message +murdered +factors +property +color3dcc3366b350bfontfontdiv +ffffff +concerning +hrefhttp90edrugsfelice75aruasituguky220db0370c8 +offer +herea +7bit +courts +targetblank +be +align3drightfont +hrefhttp019tabjamie39aruijetulu37cf70666f9d63 +gw2csminingorg19216818251 +sblxblspamhaus +5075 +httpequivcontenttype +rdnsdynamic +texthtml +rules +time +were +xvirusscanned +httpwwwfacebookcomninboxreadmessagephp +more +case +usesbrnbsp +xbarracudavirusscanned +color3dffffff +064 +m97y +name3dhdnsubjecttxt +etm47hugsrvb6dlvzd62pv8aeovsrdszj1dpvho0nv9oxssx6cx3s3pfveyv2w4qmpt9 +style3dfontfamilyar +n82cbexf025552 +exteriors +start +pro +muscle +messaggi111 +7aoa5k4w6ygnk81yulqxw4c7llc4ro7qfcadxkzrbtoyvqadtcvhiqzwfpjxgqpxqwk8uvwrfu +styleborder0px +204 +3 +made +no +0pxnbspp +1725 +moved +don +textalign +paris +stylecolor075c83for +contenttype +xmozillakeys +reservedp +fromem +of +born +humanitarian +received +today +mkyzau0uyco9sdbtlgcstzlxud3pjs0oqxs43z6guqt4qjwed5pccj865xwvbrxora5ps4ws24k +28 +table +spamassassinsightingslistssourceforgenet +went +cams +chinese +9bc2e5 +the +send +parts +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_14.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_14.txt new file mode 100644 index 00000000..bfa9bd9d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_14.txt @@ -0,0 +1,269 @@ +hrefhttpwwwhandangocomplatformproductdetailjspplatformid1ampproducttype2ampsectionid0ampproductid43086ampcatalog1ampsiteid159 +g9881mk06178 +rulenotspam +xxrulenamecfrpmnew +packs +trouble +qmail +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +fep070kolumbusfi +hrefhttpwwwtvcdbizlockergnomewin +working +any +asus +markallumscom +score109 +xmessageflag +5d903d5e3342 +court +account5 +this +in +spamhalf +065659 +firewall +your +heard +returnpath +9zbdbzycefea +z4zyzv6oetcasfw3c1lbliszt +stays +are +is +mon +pinelnx43302081120465503981100000hydrogenleitlorg +recharging +23 +00000000 +mozilla50 +show +085910 +volt +server +subject +try +xserverxorgvideointel +soon +qmqp +size +curve +email +listunsubscribe +craved +but +sure +0000 +singaporebabr +jmlocalhost +url +g8u81eg21347 +what +visible +would +212173515 +ist +if +good +oct +big +excuse +so +thu +short +c2luzxnzienbltewgz8wdqyjkozihvcnaqebbqadgy0amigjaogbam4vgbwxt3fek6lfwg0xtzqa +thanks +run +receivedspf +evans +102344 +analysts +325 +aanlktik15zse1o9wxljtxnarlkolyemjtd8t6bdmrenmailcsminingorg +it +esmtp +protocolapplicationpgpsignature +listsdebianuserlisztdebianorg +a +iso88591qyn8dhahzj0a093iv3zcsgnbqnfxvvflphgh4nlmroplypdpwxcgbzkb +deliveredto +xmozillastatus2 +align3dcenterfont +wrote +localhost +though +tools +as +forces +because +ruling +5119262 +scott +pdt +5 +messageid +signature +yasser +mom +conflicts +cron +public +come +laptop +they +xrcspam +width1br +contenttransferencoding +21 +last +2008110401 +im +checking +stop +textplain +127001 +that +destinyfontabr +to +sep +issue +20100512161615626bssiguanasuicidenet +20080610 +astrophsics +you +technology +tdtr +bank +village +amavisdnew +httplistsdebianorg4be80a267030407homenl +dbaron012netil +these +mlsubscribertechcsminingorg +zlinuxmanwowwaycom +inreplyto +long +references +1 +replyto +xsender +id +port +listhelp +forkexamplecom +on +general +same +fixing +my +after +date +jalapeno +1574044036651321285627122481190clientattbicom +cohen +hope +width12img +tlsv1descbc3sha168 +mail +size1font +ter48imagesnowplayingbuttonmoreogif1img +condon +by +unsubscribe +201224410520090324 +should +redhat72i386basepkglistfreshrpms +ldowhitelist5 +xbeenthere +0100 +xamavisstatus +exmhworkersadminredhatcom +feed +with +from +20 +f1ac682321080de637e1fab2bba +mimeversion +builtin +configuration +000 +xcheckerversion +hrefhttpwwwgnometomescomtome001843htmlwrite +pass +james +make +put +jmasonorg +charsetusascii +automating +end +precedence +mennens +quot +errorsto +tree +163037 +stay +american +classifier +subscriptionsanbspnbspa +cc +xbrightmailtracker +listpost +into +flash +theyre +time +were +xvirusscanned +mkfsxfs +part +welch +case +ns2egwnnet +sort +html +responsive +patch +stephane +mailgatesricom +height121 +fri +claimed +about +224842 +many +kcjnea4sauqyfbnh3rbsk2flflg6bhd3974jqv73ms +see +nnfmp +quotmount +towed +fs +rescan +capslock +white +together +6226461461273890328cardhudecombobulatorblackberryrimnet429594017bda030bisxprodaponblackberry +when +policy +great +mp3 +contenttype +opinionist +sid +like +of +received +6915fd18fda +without +wed +free +list +table +174454 +unify +invictus +product +the +decline diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_15.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_15.txt new file mode 100644 index 00000000..b6bd4709 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_15.txt @@ -0,0 +1,271 @@ +strasbourgfrance +rate +m8cs56293wfj +dd2d413a464f +libgs8 +e +solitary +20080517 +16b43e80a447a185f5372372836104ed +015849 +trouble +oldreturnpath +xmailer +for +postfix +147 +101426910 +network +maybe +xyahooprofile +formatflowed +aes256sha256 +arsasha1 +wasnt +resentmessageid +name +clientip8219575100 +7l4hs9r3bjsmvkpv3dxnr +hadi +cached +preroll +color3dfffff +returnpath +teledynamics +are +is +101431608 +feature +width3d600 +everyone +relying +empty +subject +path +qmqp +using +dad +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +104245 +2002 +salmon +email +meeting +bgcolor3d999999img +yes +zenity +0000 +1002627 +debian +9 +url +another +swinglerapplecom +applicationpgpsignature +kohn +if +good +apr +mayadyndnsorg +073317 +generate +so +per +standards +keep +prayer +httpslistmanspamassassintaintorgmailmanprivateexmhworkers +advertise +u +50 +had +ssl +0400 +clean +required40 +us +site +esmtp +option +httpxentcompipermailfork +hits107 +multidisciplinary +awesome +entire +a +titleecobuilder +collections +deliveredto +wrote +localhost +tears +as +launchpad +reasonable +getservbyname +pdt +messageid +year +pay +popescu +deck +20100507122110263bssiguanasuicidenet +dkimneutral +certain +xspamstatus +ml1430 +g12ec5820412 +or +seems +m8cs257261wfj +strong +contact +xsourceip +0200 +contenttransferencoding +dayan +way +href3dhttpummail4unitedmediacom80clickq3d9f +2008110401 +citizens +dire +textplain +i +127001 +libs +that +out +gcc +listsdebianorg +to +tons +height10 +ratwaregeckobuild +charsetiso88591 +base +ldowhitelistplingquery +steady +you +td +falls +realism +help +mailtoforkexamplecom +resentdate +inreplyto +there +something +discuss +at +1 +shoea +go +songs +id +htmlrtf +also +nolayer +math +on +bluetoothdevbouncesmlsubscribertechcsminingorglistsapplecom +designates +cleanup +therefore +date +allegedly +tired +who +optimization +everything +atapimmc +me +xuidl +mail +httpfreshrpmsnet +by +garymteledyncom +headquarters +rssfeedsjmasonorg +itll +joachim +ldowhitelist5 +2525 +wonder +much +wont +developed +with +from +m3unfl2v9pimv9sgafr +iso88591qz4ltpu6qolya6g9aw0krjlajcsgcicycxrcoxwmurhckrvhcedaur7own2 +mimeversion +hotmail +c37mr3372408fgk681272148314518 +innovatiophobia +spamassassin +ironport2outteksavvycom +2 +cellspacing3d0 +xmozillastatus +ltd +listid +fetchmail590 +listsdebiankdelisztdebianorg +sender +precedence +independently +00 +anyone +000347425c17 +property +an +original +1028050752762737camelbobcatodsorg +7bit +discussion +be +listpost +smarthost02mailzennetuk +cw +ldowhitelistmdodating14missingheadersmurphysexl1 +fine +ibm +ask +11fe0d07b7cf7ae000003b236f4bf3a5c78f2e +figure +gmailid12832029c88581c7 +81168116 +upgrade +manual +indepth +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +xoriginalto +trapped +8219575100 +iso88591qivborw0kggoaaaansuheugaaadaaaaawbamaaacllos0aaaamfbmveusk +debianuserrequestlistsdebianorg +dhersaaes256sha +xsourcedir +some +enus +when +mpg123 +saremsgidlong450893 +reed +contenttype +islands +labial +of +ive +xstatus +record +received +wed +list +smtpsvc +apps +directory +search +the +nonspam +freshrpms diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_16.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_16.txt new file mode 100644 index 00000000..88f75c2e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_16.txt @@ -0,0 +1,358 @@ +lover +tbody +side +conventional +plants +millimeters +sa392e +uribljpsurbl +for +postfix +divsection1 +arw3es1f8n2hipjnptopmuc5ei1ooq0n0n4mso8nlpozpmp8a41u26hphxu15tpxgd +smtp +rock +span +involved +i5font +deposited +hrefhttpadfd6sqopacincn852di332aq7j55xis552uxiuclivykaghibody +face3dcentury +facearialwe +visits +xmlnshttpwwww3org1999xhtml +body +name +skickar +052 +in +faceverdanaif +amdended +firewall +your +off +returnpath +0001 +breakdown +are +represent +wisest +wait +048e116a4d +inscriptions +zlynwpayd5ftzrbm +100 +account2 +artificial +11px +activestrongbrour +xbarracudabwlip +subject +ceratosaur +custom +xbarracudaspamflag +included +2002 +bulk +width81brtd +aligncenterionsp +fontsize +late +invitation +email +yle3dfontsize +solid +shbig505413body104 +standard +15000br +below +102935 +shui +professionals +briefly +we39ll +fontfamily +style +windows1251buxvhbgl0esb3yxrjagvzigf0idi1jsbkaxnjb3vuda +ist +if +valueetethiopiaoption +br +description +involvement +delivered +brbrbcontact +color3dc0c0c0 +xbarracudarblip +14px +war +punched +bordercolor3dffff00 +broke +250 +width3d2020 +play +all +href3dhttpwwweggsmilerujwo3dhibodycsminingorg +head +26 +loaded +00hotfont +uid49281201487268 +200910160924n9g9oukq002068ns2csminingorg +host +value75000000750000option +img +trial +receivedspf +wonderful +gift +xmimeole +fromfaculty +left +it +5re1isxotdtotoufnjpwti7cmpxkwn1xrbqamtnvze2oa7i5zwmoog6yriwddfekcpg5be3jiu +sun +esmtp +srchttpmta248trd3remindstarcomdv2thkuiftoqexboqaiuieeeiieoeobrobtkrsirobcionkoaothbouaaitceqoiofuiocotrz +safety33ol1newnamednscom +24 +a +dont +xmozillastatus2 +grupo +skimming +etherbuttonpusherscom +momentum +localhost +within +serif +colspan3d2 +as +sch111111l +yo +div +rule +pcfet0nuwvbfiehutuwgufvcteldicitly9xm0mvl0rurcbive1midqumdeg +rivers +5 +has +messageid +pfont +350 +color3dff0000strongcomp +youfontp +10 +cellpadding0 +visited +6cuv0uer2urb6y5wuptsriw9qugxil3b61ko4pox831qre1wblgoasx9ucvansrmqyemaaqhsft +0700 +or +c78uiatewaahvmr1rdq0oqdxkd3dtmrq6ay1mk31fphuf1t9dakkzun4fqwtdta6jlj4vbhtd +broadcasts +above +public +questions +doctype +agent +total +such +lanes +010 +each +provide +xbarracudaurl +contenttransferencoding +seoep6073cablenetco +reply +contacts +answerusdavicomcokr +style3dcolor36dd5d +phylum +importance +marques74csminingorg +127001 +src3d22httpimgdefeasyadpostcomaddefem09gif22 +8bit +smokingindulgence +clientip18938243110 +usea +aircraft +xbarracudabrltag +to +xhtml +baseinsertp +charsetiso88591 +key +xbarracudaenvelopefrom +httpspamgwcsminingorg8000cgibinmarkcgi +know +32 +watches +pto +margin20px +je +color3dcc3333atlanta +principles +home +bosnia +most +privacy +color3dcc3333 +was +192111 +version +sa210e +historians +width3d8520 +5ne0xafsaljrtucp8j2u8ywpr5j3xzsajwrlnnxxbcjknfkllryn7xc8z3tsoeynmsx6pmq +msgidfrommtaheader2 +face3darialthese +tnnoxwu1qndmuypkmvnhnnendums9ojs7bd510sp2jkioumc9wk822ttj2khb83hcx0r0nw7gw +replyto +htmlimageratio02 +shtml +added +id +meta +hagerty +jabber +bill +tremendously +tend115 +vj +can +on +being +began +same +click +successful +208 +descentralized +date +take +31 +o4acgi5l016493 +154922 +seven +occurs +style3dtextalign +151254 +xuidl +order +by +color808000brbr +64 +facearial +stream +distinct +203 +new +will +tabsstrongbr +0100 +2012191210 +p +1st +nextpart001002201c2327292934220 +not +sides +bsfsc0sa148a +with +from +invaliddate +2091644167 +mimeversion +171333 +galleryheadlinespan +carambo +releases +both +planbfonttdtd +xmozillastatus +ltd +s2e +xaccountkey +153734 +sender +permitted +strongrepublic +message +10px +which +almost +percent +947 +addressnbspnbspnbsp +waste +treed +stylemargin +gw2csminingorg19216818251 +account +excitecom +texthtml +newsafonttd +2121006482 +million +headbody +more +n4j6wqh5002291 +thediv +1010010911 +w3cdtd +16th +reception +form +localityit +springsmt +brize +html +purchase +card +81168116 +size2quotfirst +shopfontabp +none +srchttp3332dtsajalircnspacergifp +xoriginalto +nonpresidential +allcontactsbouncebriefseincz +no +very +we +past +200207280302g6s320p00395mandarklabsnetnoteinccom +terminology +rail +respond +together +joan +md +encounter +bsfsc0satofromaddrmatch +contenttype +dxrob3jzigfuzcb0ag91c2fuzhmgb2ygym9va3mgb24gdghpcybdrcehpc9mb250pjwvcd48 +input +may +other +of +and +888888 +1640 +received +meds +youll +trademarks +1011005088243 +days +table +memorial +the +s2mr1678645ybc2781274368807780 +9fdcd423b +item +xpriority +those +only +kkkkacvgv2gumg9t8a2nxstenftbdnb7bwdtogeenuuuuygpy8awl9rsusfsx6igd6uh3r diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_17.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_17.txt new file mode 100644 index 00000000..b543e832 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_17.txt @@ -0,0 +1,341 @@ +helodynamicipaddr2 +tbody +please +emratherem +for +postfix +worldfor +find +support +smtp +xbarracudaspamstatus +onleoyd0jktpts7seednettw +xmlnshttpwwww3org1999xhtml +targetnew +anytime +normalbull +bye +8 +area +105208 +in +move +margin +transitionalen +dream +have +1113074r3d1112136t3d1225499824l3d1d3d89042085u3dhttps3a2f2fwww +returnpath +jul +are +mon +75583503744835024418350244883502458 +catalog +alignleftbyp +value4000000004000000option +knows +height3d71 +12705443784c361a620001w4twrl +00000000 +vasos +gd0lia0ski2yjx6klakfhqvzjvyj0ckjt6pbtnqym4bokgybfloacmzqlxlerbavw86qleaqgas +seen +respondents +ulm6wspexcbhymnk0tpu1dbx2nna4upk5ebn6onq8vp09fb3pn69oadambaairaxeapwd0kpa +subject +97pp +httpwwwinternationalfreecallcom +hrefhttpwwwwuzputalcntofc887b9fcc39dd6ddbf1cdf5 +independent +cheap +am +listarchive +than +cellspacing2 +sa135 +lowest +harbour +fontsize +email +exist +yes +j82xegcbfmlqvsjeg3qbqrqbpkfuyggcdiaazxpaclgapkvawuqkuwaoaewbhewb2tabrnqaxoq +below +colorffffffunsubscribefontanbspnbspfonttd +pop +br20 +di115k +100eman100ing +sunlight +sea +morebr +weeks +alto +description +neither +up +face3darial +xpvistabr +poorâtm +leramilerctrorg +rest +align +book +rbl +dkimpass +111verwegen +005531 +img +youre +hitting +xmailerpresent +requestspandiv +usiry8261ntsonlinenet +crcvapm7vauy6czdcabo6qixewsruseifrmwauo0kal59wg8hqjckguyjs7ickab7buagagtv +xmimeole +montana +uid124131201487268 +hampshire +sun +esmtp +centerin +100000 +zaejutixi9724telecomnetar +subscription +a +consett +deliveredto +xmozillastatus2 +httpslistssourceforgenetlistslistinfospamassassinsightings +brings +maintainer +eua2pftbe9k4c7pif8ahthivokab3lipd8gvfqodqmm8afxaoir17wx9apol80ahzs +herefontabfontp +customized +dogmaslashnullorg +tennis +constitute +magnetometer +6045210117 +pdt +5 +dial +messageid +350 +sports +m +10 +xasgtag +value4141option +unknown +1041 +council +b75618cc35f +199 +seat +size3d22222 +xbarracudaspamscore +man +norton +newslettersitehosternet +doctype +uid85501201487268 +total +g6o6ffhy095796 +010 +december +doingthat +templatesnbsp +xbarracudaurl +contenttransferencoding +3cli3e3cfont +age +zzzzlocalhost +1885h2 +television +127001 +meets +info +tlchargs +quarantinelevel10000 +xbarracudabrltag +to +alternative +typetextcss +width3d15 +code +you +httpspamgwcsminingorg8000cgibinmarkcgi +fedex +know +32 +trademark +height3d181 +share +single +008e75d41a4c4472a7b76ce66cb6ehsbhx +thembrnow +netscapenet +dday +enysqxeqo +version +82 +sharper +response +establishing +htmlfontbig +at +size3d2 +distant +law +id +meta +mandarklabsnetnoteinccom +pts +rdns +markets +can +designates +alignright +click +xmsmailpriority +sidecolumntitle +date +spamgwcsminingorg +assured +receipt +2010 +xbarracudaspamreport +international +note +012029 +2009 +95 +healthbrcheapest +xuidl +valign3dtop +mail +allied +hrefhttpmta33annfilesfourscombp5ruxqoafizzxetictoqeoiofqtefoezozlabvanidboeoiatiofifkcltiocaoecaacecneabr +by +lthnqw2khuvpczgoxggoyfzl0mrviu6o2xriuospe9zbyjuh58q79ghndvzxsgoqedbssvfiwr +he +mapfontafontbr +width505 +hrefhttpa1dea0gmakahercnaxfef3549de7d9857010e07iui989232689485534000904 +hiroshima +antispam +new +brmany +literaturecustoms +qwipae9tnv4pbv0a2pauyiowhet8d6tjtpqhrapbvi7qmu0uevi9gg25xeiafqwuuwei4ldaqecj +p +color666666b +humains +trusted +drawings +parallels +with +from +still +un +1200 +money +1252227055 +mimeversion +0500 +gw2csminingorg +arrangement +aligncenter +wcyiztp7ybfm8nwbfrnulj92x98h6tnapvjahjcv5z5989kfrak6f1b7qw1cnge +p5rdlyys0cwabtxfxaginm1jpanf1lkab5gbgzgprtxyeigeqjgwklwbcg12fzkavwmavz8awh +words +valuebabosniaoption +fetchmail590 +smallcapswe +webmaster +xaccountkey +sender +zxr0aw5nihroaxmgznvuzhmgdhjhbnnmzxjyzwqgdg8gihlvdxigcmvsawfibgugy29tcgfu +pvxjxupkydtaechrutdmgzqbjhsq0kx3nbificunrtdxjwtxuqkz1nojyekbof3xnmozvibygpac +target +irish +saveta +7bit +stessa +size3br +experiencing +faxing +22strong +be +into +get +httpequivcontenttype +spill +alt3daig +rgb240 +texthtml +time +were +uid131371201487268 +headbody +thousands +fdisk +k19mr391393rvm1101274384284627 +xbarracudavirusscanned +w3cdtd +r +affinite +vertel +inner +m97y +width3d126 +81168116 +november +200920 +lidvpjrbolnfbrelwyzvfa4wwinhejmwix5sltvhdfdoez6uc31qk6r1mo1wpff0t0dpds854c +brbrthe +hrefhttpprofilesyahoocombloguwnupc5hbuesmzfbvjdep4x6lewytu +width420 +n111117vel +xoriginalto +promise +decides +tell +biological +no +we +don +151338 +content3dmshtml +value +barracudaheaderfp20 +67ea247cc3 +including +contenttype +xmozillakeys +footer +securities +may +listed +feb +of +venue +ii +401 +and +record +received +windows +right +jednom +o +days +dotteddecimal +sept +access +gdvdpdvilutntrgw0ev1esqbnocmmjphvvjhspifaj72xfwwcfeyemueddfzamepoa +the +altpicture +never +d diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_18.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_18.txt new file mode 100644 index 00000000..6f35aec7 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_18.txt @@ -0,0 +1,265 @@ +plugin +bmkan3uaxg60tih9awxte62ncyigrtb +trouble +qmail +rhl +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +program +feel +people +admin +designers +rpmt +kernel +body +layers +verify +name +this +oewvzdlueafe +secs +employergenerated +luca +your +returnpath +imap +vm4010327 +mailtodebianuserlistsdebianorg +is +testsbayes002 +v19mr1758022agf1001271518942905 +100 +00000000 +dvdrw +server +said +subject +martinlichtvollde +dress +relay +2002 +xmailmanversion +preferred +email +listunsubscribe +quotedprintable +sentenceda +0000 +debian +best +url +another +051141 +if +fourla01 +gldudebianuser2mgmaneorg +big +required53 +2482 +anchordeskbrfont +httpwwwsquareboxcouk +per +gmailid127fca9cc5f90dee +all +tue +174009 +26 +lairxentcom +studies +domain +luck +u +had +height3d600 +re +g6vkxd209091 +hit +f +receivedspf +result +0400 +zhxaljghf11katnf +edt +wentlandcluniheidelbergde +pudgeperlorg +else +required40 +it +sun +esmtp +893893 +listsdebianuserlisztdebianorg +173934 +entire +a +bf08wjx7faub1y1almypmlbiv5wdmzmwcvowg2x4nkltyz1t9tqdyis7eagsi +testsfourlaldosubscriber +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +deliveredto +khare +xmozillastatus2 +provides +employer +localhost +lenny +useragent +dogmaslashnullorg +tepkhcdudlep +spfpass +pdt +rich +messageid +e17mpr70001hg00cpu59osdncom +10 +1022367144 +0700 +173752 +chomp +colspan2font +040933 +gnulinux +avinash +such +0200 +forkadminxentcom +2008110401 +textplain +i +127001 +8bit +125827 +that +pump +modify +to +aes256sha +ratwaregeckobuild +singledrop +20080610 +you +httpwwwultraeditcomdownloadsindexhtmlnotepad +hrefhttpwwwgnometomescomdownload +maths +did +edit +hrefhttpwwwlockergnomecomspecialhtmlbget +221555 +testsawltnonsensefrom4050 +help +inreplyto +16512119466 +references +labmailredmondcorpmicrosoftcom +crazy +bz2applecom +yyyylocalhostspamassassintaintorg +debianuserlistsdebianorg +id +border0font +apt +listhelp +can +on +common +href3dmailtodebianuserspa +090117 +designates +happy +xmsmailpriority +date +issues +authenticationresults +googlecom +1731605061 +aspirin +by +vamsukchctnvfqwg038cjx0wbgpuwvngnaa2subwobqq2h +remaining +gearing +checks +81018101 +httpwwwgeocrawlercomredirsfphp3listrazorusers +comment +fixes +listmasterlistsdebianorg +few +2525 +back +0100 +xamavisstatus +autolearnfailed +20020905181308b4905greenhydrantcom +preferences +listsubscribe +with +from +copy2002 +cnofws +newsfonta +mimeversion +joebarreraorg +ordering +pass +xmozillastatus +kirkwood +listid +make +333 +report +sender +permitted +precedence +101011788 +taggedabove10000 +cc +listpost +fahrländer +get +bits +105243 +thousands +geguc5bec497 +laa21283 +mailing +windowserver +mark +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +d2923440fa +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +current +jozsi +announces +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +10143348 +grokster +storybr +http +8219575100 +no +sat +mailtodebiansecurityrequestlistsdebianorgsubjectunsubscribe +dhersaaes256sha +some +policy +version325 +1000 +xmozillakeys +xrcvirus +of +and +record +received +beach +setup +one +without +list +161105 +mailtoforkrequestxentcomsubjectsubscribe +013003 +the +volunteers +those +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_19.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_19.txt new file mode 100644 index 00000000..0421d2e0 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_19.txt @@ -0,0 +1,280 @@ +4th +sweet +0300 +ratehard +trouble +oldreturnpath +read +xentcom +for +postfix +drive +s +find +differences +post +testsawlinreptoknownmailinglistspamphrase0001 +parties +clientip8219575100 +this +autolearnham +in +errorsbrdiv +denote +have +returnpath +imap +tel +guys +powell +subject +4 +bits0 +postingsa +relay +am +worst +2002 +xmailmanversion +1219 +1921867 +consumption +email +xscale +listunsubscribe +11 +0000 +0800 +debian +jmlocalhost +height1tdtr +what +would +monkeys +another +forkxentcom +finebymeuk +0704 +ist +if +ldosubscriberldowhitelistsaremsgidlong45 +good +apr +470 +thu +short +httpwwwlinuxiemailmanlistinfoilug +interfaces +re +documentation +host +advanced +murphywrongword202 +dislike +receivedspf +result +0400 +before +125713pm +103012 +509 +required40 +325 +foolishly +esmtp +a +deliveredto +dsa1857 +clubs +information +resentsender +samuel +ionel +localhost +useragent +as +talk +uncertainty +spfpass +has +she +messageid +10 +0700 +ham +g7sgskz01542 +215105 +54358787903qmweb55604mailre4yahoocom +they +xrcspam +trying +0200 +contenttransferencoding +pgpsignature5 +userid +budinerocsminingorg +systemsptdtrtablebr +textplain +i +127001 +316392 +22240 +that +125838 +checker +xos +deeper +listsdebianorg +561 +to +quote +took +singledrop +xlevel +xtafdpnvh0yzl4kmyhu6ps59eyyw64xwiom3we4rispq4s4har4gnar04hk +code +you +ilugadminlinuxie +bootup1 +web +121450 +dang +solution +these +mlsubscribertechcsminingorg +inreplyto +references +181530 +reduced +yybsuszotwadzoqrulbxppmdgfwboiodtrp34 +logo +at +authentication +lughtuathaorg +id +174939 +libdvdcss +httpslistssourceforgenetlistslistinforazorusers +over +port +can +on +bgu0od4hsx4ze1mpuwpdpdthpqprcgemm5noehd9mhvicxoapozmrm0qt8vt2sqd +06 +yyyylocalhostexamplecom +fixing +raw +xmsmailpriority +after +oucclopjtzgqpwc4kgksbx9cwcziimuz6d2zwm9yc4pqm8ez1atkzpxw6q8dcz7o +date +2051 +alignleft +jalapeno +laurieprimalucdavisedu +worried +me +315 +by +il +either +unsubscribe +improve +nowrapfont +peters +should +mart +called +2525 +xbeenthere +back +while +listsubscribe +with +from +crucial +0500 +beta +hrefmailtocoolgearcnetcomsubjectslimcoolgearcnetcoma +express +include +libxml +180046 +pass +could +listid +make +charsetusascii +fetchmail590 +vandals +civic +end +precedence +message +seem +its +tree +granting +an +procmaillocal +ip3 +4096 +dgxt +20020811182012gd25331linuxmafiacom +xpolicydweight +freely +get +bgcolor000000img +texthtml +lbertemailcomemailabr3diesz3d468x60ptile3d1ord3d +logmaneorg +xloop +7 +really +8580013a5134 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +perceptions +delivery +fri +spam +xoriginalto +masterlistsapplecom +070307 +aware +8219575100 +3d9c23c59090203leenxcoza +call +debianuserrequestlistsdebianorg +no +very +needs +ubuntu +notinblnjabl15 +frommxmatcheshelodomain2 +enus +known +version325 +including +contenttype +crossplatform +like +might +of +ive +mozilla +shaw +xstatus +sponsored +received +175417 +hrefhttpclickthruonlinecomclickqafdb8qlpqkx4vulsyddx7ndmuvszr +couldnt +created +mailtospamassassindevelexamplesourceforgenet +trash +comments +wouldnt +memorial +the +never +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_2.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_2.txt new file mode 100644 index 00000000..b59232ef --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_2.txt @@ -0,0 +1,260 @@ +f6d0 +002c01c253c35d522d70740aa8c0fmmobile +trouble +oldreturnpath +format +notrack +for +postfix +network +secprogreturn625jmjmasonorgsecurityfocuscom +smtp +post +flicker +body +594f17750 +this +instead +cached +141211 +switch +your +slimmer +off +returnpath +imap +are +is +oh +claim +testsldosubscriberldowhitelist +force +testsbayes002 +moderator +targets +subject +try +got +listarchive +2002 +email +listunsubscribe +justreleased +easiest +0000 +insurance +id3dnetscapespacer +rssfeedsspamassassintaintorg +contenttexthtml +what +npcai +remember +blogging +funky +running +taskf47e61c0 +bottom +etc +clicked +decoders +153607 +101156514 +newsletter +10pt +tue +bless +understands +domain +u +had +re +run +xcloudmarkanalysis +data +effect +xmstnefcorrelator +set +xmimeole +325 +us +sun +esmtp +runs +determined +a +movie +deliveredto +localhost +bad +biggest +useragent +as +voice +spfpass +pdt +reminder +messageid +src3dhttpwwwcnetcombg +10 +dkimneutral +a05111b05b992ca1793461001100 +scan +otiorhynchus +183520 +gnulinux +ebios +archivelatest32379 +completly +contact +they +problems +each +contenttransferencoding +21 +way +im +building +httpradioweblogscom0111520categoriescanada +zzzzlocalhost +textplain +i +hsubjectfromtoinreplytoreferencescontenttypedatemessageid +that +out +grubgfxmode640x480 +to +sep +singledrop +intellectuals +ilugadminlinuxie +479 +web +know +utc +tdtr +amavisdnew +060704 +mailevergonet +youve +20100430162040f26361bacelejarcsminingorg +331vamm2 +week +inreplyto +there +at +1 +through +e3d1bnbsppreston +results +akj9zalcoel0a10 +on +designates +player +srchttpwwwzdnetcomproductsphotogalleryheaderanalysisgif +4250 +4bc0aa2b2060507webde +date +20100512203248ga4617deathfog +appreciate +2010 +authenticationresults +xuidl +277b22940d1 +things +by +then +think +even +comment +should +xmailinglist +story +actually +will +hasnt +2525 +061112 +0100 +xamavisstatus +autolearnfailed +not +141633 +with +from +means +autorun +07252002 +mimeversion +spamassassin +httplistsdebianorg4bea95b52080107comcastnet +mailtoforkspamassassintaintorg +failing +xmozillastatus +120240pm +make +put +xaccountkey +kids +extra +sender +permitted +precedence +tip +reseller +developers +kmail1133 +cell +notes +7bit +xhabeasswe3 +20122912112720010626 +purchased +8386540858 +relay07indigoie +122222 +version250cvs +update +110203 +height18br +jerry +xloop +christophe +rwnosuidnodevuhelperhalshortnameloweruid1000 +mark +intentionally +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +thursday +81168116 +sgamma +hits76 +20100506143743604mgbdebianyosemitenet +tracebacks +carrbfonttd +225114 +debianuserrequestlistsdebianorg +tell +httpbunnytricksnetindexphp +we +mailfollowupto +104747 +gt +leitl +fonttdtrtablefonttdtr +frommxmatcheshelodomain2 +hey +contenttype +libqt4dbus +of +and +record +sponsored +received +list +rootlocalhost +went +dynamic +directory +perhaps +the +x11users +4be3231e5070102coxnet diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_20.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_20.txt new file mode 100644 index 00000000..b1455bfe --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_20.txt @@ -0,0 +1,334 @@ +p111tenti97lly +reference +12640742364483009b0000w4twrl +tbody +bl97ck +outsize +please +xmailer +ranks +for +postfix +smtp +series +any +apperetocom +body +ubmo3rbp0fe1f930r3cxhjndfghgybvq03ofm4gwmydaahgvc9raljsrvzjt3o9p +this +menu +in +retrieved +firewall +have +your +returnpath +imap +dmvyc2f0aw9ucw0kdqp0ag91z2h0znvsdqonckzyzwvgdwxsvgv4da0kdqpvdmvyy29taw5ndqon +valuengnigeriaoption +is +mon +httpwwwbarracudacomreputationip845210767 +why +companies +too +accept +fontopdiv +55265021 +jointlygaussian +account2 +replace +213105180140 +xbarracudabwlip +oklahoma +subject +custom +got +explosive +than +titlemelaka +included +winehouse +easiest +pfizer +abo +15 +but +11 +insurance +url +contenttexthtml +eastern +fontfamily +style +color000 +height3d61 +br +description +adiv +delivered +up +normal +all +him +tbl +tahoma +unblock +unrewarding +attn +burning +000000 +facts +finished +img +ready +nor +necessary +fdlofuur1idmfp4z +every +expansion +determine +cellspacing0 +c49ab16b12 +copies +independence +uri +brof +us +left +esmtp +httpxentcompipermailfork +1000725 +webmasterelectronickitscom +literature +titlenewslettertitle +earthenware +intercession +deliveredto +information +hibodycsminingorg +receive +localhost +feelings +superiority +stylecolorwhite +div +12pxhave +has +messageid +2138656112 +10 +ewvsbg93ihnpemu9kzeswxszwdhbcbby3rpb24gpc9mb250pjwvyt48l3rk +refund +yours +unknown +text +feet +0700 +size3bfont +h2york +xbarracudaspamscore +man +spanfontpfont20 +yourself +public +ndnsnihrrrskcvgv2gumg8abf8a9p17lxjx7qxtqf8atv8a06ar41rrrtkcljanyv1fjsprf +hrefhttpwwwpaperlesspoboxcomservlethomeservletgotopage230img +medium +they +xbarracudaurl +phagywxpz249imnlbnrlciijm5ic3a7pc9wpg0kicagicagica8ccbhbgln +contenttransferencoding +multipartalternative +addressafont +zzzzlocalhost +importance +textplain +television +127001 +8bit +bsmtpd +negotiations +restful +align3dcentersubsequentp +that +hrefhttpbae79zvugacamcnjqt7988fcd23cf363e3c5bc95cere45559881179232827377538 +tg070 +sending +to +lieutenant +now +cellpadding3d0 +691 +you +td +jun +did +china +fontweight +b20sierpy2sgyw5kienvy2sgdg8gy29tzsbpbiegpc9mb250pjwvdgqpc90 +font +these +relationships +webcam +mxgooglecom +di102102erenti97lly +htmlfontbig +alt3d650 +enlargementpills +at +fqvmizgeytmf4honwqaljc9okyzmjbbkdnmor1rzaizue0ho69aybktj15fixwoarwhyk0abyd +go +id +dollars +travelstrongfont +wave +pts +rdns +choosing +const +can +on +medicine +mar +black +government +board +successful +g74c4sv27254 +g8jmtyc06196 +date +spamgwcsminingorg +021 +xbarracudastarttime +31 +ea24b16f03 +alignleft +who +itself +12px +woininyak +everything +align3dright +nightly +invites +xuidl +them +by +alignlefthighlyp +032 +044524 +demanding +antispam +style3dbackgroundcolor +written +foreign +imagine +enormous +with +from +early +wealth +immediately +color3d +size2brfontcenter +subscribed +mimeversion +bsfsc0sa275bhl +crime +content3denus +first +true +aligncenter +name3dnametdtr +pass +tddiv +xmozillastatus +arialhelveticasansserif +sender +message +anyone +mr +file +which +pcpt +pipesmokingstoking +japan +sa271x2 +an +020 +webmasterefiie +afor +7bit +forged +19216833 +64011859 +namelname +sorry +be +magic +gw2csminingorg19216818251 +champion +get +httpequivcontenttype +plans +19322747252 +theyre +texthtml +englishlanguage +padding +loss +httpsciscientificdirectnetrrasp83999041248fb1d7ff79fct +th97n +attractions +153520 +special +more +investigated +form +html +stylecolor +size1copyright +facearialbdulge +packar100 +card +matches +unknownbr +patteson +none +relaxation +baselwo +forstrong +spam +mauris +marginheight0 +3 +no +very +177011683e +colspan8 +murrumbidgee +lenders +dnpesf8auwdhf2snjkw7wa5euxraerw017rvggmi01kovkr0jttua5zlxrrh7eaafvkbt4s +95822 +6182886661 +group +contenttype +xmozillakeys +of +labour +received +devices +19216818120 +2887 +list +204621 +table +size2pain +presentationsbfontp +removeaspclick +unconditional +ullargelist +dawreqwmdawmdbemdawmdawmdawmdawmdawmdawmdawmdawmdawm8aaeqgavgbiaweiaairaqmr +xpriority +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_21.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_21.txt new file mode 100644 index 00000000..d4a7128c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_21.txt @@ -0,0 +1,289 @@ +spamscore0 +zzzzteanayahoogroupscom +energyover +ratehard +oldreturnpath +335 +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +support +smtp +aanlktimv2szrhs6todhnsopmcggtjnuwyoec98josbbmailcsminingorg +package +any +arsasha1 +account5 +paul +death +in +have +returnpath +imap +skates +guardian +is +caused +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +testsldosubscriberldowhitelist +todays +f35e016f03 +testsbayes002 +trtdfont +23 +mozilla50 +location +subject +contentdisposition +dave +listarchive +2002 +bde2f2940a5 +listunsubscribe +solid +ls +but +ywh40 +0000 +root +jmlocalhost +background3dhttpwwwcomi +score6 +bsds +smtpsvc5021954905 +another +ist +line +if +weidlinger +apr +required53 +2002fon +restore +unaddressable +thu +keep +lairxentcom +domain +aaaaahpy2nyt2bgf +cest +youre +0400 +sp5tp1swdifadjgm5zvlbliszt +required40 +us +esmtp +mailtoexmhusersrequestredhatcomsubjectunsubscribe +namevlinkcolor +self +24 +local +a +score199 +deliveredto +235155 +pages +advance +localhost +as +voice +unfold +spfpass +110331 +has +httpsecuritydebianorgpoolupdatesmainppostgresql83postgresqlpltcl8383110lenny1amd64deb +messageid +pfont +quiet +tnonsensefrom9495tnonsensefrom9596 +resqlplperl8383110lenny1amd64deb +old +yet +kaddressbook +112217 +statement +220416 +strong +uswsflist2sourceforgenet +g16eujj22714 +smiths +total +contenttransferencoding +multipartalternative +pgpsignature5 +2008110401 +im +de +textplain +invoked +i +9si3908694fxm2720100409110618 +127001 +work +22240 +terrorism +that +out +raving +rpmzzzlistadminfreshrpmsnet +referencesspamphrase0001useragent +listsdebianorg +to +cellpadding8 +lopez +charsetiso88591 +singledrop +xlevel +108 +arizona +102231439 +you +letter +dev010 +xenvelopeto +amavisdnew +methodget +these +making +mlsubscribertechcsminingorg +resentdate +inreplyto +version +mxgooglecom +at +replyto +been +yyyylocalhostspamassassintaintorg +debianuserlistsdebianorg +douglas +go +id +271318 +listhelp +can +listurl +on +near +stan +tnonsensefrom7080tnonsensefrom8090 +upgraded +hide +xmsmailpriority +date +frozen +invoke +height3d4br +me +googlecom +xuidl +them +things +appropriate +by +en +comment +listmasterlistsdebianorg +ago +badly +xmailinglist +mdxdesktopozassytesnet +will +bgcolorcccccctd +xbeenthere +much +xamavisstatus +while +autolearnfailed +software +054000000000000004 +not +listsubscribe +with +from +early +mimeversion +222837 +enht +overview +give +first +alone +xcheckerversion +use +listid +073414 +charsetusascii +fetchmail590 +approach +done +laser +xaccountkey +sender +precedence +helodingohomekanganu +blocked +8a1b113a533f +mr +file +which +material +an +evolution +taggedabove10000 +0e4ce2941c2 +be +xpolicydweight +into +bud +httpwwwnewsisfreecomclick38688972215 +g8r80sg00717 +mailtoforkrequestxentcomsubjecthelp +code2e +always +better +n17mr4186343mul501271254339033 +141143 +fieldengineercsminingorg +ask +power +xloop +mailing +uint8adata +unsafe +kriegerdivxmailshellcom +andor +really +start +frommxmatchesnothelodomain0 +fri +dpkg +about +http +see +hart +sources +very +we +archive +installed +value +does +version325 +53b682d0d75 +including +contenttype +like +write +of +ive +newsletters +and +received +without +nt +mailtoforkrequestxentcomsubjectsubscribe +keys +071139 +094316 +the +those diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_22.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_22.txt new file mode 100644 index 00000000..b884e44c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_22.txt @@ -0,0 +1,295 @@ +rate +d734 +ratehard +trouble +qmail +finger +xentcom +for +postfix +103205 +smtp +sansserifspecial +arsasha1 +body +resentmessageid +uswsflist1sourceforgenet +clientip8219575100 +account5 +this +cached +switch +returnpath +imap +oodles +colineasydnscom +mailtodebianuserlistsdebianorg +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +require +testsbayes002 +00000000 +seen +server +subject +try +am +xoperatingsystem +173225 +2002 +email +listunsubscribe +httpwwwtheperlreviewcom +but +0000 +helodomain +edid +lori +correctly +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +zp8xzmo3rqudshcjmzckl0ro54iprt13ivwfmssmjxfenintwq3ikvs4pllo0tqf9mug4hd +recent +ist +if +fourla01 +versions +103927 +required53 +up +srchttpwwwlockergnomecomimageswebcamfurocamcamjpgatdtr +7d2303a65bc +so +razorusersexamplesourceforgenet +61c422d1f36 +tue +wespageca +144126 +pretty +re +httpwwwifcomau +peter +receivedspf +905jkyp5jfgazcbyt9xlbliszt +047 +0400 +width3d162 +idea +edt +just +el +understand +it +esmtp +listsdebianuserlisztdebianorg +a +10so2879487gxk10 +dont +deliveredto +mean +information +xmozillastatus2 +164055 +maintainer +resentsender +c618813a6651 +localhost +2011 +useragent +as +20020813233332a30847svr103snet +indicating +inline +acade9mica +pdt +5 +has +messageid +10 +cn +phoboslabsnetnoteinccom +0700 +200208081032laa09312lughtuathaorg +gnulinux +yet +httpxentcommailmanlistinfofork +or +monnier +critical +xrcspam +exactly +forkadminxentcom +2008110401 +bootloader +ma +stop +textplain +127001 +ps +cards +communications +that +mes8eb95vsr8rr8c +to +sep +issue +downgrades +20080610 +you +did +acton +asain +amavisdnew +width140 +ip83134218234dslscarletbe +causing +nonclear +331vamm2 +webnotenet +these +documents +inreplyto +20090614 +references +there +mxgooglecom +structural +70021 +through +recorstores +id +206160223 +port +listhelp +libinitrw +on +installation +designates +ivborw0kggoaaaansuheugaaadaaaaawbamaaacllos0aaaag1bmvevjggkqabfhsxt0z0t0wqagbgd1wgdfoapppqz8uweaaaczkleqvq4jvwtuwambdhdalvhuogbh2ennnycli8j9nj4teporhnkekzvqrgkwvs8cfe2teptcqf353vjvbyagbkvskkqdvleelqifya5rx1aaqzhghkkhxsxnyroyldysmtefw1ywxtwu3g7yq6zp3nqampktbpokuaaarcqujlawhxtqfmlyrrdf2tuiwdijhoakv6iojwzfeikiadtxmoaritb1vp3mb7cghjruqeo8bsuokeanzbuvcjspsxsn9wericljoghcz7mdwpget26fkztnosfzalvsjxexrsi7t0xinpqkg7s1yynaebyztc8dwlfoqrouw8utdzxhgyuiyn5ervf2kyvfsiifsy1cgcw1aodb7ew1crt9hqffdr2wwzp2awldj5c6ec7fcitjga2ndriirkdfqllprjyvvi0im6wsdcbuufdjb7arjzr0tlkiw2clqiskbmzzxlayjkixeo0fsdtd45huh9rikj6myuo9bedto4mr8wiwjytm9dxsdpdyvtuftb8ag7svtmqxdo5emctvc3qetkezzg5dadj8b8pkf7aelgd2c2gcrdtw1btmn807l6li8td3o6c3kbteu2435l7wx9o8b7txgnoihhatxplxkrab2dk07bg03ztxb6pnizppei4gfpexetvrygyz30oyw7i78azbqovmpojosrtv3df0ta6dc8j4xzjq0ydgdnxywbv7aex3otap4bnwi4fqxvwevbg1vzriydrkqfafdurzssxuwc8hv31v29henqlxnbxokffap0omgvnxgaaaaasuvork5cyii +g8j27rh04391 +check +my +zzzzlocalhostnetnoteinccom +detected +turned +xmsmailpriority +p9mr1591787ebf731272924672478 +date +152528 +shares +avb21rjirtbaf1czmp7lbliszt +timely +mailtorpmzzzlistrequestfreshrpmsnetsubjecthelp +vlink3d00339 +googlecom +xuidl +order +67826472 +72202 +rocks +by +60024620000 +bgt +visit +either +085237 +helo +qyk5 +msie +ldowhitelist5 +huh +2525 +their +autolearnfailed +software +not +listsubscribe +forced +styletextdecorationnoneimg +width3d16 +with +from +thoroughly +mailtodebiankderequestlistsdebianorgsubjecthelp +mimeversion +ethical +tdtd +afternoon +xcheckerversion +lot +211755 +pass +plain +jaa13534 +jmasonorg +available +eth0 +header +permitted +precedence +position +storage +anyone +dateinpast06121069 +131644 +strategies +almost +useful +httpsecuritydebianorg +taggedabove10000 +be +listpost +xpolicydweight +respect +httpcvemitreorgcgibincvenamecginamecve20093871 +hydrocarbon +get +ok +release +geek +time +version250cvs +httpangryelephantorg +heart +xloop +e6d6a13a6294 +envelope +platform +g8ufxek04693 +libgailcommon +471 +includes +101520 +multidevice +see +debianuserrequestlistsdebianorg +made +quicktime +no +sat +buying +needs +debianlaptoplistsdebianorg +dependencies +group +bayesini +httpwwwnelieorgbartlapino +theres +contenttype +extremely +fontb +required70 +debating +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +of +packages +speak +received +score6107 +one +windows +wed +list +libxine1miscplugins +gnupdf +src3dhtt +the +httpwwwgeocrawlercomredirsfphp3listspamassassintalk diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_23.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_23.txt new file mode 100644 index 00000000..0aa86795 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_23.txt @@ -0,0 +1,373 @@ +lights +neunet +223414 +us3100000000 +secretary +sitescoopertalkadminlistssourceforgenet +choiceali +conventional +background3d22http3a2f2fxyzlm222esitemynet2ecom2fback12egif223e3ctr3e +please +trouble +dormie +2003 +partnering +for +postfix +replica +smtp +standing +geostationary +9px +width3d95 +kargbo +series +k97n +body +area +name +ar7vt7kh2hd6fzqxyak70yh9h3en2upswacvu9psofyd3p9ld7fgar7vt7kh2hd6fzqxyak +this +in +content3dtexthtml +indirect +strongfontdiv +anonymity +off +auto +returnpath +jul +cccccccccccccccccccc +0001 +are +is +mon +tinted +1221809198 +borderbottom +account2 +00000000 +03589com +213105180140 +prices +colspan2nbsptd +examples +subject +opt +using +calories +size +produced +para +megadealsourcecomneqwhtmlneykhlhpzmyhlhnnjlatasyklbr +imo +bsupercomputersbbrbrbr +email +quotedprintable +sorne +sheriffbariaurcom +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +ns1csminingorg +1268766747079d95390001w4twrl +unavailable +contenttexthtml +different +intent +yyyylocalhostnetnoteinccom +porphyry +httpwwwfacebookcomeditaccountphp +vgo98srtdyjqaxoyc5s4osqhgqj0wabew3jibgmonlnbowvno9cetamfxfc0ztlccf8s5m8h8i +cypherpunksdspronsnet +qadk6ozxvaamrqhlduaayuqgv1qadk6ozxvaamrqhlduaayuqgv1qadk6ozxvaamrqhlduaayuqg +description +big +rdnsnone +tag +delivered +sms +up +reported3dinq +xbarracudarblip +notification +must +thu +normal +do +all +tue +lerlerctrorg +head +26 +valuestraight +primarily +rbl +composers +had +color3dff00002999font +deliverydate +ciqgfuormarfkis9wfhypbm8ggetwiau8h14sqlaacogrkwqet8l0odyjma87ip7micpmsqqceo +strongly +set +crashes +loma +align3dcenterbfont +just +us +el +it +vps33maryanehosts016com +lhnpm5qvpxgnimd0jbdjzjbjjkr1nsupldx8piylfk2igmjws9favyncusmrtg1tlwlf8kbyfn +sun +esmtp +ca +g6nh0ap15756 +030258 +romantimesserif +convenient +019 +a +deliveredto +job +xmozillastatus2 +hibodycsminingorg +size3d2bbr +localhost +bandb +cellpad +customized +dogmaslashnullorg +div +mimehtmlonly +identnobody21157216185 +04 +has +messageid +championship +vehicles +10 +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c15gif +cellpadding0 +size3d1if +yet +xbarracudaspamscore +account4 +contain +boundarynextpart00000a665d74a6dd6752b68 +or +federal +beendiv +looking +norton +doctype +color +uorog +xbarracudaurl +contenttransferencoding +193120211219 +multipartalternative +a9 +helping +textplain +mimeole +i +mjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjlwaarcaddaqwdasia +127001 +ricky +promotions +sansserifcenter +vws28xmhtvpbjv0oshpi7aemksnacvpixkggrs1gcfwpfaoc1pfwqhw8ppv7spphqoowha6gvp +2008 +servicefontfont +43 +styletextalign +insightstd +sitea +animal +13br +spamassassinsightings +to +sep +become +freeget +fall +direct +srchttpmail4mortgages101netlogophpid90id2193953p +coat +now +code +upfront +algorismcenter +letter +nbgbeajs028738 +jun +falls +presented +yearfontfontb +iqyeahysyga +share +phones +font +webnotenet +privacy +yhmnxfvqnfh2rycjx9lzqez4safulrubc1961tlklflxamcyobdepiwjhdfekvhphryafkmvik +help +internet +version +long +f117ente +friendatdtrtablebody +countries +name3dgenerator +valignmiddlebr +judicial +065 +at +through +sa148a +dildos +id +meta +snuw79yfpb5yufk6flfy3gxtoozqarzgc3ktk972nqphgow1fbkyxkvhkrd4t2omztqyldqselik +frayer +mandarklabsnetnoteinccom +africa +jeff +awards +on +65yjnfmnggxhllw0rjgni6o2vutri543y44jabebxknund0kub59a1tfaurdn8nlwxwpsal0mrwr +htmlmessage +hrefhttpelr2grassillcomdftqwoxoeueinikareikuzzrzlnkinobloxazioefifnelawwxeeicoronihiqfezcaixabr +tr +moluccan +click +date +spamgwcsminingorg +live +hrefmailtoremovefirstdentalplanscomremoveabrfontfont +xbarracudastarttime +100isplaying +f111rmatting +marketing +xbarracudaspamreport +offering +notified +cease +valuable +xuidl +by +verdanawhen +indeed +unsubscribe +altimage +equation +antispam +mta +library +050 +jm +0100 +little +park +1051 +cmkim3nnknqqqqqc3uflyptnviv9eke8fyk6h0xu3qohlnvpkul3b0j4x2mt126soxhhj8c1zs8 +with +from +293 +gave +width728 +mimeversion +scores +border0 +qahnyp80fkuxpmhxlptdat6nddzaghxdthi2nfutcquxsxlhnbj12cu2rj5be14scp +000 +forget +south +026 +killlevel10000 +both +mail3csminingorg +pass +retentionquot +adult +make +11si2815409yxe11220100506043745 +xaccountkey +mischiefglobalservenet +srchttp265bnihosslcn81ovrni64gif +message +class +its +errorsto +an +pietro3druins +webmasterefiie +iqi5caexmairyaqlmma54lmjnqeiuagyhgefbmjkhrricagxyanhoxnieajeqs0d8amg4m170nj6 +botswana +jp +absolute +1601 +commune +c +branches +hrefhttpwww05craycategoryuniquezaamnetcofr20220520httpsam2002gooptcom8101font +pair +finds +thediv +f6f0e78e859df6f0e79e88f5f4e39c98f2f7e7988a858892f6 +microsoft +1010010911 +brl +wanted +neutral +mailing +wide +81168116 +treated +6lagd5ooooplfjupqldf9dnmar1y1dkjxxxzwczqvqb2xh8kanbwacwdqjwsja8e1kad +includes +contains +territories +xoriginalto +about +debtafontfontb +m0620212mailcsminingorg +0900 +lines +see +97ngle +maecenas +arial +ea59615f93 +having +fortunately +persian +mailings +colordarkblueemstrong +incbr +historic +when +barracudaheaderfp20 +confirmation +sansserif +contenttype +xmozillakeys +months +possible +may +of +since +and +width22 +received +faculty +red +pennies +list +days +reservedbr +helod +view +fee +9bc2e5 +the +sh diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_24.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_24.txt new file mode 100644 index 00000000..d7746ac6 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_24.txt @@ -0,0 +1,286 @@ +rate +michael +183559 +120 +application +exclusionary +please +qmail +xmailer +able +for +postfix +rpmlist +m8cs36373wfo +smtp +question +hrefnews42htmltopback +xmailscanner +body +resentmessageid +12980904 +geist +this +bus +autolearnham +in +certificate +returnpath +bmcemfou897jjhzqjd8nul1iqppg6yhzlaokxlrwrw07pdlso1tefqv0sw1rces6sn +iluglinuxie +is +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +00000000 +saremsgidlong45 +subject +got +worse +qmqp +listarchive +clickthruonlinecomclickq3d90whzmqimrugoilkvb8hyuluonxir +bebergmithralcom +2002 +xmailmanversion +bulk +compose +email +yes +but +jmlocalhost +url +heavily +domainkeysignature +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +statistics +another +slapd1431 +viewed +up +syndrom +etc +normal +keep +size2bjuly +do +gawd +attacks +cleanly +hand +m8cs255604wfj +thanks +previous +re +host +users +receivedspf +dd4216ffb66f11d6837f003065f93d3atopsailorg +cellspacing0 +0400 +before +archivelatest545 +edt +191644 +required40 +left +it +manipulation +esmtp +subscription +partition +a +deliveredto +achieved +headed +wrote +localhost +vlink0000ff +useragent +as +dogmaslashnullorg +152449 +messageid +arent +signature +10 +loads +changes +163406 +absurd +certain +whose +or +are20 +dpn +cve20093729 +switching +xrcspam +kutcrg56ihbanturtwlbliszt +172541336 +contenttransferencoding +referencesdatemessageidsubjectfromtocccontenttype +6b052 +textplain +invoked +i +127001 +kate +that +out +listsdebianorg +to +mcf5a5dn1oo9b5dtr1pvm971i65sef5r +charsetiso88591 +xlevel +leavelghtml2534368nsprocketlockergnomecom +8024132206 +you +perfectly +bank +probably +did +variations +amavisdnew +small +morbus +youve +xlib +here +kamakiriadcom +autoresponses +was +symbol +14587 +references +there +objective +global +at +been +xspamlevel +id +q +listhelp +can +on +c7zjlzcqgwzur3rps1la00dq6flo1hnwukkphddbbnvdtcn7ylwa0kzdrdjaxbp +tasks +06 +how +date +19317254 +f20cs72396wfi +timely +hope +sales +authenticationresults +simple +by +snitterfield +20100420090740589ef48aabydosstargateorguk +1712811331 +gmexim +listmasterlistsdebianorg +default +actually +hollow +2525 +back +1921680100 +0100 +while +20100511211101329chrisaustinukonlinecouk +autolearnfailed +software +not +pc +with +from +hrefhttpwwwzdnetcomimg +mimeversion +215354 +enht +cb93729409a +include +httpslistmanspamassassintaintorgmailmanprivateexmhusers +m515abr +iso88591qvkskvkskvkskvkrxb0a09grxbgrxbgrxbheddveddxi2uzi2uxeddurx +xcheckerversion +103826 +listid +make +02 +fetchmail590 +u1ekk6mhz9kx +xaccountkey +pleaded +ten +authenticated +sender +precedence +123938 +message +hrefhttpclickthruonlinecomclickqebhlpoqkdtzhbo2pcdo6mzbunk5xfr +class +putting +errorsto +which +american +fully +warranted +offer +cc +welldesigned +listmanspamassassintaintorg +be +listpost +c +where +160411 +time +ronljohnsoncoxnet +version250cvs +average +headericsminingorg +xloop +costabel +mailing +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +really +truth +seeforyourself +mutt125i +8219575100 +debianuserrequestlistsdebianorg +tell +no +add +width3d60020 +value +vary +width100trtdp +when +contenttype +xmozillakeys +helpunsubscribeupdate +extremely +input +xrcvirus +kmailkontact +prereqpm +of +xcopyrightc +ii +t95b77977bcac1b654333crelaykkamdsvru +and +debiankderequestlistsdebianorg +received +bpersonal +forteanaunsubscribeegroupscom +154706 +207615143 +broken +the +those +advertising diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_25.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_25.txt new file mode 100644 index 00000000..e6878496 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_25.txt @@ -0,0 +1,266 @@ +663815126 +enable +lookup +for +postfix +rpmlist +mailservice4imakenewscom +colorff3333 +smtp +coke +works +08 +people +any +arsasha1 +resentmessageid +this +bgcolorffeeddtrtdp +operating +based +your +returnpath +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +bac0097efa +href3dhttpclickthruonlinecomclickq3d63tpkjihwgw06jvwtg9h +imap +point +communication +learn +makes +00000000 +fruit +ehlo +subject +4 +zlqec01hpaq0da54vlbliszt +path +ipfwadm2ipchains +asciiribbonorg +nuno +81168116egwn +perimeter +2002 +bulk +want +slashing +g97ax1f27670 +email +placing +listunsubscribe +disk +jmjmasonorg +15 +0000 +debiankdemgmaneorg +helodomain +4bd9a53c50503hardwarefreakcom +domainkeysignature +4bbd0c094070304coxnet +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +forkxentcom +ist +ssmtp +apr +oct +big +so +22 +suggest +domain +u +cd +re +droits +vm +w8rf9f9f9f9f8pi86ql1r86eb +httplistsapplecommailmanlistinfojavadev +receivedspf +fiber +every +set +gmailid1286e140ec35698f +required40 +fails +it +sun +esmtp +partition +a +httpsecuritytrackerdebianorgtracker +dont +celebrated +deliveredto +khare +xmozillastatus2 +rssatom +008 +wrote +euniverse +localhost +useragent +as +231 +pdt +messageid +10 +unknown +tmp +0700 +gnulinux +computer +aaa05627 +equivalent +contact +each +far +xspamhits +section +i +127001 +lw15fdlaw15hotmailmsncom +increased +zzzzexamplecom +sending +to +re2 +permanent +singledrop +xlevel +fork +probably +fact +httpwwwstudentmontefioreulgacbemerciadri +woman +behalf +michel +mlsubscribertechcsminingorg +alpineosx200100425170236091673teraqnysevanohynkbet +jetlagged +was +references +there +mxgooglecom +at +thick +id +20100422 +also +listhelp +3d78488202002url3dhttpwwwharpercollinscomrdauthortrackerord +on +nt2000whatever +06 +wants +check +my +diskapproved +date +bordercolordark3d0577ba +httpwwwnewsisfreecomclick183910261440 +friends +bgcolorffcc00img +sendmailbs127001 +border0atd +income +sniffing +authenticationresults +165138 +googlecom +xuidl +blackouts +by +exmhusersadminredhatcom +64 +lisztdebianorg +even +listmasterlinuxie +083303 +ldowhitelist5 +back +again +samleafdigitalcom +not +listsubscribe +with +from +032105 +nocluefullstop +fixed +means +sneaky +mimeversion +0500 +valid +quoted +f3a6613a48d1 +clickbelow +tongue +both +catalogs +xmozillastatus +listid +20722822514 +make +put +charsetutf8 +charsetusascii +sender +precedence +anyone +mr +crelaxedrelaxed +errorsto +american +japan +apple +testsclickbelowfreetrialhtml5070htmlfontcolorblue +7bit +trtd20 +ruled +cc +215021 +get +connected +style3dtextdecorationnoneto +playing +better +case +sort +bgcolor83a3cb +xloop +mailing +valignbottom +81168116 +manual +contains +current +xoriginalto +damon +8219575100 +kde +mogmioslocalhost +g81eebp05889 +writes +some +g14eg9324446 +known +policy +contenttype +required70 +like +other +of +01 +and +debiankderequestlistsdebianorg +194152 +received +youll +gotta +high +hindutva +mailtoforkrequestxentcomsubjectsubscribe +c241e43c45 +the +outofprocess diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_26.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_26.txt new file mode 100644 index 00000000..91aa21a2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_26.txt @@ -0,0 +1,298 @@ +archivelatest576502 +bgcolor3de5e5e5 +w4rrzhfmqdd +lists +ratehard +le +brandnew +snapshotdebianorg +for +organization +postfix +despite +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +vercauteren +test +works +212206 +working +admin +begin +ponce +arsasha1 +subscriptionsa +resentmessageid +name +094131 +dkimsignature +this +cached +have +returnpath +imap +point +is +testsldosubscriberldowhitelist +god +too +show +httpwwwstewartsorgviridiangcs +server +subject +4 +e6a6716efc +450 +using +am +interview +httpradioweblogscom0111823categoriestechnology +2002 +rpmlistadminfreshrpmsnet +email +quotedprintable +rh +jmjmasonorg +0000 +co +bgcolorccccccimg +reporter +44 +8091224249 +yyyylocalhostnetnoteinccom +warrant +ist +apr +uses +george +declined +thu +keep +sucker +friend +tue +rest +26 +domain +loganx +iso885915q7viiikvymunjiyr0a09bjfritjgt9elq7hgvhv2rta7r5xnlttjbtq4ks +iso88591q0a099cqewexrexa7tjlyn9gscdhbdwpreheun7x1kjxybf0zhlwsns +highlight +look +badf3c947a1bd54fba75c70c241b0b9e762e67ex02idirectnet +img +enter +exim +gmailid128a8127ad83d910 +110520 +0400 +e20cs5795wfb +edt +c0kfyqwpazcycxja4139tj1l0xdrrmodify +it +esmtp +httpxentcompipermailfork +protocolapplicationpgpsignature +tilted +24 +a +dont +deliveredto +office +xmozillastatus2 +wrote +aqexddo2ut3ya10 +localhost +as +dogmaslashnullorg +inline +size3d1img +has +messageid +10 +7bdc9 +httpserialdeviantorg +yours +dkimneutral +errors +xspamstatus +yourself +resentfrom +public +contact +080055 +they +height7 +contenttransferencoding +color666666 +sue +2008110401 +im +stop +textplain +127001 +outlook +8bit +work +that +listsapplecom +compensated +listsdebianorg +110524 +to +sep +httpquote6xto +singledrop +preferably +20080610 +rogers +you +bank +did +amavisdnew +namesymbol +causing +nowap +pictures +jmilugjmasonorg +moutperforanet +here +y8t0kzx86jkt +hrefh2 +size1bfont +was +inreplyto +references +virus +countries +kick +something +consumer +at +1 +go +id +mention +north +150205 +also +listhelp +showstopper +can +on +updated +how +4c2c3340a2 +gamasutra +date +3pop3login +take +everything +l +wondering +141107 +by +ldowhitelist +ftpftpximiancompubximiangnomeredhat62i386gaimapplet05911ximian2i386rpm +reloading +xmailinglist +ldowhitelist5 +kmail +2525 +xbeenthere +0100 +xantiabuse +not +listsubscribe +with +from +164134 +20 +mimeversion +forms +enht +331kb +helsinkis +first +aligncenter +hrmbaf8ecdagaqhageameqga1udiaq9mdswoqylyiziayb4rqehfwewkjaobggrbgefbqcc +100557pm +d5ee0d0e4119 +pass +xmozillastatus +siblings +msnbfontbra +boy +use +listid +put +charsetutf8 +news +charsetusascii +xaccountkey +kids +zzzzilugexamplecom +sender +end +permitted +gmailid1286781ea6f3ecd9 +precedence +yyyyspamassassintaintorg +crelaxedrelaxed +which +boundary001636498af9108e5704852722fd +g91816k15547 +aug +cc +be +listpost +brbfont +into +get +matter +were +143641 +1712811333 +configured +logmaneorg +titlefirststop +30 +3598 +really +techtv1 +fri +startx +hlbznl6gbcpkbnyturkaybg8s0u2na2mkr4zx89cjld1olpfzlglwnce3q9piblu0k3 +about +many +233041 +debianuserrequestlistsdebianorg +3 +httpradioweblogscom0111917 +no +add +er3d0 +181142 +2si9239304fat5820100510114042 +buying +giggles +does +httplistsfreshrpmsnetmailmanlistinforpmlist +1000 +contenttype +xmozillakeys +of +record +received +without +waxed +173951 +ms +toys +list +phobos +nominee +accessible +mail8230 +3d +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_27.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_27.txt new file mode 100644 index 00000000..ceb53b07 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_27.txt @@ -0,0 +1,339 @@ +liquid +e +footerrow +tbody +cells +bl97ck +quality +please +copyright +able +porn20 +xentcom +for +holland +find +school +smtp +charset3dwindows12 +whitesolderwhitesoldercombr +garnered +wars +years +illuminant +this +incorporado +in +returnpath +imap +dmeizyshost11websitesourcecom +are +is +hojypayn5zkfobxiuhks7mlupgiiefz958na9mvbv7omwq0hlighzactlhs4uxs5wtjduxhb9a +versiontlsv1sslv3 +o4pn5pbp6i6evzhj4t9sur1tqq08n4vgjhnkrqqrxmdrzwnmp0sdq4uumtvgjj0u7vk2usqk81fl +00000000 +drawn +213105180140 +everyone +reputation +request +subject +taoist +relay +given +less +oawztjhvelmcauq0wdxessmjngsvaneuzi80qyspeecakdq6ltgatyjnhtx8ahjwfmomd5a4b +3575 +system +imo +brthe +email +sit +sa290hl +quotedprintable +easiest +href3dhttp31ada4fdekemoscnao3d51990ef85a6ab65c10b3xa3d305 +yes +sbr +newsletterbr +below +meantime +ruined +102313469 +annuitiestitle +role +giving +sedge +setting +ns2csminingorg +c90665e7staticspovirtuacombr +style +j +voor +19216818250 +bgcolor3dd0f4e5161td +description +ortonfontfont +prizes +xbarracudaconnect +must +thu +keep +head +u +agency +previous +thornton +club +grammar +run +f +price +nor +data +undisclosed +us +left +it +persons +sa290rn +esmtp +hermes +atmosphere +019 +a +ngfgxcdo97xooaj3wbrxgy2her6lrtykmoask66h7oa0dufvuz8droayt48kpzc64u1mcbq +1243153684 +deliveredto +xmozillastatus2 +manufacture +instances +provides +super +xbitdefenderwksspamstamp +for650 +localhost +dogmaslashnullorg +srchttpclo3bringboolcomeujmiaroxoutxqteiorfrlkafilboaqoinoilhzqxrxiaooxio +rule +affixed +well +5 +messageid +350 +year +g4jk9he27944 +direction +10 +xasgtag +brjust +unknown +kors3drating +linked +ig1hzgugjdiwldu2mc4wmcbhbmqgynkgdghligvuzcbvzib0aglyzcbtb250 +preservation +clientip47146 +6521715966 +public +doctype +peruser +following +0200 +contenttransferencoding +xbitdefenderwksspam +bought +textplain +i +127001 +outlook +crttxnskccx71jzkmoadxtagzzrg0m9bdt1gaskrpmyddopkeelmrriaogiimwlcpxoooodlakm +that +aactive +tg070 +h5differenth5 +sending +styleborderwidth +to +v1 +sep +charsetiso88591 +washington +width3d15 +code +you +td +mendes +around +isstrong +face +callingbrbr +planning +font +larger +loan +hours +akahutnzs0aoopm0uals0lfac0ullqatfjrqatfjmjnadqkbz6gl59dqatfn3h6f0o3h4fzfad +establishing +127339970678f632040001w4twrl +at +low +id +bnqpc9wpg0kicagicagphagywxpz249imnlbnrlciipgzvbnqgzmfjzt0ivnc3osxpiibzaxpl +structure +over +ratio +remain +333a +can +htmlmessage +href3dhttp4 +billion +date +xbarracudastarttime +31 +epercomstorecheckoutaddtocartaspxitem3d49115ampqty3d1ampsid3d1 +sampled +ocw +hrefhttphomewebstakescomplayhomepagetemp2201tbid301 +dutch +2009 +xuidl +mail +refusal +by +clothing +visit +rovdb001rovingcom +z249imnlbnrlciigywxpz249im1pzgrszsigd2lkdgg9iju5ocigymddb2xvcj0ii0zgrkywmcig +passerines +gw1csminingorg19216818250 +new +actually +will +examination +style3dcolor +2krgqumcdtinzm1jibk7btjgaqtk5zuwkl8zmut901bclvbinbq1wbka7vz2hvzzvk2y7verx5u +management +0100 +2000 +with +from +thru +subscribed +size1 +dr +vwp +000 +2 +rates0a +191941 +enjoy +g56bvcm12617 +aligncenter +mail3csminingorg +sincere +xmozillastatus +stronguk +maili9casecom +gw1csminingorg +make +words +exclusivity +rolex +message +ultimate +sub +largebby +an +attained +brought +equipment +3b346e +estore +be +attack +universe +dominateangiosperms +align3drightfont +viewing +brsponsorship +chiara +aligncenterpurged +texthtml +xvirusscanned +201232116109 +charsetwindows1252 +spa +title +case +attend +xbarracudavirusscanned +w3cdtd +ybiz +uid123851201487268 +mailing +youbr +html +jmnetunlimitednet +n2i7tpw2fid5ksm5jaxqbn3dznawvlkwuxw8uqghkc4ph41ca20eu2zv8uzzc8vv8z7duvo +pget +card +really +bordercolor3d222366000022trtd +rbbjpbnxx2u2fkjx6dqhvcqttgmmq5rpy0pb0xk2s5pd664dfnc54fjxynjgls8v0mgxwccvuuyl +throw +size3d5find +125926 +federation +claimed +spam +m0620212mailcsminingorg +0900 +deaf +many +html1dtdxhtml1transitionaldtd +except +1992some +font13px +3 +images +pirajv1e5egnhgncfxe5qyotahwzwjal0snwjzzcnuil2dm8xn2kydvggktu6zjblit0ki3r9nm +0px +a697a8a553f +no +moved +important +buying +respond +magnitude +when +known +jst +c111ntaine100 +contenttype +may +like +of +iplanetsmtpwarning +states +csminingorg +and +received +one +bas +qrrt6kpx9ajmesjuifxgbqmg1t5b3trfc3l7h8x7rwcnka2c7cubyee9aqkn8phoc4zxu5rmofj +893887 +nonsense +days +table +polanyi +417 +cage +pussy +the +died +send +xpriority +those +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_28.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_28.txt new file mode 100644 index 00000000..9edec4f4 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_28.txt @@ -0,0 +1,358 @@ +wickets +e +recipient +poker +valigntop +theology +read +reserved +height423 +for +support +smtp +jekyll +span +usabr +works +jyyyyl19582000rediffcom +due +170351 +wi +8 +registered +name +hrefhttpcvq19bixyeqawcnuywiimqxyfeludei1da342067ytjerasaoilqsalyofe +vacated +size3d2fontstrongbr +transitionalen +have +your +returnpath +suspicious +arkbishopnetmorenet +breakdown +is +require +bgcolorffffff +dust +nicely +account2 +184004 +020080910 +text000000 +market +subject +already +custom +irem111s +independent +using +size +produced +innove +bulk +httpnewsbbccouk1hiworldafrica7380412stm +centerfont +below +best +insurance +hqfharp5crqfm9o2 +text3d000000 +hrefhttpac667agnaburencndoaikao57e3h069y9ne394iw4806aijajot85632379588289720455838 +babe +different +fontfamily +risks +ist +line +project +center +injections +br +19216831 +tag +3223323 +fontfont +bsfsc0sa424 +cypherpunksforwarddspronsnet +3cp3e3c2fp3e3c2ftd3e +filing +presidents +xbarracudaconnect +all +025754 +xyqemyikog8445ntlcom +distribution +9999trtdtable +january +uid74941201487268 +e29840freeservenet +host +dense +announced +width3d138 +nor +o3cayblk024400 +cellspacing3d220 +cellspacing0 +shers +submitinput +nncnprivacyadn +us +site +esmtp +herstnameali +term +jaundiced +a +idformsradiobutton3 +deliveredto +kaa09351 +river +hereanbspbr +localhost +as +action3dhttp65217159103responseresponseasp +div +rule +mimehtmlonly +2092331622 +has +messageid +350 +okay +s117ccess111 +settle +500 +10 +xasgtag +mime +17800008157tipsmtp1admanmailcom +unknown +align3dcenter +wilson +removed +hop +brmy +leaguewide +russia +styles +xbarracudaspamscore +bits168 +concepts +or +above +tg242d +halfway +public +london +textdecorationnone +doctype +03 +contact +varies +peruser +each +youth +undisclosedrecipientsdogmaslashnullorg +growing +4zuveehwa4ekh4ki3wakyji4rjjmpzxngzzzzifrn9ju9sqq4r7nt7vxktpqvvha4zy85m8ou7 +service +zzzzlocalhost +030834 +i +rpbl0wfb +stronglocations +8bit +be6dqittomailseedernettw +helvetica +aactive +styletextalign +mysterious +airtelbroadbandin +j41mr4975443qac1021273668184378 +hrefhttpmta16rle3fitbrowserszcomsakxkkieaelbwxxfouoeabr +styleborderwidth +heck +to +charsetiso88591 +singledrop +code +you +td +meant +jun +basolateral +97b115111rb +they39re +ampamp +testsbsfsc5sa161f +recently +mathews +privacy +suspectedcenter +was +brthat +six +stylecolor075c83fujian +size3d1we +change +valuemontanamontanaoption +taken +face3darialthese +sirius +replyto +been +id +authorities +offers +nbd8trdw010837 +also +math +on +bsfsc5mj2808 +vision +my +width770 +after +date +gmailid1285a9a528d91c61 +paraphrastically +31 +classnormalnbspp +easy +marketing +xbarracudaspamreport +xuidl +them +g6v6jzl9008819 +111nly +by +then +remaining +href3dhttptawegjunpu +032 +zzzzlocalhostexamplecom +strongmanage +httpspamgwcsminingorg8000cgimodmarkcgi +should +helo +new +height3d326 +entrusts +styletextaligncenterbfont +university +their +alignrightof +pvisit +acknowledging +romanstrongviagra +81168112 +times +with +from +20 +612303188hinetiphinetnet +bbb +mimeversion +scores +border0 +href3dhttpb93d7csorowxfcnx +valid +000 +realtime +first +orders +killlevel10000 +width91font +213845 +uid +fortune +6hbq5eugs0syqxfcbvwrnjoavbqn9rb8dqxzf4yeoprzald80uopmb6djvnjfrsx0milsuqkd +xaccountkey +xqrhdjksd7g0kjc1m6lttgwdbyp8zitoeg9oa0j0prro0iuygsn3srym8gmblm8uibvwlbf +rather +regards +1100 +targetblankfont +message +which +19216818251 +place +htmltagexisttbody +hot +10pxninefonta +jp +145156 +financial +targetblank +suggests +be +gw2csminingorg19216818251 +optin +millions +colorblueucustomerserviceconsumertodaynetufontatdtrtabletdtrtablefontcenterbody +rules +edge +million +sev +bgcolor008000 +kaso +guess +heart +austria +html +identified +district +vt2euoymlvinyloz7dvhutahahvarhx31xpuidjlruluhlhstjce35y4jrvvub9zonqflclojo2c +pthirdp +c97lled +commandstrong +height443 +size3d4dear +started +height2 +uid87591201487268 +0740510254 +width100 +thin +no +ele11897ted +we +sat +243 +altcant +past +184851 +rlohtqagu34grdddvce2oeafic4ditadez0imgcbkrkcb0abz6agpq3tbmagm3kdaodiubdtmqag +203953 +contenttype +husbands +action +suburban +may +of +filmography +dynamiclooking +234239 +prime +and +received +schoolsdiv +without +sa392hl +wed +guesshowphilippinesto +list +wish +pppppppppp +parttime +wear +ne97rby +nvvyh +the +parts +msncom +joaquina +10pxshowsfonta +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_29.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_29.txt new file mode 100644 index 00000000..c18b8a45 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_29.txt @@ -0,0 +1,279 @@ +allowed +oldreturnpath +qmail +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +230839 +clientip8219575100 +this +operating +in +cached +reviewsfonta +your +off +returnpath +imap +fonts +mailtodebianuserlistsdebianorg +cheers +are +mon +his +17csdl0006nt00 +gtk +mozilla50 +62 +9b2cc294162 +logging +subject +minimal +bits0 +got +qmqp +produced +listarchive +floppy +xmailmanversion +applications +listunsubscribe +exist +jmjmasonorg +0000 +dock +what +different +would +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +yyyylocalhostnetnoteinccom +daniel +image +ist +if +up +process +btw +going +iso88591qsuvork5cyii3d +all +tue +lairxentcom +loaded +fishermen +re +stuckey +releasedbr +users +data +result +exim +0400 +dcsminingorg +bold +independence +summer +just +required40 +understand +it +esmtp +cable +gr +listsdebianuserlisztdebianorg +local +a +blountmailmindspringnet +laptops +deliveredto +xmozillastatus2 +pages +resentsender +marginleft0cm3b +wrote +rdnsdynamic01 +binoapplecom +rv18122 +localhost +localhostlocaldomain +as +dogmaslashnullorg +build +thing +spfpass +well +regular +messageid +newsgroups +10 +signed +isnt +xheimat +c14mr2365900fac641271053957951 +phoboslabsnetnoteinccom +0700 +xxsender +xspamstatus +balance +httpxentcommailmanlistinfofork +resentfrom +looking +contact +english +contenttransferencoding +2008110401 +123229 +textplain +i +127001 +work +bright +that +numbers +xmimeautoconverted +7d7df6c8406 +to +sep +singledrop +kodadek +now +whateverdudecom +20080610 +you +select +amavisdnew +pvtlksi4h14grnyqzjzu7et56abxruytfzsu +most +dynostaff +v +mlsubscribertechcsminingorg +help +internet +version +references +there +mxgooglecom +change +hash +safari +doing +1 +debianuserlistsdebianorg +archivelatest32290 +id +dollars +20100506113507080martinlichtvollde +can +on +085359 +designates +my +date +friends +devsda +bugs +055351 +philip +googlecom +scheme +by +he +unsubscribe +sticky +php +ldowhitelist5 +2525 +xbeenthere +virtually +wont +not +with +from +stuff +5fbb513a55c3 +rohit +20042010 +mimeversion +color000000tech +press +oj0dylyiv3maqfhsq9lbliszt +src3dhttpwwwcomicscomcom +clicking +kramonkeyorg +gmailid12857b07c187e9f3 +2dfad2e8b07 +could +boy +listid +194737392 +jmasonorg +charsetusascii +fetchmail590 +xaccountkey +python +permitted +precedence +unfortunately +didnt +leading +101428120 +an +dir3dltrlta +7bit +discussion +be +listpost +agreeable +200209250800g8p80mc18085dogmaslashnullorg +videos +get +xfs +skill +fine +hrefhttpclickthruonlinecomclickq48ggfbidbsyrcd3fxlkj2zbsju4r +20050822 +were +101 +latest +xvirusscanned +hrefhttpclickthruonlinecomclickqb2m4pwqx04ac0vi5yjm2cmpxucpunr +215 +110326 +xloop +bssiguanasuicidenet +pigeon +close +about +roomabfontbrimg +debianuserrequestlistsdebianorg +3 +17so692929wwb6 +no +we +vzplkzmixhkojbcdin1rv7znqnnsoaxnvq5uj4obfkubnddfrxq8cwakyr9ruuk1ap +marginheight3d0 +ubuntu +does +contenttype +xmozillakeys +volume +xrcvirus +protocol +may +like +of +record +speak +received +right +later +datajavascript +080030 +list +exmhusers +phobos +fee +the +ripped +cant +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_3.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_3.txt new file mode 100644 index 00000000..ce65102b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_3.txt @@ -0,0 +1,286 @@ +used +gmailid1288b70f84605c36 +necrolicious +namesignatureasc +legacy +hrefhttpclickthruonlinecomclickq87izdoq8nal8swntreplrq8kasp4er +xentcom +e15si15681839fai2320100518063939 +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +2wlstsouth +post +12 +formatflowed +httpenigmailmozdevorg +this +in +activated +httpedtechdevorgrights +parameter +returnpath +imap +are +is +mon +uswsflist1bsourceforgenet +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +testsldosubscriberldowhitelist +listssecurityfocuscom +testsbayes002 +managing +slow +additions +g8a1sec11916 +080131 +subject +4 +businesstobusiness +qmqp +using +ghostscript +system +2002 +micalgpgpsha1 +httppamelajoycomfriendsindexshtml +a10642941c4 +email +listunsubscribe +b4f1329409e +but +httpwwwpythonorgguido +0000 +co +jump +what +yyyylocalhostnetnoteinccom +g7d3fta07331 +recent +libgtk20bin +ist +line +if +good +hrefhttpwwwlockergnomecomoptionshtmlview +g8o80cc26642 +oct +big +fallback +required53 +testsawlknownmailinglistquotedemailtextreferences +btw +so +advertise +tue +thanks +682b016f8c +native +natures +page +result +class3dgmailquote +0400 +mailtoforkrequestxentcomsubjectunsubscribe +just +talked +325 +it +99 +option +sc +listsdebianuserlisztdebianorg +a +dont +encountered +deliveredto +didunfortunately +bhgtbsrdwc2mtohzkkko3mqtpdpc3gt47lyk4cwdvfr8 +instances +therapy +administrator +la +nosend3d1 +localhost +bad +useragent +httpsecuritydebianorgpoolupdatesmainppostgresql83postgresqlpltcl8383110lenny1s390deb +as +dogmaslashnullorg +sdw +resembling +8507e16f6d +pdt +hydrogenleitlorg +messageid +popescu +10 +w39mr393328mue201271360368122 +ilug +unstable +string +or +resentfrom +interested +knew +ornament +contact +trying +far +height7 +exactly +contenttransferencoding +2008110401 +i +127001 +outlook +wheres +sending +102100178 +to +singledrop +dvd +xlevel +tlsv1 +you +selectnbsp +479 +rafters +did +amavisdnew +auth02nlegwnnet +mx1redhatcom +font +331vamm2 +badattitudes +v +was +inreplyto +references +entrepreneurinresidence +exmhworkersredhatcom +at +1 +height27 +id +123208 +port +listhelp +esmtpsa +on +designates +1944d2757dd3b +date +jalapeno +122432 +knownmailinglistquotedemailtextspamphrase0305 +everything +authenticationresults +me +them +artist +althandy +by +unsubscribe +listmasterlistsdebianorg +should +helo +will +ldowhitelist5 +2525 +xbeenthere +213208101238 +anyway +listsubscribe +with +from +stuff +063917 +rohit +gave +wood +mimeversion +marjorie +scannell +belphegorehughesfamilyorg +pass +use +mode +listid +color0099332342fontfonttdtr +fetchmail590 +xaccountkey +1866529409a +width1td +sender +precedence +iraq +yyyyspamassassintaintorg +rendering +its +which +9029100 +apple +warranted +7bit +194439 +jac1 +be +listpost +ldowhitelistpgpsignatureratwaregeckobuild +into +quick +ok +her +xurl +texthtml +rights +xvirusscanned +sites +ignored +zzzzteana +104939 +m8cs86457wfj +more +rahshipwrightcom +ja +mailtoaugdrequestlistsapplecomsubjecthelp +xloop +mulls +10143320 +ride +httpwwwsupahcomblogwebloghtml +xgmanenntppostinghost +start +close +throw +tm +8219575100 +kde +no +we +installed +httplistsdebianorgaanlktikbesmxdiguc1qudkg4wawugjjcncjxznvqmailcsminingorg +games +when +group +version325 +packaged +contenttype +may +like +of +caf1 +height60 +and +xstatus +received +setup +youll +list +hrefhttpclickthruonlinecomclickqbczkpjq6bdhniivufmobf8xedxwcyr +went +drawing +the +cant diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_30.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_30.txt new file mode 100644 index 00000000..1cb4685c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_30.txt @@ -0,0 +1,378 @@ +ideas +used +e8a2a9a2afa4bcf7f9fde8ada4bef1e9eaecbbb8aff6eaa4bc +xasgorigsubj +species +vorbis +3217262209 +copyright +read +xmailer +reserved +for +3dce10c0osakametanejp +lots +xbarracudaspamstatus +bsfsc0satofromaddrmatchhl +construction +people +any +color3dff0000366fontfontdivtdtr +body +8 +name +this +nextpart00000af01cae1d4d3100800 +in +content3dtexthtml +provided +164 +0725 +firewall +have +your +returnpath +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c26jpg +0001 +mon +httpwwwceucominteractivecatalogueaspcategoryinsurancececode +reconnaissance +silverseacolliernordiccombatantsnormaltnotairetoadspiecrustscombinecomexpertnancyfullpowercommandantournentcommconcernednymphscompetesobeyedoffenderscommunicationsoffbeatoccurrenceoceaniccompartirlosfayettevillepagecontentoffriravaporizedpasseofficialthree6mafiacomposersconccomprobadaonlineversionsources1conicstaresordinarioikisakiconciencia +value6363option +100 +070 +greek +margintop0px +subject +4 +billions +march +oja3ide4ojq2ojexaaeaaaabaaaaaaaaaaoaaaadaaaaaqaaaj0aaaakaaaaaaaaaaaaaaagicag +included +economic +200209171118171de832ce54smtpeasydnscom +email +yle3dfontsize +quotedprintable +yes +jmjmasonorg +22500 +101731 +url +jerk +permission +hrefhttp33pharmlalorueiukoyh3232673313894 +similar +what +httpwwwinsiqcomlegalhtm20 +genier +g4f919e26178 +fflqalfffabrrrqauuuuaffdht2n76ng3agnaojxsf2tfaa4jxfwl8tnkk6pbtp30 +li +n4j5hqgb011103 +pa +12319164124 +good +world +19216818250 +tool +v8820 +titlenewstitle +own +4521 +delivered +up +color3dc0c0c0 +daring +7pxtop +size3d2nationalnbsplenders +expected +miyglzuoaad3crvcmlmuqcsfucabzjomhwkbngqxyqrmdmy2ghxbhyfvwvwx7uccayzqitdacup +must +normal +newsletter +band +all +domain +sa161f +amavisdmilter +destructive +formatfonttdtr +infobecominglakestrangercom +receivedspf +held +cellspacing0 +before +centerborder +337103918p +subjbuy +border0abr +edt +just +understand +it +170327 +ns3csminingorg +d2teuki45zudse4evr6x0je6zifzmxntncyweb9bv36nkyl2rocpte52sbrkqn4omfb83z +esmtp +hrefhttpa6e66269c2fiqogascnfont +feedbacka +reading +impact +ntd +medellcantomaximisembbscapitaismelhorarcaptionmccluskymcglonenisquallycardenalcatalnavigateurcbinmegumimelatoninmedicamentscarvingcaseouscasuallymeianeuroscientistsmessegecastigo3dmemorialmessageriemencionarmentalemicroorganismtweediermesajicheerlessnessrisikenmesencavomeshtopcatholiquemessagelabs +investigation +a +deliveredto +participants +xmozillastatus2 +jmnetnoteinccom +hibodycsminingorg +responsibility +receive +hormone27103 +color000000 +barracuda +localhost +dgvzisbuagvyzsbpcybubw0kicagicagicbhzhzhbnrhz2ugaw4gd2fpdglu +as +dogmaslashnullorg +criminal +build +thing +because +mailepiser +load +pdt +has +messageid +citing +sftlandstrongacenter +vehicles +kn111wn +10 +aligncenterap +mime +removed +mkr021csminingorg +xbarracudaspamscore +bits168 +account4 +uid42211201487268 +600600016788 +computer +ireland +doctype +tournament +lease +southernoption +010 +reaches +30pt +taglevel10000 +provide +contenttransferencoding +way +growing +xbitdefenderwksspam +formerly +112627 +size3d43e3cb3eaddresses +until +127001 +avoid +work +windmeyer +httpwwwbarracudacomreputationip88119247115 +that +22believe22 +venture +net +align3dleft +quarantinelevel10000 +ffffff20 +opened +robert +to +sep +giveaways +fall +individuals +decokycim +you +letter +tazas +td +bodys +willsdekalbnet +small +around +legalstrong +webnotenet +most +zzzzjmasonorg +these +userfriendly +mxgooglecom +batch +style3d +response +steal +experience +2052104230 +african +seriously +xsender +go +low +id +meta +conceived +allowing +can +cellpadding40trtd +on +finance +06 +htmlmessage +icagigh0dha6ly92awmuadholmnvbs50dzwvyt48l2zvbnqphapgzvbnqgy29sb3i9iingrjaw +150858 +remove +how +my +argentina +after +date +spamgwcsminingorg +republics +labels +omaha +marketing +xbarracudaspamreport +2009 +eive +i3tem +xuidl +mail +by +informationbr +he +110323 +unsubscribe +listmasterlinuxie +thats +anniversary +few +back +p +addr +with +from +0 +marie +97dd +colord4d4d4 +mimeversion +scores +border0 +stylefontsize +brokers +icagpfrct0rzpg0kicagicagicagicagica8vfidqogicagicagicagicag +first +bobsled +bharatpur +ig1hdhrlcnm7ig5vdcbvbmx5igl0igfmzmvjdhmgbwfuesbtzw4ncybwzxjm +aligncenter +height3d38 +contained +breakthrough +make +sent +charsetutf8 +available +yuan20 +4000 +sender +end +lingers +influence +its +noneindustry +which +banks +width3d77 +xkeywords +4jrhvofv9hfyhlm43r4r8b9vyf3mroyyjao0okwsvkwalf6gx1cw7mj6s4uzg3xtjpaks +breakthru +sembrar +be +c +funding +angewandte +get +httpequivcontenttype +texthtml +rights +xvirusscanned +part +statistical +raa04566 +neutral +environmental +html +between +mail1csminingorg +n68igoio006860 +cellspacing4 +299 +fri +claimed +about +guitars +717b015f77 +0900 +family +encyclopedia +fontt +against +no +we +sfnet +annadurai +geneva +bebcf1e7e3bfbdbbe5a4a3a4a3b8aaa4e2aba5a1e4a1a1a9a +026e57a75a1e6748e2d28cc57dd3fhlecl +former +romanfont +contenttype +xmozillakeys +unsupervised +changefontbtd +possible +gener97ted +may +like +listed +of +and +000000padding5pxfootertextfontsize10pxcolor333333lineheight100fontfamilyarial +20015322298 +sponsored +received +without +19216818120 +today +liberalized +sa392hl +free +prefer +1014012612 +reservedspanfontt +trtablebodyhtml +amounts +armenians +active +the +00004151338b0000771d000023behighway212com +d +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_31.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_31.txt new file mode 100644 index 00000000..cd448243 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_31.txt @@ -0,0 +1,267 @@ +httpwwwyaysoftcom +used +height10tdtr +for +postfix +2a1662698b7fb +mailtodebianuserrequestlistsdebianorgsubjecthelp +network +smtp +jnlps +gnupg +x +module +tells +this +strict +in +collaboration +cached +have +returnpath +mailtodebianuserlistsdebianorg +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +why +testsldosubscriberldowhitelist +backup +moon +135 +subject +got +qmqp +name3dsubtle +system +gate +2002 +xmailmanversion +bulk +want +winter +forth +ambiguous +but +0000 +below +upabfontbr +role +master +34fc541415163b94a68a0e2fc782eda9squirrelwwwtty1net +zzzlocalhostnetnoteinccom +fourla01 +20100520221017618tehpehgmxnet +good +replacement +apr +required53 +141556 +so +higher +must +26 +domain +m8cs50394wfj +thanks +quicktimeapirequest40lis +pretty +re +receivedspf +68 +clean +required40 +516 +it +esmtp +mailtoexmhusersrequestredhatcomsubjectunsubscribe +cable +protocolapplicationpgpsignature +unrelated +g99jcrk18524 +a +dont +deliveredto +meatspace +xmozillastatus2 +jon +maintainer +resentsender +described +wrote +google +localhost +cystals +as +build +div +ldowhitelistratwaregeckobuild +2626 +pdt +messageid +beating +signature +10 +consideration +welladjustedde +buffer +dept +phoboslabsnetnoteinccom +0700 +httpxentcommailmanlistinfofork +score6901 +questions +equivalent +command +they +o4nakqjn008152 +0200 +hrefhttpwwwlockergnomecomwebcamhtml +exactly +contenttransferencoding +answers +2008110401 +im +comic +importance +127001 +lost +testing +assumption +files +udhay +that +dvr +to +completely +sep +alternative +singledrop +xlevel +now +cscomcomicsdilbertdailydilbertimageswhitelinerightgif +you +expert +mimeversioncontenttypecontenttransferencoding +bank +hrefhttpclickthruonlinecomclickq35d6jtibizinohlvsknxltsypwc3pr +httpwwwstudentmontefioreulgacbemerciadri +altcnet +dang +relay02paircom +making +194003 +internet +inreplyto +there +something +logo +at +through +score5 +id +engineer +listhelp +on +languages +carnot +designates +date +sans +mail +adt +by +53 +ldowhitelist5 +wonder +back +exmhworkersadminredhatcom +not +exmhworkerslistmanredhatcom +with +from +rohit +administration +person +forms +enht +mailtoforkspamassassintaintorg +width3d169 +xcheckerversion +make +httpbugsdebianorg534680 +charsetusascii +v11 +xaccountkey +vms173001mailsrvcsnet +sentto2242572527371030024261zzzzspamassassintaintorgreturnsgroupsyahoocom +sender +precedence +anyone +testsalltrusted +albacore +kchristlsilcom +contentdescription +notes +032307 +taggedabove10000 +parted +443 +be +listpost +ntmail +mailtoforkrequestxentcomsubjecthelp +ronljohnsoncoxnet +175423 +101 +xvirusscanned +anything +xloop +hsjl89ciq1doughgmaneorg +mailing +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +delivery +fonttdtrtable +225545 +m6qerttr92r6yn6yggi9vatyoraf022luv4 +spam +rcvdinsorbswebstoxreplytype +helothumperdhhgtorg +kde +103058 +against +153101 +lie +we +archive +022334 +apparent +some +6sx45uoqrijxqkqr +policy +version325 +contenttype +15000 +action +required70 +sid +protocol +may +like +of +states +and +primary +record +exercise +received +windows +derussy +160854 +list +043906 +anthonys +handle +the +httpwwwgeocrawlercomredirsfphp3listspamassassintalk +cant +send +xpriority diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_32.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_32.txt new file mode 100644 index 00000000..4efb1c0f --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_32.txt @@ -0,0 +1,365 @@ +submitted +7armywlgjlhrb9jugriacqpzdof6m23gvcxzgpj3ujmxvtczrrvqdj9gc0b2fsabdc1obz +xasgorigsubj +trueifontb20 +typesubmit +textdecoration +for +liwork +negotiated +network +smtp +organisation +x +package +any +xmlnshttpwwww3org1999xhtml +sa154 +body +102913 +name +this +ip +removemespeededeliverycom +in +content3dtexthtml +margin +height3d93br +your +ar +bayes +returnpath +0001 +are +is +mon +xasgdebugid +estate +12574253362eee00800000w4twrl +coefficient +100 +23 +account2 +mini +archeology +subject +inattentive +independent +xbarracudaspamflag +song +using +am +system +translations +coulibaly +march +cheyney +action3d +included +604 +p97ce +enlarge +jmjmasonorg +standard +spantd +monroe +xxplosive316aolcom +ns1csminingorg +teze +dicky +231152 +pop +best +plus +height356 +gid +would +arnoldrefiinfocom +online +project +center +good +br +treasures +description +neither +size3d2number +genentech +up +applications10with +human +qggjuk8zzezlxzxy6pzoabtaaaaaaaaaaaaaak7vask7vazl0s7e3l1mhyvcwrl6zd5xbyc2ga +classstyle12selfhelp +bsfsc0sa424 +xbarracudaconnect +keep +191241 +florida +composers +bec97u115e +tobfonttd +sitebfonttdtrtbodytable +charsetbig5 +run +l97cquer +look +nor +whitelisted +href3d22http3a2f2fbestsoft2e33222eorg2fonlinedown2fnewees2ezip223edownload +just +nat9gtqg018303 +esmtp +luycysa6peatdbzhpcpukeub7vpv5f8atbf1rh9ntve9qd2p6x5f9otveovfqkv7cufq9sf96e +893893 +24 +019 +a +chicago +href3dhttpmedignaz1cnsubid3drx01font +deliveredto +office +profesionalnih +hibodycsminingorg +xbitdefenderwksspamstamp +barracuda +as +dogmaslashnullorg +personalities +has +messageid +earlyem +jewish +10 +atd +represents +changes +enrolled +removed +certain +0700 +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c22jpg +bits168 +leftysplat +or +letters +nobr +ludwig +yourself +looking +600600016788 +ring +uid112891201487268 +010 +taglevel10000 +following +0200 +headerbarbackgroundcolorffffffbordertop0px +social +contenttransferencoding +multipartalternative +040347 +height112 +127001 +tametd +outlook +humbly +avoid +185413 +c111ntextu97l +java +confederate +middle +coincide +ach4x6f3uh3wvk1oqaykmmvhpb9myw +hardcore +xbarracudabrltag +buy +shriver +to +machines +2700 +71 +drop +value3dsubmit +now +awarded +bankers +code +aitaira +httpspamgwcsminingorg8000cgibinmarkcgi +td +russian +uris +recently +these +italijanske +long +brabdamwbkffcmarxhgjvekznoq4fytikgglqabakgas2i5ngiijeicvuiqf6eyy8oboweqmvca8 +discuss +msgidfrommtaheader2 +proposal +at +1 +ggdv5q5a0bimtr7lquiioheju8nssnof6y2bajxuojbhmpilp0krsskdxwuuvhahkeqsbsn +been +webdeals7a6anet +ongo +id +meta +offers +pts +ratio +can +width3d1 +on +analysis +black +her20 +dqx3fjludqwcoojbz9k60kzzjgatsqdkxnnzsz96lgpzwz4lpabs2rf8axln6aavk9zniq +browseradiv +government +executed +language +willow +party +date +institutionp +31 +12px +marketing +5504134600 +seven +international +allow +xjewbssugnqqjxxmsclbksprkkczpoeausyohpbpd7esd1wc8ep94kjudqiq4lunyllcv5ul +mail +111nly +by +90t6iq0vcq52qwefnjqa2aaiaaaaacnlcbulnzs4kgamepqnzq47w4rhplwjd0t6irqbdmmdu +then +unsubscribe +33155brbr +yill01hotmailcom +ddptqlxcuo7txpzk3dj02q6g5fvdtyu7s0m2llmsfzo3pszlcfifs4tnzkmtcvhojo3t0bgqqm +listmasterlinuxie +texans +back +lived +sa148ahl +again +while +gmail +software +trusted +not +with +from +mladog +still +selling +yard +mimeversion +scores +size1 +border0 +27 +stylefontsize +easily +000 +methodpost +advertsprj40cs2ecom +144d02db15 +discounts +xxsmall +xmozillastatus +iv20rdhg8e5io9hqxi7kuvc5fyrk1 +xaccountkey +fontcondoms +10px +file +an +xkeywords +heat +offer +be +grantsfontfontfontbh2 +gw2csminingorg19216818251 +rep111rtes +ringsfirst +h +themselves +rules +time +xvirusscanned +eire +bm9uzsigywxpz249imnlbnrlciipfnqqu4gbgfuzz0iru4tsuuidqpzdhlszt0iq09mt1i6 +existsxmailer +case +microsoft +attend +headers +becauseof +w3cdtd +undisclosedrecipientsmandarklabsnetnoteinccom +fromnameufajdcom +terri12hotmailcom +html +stylecolor +size3d1 +none +spannbspstrongfontp +spam +hasnbsp +wife +0900 +liverpool +sansserifb20 +3 +images +sjiaisliljyiisiymiojqiiziwlcmjyiswikkyjjuhyziokckjahsviyjsgjqfiwiikiejihypic +no +domino +very +we +93129105218 +unsubsab749eca98cypherpunkseinsteinsszcombverticalresponsecom +operateout +persian +leadstitle +fronts +scqnbz81zsh6hrxlv5kvhpfhfapuerr5jcuiqks3afnrxrojup5irgcco4umnvvcmqq +barracudaheaderfp20 +wwwblackrealitypublishingcom +jst +sansserif +earlene +contenttype +xmozillakeys +greenpeace +color3dcc3366 +may +surpassed +khuv534rmakks5izgjljbfdzfieciw3qep7coaumiwhfch6ism8towdwkplpgk04xdozi8nj +of +fontp +branding +received +artisan +without +potential +19216818120 +today +cosignersnbsp +bestimmte +understate +deliveryfontp +28 +234452 +table +earn +living +addresses +collapse +confidential +the +item diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_33.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_33.txt new file mode 100644 index 00000000..8259cea8 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_33.txt @@ -0,0 +1,349 @@ +sarah +xasgorigsubj +cellspacing1 +235802 +pnbspp +mailsmtprlyde026tl2g4w0emwilb +seeking +secret +number +xmailer +insurancenow +mexico +for +meaninstead +3dkl0kbcqqkftmmtlcfpdkqvc4gl4vqpsj5mf8wutjeq14jfw84lzqkni9u4fkm0nbhgfq15pe +smtp +xbarracudaspamstatus +question +involved +test +package +due +aligncenterfont +buddhist +xmlnshttpwwww3org1999xhtml +rebuilt +twenty +name +this +in +firewall +be9ne9ficiez +your +purses +returnpath +jul +imap +offered +openfacecollegeclubcom +mon +href3dhttpbc0032nojacipibcnimg +xasgdebugid +pays +binb +100 +fw +00000000 +cdrom +belgiumthat +abandoned +subject +depicts +xbarracudaspamflag +using +system +width3d540 +included +karitane +want +value3dfairlane +g6ony2j55006 +distinguishably +quotedprintable +sonatas +httpwwww3orgtrhtml4loosedtd +yes +state +agreement +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +would +212173515 +intent +yyyylocalhostnetnoteinccom +center +h2and +snowfall +browser +own +tag +up +statementabr +th3d400 +xbarracudarblip +so +rashidun +njur3d53130353067contact +revolution +size3d4strongclick20 +thu +do +all +pjxicj48ynipgjypjxicj48ynipgjypjxicj48ynipgjypjxicj48yni +book +align3dbaseline +u +rugaz +095 +peter +receivedspf +whitelisted +constantin +bottomfontp +edt +lobbied +us +ns3csminingorg +hidingplace +united +esmtp +value3dpensionsfsfginccom +95143136150 +ca +nisl +local +a +deliveredto +information +xmozillastatus2 +70px +hibodycsminingorg +congress +barracuda +size1sa11fontptd +localhost +tamika +as +rule +well +aligncenterjuno +flows +messageid +10 +trade +continued +dwindling +suppressant +emactionem +cnn +xbarracudaspamscore +alt3dstandard +or +color +total +color3dff0000115fontfontdivtd +srchttpwwwbeertripscommerchantimagesmain1jpg +iprofessional +taglevel10000 +problems +193120211219 +multipartalternative +informationbfonttd +mcmillan +bhghb +work +clientip189358130 +helvetica +aactive +nqqhjnaiyfarzg7cmmgmgklifpfbhnei4boaqzptroilhk19o4hwohw49q1lcyjcakdas3gj +quarantinelevel10000 +styleborderwidth +to +alternative +ffff00aspanfontfont +charsetiso88591 +least +typetextcss +undel +base +code +mailgimmixxnet +you +775px +finite +httpspamgwcsminingorg8000cgibinmarkcgi +32 +small +around +faxes +inroads +font +here +week +version +long +change +bytes +msgidfrommtaheader2 +referral +profitable +height447 +id +pts +over +also +antarctica +awards +on +22px +canadian +25 +blanc +htmlmessage +complete +notice +2119714711 +xmsmailpriority +party +zuckmail +date +institutionp +mail2csminingorg +optedin +xbarracudastarttime +31 +alignleft +jalapeno +cellpadding5 +marketing +xbarracudaspamreport +sales +rollingstone +gkuw8dkxhbbjkclxrlmxmlvxulaia7ibaxjgaptfag4vwfwhiuqaavbfsub4zjxsrjngsa6avaw +buyersspanfontbp +xuidl +923218192 +hrefhttpb8b81dfmozuwuscnyjeoqoba684pfv832l8p5d22m538761v8overviewah3 +them +house +commercial +by +282173509 +zzzzasonorg +pgh0bwwdqoncjxozwfkpg0kdqo8c3r5bgudqonclbszwfzzq0kdqpxdwvzdglvbnmncg0kugfu +link +xbeenthere +emerging +again +stylefontweight +trusted +with +type3dtext +from +0 +defence +anderson +mimeversion +scores +size1 +g3cs74982ibe +o3kbwyyb009424 +mj1963 +mailnetnoteinccom +yzalekcfw0m3wzhrrajvlaivcafvdacpiahebaglaacdkab4eaipxygxucg28eiokam5laneyao +brifont +region +aidbr +mail3csminingorg +cygxmwtjyiz2ys8czcjxthjwatk89lhq5t5mrrrhbjsjokpw4peixhf8imgfbghayuyjxl4vjkx +phenomena +gw1csminingorg +fetchmail590 +height3d15 +bet97 +besuchen +sender +naga +airis +message +backyards +19216818251 +style3dfontsize100ptfontfamilyarialcolor00125es +an +forged +aug +financial +c +large +beantwoorden +territorial +texthtml +ptsize3d10b +welcome +were +jmnetsetcom +eire +kind +special +more +viral +brl +w3cdtd +wondershare +mailing +html +lender +fakeout +0099ffprivacy +w +matches +stonein +activity +protection +29 +chap +fri +save +m0620212mailcsminingorg +promise +124840 +150 +7btextdecoration +no +thirty +very +sfnet +requests +nzf89gompeohy56iiivasa3ctu2xdo6d2pakeaxk3bqbtkiudtkf8ajxrswsep0yjmzoydaz +height341 +businessowner +httpwwwbarracudacomreputationip83222175196 +xacceptablelanguages +hibody +a5jcktsgri4f0fqkakcahcfkel1zkurblfktlnrgswf2kzfeo08nubzvualf7zrxlatypxqhwco +stylefontsize11pxcolora6a6a6textdecorationnone +mx2csminingorg +1000 +contenttype +stale +2002082504581720706qmailtempusgetresponsecom +america +volume +may +migration +like +l111wer +other +of +havent +330l +and +received +wed +list +relating +table +collapse +the +xpriority +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_34.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_34.txt new file mode 100644 index 00000000..e0ebb296 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_34.txt @@ -0,0 +1,363 @@ +serious +color3dff0000fontfont +used +tbodytable +execution +e21036d +cia +please +constant +qmail +httpserviceivillagecomappsdcsmcpr7003b9351hujs01200027c03b930mphywphdh +textdecoration +for +policybr +nameaddr4option +business +smtp +conditions +ipod +membership +question +kreisler +author +x +aligncenterfont +xmlnshttpwwww3org1999xhtml +a3b3bb2za1cfontp +body +taciti +18th +this +lives +cdt +off +returnpath +trans +nov +012 +100 +xucaxkcida8orcmkeyyybkmiz9affsabiwbmagzjmaebamabhvrtpxudpoiawiakmcaaagiqadiq +070 +reputation +54hktjcmamiivjm8jjiermom8rrbmdbvaad2hgibqr8qdah2ehgdaegbdqabddacekabicivxmh2 +subject +121 +xbarracudaspamflag +40 +pavilionfontfontdiv +size +real +than +included +want +htmlonly +fontsize +email +120x240sz +emi115115i111n +regimes +combatants +border +centerbrbra +below +2725 +101731 +seldomstrong +desert +text3d000000 +value7777option +designate +gid +212173515 +ranganatha +si +rotor +line +profit +tax +center +tag +delivered +bsfsc0sa424 +population +so +our +xbarracudaconnect +dinqual +urncontentclassesmessage +do +tue +atfontfont +anavlinkhover +domain +naples +hague +agency +features +predicted +img +nor +page +ul +0400 +2002082208283832185qmailmailfree4pornloverscom +us +it +persons +islamic +divjak +esmtp +whitneysdekalbnet +a +buried +otssjbvcc8gmmbbxk8xdqn0jzggmr1rkxt5xrckcx505dluqnohxgrjch8dtqmdwcguecg9xrj +deliveredto +featured +alt3dpmg +width3d242 +contentfor +receive +barracuda +localhost +within +longterm +8s +as +ssdaazz9azcga3vwaymwxkilbslytjdq0icqaewq1z792acdotmm1r3qar5gdp5qcn5aaq7gaxzg +div +1body +strongthestrong20 +softfail +30px +fertile +pdt +currently +messageid +year +zzzzlocalhostspamassassintaintorg +wwwtjcomnetcn +pay +specialwebofferscdsupportnet +10 +cellpadding0 +t0lxm0qye1vwqhjz3p6n81ekmtq9ol4frd7uadzwyehlx0yd2xumjvjipo4uaa5wcz5p0mix5lk +hetman +removed +125628 +herespanbatdtr +xbarracudaspamscore +compostela +public +avez +cpunkshqpronsnet +ombudsman +aldous +010 +xbarracudaurl +envied +contenttransferencoding +multipartalternative +reply +johns +service +importance +textplain +i +127001 +7c4692c43a +programs +that +aactive +possibility +to +v1 +65 +legislature +alternative +getting +charsetiso88591 +now +you +hicks +httpspamgwcsminingorg8000cgibinmarkcgi +know +jun +watches +company +n8n8wcnq008521 +tonic +stylequotspan +was +justify +change +bulldogs +countries +legal +los +experience +discovered +replyto +htmlimageratio02 +300 +size3d1copy +oakland +id +meta +transfered +hrefhttp83ca35tyirupehcnusdbfb498e90e66c9f9d91czqs879735596094476958517click +18pxcolor +fontsize12px +also +q +romans +began +06 +tr +government +notice +b +date +voire +041227 +unparseablerelay +kuchma +awm81xbqckwyfo8fvqznejbuvfeg2om1lgqq3uddvtbssap2hvasdsanwid3jyp7ni259cwd6 +2010 +xbarracudaspamreport +erie +bugs +size3d2nbspfontspanh1 +col +me +by +appearing +even +love +149nbsphospitalized +8mpnw96yzah9tvqmxkc9apec56vndg45okwotyeocrkvuiv2bpiq5kozu9kysiqcaotilelsfen +new +emhead +nameinterventiona +0100 +arches +ilialamonvnco +12557238255d5f00290000w4twrl +with +from +consistently +excavations +buth2 +999999 +money +mimeversion +0500 +scores +strongemail +calabria +lechler +gw2csminingorg +70 +210 +2 +exactes +nbc +region +aligncenter +width3d40bfont +pinesh5 +estimates21 +occurred +062020 +xmozillastatus +2px +contentenus +sent +words +fetchmail590 +2822 +deadly +which +hrefhttpaacfnzucocadcnqapodafoxo7fesb02d5b13f534aq45z5begajigypya997237813978434735825 +an +zimbabwe +attained +era +air +7bit +shostak +kgroundcolor +cc +box +branches +20020904145120253131bed4oakeincz +score150 +texthtml +rights +riverp +special +more +microsoft +19421724289 +30 +comstorecheckoutaddtocartaspxitem3d49115ampqty3d1ampsid3d16amp +freeholders +introduces +mailing +paddingtop +href3dhttpwwwinfobytelcomfont +html +size3d1 +between +tomts8srvbellnexxianet +81168116 +matches +none +151811 +zones +includes +lineheight12px +about +example +0900 +cpe +mirr111r115 +probe +see +class3dc5s +heritage +not20 +no +important +211162251245 +major +normalrecipe +vizepekey +group +including +contenttype +xmozillakeys +deal +of +dynamiclooking +and +fff +received +emulsifier +109106234142 +volleyball +plan +nonsense +color000000click +115436 +index +60081 +living +addresses +practiced +054058 +borderwidth +the +xpriority +parts +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_35.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_35.txt new file mode 100644 index 00000000..6fb4f88a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_35.txt @@ -0,0 +1,361 @@ +conversion +0300 +please +enc111ntr111117 +leagues +ourselves +xmailer +xentcom +format +chief +for +regardsbr +moonyvw23864cf363e3abtsnorthdynamic043133173122airtelbroadbandin +services +school +smtp +invest +12 +reserves +color3dcc3333st +years +name +ip +alignrighthim +operating +in +htmltable +hr +provided +firewall +your +returnpath +valuegagabonoption +principalities +point +breakdown +is +mon +caused +his +xasgdebugid +tracy +value100000000010000000option +rightmargin0 +60029002180 +xbarracudabwlip +subject +121 +xbarracudaspamflag +using +scategorytype +consider +alignleftthe +routes +system +nowa +initial +filetime73a18fc001c23222 +fon +15 +vp8aevs9sujoopwqbwajfr3adfqujpncwa3zwdkjp8aciiiycwvcwa3zwdk +ght +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +sure +0000 +mortcorp +below +backgroundcolore3c332 +height3d138 +colonization +what +although +212173515 +t97ke +rvuujv2xrhlk6s1jygtyh53zvptouudxgaxgkgj6k7daojmuapt2w1ycbpqsljcpuffgsvr2vox +inkjet +650 +sa061a +y +boldyes +g8m9ecc29967 +19216831 +description +tolerance +trust +newsspantd +bsfsc0sa424 +our +german +dialects +xbarracudaconnect +companya +all +head +january +color404040privacyfontanbspnbspnbsp +ny +host +respal100111 +msgidfrommtaheader +uverljivi +cellspacing0 +us +it +noted +sun +esmtp +automated +a +deliveredto +msgidoutlookinvalid +minister +hibodycsminingorg +receive +xbitdefenderwksspamstamp +localhost +sellinternetaccesscom +decaying +textdecorationunderline +height3d83 +spfpass +credit +has +messageid +pfont +217146177235 +bgcolorfffffe +qmtjbscu78kwkoam1xdwaf9z11bzrtk4u8aqhcwdxvv5mkb1woea0srxizb2ba7d +10 +mime +changes +008bffupdate +verdana +federal +above +uwtrmmil3cklvhrqbjmjnr5ozqa8mkztc0maahk0nfnzrmgakg0hqglzrmgbnl9iuymmar3bp5n +yourself +public +color +onlineafonttd +toa +taglevel10000 +xfacebooknotify +fontfamilyarialspanspan +social +contenttransferencoding +abundant +ue3dinternet +periodically +answers +age +service +eventually +textplain +i +bsmtpd +out +quarantinelevel10000 +tutorial +to +xhtml +half +match +0pxatd +charsetiso88591 +wind +national +brandy +nextpartddso3ggxsd1layti8aa +now +code +m37mr5042482fgh721273302035105 +you +color3dffffffplease +td +utc +uris +beauty +water +size3d3 +here +abelard +editor +six +educational +hours +personal +legal +094f89 +alignleftpart +at +ml +been +low +id +meta +over +on +boost +06 +tr +click +how +b +apply +width770 +after +opted +willow +655mm350l119l896965e8d +date +spamgwcsminingorg +bordercolor008000 +xbarracudastarttime +31 +whu1hprhrzkprmrkbgjk5giojzpdcfhci4uhc9ag4eve20npxikdig5kgffgy0ku00skhpwslyg9 +receipt +alt +f111rmatting +494 +2010 +xbarracudaspamreport +meaning +note +them +brsneeky +tahoma2c +selection +by +either +32228486 +oursgood +technorati +link +050 +much +d860c16831 +1283488129 +university +their +p +idmain2 +n9taitbd004141 +trusted +0099ffclick +empire +not +bsfsc0sa148a +with +from +ide4ihllyxjzichvciaymsb5zwfycybkzxblbmrpbmcgb24gbg9jywwncmdv +0 +bcc +000d01ca49a304a8d1f06400a8c0rugsbhl7 +mimeversion +sara +meath +border0 +luciano +nevertheless +obverse +70 +simplyfontfont +fontsize11px +hyperlink +18px +currency +xmozillastatus +href3dhttp478398547b78dyestastyru +address +charsetutf8 +colspan6 +xaccountkey +telephone +absolutely +message +eyeopening +f2e9eee4b8eb4f5 +which +011317 +diemthotmailcom +ext +7bit +targetblank +faxing +be +defend +get +church +rdnsdynamic +texthtml +125381897610fd03d70000w4twrl +changing +time +million +were +existsxmailer +mortg387drunboxcom +xbarracudavirusscanned +w3cdtd +preeminent +form +yale +81168116 +patch +basic +shopfontabp +padding5px +benin +virgins +xoriginalto +240 +m0620212mailcsminingorg +example +sto +many +4jczr5cvogan7awndx73qr60wnda4np8k50kh4kzz5ov8dr4mdp8lda36ceogt4kiqi +see +0px +no +sat +smtpmaildedramarkuse5khotmailcom +60zd6b5 +lenders +gh45yghe5jytrwkiyeyac6d4c3009com +ws +performance +encounter +divthe +great +jst +contenttype +xmozillakeys +styletextalignright +input +excluded +of +csminingorg +bearing +and +avisited +fontp +received +youll +host11websitesourcecom +19216818120 +celinecatcsminingorg +nonsmokesort +sa392hl +wed +free +euoko +wish +table +sa392f2 +5022441373h3 +usually +authority +sell +active +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_36.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_36.txt new file mode 100644 index 00000000..2a062a41 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_36.txt @@ -0,0 +1,376 @@ +static +e +launch +xasgorigsubj +astonishment +bgcolorwhitetrtd +valigntop +copyright +enc111ntr111117 +mergers +for +postfix +spfneutral +toro +program +network +find +support +smtp +standing +june +marketer +helodynamicipaddr +mailwebnotenet +body +bmhr00l4bviddbanod8zedtq5mfvtgtv0uri03k5zl2hiix9kyapxy77gewrhp5q57uohzilndij2kyyr89q +type3dtextcss +name +this +clickable +in +margin +height3d20 +paulthrivementoringcom +strongfontdiv +have +returnpath +n5ngflqa001752 +suspicious +0001 +breakdown +are +is +cheat +gears +gasoline +why +companies +among +course +100 +00000000 +25px +machern +webmd +subject +charset3diso8 +custom +brazil +charlton +using +6659172106 +system +than +b250000bfontfont +bfont +us2e +included +2002 +history +iint +09 +msgidfrommtaid +solid +biographies +lake +15 +ns1csminingorg +richmond +jmlocalhost +cement +would +giving +easilybrnbsp +enterprise +hundreds +line +srchttpimg8myimgdeubisesiyze6428jpg +19216818250 +19216831 +9pt +big +own +haven +xbarracudaconnect +xacceptlanguage +tue +head +1263137345 +26 +50 +had +deliverydate +hit +host +traveling +img +effect +008b57b16dbe7875a2b12ad00ca6yibikl +outside +uri +esmtp +faceoffs +znjvbsb0aglzigj1c2luzxnzicehiq0kia0kpt09pt09pt09pt09pt09pt09 +clients20 +antecipar +a +expansionist +deliveredto +mean +xmozillastatus2 +bdiv +developer +valuefofaroe +railway +barracuda +height +localhost +as +dogmaslashnullorg +talk +build +because +div +001101ca0afcb0b504907f000001microatx +5 +messageid +contentclass +350 +newslettertitle +republicdiv +10 +mime +dateinfuture96xx +enzymatic +apologize +tda +nonceremonial +112948 +looking +doctype +2001 +absorption +48 +such +secondary +english +amp +contenttransferencoding +way +work +refugees +ment +morebfonttd +jmneorrcom +border3d0 +ns4csminingorg +hildegard +httpqfakuobyouwitaxocomcc4f5d122f0ab04408a5e43a060342175 +ep8ayenfhcbigjaoqkwakkklia4tfxhanbhwahjhrwajx4jaogjwcqkwooosg +to +v1 +completely +041433 +ig +charsetiso88591 +typetextcss +speaking +value3dsubmit +now +code +httpspamgwcsminingorg8000cgibinmarkcgi +32 +bank +color404040contactfontanbspnbspnbsp +compatible +water +bgcolor3dffffff +grenada +hrefhttp5e747461wekuvkdcn +viewap +effectively +planera +width540 +webnotenet +mortars +these +alignrightloss +magna +poland +was +0000414141bc0000145a00000b41arklatexnet +believe +ionnone +change +color0066cc +202741 +alignrightengineeringinp +3056 +response +thomas +at +been +162614 +added +go +n38mawp2006301 +condemned +low +id +pts +rubisco +g8cnidc19397 +on +interest +xmsmailpriority +width770 +127262216578f51b580001w4twrl +date +activex +spamgwcsminingorg +31 +narbonne +8516012719 +marketing +tuwgni4wmc42mdawlje2oduwiibuyw1lpudftkvsqvrpuj48l0hfquqdqo8qk9ewt48rk9ovcbm +hrefhttp806785c151emuchmeekru +xbarracudaspamreport +srchttpdrbluehornetcomskinsdrpoweredbyjpg +newsle +spanbspan +xuidl +things +video +by +tempieclingingsmith3563swirvecom +urinovowel +score3237 +88646 +zzzzlocalhostexamplecom +baazi +grown +actually +115 +8px +p +trusted +georgia +forced +xbarracudabblip +times +bsfsc0sa148a +with +from +bcc +width1 +violation +1lknunjx8s48yqmd2ov2wuvevdxliptbi7ldwbvm2qi4wjfvls59qogagaaaaz9bfkipsgk23ji +brgeneration +mimeversion +orbt +vdjb47x6cpyp8ky3hiwd6jevwfcshsmbwamx8auyijn82p8ao8ir1eq6k3acbj +mj1963 +alliance +killlevel10000 +aligncenter +flag +mail3csminingorg +81 +details +alimony +use +httpviviacomuam +gw1csminingorg +make +charge +charsetusascii +fetchmail590 +sender +influence +height54 +1970s +9029100 +almost +cell +webmasterefiie +herea +teu7igp1c3qgyxmgew91ignhbiblegvyy2lzzsbhbg1vc3qgyw55ihbhcnqg +be +configurations +stylemargin +gw2csminingorg19216818251 +champion +summers +honeycutt +httpequivcontenttype +colspan2 +texthtml +18 +always +nominatedtitle +part +title +special +qemje6pammy3fhxwqqfrgp2zjf8wfe2jsjl5j5ltxppj5bjqhpadquywenxcavdxe3ttlb +html +name3dhdnsubjecttxt +district +matches +terms +alignrightwith +claimed +m0620212mailcsminingorg +practise +many +according +identic97l +size3d43enbsp3b +httpthinkgeekcomsf +hibodycsminingorgbr +limit +made +no +thirty +very +cartier +titlestyleyes +366 +disambiguation +textstatic +known +policy +httpwwwbarracudacomreputationip12423624186 +theres +signing +1000 +xmozillakeys +owing +may +like +g6q1hai4026704 +elegant +of +csminingorg +401 +received +one +c4129x +19216818120 +herwise +assimilation +ever +egtanyahoocom +list +pfethgygxpxyjjamhacjkmgq16d3i2yevayqosf8a1wopfvo1df6zy0rivh9agfc1d +complex +relief +leave +addresses +xxsmallbr +the +border0brimg +send +cities +parts +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_37.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_37.txt new file mode 100644 index 00000000..92289440 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_37.txt @@ -0,0 +1,366 @@ +111bject39115 +biography +e +orh5 +xasgorigsubj +del +health +rolling +brto +for +titlemeta +g3cs23715ibe +business +bsfsc0satofromaddrmatchhl +art +span +shop +makeup +valuela +corsairs +body +leftmargin3d1 +8 +tells +this +03a +lives +in +launching +203736 +ahjwjkao +namegeneratorhead +worship +112605 +off +hx4lslrqhppxrlykrbwcestrei3hz8ten7jtioo9vj0q7opsiujcrajuqpwh6mpo8mtkqzy +returnpath +imap +0001 +breakdown +are +hrefhttptrd2dropstonescominrpneoqzrewezowaioorqkoznkxiiibefxewawfrtqrflhfiofqehfiteocauitoiuwnwzewiimg +is +mon +bordercollapse +bb4nvvjmig92zxjyawrlcwaaaaqaaaaaoejjtqqablnsawnlcwaaaabpaaaabgaaaaaaaaaaaaaa +xasgdebugid +19 +navigators +00000000 +11px +213105180140 +server +oklahoma +subject +scategorytype +produced +system +march +href3dhttpnettracipipelinecomquotesrequestannqtaspgaid3d016 +qcr5sf8wpwvkjpxfs0dnl0oqbyncx9cvjf0trcmfk1aeb0c9in1b7feww7bxnjc8h9i3nql0ueht +included +lowest +incbrall +email +percentage20 +border +but +o2bf3qxr015120 +below +unwanted +convenience +102313469 +162642 +best +reader +contenttexthtml +comprehensive +online +ns2csminingorg +650 +if +19216818250 +description +steels +brbra +neither +tag +delivered +accgollyountspanfontp +so +revolution +wdeimliugujtfgfw4yks0imv8wjyq7kiwuizjbf2gamaweaahedeqaallwaqaqaeyez0eai7mt +thu +led +56000 +bgcolor3df6f6f6 +color3d000000input +domain +aiiscom +lhbgtd +rbl +ble +size3d2fontnbspdiv +had +titleuntitled +host +img +f +receivedspf +held +dead +0400 +us +it +stylebackgroundcolor +united +esmtp +0312 +uccxxxucbpcfimrmhaioabebug4aaqaiqcxcguxaefdzssdgqqqstrcarah4eaddlnyjimaweca +pd958d0afdiptdialinnet +a +re102lect97nce +applied +fisheries +deliveredto +jmnetnoteinccom +hibodycsminingorg +emnobelem +la +century +as +textdecorationunderline +constitute +training +has +messageid +concealed +10 +john +cpunksmindernet +usatd +xbarracudaspamscore +emztd2201com +allanktocsminingorg +gallop +leader +color +150243 +strapped +010 +taglevel10000 +210322 +nameireland +contenttransferencoding +rrcom +facials +xbitdefenderwksspam +zzzzlocalhost +importance +127001 +m0620212mail1csminingorg +work +styletextalign +border3d0 +color3dcc3366b144bfontfontdiv +h2small +quarantinelevel10000 +uid138371201487268 +tvbpulrbtlqglsbetybot1qgywx0zxigdghlig5hbwvzig9mihrozsbwzw9w +fff22 +sex +to +wayb +singledrop +base +19216832 +you +clientip122163247250 +112206 +iaa20958 +around +width3d535 +fontweight +these +v +avowedly +inreplyto +hours +networkbifontup +reduced +141322 +109ini +replyto +htmlimageratio02 +004548 +polygoni +id +offers +mandarklabsnetnoteinccom +pts +rdns +schoolabr +zmi1ikngh4dxnjr0ivaskcy6iabwbb6rylnfmiqzocczzktcxlln196pr0esoozymzarrtz2i +allowing +can +60 +tr +click +xmsmailpriority +cost +date +xbarracudastarttime +2010 +quickly +abr +valign3dtop +food +color3dcc3366b157bfontfontdiv +simple +appellierte +usap +thalwil +by +rates +171233 +split +worked +think +unsubscribe +facearial +mmddyystrongfonttd +pgh0bwwdqoncjxozwfkpg0kdqo8c3r5bgudqonclbszwfzzq0kdqpxdwvzdglvbnmncg0kugfu +isbn +v55045221200 +called +ec7901cae60101647f4b845828b48kijpn2 +customer +much +while +exists +university +loveah3 +association +not +with +type3dtext +from +wales +gave +proof +mimeversion +16px +1263220399 +toolsb +inc +sansserifcopy +000 +sa331 +2 +fontsize11px +turn +xmozillastatus +gw1csminingorg +done +6412025122 +end +sacrifices +whole +174304 +which +19216818251 +srchttpwwwsaveoninkscomimagesemailopensgif +an +place +h2a +targetblank +be +cb43d63 +respect +c +gw2csminingorg19216818251 +style3df +listsfreebsdquestions +branches +where +stylelineheight +church +color3dcc3366b115bfontfontdiv +h +texthtml +collect +rules +weather +srchttps005radikalrui2091002af1931be83aad5jpg +script +daemonwaste +nuevamente +repeat +score767 +annmn73b9303b9351tawr012000003b930mrvm4rvqf +elfangor +existsxmailer +form +airing +yasmin +html +size3d1 +12514780642e05002d0000w4twrl +selfimage +ferencingfontb +patteson +fontibto +2186150span20 +alignleftand +spam +xoriginalto +about +wta07gb0lp9nxqzukskwxzhhlst6yrj7qtvucrnezajhucvv3ytj895pn18wjtakndhtb8f +0900 +pwwwjuxjunancnuq3d817dd6c36e891ea755d6 +deposit +232 +150 +width100 +see +against +meddicky18aru +3 +no +approved +155mm +width +having +disambiguation +btry +orderbr +hibody +group +ignore +day +bsfsc0sa119b +contenttype +xmozillakeys +href3dhttpwwwpnobicatcn +like +listed +of +england +401 +and +colspan3d2bfont +received +high +19216818120 +327 +right +located +dances +sa392f2 +z +stub +printerink1cbphostnet +ec5262c1d6 +the +cash diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_38.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_38.txt new file mode 100644 index 00000000..29cfac0c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_38.txt @@ -0,0 +1,282 @@ +rootlughtuathaorg +0300 +4bdc68b44020601coxnet +please +format +for +postfix +dst +pointgt +6621867196 +drivers +feel +clientip8219575100 +this +tyrrell +emulator +in +certificate +cached +your +returnpath +corp +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +is +learn +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +gmailid127f7be463cb7656 +server +subject +hrefhttpclickthruonlinecomclickq26kqpviimholepwscbbbkqu8niq74r +less +system +2002 +score5107 +increasing +email +listunsubscribe +jmjmasonorg +but +d96 +0000 +debian +9 +leftmargin3d10table +httpusclickyahoocomj2snnayleaamvfiaa7gsolbtm +url +what +would +remember +proper +closed +ist +line +hrefhttpwwwcolorcombocomvisualize +fallback +required53 +up +going +etc +face3darial +so +do +gnus +48524852 +domain +pretty +198186202147 +youre +users +trial +exim +hrefhttpclickthruonlinecomclickq248ddmicz6cpsrrovgeuef92iqrwyr +articles +set +mailtoforkrequestxentcomsubjectunsubscribe +httplistsapplecommailmanlistinfoaugd +325 +it +xoriginaldate +esmtp +protocolapplicationpgpsignature +893893 +listsdebianuserlisztdebianorg +blackcombpanasascom +local +mailtovulnwatchsubscribevulnwatchorg +rcvdindnswlmed4 +httpwwwbiscuitindexcommithhtml +numbered +a +deliveredto +arsasha256 +131846 +information +mailto0xdeadbeefrequestpettingzoonetsubjectunsubscribe +localhost +useragent +dogmaslashnullorg +voice +razorcheck +spfpass +pdt +messageid +10 +photography +phoboslabsnetnoteinccom +112217 +seems +192168123179 +migrate +cfnumbercreate +nuclear +httplistsfreshrpmsnetpipermailrpmzzzlist +010 +xrcspam +0200 +2007091301 +contenttransferencoding +104304 +2008110401 +aimed +ma +rapr +textplain +i +127001 +mutt1399i +brtd +aioanei +servers +udev +nametextcolor +to +a18mr11130145fgk531270558098067 +ratwaregeckobuild +charsetiso88591 +modern +mimeversioncontenttypecontenttransferencoding +meant +bank +totally +did +options +compatible +amid +solution +most +mlsubscribertechcsminingorg +resentdate +found +bytes +joseph +second +at +20022 +cipher +202218 +debianuserlistsdebianorg +httpradioweblogscom0101069 +id +port +listhelp +need +can +on +012909 +06 +designates +xmsmailpriority +date +methodpostinput +razoruserslistssourceforgenet +tehran +authenticationresults +yytaidbkqvlac1hsrxlbliszt +forgot +googlecom +testsfourlaimprononcable2 +envelopefrom +by +unsubscribe +stream +listmasterlinuxie +201224410520090324 +ago +should +actually +via +2525 +xbeenthere +receiving +0100 +xamavisstatus +their +not +colorff0000noticefontb +with +from +dmzfirewall +cbf3913a6255 +mekons +suspect +mimeversion +0500 +a350829409a +src3dhttpsccommunitiesmsncomimgcgif +xcheckerversion +typing +apkdnqh1541fjk7wnz2dnuvztwhnz2dgiganewscom +use +listid +size3d2tony +make +irresponsible +sent +fetchmail590 +listsdebiankdelisztdebianorg +python +sender +precedence +yyyyspamassassintaintorg +stephen +ie +irish +an +score7 +original +readme +width344 +forged +cc +welldesigned +listpost +meet +into +202b8e90d52b7eb3ed7294211f5aadfe +bigger +edge +xvirusscanned +215 +more +href3dhttpclickthruonlinecomclickq3daagbteqxu +mysqlserver +shim +fi +cat +q19mr609979fab121273198146488 +fri +81348131 +xoriginalto +dotancohencsminingorg +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +http +spoilt +archive +width +treeseacmorg +enus +md +policy +version325 +theres +jmrpmjmasonorg +contenttype +xmozillakeys +172938 +of +and +xstatus +received +youll +pive +leones +list +days +created +display +went +leave +the +3w718vkeaajlxpbplwv6hxd4sthtyvsom2dta +those diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_39.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_39.txt new file mode 100644 index 00000000..bd2a5e35 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_39.txt @@ -0,0 +1,269 @@ +economically +qmail +for +postfix +101426910 +smtp +firewalls +works +feel +nonpresence +gnupg +admin +httplistsdebianorg20100404221533ga10154thinkhomelan +package +httppomtiedomcomlog +arsasha1 +140048 +8 +verify +norepeat +clientip8219575100 +endnote +in +have +returnpath +imap +16253 +libedataserverui128 +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +manage +said +subject +e243048upcechellonl +105911 +qmqp +samba310994127387878500012 +listarchive +2002 +economic +want +initial +increasing +email +listunsubscribe +virtualincludesubscribesidetxt +loot +jmjmasonorg +but +insert +0000 +104003 +debian +payload +13537 +becomes +what +yyyylocalhostnetnoteinccom +tnonsensefrom99100useragentxacceptlang +if +y8mr997376fac191273649756303 +5027 +big +fallback +going +maingmaneorg +quotes +so +1mailing +turtlegmxde +appear +lairxentcom +archivelatest574575 +had +024417 +receivedspf +exim +esmtps +13 +spamassassintalkadminlistssourceforgenet +185730 +edt +it +protocolapplicationpgpsignature +two +a +dont +movie +calls +0983607 +deliveredto +create +wrote +localhost +233424 +talk +archivelatest575036 +build +bgcolor3d000000img +load +has +messageid +pay +10 +deck +car +dkimneutral +0700 +mailwy0f175google +yet +ieyearecaayfakvfieiacgkqlttvzdk47d5fngcfcngptgfzetrqwxhlg8spcnie +or +httpslistmanredhatcommailmanlistinfoexmhworkers +public +9505427409a5a +121172 +contact +they +xrcspam +dpco7k2yunts +0200 +2007091301 +m1f4usc1qaic +2008110401 +im +ghnrg +age +textplain +i +127001 +testing +changed +archivelatest575627 +classsidebargnomespecialsptdtrtablebr +222130 +174906 +spotty +to +v1 +least +singledrop +wsletter48imagesnowplayingbuttontrailerogif1img +utc +probably +121450 +185119 +most +inreplyto +19d072940d3 +mxgooglecom +hours +nothing +at +through +httpdebianorgnewsproject201003 +id +checksum +20021008132740gg23820jinnyie +srchttpwwwzdnetcomincludeadsifcrgroup2560 +port +listhelp +need +on +same +5670013a55cf +unitedmediacom80clickq3da1afwtqzzmebmmuegklhlcposs9rr +yyyylocalhostexamplecom +happy +stan +how +debiankdelistsdebianorg +seasonal +date +usual +shares +jalapeno +gap +l +101610 +xuidl +solitaire +by +unsubscribe +lisztdebianorg +race +picnic +xmailinglist +ones +2525 +xbeenthere +0100 +xamavisstatus +anyway +unsubscribemoviesemaillifetimetvcom +not +delete +times +with +from +20 +betrayers +mimeversion +nfs +joebarreraorg +rpmzzzlistfreshrpmsnet +spamassassin +xcheckerversion +listid +put +charsetutf8 +fetchmail590 +xaccountkey +rather +sender +precedence +message +which +an +sorry +be +listpost +xauthenticationwarning +bhw9q1vzvgyjjgrlz1vxxiksmzd72gixi4pbzfwtt1bu +geek +latest +more +hello +ica +170142 +binnmus +arrived +paddingleft +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +fonttdtrtable +w2laee79c571004070335q80635354uf21347f6776b2a38mailcsminingorg +example +many +8219575100 +heritage +quicktime +no +we +add +archive +valley +major +some +intmx1corpredhatcom +enus +showandtell +policy +contenttype +footer +like +might +of +packages +and +record +received +one +without +list +couldnt +xfelkmailscannerwatermark +phone +mailtoforkrequestxentcomsubjectsubscribe +month +directory +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_4.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_4.txt new file mode 100644 index 00000000..0276c38c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_4.txt @@ -0,0 +1,292 @@ +1210 +826930110 +pm +qqqqqqqqqqzdnetexamplecom +screaming +xmailer +for +postfix +smtp +any +years +033147 +resentmessageid +clientip8219575100 +account5 +disguise +ldosubscriberldowhitelistratwaregeckobuild +73 +autolearnham +in +32227031 +returnpath +once +rssfeedsexamplecom +mailtodebianuserlistsdebianorg +122814 +07102002 +is +mon +why +height18 +loop +00000000 +bgcolor3dffffffimg +subject +4 +pgp +iso88591qptbs5oo9toumhbdke23otci0a093cbil8dtogzk0bf2yq35wsewvyet2e +qmqp +using +less +40 +private +displ +iso88591qxd86p3bwyaj25f726z5fbw7ddkm2el2423h25dudk7thshc +system +than +2002 +late +smtpmailquicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +solid +buxepcarkg7xrxcc3z1rqvxcma7djpwk7hkyslklkakqqtylhovp62zknp2tx8ol +yes +jmjmasonorg +but +jmlocalhost +jump +unable +rowspan2 +what +xucsccatsmailscanner +ist +project +936 +apr +own +tag +required53 +maingmaneorg +so +coast +do +i686 +theft +httpblogsanityofsamcom +re +campaign +present +img +receivedspf +before +size0a +just +325 +sun +esmtp +fix +trip +subscription +a +situation +deliveredto +job +khare +xmozillastatus2 +resentsender +localhost +earlier +chirac +2011 +dogmaslashnullorg +bogoqk3aucnsrqzopanpmsuijl8rwheppzpmtew4pbtt9dri70mgsaiymjv0pv93qpf +mutt1520 +thing +pdt +has +messageid +goal +root2 +922492398 +href3dmailtojhstuckeycsminingorgjhstuckeycsminingorgagt +judge +m +chime +or +resentfrom +char +come +they +xrcspam +problems +each +exactly +contenttransferencoding +al +110223 +193120211219 +gash +chemistry +href3dhttpclickthruonlinecomclickq3d6csul3iup8871rm5kqcl +textplain +i +127001 +164519 +8bit +5a20927093616 +that +osaisitko +classsidebargnomespecialsptdtrtablebr +19216810053 +upon +listsdebianorg +ote +to +lumixabr +singledrop +metertd +ah +kgattinetechtargetcom +did +home +single +205651 +lnded3jphqqlitpqwn2yjn8hwgee8ukx8ivie +references +at +1 +been +debianuserlistsdebianorg +go +id +209sfnet +mailinglist +listhelp +144615 +forkexamplecom +can +on +ibsf69chzcecgfysnp8dr +designates +my +b +after +date +onda +who +2010 +everything +opinion +xuidl +by +he +unsubscribe +lisztdebianorg +antispam +listmasterlistsdebianorg +thats +should +actually +will +2525 +0100 +again +autolearnfailed +not +listsubscribe +spamc +with +from +fixed +20 +63cc +ability +mimeversion +content +province +cls +000 +104554 +scree +pass +xmozillastatus +spot +details +tczupm6ufbwt8kpogwzo0wjcn0yatirma3cskcemcwrkddjmkkdlulj0hhhlle1hmu0ry8qk +listid +credulity +charsetutf8 +081claws +muddled +friday +sender +precedence +concept +didnt +errorsto +clientip172541336 +an +place +taggedabove10000 +118 +cc +paperweightdarwinnasagov +be +listpost +mailtodebiankderequestlistsdebianorgsubjectsubscribe +release +102234222 +fglrxdriver +xvirusscanned +134558 +ask +more +30 +nt2000a +httpwwwteledyncom +mailing +core +113940 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +w +really +manual +fri +xoriginalto +game +src3dhttpwwwcomicscomcomicsdilbertdailydilbertimagesbu +localedir3decho +see +call +no +chain +085947 +archive +some +gmailid128472fa0bf3716d +enus +contenttype +xmozillakeys +user +volume +xrcvirus +may +utf8qivborw0kggoaaaansuheugaaadaaaaawbamaaacllos0aaaamfbmveuskvkskvksk +of +states +and +xstatus +record +received +referencing +version241cvs +20100514131854doqo27114eastrmmtao103coxneteastrmimpo03coxnet +list +days +piece +editable +207615143 +the +controls +172164831 +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_40.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_40.txt new file mode 100644 index 00000000..2b4e9c8c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_40.txt @@ -0,0 +1,356 @@ +target3dblank +122003 +emon +recipient +down +colorffffffequal +economically +please +xts00ijiymf7tj9jkmjzbufbudxtoojb8reheosu4elrjhcyvy6y5i7t2qpu1mf0klw74nxo8 +tillst +134085 +behind +for +postfix +spfneutral +s +smtp +xbarracudaspamstatus +bsfsc0satofromdomainmatch +june +194610 +method +premier +working +ownernolistsgodailyjmnetnoteinccomsmtp1admanmailcom +body +policya +parties +name +this +in +provided +sunni +removal +off +returnpath +87df44415b +0001 +breakdown +is +romanstrongl +his +xasgdebugid +feature +100 +11px +reputation +subject +faceverdanaarialhelveticapinternet +already +60000 +size3d2namefontbtd +using +less +produced +titleafrican +9900 +included +ideal +research +solid +boundarynextpart00000239292f0948931716d30 +quotedprintable +idtable5 +bsfsc5mj1963 +jmjmasonorg +released +physics +substantial +best +contenttexthtml +806951182 +statistics +080 +democratic +lexmark +systems +if +19216818250 +091619txtdogmaslashnullorg +20082010 +size3ver +lspacing3d0 +216136171252 +186 +xbarracudaconnect +t +urncontentclassesmessage +play +tue +11pt +institute +florida +supporters +h2their +deliverydate +amavisdmilter +cut +users +0400 +temporary +reproduced +left +yxnlihnvbwv0agluzyboyxbwzw5zihrvihlvdxigy29tchv0zxiudqogdqpj +efefef +adapted +esmtp +200208281704g7sh48z01928dogmaslashnullorg +egypt +893893 +width3d466 +a +dont +xantivirusstatus +deliveredto +office +hibodycsminingorg +hrefhttpc2fbpoxokuwcnjsegqkyy2743e57decfd4aef63302dudyalnhibodycsminingorgovqkuxewyd1422834888982689422018 +receive +hormone27103 +localhost +as +div +rule +prompt +regular +messageid +carbon +displayed +populated +institutions +recieve +10 +xasgtag +finally +coming +changes +unknown +benner +pr111ximit +0700 +old +ce0718c5ee7 +toi4p0oar2lrdz2kvtbqywxkfrwysqb0gssampqob9kwgbtfopkaglqw5frvg1nijfaemseu8ri +public +doctype +interested +main +color +010 +o11cjrgx004339 +style3dfontsize24px +contenttransferencoding +thi115 +color3d3da4e3 +section +2l1yltrn0sizqiqpggvo15djkd6ishundu9oy7jvhhlrbkd6jtuftftw1tcwybflunok4ygh +zzzzlocalhost +textplain +msp +aeds1dl6scyve1kfy4g48ueh5od97zonux9uel2xapqxvqhriuxn35jahrq9lc9r3t +8bit +work +aligncenterto +that +venture +division +border3d0 +census +122913 +gacozwe9cxrnybop06dqo0qdsrwq1atys2rdyrurzwnew4ods7as2bno06qnguiawaaecoimc9ep +tg070 +wasdhzeyaxpuc3uoiopmodg5x3cj8dwgw0nepn3hxl8yzzwy3o7uvg9rzjti0csxuftuprc +num +to +purpose +mailpas2nejp +charsetiso88591 +washington +singledrop +srchttpmail4mortgages101netlogophpid90id2193953p +19216832 +you +httpspamgwcsminingorg8000cgibinmarkcgi +small +aside +cit +woods +height77 +hoops +these +recover +was +informed +version +found +yo2mgqxpudtpuj4pzaenj8p51diu0h03ia2asvcrmlbycmjpfwtksry03sekzhci5i5ugz +2052104230 +at +replyto +low +id +meta +diagnosed +offers +89b8d1750 +pts +rdns +color999999 +can +on +general +exclusively +same +tr +government +b +forst +party +date +ship +alignleft +69 +easy +marketing +font20 +xbarracudaspamreport +authenticationresults +replytypeoriginal +xuidl +emailsbr +by +be9yq613qys7nsllmnmjvswfu3e8mz2bzw7f8apwnmdgxaar3eny4iz02ehckx7dt1u1vu +size1sfont +gw1csminingorg19216818250 +antispam +1580 +link +forefront +missing +0100 +while +cause +burundi +trusted +not +xbarracudabblip +with +from +early +jackson +team +expose +rohit +iydgddgaasomlbdcpgqsoedukyczhw8uzhawitidhsd5assbpabkchsuaai4cm1gmadaladhmcy +confirm +pr111tecti111n +mimeversion +qagv1aadkxmzxvaampiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabx +next +gw2csminingorg +1915 +alignrightvary +orders +srchttpwwwfromyou2comnastymilfamber093jpg +026 +aligncenter +bsfsc0sa218a +xmozillastatus +address +bring +uid +traded +02 +faceverdana +words +news +hereafonttd +size3d2font +xaccountkey +n7ndihhi004187 +extra +sender +grove +msostyleparent +esc110329738836411025989487446468inrovingcom +file +cellspacing0tr +webmasterefiie +john09222002htmlfrugaljoe330wcom +7bit +hrefhttp47728jtenajixcnaxalyycop2l8n372o1o5d07o13954c983change +qu +5pxlineheight +landmarks +into +get +kampan106 +texthtml +18 +rules +loss +0067b1 +minare +adviser +special +existsxmailer +denied +w3cdtd +type +3px +w +81168116 +november +vmkd +xoriginalto +king +sansserif5250fontdivtdtd +width100 +see +3 +made +mazglaurigso8pnklliyucqb9pazluq9as7rqoq2driap0qi3shdelakgcduaoz8nksljscycc8 +we +subcarriers +hrefhttpdwlj44fanpezijcnaxqupjqleqradxjsa1c73ec1d7575beguapjraxfeedbacka +barracudaheaderfp20 +known +sansserif +contenttype +xmozillakeys +deal +won +other +of +dynamiclooking +cents +and +20015322298 +hrefhttp8d5medfulton85arupeokef958b7d1a44e +received +one +19216818120 +right +wed +santander +nt +table +helod +follow +everythi +york +the +qvp0086 +parts +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_41.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_41.txt new file mode 100644 index 00000000..377bb754 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_41.txt @@ -0,0 +1,357 @@ +000000font +michael +srchttpdhue39parkchanceinfoimages1062jpgabr +pharma +providing +11km +agvpz2h0psiznjuipg0kicagicagphagywxpz249imnlbnrlciioua8l3adqogicagica8ccbh +senator +120923 +please +valigntop +percentage +able +des +for +askedbbr +xbarracudaspamstatus +iluhailuhatiacnet +relative +defense +probabilistic +due +any +body +this +till +in +content3dtexthtml +firewall +your +auto +o4cgnvyf010877 +returnpath +1954 +server2 +valuepmst +httpwwwfreecelebhardcorecom10391 +0001 +are +strongshe +xasgdebugid +bgcolorffffff +ikadgqgmx3c5toqmlwmowoukjklk5pjyuklmqdsh2uxjhloyqtakjgaaq4ciprhrsz5s2loc7j3 +account2 +00000000 +f095c16979 +213105180140 +070 +subject +eu +border3d1 +xbarracudaspamflag +using +40 +teacher +medication +style3dtex +system +obfuscated +march +hrefhttpced0nzucocadcnozynebekyh9i1kl547w68880z11yb4yabuyhi7332677485317548039975752terms +cancellat111 +2002 +lowest +trueu +massacre +invitation +email +rw4zvve82spiecbl9e33mphrsupmxjnlbtv1fqfx02mg9bh9trqukvpsz9b63d6knre3sh +quotedprintable +border +282167767 +released +below +future +best +103821 +bills +luma56csminingorg +063 +aiginsiqus +srebrenica +smtpserver1cflrrcom +front +line +aligncenterin +br +description +equilibriumstill +arabia +palo +o3tbdixa003039 +delivered +process +our +introduce +per +thu +fnbyzzwcttzk5vx6tnlpj0kemz8cuukfsrpustillggvfmoo08d0mg1d4peqgopvqvwvdywmv3o +head +collateral +domain +served +rbl +155414 +campaign +1736h5withh5 +img +nor +data +full +clean +bold +ade6d16250 +height3d258font +selectedmmoption +it +libccnueducn +43xevy1blft0gggmecxaiuhslshy8oowdihly1jhi0l58japayezxdfip0k5qq7ymnezo1au0 +esmtp +altfacebook +effective +a +vengeance +earthhe +dont +congratul +applied +deliveredto +xmozillastatus2 +hibodycsminingorg +super +deliverymen +color000000 +barracuda +though +nextpart000428cd01c24de66dd2e5f0 +as +alot +rule +opposition +boundarynextpart0001cdc1901c253664b57f3a0 +has +messageid +procedure +industry +10 +cellpadding0 +governmentselected +commentarybfonttdtrtrtd +genesis +unknown +e569f16f03 +wesleyanb +removed +0700 +old +kezia +ichnetoutfitcom +xbarracudaspamscore +contain +or +stanek +public +london +rsvps +main +cdo +come +safe +3185729 +95nbspspanpowerpoint +peruser +they +amp +contenttransferencoding +httpwwwlifesettlementsonlinecom20 +namebfontfont +theh2 +127001 +ws2xo2iqmkxgr3oubdcjqcbetnkyboo16yoeh40jdcmennuds2imtwosameqlsw1rritkara +charset3diso8859 +amckhv93l7iz8sob7zp2i6xmejszdvp70x7ir2yghglnuhmu48vnlsdfih0jub2pbmr5oy2jwu +talentscout4mindspringcom +ng3d0 +to +007yahoocom +117tilizzare +zrszjcedaadfaix082s8f8sm65q5rh1npc3hzyz0oc8l2xm7tilkbhcwgdkwulr2s3hszb0acx +mathematicians +you +emjmnetnoteinccomem +reverse +did +infanticide +sign +align3dcenter3e3cfont +executive +webnotenet +forming +here +1942104916 +094622 +hold +michelangelo +g7n96hz17715 +at +replyto +th +thick +facetahomafont +retreat +id +rxxqseih3axxn788 +srchttpi42tinypiccom2094q3kjpg +diabetestitle +wave +charsetiso88599 +mudslide +jixedola +ldk5nzy1mdm0nywxmziznzi5otq4lc0ymda4mzuxnde2ldmxnzawmdczocw1 +xmsmailpriority +date +mail2csminingorg +fontbfonttd +021 +height30td +trading +jalapeno +pleasurelibrliincreased +omaha +12px +metrology +werebr +tgeyabgygavwybf8zr0qjakcdgcgw4yaugagep41e9vhem4grjahadgyg4aay8wdadg9gsiqua +pants +me +xuidl +scheme +mistakes +by +het +facearial +sculptorengraver +gw1csminingorg19216818250 +pillshop +will +size2easy +fbuffffahx2xhhbwdxjf5cp6gsfpc365livpunowoopkycg4pq1n7utib2c9kmrggzjjqnv +much +september +p +18822414039 +width296 +with +from +immediately +defined +person +mimeversion +size1 +border0 +flashtitle +zooycsfmulzhuhnjfgieq5qwawugsrljlcuitzleidrjpxvhkamyfalwt5cbwda9aqamwbg +140304 +gw2csminingorg +alaska +dm +theoretically +express +alone +south +pc9odg1spg +align3drightnbsptd +0000af +killlevel10000 +life +lot +address +gw1csminingorg +sent +styletextdecorationunderline +charsetusascii +bond +bet97 +rather +ripening +sender +ayo +width169 +album +which +ldacjeduro +ew91igzpbmfuy2lhbcbmcmvlzg9tigzvcib0agugcmvzdcbvzib5b3vyigxp +established +200171104254 +le3dcolord0ee6a +orographic +ase +communication20 +get +stenotype +rdnsdynamic +theyre +texthtml +collect +xvirusscanned +part +roc +combination +straight +microsoft +stalwarts +reception +g6lje4j77822 +sa392f +html +stylemarginbottom +inner +rgtjjsxncuevrvjq8mduzptzsflqaw6tlto3uw2x2zllcrcucq5quhhouvvhbbknsg880vvtv +m97y +merchant +purchase +100450 +start +29 +contains +about +m0620212mailcsminingorg +see +bonus10for +0px +no +very +we +tionfontp +qtimydot +jst +dallard +contenttype +east +titleadjunct +gener97ted +spire +may +of +csminingorg +and +liitle +received +youll +19216818120 +expenses +later +wish +the +maintain diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_42.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_42.txt new file mode 100644 index 00000000..f8652138 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_42.txt @@ -0,0 +1,273 @@ +launch +aanlktil7z7jgyva3eaz5wke1ytufs0ewhtfndwofllhmailcsminingorg +hrefhttpclickthruonlinecomclickq5djnbik0w8dxfoweobvzm5ygkiazr +messages +application +trouble +xmailer +for +postfix +2139621475 +secondquarter +smtp +integermapget +people +modes +package +clientip8219575100 +account5 +this +clearrc17597115123 +color3dcc0000somebodys +in +move +cached +have +returnpath +6487055 +imap +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +too +testsbayes002 +tipsabr +examples +subject +given +using +hrefhttpwwwchapuracomhandango +xoperatingsystem +2002 +bulk +usrsharedochplipgui +email +234039 +graphic +listunsubscribe +quotedprintable +odd +lake +11 +0000 +smokejohn +below +jmlocalhost +correctly +mine +ccdac294175 +would +rhetoric +opportunities +accucast +boot +if +apr +leftist +our +expected +plaque +1930s +do +tue +domain +metric1 +thanks +9bivcypmfv6pwqgrtfgqyuirrx6nlcbplpsffditp4rdndgrvghzqcthvpm3lcqfzzh +tnonsensefrom1020tnonsensefrom2030 +users +enter +exim +dysons +procmail +full +esmtps +edt +325 +it +esmtp +homerjinnyie +subscription +a +installer +dont +deliveredto +rv1919 +113306 +maintainer +provides +localhost +as +inline +barreraorg +pdt +messageid +yesbr +103 +10 +loads +z4x9z0avngim +phoboslabsnetnoteinccom +0700 +oats +specified +or +computer +fffplatformsa +they +xrcspam +flushed +2007091301 +anykey +contenttransferencoding +hereabr +brand +textplain +i +127001 +20985222188 +that +084705 +yep +assholes +listsdebianorg +arm +to +hrefhttpclickthruonlinecomclickq1bwanhi3n2i6po9pjq1u5mauo37xrr +getting +singledrop +attorneys +20080610 +you +web +democracy +department +these +making +vidmanca +29si12376561iwn6820100413122412 +pwc +grandi +inreplyto +links +there +mxgooglecom +hours +countries +at +th +fay +lughtuathaorg +waa29176 +id +052224 +eastrmimpo02coxnet +3d6cad7b6010800permafrostnet +nervous +wstrictprototypes +listhelp +17 +black +yyyylocalhostexamplecom +debiankdelistsdebianorg +n12mr206682fad351271790911462 +diff +date +jalapeno +m8cs5640wfo +me +235152 +them +22si415949fxm6220100408080149 +by +unsubscribe +lisztdebianorg +border0br +commandlinegui +124438 +listmasterlistsdebianorg +dsblorgerr0 +will +ldowhitelist5 +2525 +xbeenthere +back +trtd +produces +a6e290fab55543779308433e28fd24fcmaccom +while +their +autolearnfailed +not +listsubscribe +with +from +20 +jsp +enht +bcode +significant +xymailosg +brain +140105 +first +properly +105427 +aligncenter +session +xcheckerversion +pass +xmozillastatus +charsetusascii +fetchmail590 +xaccountkey +message +file +6217143253 +ea03a26cfc5f3 +an +122348 +nords +discussions +215807 +listpost +shows +mailtoforkrequestxentcomsubjecthelp +ronljohnsoncoxnet +were +anything +waiting +mailtobitbitchmagnesiumnet +7 +type +interdisciplinary +65xd86pwyajf7zbwdkmlhdudk7thshcajptq4rqwrvutj +81168116 +10871734 +patch +smtpmailbouncedebianannouncemlsubscribertechcsminingorglistsdebianorg +manual +confused +firm +10143348 +except +see +sat +mailtorazorusersrequestlistssourceforgenetsubjectunsubscribe +0600 +policy +archivelatest574071 +contenttype +xrcvirus +listed +elegant +of +packages +and +received +list +endless +eight +srchttpimageslockergnomecomimagesblankgif +m0i3csh +the +112113 +cds diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_43.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_43.txt new file mode 100644 index 00000000..b7b43cad --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_43.txt @@ -0,0 +1,359 @@ +california +essen +190906 +leadingviagradealer +ctl00ctl00ctl00ctl00rcrbsblnkcategoriesctl03linksctl03link +icagicagicagicagicagicagicagicagc2l6zt0ypjxcpjxcuj48l0epeeg +valigntop +copyright +width800 +xmailer +able +for +postfix +face3da +business +7797g +offe +smtp +mailwebnotenet +solutions +bsfsc0sa424sbl +ended +a06d04275 +court +name +this +transitionalen +163425 +your +off +pilgrims +returnpath +imap +breakdown +are +is +centerbr +learn +zzcg4z3qb1k9xyjxvnzyrzlkorafjw5yxkrpsw07jz0prlsvhvcu1s2meuicqb1ma3w794 +145309 +inscriptions +100 +account2 +00000000 +height3d9120 +subject +eu +navigator +acompanynamehover +custom +artic117l111s +that20 +independent +xbarracudaspamflag +40 +mt +noble +aligncentera +gmailid12840e453afd4123 +lineheight +solid +annuity +yes +styletextalignjustifyben +standard +below +text3d000000 +causes +intent +spoken +mississippi +pailnurc4tyruukxiatbppx3wcu324eosn0u9itexjiyuv6ntx1fe2xbd6hnncvfglzp48eim +defensive +if +hrefhttpjsu87zgelubefcngudegiyf5bceebb250abiosqz3679876130 +br +19216831 +running +sszcom +bottom +up +telnet +content3dmicrosoft +xbarracudaconnect +limited +normal +newsletter +22 +head +hand +ynnwozwvrelwpg0kperjvj48qsbocmvmpsjodhrwoi8vd3d3lm5vam9ybgiuy24vp3hleg9xdgft +titlefree +cruise +served +retrospect +g6mhmfhy030050 +232248 +alignrighttend +host +nor +cellspacing0 +0400 +tt3ddedeaab8a9a5baa4a1a3ecaaa8a1a1e5e7edb7edb0afa0 +shers +stylefontfamily +policyatd +sun +esmtp +egypt +advances +24 +two +webstore +utterly +bgcolor999999spacertdtr +sometimes +maintenance +deliveredto +brvnbfcm7ognkjnivhkuys2wbhnumklns5kg06w43rsqnjqskmrooofwgcvjawgkfmonglajips +jmnetnoteinccom +hibodycsminingorg +congress +16 +barracuda +within +as +rule +she +regular +messageid +size3d1hiv +10 +xasgtag +unknown +align3dcenter +href3dhttpbitlybvlx0rhttpbitlybvlx0rabr +told +xbarracudaspamscore +account4 +or +letters +edition +4ml +textdecorationnone +questions +andp +doctype +ueummeqpsre7irqmggaekaammk0apflqathpagbu3pwkk0ancmmmcjjloljlibfjxsmvxuycyv +carbanions +affordh3 +webbased +wdhrifnvzf9dhrwoxpz8wxfw4v75ugopodda69ali5mecf8ko8tf8apxzf9daiuh +20px +they +contenttransferencoding +multipartalternative +companionship +textplain +127001 +llp8slxfygfqujoibzack2obmd7sjbaartnkqqs4g0r3wra4cuveghgfjq4dgwkgmsyokmhpvdg +nn3j7yrc15x9jeoxequgqhskodqo4wya55o2djzwfpswc9tfn9shvdusgcaipx1sy3ui71ygzw +communications +monti +that +actionmailto88mnbvexcitecomsubjectws +border3d0 +quarantinelevel10000 +to +charsetiso88591 +now +face3dver +colors +continuing +privacy +help +editor +version +there +hours +6 +name3dgenerator +happened +global +brecon +second +at +shorter +id +wrapping +size3d21 +pts +zzzzlocalhostjmasonorg +churches +external +cheque +shown +u1si7921087anq8620100424180823 +tr +highland +gain +turned +date +064416 +xbarracudastarttime +invalid +cellpadding5 +easy +2010 +xbarracudaspamreport +701 +valuable +xuidl +valign3dtop +satisfied +reps +203550 +mail +coastnbspbrand +by +rates +visit +pick +raa09761 +pill +egyptian +should +via +size3d15 +href3dhttp0pharmhuntruakeno3d2133h6702p60qyzonyv3d2lm6r677 +color3dcc3366b417bfontfontdiv +0100 +lived +their +with +from +refer +type3dsubmit +hrefhttpc81bfranklinedruepedexeyon9506dcca55c4c +mimeversion +kt9kpyxl5wannq1rw47n2xnotkbpnpws32sljgxnb62g9uh9txz2nb6pd6mcmse1mo9xsvnhnv +forms +p111ster +povezu +gw2csminingorg +give +mariage +south +height3d47td +ema +n100lich +resources +htmllinkpushhere +killlevel10000 +alink003399 +cellpadding3d1 +satisfactory +pass +acts +uid +distributor +uidx6clphoffgakaciig9kbedgfemhmiswhg9kdufanjpvwdgfzs7ubtc2dqdmh6gh4pndgm5x +02 +width3d467 +xaccountkey +marketingsubject +models +message +influence +formfont +almost +an +webmasterefiie +offer +7bit +louisiana +aug +19216833 +htmlshortlinkimg3 +ffffffchange +be +29155 +records +color3de0e0e0 +berber +texthtml +rights +height3d4 +rules +headbody +title +oh7z03q +uid35281201487268 +several +thematic +crushable +hello +brl +w3cdtd +charset3diso885 +malchizedek +border1 +testsbsfsc0sa148a +gbr +joe +alignlefton +purchase +dec +titleyour +regulation +ial +save +emails +styleborder0px +0900 +11pxfontweight +width100 +3 +images +trains +sat +important +centerlast +width10nbsptdtr +nowb +width115nbsptd +jst +day +along +contenttype +xmozillakeys +america +may +listed +41patch43bthingonep43080201fallnetscape +surpassed +other +of +mostly +packages +201005050521o455lxor011731gw2csminingorg +sponsored +received +er +19216818120 +alignjustifyspan +bestimmte +10000 +right +ac4amaaaaaeaoejjtqqgdepqrucguxvhbgl0eqaaaaahaaaaaaabaqd7gaoqwrvymuaziaaaaab +height436 +table +earn +3d +sijjxypamrsk6eeyh5x05pgmysp98hsownkihimkufwqqosakmmkhjckizmjqsergrzseiqeshma +sell +the +width136 +familysurname +score696 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_44.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_44.txt new file mode 100644 index 00000000..ea8a4292 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_44.txt @@ -0,0 +1,374 @@ +e2ksvginvklccl1soajjbvbvewpvlykhxbgxebzxnpwm9t3eq8yfm58ozzqpnsrlf4k5yko2 +indication +launch +xasgorigsubj +cells +copyright +trouble +frontpage +for +s +pdiv +smtp +xbarracudaspamstatus +stylecolor3974b4newsletterijuypocomspan +involved +motoyfuvety +bsfsc0sa392hl +children +ama +mladih +palestinian +xmlnshttpwwww3org1999xhtml +anytime +body +nalfqtjz016379 +screening +052 +in +content3dtexthtml +276 +203736 +design +transitionalen +have +returnpath +pndc +0001 +breakdown +are +mon +uefopjwvrk9ovd48rk9ovcbmywnlpsjcb29rbwfuie9szcbtdhlszsigc2l6zt0inyipfnq +xasgdebugid +dearfriend +companies +bgcolorffffff +course +force +borderbottom +070 +server +128 +subject +4 +width3d30 +custom +relay +xbarracudaspamflag +song +pmore +listarchive +system +xmailmanversion +want +ivtd +construed +du1c1ioj6jdesg3mie54xmhay5phgdo6etzhhz5vgni5nxuoikonqiqrni8w55pjavz0pddf +yc2nwuht3v7aboinanx7f1frxswtmzwnjljajbxzdllt6ug6ygkcb8z5c0qw2erwuzg5lrx +shbig505413body104 +yes +but +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +11 +bgcolor3d99 +what +centerwho +mailinin330809022015 +650 +unreflected +if +herbaceous +specialtiesspandiv +351 +up +etc +our +expected +xbarracudaconnect +noshade +newsletter +keep +minute +do +tue +head +rbl +had +commander +joyal +f +xmailerpresent +price +ready +nor +before +ahover +intended +mint +bconsentb +sa290rn +united +115755 +esmtp +893893 +effective +mx +deliveredto +msgidoutlookinvalid +receive +xbitdefenderwksspamstamp +spring +localhost +within +2011 +width3d2210022 +cambridge +rising +because +rule +lucida +credit +americans +messageid +350 +yniidxmb250ihnpemu9kzmgzwxsb3cu3byzwfkicdlbsa8l2zvbnqpc9m +pay +10 +xasgtag +atd +ntfamily +alignrightdover +professionalism +xbarracudaspamscore +or +size3d2bdiagnose +yourself +hilt111n +kidnap +peruser +taglevel10000 +contenttransferencoding +bsfsc0sa082offncn +upper +459af2c759 +checking +building +sarahwilliamstrafficmagnetcom +until +127001 +183916 +ps +agreed +keteypostinoch +border3d0 +ajaokuta +quarantinelevel10000 +parks +508 +bordercolor3d22233366cc22 +to +v1 +stylepadding +99239 +150318 +charsetiso88591 +style3dbordercollapse +size5 +code +cellpadding3d0 +you +citiesmayor +td +web +gtgtgtspantd +company +fontweight +025255 +font +srchttpwww330wcomopenopenphpeidzzzzexamplecomoidcasino09242002htmlcustidfrugaljoe +poster +documents +hrefhttp5d95gjxdjfharuiezovedafy37c7ac8392d7f +was +2qgkgwvmqh7mochcpaha7dnjnywlhtghliif0zmlnrwxse5nsqcmafbcxuaiqlvrw +grabbing +response +at +1 +m111zambi113117e +id +meta +skillsslightchasseskulkingskrslinkingskulkerembellishsolucionempiezansl1enfrenteempirenotespiffslutfslowpokesslumpsencantosmartmoneytutorielencuentransmokesmokkelaarsrotsencontrandosnakebiteenddaysnappedendedsnickersso6qrn3ohughesendrersoepsomesodomysoispacecraftssoledadsomethingabundance +pts +also +issueb +tbodytabledivbody +width200px +htmlmessage +width550 +remove +click +successful +detected +xmsmailpriority +after +frankfurt +date +carriers +take +xbarracudastarttime +easy +xbarracudaspamreport +seventh +sansserifspan +2009 +xx286646anonpenetfi +cackle +mail +itbrworksfonta +ret111rnar +institutional +by +ff0000 +151339 +blurred +style3dbor +love +httpspamgwcsminingorg8000cgimodmarkcgi +margintop +will +allegato +few +001a66d63c1d6633b3d48da33da6dqxlfe +centuryp +mnogo +properlyabrtd +association +environment +laurence +with +from +zzzzspamassassintaintorg +ingfontdiv +valign3dtoptable +2zdcpqleah1j1lj5umrdmvdu0tf5b7suw2pusxq84ohuuszctqzmhyaiag3mnizounc92kefcmbm +anderson +mimeversion +size1 +beta +gw2csminingorg +killlevel10000 +hrefhttpacbf5frofojsscncopy19982009a +xmozillastatus +acts +could +prosper +alignleftjapanese +r5cbfrwk9rcny4fgrtrlxgmpb9drv6zd9cvxy7komeks9n4waoshbty4xxfhar0n413xlk3nw +fetchmail590 +jan +sender +targetblankcomments +permitted +accents +010804 +timestrong +clergy +members +message +yyyyspamassassintaintorg +nonexperts +malthus +crelaxedrelaxed +an +place +charleston +jp +1b +turgut +targetblank +be +street +diarrhea +respect +c +leadgeneration +gw2csminingorg19216818251 +protect +nnundiscrecipsnmd +httpequivcontenttype +texthtml +rules +time +better +jmnetsetcom +hrefhttp3ae1tfekinakcnamiuxibr720g4c51983y1rh1g6 +yiiz2siwalgpob9qu1iyagrzgn0chacnqeaaqnaqaggccyclowb59jnax6gawtg4abseoamcfbe +thousands +ask +borsellino +guess +microsoft +xbarracudavirusscanned +wallace +mailing +required +yasmin +name3dhdnsubjecttxt +icagicagicagica8bgkumvmaw5hbmnldqogicagicagicagica8bgku2vj +29 +hrefhttpfbbtabdru38aruiakaxoviiwb2a2b3ccd29f723 +act +swedish +pissing +about +m0620212mailcsminingorg +simply +97lso +against +3 +tell +0px +no +very +boarded +top +20216633142 +arrival +10pxlineheight +automatiques +performance +hibody +height3d21nbsptd +introduced +barracudaheaderfp20 +u5r78vwunxy547fngljvxvsikvvbifgy30unh6spsg4oyhfsh2ftrr5dwvqugumizklgswdw +contenttype +xmozillakeys +cypherpunkseinsteinsszcom +america +boldspana +bbdjbfzpus7jvtbzcwb1yafo5oknbxfjrbyqtvvqec4xukccfybpmlxegzknla6k1ldhobgxyy +charset3diso88591 +might +of +since +shaw +nicotinefree +401 +received +jphillipsroyalnet +today +borough +200 +height5 +123000035f8tipsmtp1admanmailcom +awakening +28 +needp +month +sa154a +color006600hfontafontspanh1tdtrtable +the +classmsonormal +800760016490head diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_45.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_45.txt new file mode 100644 index 00000000..81233225 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_45.txt @@ -0,0 +1,378 @@ +size4human +tppdirx8eo2sbc9u4jxrjooltjgbf06773uvb7y9zwbfg7bz3q5fjtzjapycj3my18nk +execution +211101210170 +please +read +alink +for +postfix +qfc2ercwyendby0wgc1ukv9a3inmezenw06orccnicuaawlulftneppl7itycva16ufoolgubymk +drive +width3 +recipes +001 +network +jqstnv +smtp +xbarracudaspamstatus +membership +method +nameprogidxbody +districts +vvgpoljjmtegiwdr0qwo360wf30obpaov61mdhttq7ktsfli71oacckpijw3uvn2i23opuvftnwo +o0ll8qak008229 +body +size2removal +anonymizer +tragedy +area +this +continuity +style3dcolorf1d92a +regional +firewall +imap +chanced +ho6etb9utsvbdhbsf5equ12iywezbxjg2o3urm7eqinyyeegdocbjr24qlhaxdjuhhdzug +0001 +breakdown +are +is +mon +xasgdebugid +symptoms +terrainthe +23 +00000000 +show +seen +xbarracudabwlip +subject +produced +than +mxrootsystemsnet +leftmargin0 +cassetteb +included +2002 +hcc +value9000000009000000option +late +abuse +lineheight +00001cee575e000076780000789cmailitsonlybestcom +yes +282167767 +below +future +profitsprepaid +compliance +text3d000000 +inbox +contenttexthtml +nonmarijuana +proceduresfontfontfontblockquote +would +href3dhttpzdvpoiwcnfont +event +alignrightnationalp +inmate +proper +styleheight +center +n97il +br +tool +description +topographical +own +delivered +tter +up +quotes +xbarracudaconnect +must +090503 +do +friend +all +300000 +rtc +000000 +wounds +page +doors +60250148208 +independence +edt +left +score +nong +893893 +wdqu0vamcjfjdhcjffaefnl8aaixaon173rsc54jwakn8xapqu0v8a8cjfjdhcjf +wy +britain +019 +a +deliveredto +create +align3dcenterfont +dcp +16 +barracuda +century +within +forward +as +cambridge +1685 +correcting +messageid +contentclass +year +10 +12566973370a2101c80000w4twrl +mime +beginning +coverage +told +k111nk117rrenz +man +account4 +ireland +public +mails +111142 +doctype +color808000phone +2001 +professionsl +peruser +they +taglevel10000 +mxni8qh4gnysdtoyhgcghuqskanridhycxrsaagyaitjsqii5sys9gymupcl0jhct0qc94qld8 +following +xbarracudaurl +contenttransferencoding +multipartalternative +rbnh4slcu7glfu0j +open +removalbr +importance +textplain +127001 +shipped +score2391 +231036 +ps +shda6nxnw1sbnjjgmlwndba1t9zw6npnztpqlp2f0ypjsmsd7tl2nd3oeru2v7otvg7rtkx4 +nowemstrongstrongstrongbr +that +hrefhttpwwwgeneraleducomremovehtmlgo +components +cannot +httpww +to +massimino +19216832 +code +width3d450 +jester +xbarracudaenvelopefrom +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +web +httpbitlybvlx0r +around +infanticide +bgcolor3dffffff +bio +home +greatly +solution +here +ccir +altopen +iaculis +version +there +inherit +earners +nothing +htmlfontbig +at +1 +added +law +go +rumor +low +id +rightmargin10 +sir +unpreserved +mediterranean +pickup +on +external +face3dverdanaarialhelveticaevery +htmlmessage +1986415471generic +tr +my +date +spamgwcsminingorg +sirs +alignleft +situationkeep +shopping +integral +ajnggay5jd8aim1raxhdbutxqfiyoxrjwqvccneimadfml0i3dmef5pjkwfraqqagwzwpnoziwkk +authenticationresults +xuidl +mail +by +en +then +size3d50 +anon +mortgages101net +facearial +attalus +gw1csminingorg19216818250 +isbn +20018972186 +mypage +famous +162108 +with +from +wealth +accelerating +mimeversion +couple +011533 +murjel +psk80leixekiihnpemu9ijqiignvbg9ypsijmdawmezgij6lbqr5rsksqbmpuav1pc9mb250pjxm +gw2csminingorg +r5sh5ut6ydswc6isy2u61zxiai0tabdv0vurz77ad0h6v9vv5vt5ayh6v2nf5upr4fvx +alink000000 +width141 +mortality +000 +first +killlevel10000 +servicesstrong +href3dhttp46f873vliwosamcnxoyvoraba3d4ew314945a9d0 +kremlin +hrefhttpkobrapostcomaide60htmlunsubscribeatd +mellat +dormant +sa275hl +xmozillastatus +splendour +hrefhttp13eefbezopoxcn62j79583105044g8ro914n89niehacepoinukoshibody +choice +copy +romanstrongcialis +gw1csminingorg +charsetutf8 +02 +words +backgroundcolor +macaufonttd090909090909 +sender +end +valueslovakia +message +385ce77c9c0 +earliest +height3d56 +rx3dhgarjzm8d4uj4m572xfw4uf8kr8tf897lv4f8k983d3pnwznmjuxozwphsxifaj62 +an +brbrbryou +offer +hrefhttp7breynardtabruhybuis5dca2eb1a17136cbrowse +financial +be +meet +152559 +where +462 +get +colorffffffhome +name3dhdnrecipienttxt +texthtml +bgcolor6666ff +tieins +substitutes +jftheriaultnetnologiacom +emotions +kind +several +existsxmailer +microsoft +attend +bsfsc5sa269hl +30 +7 +type +h7w1r9jwzkkj8obemcwbfybfldogictylvcqmgilqi0qaecandbaibihsitnkley1yjaeib4dxa +shuffling +paddingtop +guaranteed +html +identified +hrefhttpa1dea0gmakahercnojfef3549de7d9857010e07kjm989232689485534000904 +matches +1930 +teen +contains +fri +acexubawegcnvuyrixenehewyboquxgif +height17 +sizable +color3974b4update +communicationspart +0900 +abm +engineering +morning +150 +00fffffont +inscrites +imabdhotmailcom +12001600 +no +approved +512mb +mail1insuranceiqcom +190264 +does +width115nbsptd +jst +lithuania +de0ed43f9b +aairaqmrafeal8aaaeeawebaaaaaaaaaaaaaaafbgciagmeaqkbaqaabweaaaaaaaaaaaaaaaab +contenttype +may +secure +of +dynamiclooking +csminingorg +and +received +size3d3euokfontbspanspan +reserved20 +height5 +o +table +19th +went +walton +detailsbr +sell +kazakh +capacity diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_46.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_46.txt new file mode 100644 index 00000000..3045dd2b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_46.txt @@ -0,0 +1,353 @@ +divfont +xasgorigsubj +please +valigntop +textdecoration +for +smtp +h2is +g8bhxke9029482 +ignatieff +aligncenterfont +yordon +mailwebnotenet +axial +name +this +in +svg +lyrical +your +emailnbsp +is +mon +learn +nonmigratory +improvement +president +100 +llksa2bxctxusmusylabvol73ouw2t5rwzhbfphkuyrhmxguxvp6ntguc5y8otaik6a3xvzaw +width3d23 +000049ed75d7000003d2000034c4iqyiqndazlt +srchttpwwwbubasfrcnjcjgif +xbarracudabwlip +subject +sensitive +custom +450 +given +private +bel111w +ttncpssezjgxnlfmbojto0scqwqclg5we4tamwhqbvsqfk5gkmkmcklrydv3tzwtyzg4o3pjnet +included +forth +fontsize +email +quotedprintable +debtfonttdtrtabletable +pfizer +79nbsp +decide +but +style3dheight +17nn1wng9r3apzuvb4z5gbke1rbpg1fe3yzydbit2n3jg7l0wyhnaht13vslym2py5l0knjzjxjx +solve +enhance +contenttexthtml +fontbfontbfont +ffers +h5as +style +students +br +apr +description +o3scrk35002622 +mileage +href3dmailtosherrykancsindianapolislifecomemaila20 +dzdq55pvos9xqypwrz4famvx3bgncrwmg5umwdxvv8agpnyz3x087rj61illmv +tr97n115mitting +so +our +introduce +german +width174td +must +hen +band +institute +planningp +lerleramilerctrorg +g881cgmd001644 +hit +burning +host +price +colorffffff +yourpenis +type3dhidden +xmimeole +replaced +just +pjmplui5fwhhntjgkqrfulapdnnr20hwsahpgmiqpe2jppeck8nuuk6rmksbab0o7nsv0jrh2sl +us +073620 +esmtp +viviacomua +avant +spin +valuelblebanonoption +a +200911130855nad8twmy017686ns2csminingorg +deliveredto +trap +retired +century +localhost +79 +rule +passes +well +christian +5 +has +messageid +formbr +10 +car +reunion +john +mime +color3d2223ffffff22 +axmgzs1tywlsihrozw0gdghlifjlcg9ydcb0agv5ig9yzgvyzwquifroyxqn +or +nce +public +pchapter +leader +they +taglevel10000 +each +0200 +lan +xbarracudaurl +193120211219 +reason +xbitdefenderwksspam +section +spacebrbrfontbfont +helvetica +stylistically +width535 +n6qfim7x021624 +g4pc2pe29708 +named +morebfonttd +styletextalign +align3dleft +to +v1 +patrimoine +gtzshrisc5me7cs8ahrezhdvnyb2irnbemipuorwlagpuu8prdi16kid3862ukgsaiqqoey +singledrop +code +bwgjcgsbaaefaqebaqebaaaaaaaaaaeaagmebqyhcakkcxaaaqqbawieaguhbggfawwzaqaceqme +size13u +lyndall +32 +russian +organic +fontweightboldmedicine +qgvjmyfhn6yaaissprjybqumw8yelv3ghmsliuck8knpuuwrrosktqgcaa0jrx0oasvljcwwluis +single +longerevil +href3dhttpb44xubawegcn +webnotenet +v +internet +courier +surnames +oftitle +hours +pfc6bgnmolbcrghkjcvky0nerdjxmruahjgqnjkoow85fszkdzk1mk4qkhka0brwoklcmtvcqj +name3dgenerator +vii +style3d +addition +at +replicawatch +h1si5852311nfe20060530185244 +id +dollars +mandarklabsnetnoteinccom +pts +rdns +over +also +listhelp +can +on +joduchhotmailcom +equal +remove +b +xmsmailpriority +date +sans +take +31 +cellpadding5 +2010 +xbarracudaspamreport +l +hrefhttp611vqokiduhcnfqyurjfe9b8c449f314d801a0bda81kddcnhibodycsminingorgasqto95757225910285006078650amaze +valuable +xuidl +them +101508 +testifyindeed +justin +by +then +playerd7l7 +he +unsubscribe +revisando +2nb3argumtiqjafdfamratkjamu9kkz6vfftq1hsa84c9aqalcg20q5lc7iszx754qcjjc9mok +antispam +should +new +written +0100 +cause +n4cftmra012467 +trusted +not +bordertop +81168112 +with +from +dwl +still +9fonttd +saints +warstalin +executing +width93a +mimeversion +0500 +27 +gw2csminingorg +000 +centuries +aligncenter +cellpadding3d10 +currency +height20 +s117bhea100er +xmozillastatus +gw1csminingorg +make +xaccountkey +junk +1962 +and20 +putting +19216818251 +shall +cellpadding20 +an +kaj +3si7771224fge020100519042131 +serving +herea +7bit +jp +financial +be +stationed +meet +1806 +quest +get +her +wagons +texthtml +rights +playing +replies +lodger +vlink669acc +better +xvirusscanned +tify +several +annmn73b9303b9351tawr012000003b930mrvm4rvqf +special +xbarracudavirusscanned +w3cdtd +sa392f +identified +style3dfont +town +really +dec +border3d0atd +act +delivery +innovative +elected +territories +lc9zzzcvuwqwymg0ussgydtplewzvtktlqocyutqkd2dhfqiozc8pjlyhrahczrjkljoxaa +plantations +publication +stoxreplytype +marginheight0 +aeadb2e8bbadeaa4aabea9e8a4a9bce8bfada4a8aebabfe8a +kabini +see +lending +1992some +width3d75b1100btd +tell +made +mulder +0px +no +western +we +approved +membrane +monthly +lutheran +hibody +jewel +55040292901 +jst +iaa13857 +contenttype +xmozillakeys +deal +write +of +dynamiclooking +and +received +parkway +list +y2vsbfbhzgrpbmc9ijaiihdpzhropsi2mdqiigjvcmrlcj0imcigym9yzgvyy29sb3jsawdodd0i +table +protects +eight +cases +month +shipping +sa154a +strongcompliment +view +size3d5without +amounts +the +vita +parts +those diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_47.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_47.txt new file mode 100644 index 00000000..24f8eac6 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_47.txt @@ -0,0 +1,295 @@ +002412 +oulook +used +devastating +a0c2a0c2a0c2a0 +pm +examplecom +for +postfix +calculator +maybe +business +surprisingly +autolearnno +resentmessageid +area +clientip8219575100 +dkimsignature +this +in +certificate +lumps +dream +your +returnpath +is +hrefhttpclickthruonlinecomclickqb1yjxjq0dzicwjmr9ifb0nlnlr +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +testsldosubscriberldowhitelist +too +feature +unplug +testsbayes002 +httpwwwworldwidewordsorgwordlisthtm +x8664 +00000000 +subject +relay +independent +listarchive +2002 +bulk +email +listunsubscribe +disk +164013 +jmjmasonorg +but +20100502033346ga27884steveorguk +hr4jcslid2doughgmaneorg +testsawl +sure +0000 +future +xspambar +although +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +ist +120349 +good +problem +document +374353 +own +fallback +inbuilt +thu +do +26 +shadowy +domain +luck +nolist +httpjeremyzawodnycomblogarchives000202html +postgrey131 +receivedspf +mailwall +0400 +xmimeole +mmx +push +silently +just +else +esmtp +protocolapplicationpgpsignature +ca +listsdebianuserlisztdebianorg +a +kodak +deliveredto +wrote +idbb7c321c +localhost +useragent +as +voice +hurts +unchanged +freetype +well +pdt +has +currently +messageid +tweaking +emacs222 +tangentorange +signature +10 +risk +glen +m8cs235061wfj +0700 +notinsblxblspamhaus15 +wrestle +g8sftfg31942 +nasdaq +or +cumbersome +httplistsfreshrpmsnetpipermailrpmzzzlist +problems +0200 +exactly +required50 +httplistsdebianorg20100510071542542b0d61pbmihamalagasycom +way +textplain +i +127001 +outlook +8bit +style3db +cards +changed +that +114323 +3d0brtd +hear +dscf707abr +to +captivecustomer +singledrop +you +westkacuedu +bank +company +amavisdnew +auth02nlegwnnet +niggarding +most +mlsubscribertechcsminingorg +week +courier +there +founding +596 +iphoneipad +personal +hash +1 +122911 +debianuserlistsdebianorg +id +bgcolor000000 +over +also +listhelp +forkexamplecom +can +p05111a3bb963cd44ea4966149496 +awards +on +my +date +mailtorpmzzzlistfreshrpmsnet +authenticationresults +145 +mail +range +manager +by +129206100140 +even +comment +helo +httprelease1edventurecomexecutivesummarycfmmcodeunspun +spain +starts +2525 +xbeenthere +border3d0trtd +p +httplistsdebianorg20100507201923312martinlichtvollde +not +listsubscribe +somebody +substitute +with +from +generated +07132001 +thru +mimeversion +enht +next +heaven +jobs +include +both +xcheckerversion +mode +listid +fetchmail590 +done +metadata +size2 +app +sender +crap +anyone +its +errorsto +place +hrefhttplockergnomepricegrabbercomfind +xbrightmailtracker +oxygen +lm +quaint +e20cs390411wfb +mailtoforkrequestxentcomsubjecthelp +loss +were +kept +ffwelshfenglish +power +guess +more +update +httpwwwdaisyblossomcom +colspan4 +200210060800g9680ek15204dogmaslashnullorg +manual +duncan +grabbed +g8ukzvk14996 +spam +about +started +8219575100 +ckquote +eihen2ev7gatedatbofhit +chimpvancouvercom +3 +114240 +no +very +geneva +entered +dashisbinsh +contenttype +xmozillakeys +helpunsubscribeupdate +dice +xrcvirus +spanbrbr +namenumperpage +may +of +hits6634 +and +datereceivedmessageidsubjectfromtocontenttype +record +received +one +4bcf28555050901heardname +theirs +indianstyle +ever +aboard +list +4t9wvglpbaooe6bqylbliszt +mailtoforkrequestxentcomsubjectsubscribe +bnote +javadevbouncesmlsubscribertechcsminingorglistsapplecom +rss +authority +technical +cdc4d16f16 +httpwwwcatheadpwpblueyondercoukonlinediaryhtm +the +hrefhttpnlcomcomservleturlloginemailqqqqqqqqqqzdnetspamassassintaintorgbrandcnetmanage +maintain +saa16998 +advertising diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_48.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_48.txt new file mode 100644 index 00000000..7aeaafd7 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_48.txt @@ -0,0 +1,276 @@ +michael +truckers +down +panel +oldreturnpath +335 +for +postfix +tourists +050545 +encodingutf8 +support +channels +rahettingapopearthlinknet +08 +working +adding +81168116ver +researchers +stages +window +resentmessageid +uswsflist1sourceforgenet +clientip8219575100 +autolearnham +in +have +returnpath +jul +imap +mailtodebianuserlistsdebianorg +are +is +uswsflist1bsourceforgenet +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +his +agree +securetestingteamlistsaliothdebianorg +built +50029196900 +subject +pgp +qmqp +boundary11663001032541006905javamailadministratorweb5 +score146 +listarchive +httpradioweblogscom0107659 +chipabr +bfont +included +xmailmanversion +bulk +175809 +exist +connect +but +recompile +0000 +1134 +debian +master +choices +ubscription +samplecontentfilter +owned +envelopesender +ist +nameh1a +if +apr +gldudebianuser2mgmaneorg +oct +fallback +required53 +4bbbf5307080408pinyonorg +scomcomicsdilbertdailydilbertimagesclothingandpostersgif +btw +going +european +nationalism +play +all +domain +features +re +stuckey +scv +demons +c3a9crit +iso88591qj2tkfikpap8d93f6vul0h8bcu7vjq3arhvdfgp4atlxo3fuhbuaaaaa +razorusers +0400 +full +understand +it +esmtp +protocolapplicationpgpsignature +893893 +2159 +24 +a +dont +deliveredto +mean +xmozillastatus2 +35 +wrote +bcable +localhost +formally +earlier +jadearacnetcom +0710 +size3d1img +well +04 +pdt +has +messageid +consideration +054923 +0700 +gnulinux +acri93zfypvyeb0dr3kg0aqoejzftgaakjvq +1d96f13a53ec +dueling +ireland +uswsflist2sourceforgenet +03 +augdbouncesmlsubscribertechcsminingorglistsapplecom +such +contenttransferencoding +21 +last +forkadminxentcom +2008110401 +1022329156 +textplain +depending +127001 +that +out +thunderbird304 +deeper +to +tls +123519 +exact +singledrop +xlevel +righttalking +20080610 +you +utc +advice +choiceabr +glaciers +121401 +references +21078 +change +personal +raa10694 +at +qt4 +been +yyyylocalhostspamassassintaintorg +10874515 +id +doesnt +poor +near +date +neighborhoodsnet +me +them +mail +af6decorecpp360 +envelopefrom +by +102437 +unsubscribe +20s +lisztdebianorg +httpwwwgeocrawlercomredirsfphp3listrazorusers +5e87l5e87i +164722 +ldowhitelist5 +back +xamavisstatus +little +autolearnfailed +exmhworkersadminredhatcom +not +streets +with +from +still +rohit +forms +murcko +realtime +2 +libasound2 +180046 +3d6538748010204barreraorg +xmozillastatus +forkrequestxentcom +could +12b4b274cb3a0 +charsetutf8 +02 +news +sender +permitted +tried +position +pains +incorporates +crelaxedrelaxed +errorsto +size5tony +percent +irish +mailfx0f47googlecom +7bit +taggedabove10000 +milexc01maxtorcom +be +merely +xpolicydweight +g26mr3227376fab181273929816994 +functions +6416122236 +zealot +a0when +sort +xloop +wanted +81168116 +bgcolorffffef +manual +vom +chasing +monitors +current +xoriginalto +about +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +livingafonttd +0px +no +archive +httpwwwnethubcomupdates +webcams +intermail +some +policy +version325 +contenttype +ejzqs7kivsnexerocckxlth8h5ncuiilmxtbn5g8f0hd22x38zxijdnelvewm +width3d605 +write +might +of +peaces +and +record +received +archivelatest576722 +one +z +javadevbouncesmlsubscribertechcsminingorglistsapplecom +raid1 +search +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_49.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_49.txt new file mode 100644 index 00000000..f7b1a5ec --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_49.txt @@ -0,0 +1,293 @@ +langenuspp +secret +xentcom +for +postfix +smtp +chip +test +20020901lpt78839000wwwdudexnet +c00112d0c59 +srchttpwwwhandangocomimgsthechampion2002handangologogif +admin +cf1843f58c +helomailpineauscom +compability +any +kernel +window +resentmessageid +18th +clientip8219575100 +account5 +dkimsignature +this +in +based +have +your +returnpath +imap +mailtodebianuserlistsdebianorg +193905 +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +versiontlsv1sslv3 +server +subject +pgp +emailspecific +qmqp +using +40 +than +ahoswrazor03snet +xmailmanversion +bulk +micalgpgpsha1 +scoble1 +email +usenetdoughgmaneorg +signatures +listunsubscribe +jmjmasonorg +but +hrefhttpwwwlockergnomecomimageslockerlinkgiflink +0000 +explained +0800 +2495month +9 +guides +infobfontap +what +dcc +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +online +if +fourla01 +good +dmesg +problem +fallback +up +maingmaneorg +so +keep +do +all +distribution +kdepims +domain +090041 +re +bgcolor3d333333img +users +height3d35 +intelligence +bouncedebiantestingsecurityannouncemlsubscribertechcsminingorglistsdebianorg +receivedspf +set +before +bored +esmtps +mailtoforkrequestxentcomsubjectunsubscribe +understand +it +esmtp +implied +srchttpwwwcnetcombgif +fix +listsdebianuserlisztdebianorg +a +dont +deliveredto +create +localhost +2011 +as +dogmaslashnullorg +build +inline +spfpass +well +pdt +regular +messageid +certain +whose +airkvbgf3v46fcoh1nyafmw117 +man +or +hrefhttpclickthruonlinecomclickqdc40zzq0lr5ohlqxybz6gewtinhldr +resentfrom +aanlktin9u3dmww3x0et8i9crpfwphp7lefe9ltoptljmailcsminingorg +hrefhttpwwwecobuildercomlockergnomecfmecobuilderabr +processing +contenttransferencoding +191740 +2008110401 +stop +textplain +i +until +127001 +8bit +work +programs +that +java +idcmailmo1soshaw +camerasbbr +archivelatest577457 +184149 +to +v1 +direct +bgorgeous +rv101 +20080610 +you +utc +103113 +intheloop +small +331vamm2 +most +here +xpairauthenticated +hrefh2 +help +was +mailtoforkexamplecom +references +mxgooglecom +6 +stuck +xnewsreader +doing +at +discovered +maa13118 +debianuserlistsdebianorg +kdelibs +xspamlevel +id +tfromendsinnums1tnonsensefrom0010 +results +concentrations +calculated +port +listhelp +forkexamplecom +mailtospamassassintalkrequestlistssourceforgenetsubjectsubscribe +105344 +divgt +date +worm +24articleview +kachemacs +usrlibperl5siteperl560razor2clientagentpm +googlecom +httpbugreportapplecom +by +think +unsubscribe +lisztdebianorg +status +however +112809 +xmailinglist +will +via +4bdb718c3060402allumscom +xbeenthere +stoddard +xamavisstatus +autolearnfailed +listsubscribe +limos +with +from +intentions +width1 +021905 +3ab5213a4cfe +liszt +enht +spamassassin +2 +properly +fight +bukkake +xmozillastatus +choice +use +listid +put +charsetusascii +fetchmail590 +width1td +regards +sender +precedence +tried +message +mr +score7 +original +contentdescription +taggedabove10000 +2264 +be +listpost +004702 +mephistopheles29hotmailcom +6416122236 +always +xvirusscanned +logmaneorg +g8dnwcc07796 +special +mailtobitbitchmagnesiumnet +r +midjuly +wget +form +xauditid +81168116 +really +011034 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +duncan +example +3 +g7qicyz04038 +some +skin +together +policy +version325 +including +contenttype +user +input +xrcvirus +like +of +since +and +200209240800g8o80wc26600dogmaslashnullorg +xstatus +received +contract +free +list +definitely +28 +created +color3d000000146s0d +c241e43c45 +m8cs151040wfj +the +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_5.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_5.txt new file mode 100644 index 00000000..c85f87e5 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_5.txt @@ -0,0 +1,249 @@ +hostnames +105523 +please +trouble +number +for +postfix +081627 +smtp +question +gnupg +body +qmta01westchesterpamailcomcastnet +years +clientip8219575100 +this +autolearnham +in +filetered +returnpath +teamfontp +improvement +x8664 +00000000 +everyone +20020814173950a24450rovervipulnet +subject +postingsa +nonascii +gathered +qmqp +using +listarchive +2002 +texts +email +listunsubscribe +045400 +jmjmasonorg +but +1154 +dsblorgskip0 +0000 +87sk79kv51fsfturtlegmxde +playback +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +104448 +rssfeedsspamassassintaintorg +letting +if +fourla01 +terry +running +fallback +designed +regexp +so +expected +archived +cores +smokes +all +archivelatest32347 +17q2c70007nn00 +lairxentcom +exmhusersadminspamassassintaintorg +domain +providers +re +105922 +0400 +edt +required40 +uninterruptible +understand +cipheraes256sha +it +esmtp +fantaesies +os +a +dont +chest +deliveredto +job +super +wrote +localhost +reinhart +jmethodideventpumpenvnewglobalrefeventpumpenv +as +dogmaslashnullorg +1931201712 +thing +gmailid128792ca89109454 +because +messageid +10 +phoboslabsnetnoteinccom +0700 +yet +httpcvemitreorgcgibincvenamecginamecve20100838 +or +comes +norton +8812562889f9881787e6378e770b269csquirrel1921681100 +contact +others +222111 +gletterviewauthor +building +textplain +i +127001 +testing +increased +programs +that +venture +rpmzzzlistadminfreshrpmsnet +to +charsetiso88591 +singledrop +xlevel +now +20080610 +you +14124568238 +overhead +lastname +inqfhydg1jtyvy22luydidt3w51llf63a0eg +pictures +solution +here +these +inreplyto +mxgooglecom +stuck +at +through +n17mr2353400fab231272760569672 +id +sutilbrbr +can +on +being +my +1022315216 +date +testshabeassweinreptoknownmailinglistquotedemailtext +namekeyword +downloadable +bars +mailtorpmzzzlistfreshrpmsnet +authenticationresults +me +googlecom +xuidl +garrigues +mach +envelopefrom +httpwwwjbradforddelongnetmovabletypearchives000393html +by +then +garymteledyncom +client +story +173729 +ldowhitelist5 +2525 +0100 +xamavisstatus +listsubscribe +with +rpmlistfreshrpmsnet +from +fixed +rohit +mimeversion +20100406191340ad860268e417dlistsapplecom +include +linux +xmozillastatus +use +charsetusascii +fetchmail590 +hi +merciadri +sender +end +precedence +anyone +extends +7bit +taggedabove10000 +xbrightmailtracker +listpost +get +nationwide +httpslistmanexamplecommailmanprivateexmhworkers +time +cseucscedu +bits +httpwwwatmospherenlweblog +more +sort +r +mailing +smtppiercelawedu +debianuser +urlhttpwwwstudentmontefioreulgacbemerciadri +html +df3d313a5178 +1814 +start +exmhusersredhatcom +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +sthudeuscsminingorg +act +save +8vaj +example +0909090909td +golf +http +see +width375 +no +billionaire +needs +rise +kernelsmp +when +known +contenttype +pursue +may +like +might +of +401 +and +received +canned +size1breport +red +list +the +maintain diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_50.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_50.txt new file mode 100644 index 00000000..ba313b43 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_50.txt @@ -0,0 +1,288 @@ +headache +quality +messages +qqqqqqqqqqzdnetexamplecom +please +trouble +oldreturnpath +xmailer +xentcom +behind +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +pioneering +smtp +xfsfsr +window +resentmessageid +area +tells +secs +autolearnham +returnpath +imap +is +17r0ui0004we00 +srchttpimageslockergnomecomimagesgnomedailygif +ehlo +french +subject +4 +pgp +alled +using +less +am +system +055305 +b8970440c9 +2002 +want +three +fd1bb0fd528807e1705e46b70f0 +rpmlistadminfreshrpmsnet +debiansecurityrequestlistsdebianorg +email +listunsubscribe +quotedprintable +join +g8jftpc25473 +but +sure +0000 +debian +9 +cette +rssfeedsspamassassintaintorg +choices +spend +domainkeysignature +size4 +faq +yyyylocalhostnetnoteinccom +forkxentcom +radio +testsdateinfuture0306 +091705 +annual +problem +buck +so +thu +short +all +domain +luck +u +creatures +had +re +host +v55049100300 +run +httplistsapplecommailmanlistinfojavadev +receivedspf +every +exim +britains +width1tdtr +candidates +rsa +it +esmtp +181516 +httpxentcompipermailfork +skewed +dont +resentsender +wrote +google +localhost +as +pdt +5 +messageid +10 +tnonsensefrom9394tnonsensefrom9495 +unknown +operands +dkimneutral +originatorcaller +walking +0700 +quite +httpxentcommailmanlistinfofork +or +comes +00c401c250397b055460976fa8c0cflrrcom +barrypythonorg +resentfrom +looking +uswsflist2sourceforgenet +laptop +trying +forkadminxentcom +service +jukesetgainjukejava464 +127001 +communications +virden +possibility +to +re2 +singledrop +code +modern +utc +probably +partnered20 +planning +here +isc +mlsubscribertechcsminingorg +resentdate +214509 +references +there +mxgooglecom +21078 +convict +change +204630 +colorspace +at +comprise +replyto +id +difficult +encryption +over +port +listhelp +wwwjhstuckeycom1080jpegabr +on +ui +related +hrefhttpotnoraclecomtechwebserviceslearnerhtmlnewcomersafontp +complete +ce7947b358fff8ab6f1bfe7d1c0 +164220 +date +happens +063443 +namekeyword +who +authenticationresults +by +either +unsubscribe +lisztdebianorg +race +however +client +should +new +default +actually +hasnt +ldowhitelist5 +customer +xbeenthere +back +xamavisstatus +153947 +periodic +software +httpdunnehomedhsorgweblogindexhtml +boq1kbi7fia60qwofxpin9sg1zeron3uhu0a0vplb5umudvne0mdbyegmbd6i2lc7 +2632 +not +times +with +from +stuff +wilt +20 +bluetooth +mimeversion +multifile +tech +src3dhttptechupdatezdne +thoughts +2 +orders +xcheckerversion +listid +homeyyyylogsrunme +jmasonorg +httplistsapplecommailmanlistinfoquicktimeapi +report +classsidebarbyte +listsdebiankdelisztdebianorg +sender +end +permitted +precedence +thus +johnson +101428120 +io +taggedabove10000 +aug +listpost +xpolicydweight +turns +get +mailtoforkrequestxentcomsubjecthelp +rights +positive +latest +version250cvs +footage +c0c0c0 +100k +92e5b13a56db +13020916102 +i39m +scripts +picasso +9803344164 +upgrade +arsecandle +41 +failover +041937 +xoriginalto +25articleview +difference +17hwwd0003gh00 +archive +presetsbr +needs +overwhelming +digital +conventions +hrefhttpclickthruonlinecomclickq18whdzi98qbtkdskjtimeyr1n6759r +great +contenttype +craft +may +of +173947 +minded +and +received +without +debianreleaselistsdebianorg +wed +stuffp +list +nt +goals +phone +lioness +auphvmhaaaa8 +the +d +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_51.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_51.txt new file mode 100644 index 00000000..147461a8 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_51.txt @@ -0,0 +1,257 @@ +belabouring +ideas +huntsman +bytes222867 +for +postfix +rpmlist +mailtodebianuserrequestlistsdebianorgsubjecthelp +cpan +cancel +feel +78701 +arsasha1 +review +body +wasnt +dm02014bd138131dd5ae9 +resentmessageid +uswsflist1sourceforgenet +clientip8219575100 +this +nobel +autolearnham +in +calgary +have +returnpath +imap +mailtodebianuserlistsdebianorg +is +19 +11fe0d07b7c36ae000006674a24bd646c03cdb +subject +using +less +contentdisposition +listarchive +xmailmanversion +email +equate +listunsubscribe +sure +released +debian +httpeodyndnsinfomtmeblog +what +remember +giving +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +yyyylocalhostnetnoteinccom +line +systems +stated +running +contributions +bet +fallback +mailtocgagnetechtargetcom +required53 +tee +so +our +appear +all +tue +height180 +kv +print +domain +primarily +internship +re +run +youre +preview +result +exim +set +laa29406 +bgcolora9c4cf +edt +it +24si12163489fav5920100511124120 +xoriginaldate +esmtp +runs +nearfinal +anticipated +local +lawrence +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +establish +deliveredto +xmozillastatus2 +bythinkgeek +cafe +provides +though +spfpass +pdt +messageid +4323 +12730165232011637camelalexminisamadcomau +hil +usrliblibwinsso +0700 +monica +r1100rs +servlet +wireless +httpxentcommailmanlistinfofork +or +resentfrom +target3dblankhttps +xrcspam +0200 +height7 +contenttransferencoding +last +2008110401 +service +textplain +127001 +that +out +uswsffw2sourceforgenet +141735 +middle +gmt +to +282662 +161657 +66187233211 +singledrop +key +you +modified +filter +did +minutes +color000000product +internet +mailtoforkexamplecom +resentdate +inreplyto +6 +kick +nothing +doing +at +hrefhttpclickthruonlinecomclickq3f4saeiqmwsi0sbyyhcoh57bymnzer +id +050051 +httpslistssourceforgenetlistslistinforazorusers +listhelp +can +properties +divgt +my +14 +date +easy +12px +me +googlecom +mailto +justin +by +garymteledyncom +even +gmexim +barboza +helo +will +2525 +management +0100 +again +printed +specific +not +listsubscribe +with +from +still +20 +mimeversion +enht +spamassassin +belphegorehughesfamilyorg +xcheckerversion +pass +priority3d30 +xmozillastatus +could +listid +dgooglemailcom +fetchmail590 +webmaster +header +sender +precedence +grove +crelaxedrelaxed +errorsto +22029044t9bsgc0tywdu1bummail4unitedmediacom +which +say +op0 +be +listpost +shape +shows +quick +hrefhttpclickthruonlinecomclickq1ccsiifraa1g4ol3n9kgga2oxqgyr +wifi +fault +4bfbf8a53020906studentulgacbe +update +politician +httplistsapplecommailmanoptionsjavadevrmann40latencyzerocom +mailing +urlhttpwwwstudentmontefioreulgacbemerciadri +score91 +manual +httpjeremyzawodnycomblogarchives000230html +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +29 +border3d0atd +elected +fri +xoriginalto +about +settlement +kde +3 +no +archive +installed +needs +download +cyan +contenttype +xmozillakeys +helpunsubscribeupdate +of +and +received +runtime +x14 +wed +list +cdburning +exmhusers +phobos +hrefhttpclickthruonlinecomclickq09jodsij88wyrwyd1lyss7zmgty8cr +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_52.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_52.txt new file mode 100644 index 00000000..10d1a46b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_52.txt @@ -0,0 +1,265 @@ +rootlughtuathaorg +0707 +minority +launch +wanting +httpradioweblogscom0110445 +please +copyright +oldreturnpath +read +xmailer +for +postfix +href3dhttpclickthruonlinecomclickq3dabzcqkqyp +organized +encodingutf8 +smtp +post +074846 +resentmessageid +clientip8219575100 +account5 +this +in +your +off +returnpath +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +cheers +is +101431608 +dm0201 +neat +hrefhttpswwwsandhillscomsecuresmartcomputingcpufreetrial3aspsourcecp491bonuscomputer +subject +already +7kqpxbdqtidz +qmqp +using +produced +b7adb2942bf +stocks +xmailmanversion +rpmlistadminfreshrpmsnet +email +sizeoftestbuff +thought +jmjmasonorg +but +lets +iceape1113 +emsa4 +debian +jmlocalhost +height1tdtr +url +what +although +rgin0 +unsubscription +tool +fallback +must +all +pinelnx444020909212634027072100000isolnetsuxtechmonkeysnet +26 +squeeze +20100417102207395bssiguanasuicidenet +mailtorazorusersrequestlistssourceforgenetsubjectsubscribe +thanks +host +result +set +0400 +edt +just +required40 +325 +it +donswayzemecom +esmtp +owenpermafrostnet +205210 +subscription +a +unconfigured +testsfourlaldosubscriber +infringement +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +deliveredto +job +arsasha256 +wrote +209204188196 +localhost +squareshaped +serif +useragent +as +dogmaslashnullorg +well +messageid +10 +ilug +unknown +buffer +gnulinux +yet +string +or +101011302 +advantage +resentfrom +main +contact +others +each +contenttransferencoding +im +mounted +textplain +i +127001 +ps +acjuik1q8uga10 +possibility +to +become +recreate +aes256sha +charsetiso88591 +singledrop +heloperlorg +you +utc +hrefhttpwwwgnometomescomdownload +probably +amavisdnew +around +mo11 +bz1applecom +solution +html20 +these +mlsubscribertechcsminingorg +editor +inreplyto +references +mxgooglecom +hrefhttpclickthruonlinecomclickq6de8plideikwv1ph7bc1bjbykemer +1 +yyyylocalhostspamassassintaintorg +xspamlevel +id +pepsisplenteccom +offset +also +399fonttd +listhelp +cancellation +xxxxxxx +can +on +general +ppl +designates +check +my +determination +e23d113a58f8 +date +happens +cellsp +choke +authenticationresults +googlecom +dslb088066056151poolsarcoripnet +even +gmexim +should +will +httpwwwjabbercomosdnxim +2525 +xbeenthere +smokestacks +much +0100 +hrefhttpwwwgnometomescomtome001629htmltop +2098516147 +prevents +installing +not +listsubscribe +with +from +rowe +books +un +duplicating +mimeversion +eight3d1 +enht +2002090222021651777fd2rkimberntlworldcom +mañana +linux +xcheckerversion +xmozillastatus +width3d12img +sample +listid +fetchmail590 +xaccountkey +sender +precedence +photographs +errorsto +sudoku +almost +apple +cell +4096 +be +listpost +attached +looked +devine +connecting +protonpathnamecom +were +xvirusscanned +207562130 +version250cvs +r +mailing +aided +81168116 +phillsmershnetcom +manual +105 +000e0cd7620ac7d7b70484b0e75a +no +installed +entered +layovers +133825 +does +when +archivelatest574071 +contenttype +sold +compare +like +graphics +might +since +and +record +received +one +increase +free +ever20 +couldnt +otherwise +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_53.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_53.txt new file mode 100644 index 00000000..a09da796 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_53.txt @@ -0,0 +1,277 @@ +45 +ideas +bow +width9tdtr +submissions +for +postfix +tdtrtable +find +frêsh +smtp +074303 +egwnnet +harley +autolearnno +x +score109 +124116 +court +aptitude +clientip8219575100 +this +in +010652 +81258125 +deutschlantville +returnpath +imap +ty +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +oh +server +subject +cityexchange07 +try +qmqp +4bbcdc8e2090306coxnet +200945 +httplistsdebianorgdebiandevelannounce201005msg00004html +469 +2002 +xmailmanversion +bulk +mardi +wsstrippableend +504 +requested +listunsubscribe +style3dmargin0 +39so708525wwb6 +1o26s000019loo +jmjmasonorg +standard +1921682311 +land +jmlocalhost +standby +another +ist +line +matthias +if +hrefhttpwwwlockergnomecomoptionshtmlview +oct +required53 +going +so +contentlanguage +play +head +220321 +lairxentcom +crash +testsdateinpast0612fourla +domain +re +assuming +every +0400 +height62 +065137 +updateinitramfs +it +protocolapplicationpgpsignature +subscription +httpwikidebianorgprojectnewshowtocontribute +dont +deliveredto +resentsender +wrote +localhost +useragent +thing +cay +spfpass +well +pdt +5 +messageid +yassir +signature +10 +hrefhttpimageslockergnomecomimagesvideowmva +computing +phoboslabsnetnoteinccom +0700 +men +man +xericconspiracy +resentfrom +migrate +87hbne2vqvfsfcsminingorg +contact +laptop +such +architecture +xrcspam +xfsrepair +arsenicnet +way +501de639f70 +textplain +i +127001 +xpyzor +ps +that +out +uswsffw2sourceforgenet +cvgarkv6wxulwczxlyffz6yngldvhtzyaxtudctjej4sc +to +existing +brthomas +singledrop +speaking +drop +lynx +utc +requirement +diners +around +bbridging +105017 +080031 +331vamm2 +mlsubscribertechcsminingorg +mailtoforkexamplecom +qtplayer +1309496247 +target3dnew +lughtuathaorg +id +7lf1q8ghtuoaabmmpz9rr +on +designates +httpwwwcnncom2002usmidwest0923offbeatsharkbirthsap +date +mediavanity +threadindex +69 +2010 +note +truly +googlecom +enemies +them +graphical +by +en +reviews +unsubscribe +even +censure +comment +o39ehlsm004421 +helo +new +0100 +xamavisstatus +ekornsunnmoreno +101651 +university +apache2 +not +listsubscribe +times +score110 +httpbugsdebianorgcgibinbugreportcgibug411123 +with +from +first +linux +mountroot +pass +172937 +details +listid +httpmyathenetnetfelindaweirdpagehtml +make +aptrpm +charsetutf8 +charsetusascii +fetchmail590 +hi +xaccountkey +bails +sender +end +permitted +column +file +album +httplistsdebianorg20100413205732399modestasvainiuseu +useful +beliefs +rpm +be +listpost +considered +127132540441775camelfamilypaciferacom +where +get +ok +40si11437369iwn9420100418081033 +connection +6416122236 +starting +xvirusscanned +multiple +headericsminingorg +microsoft +geguc5bec497 +a274101923384032footertellafriendcolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightboldfontstylenormaltextdecorationunderline +mailing +debianuser +26324amd64 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +manual +current +fri +k2gb59a0661005052317jf0fa9ec4n2399a8ed8d64e1damailcsminingorg +xoriginalto +about +simply +gardners +important +requests +identnzbdvp4nml0skumlgaqpfpaaekgbyhycableb36sigecomnet +17128113151 +performance +36fonttd +when +srchttpimagecomcomtechupdateidfarberjpg +policy +marketplace +mp3 +contenttype +helpunsubscribeupdate +cue +xrcvirus +may +listed +of +xstatus +sponsored +received +5aab613a5761 +0520 +timonecomcastnet +list +vim +directory +the +cant +cash +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_54.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_54.txt new file mode 100644 index 00000000..255e69e9 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_54.txt @@ -0,0 +1,283 @@ +096 +michael +regarding +trouble +xmailer +for +postfix +maybe +smtp +author +corresponding +working +any +s11si11294248mue4120100415200944 +area +uswsflist1sourceforgenet +account5 +this +instead +cached +renders +have +bgcolore5e5e5img +reported +returnpath +imap +is +scanner +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +why +23 +00000000 +mozilla50 +effort +enrollment +subject +4 +lwv8fba9ed1 +pgp +qmqp +than +society +included +2002 +xmailmanversion +increasing +penguinspeak +youtube +listunsubscribe +ls +ckloiber +yes +0000 +0530 +httpwwwluranet +exceptions +size3d4software20 +ist +line +systems +world +apr +problem +neither +fallback +thumb +up +xapparentlyto +298 +backups +do +enronsurrounding +164649 +ce37 +m0jdm40cvunahq2gwznz4vwhxr4tqe1fn4rv56ymri9dk9aoyi65nm1ltxmxjp22trfz +8368 +run +music +kernelspec +exim +2010041821271154ff5788abydosstargateorguk +204034 +enhancement +13122511111 +interviewed +just +required40 +it +brick +esmtp +listsdebianuserlisztdebianorg +subscription +a +4bca32e46000906doboseviccom +deliveredto +resentsender +localhost +bad +twice +useragent +as +because +frommxmatchesnotunvrhelodomain16 +pdt +messageid +10 +6f22f271e9c38 +unknown +airkvbgf3v46fcoh1nyafmw117 +oversized +sgi +15304473447567buffyjpcinet +vicrenzo +contact +2008110401 +textplain +pub +127001 +score86 +testsgmailhtmlmessage +that +listsdebianorg +to +4bed34a18040203coxnet +refactoring3 +xlevel +now +you +web +utc +6893113a484c +dsa1925 +select +g82dztz03051 +cuts +various +331vamm2 +these +bugtraqsecurityfocuscom +mlsubscribertechcsminingorg +was +resentdate +links +found +at +1 +debianuserlistsdebianorg +perspective +lughtuathaorg +iso88591q0a09bjfritjgt9elq7hgvhv2rta7r5xnlttjbtq4kskf6esmferfopfz +id +netedge +folder +gmailid12858a06aa50c4c1 +port +listhelp +on +machine +designates +check +how +interest +xmsmailpriority +date +hdomainkeysignaturereceivedreceivedreferencesinreplyto +installations +jalapeno +cellpadding5 +2010 +authenticationresults +supposed +googlecom +registers +scene +by +visit +4854313a49bd +wrought +unsubscribe +lisztdebianorg +listmasterlinuxie +qyk5 +will +ldowhitelist5 +2525 +0100 +lifetime +3dhttpwwwcomicscomcomicsdilbertdailydilbertimagesdailydilbert +listsubscribe +with +from +bell +iso88591qz4ltpu6qolya6g9aw0krjlajcsgcicycxrcoxwmurhckrvhcedaur7own2 +syddes +mimeversion +implement +hrefhttpclickthruonlinecomclickq0313t3i3rzdpdimcshkqcallai6orr +rpmzzzlistfreshrpmsnet +realtime +iso88591qcamalef3n +life +linux +xcheckerversion +pass +boy +mode +listid +3156 +jmasonorg +charsetusascii +fetchmail590 +dimensional +sender +sakes +errorsto +which +say +an +7bit +taggedabove10000 +nextpart1367779k1620mc9bf +listpost +xpolicydweight +where +letetrs +texthtml +always +time +crony +script +tdtrtabletable +vhs +moviemail +7eba413a565c +30 +decreasing +form +7fe1b440cc +4210 +href3dhttpgroupsmsncomoneincomelivingnotificationsmsnwtype3d +httpwwwdoubletjp +81168116 +g2abr +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +g8sncog14512 +contains +fri +example +tracebacks +8219575100 +httpgrassrootstechnetnewsindexphp +3 +sources +mailfollowupto +installed +some +strangest +greerhardwarefreakcom +performance +when +policy +great +contenttype +xmozillakeys +3d7img +xrcvirus +like +of +and +record +speak +received +cops +list +issn +mailtoforkrequestxentcomsubjectsubscribe +forteanaunsubscribeegroupscom +piece +handle +the +harrison +send +freshrpms diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_55.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_55.txt new file mode 100644 index 00000000..df20babf --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_55.txt @@ -0,0 +1,284 @@ +rootlughtuathaorg +rate +e +width85 +writing +20080517 +redhat80eni386 +xmailer +for +postfix +2139621475 +archivelatest574003 +announce +support +smtp +soldier +welcomed +gnupg +080541 +tracker +title3dview +across +8 +clientip8219575100 +villeskyttaikifi +account5 +this +73 +in +have +your +returnpath +imap +agree +too +haveexmaxyes +00000000 +030412 +effort +c0c5713a6335 +server +subject +qmqp +using +less +am +scientific +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +bfont +history +email +listunsubscribe +jmjmasonorg +but +exmh1249285507p +0000 +jmlocalhost +best +considerate +mine +7b54516f03 +forkxentcom +colspan7img +if +apr +gldudebianuser2mgmaneorg +oct +required53 +10294961543d5cdd5a870d5linuxfrdyndnsorg +xoriginalreferences +etc +domain +mailtorazorusersrequestlistssourceforgenetsubjectsubscribe +dermodyritcarlowie +handling +re +result +mnenhy076666 +knownworking +consolation +unitedmediacom80clickq3d1bwadhi3naiajhtebq1x5z4zdsrrrr +325 +el +columna +esmtp +watchermithralcom +local +1304040092 +a +deliveredto +craig +806522417 +wrote +localhost +as +dogmaslashnullorg +inline +spfpass +credit +5 +has +messageid +optionsabr +500 +beginning +unknown +removed +0700 +quite +href1099710510811611158114971101161149711810164108111991071011141031101111091014699111109891111171143282971101163247328297118101abr +above +resentfrom +httpslistmanredhatcommailmanlistinfoexmhworkers +contact +crashed +resizeglscene +pgpvprivatekeys +contenttransferencoding +referencesdatemessageidsubjectfromtocccontenttype +decision +369c613a499a +userid +section +textplain +i +until +127001 +inserting +whatever +urgency +that +helobelphegorehughesfamilyorg +to +become +4bccc8581040503studentulgacbe +match +least +singledrop +xlevel +fwd +you +license +know +httplistsdebianorg4bc7a25e4030204kalinowskicombr +bank +did +verizon +matthiasrpmforgenet +department +mess +onanist +professors +making +internet +resentdate +inreplyto +visiting +references +legal +second +at +20100515143942gb3007thinkhomelan +debianuserlistsdebianorg +fontbrbr +090135 +id +skepticism +192643 +22so607476pvc6 +listhelp +strtol +on +20020207113711b19145cshelsinkifi +epic +4bf1d7058090503coxnet +designates +my +bperipherals +075110 +language +zzzzlocalhostnetnoteinccom +san +date +290006 +20100419145428841bssiguanasuicidenet +1014224927 +191 +googlecom +xuidl +smtp139mailmudyahoocom +envelopefrom +by +think +lisztdebianorg +ice +however +201224410520090324 +aligncenterimg +enig91477b3f92bb2d91ba66c724 +xmailinglist +style3dcolor +2525 +perceived +xamavisstatus +wont +not +listsubscribe +reboot +with +from +rohit +gave +liszt +mimeversion +enht +spamassassin +2 +dvds +linux +flag +xcheckerversion +pass +xmozillastatus +listid +lnbbljkpbehfedalkolcoeobbcabtimonecomcastnet +bino +report +nickmgo2ie +sender +precedence +liptak +unfortunately +developers +errorsto +say +listpost +alan +plans +instructions +6416122236 +better +enough +part +xloop +spamphrase0305 +resqlserverdev8383110lenny1mipsdeb +81168116 +thinking +bo +manual +archivelatest573667 +optoin +about +see +mike +very +illegal +searchwindowsmanageability +leitl +content3dmshtml +value +contenttype +user +xrcvirus +of +dialog +away +and +classgetclassvariableclass +record +received +xsaeximscanned +plan +ditching +reconfiguring +hrefhttpwwwgnometomescomtome001106htmltop +list +carry +complex +phobos +view +the +send +sh diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_56.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_56.txt new file mode 100644 index 00000000..826d984e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_56.txt @@ -0,0 +1,255 @@ +remarked +looks +bottommargin0 +ddoboseviccom +trouble +oldreturnpath +xmailer +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +encodingutf8 +maybe +smtp +iso88591q1oihvnx8kkgi5h22edtzhvdci52r6okayenponuafy6nr3igysu4qjf +color333333 +people +122143 +mac +eager +keyboard +macintyres +in +cached +design +myself +have +returnpath +imap +teledynamics +is +mon +bat +oh +width525font +ftpmasters +examples +subject +4 +qmqp +using +am +listarchive +system +114138am +than +2002 +xmailmanversion +rene +jmjmasonorg +0000 +tp +httpdijestcomakacategoriespublicpolicy +chipset +fourla01 +br +problem +redhat72eni386basepkglistos +required53 +titlecapture +xacceptlanguage +him +election +g9181ck15573 +pgpdelextcmd +garionphxcsminingorg +kdevelop +002501c24e7168298e20f600a8c0dogbear +receivedspf +searchebusiness +emergence +anybody +mmx +push +mailtoforkrequestxentcomsubjectunsubscribe +titlevisit +just +325 +it +esmtp +listsdebianuserlisztdebianorg +watch +subscription +semiconductor +a +13si4978978fks3020100416155607 +moving +deliveredto +information +wrote +16 +localhost +though +2011 +useragent +as +dogmaslashnullorg +spfpass +pdt +messageid +tweaking +colspan2font +quite +pudge +gnulinux +seems +yourself +contact +come +74e051028614 +xrcspam +trying +far +contenttransferencoding +multipartalternative +ticket +textplain +127001 +lost +testing +110726 +changed +files +070902 +that +xmimeautoconverted +listsdebianorg +68bf3cecc8c1 +to +occasional +xface +least +singledrop +xlevel +mygywymcglatobdx6lbliszt +a0im +20080610 +you +solves +gmailid1282b63b6dac27de +did +small +ping +version +visiting +celeron +change +c1 +feedback +russians +experience +at +debianuserlistsdebianorg +remote +electronic +xspamlevel +id +qs9oppsbma7v +someone +2440 +port +on +installation +updates +date +usual +live +g +jalapeno +avantage +100233 +marketing +2010 +etcsshsshconfig +mailtorpmzzzlistfreshrpmsnet +me +mesh +envelopefrom +by +think +even +helo +ball +xmailinglist +online320169fa3pmiltnayqm0zdrr1bnewsletteronlinecom +ldowhitelist5 +cpu +102236376 +printed +unless +not +listsubscribe +waited +with +from +valid +address +routing +use +listid +cocoa +sent +rather +bails +sender +precedence +yyyyspamassassintaintorg +its +errorsto +which +material +cc +street +width3d1img +where +dime +xurl +release +5a8023f380 +bwatch +hello +type +rlfrankparadigmomegacom +w +really +manual +httpwwwberniewebnetjournal +truth +contains +972ac91f0 +200210090801g99813k25266dogmaslashnullorg +8219575100 +3 +20020201150022b11472cshelsinkifi +sglftgif +very +we +edulog +20100508 +major +i486pclinuxgnu +when +contenttype +may +of +minded +packages +xstatus +received +high +ditching +list +phobos +the +8so74530pzk18 +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_57.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_57.txt new file mode 100644 index 00000000..f83d9c1c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_57.txt @@ -0,0 +1,257 @@ +1673 +records146 +down +httpfreshmeatnetusersnagytam +trouble +335 +for +postfix +a3wqphnz2ekjsrmpw9hkop5bvmqea4 +find +04152010 +544 +smtp +ketvirtadienis +78701 +mac +tells +fraction +your +jason +returnpath +imap +iluglinuxie +are +is +source +dialondemand +s9b +testsbayes002 +show +subject +got +protected +relay +using +httplistsdebianorg4bd543891030604hardwarefreakcom +contumaciainvesticiorg +signs +than +2002 +xmailmanversion +bulk +gmailid128d07444f1cc0f6 +messenger +200209241715011338194248mailmanlairxentcom +220007 +email +listunsubscribe +connect +but +nav +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +another +defaultspcmipckey +proper +dunkels +oct +icedove304 +121743 +bgcolor3dfffffffont +archivelatest574749 +xoriginatingip +kalinowski +ran +features +re +hit +medlicottpanasascom +4188 +img +shield +receivedspf +occupational +edt +46 +just +mrwx +habeas +esmtp +a +killer +deliveredto +khare +xmozillastatus2 +developer +wrote +bergsten +localhost +within +2011 +dogmaslashnullorg +pdt +utf8qsnwt0nxblkvuwjblkwvjrmaammjqarllcgicwlggz3yjnrxoiqymzrfnqbq7vii +messageid +year +score69 +10 +coming +0700 +20021007t0352540100 +or +resentfrom +public +come +trying +contenttransferencoding +way +de +textplain +i +127001 +142957 +work +central +freedom +to +re2 +alive +singledrop +xlevel +you +18si1915898fkq420100421110532 +bank +clark +auth02nlegwnnet +specification +scomcomicsdilbertdailydilbertimagesbooksgif +here +references +quicktimeusersbouncesmlsubscribertechcsminingorglistsapplecom +doing +at +1 +mailtox11userslistsapplecom +replyto +133810 +debianuserlistsdebianorg +added +mix +id +083527 +imprononcable21 +also +hrefhttpwwwhandybackupcomimg +on +designates +39si12579283iwn11520100418173910 +remove +executed +my +party +date +69 +jalapeno +tired +12px +2010 +note +me +them +20100404 +by +flip +even +rssfeedsjmasonorg +listmasterlistsdebianorg +actually +phil +xbeenthere +0100 +plumatd +university +1st +not +listsubscribe +times +with +from +jackson +mimeversion +0500 +implement +panels +linux +address +copy +listid +charsetusascii +fetchmail590 +13497 +sender +permitted +precedence +xcmaecategory +strawman +101428120 +errorsto +boosts +almost +irish +daleenterprisecom +listpost +c +083208 +tickers +where +returning +xauthenticationwarning +fine +competition +042504 +better +million +xvirusscanned +required +really +fi +manual +c79db13a5377 +about +8219575100 +jmdeadbeefjmasonorg +see +102163776 +debianuserrequestlistsdebianorg +7454107139 +no +razor +we +sat +having +installed +182809 +group +known +testsemailattributioninreptoknownmailinglist +contenttype +xmozillakeys +required70 +xrcvirus +mentioned +other +of +and +received +855f416f69 +afont +160640 +phone +the +inputx11optionsshmconfig +died +cant +752d46e7e674d311842600a0c9ec627a03eaab3crockford +httpsecuritydebianorgpoolupdatesmainppostgresql83libpgtypes383110lenny1s390deb +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_58.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_58.txt new file mode 100644 index 00000000..212acc89 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_58.txt @@ -0,0 +1,297 @@ +oulook +panel +oldreturnpath +for +postfix +40ee616f1c +encodingutf8 +charsetiso885915 +span +autolearnno +bea +module +uswsflist1sourceforgenet +mailtodebiansecurityrequestlistsdebianorgsubjectsubscribe +clientip8219575100 +autolearnham +operating +in +cached +retrieved +coolest +vulnerabilities +have +reported +returnpath +imap +zdnetbfontbr +mailtodebianuserlistsdebianorg +are +is +mon +gmailid128c7c8fef59b72a +srivastavas +workflow +excuses +00000000 +mozilla50 +1014121327 +i9znwly2sjtv +seen +robomodnewsnicit +subject +higgins +got +qmqp +listarchive +specifications +4bbe938c2080809doboseviccom +listunsubscribe +2122188179 +yes +but +0000 +pdflatex +debian +culprit +identazhljbycnzyn5y8xd95u7kw4egh1arjjumpaustinvirciocom +different +methods +event +recent +ist +if +good +d172013a51be +problem +oct +eugenleitlorg +required53 +hotmailcom +up +btw +snowdealorg +250 +absent +all +ieyearecaayfakvsceuacgkqm0llzlt8mhza0acgnyqapvzbiw8ooi4e9gdvhb1v +size3ba +had +m8cs26431wfo +gmailid12879dcad6721d7a +libxml2utils +edt +just +hettinga +it +esmtp +protocolapplicationpgpsignature +listsdebianuserlisztdebianorg +two +t3danchordesk +gag31mes2l +a +deliveredto +hrefhttpwwwnytimescom20021001opinion01krughtmlhttpwwwnytimescom20021001opinion01krughtmla +resentsender +hicsanchordeskfrontpagebgif +reiiguanasuicidenet +localhost +m3t3t3t3twy6o3t3t3t3t3t3t3tfms +though +localhostlocaldomain +as +because +cupsclient +tpf +pdt +messageid +pay +judge +10 +isnt +hrefhttpotnoraclecomsoftwaretechwindowsodpnetcontenthtmloracle +dkimneutral +unstable +feeding +0700 +hspace3d0 +yet +wireless +advantage +resentfrom +nonfunctional +such +2007091301 +whats +surfers +monique +framework +forkadminxentcom +2008110401 +de +textplain +127001 +work +that +4bcb51201000205coxnet +to +holidays +singledrop +tlsv1 +you +web +know +025553 +utc +filter +amavisdnew +6qaaai0surbvdjlbzrnbhqxeiu9smsqcwukmnmjitkdnaaccbw2sh1fxqk3ijrtknzsolyn8dhahzj +face +hrefh2 +coreabr +version +bdell +references +mxgooglecom +173728 +at +gmailid1284122bacb475a6 +eveything +ss1024 +debianuserlistsdebianorg +xspamlevel +id +dnsfromrfcwhoisfvgtmmultioddgmailimprononcable1imprononcable2 +listhelp +on +reports +my +exhausted +081634 +date +live +bug +hope +height3d4br +me +sat24apr10 +xuidl +1014211926 +by +garymteledyncom +he +unsubscribe +81018101 +client +new +written +ldowhitelist5 +2525 +011043 +again +httpfiachraucdiegavinpublicgpg +specific +with +from +means +fun +193656 +balls +objclanguagelistsapplecom +mimeversion +postscript +pet +12so50425vws6 +inc +first +2 +nbc +linux +both +xcheckerversion +catalogs +use +words +done +xaccountkey +extra +precedence +values +s31mr707987fgj601273116262185 +developers +file +upstream +basically +an +place +bymcsqs1eb8a +avi +msgshow +fanatics +be +listpost +universe +c +mailtospamassassintalkexamplesourceforgenet +92 +get +prospect +222427 +quotedemailtextspamphrase0102tolocalparteqreal +refute +geek +were +anything +incentives +hdomainkeysignaturemimeversionreceivedreceivedinreplyto +update +30 +recommended +dynasty +xloop +xgmanenntppostinghost +httpfreshmeatnetprojectsamsynth +81168116 +manual +wife +jmdeadbeefjmasonorg +debianuserrequestlistsdebianorg +arial +420 +very +iaa13280 +153236 +mailfollowupto +ftpftpximiancompubximiangnomesuse73i386gaim05911ximian2i386rpm +score95 +some +tfont +white +policy +great +contenttype +xmozillakeys +xrcvirus +may +like +ownerworldwidewordslistservlinguistlistorg +of +ive +and +xstatus +record +received +device +074023 +vista +definitely +days +protects +sept +javadevbouncesmlsubscribertechcsminingorglistsapplecom +mattsliststribalmediacom +active +the +jehan +foo diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_59.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_59.txt new file mode 100644 index 00000000..b683c5ad --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_59.txt @@ -0,0 +1,379 @@ +srchttp1pixelhostcomimagesfoh7q9r0q7op6xtb8hxjpg +romefiumicino +hickey +sweet +secretary +recipient +tbody +killed +economy +application +5px +for +001 +hrefarticle1top +business +smtp +conditions +xbarracudaspamstatus +bsfsc0satofromaddrmatchhl +chances +757575local +08 +strongothersstrong +ownernolistsgodailyjmnetnoteinccomsmtp1admanmailcom +body +buuren +8 +name +this +in +content3dtexthtml +professional +174836 +firewall +value3dimpairedrisktbacom +lay +returnpath +are +is +solid20 +tinyurlcom +n9efntct019326 +his +nextpart000002101c2327292934220 +baltic +ittitle +xasgdebugid +borderbottom +100 +00000000 +hgeicvdahqpzlqnbedx10aesauhaz1zsrqaywwighwdvjqeaimyj6gamjwbwjqm7tjqytrw8nbad +elections +width72nbsptd +size1equal +reputation +said +subject +relay +leon +xbarracudaspamflag +using +produced +size3d5n +mailer +included +xmailmanversion +email +153313 +alignmiddle +yes +inhalt +jmjmasonorg +but +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +11 +released +0800 +sa154d +conferencep +best +contenttexthtml +would +1991 +online +irrevocable +proper +98489f6f688829eeca9a6a5aca6f1abafa2bfadbef6f68e99 +ist +good +br +size5149149149149149click +description +rdnsnone +delivered +color000066today +weer +face3darial +xbarracudaconnect +100irigir +all +head +domain +u +strongit +argwgkczzjklptlgga3tys8advbeskujoppwryuotplrybgszcf4aab5hjnaegamqkmfj3h2ae9 +size3all +styletextdecoration +look +img +y03n7n3reclgd4sepgejlkta2mahumnztlsxhh4m1ri7gomutd0znsvjaupvmgjdhsudlwflzs +0400 +xmimeole +align3dcenterbfont +edt +us +it +esmtp +size3d5strongwe +two +a +acaangauadaaaaabadhcsu0ebgxkuevhiff1ywxpdhkaaaaabwaaaaaaaqea4adkfkb2jlagsa +omega +deliveredto +manufacture +6500 +night +became +localhost +earlier +stylemarginright0pxmarginleft0pxmargintop0pxmarginbottom0px +align3dleftfont +div +sunny +mimehtmlonly +has +114641 +messageid +ppnbspp +10 +cellpadding0 +qawc7mybceid1jcgoywbdx5oqrv8uigh4iatvetejojghkloorhvommzrlciojdqypgsyn5wwaaq +text +swiftly +flemish +lower +styles +v117lnerable +lup +xbarracudaspamscore +or +federal +above +dezember +public +gallop +doctype +total +124528 +010 +they +chipstrongfontp +encoded +contenttransferencoding +dersiniz +color3dff0000models +sansserifand +nterba +borderleft +zzzzlocalhost +textplain +bsmtpd +spudleymcdudleycsminingorg +theem +m97nner +out +051 +border3d0 +sending +to +become +mailtocustomerservicerosterfinancialcom +bgcolore6ecec +separated +n8d8kr1n023178 +kated38excitecom +you +modern +httpspamgwcsminingorg8000cgibinmarkcgi +32 +ywnvldo6kvwufoc7ptkbaut7d8vzv7v0rx0xx90nuqoipaysro4f3f1rqoqfoplfwm580nd6n +width570 +charset3dwindows +wiktionary +series12550br +minutes +80 +em +bombs +alt3dpreferred +greatly +overwhelmingly +font +here +valuebfburkina +v +accountnbspin +internet +divsquad +hours +color3dcc3366b282bfontfontdiv +at +htmlimageratio02 +grangetown +id +diagnosed +bnqpc9wpg0kicagicagphagywxpz249imnlbnrlciipgzvbnqgzmfjzt0ivnc3osxpiibzaxpl +pts +rdns +wellknown +on +pressurebr +canadian +began +reports +htmlmessage +style3dfontfamily +focus +complete +how +o2g6g7lh012786 +hide +seas +sa424 +xmsmailpriority +date +spamgwcsminingorg +member +scottish +padding0px +titleunsubscribe +alignleft +2010 +nisu +i4ztboitjiiwoulwf6vymgpyyfk4waaie0t3bg1kzcph5ugj0pzn9bwoovnjeusrim5a6 +third +marginwidth10 +xuidl +12536963202d6401830000w4twrl +mail +href3dhttpwwwinsurancemailnethttpwwwinsurancemailnetaf +by +vavfzgl9qn5hoyxwm0auco3ahv71n5gigo3uzroeha4om0nnnopwvo5tdxlrlqpyc5oroeytfci +ff0000 +173115 +even +love +httpspamgwcsminingorg8000cgimodmarkcgi +new +will +missing +back +much +sa148ahl +2000 +p +qqggbjgabqn6xp9ihoc114vtihtuaxkmgacmfjrhhc6pjxpaqe6aqbuwyajtc0xtuejajvsmupx +with +from +height278 +money +carnival +mimeversion +unsubscribefontabr +gw2csminingorg +give +width141 +august +express +jryeltearthlinknet +first +cellspacing3d0 +127136475066f106f70001w4twrl +catalogs +srchttp211117193159bookhome1jpg +investment +bgcolor3ddbe8f4 +sent +words +asmtp +extra +105114 +sender +3d6767b54ba85639caff829jriguhaoe3d343473276533267699516dqlbcacdsi3dn +rec +which +transp +offer +pr0nsubject +be +backgroundcolorffcc66 +large +ntinental +stronggroupsstrong +htmlheadmeta +texthtml +rights +accountants +were +several +more +173734 +microsoft +xbarracudavirusscanned +soil +brl +iq +stylecolor +123612 +81168116 +healthcare +metal +pissing +carafa +10997y +m0620212mailcsminingorg +australia +arialstrongfont +simply +many +especiallystrong +see +no +we +sat +add +breedp +games +hibody +does +stamps +when +stylestyle +known +policy +sansserif +contenttype +bsfsc2sa154 +mailnbsp +fontb +uid60971201487268 +like +other +of +since +hadp +away +states +immerse +csminingorg +and +received +one +windows +94hqv9thjcljrngfotnrmugegqhdcc5rjlmo80sljri5pkula6cdthgvtgn6k20llvcaarv8h +raterepublik +rifles +lino +later +days +dear +table +created +bfgspeedicon2jpg +stylefontfamilyarialfontsize12pxfontweightnormalcolorfffffftextdecorationnone +titledecember +brit +the +xpriority +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_6.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_6.txt new file mode 100644 index 00000000..01b17afa --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_6.txt @@ -0,0 +1,376 @@ +purchasing +s111me +2009div +nbhib5wm010164 +used +recipient +tbody +27607spanspanbr +please +opuaxvimgpxnvvvicaremedicaldk +headline +format +textdecoration +network +iavuiqu5ij6fuqao8gxrj55laugkymfaciqtrkpazsemtv6ybl41ctofquyvgmgr9k8qichuuu +dst +smtp +xbarracudaspamstatus +membership +bulletin +aligncenterfont +xmlnshttpwwww3org1999xhtml +chrbadertelecomat +body +wasnt +gradually +parties +area +this +in +faccedilades +mortgage +benefits +design +professional +have +your +heard +returnpath +imap +suspicious +282179658 +0001 +is +mon +http3bckugitifcn7wmt247je392k0047192w723dvyqopalalohibody3dn +his +bgcolorffffff +halt +too +accept +qdgcz6d2ooocykffffazz +100 +account2 +00000000 +11px +y86uwa9ywtjea4jeyrqnrmj44wezr46t0z5rrvw35d4rdxsprjbw8rrdbslrhbbahq +reputation +circuitry +subject +hrefhttp5donlinemaje90aruhegiz7b799d2aa2797fc +nanyang +using +am +system +donations +src3dhttpiiqusimagestba072920020729102gif +n +included +winter +three +lineheight +twelve +solid +yes +debt +divoire +released +hrefhttp669fmozuwuscnigyeepuo79o069zl18c2gm5b70j852more +below +colspan3d3input +size3d2b +officerb +janissaries +elsewhere +eastern +dramatic +another +yyyylocalhostnetnoteinccom +cosmonaut +front +style +image +meter +if +center +taceemorabaq +19216818250 +adiv +540 +htmlimageonly20 +200207232211g6nmbnj49684locustmindernet +crises +face3darial +142308 +noprescription +xbarracudaconnect +normal +jesse +friend +all +head +implemented +u +rbl +geobates +size1bstarter +img +youre +strongly +xmailerpresent +receivedspf +166 +325 +site +vanessa +esmtp +efforts +issoudun +broadcasting +100000 +xay74faokoop8qaxhyerapptzf8afdabvhcxnx59rlvh8a4qiijlqbwald1faj9r +a +tz +no6q7d6nhoorodtsytblbctwpxbiti5wcvla34ki6aqelkmekgqjb0tzbsxhgvjjfdjfws7x9 +170634 +052104 +deliveredto +khare +xmozillastatus2 +ihrob3vzyw5kcybvzibhzhmglsbzb21ldghpbmcgzm9yigv2zxj5b25llcbt +hibodycsminingorg +reproduction +xbitdefenderwksspamstamp +n7id4udc007640 +040432 +pheromonesfontfont +localhost +dogmaslashnullorg +div +benefiting +30px +g48usvknwersjaaslrbrpgp1z3bklzkzuik1gvuvsirn5b1xpmuovt1vzpomfzfu1glwxtzku +nl +pdt +has +messageid +350 +rooraw +zzzzlocalhostspamassassintaintorg +201314089nhoce701dslbrasiltelecomnetbr +10 +xasgtag +cellpadding0 +mime +valuesrsurinameoption +rehberi3c2ffont3e3c2fb3e3c2ftd3e +face3darialfont +goethe +resembles +products +protests +or +doctype +agent +safe +010 +xfacebooknotify +ach +sa269hl +xbarracudaurl +contenttransferencoding +characters +stand +reply +bp8ukaidyaiyoeywuaahmmneq0t6ley9heh7pe2w6mwxnm2he01egz3207pahmjw1epxxmob4usc +importance +idcontactdetailsastrongp +textplain +mimeole +outlook +8bit +coincided +that +freebpfont +hrefhttpwwwdigitalrivercomimg +xbarracudabrltag +buy +sex +to +iranian +170548 +charsetiso88591 +lsmtp +code +cellpadding3d0 +link000080 +12105 +discriminatory +bgcolor3dffffff +nigerian +184037 +183746243ad +here +aligncenterdiv +at +1 +300 +added +id +mention +north +lemwnife9l6xcyducij6lacwciigcajygiuasaiykll8oii1atp8xgweaey1emapawxemd5inm +hrefhttp7c44holdeveryruuuymynawan7d789cbc3e18 +pts +miami +zzzzlocalhostjmasonorg +n93eaa9i027680 +none7dah +fffffftextdecoration +undiminishedang1vasfontweightunikunilfooterrowformatosunderstated3fscunhurtuppgifternaunhelpfulunnaapproximateunitsunmanageableunleashunmarkedunrecognizedunmittelbaranimauxfoughtformalmente2ecomuntitled8formidable2fchineseuntitledanlagenutiupswelledframeborderfostersverifiesfrontmanurenkfoyers +click +language +xmsmailpriority +party +date +spamgwcsminingorg +deep +big5bqfap9qxnrkg69a +h111115t +dynamicadsl943614610clientitiscaliit +income +color3dwhite5849796176976731484864674617967 +gap +xbarracudaspamreport +offering +sales +note +third +paid +suitebr +10pxsubscribefonta +xuidl +mail +by +y3gdevhi57gljjqjxztt9wtghe4rk8unwaunrhxjwnwdrbvpfs1lejzxsmrapxjp6aaylc +contentmicrosoft +pick +eff846d +regulatory +gw1csminingorg19216818250 +charset3dwindows125 +should +new +stored +few +management +trtd +0100 +titledaily +gagdgcpsixqgg0dwgsxoiy9ibenyghzygkljguijfckagaxyq3e8gumaejlackipnsmgcmqgb2g +p +zealand +addr +161312 +with +from +jackson +wood +mimeversion +them115el118e115 +forms +latinoamrica +entity +000 +mj1963 +es +t7qdphzluxlyrictgj2pde5qylai72jetxqa5agdsu1tczpp2cdj5bg6uze0vbwy4qlrpkpk3w +selectfonttdtr +axe +ef3dhttpfreesite006friendlyhostingcomintroducing +vizynqzsenabqf7kc0hccjr1rf0m222wb4nzdlaea4c9q3o1zeukbxtqy3qcc0ilzckeff7tuj +divdecentdiv +christopher +gw1csminingorg +put +words +charsetusascii +rather +httpwwwbarracudacomreputationip11724120195 +sender +samplesbafont +marketingsubject +message +src3dhttpimgwebmdcomnlwebmd +duchy +19216818251 +surprised +established +walmart +herea +ill +vysgs7pdufdawcxr8x9wln4udqjflmagageal5y29xyxxcynn8ddgjihgk56spoc4aqmuhebfqb +cc +videos +texthtml +armed +v60029005512 +yyyynetnoteinccom +pink +vhs +qvlitlhdailwrfo7ndqnj5kudoah1sp1wbypkbss2ejvec0agazaaadh2bu1vqdk4qomklnhix +guess +case +participate +microsoft +brl +w3cdtd +testsbsfsc0sa148a +html +httpequiv3dcontenttype +safer +crane +81168116 +15px +200920 +fri +spam +xoriginalto +delane4uyahoocom +recordings +http +nuvoxnet +3 +images +no +aoaffikikt +zone +djc5c3jyrdsbadhvcrup0tv7aot7vphji7uhbiijcgx1rs6hxhuyjaak5fz0ten3wo +does +013 +size3d2bnamebfonttd +targetedbr +bgcolorf8f8f8 +day +transformed +contenttype +instructed +may +like +l111wer +of +and +labour +non +received +youll +wed +free +later +hrefhttp9032jtenajixcnebiyecusul71o8jq056i995t1120777v27privacy +table +3186145 +viruses +the +georgewarren diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_60.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_60.txt new file mode 100644 index 00000000..ecee9dc5 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_60.txt @@ -0,0 +1,258 @@ +ditch +hrefhttpclickthruonlinecomclickq2eemtgiikj5wgypktq3dlpfp8n4r +10024 +pm +oldreturnpath +read +papers +for +postfix +network +find +working +configdmmultipathqlm +budgeted +body +this +busid +have +pcmcia +returnpath +mailtodebianuserlistsdebianorg +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +height18 +235528 +built +100 +artificial +influenced +server +listsoftware +intercepted +2d20 +size +2002 +bulk +email +libvte9 +quotedprintable +hesty +0000 +6621867199 +9 +root +different +would +092955 +yyyylocalhostnetnoteinccom +hrefhttpwwwcolorcombocomvisualize +systems +10873520 +problem +designed +going +956fa13a57aa +brandon +archivelatest575064 +normal +xhabeasswe4 +gnus +tue +m8cs43403wfj +httpwwwwebminknet +thanks +result +somewhere +esmtps +mailtoforkrequestxentcomsubjectunsubscribe +required40 +hearing +it +scarab23adobecom +inter +esmtp +mailtoexmhusersrequestredhatcomsubjectunsubscribe +listsdebianuserlisztdebianorg +a +utf8qmhbdke23otci0a093cbil8dtogzk0bf2yq35wsewvyet2eeknhrzorfmmgdmh +deliveredto +resentsender +palmos +andronic +localhost +163349 +as +dogmaslashnullorg +thing +080112 +spfpass +pdt +has +messageid +2eand +191901 +10 +especially +httpwwwintelligentcombrcharlie +skip +removed +3dborderleft +dpatsynnetorg +height3d6 +gnulinux +yet +or +resentfrom +contact +httpcodeclearfoundationcomsvnlistingphprepnamel7filter +they +size2in +each +4bed4c6a30303cyberspaceroadcom +2007091301 +monique +contenttransferencoding +13916532111 +crystal +service +zzzzlocalhost +textplain +depending +127001 +rovervipulnet +whatever +04408 +xoriginalnewsgroups +upon +to +hrefhttpclickthruonlinecomclickq16tcji4cfvqywiqxdblruabdxru4r +qmailldap103 +6278128237 +cache +singledrop +now +spamassassintaintorg +utc +forteanaowneryahoogroupscom +aaaaaxo1wc4ttlhje7acgg +listmanredhatcom +17kdru0001sk00 +these +1058 +mlsubscribertechcsminingorg +f20cs11462wfi +resentdate +there +defaultsctlcard +ontpagetrimbluegif +been +yyyylocalhostspamassassintaintorg +debianuserlistsdebianorg +cb687ea01 +id +someone +port +listhelp +forkexamplecom +can +on +general +tasks +sequence +designates +19812316027 +g8fhf8o22539 +height3d2tdtd +date +2010 +exmhworkers +incorrect +friendly +authenticationresults +me +them +belorussian +envelopefrom +by +exmhusersadminredhatcom +thats +xmailinglist +actually +width75 +spain +failures +2525 +xbeenthere +much +says +not +with +from +20 +reality +cnofws +q321305ali +mimeversion +enht +builtin +143955 +xcheckerversion +xmozillastatus +listid +compiletime +href3dhttpwwwtwelvehorsescommmclickthrough +make +fetchmail590 +xaccountkey +permitted +precedence +tried +developers +class +errorsto +which +control +american +unseen +be +listpost +xurl +collect +xvirusscanned +n17mr4186343mul501271254339033 +version250cvs +charsetwindows1252 +guess +xloop +simultaneously +scripting +81168116 +sgamma +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +xoriginalto +about +narayans +width100 +major +endeavorscom +does +contenttype +helpunsubscribeupdate +boys +xrcvirus +mentioned +like +devselfast +since +xstatus +received +windows +ms +obrien +speed +list +nt +1000custom +the +1976 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_61.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_61.txt new file mode 100644 index 00000000..0a1f00d2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_61.txt @@ -0,0 +1,361 @@ +eclipticcomparison +qualitiesp +tppdirx8eo2sbc9u4jxrjooltjgbf06773uvb7y9zwbfg7bz3q5fjtzjapycj3my18nk +classbodytable +1to +titledo +please +class3dbdiv +xentcom +format +for +ss +organized +smtp +genes +due +51 +body +area +name +this +shoesand +colorredyourfont +in +1971 +z3ehl6e3xaaawdaqaceqmrad8a6qu5zahy5botq6feolmagagt4o2k1t12x49pbmjklly1vlq +patrons +firewall +memberthe +off +returnpath +imap +0001 +are +is +mon +bat +xasgdebugid +ents +19 +width440 +ages +wasdiv +stars +subject +tiny +200904291833014892 +custom +xbarracudaspamflag +less +234413 +included +2002 +bonus +httpwwwenvexnet +bulk +fontsize +0c16621 +clientip1906517166 +yes +substantial +below +hrefhttpeffwalkcolonyruexyzymyea5bd86b67dc20 +bordercolorb000b0 +0530 +url +would +deaths +developmentbr +separately +students +if +eliteshe +br +19216818250 +19216831 +color0000ffcheck +own +liresumesli +happiness +valueuganda +cited +bsfsc0sa424 +our +201002041736o14hayt5027498ns2csminingorg +per +xbarracudaconnect +head +phd +tahoma +1239203795 +fishing +fx1 +florida +naturalistic +gl +had +013730 +ny +marginheight3d22022 +102urther +img +patterns +ieriiy +respal100111 +music +assembly +barbara +cellspacing0 +0400 +constitutional +bold +moody +us +it +hidden +united +sun +esmtp +lyrics +a +colorc7746c +dont +award +jibbneh +financing +sometimes +maintenance +deliveredto +featured +xmozillastatus2 +hibodycsminingorg +medievalists +sayfamfdzi +build +thing +div +rule +mimehtmlonly +training +christian +taxation +messageid +contentclass +year +10 +cellpadding0 +score279 +proprietary +0700 +7791targetid +color3d220000ff223eb4f3ccd8bcdba3ba3c2ffont3e3cfont +giveaway +xbarracudaspamscore +man +mailservisttnetnettr +or +country +yourself +600600016788 +london +uss +mo1sayyyy1r4684hotmailcom +partner +they +solomon +contenttransferencoding +wuo9fqu19pvfb068b9l62tegspdvz7xpvtorlxjrds5xydieuc4btih7kjmvtk6xic5cxrb +im +hf3efcpcvtdzxvq8zp1runtxh0gl7bi25v0d36nz7x5nzv5ddk5q8a5mxfnvacy6hk4u +brand +divided +i +sincerely +127001 +meets +fontsize10px +bsmtpd +href3dhttpnyavekepcnimg +yfobu +central +sturb +styletextalign +cannot +to +become +singledrop +now +cellpadding3d0 +you +msa +pharmacy +around +fontweight +lehi +visitor +70949 +kevin +203838 +y29sbgfwc2uiia0kicagicagym9yzgvyq29sb3i9izexmtexmsbozwlnahq9 +version +ht +at +betray +occurring +replyto +xsender +004548 +fairfield +id +286 +korea +profile +on +17 +tr +click +language +involves +discriminate +struggling +date +viagra +trading +jalapeno +2010 +everything +politically +dysfunction +floor +xprecedenceref +hrefhttpmta33annfilesfourscombp5ruxqoafizzxetictoqeoiofqtefoezozlabvanidboeoiatiofifkcltiocaoecaacecneabr +cigars +by +zzzzasonorg +2005yet +antispam +new +curious +customer +forum +endlessfontfontblockquote +while +their +p +varum +href3dhttpwwwroyalmedscommain2phprx +not +xbarracudabblip +times +with +from +zzzzspamassassintaintorg +09138txtdogmaslashnullorg +books +mimeversion +scores +border0 +1915 +000 +region +aligncenter +height20 +bgcolortrtdtable +104104 +o4gnvuou024307 +1250237994 +spanstrong +words +available +jan +otonysucal +lifebrfontbfontfont +marketingsubject +class +75px +under +which +postmarked +communities +loc +an +mediumwave +cc +discussion +attack +karachih5 +grew +c +branches +greenish +get +httpequivcontenttype +church +rgb240 +d9b8d16f03 +deserted +hermeneutics +loss +30168 +positive +repeat +case +objects +microsoft +xbarracudavirusscanned +w3cdtd +overall +html +joyful +mail1csminingorg +really +gvba7steqy8kcahbotsimidauswbe1l5e8sh2oyphowuiysbuw2qy0awrsgtkscrqaxrjipig2 +fri +mj3661 +m0620212mailcsminingorg +0900 +menseveral +tyranny +call +basis +0px +no +having +titus +atjayynxa2ioe1gknpao6dhkdciij9evbfwkjferwbjkyqbmqn0opehmidsd7slxty84ixz +mail1insuranceiqcom +known +great +bsfsc0sa148ahl +sadness +1000 +contenttype +originalviagra +pjwvcd4nciagicagidxwigfsawdupsjjzw50zxiipjxmb250igzhy2u9irzqt6lf6sigc2l6zt0i +wdiqthy2sf81lwb8p8afuuurbyoxhysf81l3wamvrwalg1jaj9rlvhwd4qiim +compare +of +since +dynamiclooking +csminingorg +401 +4tuel6ali2ifvdsriw4wgmjhstguch1vbimjkckg656g1t2vylq2vwckcn9a5z03hc7qdwm9mwk +and +isvrfbe7k1syzgxwntjyp5fjwjc2s93ayw8rxorgfmojiwqe0px06cw1hhmxcyluk4vub +hrefhttp8d5medfulton85arupeokef958b7d1a44e +1600 +received +arialcolorwhitefontstyleitalicflspanfontifont +wed +070606 +free +color000000click +wish +advantages +shipping +product +follow +discrimination +k1onopinbr +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_62.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_62.txt new file mode 100644 index 00000000..b7c4ad41 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_62.txt @@ -0,0 +1,358 @@ +alignleftnations +xasgorigsubj +elements +pnbspp +qmail +read +xmailer +for +st +001 +network +84e97109 +smtp +xbarracudaspamstatus +bodyhtml +people +noneseveral +any +666666 +website +mailwebnotenet +body +weightbr +hanage +in +content3dtexthtml +danced +transitionalen +have +your +hrefhttprvrespcoma40ebe42446order +returnpath +feeling +0001 +are +is +hpom1ox9o1a8tvelwysibjzcjpjlm26awq5dzjhp4adawb6vfywjapqmtp8avyk0o5xtnpoa +1264600872241300140000w4twrl +companies +cnil01400702 +reserve +00000000 +show +subject +custom +relay +xbarracudaspamflag +40 +size +leftmargin0 +seed +2002 +capitalfontb +email +solid +href3d22 +border +shbig505413body104 +yes +frederick +below +plurality +best +benz +g4m2ejd28811 +what +align3dlefttable +intent +lv8agmn4lvh25hgu5hytjrrsatcyrlwajbqgoyrc8bbabq1d41wbdqwwhhujfabvffllq +style +rankbelgian +ist +bwbmagkabaaaaaaxqwxsz2vtzwluzxmgukdcifbyb2zpbaaaaaaaaaaaaaaaaaaaaaaaaaaa +center +sansserifhr +happen +turbines +delivered +up +going +013935 +haven +tim +franchise +our +monitoring +content3dmicrosoft +normal +excellent +head +displaced +score152 +announced +immigration +full +he106 +track +artistic +src3dhttp632118187imagesroster200204235gif +us +it +sun +esmtp +sc +two +local +watch +entire +a +enunciated +deliveredto +happens20 +xmozillastatus2 +receive +barracuda +h2ash2 +localhost +textdecorationunderline +rule +5 +messageid +championship +bowled +nonejr +500 +10 +unknown +srchttpimg101photolavacom2009120816994bvz2iqj4jpg +flemish +width3d602 +cick +valueinvestorsinput +article +6521715966 +questions +doctype +175109 +they +following +encoded +contenttransferencoding +xbitdefenderwksspam +gerard +textplain +invoked +theh2 +127001 +kingdom +helvetica +that +out +quarantinelevel10000 +color3d22ff0000223etr +newsbrfontstrong +489ee2c427 +1947 +to +charsetiso88591 +washington +singledrop +pachacuti +code +you +tdtr +1245675436 +activestrongbr +fontweight +face +030544 +carlyle +webnotenet +most +width600 +these +boxes +hyperlinks +believe +075946 +mxgooglecom +3tax6pswz3o2jfvqx3z1owuippx22ctlyzeh1x2j1pvhspdftglp4rhjxdjalgpufzkf +targetnewimg +bytes +altview +proposal +at +through +replyto +go +id +meta +bjmwyroyouq6ozkwmszb3ingkijhegrns5ciuivxotsd6c2goc6ikjfaipaiaadta4beiaeulpc +results +pts +20000 +over +pickup +bill +listhelp +ecosystem +samebokauymailcom +fontb20 +shown +navy +ilugsocial +complete +click +riversi100e +individual +stylefontfamilyarial +xmsmailpriority +date +mail2csminingorg +spamgwcsminingorg +take +31 +unparseablerelay +width3d51font +geschichte +gordonstoun +9oiajwqnp0y5ozpnbzlhgpbhetrpwdro0ny00nwyr1x48jpv0ncbug22jlr5ssykyh48ewlmg +cellsp +024941 +hotbfontp +supposed +reservedtd +googlecom +xuidl +simple +333333 +nbsptd +height3d5 +manager +by +jennifer +think +src3dhttpiiqusimagesmandoannuity081420020814102jpg +abdurrahim +dramatically +9fde3fefef5e1e2ede99eada0b7aaabeae2edebb9a8a9a2af +paper +reentry +yellow +aaaaaaaaaaaaaaaaaaaaaaaacuxrmplf8fhxl1mhyv8agbl1mhyvcwrl6zd5xbyaaaa44xk02k +receiving +back +shopnetscape +g4r0br722433 +their +developed +trusted +not +760 +with +from +b0000877951msdmailcokr +early +neglecting +mimeversion +scores +tbodytablebodyhtml +countless20 +000 +ayhgbtfqicuxjocklkkwfn2oae1akaghmowskk6mil1xjioldhhyhldsopxw1pd3bb5kkiricms +20717896147 +killlevel10000 +aligncenter +gw1csminingorg +put +phyla +xaccountkey +equity +message +gjxdjfharu +its +ffffff +an +place +consent +160 +natural +5pxlineheight +c +shows +httpequivcontenttype +measured +colspan2 +millions +texthtml +padding +million +petcaremta89qwerty657autowealthcom +aeoliccenter +e4zt8acmrkey9hdhhsfefyf8aqjjkqmocpl9t0vqs9snxzsduj97ztthbzbu680vfbhit +111rganiza100a +smheader +form +overall +runic +between +meanings +matches +wellington +electricity +serverntsaricomjo +063734 +fiveyear +contains +bsfsc0sa271x2 +mj3661 +about +many +targetnewbr +sovereign +presses +edward +width100 +3 +0px +no +we +profiled +name3dstate +content3dmshtml +k +uid118091201487268 +textalign +4750span +p97st +bsfsc0sa119b +sansserif +contenttype +href3dmailtonospamwsyahoocomsubject3dabuse +extremely +user +may +of +airphone +csminingorg +and +154648 +received +psjodhrwoi8vd3d3lm5vam9ybgiuy24vztiwntq5ytmwmgexmjbhmtmynwqxntuuz2lmp2hqaxfp +200 +185442 +free +umiokyusebe +v286n4u852urn07savdsauihwneiapzwagp4aeuawno6ave4arbyaf38aqz0aqu4aqe0ainqavy +wish +table +height3d109 +cry +pthep +removeaspclick +alt3dclick +the +those +10pxb +kkkkacvgv2gumg9t8a2nxstenftbdnb7bwdtogeenuuuuygpy8awl9rsusfsx6igd6uh3r diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_63.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_63.txt new file mode 100644 index 00000000..58c63241 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_63.txt @@ -0,0 +1,257 @@ +particularly +allowed +aliens +down +trouble +for +postfix +network +smtp +width1img +12 +gnupg +autolearnno +chris +divdivrootdebianhomestuckey +this +lives +lta +have +divxatlistsdivxcomdivxatkriegermailshellcom +returnpath +point +is +psyche +versiontlsv1sslv3 +testsbayes002 +precision +pinning +india +show +bit +99516 +subject +try +aged +already +relay +conservatively +0c56d2d0f25 +included +2002 +xmailmanversion +email +listunsubscribe +quotedprintable +g929duf01097 +1087383 +but +11 +debian +jmlocalhost +best +score6 +line +194914 +if +oct +072809 +up +httpwwwisiloxcominfobetaisiloxcbsdhtm +eat +normal +lerlerctrorg +archives +domain +luck +re +endian +present +xtlssubjectcfistespoolespooonokiaounokiacnsmtpnokiacom +68116120 +0400 +bjust +just +required40 +325 +it +xoriginaldate +esmtp +protocolapplicationpgpsignature +listsdebianuserlisztdebianorg +0e0a7202a8 +a +dont +deliveredto +arsasha256 +xmozillastatus2 +humans +localhost +earlier +2011 +mutt1520 +050423 +pdt +has +messageid +tonight +signature +10 +phoboslabsnetnoteinccom +paranoia +errors +disktoolc +4op1ayfvwl7c +particular +contact +icielabcsbyuedu +hrefhttpwwwlockergnomecomwebcamhtml +exactly +contenttransferencoding +way +2008110401 +jones +userid +textplain +that +virtual +excitedly +httplistsapplecommailmanoptionsx11usersmlsubscribertech40csminingorg +rpmzzzlistadminfreshrpmsnet +234325 +aptget +to +wishes20 +base +fwd +license +know +17254137 +quicker +amavisdnew +151520 +123451 +billy +inreplyto +version +195544 +references +mxgooglecom +200209080338g883clw17223pcp02138704pcsreston01vacomcastnet +zarzycki +1157 +id +offset +over +listhelp +on +17 +designates +bhandheldsbbr +referencesspamphrase0102useragent +date +take +border0tr +lack +1604 +2010 +mo6541216221staembarqhsdnet +mailtorpmzzzlistfreshrpmsnet +sourceforge +me +xuidl +mail +lc8sxvcngjajibmf5lbliszt +scene +by +unsubscribe +lisztdebianorg +even +listmasterlistsdebianorg +itll +director +actually +will +missing +libraries +xbeenthere +0100 +september +autolearnfailed +not +listsubscribe +with +from +httpwwwantwoncom +3d0 +20 +11fe0d0ab7c42ae000006518e14bc89759f33b +spamassassintalklistssourceforgenet +mimeversion +651671251 +give +spamassassin +pain +xcheckerversion +pass +listid +4bde4a556050707csminingorg +app +sender +precedence +marvinhomepriv +7bit +compiled +cc +be +srce50520011112filesrhcdigitalcamerasgif +xpolicydweight +c +looked +time +xvirusscanned +logmaneorg +crossed +install +eh +slid +81168116 +start +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +stonehorse +emails +claimed +xoriginalto +simply +http +8219575100 +asking +debianuserrequestlistsdebianorg +biglangnu +made +mike +no +contenttype +xmozillakeys +may +of +e9crit +01 +and +primary +xstatus +record +received +setup +right +list +gnomies +otherwise +mailtoforkrequestxentcomsubjectsubscribe +v1410 +cnetcom2fcgi2fpremierasp3fcid3d1000015 +211100 +the +httpwwwgeocrawlercomredirsfphp3listspamassassintalk +mdt diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_64.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_64.txt new file mode 100644 index 00000000..fa8331cc --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_64.txt @@ -0,0 +1,287 @@ +kills +trouble +for +organization +postfix +tiled +ultimately +smtp +defense +works +people +x +package +any +8 +in +cdt +have +returnpath +imap +g16gtu808597 +mon +terrorists +course +gtk +testsbayes002 +brcomplete +08bc143f99 +091806 +20m +subject +pgp +already +relay +contentdisposition +n +want +posting +applications +09 +listunsubscribe +jmjmasonorg +0000 +debian +ot +jmlocalhost +1112 +would +envelopesender +bcd0e13a539c +ist +if +tax +apr +designed +going +etc +so +dossier +lilowest +do +head +opengl +archivelatest573723 +deterioration +re +weak +receivedspf +imagined +exim +outside +0400 +gps +325 +us +it +sun +xulextmonkeysphere +esmtp +lhelveticasansserifunsubscribefontapp +100000 +a +217168153160 +cnet +mailiw0f175googlecom +arsasha256 +xgnupgpublickey +213329 +google +localhost +concerns +twice +as +dogmaslashnullorg +spfpass +5 +messageid +4bcdff394000400physikblmtumuenchende +steve +m +10 +unknown +discuss2 +boundarynextpart5258836hezamvtkus +httpxentcommailmanlistinfofork +article +httpslistmanredhatcommailmanlistinfoexmhworkers +e20cs2209wfb +141732 +contact +httplistsfreshrpmsnetpipermailrpmzzzlist +they +004215 +xrcspam +each +2007091301 +122010509 +highquality +textplain +i +127001 +outlook +acceptable +listsdebianorg +to +getting +140012 +ifcp +apples +october +20080610 +you +pan20100419184004csminingorg +utc +256256 +32 +theyve +mailevergonet +array +x1120081209 +here +httpsixtendyndnsorgblog +mlsubscribertechcsminingorg +hrefbooksnew +142200001012602017spawnse7enorg +resentdate +inreplyto +version +references +height72 +clobbering +draw +been +fsy6ivqicdycwx0kbz98kmiddncppaf7rwx1ca8cs38d72uch6odgkktextq8 +id +209sfnet +governmet +httpscriptingnewsuserlandcombackissues20020924when35956pm +port +perished +supporting +designates +highest +my +language +sshing +date +take +easy +35x +wondering +authenticationresults +me +googlecom +palm +mail +selection +by +screen +nowrapfont +listmasterlinuxie +comment +dsblorgerr0 +new +ldowhitelist5 +fa3b262286ca1bdb4 +back +0100 +autolearnfailed +specific +not +listsubscribe +environment +4bd2b9388050402hardwarefreakcom +with +from +0 +4bcc973590108physikblmtumuenchende +mimeversion +dbacars +enht +conkeror +content +easily +give +rpmzzzlistfreshrpmsnet +spamassassin +2 +turn +use +listid +put +fetchmail590 +xkb +bgcolor3d999999 +sender +stephen +yetanotherorg +its +errorsto +prematurely +err +msg +say +tnonsensefrom1020useragentuseragententourage +taggedabove10000 +cc +be +listpost +g7slxet06253 +shows +considered +get +fine +mailtoforkrequestxentcomsubjecthelp +6416122236 +httpwwwthesentimentalistcom +height30nbsptd +tratare +xloop +utf8q3vr9szxa5tvz2dbl2dyyrfv6k76a93l6g0a098gbpej2tkfikpap8d93f6v +uaa01425 +21319015625 +thursday +81168116 +basic +manual +delio +confused +20020823005705d63aec44eargotech +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +evaluation +10143348 +3 +made +no +very +add +sfnet +itc4wxyrfswqcmqfalkh +some +openpgp +digital +masquerades +policy +version325 +recentlyselected +conspiracy +contenttype +xreturnpath +other +of +sum +and +width120 +received +today +free +list +daveg65comcastnet +table +systemoutprintlnoutdata1i +memorial +the +172164831 +httpgoodiesxfceorgprojectsapplicationsxfce4dictspeedreader +cds diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_65.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_65.txt new file mode 100644 index 00000000..6e439b5d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_65.txt @@ -0,0 +1,329 @@ +rate +recipient +8rj5kagl6fafctsscl8qgzmksccbayzuelulnkdjq23eru5e0czn3mliejimrcgiog9loaqzmggd +xasgorigsubj +health +en118ir111nment +format +for +network +84e97109 +dependable +smtp +conditions +plateau +25000 +relative +people +racun +series +51 +sa154 +body +rk1pnnoh3e85ku77f9ckpgkxzztmq3pmlixzhtkpck40zwyjgrxgwyvxckiszdfgdkvsew2eu +name +this +fragmentation +hereap +310coming +transitionalen +your +returnpath +imap +weiss +breakdown +are +is +mon +episodes +educated +ck3dffffff +1172000 +kwwarrenpacbellnet +60029002180 +23 +elections +influenced +subject +billions +xbarracudaspamflag +mt +size3d22122 +brin +walked +email +yes +throughout +jmccarthynorthstarcounselorscom +standard +land +9 +best +contenttexthtml +tenant +facetahoma +si +hundreds +n4j5hqgb011103 +if +center +19216818250 +uniwersytetu +bsfsc0sa392f +mx1csminingorg +face3darial +width47 +gtbrfontfontfont +do +head +unparseable +minutebrvideofonta +singular +rbl +width3d60td +instate +cassation +host +transue +img +easyspan +andbr +us +it +massachusetts +islamic +hampshire +esmtp +connectives +ca +score071 +a +applied +passed +800am +technology22 +target3dblankhttp +deliveredto +office +mean +xmozillastatus2 +lmgb7ekry3vjibr8uiyhhoqgsw3akjfci90fbfzcsqtgzxrddygm1dfauasmdahm1adyoot +ay1arrpgag7qcct3uqzbfcftgapra0eqvbh4atlianduauyxqup3qxsxbpcyge2mbcjwakpwnh1 +barracuda +233b +localhost +resistance +fontabrbr +as +santana +div +hhsabjgginvmmcqagja3pkwxqxovssacdibnxad4xwdq1afvzagaegelpuglfufq3gdbqhredw5 +messageid +350 +10 +colorcc0000who +cellpadding0 +mime +1273350607quadrating +yours +performing +apologize +russia +k5cs11867ibd +care +or +listing +doctype +total +48 +010 +style3dfontfamilyverdanafontsize9pxcolor595959paddingtop +peruser +each +english +contenttransferencoding +quarter +age +size3d2td +214838 +world20trade +textplain +127001 +misdeptglobalservenet +lounge +that +frederic +cannot +buy +to +8319121253 +222032 +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +6321bdebt +00text3dffffffthtext3dfffffflink3dfffffftb3dffffffth3d333333td3d +111nclick +minutes +sign +ltfontabtd +pg0kica8ynidqogidx0ywjszsb3awr0ad0injawiibib3jkzxi9ijaiignl +here +these +documents +ugnie3d0771841692226026811click20 +lrhsik3gh6gpznqatokbk00nr9aqdyetmzzsc4nnrtdakt0o4z1puatavu0hdiw9aqscotnifjg +hours +191849 +legal +logo +at +discovered +replyto +bgcolordbe8f4 +added +go +id +2005 +align3dcenterptrptd +3366ccthanknbspyouspanfontbfont20 +1oz +on +25 +tr +alignright +government +luisconchawebboliviacom +facebook +worth +date +width548 +color3d0000cc +threadindex +31 +departments +pleasurelibrliincreased +2010 +xbarracudaspamreport +liinternational +xuidl +order +simple +final +149nbsp64quot +by +rates +then +think +antispam +helps +050 +few +trtd +5917523051 +while +p +network15913423778wendelleircomnet +6919246 +not +backgroundcolorffffffcannot +with +from +mimeversion +soviet +70 +express +south +true +exclusive +srchttps2redeyeemailcom80storestoregraph54509597042gif +killlevel10000 +aligncenter +xmozillastatus +could +constitutionstrong +discipline +use +done +brpuu2ns4ftjajgzznzjhynjiwdscgdiacycrktbrco6juimssjs8eneznzjficuox5adaojeak +xaccountkey +ricooption +header +josephus +113117111tas +ad +anyone +its +an +xbarracudascore +ill +aug +be +attack +5pxlineheight +texthtml +18 +always +better +gothic +were +045820 +ztgcu8jnutng4mtnlm1qa6kjgsoeuqidoljv99n930vevuhucrhlzmpsgrtjzrg1vgczsg8 +hrefmailtodspruytenburgodovafuycom +html +size3d1 +stylemarginbottom +httpequiv3dcontenttype +c95642c493 +tofontdiv +81168116 +bagdadbailiffsmat3receiveallcom +decided +none +protection +200920 +ef5dd169d6 +current +161859 +about +0900 +promise +aligncenternbspp +call +arial +no +top +major +dakota +performancebrbr +reputationquotdivtd +305 +95nbspdreamweaver +some +does +0600 +barracudaheaderfp20 +sansserif +contenttype +xmozillakeys +d1lroanoldyun02tnyzsqqam6y6psf5igvncr43zzzzflhltfaymlyl2y3baijafmcnhq4pssst4i +listed +feb +of +dynamiclooking +csminingorg +and +2009td +htmlshortlinkimg32 +received +without +sansserifb +myanmathadin +days +wish +shapepoly +b97115ed +the +died +blood +yytorma9r8hbamo5hwaayll8tio8lqmwafugxa7vidtokuhgsuq0ye0g0lv0im98ckvkhnjsrri8 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_66.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_66.txt new file mode 100644 index 00000000..40ed13c1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_66.txt @@ -0,0 +1,295 @@ +particularly +m8cs13574wfo +oldreturnpath +xmailer +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +dav39law15hotmailcom +smtp +june +question +libmonocairo10cil +admin +review +body +resentmessageid +biter +account5 +superceded +this +quicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +operating +women +have +your +returnpath +imap +mailtodebianuserlistsdebianorg +are +is +mon +jets +bgcolorffffff +testsbayes002 +00000000 +width3d600 +mailtoilugrequestlinuxiesubjectunsubscribe +subject +try +independent +using +shouldnt +produced +contentdisposition +listarchive +xmailmanversion +bulk +070039 +6614513691 +forecast +email +listunsubscribe +124750 +15 +but +jmlocalhost +materials +url +businesses +domainkeysignature +visible +tab +would +remember +defaultspcmipckey +image +ist +project +102022161 +problem +big +human +maingmaneorg +3d5e90771090105divaie +so +melissa +brandon +rowspan2img +implemented +httpwwwlinuxiemailmanlistinfoilug +borderright +enabled +required40 +left +it +esmtp +failed +subscription +dont +deliveredto +xmozillastatus2 +resentsender +localhost +as +dogmaslashnullorg +solinuxbr +well +pdt +5 +messageid +gecko20090706 +citizen +name3dstarmore +mgraphicsanchordeskfrontpageheader2newsgif +500 +10 +0700 +sven +man +discuss2 +httpxentcommailmanlistinfofork +or +above +75red21396214staticiprimatdenet +resentfrom +edition +divdiv +svanstromcomt +133343 +xrcspam +nextpart1495220v7augafcxf +problems +2007091301 +far +contenttransferencoding +postipphtvfi +service +centres +textplain +127001 +8bit +files +that +out +profiling +rpmzzzlistadminfreshrpmsnet +httpwwwblognewsnetworkcommembers0000016 +to +shot +singledrop +code +20080610 +ilugadminlinuxie +know +issued +bank +forteanaowneryahoogroupscom +gnomiesabr +michelle1 +publish +brbrin +making +mlsubscribertechcsminingorg +resentdate +inreplyto +httpwwwdebianorgvolatile +there +found +nothing +yiddish +at +id +results +wrap +ownership +also +xcomplaintsto +listhelp +driver +gmailid127e542720db2734 +can +on +analysis +procmailrc +calculate +g8t81hg03428 +appears +zeroes +date +friends +l +authenticationresults +googlecom +mail +things +robust +by +then +ipad +ldowhitelist +rssfeedsjmasonorg +listmasterlinuxie +xmailinglist +library +will +4ykpizwu8rji4pxcxr4xhz4iihmdsdl2dnhbctd5gzoaiqxtbfghsrh80tz82orgalfsfc +kmail +xbeenthere +much +september +anyway +software +listsubscribe +g8a3c3c14737 +with +from +rohit +wood +gnome +hrefhttpclickthruonlinecomclickqecmhyxqvy13zxvhpz1koqfb2gvagyr +money +mimeversion +tested +spamassassin +iso88591qcamalef3n +include +longest +8024132206ucnombresttdes +004518 +use +kundenserver +listid +answer +jmasonorg +rather +app +sender +precedence +message +errorsto +happening +score7 +7bit +taggedabove10000 +listpost +711918151 +quick +plans +relay3applecom +6416122236 +10875328 +stdcout +latest +version250cvs +title +7 +wanted +mount +af6decorecpp209 +spamassassintalk +gpg +j18mr2281654ibh881273523127798 +81168116 +043909 +really +071942 +decided +approximated +fri +demands +about +8219575100 +64e1a13a5aef +debianuserrequestlistsdebianorg +no +sat +archive +atkins +rumsfield +major +amounting +when +policy +version325 +jmrpmjmasonorg +contenttype +xmozillakeys +of +and +xstatus +record +received +width33 +portability +right +free +list +days +phone +table +created +rootlocalhost +xinitrc +the +altalso diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_67.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_67.txt new file mode 100644 index 00000000..bb2b823d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_67.txt @@ -0,0 +1,294 @@ +131bjmpnjfbi +decibels +size1tell +0300 +register +xentcom +buildings +for +postfix +cwgexmhdeepeddycom +working +gnupg +people +xmailscanner +any +network159134147129pc897as1galway1eircomnet +httpradioweblogscom0108189 +clientip8219575100 +21318539113 +in +your +expressed +returnpath +imap +g6o21k413499 +mon +combat +pan20100429200303csminingorg +stupidity +x8664 +100 +market +subject +teacher +xenigmailversion +212405 +2002 +want +wr +forth +email +explain +is20 +sure +0000 +1002627 +degrees +unwanted +drives +future +couples +another +forkxentcom +grave +if +running +tag +required53 +gforce +so +ontpageauthors144gif +dan +thu +madduckdo +lairxentcom +httpwwwlinuxiemailmanlistinfoilug +ptdtd +kde434 +re +campaign +onto +ipaq +anybody +esmtps +mailtoforkrequestxentcomsubjectunsubscribe +just +required40 +us +it +hrefhttpclickthruonlinecomclickqc1piljqpw7b4rlth3wiopyolipujlr +bt +united +height600 +sun +esmtp +902416 +listsdebianuserlisztdebianorg +managementvoipwireless +marginwidth0 +a +123337 +deliveredto +xmozillastatus2 +resentsender +create +bergsten +localhost +2011 +voice +spfpass +g15e72316923 +messageid +offended +pay +helveti +500 +deck +cellpadding0 +fraudulent +testsfvgtmmultiodd +0700 +whose +initramfs +care +xericconspiracy +or +4bed20469000006coxnet +21613683173 +daemonlocalhost +such +httplistsfreshrpmsnetpipermailrpmzzzlist +they +0200 +2007091301 +rs232 +contenttransferencoding +utilities +achievement +etcaptaptconf +textplain +i +127001 +reject +urgency +ps +shore +19412514545 +that +usrlocallib +to +xface +charsetiso88591 +dvd +130823 +20080610 +you +utc +amavisdnew +2283 +imitate +slrnpre099111 +incredibly +uplevel +httpwwwtributecamoviespec4990htmnewsletter3dnumber4820 +resentdate +version +41140153155 +references +there +mxgooglecom +6 +vertical +taken +at +1 +debianuserlistsdebianorg +id +checksum +listhelp +holds +can +067000000000000004 +on +17 +being +designates +yyyylocalhostexamplecom +172541311 +date +2010 +mailtorpmzzzlistrequestfreshrpmsnetsubjecthelp +authenticationresults +me +logical +googlecom +xuidl +simple +hrefhttpwwworaclecomoracleworldimg +federalists +by +either +unsubscribe +lisztdebianorg +even +listmasterlistsdebianorg +hate +should +xmailinglist +ldowhitelist5 +mutt14i +0100 +again +autolearnfailed +wont +v8mr1669955wfp1621273016400842 +not +with +from +bgcolorfffffffont +mailtodebiankderequestlistsdebianorgsubjecthelp +153151 +money +mimeversion +mouths +spamassassin +media +xcheckerversion +pass +mailunilogicnl +use +listid +make +v11 +xaccountkey +regards +sender +precedence +kernels +07 +tip +095151 +errorsto +basically +pda +place +7bit +taggedabove10000 +aug +be +listpost +xpolicydweight +mailout07serveribbs +where +jammie +preprocessor +geek +bits +1362441161 +fieldengineercsminingorg +xloop +smith +type +mailing +mark +card +automatically +154385b590894055b1ee88d470f19ac4infethzch +monitors +about +remodeling +mlbwireless +8219575100 +httpthinkgeekcomsf +envy +call +no +add +literally +top +011924 +httplistsapplecommailmanoptionsquicktimeapimlsubscribertech40csminingorg +classm1font +creators +contenttype +college +of +since +ive +mozilla +01 +and +16f4 +record +received +one +posted +days +javadevbouncesmlsubscribertechcsminingorglistsapplecom +1087113 +height3d8 +207615143 +faqanbsp +the +volunteers diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_68.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_68.txt new file mode 100644 index 00000000..150c7aee --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_68.txt @@ -0,0 +1,371 @@ +litter +sweet +e +providing +xasgorigsubj +wanting +trouble +rial +for +postfix +services +smtp +height3d151 +diriyhola +any +ahrcwaaykfq8fmpmjfyeipt9een5t9xj8xj0hrcpqdmeml2qlvzfzlcsxvoievrvjq8hc0wxa +sa154 +wars +across +court +mzde9znucini62ixljvw6xpbez9paa1937zctvi6n0mzatzmcnfcggucxkuauzltc7 +name +125320234972fe00030000w4twrl +this +ba +in +margin +height3d20 +o11lr7nz000092 +c9mgpt7y3ke2ir5facnnxn0xumqbondyzwoqeen6wb2wielqnjqi0pkiqzoth8ddaubdsqubr4 +twitterptd +classstyle12job +naa8ok22007689 +transitionalen +have +your +returnpath +normalfontvariant +imap +h5version +are +is +charsetgb2312 +wait +xasgdebugid +filesfontbfont +energizers +arsenal +account2 +11px +reputation +scattered +fam +subject +4 +fontbfont +using +addressb +color336699bunsubscribebfonta +prior +mxrootsystemsnet +000047c264ab00004c8200000e53mailcbsch +three +base64 +email +akyanmoayx0svonoetrvuqsysuffxxajwqbl9nv8eyfxzncbcumpjabt2petgrcsgktbirqlztg +quotedprintable +href3dhttpwwwinsurancemailnethttpwwwinsurancemailnetabr +87109210138 +but +goj5xa70aopo6t9f6uopxniorfxlax1ffiekaanfzhiuf8aevjj6ap5gtcepri8s8gp +width3d50td +cannabismarijuana +methods +intent +another +yyyylocalhostnetnoteinccom +atographic +style +ballal +commentary +21039350 +if +tolerance +neither +up +htmlimageonly20 +our +b0000426157 +excellent +mgh5c9swdkcqohwkbxnyc7wkc55pdwhgcisrcphyhnrpka6rctmcdh0antegnsuom5dicar13om +all +contentselling +pquebec +celim05ltod5q5wr2qwtllyet1qkm5uoekglz8rglqa0e5uydxpxhjq9bphm8gqfhgavlosu46 +hit +390 +versionfontanbspnbspnbspnbspa +valuemtmaltaoption +bluestuck +xmailerpresent +full +clean +bold +artistic +advertisement +width97a +it +united +sun +nappade +esmtp +httpwwwinsuranceiqcomoptout +oaa12179 +893893 +two +wisely +watch +a +residences +deliveredto +gf6nderiyoruz2e2e2e +httpslistssourceforgenetlistslistinfospamassassinsightings +70px +domains +became +ab +though +085133 +as +build +width3d589 +colspan3d2br +spfpass +store +pdt +has +messageid +industry +500 +10 +xasgtag +subjectexcessbase64 +mime +eify +council +text +or +spite +ireland +cpunkshqpronsnet +indicate +contact +hrefhttpwwwqksrvnetclick10955485684912 +others +48 +west +they +todaybfonttdtrtrtd +uriblphsurbl +81128112 +contenttransferencoding +193120211219 +induction +glise +size3d2td +textplain +mimeole +i +127001 +girls +helvetica +specials +citation +quarantinelevel10000 +pspmetgfuacfznbgaafopm0ualrrrqazoyakkafzrmkooawikooawikooawijngaafoztakahzoz +offaly +buy +to +brbtodays +singledrop +19216832 +cellpadding3d0 +you +td +exchange +id3dmasthead +stores +object115 +webnotenet +most +252 +documents +border3d22022 +long +valuejojordanoption +bytes +width3d51nbsptd +relationship +gdzo3y59n86vyvzvyqxwrwf9fo2dg0cxspnhga69onyjr60yyzztwvy4g3kkhtykjdjjgj1 +at +replyto +seriously +points +zywo +id +authorities +mandarklabsnetnoteinccom +largest +poetry +opipy6jijbwupyxdgskkw2d5khv0ukcmom5nrdiubsybtte3nr8ywmyfnzlixrpbamdq6scl +reports +populations +201630 +planned +color000080 +date +wastemindernet +xbarracudastarttime +31 +appreciate +width657 +alignleft +size2bfont +tired +c1b272e801f +xbarracudaspamreport +offering +xuidl +critically +envelopefrom +buttonville +by +zzzzasonorg +visit +he +laps +called +0100 +again +p +chargea +georgia +not +chapungu +times +with +from +bsfsc0sa392d +220541 +sansserifa +mimeversion +0500 +todaya +tdimg +methodpost +killlevel10000 +054 +aligncenter +smtpeasydnscom +xmozillastatus +address +make +sent +minimize +category +fetchmail590 +182125 +xaccountkey +countrys +hrefhttp6cdbwayiguvcn612392123cy7oxu8kj0w9uw0seumaibiobhibodytusywunetujafxuipojinysakiludaj +message +whole +stephen +speaks +monthfontdiv +host847224849conversentnet +19354215253 +fully +bwhyb +7bit +machoworld +be +into +where +get +expectations +rdnsdynamic +hypothesized +h +nigeria +texthtml +rules +possess +were +101 +fonttd +ranging +title +httpezewifuigaacom804e15d46238de64d9ac07850993aec +profits +xbarracudavirusscanned +dynasty +brl +ibrahim +href3dhttpwwwfairlaneannuitycomimg +moravianlyceumcom +ormaltextdecorationnonecolor333333 +fbi +bordercolor000000 +3px +html +httpequiv3dcontenttype +k1onopin +between +81168116 +border3d0tr +size1fundamentals +wqbgmuarhitguylloq7eewqdgfusmcoguyis20a5tnnibjsbkd5tjghvurtzuoehhpva21ndqmcb +current +gebeurde8bngeboortedatumwellington2fchannelgfgappropriatenesswnnggttffkelly +territories +started +133511 +3 +arial +no +barracudaheaderfp3043 +deputy +bne +major +soxifadi +vidula +uaa27006 +uid52341201487268 +jst +hey +1253111761 +vegetation +bsfsc0satofromaddrmatch +contenttype +xmozillakeys +66925374 +america +like +inquiries +v55049202300 +designs +fontp +align3dcenterbuy +received +ms +free +173842 +o +qualité +size3d7brfont +helod +went +leave +417 +xgmailreceived +bgcolor3d666666 +search +the +send +xpriority +boards +cod +width3d550 +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_69.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_69.txt new file mode 100644 index 00000000..605c8446 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_69.txt @@ -0,0 +1,262 @@ +123803 +used +color3d888888br +read +reserved +for +postfix +find +smtp +profileabbrbrfont +feel +argc +people +due +any +g4si4640448fac120100508094412 +linuxuser +clientip8219575100 +this +operating +in +returnpath +point +is +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +101431608 +testsldosubscriberldowhitelist +too +multipartsigned +ee87d2d0dc0 +132115 +rowspan +moderator +121908 +subject +qmqp +using +bulk +nameserver +report98557 +quotedprintable +184124 +jmjmasonorg +111948 +15 +httplistsdebianorg20100504173725ga22897columbusrrcom +e221713a542b +0000 +debian +root +jmlocalhost +best +would +yyyylocalhostnetnoteinccom +ist +if +unsubscription +good +required53 +anchordeskbrfont +going +xapparentlyto +war +so +b975413a4a48 +must +t +do +200209101902027528f4a2matthiashaasebennewitzcom +lairxentcom +thanks +161038 +murphywrongword202 +grallabfonttd +laugh +3lj2qfvutsxwlermen4kxnjtuyganr1katcpkym9pqbegke80scby6wfxjuunuwvz2 +xmimeole +anybody +spamphrase1321 +track +it +prof +sun +a16mr2550190fac791274527526739 +boundarynextpart3240376t7vxzdoi9q +esmtp +listsdebianuserlisztdebianorg +watch +212605 +6603 +a +newer +deliveredto +mean +wrote +localhost +nua +2011 +subscribe +as +inline +spfpass +store +pdt +has +messageid +m8cs29805wfo +helveti +finally +yours +0700 +ian +or +g7sannz19961 +resentfrom +main +0200 +2008110401 +textplain +i +127001 +that +digitally +robert +to +satellite +completely +809122912 +problematic +81138111 +66187233211 +singledrop +speaking +hawkwind +you +tuned +43045237949 +utc +hat +returned +movies +mlsubscribertechcsminingorg +ikifi +resentdate +mxgooglecom +change +connectedattached +uc +something +hash +gmailid128bb50055819797 +1 +xp +xspamlevel +id +xcomplaintsto +can +on +designates +yyyylocalhostexamplecom +decidedly +how +my +g8j20cc31838 +zzzzlocalhostnetnoteinccom +appears +date +yeah +width3d430 +uploaded +them +house +by +even +intels +however +rssfeedsjmasonorg +ago +xmailinglist +helps +written +much +not +listsubscribe +with +from +rohit +executing +valentuathaorg +mimeversion +0500 +next +first +ignorant +linux +theory +xmozillastatus +jmasonorg +fetchmail590 +151340 +available +xaccountkey +automating +dull +sender +permitted +precedence +101428120 +which +taggedabove10000 +nowrap +src3dhttpwwwapplecomlibraryquicktimescriptsacquicktimejs +forgedyahoorcvd2297 +be +listpost +longer +mailtospamassassintalkexamplesourceforgenet +get +xauthenticationwarning +mailtoforkrequestxentcomsubjecthelp +better +enough +xloop +informative +lia +utf8q3vr9szxa5tvz2dbl2dyyrfv6k76a93l6g0a098gbpej2tkfikpap8d93f6v +html +joe +paddingleft +7309344157 +05142010 +09cb013a6079 +royal +stable +no +very +add +archive +ajatuksia +requests +125214 +theres +contenttype +wmmrcsdirrrunsubscribeafonttdtd +helpunsubscribeupdate +bash +might +record +received +right +wed +list +beyond +table +access +210056 +droid +search +the +cant +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_7.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_7.txt new file mode 100644 index 00000000..f385cc08 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_7.txt @@ -0,0 +1,282 @@ +httpindiagoblogspotcom +175406 +euphoria +wow +gmailid128a5891d3cfae01 +application +oldreturnpath +125322 +for +postfix +database +valuedetailquote +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +children +any +020033 +clientip8219575100 +this +synaptic +in +dlpxppyioyce +buuedxawwexl7q5ksx0krwkhutww9upeeihfkmsjqrexrm71zi71bd7i88x6deure2tzoxiudghlvenqbrrcwbgca22t8wt7i8bk2ygra3geqkiv7p4bsjyybg0m +returnpath +mailtodebianuserlistsdebianorg +guardian +are +m542mp1cvx2cnthdialntlinet +113840 +knows +testsldosubscriberldowhitelist +too +seen +subject +try +using +voices +lurking +bulk +df71916f16 +inreplytouseragent +debianlinuxhomenl +disk +1087383 +but +atlauoxdrhduznavbljjes7ua4 +gw +0000 +20so1896001vws6 +httpwwwanzidesigncommt +debian +jmlocalhost +url +205747 +urgentrugacbe +would +hyperpublish +event +owned +enhanced +image +javatm +line +if +good +openpgpmime +newslettertitlehead +excreta +must +do +opengl +lairxentcom +testsdateinpast0612fourla +asap +warmer +re +documentation +powermtatm +page +every +0400 +before +portfolioabbr +track +edt +pudgeperlorg +8vbxj5tztvhaohgf1eylbliszt +required40 +vyur1j3cbqa6z32zci2b4q5xherenzxzfkjprgburxcvaffjxyf8hf4x8ffjxyf8hf4x8 +it +esmtp +striped +completion +convenient +dont +deliveredto +xmozillastatus2 +devastated +c906a7fc48 +localhost +bandwidth +05 +razorusersadminlistssourceforgenet +c5 +dogmaslashnullorg +messageid +m +10 +isnt +changes +unstable +phoboslabsnetnoteinccom +0700 +resentfrom +mailtojavadevrequestlistsapplecomsubjectunsubscribe +groups +th3d37 +2007091301 +contenttransferencoding +way +im +gary +127001 +uidgid +that +to +v1 +singledrop +now +you +ilugadminlinuxie +n33grpscdyahoocom +fork +200209261532g8qfwlg25210dogmaslashnullorg +103113 +did +company +amavisdnew +auth02nlegwnnet +single +chainofcommand +xllwfw6e5dq0lq97kpctvae34fj1v55e66y +was +resentdate +inreplyto +hrefhttpwwwlockergnomecomrecommendhtmltell +vs +at +replyto +been +debianuserlistsdebianorg +go +202891285 +xspamlevel +id +largest +rfc +condition +http3adamien2edouxchamps2enetieee1394cameras +xcomplaintsto +port +listhelp +need +can +on +cleaned +installation +yyyylocalhostexamplecom +102299674 +notice +check +my +individual +date +httpastraycomperl6now +gap +2010 +mailtorpmzzzlistfreshrpmsnet +googlecom +opening +055608 +xuidl +resolution +things +by +gmailid128889745b86e5a5 +unsubscribe +1712811331 +however +comment +xmailinglist +0100 +again +september +listsubscribe +with +from +07132001 +mimeversion +enht +next +spamassassin +t1272354010 +vm7090204 +1296591128 +life +linux +ext4xfs +xcheckerversion +pass +james +choice +copy +use +1222 +clamavrelated +listid +51510 +fetchmail590 +regards +sender +permitted +precedence +occasionally +anyone +211614 +pull +imagemagick +say +an +7bit +cc +be +street +nations +mailtodebiankderequestlistsdebianorgsubjectsubscribe +geek +fieldengineercsminingorg +215 +headericsminingorg +install +login +spamassassintalk +close +protection +executives +10143348 +8219575100 +made +no +very +writes +dhersaaes256sha +gnus513 +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +policy +version325 +hurricane +contenttype +hrefhttpclickthruonlinecomclickq4bkl7ibuwwfh8aswp9va35c54jmcr +user +fontb +required70 +months +123623 +may +of +integrity +and +record +received +waxed +href3dhttpclickthruonlinecomclickq3def04uyqmkhvfv1bkqbxdv +right +list +152037 +6445128110 +rootlocalhost +98106251222 +the +memory +172164831 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_70.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_70.txt new file mode 100644 index 00000000..a7bff55e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_70.txt @@ -0,0 +1,287 @@ +471782 +pm +for +postfix +experimental +2139621475 +mailtodebianuserrequestlistsdebianorgsubjecthelp +ulises +221954 +egwnnet +gail +any +operations +resentmessageid +clientip8219575100 +account5 +in +brdivdivblockquotedivbri +have +needed +your +returnpath +jul +iluglinuxie +are +is +mon +19 +too +multipartsigned +spamassassindevel +testsbayes002 +v17cs15893wfj +hrefhttpwwwlockergnomecomissuesdaily20020712htmlhttpwwwlockergnomecomissuesdaily20020712htmla +62 +prices +irix +subject +extended +emailspecific +am +4bd9ce8690009hardwarefreakcom +xmailmanversion +bulk +micalgpgpsha1 +want +himself +socialism +email +listunsubscribe +jmjmasonorg +but +0000 +wrong +url +what +would +remember +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +setting +forkxentcom +style +ist +world +gldudebianuser2mgmaneorg +document +required53 +clicked +thu +bgcolor3dccccccimg +do +httpslistmanspamassassintaintorgmailmanprivateexmhworkers +all +attacks +dga +correct +re +curtail +025 +61 +0400 +esmtps +nutzso +ba70647c68 +xoriginaldate +esmtp +httpxentcompipermailfork +henningsen +a +movie +deliveredto +xmozillastatus2 +resentsender +provides +href3dhttp +wrote +unfortunet +localhost +httpusclickyahoocompt6ybbnxieaamvfiaa7gsolbtm +bridges +though +within +permissions +dogmaslashnullorg +acuitive +pdt +messageid +year +040031 +10 +tony +a0samba +tmp +ae3130lw +0700 +additional +contact +groups +they +contenttransferencoding +framework +0910 +postmasterryanairie +textplain +i +127001 +cards +volumes +that +java +frankly +pump +dalai +object +listsdebianorg +aptget +to +httpwwwlorenzobettiniit +20100517092244gh8272thinkhomelan +getting +66187233211 +issue +singledrop +apes +searchnetworkinged20d7b9a49e402cliststechtargetcom +utc +bank +did +6trb8vl5aid8a1miai8uqywnx4qidpejknfomfxinthr2oxt2y8pnaq1tp7iizh0zy +amavisdnew +returned +practicespost +v17cs53106wfj +mlsubscribertechcsminingorg +inreplyto +20090614 +references +height3d349 +mile +microsofts +hash +experience +at +1 +mta1grpscdyahoocom +yyyylocalhostspamassassintaintorg +go +id +gmailid128c02385d81c2d3 +can +on +bouncedebianispmlsubscribertechcsminingorglistsdebianorg +same +designates +questa +14 +classurl +date +tired +customizable +2010 +l +googlecom +xuidl +justin +by +us20bbr +obviously +function +asmtp028maccom +comment +xmailinglist +via +mutt14i +2525 +xbeenthere +193258 +their +anyway +151328 +not +listsubscribe +with +from +rohit +0 +tmobile +thru +cnofws +mimeversion +v59sm446632wec1520100426045557 +turn +033659 +pass +details +mode +listid +make +steam +1oevqd00049o0q +sender +permitted +precedence +skqaxylrpdga6wbyxgvlbliszt +kernels +thus +an +vulnerable +aug +cc +listpost +shows +large +relevant +release +fontspan +siren +time +db5wmvvxcmcxannsloq +xvirusscanned +part +pear +192168110 +more +update +100k +175653 +xloop +mailing +81168116 +upgrade +manual +httpwwwintrinziqorgasm +xoriginalto +about +8219575100 +fantastically +tell +very +intmx1corpspamassassintaintorg +archive +1ocqln0004du3t +corporation +hrefhttpwwwmatroxcommgastartnewsletterjul2002reefstorycfm +does +objclanguagebouncesmlsubscribertechcsminingorglistsapplecom +policy +version325 +including +contenttype +cuda1bluetone +required70 +may +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +of +and +084933 +received +without +wed +list +phobos +active +224141 +7414e1bc92aa3 +capacity +freshrpms diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_71.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_71.txt new file mode 100644 index 00000000..11044c92 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_71.txt @@ -0,0 +1,358 @@ +width384 +sfszp6o37zxtdxztkwg7wobp7ppe9rvorkurvop6lk1umfucdeshhe13sxsoz2nkoa5hfuy +alt3dthis +for +vspace2 +smtp +span +igzvcibob3cgbxvjacb5b3ugz2v0iq0kdqpodhrwoi8vnjyumjmxljezmy4y +facearialfontptd +painters +jakarta +ownernolistsgodailyjmnetnoteinccomsmtp1admanmailcom +industrialists +body +expense +0staackkgxxadijewa9sxfbqvzu4hhjl6b1e2ladbafraccpfrkvos1w71wpqdbrssv99sxaclj +this +in +content3dtexthtml +printing +kgn8rrfl0wedoe37kxzbnu78x6mvrz5mkvcqwp3otqt5sxktoxrv1mr5upv7orpijx6mn8awq90l +aligncenterspan +transitionalen +styleborder1px +your +returnpath +0001 +breakdown +are +lol +xasgdebugid +1nhajifbkpgfppo8kpalzxstfaxpgvgfppo8kpalzrwamf8au0fsh2yvzakaue +bgcolorffffff +account2 +213105180140 +examples +subject +higgins +4 +relay +using +listarchive +width579 +than +lowest +guide +tg545e +heads +bsfsc5mj1963 +jmjmasonorg +state +spantd +11 +heavylift +tipconnectde +quota +jmlocalhost +beastiemcwaolcom +043913 +fontbrbfont +what +koi8rbig1hdhrlcib3agvyzsb5b3ugyxjllia +intersection +080 +another +recent +bgcolor3dcccccc +flags +if +good +detect +apr +rdnsnone +mx1csminingorg +designed +delivered +bottom +process +our +expected +115ee +valuebfontbr +xbarracudaconnect +internetbr +head +citizenship +httpwwwlinuxiemailmanlistinfoilug +wit96eciscom +8757 +deliverydate +yrurimg +hit +000000 +host +1px +prada +birth +height19 +xmailerpresent +color3d000000cityfontbtd +anybody +huia +99 +esmtp +reading +a +dont +deliveredto +hibodycsminingorg +congress +receive +localhost +within +serif +as +rule +bgcolor3deeeeee +idformsradiobutton4 +well +messageid +350 +worn +e11si4648635fga820100426084457 +rasbenefitonecojp +10 +wstm8zq6kzovf0n1 +mime +especially +castles +aligncenterb +tda +xbarracudaspamscore +1846 +account4 +550 +surrender +federal +marty +supplier +computer +uninformed +total +48 +010 +value15000000150000option +taglevel10000 +xbarracudaurl +contenttransferencoding +hm +hrefhttp59145azubohqpcnfont +fontof +citizens +width3d75b1670btd +127001 +8bit +increased +helvetica +hrefhttpwwwgodigitalstudioscomwwwcolorhttpwwwgodigitalstudioscomwwwcolora +that +garezzo +httpnrwaydrugruaf2cc94e38e54178736e169958ca5d2b5 +numbers +to +v1 +sep +singledrop +lookoutfonta +79111165172 +code +xbarracudaenvelopefrom +you +41100 +httpspamgwcsminingorg8000cgibinmarkcgi +size3d4brfontptdtrtbodytablecenterdiv +bank +ðìµîorganizer +fontweight +size3d3 +phones +solution +font +these +pharmacyfontfont +version +value1010option +24th +something +15strong +at +through +replicawatch +id +meta +engineer +conceived +rdns +ratio +paperworkfontbp +2040405 +ancient +hotelbr +transferred +tr +government +notice +my +after +inform +ulli +date +instant +viagra +103508 +take +xbarracudastarttime +engaged +sense +bronze +income +align3dright +width374 +013145 +googlecom +xuidl +proprietors +9338145197ip71fastwebnetit +mail +simple +igrgougbbt48sjeaagckaqckgcaddaxwaielumxgxu8uyiig7jydwieaosgsglwfmknoghcgigq +by +60024620000 +he +bhoffmannetniconet +050 +xbeenthere +lw8lfdbfqjeavaeacrgorpbssr9uyliabgdhslgvtnil7hsbsejj4cil7rmlfujgawy4aplruab +while +8px +size2this +trusted +remnant +with +from +crossdisciplinary +qgec1udfdj8ox2q2eruqxnkuavtaoeqikyrjavbk1gokqiqlzbakcsyq5uyqxjjtsbc0aprg +bar +digibelbe +mimeversion +0500 +border0 +splintering +calabria +soviet +tracijmalwaredomainscom +orders +enjoy +penispill +komiteta +lot +ivkovic +51df516071 +gg9xtgb6vfihbkmoxbxjbjiqau8zjccjrvyyuy5sxzvyuyzqtyaqeoofxbgovpsqbag2ucco1s +stylepadding10px +restricted +irresponsible +words +fetchmail590 +message +tip +itfontfontblockquote +flavor +album +control +bass +developments +established +an +src3d +7bit +sxlc1jwzrorhggdqxl5y2kdt8epriv8ezbctv17i8ksbniuidrquxzqurdmzh5zzlabgycsjhr +juneau +be +attack +stylemargin +src3dcid003a01c1e705a111de00147ba8c0xg395local +123641 +supported +httpequivcontenttype +church +1967 +rdnsdynamic +tcvet31217a4afc65tecumsehzwpl +rules +theodore +ls0tls0tls0tls0tls0tdqpzb3ugyxjlihjly2vpdmluzyb0aglzig1lc3nhz2ugymvjyxvz +time +kind +more +microsoft +xbarracudavirusscanned +w3cdtd +r +mailing +paddingtop +html +valignbottom +mail1csminingorg +882 +thinking +baseball +uriblabsurbl +xoriginalto +styleborder0px +unicef +105 +nearby +089 +game +3198870 +see +mace +arial +thin +no +sat +tahoma11white +retaining +important +economies +013 +o368xcil006291 +jst +contenttype +xmozillakeys +settlements +underlined +may +like +listed +hill +won +dynamiclooking +pfxwfyppngrl8rxtkjk9kjidqssetplyc7pfflelp8aokabwb5f8a4il4ur4s6cem +bis +csminingorg +and +888888 +received +043935 +corporationshe +ever +table +hst15961telelanaslt +sa392f2 +decorappropriate +sufferers +shipping +ag1iyahoocom diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_72.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_72.txt new file mode 100644 index 00000000..fe5d5460 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_72.txt @@ -0,0 +1,268 @@ +wwwtwittercommkbanemcr +lsorensecsclubuwaterlooca +e +booted +oldreturnpath +xmailer +for +postfix +icq +smtp +hrefhttpclickthruonlinecomclickqb7fogpq8jho5rfloy0npbnptbdl3er +usabr +post +bhcbvv1nevrxkiowtv4tfph6sclhne5p4nsse5h9uuea +package +0930 +resentmessageid +clientip8219575100 +account5 +this +content3dtexthtml +size2bmonitorsbfontbr +have +learned +luca +returnpath +jul +mailtodebianuserlistsdebianorg +is +mon +oh +listssecurityfocuscom +mailtospamassassintalkrequestexamplesourceforgenetsubjecthelp +172541338 +built +x8664 +x2g9b3004971004241956kdd1e6ab5j6bc962f52f9ddc6fmailcsminingorg +subject +melbwirelesslist +q7quowhquzjx +private +buddies +xoperatingsystem +included +enpdf +email +4bc6378a5050700coxnet +listunsubscribe +but +dsblorgskip0 +sure +0000 +best +actionhttpwwwlockergnomecomcgibinhtsearch +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +g713rr218337 +online +sdk20050327 +br +compliant +happen +oct +commodore +12921512853 +up +2002fon +yyyyuseperlspamassassintaintorg +do +rest +lairxentcom +domain +015 +strike +xegroupsreturn +run +look +patterns +107 +irxyylrbqvyqlkngphapazctckeox4tdpa90j2gxaibh51nmarwdmfzo3zohq4ct +ba2a92940ae +dcsminingorg +drifted +required40 +esmtp +hmm +confusing +a +dont +fujitsu +moving +deliveredto +hrefhttpclickthruonlinecomclickqd3gzhkqjpayblbx02k74p01k7fy3rr +leak +xmozillastatus2 +provides +localhost +bad +as +because +5 +messageid +score69 +500 +10 +compixelturesystemhookssystemhooks +0700 +paranoia +xspamstatus +flannel +rm +or +uswsflist2sourceforgenet +groups +they +each +required50 +way +last +avoids +importance +textplain +127001 +that +ha +out +httplistsdebianorg4bef1c0f5030902acuedu +154652 +hits10259 +to +monday +singledrop +you +web +utc +bank +amavisdnew +most +httpwwwmediaunspuncomsubscribehtml +mlsubscribertechcsminingorg +was +mailtoforkexamplecom +exmhworkersredhatcom +something +block +joseph +1309496247 +at +1 +debianuserlistsdebianorg +iso88591q0a09bjfritjgt9elq7hgvhv2rta7r5xnlttjbtq4kskf6esmferfopfz +id +offers +rfc +bill +srchttpwwwtechrepubliccomimagesadsgodaddyban160x600domainsgirl3gif +kelli +on +httpsitescooperorg +25 +space +2212f13a4c28 +date +httpsexamplesourceforgenetlistslistinforazorusers +permit +razoruserslistssourceforgenet +googlecom +xuidl +gutermanmediaunspunimakenewsnet +by +unsubscribe +cleverness +pool7019164141boseastverizonnet +httpwwwgeocrawlercomredirsfphp3listrazorusers +rssfeedsjmasonorg +201224410520090324 +gmexim +will +ldowhitelist5 +back +0100 +xamavisstatus +re5mhcsjymsipzbuvwtrs3cajjcgn5fp1n42cuy0rvvtshghesmw0dkplowz6gsjvx +autolearnfailed +score110 +with +from +0 +mimeversion +stepped +enht +es +life +xcheckerversion +xmozillastatus +bring +cyrus +listid +charsetusascii +fetchmail590 +xaccountkey +permitted +precedence +tetexbin +message +hrefhttpclickthruonlinecomclickq25inepibdxxmza5pdztscr7jeqrfpr +file +errorsto +under +which +newfontabspanfontfontdiv +an +mrct6rb +notes +v13si2686537fah6520100507110016 +be +listpost +alan +into +yahoo +steveburtcursorsystemcom +time +whitehead +julkista +xloop +81168116 +start +jetzt +81348131 +about +8219575100 +mistake +debianuserrequestlistsdebianorg +204 +no +kfjcorg +archive +sfnet +console +mailtoexmhworkersrequestredhatcomsubjectunsubscribe +intermail +corporation +when +group +version325 +including +policydefault +contenttype +xrcvirus +like +isapiwc9c0acd0654b14be7db6aa58cd16mailinphtumde +of +9ae9b13a5152 +and +209123207194 +received +setup +windows +httpxptsourceforgenettechdocs +wed +list +o +couldnt +phobos +1280 +the +4f3d313a6137 +cant +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_73.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_73.txt new file mode 100644 index 00000000..1aa0603c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_73.txt @@ -0,0 +1,266 @@ +able +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +miklblitzyandexru +ldosubscriberldowhitelist +works +xsync +4bc9b04f6030805hardwarefreakcom +chris +clientip8219575100 +this +name3dnptrail211112 +fontbr +autolearnham +in +ldowhitelistmissingheaders +listmanexamplecom +players +vadkan +amazon +returnpath +jul +etxx11xorgconf +is +mon +versiontlsv1sslv3 +improvement +f35e016f03 +bhlfzfgzkfjoeipy1meyrcbthim21pmdd8f0vqkevqyc +calling +subject +relay +vvaflpdf8twqaeb46jmgugkpld081dcicd9ve2y4v7lrbt5zmnzk6bgu8qcdknu +hmtl +system +than +469 +2002 +xmailmanversion +092208 +ezmlm +increasing +listunsubscribe +subie +0000 +released +security +uninitialized +durham +what +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +yyyylocalhostnetnoteinccom +applicationpgpsignature +accucast +center +world +br +apr +neither +xapparentlyto +happiness +so +102311 +26 +compiler +domain +tolistmasterlistsdebianorglistmasterlistsdebianorgabr +apu3mmk4u60onr9hjtcya9 +re +cest +teh +page +httpsexamplesourceforgenetlistslistinfospamassassintalk +0632350812 +size9 +j46mr655719wek11272524176033 +httpsearchsecuritycomr0461900htmfreesecuritywebcast +just +required40 +it +xoriginaldate +esmtp +howd +cable +betcha +170349 +a +deliveredto +xmozillastatus2 +nutshell +resentsender +wrote +localhost +endorsing +11992 +localhostlocaldomain +as +well +bgcolor3d000000img +pdt +messageid +pcs +10 +debianlive +0700 +appropriately +or +resentfrom +rick +03 +come +fizzle +iran +contenttransferencoding +open +borderleft +smtpmailwowwaycom +textplain +mimeole +i +lorganisation +127001 +lw15fdlaw15hotmailmsncom +g7mg8hqi004189 +pint +listsdebianorg +to +charsetiso88591 +singledrop +you +eventual +269704 +httpwwwgeekychicknetblog +web +utc +184659 +here +inreplyto +there +mxgooglecom +596 +at +neverfail +eguinnessandbudweisergif +rces20 +id +slapping +engineer +width3d4 +inpmrfci01 +someone +quimbo +da39eg0o9ekh +pitfalls +listhelp +can +on +yyyylocalhostexamplecom +60 +how +my +date +082claws +live +servicesfonta +gecko20020523 +084015 +me +xuidl +envelopefrom +things +size1font +by +lisztdebianorg +function +4e90613a5123 +much +0100 +xamavisstatus +little +their +software +specific +listsubscribe +exmhworkerslistmanredhatcom +dell +times +with +from +personals +cnofws +liszt +mimeversion +enht +significant +cb01 +iqaawubpamxl8pxh8jf3ohaeqjovwcgwljdfcrlc15ohgtxy7vvrl5ian0ia +include +assumptions +uid +copy +use +listid +charsetusascii +available +xaccountkey +regards +sender +bdf3ktgztdrsdeg3xgzsn5ls4pfrzqavrjyyl0l8r8jsedjmaduy3jyggl9zwijed +precedence +errorsto +an +taggedabove10000 +discussion +be +listpost +reads +youd +where +get +aqmnnbrlxiyemzyz2qiaaaabbf6eea24e22ae3ac6fcb8ec6f4224a5d +time +better +ronljohnsoncoxnet +part +q2t880dece01005080831zae8e27a7z35e6c132e9f0c273mailcsminingorg +bulgarian +fxuel5ir +smith +224302 +87fx36gsiwfsfcsminingorg +81168116 +truth +bt48 +xoriginalto +example +linuximage26334hvw +10143348 +8219575100 +call +debianuserrequestlistsdebianorg +ff +sfnet +dhersaaes256sha +maillocalhost +component +2f27313a5735 +policy +srchttpartvcomcnet1dinlftgif +version325 +testsemailattributioninreptoknownmailinglist +contenttype +and +11005 +record +received +testsldosubscriberpgpsignature +red +list +couldnt +mailtoforkrequestxentcomsubjectsubscribe +resulting +155217 +211100 +calibrate diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_74.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_74.txt new file mode 100644 index 00000000..4e3c87b4 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_74.txt @@ -0,0 +1,275 @@ +testsawlknownmailinglisttmsgidgoodexchange +used +namemode +pm +trouble +qmail +number +xmailer +for +postfix +dot +smtp +x +body +plant +resentmessageid +8 +aptitude +account5 +this +debianorg +in +wsletter48newsletter3dnumber48 +ddrbabrfont +based +crook +your +returnpath +imap +are +communication +httpgmaneorg +20020828 +00000000 +122625 +everyone +said +subject +soon +using +rcvstore +xscannedby +2002 +xmailmanversion +bulk +email +listunsubscribe +hrefhttpwwwlockergnomecomchathtmlchat +david +mailenable +x1120100328 +0000 +httpamericanwasteblogspotcom +jmlocalhost +055438 +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +online +reconstruction +ist +if +fourla01 +good +explanations +running +assigning +up +communist +so +2456416f17 +ergonomists +keep +all +tue +marginwidth3d0 +domain +thanks +re +look +4110 +receivedspf +set +before +edt +just +it +esmtp +httpbidocom +httpxentcompipermailfork +protocolapplicationpgpsignature +20100520231833779tehpehgmxnet +extremist +101005639034 +subscription +a +deliveredto +localhost +though +within +141205 +as +dogmaslashnullorg +thing +spfpass +messageid +year +10 +siju +signed +perfect +universitya +corba +gnulinux +or +submit +crashed +they +2007091301 +whats +httpwwwasacomputerscomcgibinindexfcgactiondisplayscreentemplateid25 +open +importance +textplain +i +127001 +ipv62001053aa64c201b6c5dafbbaad1 +writ +that +nope +mailinapplecom +href3dhttpummail4unitedmediacom80click +rpmzzzlistadminfreshrpmsnet +listsdebianorg +to +completely +gkmpettingzoonet +getting +issue +singledrop +base +speaking +xlevel +libsoup241 +know +utc +tdtr +home +bxum1g7dsymw +these +minimised +help +version +references +response +at +1 +converse +id +difficult +154301 +doesnt +port +on +2041532441 +25 +my +zzzzlocalhostnetnoteinccom +error +date +friends +g +contracted +2010 +oneincomelivinggroupsmsncom +authenticationresults +me +googlecom +hrefhttpclickthruonlinecomclickqebhlpoqkdtzhbv2pedo6mzbunk5xrr +by +tracking +he +lisztdebianorg +ximian +silence +browsers +width75 +xmailerxemacs +120609 +2525 +back +while +university +062003 +agenda +not +listsubscribe +with +from +brown +debianlenny +enht +spamassassin +first +life +xcheckerversion +address +acceptedbr +details +listid +3156 +jmasonorg +sent +charsetutf8 +xaccountkey +keyvalue +sender +permitted +message +errorsto +which +camaleãn +irish +40rogerscom +httpcharliez1004blogspotcom +an +axiombraindustcom +herea +emacs +be +attack +10223127212 +into +4bd375513070109coxnet +get +mailtoforkrequestxentcomsubjecthelp +texthtml +hrefhttpwwwrhinocomfunlisteningparties78184partyplayermgi2prhiqtimg +wifi +better +bits +293644 +cycles +pgpsignatureratwaregeckobuild +headericsminingorg +20100516094359726newsdebianjetcityorg +xauditid +start +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +29 +hrefhttpwwwgnometomescomtome000626htmlcool +debianuserrequestlistsdebianorg +no +153948 +poised +0vtq9rm8ngu0osocsm10adj4gzdb80f7ksrjiqn8qkznptid1kvdcaoiyh4rwdnsi7e +contenttype +090012 +xrcvirus +8441 +baffled +like +of +003811 +settings +and +fff +received +testsldosubscriberpgpsignature +one +bugzilla +quietly +list +the +never +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_75.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_75.txt new file mode 100644 index 00000000..f60da9bf --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_75.txt @@ -0,0 +1,354 @@ +harding +dominican +funny +recipient +xasgorigsubj +235551 +please +trouble +h2someh2 +5px +format +hurry +for +postfix +face3dverdanalower +154824 +network +banned +bsfsc0satofromaddrmatchhl +june +060259 +post +further +08 +size3d2equivalent +brhere +xmlnshttpwwww3org1999xhtml +body +operations +name +in +have +letterspacing +your +auto +returnpath +17s9is0003c000 +mon +ciertos +xasgdebugid +width3d5tdtr +require +genre +among +dam +00000000 +effort +bit +subject +soon +custom +hostname +headtitletitleheadbody +using +produced +dave +titlea +recognition +included +lowest +want +fontsize +httpwwww3orgtrhtml4loosedtd +tambi3dneill +state +land +below +advisement +9 +s0usranmkemz6lt1mcbgu11izzzzreirwckukn8c9jjjlkruk7o2pcyplwfmlksisyas95tqnr +charles +ed +contenttexthtml +similar +015557 +different +divui +hundreds +ns2csminingorg +ffffffunsubscribea +stylewidth795px +ist +if +rwqymkc7b3ja4ahpc1ia7waqxfrtongabx04ga5akcsheoadhgkcf7jwhmnwiquozeetcmecrzas +stated +br +compliant +description +neither +designed +aciganda3dwoody +process +ixtkx8ipj6ydejmvj5foo9itlpnx637gl6df7rfuczfcnq5tf48akuadk6h3n +nop +httpwwwbannersgomlmcom +our +jobspartfull +higher +xbarracudaconnect +normal +recibir +ffffcc +newsletter +do +tue +head +fit +lairxentcom +advancementto +score152 +hrefhttp8ec7dforlupccneterisoem4md9u7n228g48641699gvaawivueny744727200065868172312 +users +page +set +cellspacing0 +dedramarkuse5khotmailcom +styletextdecorationnoneclick +stylefontfamily +else +left +it +sa290rn +esmtp +color0000ffclick +xcyn3onx5nhlx8tp6dl9ly3f6rzxzf8zhvftqrrhogejiqioypobppxw6bcsr5be0i1kkusm73 +commerce +term +entire +a +nameused +information +localhost +owners +nbb8xpg3u9tgktzuzv7bgfbpfvbx5zn7mrcu0fyjbmb61zppeihyurcumbdid0hcuoudpvqv5z +uscdfebkewkpshqrubvdtolawzp91185102047abtsmpdynamic24420168122airtelbroadbandin +build +because +spfpass +well +5 +has +messageid +procedure +222331 +kn111wn +settle +10 +xasgtag +cellpadding0 +john +t0lxm0qye1vwqhjz3p6n81ekmtq9ol4frd7uadzwyehlx0yd2xumjvjipo4uaa5wcz5p0mix5lk +removed +appealing +tda +topmargin3d6 +xbarracudaspamscore +popular +verh111117100ing +looking +valueqaqataroption +4csrnjzuw18bkmokmvinzorabqb1jt4xm665pzcwl8cbgias7oem7xj4c2qrajqubwvfywydia +taglevel10000 +56dc01ca70a7462f9d83bf399734actionfororphansorguk +xbarracudaurl +contenttransferencoding +xbitdefenderwksspam +addressafont +trained +collective +127001 +committee +outlook +out +styletextalign +mysterious +height15 +quarantinelevel10000 +size3d2high +xmimeautoconverted +httpwwwinsuranceiqcomlegalhtm +licence +to +shot +miss +charsetiso88591 +singledrop +base +19216832 +code +xbarracudaenvelopefrom +you +reverse +pfrbqkxfienftextuefdsu5hptugq0vmtfbbrerjtkc9nsbcr0npte9spsij +32 +purely +hrefhttp9814nugofdlcn4084c05ky5u21988zd69yxajyfhibodyovihizezucuseiazolemezumahyzyxox +tracks +hrefhttpwwwfromyou2comherbalobpenisindexhtmlfont +these +italijanske +v +therebr +relationships +was +mxgooglecom +updatetime +6223e3cfont +size3d5canadiannbspfont +replyto +b1000000bfontfont +low +id +mail2bizlandincnet +surrendered +pts +cpunkswastemindernet +churches +can +son +jb +14 +zuckmail +date +mail2csminingorg +bfontbr +blockquote +ofbrstimulating +grades +77187822tmitelenormobilno +h5campuses +mail +by +zzzzasonorg +unsubscribe +stylebackgroundcoloreff0f0 +will +link +size1nbspnbspnbsp +0100 +p +alignrightof +zealand +developed +bsfsc5sa161f +not +ignores +with +from +gave +proof +mimeversion +rory007460171i01edtnmailcom +gw2csminingorg +zsgwufwkscmmcw4viimz8fchaqkshllaifkwaraeizskisbl4uykivkkavakimyiaq1qapjxac +hemostaticas +000 +alone +g6phxluw032217 +include +cellspacing3d0 +killlevel10000 +ccenter +centerbodyhtml +xmozillastatus +stylepadding10px +value30000000300000option +gw1csminingorg +make +reset +architectural +faceverdana +propose +backgroundcolor +dc10716aae +xaccountkey +ad +slobodana +193359 +brcentera +19216818251 +belowbrbr +src3dhttpiiqusimagesflynn2002090399gif +ffffff +htmltagexisttbody +herea +7bit +19216833 +booster +attack +boom +c +large +linuxlocal +texthtml +rules +always +time +minimum +brl +r +score2441 +sa392f +81168116 +color3d000000brfont +dnermlswpbcom +paddingleft +onths +ial +save +0900 +xrovingid +erick +468 +penisenlargement +made +images +no +we +illegal +sat +major +fjgcwxnaxqdfzqgdhy8jawiwcrednbbr4ugxswhyphopljulfty8lmvhnae8zsy9xrhpapm9mywq +xacceptablelanguages +games +introduced +sansserif +contenttype +xmozillakeys +learning +unequaled +of +dynamiclooking +csminingorg +and +received +11995br +one +suite +srchttpwwwpicshackeuasdimages70ydamirysupoxugif +hereafont +wish +earth +table +re102lect97nce115 +trash +cry +view +ywnha3rpci4gu2l6asbiaxigc3vyzsbzaw5pcnnpeibzbxnkzw4gbwfocnvtigv0dglnaw1p +sell +psychiatry +the +owning +parts +advertising diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_76.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_76.txt new file mode 100644 index 00000000..4f62c1ba --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_76.txt @@ -0,0 +1,251 @@ +serious +43ac144155 +looks +cs100 +toobad +namesignatureasc +read +xmailer +httpwwwactivestatecomproductsactivepython +for +postfix +drive +mailtodebianuserrequestlistsdebianorgsubjecthelp +101426910 +medications +encodingutf8 +find +smtp +jujhr0prqwelitdvaxodpi5usnvywhhz1jdy +post +package +decisions +srchttpclickthruonlinecomclickq58mr3xn9jfoqdh9wbj8bbi9gcmrrr +autolearnham +in +have +off +returnpath +imap +mailtodebianuserlistsdebianorg +is +apparently +ehiogu +source +domestic +server +det +subject +1001 +labelled +3d8img +listarchive +scientific +jmhall +bulk +fortnight +configasyncraid6test +netrix +yes +but +0000 +debian +jmlocalhost +security +unable +hunter +ist +up +so +gletterpost +do +chartb +xhabeasswe1 +huddleston +sponsor +saveat +price +awpnlvfgteeia10 +stat +somewhere +just +esmtp +protocolapplicationpgpsignature +xinjectedviagmane +893893 +listsdebianuserlisztdebianorg +a +usemap3dclubcar +deliveredto +venusphpwebhostingcom +xmozillastatus2 +resentsender +thrown +localhost +inca +dogmaslashnullorg +9bbv1e0015qhlui3pbbvvb +height15font +store +kmail1124 +has +messageid +showed +10 +cellpadding0 +john +0700 +192168123179 +resentfrom +public +24050 +assignedto +101434317 +0x015a4e35 +holder +pgpsignature5 +2008110401 +1931201713 +open +i +127001 +assumption +that +listsapplecom +digitally +proving +src3dhttpwwwcomicscomcomicsdilbertimages +to +sep +issue +edpzn4lo11gatedatbofhit +xlevel +dynnjablskip0 +utc +bank +totally +probably +005629 +version +references +126 +at +th +debianuserlistsdebianorg +id +inherited +doesnt +130051 +modules +also +xcomplaintsto +listhelp +need +can +profile +mailtoexmhworkersspamassassintaintorg +ion20 +designates +yyyylocalhostexamplecom +date +contortions +signatureshortdensetfromhasalphas +httpsexamplesourceforgenetlistslistinforazorusers +jalapeno +unavoidable +hrefhttpclickthruonlinecomclickq43col5ibjael8grm9z5vkc3p0bcr +me +googlecom +satisfied +mail +waxworks +by +spokesman +visit +200207220049uaa11347www22ureachcom +pcmsurround50 +width3d3most +will +42 +back +0100 +again +their +anyway +wont +not +listsubscribe +with +from +pidgindata +1637 +cdaletechmonkeysnet +mimeversion +inc +2 +linux +listid +answer +fetchmail590 +available +sender +permitted +xcmaecategory +clientip172541336 +original +absolute +taggedabove10000 +aug +listpost +500000 +longer +mailtodebiankderequestlistsdebianorgsubjectsubscribe +feinstein +where +matter +a1 +peoples +xvirusscanned +215 +room +30 +mailing +182625 +1o0icn0005hy7m +81168116 +start +pumpkin +sway4rnook3ovucqybdpc6hvqku4bmwfpbdedztypfms2bamisctm79lqdushlr +manual +cvsing +simply +10143348 +except +customers +randomshuffle +hassles +archive +important +bord +mailtoexmhworkersrequestredhatcomsubjectunsubscribe +revert +reasons +known +policy +day +contenttype +classtitlenbspgnomepluginptdtrtablep +1022071133 +d8aae6c845f +achieve +censeo +and +received +18si5133119fks520100424162724 +httpwwwdebianorgsecurityfaq +list +libldlsbso3 +phone +the +d +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_77.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_77.txt new file mode 100644 index 00000000..c9a4579f --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_77.txt @@ -0,0 +1,335 @@ +h97t +providing +tbody +explore +203743 +valuesubscribe +for +postfix +kutjhurq +services +xbarracudaspamstatus +span +post +size1name +width3d95 +people +arsasha1 +siz +wbnwyzjemjv5bvg423noywnzu4lccnr5e7eh9xiojleel7pjhvusysg4yeqp0rjq2wcynb0zl4x +defeated +body +name +ip +in +content3dtexthtml +exyltitle +have +avisitedcolorffcc00 +lioperator +returnpath +bearer +pndc +32226834 +breakdown +his +bergens +xasgdebugid +informational +feature +100 +subject +4 +custom +collection +450 +xbarracudaspamflag +system +facebooks +gate +size2b +presents +2002 +want +tg545e +stylefontfamilyarialwidth617pxborder1px +rural +nation +sirmadam +inbr +282167767 +11 +below +9 +best +2010div +201536 +what +matters +visible +supermarket +divtaken +hereafontfontb +pa +center +19216831 +bsfsc0sa392f +emin +rdnsnone +human +zakelnpn1bppu2yc1geofrpvv8atn6mwt3ca5xufcn8pufakmwnygtzywiiikdauuuuaep +250 +must +head +name3dhowmanybedrooms +bgcolor3d003333 +strongnation +ieriiy +bordercolordark000000 +20012018917 +hrefhttpuxfh86dalzibevcnuxjzorqcvepjogxucyrehjce34ea8d080dylilfitqg +0400 +rockets +modalities +xmimeole +bold +src3dcid64 +it +esmtp +a +dnflbzhfws4bcgxbf7d6xuhxwtr5uh03vu7xxxktwkwbbnywtlfuig9tjq4em57xe9 +xmozillastatus2 +jmnetnoteinccom +described +wrote +localhost +prince +dental +as +div +meanwhile +credit +messageid +pfont +contentclass +zzzzlocalhostspamassassintaintorg +3188254 +licensed +10 +represents +2xxfqc8piv6mkr8ovwytudrlvo3t1zxgduljjzibfo9rp3qljqtzmxmvm3k5p6oct5zyaqos9f8 +laserwriter +0700 +6619093114 +or +article +ipc +questions +fats +3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d +such +hrefhttp26cpnocoguycnjsqzakjrik148ab9e7c5ffcada7bdfsjpecegjj02566684739976864689npwfgkvgamsbpqxttftpxpepdmsqkz +ordered +width3d540center +151350 +textplain +127001 +oz +8bit +avoid +helvetica +2947zyw5389dcc32162892839804ubngrfgvgifqmxt50218538213 +out +border3d0 +82qeiracopkahraa70qypomok9aakcumqv67majcctbkgijaqw7v9pe4yxsd1xaedreqgwb3sp +quarantinelevel10000 +to +half +issue +charsetiso88591 +met +19216832 +competence +now +code +cellpadding3d0 +xbarracudaenvelopefrom +web +utc +3758 +win3dwindowopenhttpwwwfreecelebhardcorecom10391 +did +reservedptd +reformation +most +v +alt3dwhat +indiana +week +change +evangelical +taken +draait +at +valueny +size3d2 +depends +semper +go +low +id +uid61401201487268 +largest +rdns +rfc +112610 +color3d000000 +on +general +17 +boost +asked +same +htmlmessage +discover +tr +pinch +wolfgang +b +net9370174151custdslvodafoneit +2e2ca6d +date +spamgwcsminingorg +viagra +g +color3dff00 +thedrickerectru +valuable +peace +brsneeky +src3dhttp632118187imagestba20020509102cgif +by +split +cribed +bordercolor3d000000 +official +httpspamgwcsminingorg8000cgimodmarkcgi +prostate +helo +new +will +style3dcolor +charsetwindows1250 +customer +bsfsc5mj3661 +261908501996294421526javamailrovadminrovweb003 +illinois +their +p +g6u7g3q13343 +f97cebook +not +bordertop +stopping +bsfsc0sa148a +with +from +zzzzspamassassintaintorg +baselworld +hemispheres +061351 +ofcenter +mimeversion +scores +gw2csminingorg +href3dhttpwwwwidxixidcnsjmewpyzowibiyn3d873472 +000 +ushanka +killlevel10000 +aligncenter +use +make +jody +propose +width547 +permitted +113117111tas +07 +control +almost +emagazine +an +gaa07722 +106ust +testsbsfsc0sa275bhl +communication20 +score273 +httpwwwbarracudacomreputationip18759211103 +rules +part +kept +c111nc111rra +more +rolexb +cellspacing24 +case +0f4773trtd +crossed +microsoft +w3cdtd +r +religion +mount +form +instilling +valignmiddle +html +shootouts +httpequiv3dcontenttype +1247316904097b00060000w4twrl +81168116 +width3d7 +none +width586 +beebf9d6fa2 +idautonumber2 +spam +compulsory +lines +see +word +images +212016498 +no +700 +speech +does +srchttpwwwritminikaru10gif +jst +including +contenttype +of +positron +csminingorg +and +054714 +fontp +122230 +received +zouvimoyul +19216818120 +today +beginners +laserjet +list +holding +table +style3d22margintop3a +200910140953n9e9rdga028827ns1csminingorg +view +width500 +bodyfont +level +the +parts +helomailmailboxhuby diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_78.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_78.txt new file mode 100644 index 00000000..e109a3b4 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_78.txt @@ -0,0 +1,361 @@ +nbsp +used +114105 +003a01c1e705a111de00147ba8c0xg395local +launch +tbody +facearialnbspnbspnbspnbspnbspfontfont +goodrich +number +for +74ad583cd271389jqcax3d42867426835178141255exhjdznb3dxldcmxtqrrktjy +tdtrtable +business +losing +smtp +xbarracudaspamstatus +span +relative +f64law4hottestmalecom +formatflowed +any +bsfsc0sa424sbl +gold +8 +area +score896 +hrefhttppromedinfocomimg +in +hr +113uiz +transitionalen +your +returnpath +once +breakdown +are +is +allies +learn +treats +principals +xasgdebugid +lennar +account2 +00000000 +sa016 +divinorumbrfont +213105180140 +pricesfontip +backgroundhttpa725gakamainet772510950001wwwomahasteakscomgifsemailfill00ad00giftrtdtable +calling +safely +teases +233731 +server +2006among +classnormalcharbufont +subject +4 +already +value3dnhnhoption +using +system +monuments +lowest +tdfont +modification +solid +textdecorationnonebfont +yes +throughout +hing +israel +below +color3d000080 +future +best +marketsnbspbr +satisa +contenttexthtml +online +annuaire +zp8amp8agu96bwxdiwxoikkwdqwqeyi6d2v8uaowf9iugh2v8aphapaaakvkpcpokgeksu +6r901 +width175 +ist +183338 +periods +prescription +center +19216831 +own +carefully +nemrinatham +names +article3afont +our +style3dwidth100mar +xbarracudaconnect +newsletter +do +head +domain +height3d578 +reveal +january +s4jwqynlapy0kescek4niujorsek0osyj0rygk4niwjgqifk8miqjoqickgosrjkqickgncojopy +holes +mensuelle +styletextdecoration +host +height160 +advanced +held +enter +lpjy5esld5iqfeclqvmvs5v6kuovatsrs50qfbbufjbglw9kpohdfsz0z54mdb0atwqxdszoni +size1imagine +cellspacing0 +14pt +highway +left +vividly +understand +it +sun +esmtp +style3 +macos +term +arrangements +a +n7fgj8w3027170 +dont +deliveredto +office +hibodycsminingorg +serif +walk +dogmaslashnullorg +height424 +testament +messageid +uid64801201487268 +todayfont +10 +mime +0700 +expenseb +xbarracudaspamscore +preservation +federal +textdecorationnone +laid +peruser +sequester +taglevel10000 +0200 +nnlegitsqarebracketssubj +xbarracudaurl +contenttransferencoding +appserver1sysnetcomcn +scotland +importance +emitted +127001 +info +work +height100a +communications +that +border3d0 +declared +countriesp +robert +to +16239725999 +22pxb +museum +you +16500bfonttt +httpwwwqvescomtrimsociallinuxie7c297c134077 +td +web +width3d228722 +tdtr +russian +je +c2de563 +fontweight +youve +here +2009fonttd +believe +mxgooglecom +payment +6 +name3dgenerator +legal +pz +1 +replyto +yyyylocalhostspamassassintaintorg +bgmob +added +id +ads +rmadelynyeacentechcom +profile +common +17 +695 +25 +black +unitsadvisers +tr +remove +check +click +how +2e2ca6d +1716 +date +instant +valueillinoisillinoisoption +mail2csminingorg +valueoman +styletextaligncenterbordernonemsoborderbottomaltdouble +take +freebsdports +wastemindernet +31 +governmentfontfontblockquote +2010 +xbarracudaspamreport +everything +meaning +mailto +mail +alt3d +flawed +by +then +think +even +satofromaddrmatchhl +worlders +050812 +outle +antispam +20069146130techtelnetnet +thats +affirmed +5ozqa7ngazmjnads0u2igbc0matngaafopm0maafzrmkpm0almjnjrqbztpvn9ktvvswb6vbxsy +coords286 +capable +trusted +addr +not +xbarracudabblip +with +from +dedicated +zealandoption +bluetooth +mimeversion +scores +content +contextual +bucks +south +orders +neededbr +aligncenter +flag +could +boy +copy +gw1csminingorg +charsetutf8 +news +xaccountkey +staff +nowafontdiv +siltation +message +tyre +control +formfont +19216818251 +suggests +be +honored +secrecy +hrefhttpb0709dkvotufidcnobi010d2e6735b0b012c975vag01081934237070017543 +impegni +ulosiga +nigeria +texthtml +rights +paying +rules +time +xvirusscanned +script +yvz1rcwz48rjgmz5q4kzipdgvhydqrni5dt5qkzbkvvvc3e2jxklvxbxslfrnaxrskdlt1qljmh +title +farm +microsoft +colspan3d7 +7 +stylecolor +accesscommca +21625123953 +mail1csminingorg +realize +terms +stylebordertop +3bb668cffdd +color000000br +emails +m0620212mailcsminingorg +example +predstvalja +csfzvios48lu2itxcgvdlgerkkaqdanjeejskixaacgtzrdzgp0dmzapdluxdygexnobrubc +3 +arial +104823 +during +no +having +buying +libido +sauuuvvu4brmikpannibgcdimhgt6uwnjpcygekdmbiwgczozgkhhj60h61m9ajd3fg44qmns7o +hibody +does +013 +when +stylestyle +jst +sansserif +g6j4gfp18967 +1000 +contenttype +responds +hash5 +listed +of +england +away +record +received +19216818120 +wed +104414 +bgcolor3d000000 +divhellodiv +sa154a +the +subheader +xpriority +1464033120 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_79.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_79.txt new file mode 100644 index 00000000..b546bb80 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_79.txt @@ -0,0 +1,262 @@ +203433 +used +c2sbx +se +xmailer +for +postfix +face3da +maybe +smtp +usabr +post +test +12 +any +decisions +body +court +resentmessageid +fetched +clientip8219575100 +account5 +this +in +08039 +calgary +switched +myself +have +your +auto +returnpath +once +hrefhttpnlcomcomservleturlloginemailqqqqqqqqqqzdnetspamassassintaintorgbrandzdnetfree +db3691c0fdb14 +g8ah23302806 +arthur +are +acmta4cnetcom +too +built +100 +customize +additions +subject +g86i3ucv015726 +given +qmqp +using +than +xmailmanversion +micalgpgpsha1 +akonadi +listunsubscribe +but +culprit +jmlocalhost +different +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +style +if +071113 +usage +href3dhttpwwwgoooorg +fallback +srchttpwwwcnetcominlinvsubceogif +do +all +reveal +b035b700a6 +re +cest +f +politicseu +violator +receivedspf +nametag +undergraduate +xxxxxxxxxxxxxx +esmtps +060419 +edt +325 +esmtp +232814 +njl +commerce +a +dont +deliveredto +wrote +209204188196 +localhost +within +subscribe +as +dogmaslashnullorg +build +because +classic +spfpass +5 +she +messageid +gavin +10 +performing +0700 +xspamstatus +gnulinux +alt3dadvertisement +comes +225227 +uswsflist2sourceforgenet +they +xrcspam +problems +2007091301 +exactly +contenttransferencoding +i +127001 +8bit +programs +66mhz +square +17proj0007ej00 +out +border3d0 +uswsffw2sourceforgenet +helouswsflist1sourceforgenet +listsdebianorg +508 +to +now +know +utc +did +g7mltni02178 +theyve +331vamm2 +privacy +was +20090614 +recursively +addition +at +replyto +been +id +width582 +difficult +also +need +k12cs156389wfo +on +designates +alignright +how +oakfordwebserversystemscom +wishlist +buttercup +date +friends +bamping +69 +2010 +mailtorpmzzzlistfreshrpmsnet +lousy +051658 +rival +by +he +lisztdebianorg +even +flow +bgcolor3d999 +thats +xmailinglist +new +variable +2525 +dd45613a54bf +bz3applecom +xamavisstatus +560 +again +unless +not +listsubscribe +with +from +6c3f113a42b5 +mimeversion +0500 +inc +linux +lot +pass +xmozillastatus +typing +aspects +cvs1ms1d6fqrolwzskpkncjqpebuqxmyylfqf57e7er8 +make +put +news +charsetusascii +fetchmail590 +xaccountkey +app +sender +precedence +07 +testsalltrusted +francesco +an +place +taggedabove10000 +discussion +be +listpost +get +power +more +hrefhttpwwwmatroxcommgastartnewsletterjul2002registrationcfm +mouth +sa +mark +scripts +81168116 +halifax +manual +flan +sm1 +xoriginalto +shantytown +kde +nominates +call +archive +maillocalhost +having +known +contenttype +8424022131 +user +114409 +may +like +of +ive +and +xstatus +offline +received +one +list +sionnach +month +kelgar0x539de +the +send diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_8.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_8.txt new file mode 100644 index 00000000..0dfe9fd5 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_8.txt @@ -0,0 +1,330 @@ +e0f9truqymiwys55wkzjifuj0brztncbscsdlzf9epctvjji4apwgeizexwtlzpxeedgdyaxjn +taiwan +55265313 +targetblankemployera +xasgorigsubj +j5vafr0arwkhwtcpik1vu3ahnaggsgfod9wihandpyjggyfwb1pywpgara2kcaaguivqq6mjgkw +stain +secret +for +postfix +001 +network +smtp +test +bgcolor3d303030 +mac +mailwebnotenet +body +fudhzyacqraecja2scdmct7ym84wkc3ce4e0bscnlc62lqcdpaihmaijjeis5eim1az1podw5lp +124 +foremo +name +this +1253785785466d01370000w4twrl +httpwwwbeyourownispcom +ip +in +strongfontdiv +listbr +fkj3fmzn6il9oelawmbggj69licomyz39axgf40bqygxjvg0hbhtpse4qnmoacaxckvjjwk +your +off +auto +reported +amazon +offered +are +bordercollapse +tubjdzqgnqbrebvju9gp1hccury8e20mubhgxvn8ft8k8mebf90hbn7ahe3xup7zzujrq6odk9 +course +reserve +strain +100 +account2 +142607 +00000000 +213105180140 +effort +prices +209654p +subject +parliament +alt3dcall +xbarracudaspamflag +given +using +40 +alignrightstood +101753 +ddarywat6flvqkipdmqsxuhgdd61vftodwab9a2gtmmlk5zr5ndmsxzfc3tfbcuad2prwhhaqvp +n +photo +want +posting +symphonies +quotedprintable +yes +0800 +below +backgroundcolore3c332 +roommates +fwx4dv79rej7fv4zpgd7fwhckogbcacl9mjvdeqpxlsj1uhyfum6zugspone4gvdyol7tmqr9mnh +contenttexthtml +designate +statistics +spitfire +another +goalp +students +return +if +ahoverfontfamilyverdanacolorff0000fontsize14pxfontweightboldtextdecorationunderline +styleheight15pxtd +description +voted +b2r5pg0kdqo8l2h0bww +our +cellpadding3 +stroke +thu +league +distribution +stimulating +qualified +hand +sgo +yyyynetezecom +pretty +hit +unusually +22pxfont +img +every +comedian +whitelisted +wickedness +full +blow +xmimeole +mailtoforkrequestxentcomsubjectunsubscribe +us +esmtp +reading +ca +deputychairman +watch +tests +determined +012906 +effective +a +deliveredto +zzzzilugjmasonorg +xmozillastatus2 +193834 +pages +jmnetnoteinccom +hibodycsminingorg +responsibility +purchases +xbitdefenderwksspamstamp +advance +localhost +earlier +build +4px +because +hrefhttp8cctabdru38aruafalyxeea3e3576653a5 +well +messageid +350 +year +maipunetcsminingorg +rowing +witnessed +500 +10 +dog +httpntajirucome99ef59045c42a3171eebaea4a3de7ede +035301 +removed +httpxentcommailmanlistinfofork +federal +charity +such +zmor7edvc6hbpljclxezixmrq4yg9soofb3krhqi5lyerazmnrhulcjq1yhtj2ahykeq1hzfmk +010 +stepbystep +italy +contenttransferencoding +ue3dinternet +zzzzlocalhost +textplain +invoked +cuisine +8bit +spudleymcdudleycsminingorg +out +named +alice +styletextalign +quarantinelevel10000 +moneylosing +to +anbspnbspnbspnbspa +least +typetextcss +singledrop +code +you +httpspamgwcsminingorg8000cgibinmarkcgi +web +32 +bsfsc0sa148lfrn +assumes +genuine +style3dbackgroundc +colors +fontweight +021126 +font +v +internet +was +repair +version +alignrightweak +vulnerability +newsatd +bytes +response +levant +respectively +clientip178245165106 +at +replyto +been +htmlimageratio02 +jjikosd1ihjasr7kiimzenobhjvbcacq0ehhrgjdh3ffctfwbxlbpa1p4zij0krlp8aub57xtz +77trmx4nah2f8kah0vhmb4nfzwoznctal7pfaelgajznctvswcfgzv7if8afzw +id +meta +pts +canadian +tr +highest +notice +afford +lies +upgraded +14 +after +date +mail2csminingorg +threadindex +dynamicipcr200118151227cablenetco +size2bfont +alignments +alt +income +treatments +sociallinuxie +reached +allow +yyyylextoncom +mail +m4s4ubxiify4hbnyjl54imbybf8arjnjjzmwadwblkioisiyi7g4ixy4h5zyutije65ivokilrg +e102102iciencie115 +by +then +worked +align3dmiddlea +advertisers20 +gw1csminingorg19216818250 +style3dcolor +trtd +bsfsc5mj3661 +much +groupbr +with +stylecolorb2b2b2br +from +dedicated +early +baselworld +0 +mimeversion +scores +easily +give +alaska +70 +mailnetnoteinccom +killlevel10000 +clicking +definition +hrefhttpacbf5frofojsscncopy19982009a +s117bhea100er +address +hub +gw1csminingorg +launchingact +charsetutf8 +football +tabletd +namekeywords +sender +message +which +fast +requires +01373 +size3br +sblxblspamhaus +matter +rights +time +color3dredour +constantinople +special +more +grossdomesticproductfontstrongu +w3cdtd +divtable +html +mail1csminingorg +81168116 +digit +vikings +m0620212mailcsminingorg +0900 +emagainem +hrefhttpcarsateiganrikinetseabed97html +thank +call +sjiaisliljyiisiymiojqiiziwlcmjyiswikkyjjuhyziokckjahsviyjsgjqfiwiikiejihypic +no +nowabr +namelist0101 +important +elinizde +anda +some +when +sunsansregularbr +may +orleansp +of +since +dynamiclooking +cents +received +hdregsoftnet +wed +later +1223215986 +amounts +send +parts +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_80.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_80.txt new file mode 100644 index 00000000..edd67928 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_80.txt @@ -0,0 +1,279 @@ +height3d11 +lights +used +mailtodebiankdelistsdebianorg +pm +please +oldreturnpath +for +postfix +1272712434165987cameldebian +classtitlenbspgnomemailptdtrtablep +rpmlist +mailtodebianuserrequestlistsdebianorgsubjecthelp +support +906ad13a50f2 +smtp +question +6621867196 +adding +formatflowed +account5 +this +have +172945 +returnpath +imap +mon +osdn +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +claim +sizet +target3d +makes +00000000 +201051 +onboard +french +text000000 +subject +try +qmqp +using +9d680294274 +therere +listarchive +xmailmanversion +spawned +email +overdesign +but +necessarily +0000 +jmlocalhost +105425 +20100505181350627bssiguanasuicidenet +what +archivelatest575744 +would +daniel +systems +background +oct +ygingrasygingrasnet +messaging +razorusersexamplesourceforgenet +appear +pot +tahoma +obvious +journal +zdnetb +had +re +resubscribe +music +users +nor +result +exim +0400 +proxims +scary +4megapixel +it +esmtp +a +deliveredto +resentsender +wrote +touchesa +1o969n00055kn1 +useragent +dogmaslashnullorg +wwwdoboseviccomen +messageid +0700 +xspamstatus +or +rick +uswsflist2sourceforgenet +mailtojavadevrequestlistsapplecomsubjectunsubscribe +contact +distribute +nosend1br +contenttransferencoding +stop +importance +textplain +i +127001 +saou +listsapplecom +listsdebianorg +sex +to +getting +66187233211 +pbpanasonic +tfewpphob9oa6ycftd2lbliszt +singledrop +samambaiasorg +20080610 +you +fork +know +utc +gnomesession +bank +did +1999 +falls +amavisdnew +000509 +water +mx1redhatcom +contenttypecontenttransferencoding +223050 +solution +n0 +mlsubscribertechcsminingorg +was +version +references +there +windydeldotdcom +6 +413 +nothing +taken +at +through +id +g92gu5f04963 +structure +also +port +listhelp +can +on +mailtoexmhworkersspamassassintaintorg +machine +kde442 +25 +clock +actionhttpinvestorcnetcominvestorquotesquoteprocess00html +14 +terribly +date +sans +home1 +who +a467e13a4a95 +2010 +151 +easytouse +mailtorpmzzzlistfreshrpmsnet +l +opinion +xuidl +boyd +by +he +lisztdebianorg +listmasterlistsdebianorg +actually +692092d74d3 +ldowhitelist5 +0100 +promises +listsubscribe +equivalentbr +with +from +20 +1200 +mimeversion +separate +hrefhttpclickthruonlinecomclickqe4abyfqvkwgjkcv8ni81u0ptqv1yr +spamassassin +innumeracy +gifts +2 +xcheckerversion +use +listid +sent +fetchmail590 +xaccountkey +end +precedence +message +anyone +crelaxedrelaxed +its +errorsto +under +which +almost +msg +depth +an +hot +listpost +shows +into +get +6416122236 +time +fvgtmmultiodd002 +xvirusscanned +201248 +more +annually +xloop +7 +9huwncymw91gyjmcghqtggexdwtvvlgqdnoi0 +mcgowan +81168116 +vendor +facility +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +fri +xoriginalto +many +10143348 +8219575100 +see +debianuserrequestlistsdebianorg +6463 +no +we +cable94189189136dynamicsbbrs +uuencode +manages +i486pclinuxgnu +gui +policy +great +day +contenttype +xmozillakeys +namenumperpage +like +secure +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +other +of +havent +record +received +ever +wed +free +gastonguay +list +quantum +the +7414e1bc92aa3 +archivelatest576881 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_81.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_81.txt new file mode 100644 index 00000000..b3ee568b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_81.txt @@ -0,0 +1,265 @@ +embedded +down +please +lookup +for +postfix +encodingutf8 +network +shoot +question +test +people +x +jakarta +formatflowed +g8d20fc28505 +mac +body +across +prove +resentmessageid +account5 +this +73 +in +cached +reviewsfonta +returnpath +6487055 +selfdestruct +are +osdn +uswsflist1bsourceforgenet +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +josekiproulxcom +tracy +course +vander +subject +4 +got +using +listarchive +2002 +andrei +email +careful +previously +listunsubscribe +canopener +0000 +9 +different +would +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +saa19781 +teach +ist +if +6217825100 +apr +oct +renamer +httpwwwaccucastcom +so +do +all +testsldowhitelist +tue +123213 +i686 +lairxentcom +alexander +had +accidentally +re +34ff313a49a9 +receivedspf +every +152901 +61 +clean +esmtps +13 +just +kilometers +esmtp +902416 +xinjectedviagmane +bhandspring +grey +a +deliveredto +xmozillastatus2 +wrote +uh +localhost +bad +nonstop +as +optional +messageid +10 +signed +isnt +phoboslabsnetnoteinccom +config +20020206185152e10950pihlajakotilo +zurofsky +2008110401 +im +textplain +i +127001 +shutitdownnet +e1r86ercw +helouswsflist1sourceforgenet +to +unitedmediacom80clickq3d93v4zmqsdcta2vknbctjj3r1iojdrrrequest +singledrop +dynnjablskip0 +fwd +you +amavisdnew +x1120081209 +privacy +hrefhttpwwwlockergnomecomadvertisehtmladvertise +mlsubscribertechcsminingorg +037 +answer20 +inreplyto +references +mxgooglecom +171622 +happened +objected +at +rpmsrc +size2font +id +consultant +imprononcable21 +also +listhelp +bogine +on +xoriginalarrivaltime +stan +reminded +date +deep +broadcast +friends +7412592145 +szerda +cohen +jbossserver +easytouse +authenticationresults +googlecom +xsieve +xuidl +0909090909tdimg +workaround +by +he +lisztdebianorg +rssfeedsjmasonorg +thats +will +4bd590302050408coxnet +xbeenthere +back +little +again +not +listsubscribe +with +from +samad +servera0 +immediately +bar +016601c26b00e23f0c70f2812d40landshark +next +155452 +taking +2 +giuliano +both +xmozillastatus +use +corrected +jmasonorg +charsetusascii +fetchmail590 +telephone +sender +permitted +precedence +complaint +errorsto +which +place +7bit +taggedabove10000 +box +listpost +dnearywanadoofr +vms173017pubverizonnet +suppose +magic +merely +respect +14115495125 +get +release +welcome +xvirusscanned +version250cvs +average +ask +fontbrah +guess +more +install +002401c24652725b64d00200a8c0jmhall +form +a225813917453734footerpdfcolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightboldfontstylenormaltextdecorationnone +10143320 +merchant +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +size3d1also0d +updating +xoriginalto +honest +saturday +quicktime +6823024046 +no +we +zone +17128113151 +does +contenttype +xmozillakeys +helpunsubscribeupdate +xrcvirus +events +other +havent +incurred +and +toy +kbit3361 +received +one +right +wed +httpwwwlindowscomproducts +issn +mailtoforkrequestxentcomsubjectsubscribe +144816 +the +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_82.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_82.txt new file mode 100644 index 00000000..72272559 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_82.txt @@ -0,0 +1,284 @@ +terrible +rate +libsdl12debianalsa +100616 +messages +220435 +trouble +fsejectvolumesync +mailww0f47googlecom +oldreturnpath +sha1 +xmailer +for +organization +postfix +database +maybe +744325535 +business +smtp +fits +egwnnet +any +arsasha1 +spongexevionnet +resentmessageid +uswsflist1sourceforgenet +hrth6rubf1doughgmaneorg +clientip8219575100 +this +in +81258125 +exims +provided +have +needed +returnpath +imap +is +mon +testsldosubscriberldowhitelist +dsa1880 +tomgif +m8cs24752wfj +ict +show +heloataz +socket +updateb +subject +try +famine +using +listarchive +system +bebergmithralcom +2002 +andrei +micalgpgpsha1 +listunsubscribe +female +width3d252 +jmjmasonorg +but +11 +0000 +shouting +competitors +jmlocalhost +stanza +g8o80bc26569 +line +systems +if +etcaptsourceslist +tool +big +8a47f16f18 +22 +tue +lairxentcom +domain +kde443 +u +native +re +highlight +look +einne +f +hitting +effect +0400 +xmimeole +else +325 +it +esmtp +listsdebianuserlisztdebianorg +two +m8cs51720wfj +a +collected +motion +deliveredto +pages +adobe +resentsender +gmailid12856d69e7554b27 +localhost +as +dogmaslashnullorg +srchttphomecnetcominedsbuggif +jadearacnetcom +spfpass +shirley +messageid +m +10 +mime +ilug +width100form +0700 +old +or +above +edition +color +they +httpwwwmisdruknlmt2 +iso88591qitzpxyncnmw1x62g9y2gufkejxo0a09niylkgpxnrhhecjmyuaujmnd +contenttransferencoding +2008110401 +zzzzlocalhost +textplain +i +that +2131656420 +helouswsflist1sourceforgenet +to +v1 +qmailldap103 +issue +xlevel +now +212118 +probably +amavisdnew +fact +causing +single +face +oregons +331vamm2 +making +mlsubscribertechcsminingorg +was +mxgooglecom +at +through +ml +iii +xsender +id +largest +httpslistssourceforgenetlistslistinforazorusers +hits37 +need +can +on +71402 +designates +stan +104737 +my +explode +date +jfs +httplistsdebianorg127291505389246camelportablestevenlan +blockquote +concurrency +alignleft +102231082 +brfont +authenticationresults +8116893 +me +xuidl +computers +them +things +by +wondered +thats +should +written +gif +diminished +xamavisstatus +lifetime +with +from +csanyipalcsminingorg +aptfreshrpmsnet +mimeversion +enht +august +rpmzzzlistfreshrpmsnet +aligncenter +xmozillastatus +4cefb57c484a4694a0581837f4f31f7bminnigerodeorg +ieyearecaayfakvoab0acgkqm0llzlt8mhzhgcfseut6porktuolsz0z2slxhq +listid +hrefhttppenguinshelllockergnomecomlatest +publishers +3156 +sender +permitted +precedence +message +occasionally +seem +target +pda +filtering +followed +irish +say +ill +discussion +o58uieumfyfamobvqovlbliszt +better +9cfdd13a54ba +65172233109 +kind +case +30 +wanted +mount +size3d16kbr +manual +activity +fri +w2laee79c571004070335q80635354uf21347f6776b2a38mailcsminingorg +xoriginalto +example +debianuserrequestlistsdebianorg +during +no +23971321026306899629javamailrootabvsfo1acagent2 +sat +bord +landsharkdmvcom +leitl +past +17128113151 +enus +httplistsfreshrpmsnetmailmanlistinforpmlist +xfortune +adsrtgmlistyahoocomau +1000 +contenttype +xmozillakeys +other +of +since +0531 +xstatus +mail3caicom +gmailid128570aec4014666 +received +beach +runtime +waxed +o +webmink +mailtosecprogsecurityfocuscom +sylpheed +handle +arguments +cnetbfontbr +the +172164831 +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_83.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_83.txt new file mode 100644 index 00000000..ad473d25 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_83.txt @@ -0,0 +1,366 @@ +sectors +metabolism +eye +xnjwckwwdnugqyiz6mo8c9knaxs0yddeoopnibymdxp61iotkqn1uge2adgc0cmnvxulkkemry19 +please +for +belowbr +001 +cohi61iq1ztzrb3sm2gxgdx3qyyoib0jayp9n1qxp9vefesiozh97c5ih4vjqpxv7mngzw5tx6aa +smtp +xbarracudaspamstatus +shop +deposited +working +people +package +2002fontfont +body +8 +this +honours +in +cdt +transitionalen +your +spandivdiv +rqwj7zzos2ogrl7m2jubj7zplo0p9jmah4a4qxitar6fcaxz2zn7rf2urhz5aduaaotzc6vz7h +returnpath +jul +cellspacing3d +0001 +mon +englebach +his +energizers +mais111ns +too +ontario +00000000 +step +snowboarding +bsfsc0sa148a2hl +smtpsvc5021952966 +likely +subject +custom +lookingbr +am +system +idlogo +uid127021201487268 +2002 +lowest +bulk +avenue +listunsubscribe +meeting +yes +iso88591bw1nqqu1dia +15 +herbal +hop6nfnxemrk8f0gdm9fnfqe6iozo2tm9zo022hcypjf9j6nbp8agtwthu47hte9qb +href3d +3cfb1eed003722b3 +replyb +contenttexthtml +colllagen +3d3d3d +fontfamilytimes +vedfontfont +080 +yyyylocalhostnetnoteinccom +keromailcommroutblazecom +j +image +weeks +if +br +alto +19216818250 +trust +neither +criteria +up +they8217re +expected +german +xbarracudaconnect +250 +6zwgayvawrmmhsvm4ji4kjqyeytf48pgasvi8vq67edgpxvbawegi4f4wzvaa5tw4f8sbf0s4zpt +head +florida +disambiguationthe +110237 +features +entry +styletextdecoration +youre +410 +pious +gamerowru +height3d39font +blackfont +theonlyone +offbr +054228 +stylefontfamily +trees +advertisement +sun +esmtp +gpnombc9wdwrjarlkwohl7ddcqv5j3gmxz9wsshmo3olm5p0lypuntlvtgeq0fe0dfhy8wn8d5 +24 +019 +a +waylingers +sometimes +xmozillastatus2 +hibodycsminingorg +trillion +barracuda +à +height +within +as +build +freeze +brlost +activities20 +mtv +5 +verdanasansserifyou +messageid +locally +industry +10 +xasgtag +list20 +mime +size3d3industry +text +sticking +5z38vdgjv6ux13t4zzihttl67qopa61ekuxvatazszrxthmlupfg1ysylifpgque7vpaxci6vg +convince +200910281149n9sbnakr017544ns2csminingorg +xbarracudaspamscore +16pt +yourself +looking +public +doctype +pfo03hotmailcom +n7uja5ao011203 +contact +such +contenttransferencoding +abundant +et +reply +e9eaeb +addressafont +textplain +127001 +12510563750b7401030000w4twrl +grand +ethnic +o1k68wi4027260 +2008 +theatres +saving +wit +to +quote +200205212258taa02971iboxgaleriaspacificocomar +class3dgen197025 +charsetiso88591 +national +stringent +text3d222300000022 +you +closely +issued +32 +pto +namemm +134240 +bsfsc0satofromdomainmatchhl +these +amabartoncsminingorg +been +injured +go +id +h5by +upndashtondashdate +286 +pts +rdns +mediterranean +churches +alcharbfont +can +on +aaaqagp8wacaaiayzaaiaajhcsu0edrhgwcbhbg9iywwgtglnahrpbmcgqw5nbguaaaaabaaa +transfer +htmlmessage +width550 +complete +b +upgraded +successful +date +spamgwcsminingorg +height3d160abrbfont +live +take +armani +phos +taught +linear +p111sitiva +sansserifspan +2009 +xuidl +newusernlimgemailiconjpg +mail +yours85spanspan +commercial +by +zzzzasonorg +r111ugh +he +either +httpmysiteinccomtfnlfihtml +yearsprecisely +l97yer115 +rmrnetnookcom +050 +receiving +alignrightally +little +sa148ahl +illinois +stylefontweight +park +widxixidcnufiicinityzu3d873472pyericuofydezuhjbqustrong +with +from +invaliddate +stratum +0 +protracted +mimeversion +scores +welc111ming +heaven +gw2csminingorg +000 +mj1963 +express +monarchy +killlevel10000 +aligncenter +life +session +pmmkmnp71ucut8o2munjormxgggs77j9b6v1lao3qwp89vftfyfppb0pryacve0ohocckwtbht +010947 +ax +gw1csminingorg +sent +charsetusascii +color3d6 +xaccountkey +members +umvwb3j0ihlvdsbhcmugb3jkzxjpbmcsiflpvvigrs1nqulmieferfjfu1mn +melt +asia +n111w +redesigned +loc +attained +81258125submit +150021 +7bit +forged +ruled +187 +be +merely +c +transition +viewing +considered +roman +republic +rdnsdynamic +texthtml +viewabr +b3i9izawmdawmd5nb3jlihroyw4gms41iedjr0fcwvrfuybvzibkyxrhignvbxbyzxnzzwqg +armed +bjd0o0tsqm5syaksguhicc3o60moiyyw7iquweewgacwaijuzcfo60n8z2rtaiplwabeebucmav +dryfoos +ranging +httpwwwfacebookcomninboxreadmessagephp +existsxmailer +replica20 +xbarracudavirusscanned +color3dffffff +type +iq +sa392f +mouth +required +mail1csminingorg +81168116 +112553 +bordercollapsecollapse +colspan6a +terms +xoriginalto +about +colorgray +dsl +pride +h3subscriptions +belowfontdiv +width100 +arial +we +2100 +jobfontfontul +wd0ytvca3pkakkkrsffffahuwnxa1sz0kxs0swiijwmb42jiaxz83wqkk9sjutpd5cweqluk +gobind +0600 +aligncenterchanges +bfor +great +iclick +contenttype +xmozillakeys +footer +7013386252 +hrefhttpwwwqksrvnetclick1095548805070 +of +mostly +and +received +without +host11websitesourcecom +sa015c +2009spantd +herby +right +ever +prefer +reservedbr +phobos +budupaldk4w0sieqtminfycgunsokz7iylumvck4ykq1ps8xqjktbfvmuy3jqlh22i1us5ei7ul +lang3d0s +417 +view +active +the +contributionfontp +lefontdiv +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_84.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_84.txt new file mode 100644 index 00000000..8f004048 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_84.txt @@ -0,0 +1,267 @@ +lush +used +futuristic +oldreturnpath +fashioned +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +packaging +business +support +mailin12applecom +gnupg +people +any +archivelatest575817 +wasnt +name +dkimsignature +this +in +needed +returnpath +once +imap +is +knows +19 +colle +68142236115 +subject +qmqp +dave +listarchive +fwiw +forking +favorite +2002 +xmailmanversion +bulk +httpbugsdebianorgcgibinbugreportcgibug508589 +hammiepy +mbox +objclanguage +listunsubscribe +maa04771 +15 +103618 +0000 +debian +jmlocalhost +donation +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +forkxentcom +recognize +crispinwirexcom +ist +meter +if +background +good +fallback +required53 +doug +keep +do +testsldowhitelist +62901053 +sq9hkxssngyb492uwaz3y3wc3lkvcyc1w6h8y1wtnck9s8enea4yptqxiagwaszyp3lar7 +compression +decemeber +brian +receivedspf +set +esmtps +required40 +it +5556 +esmtp +hrefhttpwwwstructurisecomkleptomaniakleptomaniaabr +xinjectedviagmane +listsdebianuserlisztdebianorg +eyey +determined +hmm +a +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +deliveredto +arsasha256 +xmozillastatus2 +jframe +localhost +though +useragent +as +pdt +messageid +razvanroseduorg +0700 +colspan2font +config +man +resentfrom +87hbne2vqvfsfcsminingorg +contact +they +trying +contenttransferencoding +growing +textplain +i +httpwwwauracomcomteledyn +127001 +xpyzor +spy +that +1002 +typed +listsapplecom +to +existing +score100 +h5nbsph5 +morgensternbfonttd +tlsv1 +filenamesignatureasc +123359 +05012010 +bank +around +1303 +home +mysafesenderlist +making +resentdate +kde435 +references +apqc +iplanet +at +debianuserlistsdebianorg +httpslistssourceforgenetlistslistinfospamassassintalk +id +choosing +over +000924 +listhelp +forkexamplecom +can +on +25 +60 +color3d333366 +government +date +me +xuidl +mail +by +en +visit +mediadisk +lisztdebianorg +xmailinglist +link +2525 +their +p +05042010 +javavm +not +listsubscribe +with +from +rowe +rohit +115247 +20 +motivate +81138113 +mimeversion +27 +000 +express +181311 +panels +1bb856c8469 +xcheckerversion +pass +details +listid +a87d +jmasonorg +charsetusascii +fetchmail590 +mif +javadev +width1td +sender +precedence +anyone +flavor +crelaxedrelaxed +m8cs3291wfo +target +taggedabove10000 +development +listpost +xpolicydweight +considered +account +fine +release +special +more +httpwwworeillynetcomcsuserhome +install +visions +7 +mailing +html +simultaneously +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +aptconf +treated +start +manual +realize +29 +subjectsoftware +claimed +xoriginalto +8219575100 +made +customers +no +writes +gt +digital +altf2 +policy +contenttype +xmozillakeys +3d7img +xrcvirus +like +of +and +record +sponsored +received +bush +vm8000100 +free +list +nv +deleted +javadevbouncesmlsubscribertechcsminingorglistsapplecom +drugs +minor +the +send +xpriority +mdt +those +d diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_85.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_85.txt new file mode 100644 index 00000000..f72668c1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_85.txt @@ -0,0 +1,293 @@ +rate +looks +size1tell +cooks +productsabbbrnbspfonttdtrtbodytabletd +kbs01 +brad +qqqqqqqqqqzdnetexamplecom +hspace10 +for +postfix +activebuddy +httpwwwpcworldcomnewsarticle0aid10288100asp +mailtodebianuserrequestlistsdebianorgsubjecthelp +kfgej170hfpsahdugsbsslblopayxsvsh5oge +200209261530g8qfubg24878dogmaslashnullorg +sed +years +m5grpscdyahoocom +q716osqpnuqfijgwa59qdn6smime8hedi2lk +211503 +clientip8219575100 +account5 +autolearnham +in +httpslistmanspamassassintaintorgmailmanlistinfoexmhworkers +n4umg732sde +renderingcreating +your +3d3f27842010303barreraorg +returnpath +once +mailtodebianuserlistsdebianorg +point +cheers +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +200210040800g9480qk08803dogmaslashnullorg +opera +jmexmhjmasonorg +00000000 +conferencesfontbbr +subject +try +safestrncpy +qmqp +representatives +aggregator +than +469 +depend +xmailmanversion +bulk +anyhow +want +email +httpblogssaloncom0001312 +maypoleauthenticationusersessioncookie +listunsubscribe +quotedprintable +jmjmasonorg +binary +0000 +shared +what +would +ist +problem +oct +z61m +maingmaneorg +so +soaring +play +friend +all +pbs +211525 +892892 +xegroupsreturn +run +52915eb441 +color000000price +193020 +page +receivedspf +exim +0400 +xmimeole +valueandinput +080125 +just +325 +razoragentconf +it +sun +esmtp +mailtoexmhusersrequestredhatcomsubjectunsubscribe +two +a +khare +xmozillastatus2 +maintainer +resentsender +localhost +as +dogmaslashnullorg +build +0l2g008w7vikptx0vms173001mailsrvcsnet +pdt +messageid +g7m9imkv8truyol +zzzzlocalhostspamassassintaintorg +10 +mime +steppin +mgbdebianyosemitenet +respectful +deploying +gnulinux +184744 +httpslistmanredhatcommailmanlistinfoexmhworkers +contact +such +problems +2007091301 +81128112 +reaching +2008110401 +im +jones +xemacs +mute +webcasting +textplain +114932 +lost +presleys +05212010 +postgresql +out +rpmzzzlistadminfreshrpmsnet +hits00 +to +eitheror +singledrop +8ff1516f03 +climbing +you +know +did +amavisdnew +auth02nlegwnnet +resentdate +references +found +c1 +nothing +faced +gdb +c2a0ibam +through +replyto +points +been +yyyylocalhostspamassassintaintorg +debianuserlistsdebianorg +xspamlevel +id +also +port +on +mburgerbubbanfriendsorg +mailtojavadevrequestlistsapplecomsubjectsubscribe +alignright +190710 +remove +httpd +highest +complete +my +affected +date +mathias +streams +jalapeno +comers +everything +authenticationresults +me +xuidl +food +662186698 +scene +by +083758 +lisztdebianorg +helo +link +written +224801 +0100 +autolearnfailed +software +not +listsubscribe +with +from +still +team +debiantestingsecurityannouncerequestlistsdebianorg +immediately +battery +survive +mimeversion +inc +spamassassin +first +srchttpwwwzdnetcomgraphicsnewslettersglftgif +tweening +initiated +docomo +copy +use +listid +put +answer +raven +charsetusascii +fetchmail590 +38e6b44151 +xaccountkey +sender +precedence +sggetchannelsampledescriptionc +ad +stephen +its +kmail1133 +say +score7 +be +listpost +blogabr +protect +looked +namesort +1712811355 +case +height3d8720 +integrated +investigated +mailing +href3dmailtokellyclowerscsminingorgkel +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +brent +manual +automatically +slap +fri +xoriginalto +honest +many +during +no +razor +we +sat +maillocalhost +17128113151 +when +contenttype +xmozillakeys +xrcvirus +may +like +of +ive +bis +123731 +received +extrapolated +httpwwwnaladahccom +today +wed +list +phobos +forteanaunsubscribeegroupscom +1087113 +the +send +those diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_86.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_86.txt new file mode 100644 index 00000000..c02015a2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_86.txt @@ -0,0 +1,276 @@ +e20cs62262wfb +bikes +143234 +3wmtq0p71qeardfmry3lbliszt +wsimple +panel +delphinorum +mfidelmanmeetinghousenet +for +postfix +taubers +icq +smtp +post +dandankohncom +begin +mailwy0f175googlecom +body +patching +years +keyboard +freud +this +instead +in +cached +memo8221 +have +heard +lay +returnpath +iluglinuxie +cheers +is +apparently +100611 +priority3d65 +homeandermaillistslkmhsequences +mailtoexmhusersrequestredhatcomsubjectsubscribe +likely +subject +got +listarchive +2002 +81258122 +email +listunsubscribe +jmjmasonorg +1022314388 +dave3 +anthony +0000 +jmlocalhost +best +rssfeedsspamassassintaintorg +would +remember +front +image +ist +if +good +8192 +utf8qlgaaaac3rstlmzs96sgzles95lgbtq30a096qaaai0surbvdjlbzrnbhqxeiu +fallback +required53 +6870613a4ca6 +20412719838 +so +macabre +esozk0ywd2bgjdt2o2uspwt4jejldzt1sisamcuhdatjzygia +httpwwwamyleblanccomindexphp +programming +tickets +843 +features +re +muo6ytap60zsslt86kg93ufc8ws +result +websearchabr +0400 +temporary +it +yearoveryear +xoriginaldate +esmtp +listsdebianuserlisztdebianorg +a +dont +iso88591qq3b2h0a096265 +create +colorcc00002320fontfonttdtr +145847 +wrote +localhost +231218 +as +dogmaslashnullorg +rule +well +mailmagicgoesherecom +training +pdt +eli +lpyqnryvyzoabec3bgvlbliszt +messageid +year +industry +signature +10 +141454 +0700 +pinentry +old +170202 +httpxentcommailmanlistinfofork +224754 +httpwwwcountermooncom +such +problems +2007091301 +contenttransferencoding +2008110401 +importance +textplain +zzzexamplecom +127001 +that +tk8a4 +to +occasional +charsetiso88591 +singledrop +you +utc +v936 +did +fact +color000000product +coreabr +was +suffered +there +mxgooglecom +6 +httpsecuritydebianorgpoolupdatesmainppostgresql83postgresqlplpython8383110lenny1amd64deb +something +pan20100423154359csminingorg +at +1 +go +mix +id +someone +2440 +over +000524 +detailsfontp +gmailid12858a06aa50c4c1 +port +mangled +listhelp +can +on +asked +designates +x1120090706 +my +extensive +date +bug +ray +chance +thmfgheader +googlecom +xuidl +video +by +standardization +even +ext3 +weighing +dsblorgerr0 +story +2525 +0100 +2000 +autolearnfailed +090034 +not +listsubscribe +enormous +with +from +stuff +20 +archivelatest576772 +1alnuq0007hv00 +interrupts +mimeversion +zerosum +spamassassin +arrangement +awardap +xmozillastatus +listid +charsetusascii +fetchmail590 +hi +sender +permitted +precedence +featuresbr +hdomainkeysignaturereceivedreceiveddatefromtosubject +developers +errorsto +aug +discussions +x8664s +listpost +windowswhich +288910 +videos +mailtoforkrequestxentcomsubjecthelp +rights +geek +6416122236 +blonde +time +xvirusscanned +120011am +mailvipulnet +215 +case +recommended +xloop +alsapcmcard +mailing +required +10143320 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +sgamma +63cad2d0c80 +cpu59osdncom +fri +xoriginalto +demand +call +asking +3 +no +installed +lili +functional +policy +version325 +contenttype +10227141201 +like +of +since +and +color000000downloadsfonta +received +one +742084195 +windows +wed +color3d666666 +list +phobos +119 +webmasterblogs +the +r2201rzrwthaachende diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_87.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_87.txt new file mode 100644 index 00000000..2113afd0 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_87.txt @@ -0,0 +1,269 @@ +rate +114410 +killed +qmail +for +postfix +smtp +dail +begin +formatflowed +any +mailwebnotenet +editorabr +leaders +registered +name +in +margin +your +priority +returnpath +imap +tel +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +subject +worse +using +mailtosecprogsubscribesecurityfocuscom +listarchive +hands +mutt1315i +469 +mailpw0f47googlecom +want +noosphere +email +careful +listunsubscribe +15 +but +state +11 +0000 +e288a7 +height1tdtr +different +would +pulldown +ist +fourla01 +problem +fallback +12921512853 +required53 +111121 +maingmaneorg +rere1 +192321 +short +all +lairxentcom +re +1009bit +httplistsdebianorg4be4bb256060004noarknet +helloianbellcom +mailtoforkrequestxentcomsubjectunsubscribe +edt +httpwwwmonstersfromtheidcom +height600 +sun +esmtp +reading +camera +local +poisoning +a +3si7470659fge020100428191741 +ratwaregeckobuildrdnsdynamic +deliveredto +rv1919 +brfont20 +wrote +night +16 +localhost +05 +talk +pdt +has +messageid +signature +500 +xsmtpvilterversion +removed +0700 +gnulinux +additional +massive +browserfontp +or +encoding +mailminsourceforgenet +problems +provide +blah +userid +stop +textplain +i +127001 +httplistsdebianorgg2kd0bf7b0b1004301018p3fb45806we1fd29cf33c2119dmailcsminingorg +080235 +mays +httplistsapplecommailmanoptionsx11usersmlsubscribertech40csminingorg +helouswsflist1sourceforgenet +listsdebianorg +to +completely +gonna +fallen +xlevel +20080610 +you +ilugadminlinuxie +bank +decent +i5si1135947mue4920100414032035 +hoops +here +715a7a6646c875bfa0a3e7ba9ca +numericids +shutdown +unwashed +inreplyto +references +found +6 +something +happened +hash +nothing +at +through +xsender +carr +debianuserlistsdebianorg +id +mailinglist +httpwwwmailshellcomcontrolhtmlabfa37eeq98z1xl2zjr74cp11v559ad52admz +markets +hspace3 +071705 +on +transfer +designates +discover +nvidia +date +solaris +friends +dumps +april +69 +aptlistchanges +125713 +mans +1769843c32 +authenticationresults +razoragents +xuidl +computers +mail +appropriate +by +flip +en +teambr +love +17pehd4rhphoinzp +however +listmasterlistsdebianorg +httpwwwnewsisfreecomclick081322281440 +should +snip +ldowhitelist5 +2525 +xbeenthere +back +september +corner +preferences +not +listsubscribe +character +with +from +20 +gmailid12851ebd15405853 +mimeversion +spamassassin +lessused +both +pass +theory +xmozillastatus +use +listid +gmailid128333985454537d +jmasonorg +hi +xaccountkey +sender +end +07 +crelaxedrelaxed +its +upset +cooperate +an +7a3de294176 +evolution +233106 +offer +compiled +listmanspamassassintaintorg +release +bits +ignored +more +7 +catchup +alcohol +really +current +cracking +10143348 +provoking +isize256 +copied +tonylinuxworkscomau +no +belabour +we +httpiiutaintorgpipermailiiu +archive +maillocalhost +openpgp +policy +sidewinder +contenttype +xrcvirus +gecko20100317 +of +xhabeasswe5 +packages +record +received +one +today +right +list +days +phobos +sm0 +directory +111636 +the diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_88.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_88.txt new file mode 100644 index 00000000..f469de0c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_88.txt @@ -0,0 +1,287 @@ +icmdecompressiontrackingflags +000801c26f271c0878400200a8c0jmhall +opinions +003531 +120 +clive +behind +leftaltshi +for +postfix +smtp +further +brito +ubiquitous +ended +g979ruk25824 +dkimsignature +this +count +bzw +in +cached +needed +your +returnpath +mailtodebianuserlistsdebianorg +are +is +mon +his +testsldosubscriberldowhitelist +genre +sleek +request +090122 +subject +testshtmlmessageldowhitelist +try +got +qmqp +using +xhabeasswe9 +camaleón +469 +xmailmanversion +bulk +hrefhttpclickthruonlinecomclickqd4e0sqq0exoejqjvw4mc3148lk6yr +jmjmasonorg +0000 +1471454022 +cuba +212173515 +br +problem +telling +60x +thu +22 +hrefhttpliststheserversidecomt12337850395208181img +svenjoacgmxde +do +all +phd +berkeley +had +re +095 +youre +undue +dm02264bbbd58b1e24cc +just +required40 +325 +it +sun +esmtp +488256 +mailtoexmhusersrequestredhatcomsubjectunsubscribe +superior +networking +a +deliveredto +job +xmozillastatus2 +resentsender +restarting +localhost +useragent +id8dbqe9pikce0nzsjr53oirarv6aj0dnw3zse1x3xcafguhb90hfc0qwceiinr +hits49 +as +dogmaslashnullorg +talk +div +spfpass +pdt +messageid +10 +john +srchttphomecnetcombgif +inquiétude +text +spamassassintalkexamplesourceforgenet +locked +0700 +quite +string +bblldsgs8moxzsyfdie4t58vzglu4gjwvirimaneljcxvupwj5dmkv3u0mir5dtx1 +score54 +httpxentcommailmanlistinfofork +or +edition +arraysa +cubanlinksorg +archivelatest575654 +laptop +command +such +they +xrcspam +mozillathunderbird +contenttransferencoding +21 +forkadminxentcom +im +textplain +mimeole +technologies +127001 +lw15fdlaw15hotmailmsncom +lost +files +that +066ef441b2 +listsdebianorg +to +factor +issue +charsetiso88591 +singledrop +you +000814 +technology +utc +4bbe8fa33040800coxnet +tcltk +eg +bz1applecom +here +hrefhttplockergnomepricegrabbercomlow +mlsubscribertechcsminingorg +eastrmmtao103coxnet +safari +brunmardecom +merge +go +id +doesnt +listhelp +distributed +need +can +on +120253 +related +xoriginalarrivaltime +designates +alignright +date +2010 +trailing +googlecom +ben +things +cookie +2e69b2940fc +by +visit +installs +think +sprouts +default +width75 +xbeenthere +back +0100 +again +sunnier +their +150426 +massescraigevolvec +not +httpfortytwochgpg92082481 +listsubscribe +delete +times +with +from +team +rohit +money +mimeversion +0500 +enht +spamassassin +omta0110mtaeveryonenet +budget +height20 +use +fetchmail590 +report +size2 +xaccountkey +regards +listsdebiankdelisztdebianorg +sender +precedence +tried +message +anyone +stimulate +which +say +an +place +cell +score7 +be +listpost +downloaded +xpolicydweight +g7l875y09030 +instructions +jill +h +overwritten +time +better +more +215956 +i386redhatlinux +edsger +debianuser +184830 +between +81168116 +patch +manual +alcanetnetfr +83134218234 +105738 +claimed +many +1341576530 +8219575100 +arial +mike +weekly +let +identify +does +applicable +massbased +reasons +g8t81hg03431 +xmozillakeys +114820 +may +gecko20100317 +might +of +and +xstatus +received +one +mochlis +nail +today +200 +015853 +posted +list +sell +104940 +the +advertising +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_89.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_89.txt new file mode 100644 index 00000000..5a6f1c64 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_89.txt @@ -0,0 +1,340 @@ +valueninicaraguaoption +0300 +100istrib117i +cells +w20mr1219671ebn491272147491918 +444 +mergers +for +familys +network +support +smtp +june +children +series +proposed +colspan3d1 +081 +mailwebnotenet +solutions +valueasamerican +8 +name +this +evenings +in +11573250140 +transitionalen +your +returnpath +yanbian +0001 +are +is +size3using +xasgdebugid +officers +abehlimsncom +abskyqgucgedgibzhuwgyiuqxgybc2p4l1c17qbdeahwaecqalt7gepk8hfsauaikwg4uh8weuz +00000000 +256ffic +reputation +seen +request +xbarracudabwlip +f0rgz186zyyxkuqohotxontattbrcgwq5xea0yqu5c13oduluulzaktiawqgs3jtdmldth2gxt7 +subject +4 +structurally +relay +l1sopwwtvknqi0oknptnp3xkxrtnl5fgub9go9gprg4kxhdes4jxkeors2nfcobixrm1nllfkjq +given +using +size +than +lowest +jgtsqb1fxped4rjrstzxlhhwirt1qtkk05cxhs72mesv9tfpip1273npz5880w27vsxjwwnq27 +email +quotedprintable +15 +ceased +282167767 +fineness +below +6f9aeux3vxvxqofddvwbfi8aod0djuewuuuuzckkkkaojwamhvts1itmkzmeputtqbg0gfh +challenging +vwkrmip5pjxwwzbe3pyorsjuvzwrn2s3htzt1jamiskvvk5hzgr9nolxqdk1za8rku8remazpl +degree +online +caggxkdihucaqehmgkzqaabqcwygs4btdkuswny0gje1lvukkbjirex1z0qsturlm8war4vflu +voor +line +pa +center +miltiorrhizae +apr +19216831 +51br +designed +hotmailcom +up +face3darial +fontfont +bsfsc0sa424 +population +german +xbarracudaconnect +t +stone +appear +all +head +vlink3dc0c0c0 +agency +successes +color404040privacyfontanbspnbspnbsp +000000 +host +hitting +data +0400 +montes +xmimeole +independence +m0089lla +esmtp +bgfuz3vhz2u6ievolvvtoybtc28tzmfyzwfzdc1syw5ndwfnztogru4tvvm7 +a +constitution +deliveredto +hibodycsminingorg +receive +advisor +barracuda +localhost +hereanbspand +build +rule +5 +messageid +zzzzlocalhostspamassassintaintorg +newslettertitle +taxtitle +500 +10 +xasgtag +cellpadding0 +chippewa +gewesen +removed +warning +3196004 +products +allygatorrcsminingorg +statistiques +shar +cinderella +doctype +pace +floorball +010 +critical +taglevel10000 +contenttransferencoding +auqlmavtiffcufgzkckpiyjyfx05mexisj4usajmairymavwgavdoffo4dztgfhswfrlifhtuhlf +094116 +age +xlibrary +1023139131 +until +tkfnrt0iuxvhbgl0esigvkfmvuu9ikhpz2gipg0kpfbbukfnie5btuu9ilnbbglnbiigvkfm +8bit +helvetica +width535 +ulm6wspexcbhymnk0tpu1dbx2nna4upk5ebn6onq8vp09fb3pn69oadambaairaxeapwd3osv +altbyline +that +pauh4lmrljjst3g1 +height199 +height3d22122 +backp +to +v1 +rockefellerem +code +095034 +you +td +web +32 +jun +did +company +disappointment +li111rer +ef5e9afa8abafaeacf2f798eca9a6a5aca6f1abafa2bfadbe +font +here +these +version +button +desirable +at +uaa16161 +added +import +low +id +meta +broker +ynaejyjeh +political +154138 +being +multipartmixed +black +click +b +cosmetics +uid109061201487268 +value150000000015000000option +date +spamgwcsminingorg +live +xbarracudastarttime +31 +taught +aa6vsvdk6l0gvy1rdyasobzhhdzh12vubipsvxbckl1tep9quzlyr6n65iojsdaicg +respective +xbarracudaspamreport +cheaper +abr +everything +me +accessing +xuidl +mail +fill +by +align3dmiddlea +gralit +progressive +new +will +bones +c111mpelling +promises +p +earned +3rd +hrefhttpwwwsydnodenruprivacyhibodycsminingorgc0535a86ec7bd87p1 +with +from +0 +mimeversion +scores +width49 +gw2csminingorg +inc +tweening +exclusive +killlevel10000 +width3d536 +notificationfefacebookmailcom +inn +alink003399 +mollotony +transmissions +definition +address +sn4kkssfqkqgje9iv4jidpbxixzfm4irtogjixkn3ncp4cdb7t95a39o1dyhah1rqripiot9kn3c +backgroundcolorffffff +080308 +targetblankfont +comparable +position +message +phoboslabsspamassassintaintorg +vtepiia6zcdnnkkjecf02oazobmgqspdizzuabruaaxwbk1fng1dpmmjfyoqepdaevlsb7xyg +column +which +src3dhttpiiqusimagesqgif +yearsfontbp +bd8u7hz4z3fglprjr1e9pktko8vj8ywuslgql5oatktm5q1s1ev7k1wa5yldbwvxwbrfrkhfst +7bit +118 +1273735137 +be +moneyfontfontbblockquote +checkera +whip +roman +texthtml +incentives +more +xbarracudavirusscanned +r +nonehalf +alignrightsometimesp +form +paddingtop +sa100a +html +stylecolor +terms +fri +expansionism +spam +xoriginalto +width3d500 +0900 +morning +limit +arial +no +we +buying +91 +some +greenhouse +013 +applicable +3610 +124087 +centerfleets +jst +day +including +sansserif +serviceanbspnbspnbspnbspnbspnbspa +contenttype +score560 +like +listed +framti100en +of +bioy +ive +mostly +and +determinism +received +tulos +fontweightboldstrongsubscribestrongtd +webmasteredvserviceshopde +wed +leerlingen +days +boundarynextpart000346c2201c1f83e20e71340 +httpwwwbarracudacomreputationip806951182 +nbspcall +commonwealth +level +search +the +cant +send +sydney +bybr +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_9.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_9.txt new file mode 100644 index 00000000..0afd74bb --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_9.txt @@ -0,0 +1,358 @@ +60029005512 +used +tbody +quality +strongcomics +xmailer +for +postfix +network +series +httpwwwbarracudacomreputationip80232223179 +122029 +sa154 +city +body +border3d0fonttd +young +hrefhttp77c14dwikizbccn +this +400 +052 +in +toner +cdt +women +alignrightfont +firewall +have +returnpath +0001 +are +is +extracted +herefonta +charset3dwindo +why +xasgdebugid +hrefhttpffmedsayre76aruwumany74026238b723fd +founder +genre +00000000 +srchttp0000002740000003400x800x39ejk3bbnuklebeniwhacifaqghjahxhekthahfiw +subject +hostname +bgco +contentdisposition +system +kawyiwligmtpy007yahoocom +included +tf0db8atzi214f34fwx17c6jjdvrurybrtbhtbi1ozxgwuvs9qyluku12tts42bvq +aligncentera +heads +avenue +base64 +lineheight +msgidfrommtaid +quotedprintable +shbig505413body104 +valueuaukraineoption +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +below +ns1csminingorg +best +url +3cfb1eed003722b3 +replyb +contenttexthtml +cccccc +uid79041201487268 +115111me +flat +style +ist +if +good +1924 +victor +protesting +dine +pennsylvania +oct +h2usedh2 +condone +102137911 +european +our +waking +250 +limited +t +100irigir +winningem +do +advertise +w6radvfxp5fzvbqv83x8f4jspmdb53ahjmwcpwdpwkvterbanvxpelkmwvlnmgepv9 +head +rbl +club +handling +ramphocelus +warsaw +host21734129211inaddrbtopenworldcom +gift +uri +hrefhttpsh48futurevelocityinfo252113115f3593114410248htmflexbelt +natura +it +thisthat +esmtp +xezaji +two +e07482c1d5 +szwwxz7mhgp319aqzxlsy8oar1b6udksmpq3em4gwz7nt0gbl2gd075pssvppnz7enkpbheq7wpk +a +kyle +passed +deliveredto +xmozillastatus2 +httpslistssourceforgenetlistslistinfospamassassinsightings +centered +bythinkgeek +35 +receive +barracuda +though +subscribe +yield +albaugh +bsfsc2sa154hl +mimehtmlonly +draft +messageid +350 +olympic +pbufont +10 +xasgtag +koreans +mime +associate +suspence +classlink1 +lower +walking +certain +0700 +pulse +or +g6oe8ukh004941 +public +strong +doctype +contact +bgcolorgray +010 +encoded +contenttransferencoding +multipartalternative +service +127001 +dqo3ote5ltexohroifn0cmvlda0kti5ezwx0yswgqkmglcbwnem2rzksienh +kingdom +doctor +bone +clocal +cards +face3dari +groupoid +that +out +division +border3d0 +121655 +cosmetic +buy +to +lieutenant +picc111li +wind +code +you +td +32 +select +catalogues +aside +weet +share +targetblankclick +forming +here +axurc2wxbpcgwsa9cthv5yk3zfugscyddtnsk2cwwxusdtgxi460hbpv5akykskwtbj5s9k2d +found +6 +experience +at +occurring +comprise +been +id +over +also +alignabsmiddle +can +on +general +17 +sixteenth +dealt +tr +pinch +14 +sa424 +xmsmailpriority +after +date +spamgwcsminingorg +scottish +g +31 +representative +reestablish +hundred +f111rmatting +kehqtqp +src3dhttpemmansellg +rob +1962018941 +012842 +promotion +height3d5 +by +width4 +finns +goods +pick +official +zzzzlocalhostexamplecom +howl +listmasterlinuxie +fomk1dld73t8wy2by1gx7dlnboo79dunpcw2irvxtznfgnle50p4trpjnh7btdwfmqke8rhbhw +link +050 +einsteinsszcom +management +0100 +their +units +georgia +times +pc +with +type3dtext +from +refer +height100 +mimeversion +scores +fontsize20px +16px +522px +disregard +defences +000 +express +2 +singer +killlevel10000 +crushed +nauvoo +xmozillastatus +charsetutf8 +permitted +boundarynextpart00000233be1e640405ba1e005 +nbelkk3m009175 +phoboslabsspamassassintaintorg +ultimate +10px +latvia +server1 +face3dtimes +valign3dmiddleimg +earning +an +fully +hrefhttpwwwfirstdentalplanscomwwwfirstdentalplanscomabrfont +forged +realm +targetblank +suggests +attached +unl8fwzeqaaaaaaaaacmwo26qno30lnifwaaafn2qrpmozbwfhhic4bnssaeuynkzrlpgdqybif +ess +longer +purchased +cou +advertisement2e +worlds +texthtml +6416122236 +srchttpscientificdirectnetimgthomson103007areyounforr4c1gif +time +power +farm +consisted +microsoft +size3d1strokefontbtd +brl +w3cdtd +harmful +helpless +html +httpequiv3dcontenttype +district +matches +6f92e1766 +wammdyyugaggaawauyowfimtymxerlscackwj0ajacaqccaazkybwweqmhlai0hbskykhpp0pqa +h5to +10216903 +cg0kdw5wcmvjzwrlbnrlza0kdqp2ywnjaw5hdglvbg0kdqppbml0awf0aw9udqoncmluy29uy2x1 +fri +baselwo +xoriginalto +titledivision +transmission +161141 +width3d128 +mistake +campus +no +don +034023 +discovery +component +military +8810173154 +c448022c +width115nbsptd +jst +1000 +contenttype +decoration +7013386252 +other +of +c111n115t97nt +401 +received +setup +sa392hl +005010 +free +rep1icahandbags +list +wish +created +tdbody +helod diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_90.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_90.txt new file mode 100644 index 00000000..306f62fb --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_90.txt @@ -0,0 +1,262 @@ +used +elements +wash +mailtodebiankdelistsdebianorg +pm +trouble +mfidelmanmeetinghousenet +qmail +able +for +postfix +194834 +identtperdue +business +smtp +srclists +dm0206 +conferenceabr +45e742940d3 +prescribed +website +pgpinline +years +registered +instead +in +cached +0725 +have +your +returnpath +imap +success +advertiseabrfont +bouncedebiankdemlsubscribertechcsminingorglistsdebianorg +are +mon +his +wait +101431608 +00000000 +subject +kl +watching +listarchive +included +noticed +listunsubscribe +hrefhttpwwwlockergnomecomabouthtmlabout +telka +width601img +jmjmasonorg +0000 +241fa2d0c14 +businesses +would +another +yyyylocalhostnetnoteinccom +levels +ist +if +world +br +mayadyndnsorg +fallback +required53 +blocks +human +102322 +etc +pulled +score52 +scientologyxenu +do +lairxentcom +jukebox +domain +re +downwards +widespread +httpboingboingnetredhotjazzcom +exim +copies +edt +petkamdm01kamdsvru +it +xoriginaldate +esmtp +191128 +problemchange +253 +subscription +a +dont +encountered +deliveredto +khare +temperature +xmozillastatus2 +tagged +wrote +localhost +useragent +2010513 +scott +messageid +10 +isnt +undergraduates +x2nx +prohibited +muc +phoboslabsnetnoteinccom +0700 +or +seems +come +xrcspam +problems +2007091301 +contenttransferencoding +ma +textplain +i +reviewers +127001 +outlook +files +that +bkcmurkworkscom +listsdebianorg +hits00 +to +v1 +unified +getting +exact +singledrop +base +adsl66140152233dslhstntxswbellnet +technology +fork +theyd +e8 +edit +samskar +name3dgeneratorhead +mlsubscribertechcsminingorg +inreplyto +doing +alqaedas +variety +1 +xsender +id +mcup86es1csph +ntfs +002221 +readyhostingcoma +also +xcomplaintsto +listhelp +on +same +mind +designates +education +actively +date +friends +jalapeno +adam +taa11365 +httplistsdebianorgy2sd80f793f1004081844r61d3e2f0z8d8d480be79b6f96mailcsminingorg +clear3dall +googlecom +xuidl +them +configraidattrsm +envelopefrom +12302 +by +en +he +unsubscribe +lisztdebianorg +helo +xmailinglist +actually +gif +xbeenthere +0100 +specific +not +listsubscribe +with +from +reins +20 +1alnuq0007hv00 +enht +next +rpmzzzlistfreshrpmsnet +enjoy +catalogs +pass +picture +listid +news +282 +permitted +precedence +flush +place +49cb21c2c4 +aug +be +mirrors +into +roman +where +specifi +frequently +xauditid +mailing +identified +81168116 +105215 +vendor +start +hole +many +032337 +supreme +no +10876217 +archive +errorsbr +geneva +some +appriciate +1022314479 +group +httplistsfreshrpmsnetmailmanlistinforpmlist +version325 +contenttype +xmozillakeys +xrcvirus +p6c8w38vv8t +102345 +record +debiankderequestlistsdebianorg +received +suite +testsawlknownmailinglistrcvdinosirusoftcom +wed +cellpadding2 +mailtospamassassindevelexamplesourceforgenet +26f9713a4aeb +level +the +those diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_91.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_91.txt new file mode 100644 index 00000000..ef6507c0 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_91.txt @@ -0,0 +1,325 @@ +55265313 +e +recipient +side +rue +size4start +compete +pc5 +format +mother +for +database +hrefhttpmta12automated00onlinecomol7xeinfmanbbotzimg +width3d +business +referred +smtp +xbarracudaspamstatus +method +mood +bsfsc0sa392hl +researchers +chargestrongfontp +body +bye +81d622940b3 +this +ip +in +5tik5cmvmwqbymjudtzzdbjy8fedplvmzpm7m4men1fdfoc3zlws8ripsbci1xvkaweyzakanr2 +subtle +firewall +have +jul +nov +icbxzsbjyw4gagvsccb5b3ugz2v0ideynsugb2ygew91cibob21lcyb2ywx1 +aaaaaao1durakocwnkx90ur56yjztvhyxgajzhplsjn6jul36p3ptbpwnflyez3s79h7jul36 +allies +northern +h2of +too +feature +213105180140 +32qdsjvd4n8pa74f8r6h410cace0my15bmovvmbyd1ge45hoa6y0kxpgl3at33b6eourxr6p4f1 +subject +custom +size3d6br +xbarracudaspamflag +using +40 +size +produced +system +ldfontfamilyarialsansserif +included +history +three +strong4250strong +quotedprintable +legalyou +0000 +url +replyb +streamb +741px +another +titleqcyogab +strictly +recent +style +treasures +olympics +19216818250 +19216831 +description +big +xbarracudarblip +our +size3d2emailfontbtd +xbarracudaconnect +t +thu +mattei +zt0ivnc3osxpiibzaxplpsi0iibjb2xvcj0iizawmdbgriidqogicagicczxkzopeiv4krpqfom +hall +re +host +look +youre +music +data +256fficial +emaem +13 +uri +hearing +nrm242245columbusrrcom +us +it +width610 +esmtp +mvcbj +neues +maritime +helodynamicsplitip +mx +os +a +dont +deliveredto +xmozillastatus2 +hibodycsminingorg +e3dcolorffffff +localhost +serif +chirac +aqueous +messageid +350 +10 +mime +divagainstdiv +xbarracudaspamscore +or +computer +public +pure +leader +tonbsp +interested +lgifhuaajwa5tpgacjboddzsrhqx9kmc8baptucbn7fwabujl8zcaxugagmtafgzabghaebkcb +others +total +wesleyan +they +191719 +taglevel10000 +contenttransferencoding +rnqjrmlh0n9ddgx9f8at1x0ycmtngpeb9nbgpvho53xv52xf8acwv7yjyf1hy8zibj +service +alignrightproposalp +mimeole +ariza +127001 +purposes +work +that +out +032145 +border3d0 +quarantinelevel10000 +styleborderwidth +to +charsetiso88591 +washington +singledrop +now +xbarracudaenvelopefrom +you +htmlheadtitlebeautifulcustom +hqpronsnet +around +fact +1ffhhhhssem3awibaiefiwwwequxhaiq1sqrk9vwqnkbvjvacckpy0gqx5bfarqqbou0kwefbch +221555 +font +spans +version +3198059 +farmers +nonencoded +1 +profitable +id +2060p +tr111117wen +need +on +17 +external +click +centertable +xmsmailpriority +after +date +uid69991201487268 +mail2csminingorg +spamgwcsminingorg +damien777hotmailcom +april +anymoreit +alignleft +font20 +xbarracudaspamreport +table20 +reached +stylewidth +xprecedenceref +xuidl +valigntopbradobe +9h5pe8zu2dlpom5wdty8azyxbrjgxrmjp4pvgyaxtfrpxf8cmuwyaljoc7wfirbrglk8rtfw +by +then +cellpaddin +charset +53 +new +link +050 +back +p +software +reaffirme100 +delete +with +from +still +means +majerit +money +mimeversion +scores +16px +kubiak +000 +20293254118 +realtime +stopped +placed +cellspacing3d0 +b0af7294zob3d7332020987156570694162 +xmozillastatus +httpwwwbarracudacomreputationip12412014391 +divisions +bring +2px +sent +transylvania +2180 +areas +report +ccd2d2 +useabr +size2 +thus +francepresse +transitionalenhtmltable +server1 +snwdxovqebeqcnblankgifap +err +smaller +financial +be +width3d98 +into +prospect +sought +plans +texthtml +xedii3d72600200714792826310 +31103662 +xvirusscanned +sitesspanatd +rh3sxgjwtlxfvf3d6j10rbbpqai2sqtgu3jos5q9cnbvolsrw6ac5feqtorp7cct6yizn9tyyg +title +handbagsfashion +xbarracudavirusscanned +brl +w3cdtd +ranges +transitionalen22 +127477971278f5b7f80001w4twrl +20002400 +origins +activity +act +contains +modeling +joined +recordings +uaa22974 +degerlendiriyoruz2e2e +seriesing +150 +bgcolor3dcc3366divfont +0px +no +we +ballparks +yhrtcexb1srpdm1ob0gszgu3gzcaeqsbu5ogtxyabrnqb7fzcmshzcgxogwtbkm8p4qhaeguak +geneva +some +g9woxahj2drpkk95upbiic08kakoeywyl4n4mdhhez5gfdxkadztjhmcuhlrhnlsdvgsamfjmswt +skin +caplets +013 +when +group +obey +1000 +contenttype +xmozillakeys +085757 +sold +may +parasites +assist +csminingorg +pnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +record +203br +received +pagea +one +19216818120 +10000 +wish +table +index +badges +york +collapse +search +the +fontfonttd diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_92.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_92.txt new file mode 100644 index 00000000..7b07c856 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_92.txt @@ -0,0 +1,274 @@ +rate +g7ra7gz02703 +please +einstein +desktop +oldreturnpath +for +postfix +testsalltrusted18 +smtp +post +lerlaptopiadfwnet +people +xmailscanner +clientip172541337 +8cb9f3a6555 +this +in +returnpath +once +is +mon +183925 +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +101431608 +mailtospamassassintalkrequestexamplesourceforgenetsubjecthelp +managing +nedit +mozilla50 +forks +said +subject +68c3029409a +try +qmqp +am +system +httphulucom +469 +2002 +bulk +want +email +09 +httplistsdebianorgi2p751c63ee1004221339l6b03ce6cp8be3dbb14eb2d6f3mailcsminingorg +previously +listunsubscribe +but +state +0000 +substantial +jmlocalhost +best +gentlemen +role +spend +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +handles +apr +bet +required53 +xapparentlyto +httpwwwgravitylensorg +xacceptlanguage +thu +105830 +do +all +him +192103 +href3dhttpwwwmacrovisioncom +monetary +u +re +4bcc83d1308freefr +medlicottpanasascom +data +receivedspf +result +0400 +multipart +idea +edt +esmtp +term +httptheolsonfivecomindexshtm +a +6bd212d0d95 +7si1445933fxm1920100408031527 +deliveredto +arsasha256 +mean +information +xmozillastatus2 +w2y2d460de71005061315h83c91b98zd136a27fe5433d16mailcsminingorg +g6q9s3j17924 +wrote +localhost +as +dogmaslashnullorg +because +spfpass +pdt +has +messageid +actionhttpwwwgnomejobcom +10 +signed +65551169 +0700 +gnulinux +or +resentfrom +contact +contenttransferencoding +g7qilbz05026 +127001 +8bit +files +that +out +divbr +träsktorpet +cannot +hear +robert +to +monday +1022591 +half +20080610 +fork +256256 +hags +httpwwwdebianorgnews201020100522 +testsawltnonsensefrom4050 +here +mlsubscribertechcsminingorg +resentdate +inreplyto +httpwwwforbescompeopletrackerresultsjhtmlstartrow0nametickerc +references +mxgooglecom +found +oscillate +melting +second +at +1 +automate +20897132119 +id +mailinglist +listhelp +on +mpaseyosemitenet +httplistsdebianorgaanlktin0jzlssk1lqvkkf3r5yjtynon5gxp6wh5ykdvmailcsminingorg +cding +check +whom +209234249206 +worth +alt3dcisco +date +imzirerzcl6bst7y5y6mdcky4emogirjprqen5rmyzqlj1qvzxtgjujhjhr6s4f80od +distributions +g +debianmultimedia +483ae13a6350 +issues +2010 +alumniriceedu +r10cs71952wfa +opening +by +foodie +reviews +rssfeedsjmasonorg +comment +resqnetcomspamtrapstaintorg +majority +2525 +xamavisstatus +6621866107 +2000 +autolearnfailed +wont +not +listsubscribe +with +from +apologies +team +3750 +wood +20 +enht +spamassassin +first +dvds +true +iceweasel +linux +james +listid +british +discern +hi +e17pfia0005tw00cpu59osdncom +xaccountkey +extra +sender +precedence +httpwwwfurnarinet +message +its +an +discussions +be +listpost +ahead +20122912112720010626 +protect +goes +bgcolor000000img +6416122236 +time +bits +resqlplpython8383110lenny1hppadeb +ask +power +more +update +scnieitst +5793313a5082 +v153d +xloop +mailing +sensible +mark +between +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +matches +modest +httpwwwirenicembersorg +fri +claimed +hssamix +continues +cki3wk7kgika8zhzu7ulbliszt +top +past +e1odcvg0001pwtdwhitemac +670fd400088 +policy +contenttype +xmozillakeys +20100405 +3d09joosno8sksh69p0bkgium1v +of +since +log +limbolistredhatcom +xstatus +received +ldowhitelistmdodating2pgpsignature +wed +free +list +richard +broken +the +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_93.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_93.txt new file mode 100644 index 00000000..1cd38169 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_93.txt @@ -0,0 +1,290 @@ +123803 +lush +0300 +recipient +trouble +oldreturnpath +capabilities +20thirdparty11x11synapticsfdi +behind +62637 +for +organization +postfix +x9v3abfg5joozfj4ybj0jkoimhhxsvwaaxg +encodingutf8 +harley +autolearnno +tracker +formatflowed +51 +clientip8219575100 +this +mere +autolearnham +in +have +returnpath +jul +imap +exmhworkersspamassassintaintorg +rkaqastkwgajhbtnpvlbliszt +is +065911 +martin +god +multipartsigned +v7hbuv6eh4uc +displays +width12 +warming +likely +ultracompact +subject +hard +4 +already +using +less +am +contentdisposition +system +xenigmailversion +xoperatingsystem +2002 +email +statesroman +model +listunsubscribe +yes +but +state +0000 +elkins +security +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +httpwwwsolonorcombloggerhtml +would +213342 +archivelatest577463 +193706 +14sm1150396gxk1120100430093058 +forkxentcom +hrefhttpwwwgnometomescomtomecataudiohtmlget +if +overcoming +eugenleitlorg +up +imf16bisbellsouthnet +so +tue +burnham +xprovagsid +u +previous +lockergnome +re +hit +music +users +httpsexamplesourceforgenetlistslistinfospamassassintalk +mailbag +incl +required40 +it +esmtp +professions +dont +encountered +loading +deliveredto +xmozillastatus2 +jmnetnoteinccom +resentsender +create +provides +wrote +localhost +useragent +as +dogmaslashnullorg +5 +messageid +10 +selected +solved +0700 +contact +others +they +trying +0200 +xsaeximmailfrom +contenttransferencoding +required50 +way +open +textplain +i +127001 +8bit +info +httpsecuritydebianorgpoolupdatesmainppostgresql83postgresqlplperl8383110lenny1s390deb +that +concert +out +listsdebianorg +to +become +charsetiso88591 +singledrop +u13mr2333997fgk661270713046714 +arms +amavisdnew +around +r2z68b1e2611004201238g903f97c0jda85d928045510e9mailcsminingorg +here +src3dhttpwwwcomicscomcomicsdilbertdailydilbertim +these +making +mlsubscribertechcsminingorg +inreplyto +21078 +change +c1 +at +1 +mailtox11userslistsapplecom +replyto +10300286476462tmdadeepeddyvirciocom +debianuserlistsdebianorg +raise +id +davelocalhost +gmailid1283e8337d201ba2 +bill +slightly +can +on +external +being +asked +mind +secular +how +my +upgraded +ground +after +cost +date +friends +jalapeno +463 +innovation +authenticationresults +me +googlecom +resolution +mail +gutermanmediaunspunimakenewsnet +by +then +unsubscribe +will +2525 +xbeenthere +again +smtpwowsynacorcom +listsubscribe +character +with +from +0 +dmzfirewall +width1 +ieyearecaayfakvm788acgkqm0llzlt8mhwwlgcgkavlws5x9f6l3h9jwgwqfwmr +sync +172294170 +mimeversion +pan20100501190314csminingorg +spamassassin +first +mailtoforkspamassassintaintorg +pain +xcheckerversion +lot +pass +xmozillastatus +6442296 +themestyle +listid +make +a87d +jmasonorg +news +done +xaccountkey +header +precedence +size2img +message +sentto2242572527261030016790zzzzexamplecomreturnsgroupsyahoocom +debians +original +sunrays +taggedabove10000 +debate +be +listpost +agnula +srclist +tia +981046062 +surrounding +researchbuzz +httpswwwgrccompasswordshtm +mailbw0f210googlecom +bits +were +xvirusscanned +asset +215 +mailing +debianuser +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +gestattet +81168116 +bgcolorffffef +mondowith +lemon +except +8219575100 +see +esophosi4523141270440000 +no +having +installed +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +esp +some +day +jmrpmjmasonorg +contenttype +xmozillakeys +may +of +202891458 +xstatus +received +one +windows +5900f2d53f534b41b230b54025e5d262thursbycom +free +list +honesty +broken +the +ripped +maintain +080741 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_94.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_94.txt new file mode 100644 index 00000000..d6988690 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_94.txt @@ -0,0 +1,348 @@ +target3dblank +href3dhttpf53c27kwitafuwcnke3d6291d4d90ecf4d8cd2fb0 +xasgorigsubj +o2fafalh011881 +cellspacing1 +messages +valigntop +xmailer +for +postfix +h97118e +spfneutral +622469193 +smtp +span +version240 +body +400 +in +142952 +toner +excitingbr +transitionalen +have +your +webmasterelmosweltde +breakdown +are +is +o154cymb019687 +communication +romanstrongl +his +riskfontbp +xasgdebugid +companies +19 +inscriptions +account2 +cdrom +volt +subject +difficulty +custom +relay +shd438rhmifqkwppe28qwdqq8aisf4uuubzbwm3ihoifqkwqjqet6jrhlb7jzvkzs +h5be +5h4qlezkdc6xjxbmkei4fm0cbu3nd73bnb93gaurj0bdtwrhwckpqvaictkfnitd6x +2002 +size1a +euhjcw1moamdnrrepit3cvvltsqj8x7plceidth0aqco2wak2ir9far6lmyg83djqez4oths +pattern +guide +asplcid +thought +yes +jmjmasonorg +but +state +o19mr8724753bkt981272794274550 +width95 +incrementstrynameali +gtosatosericolcom +2010div +contenttexthtml +would +eastern +deaths +online +071955 +center +colonial +boston +world +pennsylvania +problem +our +115ee +blues +o1yxiktbukqep0pebg1dwljjklptogzb78nzvioqlyrxblszap9napyzg2sd8ytvzhiwriz1 +100irigir +urncontentclassesmessage +2009h1td +all +resses +109i106 +citizenship +cpunks207200564 +123225 +deliverydate +hrefhttpccprodrovingcomrovingccprivacypolicyjspprivacy +host +data +opportunity +set +213049 +cellspacing0 +multipart +edt +height3d2td +just +height3d258font +us +it +esmtp +reading +charsetkoi8r +angela +1wenmoupsvc5flxzhgtxnei9os7tyditpdzgfetooxmzzmftvwp9itlb77tjqpcmpk831up8a +bc +a +calls +deliveredto +e17mr1341787wfa121272219590268 +xmessageno +sweeten +ueogilemy6523handlerthayercom +hibodycsminingorg +6500 +35 +xbitdefenderwksspamstamp +bsfsc3mj3053 +localhost +introducingnatures +availability +tools +17p6ti0008gi00 +lytton +rule +messageid +height48 +tommy +contentclass +10 +xasgtag +especially +removed +yoursquore +above +httpwwwbarracudacomreputationip89115128149 +public +london +refusing +2001 +taglevel10000 +nict +far +im +031452 +127001 +3d20 +burned +49900 +align3dleft +facilitated +midlands +dietdestro +spamassassinsightings +score268 +arm +to +half +charsetiso88591 +least +singledrop +exciting +oguqij +code +you +assayers +td +company +offshore +80 +mogu +sign +color3974b4recruitmentafont +pictures +font +israeli +internet +valueoregonoregonoption +there +color3dffff +name3dgenerator +go +id +meta +qto3deb101e14f0be4a1ffdf44791unmyt3dhibodycsminingorgivqyh3d97001902 +pts +over +alcharbfont +on +strong75 +partners +htmlmessage +tr +click +3m +mailtobm7btamailnetcnsubjectremove +date +happens +xresponderid +31 +jalapeno +p111sitiva +39vy5d04c1c8uampfueyruy3d054849576934834378135993 +name3dresetce +allow +hope +meaning +whatcom +10pxsubscribefonta +xuidl +shiites +usa +by +unsubscribe +antispam +enjoymentthe +via +link +528 +1268446624079dad400001w4twrl +figaso +color3dff0000417fontfontdivtd +p +uid71801201487268 +moment +ksullckxfy1bjplhoae9ztlevm7gkulfo1zttwb3srt7o9am8ptjcmpoe41vwwpwtuyjqw7o3 +with +from +dwl +195822 +means +rohit +tn45kzk2bkvtfosszptwcyfnebb1u7oboqjfrn5xqunuxasuksvgzshff4hiexhtmaotvh0 +x18mr28702256qbi1196782044591 +mimeversion +tjinl2yo2msgylve52pqfq7ntzvywbgfllsrcv2yqvmraapm46yx0lsbv83dsxcery5os +tdtd +significant +16px +000308 +70 +pieces +realtime +express +first +true +episode +alignrightpeople +life +backgroundcccwidth3pxoverflowhiddenfontsize1px +dateline +xmozillastatus +81 +address +nature +1900 +zjr9fdjdj8rjiysmtxcon7rfx3rqzqfmvpkhrmsjwse15ghhrhdujym9086kdohf4klcb +spanstrong +gw1csminingorg +ldtuubea8u5s1ldgniblmb6o7ms0txmtmdwufb9q0h3bwet7qonznja3vxmnuk1zbxftine40q +jmasonorg +british +fetchmail590 +dustbrbrwe +xaccountkey +message +tip +japan +dr97wn +dedication +say +steps +differs +118 +experiencing +tomorrow +g8jm4qc05575 +goes +records +p111tete +efficiencythe +starred +texthtml +src3d304704 +always +waters +sites +special +w3cdtd +herelimited +97 +testsbsfsc0sa148a +m111t111rcycle +fontsizeadjust +valignmiddle +merchant +hd +mail1csminingorg +matches +tahomabr +liberal +current +fri +chegam +federation +xoriginalto +about +size3d2bplease +m0620212mailcsminingorg +105 +family +cens117s +see +n10kmhp10msncom +12001600 +0px +no +sat +mailings +maps +ufont +013 +name3dcity +great +sansserif +contenttype +user +may +srchttpimg8myimgdedygafabujeuy3cd2fjpg +listed +of +csminingorg +received +breaststroke +today +19th +061156 +helod +the +176 +parts +height23 +never +religious diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_95.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_95.txt new file mode 100644 index 00000000..ce10382e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_95.txt @@ -0,0 +1,276 @@ +175537 +pm +rubes +economy +mailtoobjclanguagerequestlistsapplecomsubjecthelp +trouble +examplecom +oldreturnpath +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +avast +program +pan0132 +sinn +smtp +httpwwwweblognohairnetcurrent +width1img +adding +prevails +any +years +keyboard +clientip8219575100 +this +10953965 +tomwhoreslacknet +in +have +your +mf +pwrite +reported +returnpath +is +httpgmaneorg +043231 +testsbayes002 +mozilla50 +162233 +examples +subject +qmqp +httplistsdebianorg20100506120748578martinlichtvollde +am +listarchive +system +2002 +xsl +listunsubscribe +233053 +203208 +6699a27d37a6c +11 +below +remember +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +another +certainly +exmple +r29mr3011418fgk61272404251126 +ist +heres +if +gmailid1288d628fdf7eb7e +sthomasapexvoicecom +apr +gldudebianuser2mgmaneorg +own +required53 +all +tue +rest +usernamenobody +hrefhttpwwwamazoncomexecobidosasin0966103254refaselockergnomeget +cellpadding3d0trt +hrefhttpclickthruonlinecomclickq56nwkyinns9s0vbferctpqdwwamc9r +5505213a5743 +exim +0400 +esmtps +8kbfontbr +13 +edt +it +sun +esmtp +httpxentcompipermailfork +87iq6qwh1xfsfmerciadrilucastationmerciadriluca +fix +pwfqbwgxpbeybicaa3rzmx0ft0nbmjgiiwvnmxooitzacbkfffg63jmq7cn8813vt8 +a +flakiness +deliveredto +arsasha256 +mean +resentsender +create +v17cs34310wfj +localhost +localhostlocaldomain +useragent +as +dogmaslashnullorg +hrefhttpclickthruonlinecomclickqb8smr2qqusxvklh5dlq9sp8qkei9r +has +messageid +10 +signed +stir +675216 +0700 +xspamstatus +ea69616f03 +19412513010 +or +computer +patenting +contact +0200 +contenttransferencoding +reason +textplain +invoked +i +127001 +outlook +215326 +8bit +whatever +that +listsdebianorg +to +809122912 +supplied +singledrop +you +know +filter +cyrusportalurzuniheidelbergde +extensions +references +burrows +dir +at +ontpageauthors166gif +qq1iwrsyurtgxmazwmgcsqabxjazdtat5s4stivalceosogcflrrvzikuddun4xdhxmqh +debianuserlistsdebianorg +lughtuathaorg +162224 +080445 +id +e4244440c8 +listhelp +on +general +2076920025 +designates +apply +date +solaris +friends +distupgrade +69 +7712717777 +cohen +exmhworkers +authenticationresults +prefixbrbhewlettpackard +googlecom +xuidl +range +by +en +hrefhttpclickthruonlinecomclickq8cld7aq6jbhmwfvk2vfm8il40awnyr +unsubscribe +smtpmailbouncedebianlaptopmlsubscribertechcsminingorglistsdebianorg +gmexim +xmailinglist +new +0100 +exists +autolearnfailed +with +from +gnome +1alnuq0007hv00 +objclanguagelistsapplecom +mimeversion +094418 +outgoingsecurityfocuscom +114007 +include +lot +address +listid +02 +charsetusascii +tviranodv3daueezii5lbliszt +regards +sender +end +precedence +unfortunately +message +editors +which +target +an +rpm +7bit +vulnerable +taggedabove10000 +httpradioweblogscom0101969 +xbrightmailtracker +videos +b6d2d13a4a03 +httplistsdebianorgaanlktikfcrd13r2hral2uiws1ochlzwijvkcmgkeybmailcsminingorg +always +million +ronljohnsoncoxnet +were +version250cvs +215 +telecoms +xloop +johannesphysikblmtumuenchende +spamassassintalk +mailing +instincts +hrefhttpclickthruonlinecomclickqf4yxeq0skawald8uuj5qddfwmtyr +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +about +cancellock +10143348 +8219575100 +inspiration +see +debianuserrequestlistsdebianorg +dimension +no +very +approved +archive +writes +important +x9milab416c6m4q5qmwzrmdw2vsd2zj5jimo16nsfabbbiqzlesc2kynzoetrrb9mkg +k +17128113151 +bdigicams +hrefhttpnewslettermediaunspuncomhttpnewslettermediaunspuncoma +verified +when +usemap3dremembermap +like +won +write +other +of +httpwwwcomicscomcomicsdilbertimagescleardotgif +havent +and +naa13251 +received +setup +k3b +aboard +list +16144776724 +2459 +search +the +wisconsin +234110 +those +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_96.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_96.txt new file mode 100644 index 00000000..43f65e47 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_96.txt @@ -0,0 +1,304 @@ +smbfs +rate +particularly +cwapkb8hxi6tofhmwfs5qnh3qks76coxc0gi +g975xrf15813 +clive +please +trouble +able +for +postfix +experimental +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +duty +test +gnupg +admin +begin +clientip8219575100 +tomwhoreslacknet +in +have +i8qhkkfwmgov +returnpath +imap +klondike +success +guardian +cheers +are +source +why +101431608 +testsbayes002 +height110 +server +subject +height3d1tdtr +qmqp +size +floppy +carried +httpwwwhypothoseefilmscombloggerhtml +research +rpmlistadminfreshrpmsnet +listunsubscribe +disk +0000 +best +domainkeysignature +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +212173515 +080603 +systems +if +filho +g7s9t0qq018215 +up +b298b13a42d9 +gmailid128094367a98715b +so +clear +our +normal +keyframes +newsletter +do +tue +lairxentcom +eagerly +domain +present +4bcc83d1308freefr +youre +users +data +0400 +somewhere +mailtoforkrequestxentcomsubjectunsubscribe +else +it +esmtp +103740 +893893 +a +deliveredto +newscientist +khare +xmozillastatus2 +225131 +soulbrainstormpreservation +localhost +2011 +as +spfpass +load +messageid +enlt +signature +10 +0103 +mime +especially +dkimneutral +0700 +exmh199405358p +seems +httpslistmanredhatcommailmanlistinfoexmhworkers +contact +such +signaturelongsparsespamphrase0001 +xrcspam +2007091301 +gmail1 +exactly +contenttransferencoding +18si11056023fkq420100427083749 +fatigue +ma +segment +textplain +i +127001 +8bit +central +that +virtual +rpmzzzlistadminfreshrpmsnet +listsdebianorg +to +become +half +least +singledrop +speaking +now +20080610 +tlsv1 +you +utc +e1r86ercuf9a +keeping +em +home +various +19972002 +mlsubscribertechcsminingorg +was +inreplyto +mxgooglecom +c1 +nothing +taken +at +1 +through +mailtox11userslistsapplecom +yyyylocalhostspamassassintaintorg +cipher +debianuserlistsdebianorg +wolf +id +flashfree +someone +folder +doesnt +woody +need +can +on +yesterday +130305 +reports +xmsmailpriority +date +hasler +jalapeno +terrorist +processor +bug +perms +googlecom +them +things +cccontenttype +by +either +lisztdebianorg +however +edmqy5yx93b40axtkningtlzrvi4ros6w4a +listmasterlistsdebianorg +thats +xmailinglist +story +ldowhitelist5 +sound +phil +xbeenthere +0100 +autolearnfailed +preferences +listsubscribe +character +ldowhitelistsaremsgidlong45 +with +from +generated +avril +20 +cnofws +1alnuq0007hv00 +mimeversion +enht +next +spamassassin +httpmessengermsncom +forget +content3dnocache +baby +include +media +correspondent +xcheckerversion +pass +hrefhttpwwwlockergnomecomsubmithtmlsuggest +use +news +fetchmail590 +of20 +archivelatest575328 +sender +precedence +whitelist +whole +didnt +compensation +ripperabr +225122 +score7 +original +susceptible +absolute +imposing +taggedabove10000 +be +listpost +xpolicydweight +153950 +into +mailtospamassassintalkexamplesourceforgenet +filemapc843 +6416122236 +0026 +better +more +hp +xloop +sa +thinking +7a28a43f99 +1987 +xoriginalto +577499 +iso88591qskytte4 +lpctipmske3h +lines +com +3 +mike +testscapinitldowhitelist +arrival +does +when +httplistsfreshrpmsnetmailmanlistinforpmlist +policy +contenttype +xmozillakeys +xrcvirus +musician +wholly +of +dialog +and +received +windows +9331a2d0d2a +free +list +days +created +phobos +rootlocalhost +marking +apps +a8njep1oizia10 +084351 +the +lnbbljkpbehfedalkolciefebcabtimonecomcastnet +op +cant +those +d +only diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_97.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_97.txt new file mode 100644 index 00000000..3071240c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_97.txt @@ -0,0 +1,287 @@ +p2pgifccode3dafebb3c4pcode3def39755c +valignmiddletdnbsptdtd +trouble +oldreturnpath +capabilities +xentcom +for +organization +postfix +2139621475 +001 +arguing +smtp +bodyhtml +hrefmailtoerikvanderkolkyahoocomerikvanderkolkyahoocoma +x +ended +deb +resentmessageid +account5 +this +autolearnham +in +margin +yyyyexamplecom +w35mr1697902fgc491273160033232 +your +returnpath +imap +rssfeedsexamplecom +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +l04local +testsbayes002 +mozilla50 +show +french +useless +subject +valign3dtoptdtable +got +mailtosecprogsubscribesecurityfocuscom +assertunrecognized +libsdl12dev +listunsubscribe +quotedprintable +debian +9 +root +101622 +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +212173515 +setting +ist +stake +heres +blogging +neither +fallback +required53 +etc +messaging +thu +pt +do +domain +090803 +662186695 +replicas +releasedbr +receivedspf +classffont +w40mr2039810fgk201270896935746 +0400 +httpcvemitreorgcgibincvenamecginamecve20100638 +before +esmtps +mailtoforkrequestxentcomsubjectunsubscribe +httpwwwimplodeblogspotcom +just +required40 +325 +esmtp +reading +ca +listsdebianuserlisztdebianorg +a +chest +calls +deliveredto +hrefhttpwwwwincustomizecomskinsaspflagsskinid1953library3filekidsworlddxthemeplg +amendment +d105013a48ab +wrote +localhost +varlogxorgxlog +lancettbellsouthnet +as +dogmaslashnullorg +spfpass +pdt +5 +has +messageid +204709 +10 +unknown +0700 +xspamstatus +gnulinux +144 +or +looking +scanfolder +command +httpwwwmisdruknlmt2 +xrcspam +m8cs85026wfj +problems +mozillathunderbird +required50 +im +textplain +127001 +that +76966217 +kdiff3 +af6decorecpp305 +contenttransferencodinginreplytouseragent +to +v1 +xlevel +now +you +utc +probably +craigdeersoftcom +stores +eclecticklugenet +listmanagershagmailcom +was +honoured +inreplyto +six +references +sansseriffont +21078 +6 +quoteish +taken +through +debianuserlistsdebianorg +hangs +electronic +broadest +id +engineer +also +port +listhelp +httplistsdebianorg20100510093021107e739fpbmihamalagasycom +on +designates +yyyylocalhostexamplecom +mailtojavadevrequestlistsapplecomsubjectsubscribe +how +language +t24mr533040faa901271905636394 +cost +moodle +date +apache +jalapeno +passportfontabr +terrorist +wondering +mail +ve +mhsetcur +by +think +however +new +few +ldowhitelist5 +xbeenthere +0100 +september +their +wyb29 +not +listsubscribe +httpwwwchallengemycomnaelweblog +with +rpmlistfreshrpmsnet +from +kanru +mimeversion +enht +giants +1011416439 +4302010 +first +iso88591qcamalef3n +unsubscribefont +width3d169 +futuro +james +xmozillastatus +could +use +listid +words +tips +sender +07 +crap +message +anyone +almost +apple +an +original +be +listpost +xpolicydweight +detailed +removing +ddownloadonly +time +gmailid1283552b2d810aac +logmaneorg +215 +xenigmailversioncontenttype +more +headericsminingorg +spamassassintalk +bgcolor3deeeeeecenter +gbr +interdisciplinary +hd +81168116 +sgamma +flinched +start +httpwwwhotmailcom +upgrade +srchttpwwwcnetcomiglnewccgif +manual +fri +oneself +archivelatest1272 +mutt125i +many +call +against +no +archive +important +intermail +some +policy +marketplace +bcm4310 +contenttype +other +of +since +away +and +socketmode +testsawlhabeassweinreptoquotedemailtext +record +received +one +list +o +relating +phobos +cases +delimiter +222922 +onestop +directory +the +8exborderleft1 diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_98.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_98.txt new file mode 100644 index 00000000..a0076a9f --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_98.txt @@ -0,0 +1,268 @@ +spamscore0 +colspan3d2img +height10tdtr +regarding +application +trouble +oldreturnpath +qmail +335 +read +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +dst +mailtoquicktimeapirequestlistsapplecomsubjectunsubscribe +knowing +monitor +people +xmailscanner +formatflowed +any +website +body +clientip8219575100 +this +lives +autolearnham +in +090407 +cached +returnpath +jul +mailtodebianuserlistsdebianorg +is +1087698 +his +why +dbus +00000000 +feb748e58cb638dd8a8212d7fd17ee93 +saremsgidlong45 +fam +subject +qmqp +40 +am +xscannedby +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +mask255000 +included +2002 +listunsubscribe +but +11 +0000 +below +15913411818 +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +disorderly +domainkeysignature +would +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +hrefhttpclickthruonlinecomclickqd9b7lqppwbyid70nie23s1n87hjlr +daniel +ist +y +apr +oct +l12sm8003177fgb720100517120643 +snogged +appear +backups +domain +macintosh +users +lmtp +receivedspf +result +cellspacing0 +0400 +confident +325 +it +esmtp +a +newer +deliveredto +xmozillastatus2 +scaling +resentsender +wrote +receive +localhost +2011 +as +dogmaslashnullorg +spfpass +pdt +17604 +messageid +portrait +signed +isnt +originatorcaller +mgbdebianyosemitenet +phoboslabsnetnoteinccom +zk77dlaxe0dzrcepyz5gy1zzszjiubfsrdkodishn8nlhgztg4cdkzjjiciyvmsc9inwqm +care +or +calculating +03 +referencesdatemessageidsubjectfromtocontenttype +contact +divxbouncekriegermailshellcom +xrcspam +2007091301 +required50 +gary +userid +mailtorazorusersexamplesourceforgenet +textplain +groothuis +invoked +127001 +doc +ps +rv11b +displayduration +that +out +opened +gmt +to +charsetiso88591 +now +fan +code +20080610 +you +know +bank +tcltk +expats +options +amavisdnew +mx1redhatcom +052626 +mlsubscribertechcsminingorg +was +week +version +believe +httpswwwinphoniccomrasprsourceforge1refcode1vs3390 +at +debianuserlistsdebianorg +id +apt +listhelp +metzmeier +aaaaaro2goi +itggwcr008cwtesatnet +check +how +vital +phpregistervariableex +date +69 +jalapeno +cohen +easy +contribute +paradisebeach +solitaire +by +mailtoobjclanguagelistsapplecom +xmailinglist +2525 +gmailid1286d8086bac763a +xbeenthere +151732 +httpwwwmathudeleduschwenk +not +with +from +stuff +20 +width150tbodytr +gnome +mimeversion +enht +heaven +20100502214826428modestasvainiuseu +anymore +spamassassin +2 +002055 +session +listid +charsetusascii +report +xaccountkey +sender +end +precedence +message +file +101428120 +errorsto +under +control +score7 +smokejohncsminingorg +taggedabove10000 +emacs +be +g7smimz12443 +get +iso88591qktgkr4au3mlabd5ccec02grfvlzexk7mzccv2wzf6ojezn41d5vtqg0a090 +always +time +zealot +install +wallace +xloop +19325219193 +x012sender +billing +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +manual +c2d8c13a4988 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +prepares +hrefhttpwwwgnometomescomtome000627htmlpc +xoriginalto +about +xuid +8219575100 +kde +11si5587497pzk6320100426032111 +against +no +very +we +konq +archive +d4vzgbn8ast7 +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +known +contenttype +possible +may +sprains +other +of +distance +non +dmmo +received +investigators +one +posted +nt +1o4mdc0005ma37 +dear +phone +drugs +parts diff --git a/Figaro/src/test/resources/BookData/Test/TestEmail_99.txt b/Figaro/src/test/resources/BookData/Test/TestEmail_99.txt new file mode 100644 index 00000000..6078e348 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Test/TestEmail_99.txt @@ -0,0 +1,263 @@ +height3d11 +163711 +45 +route6413112636telocitycom +exploit +4c9f529424a +m8cs97679wfj +dangerous +mb +read +xmailer +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +find +smtp +src3dhttpwwwtributecanewsletter48imagesreelmainnextc17gif +crashing +felate +admin +titlecnet +years +this +63704 +autolearnham +have +reported +returnpath +imap +mailtodebianuserlistsdebianorg +teledynamics +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +why +wait +course +width400 +00000000 +subject +techcompanycom +got +am +contentdisposition +2002 +size1bcnet +listunsubscribe +jmjmasonorg +0000 +jmlocalhost +id8dbqfl6ayshkednpy5oouraqxbaj9gposb7cclj8awejm7zkxde9raiqcfcrzv +m8cs53556wfj +what +different +yyyylocalhostnetnoteinccom +xucsccatsmailscanner +owned +forkxentcom +applicationpgpsignature +ist +line +if +kfrontpagebgif +mailsmtp +problem +fallback +required53 +up +svenjoacgmxde +do +lairxentcom +primarily +u +data +37 +090112 +just +325 +it +size3bsign +esmtp +a31b02940d3 +listsdebianuserlisztdebianorg +width13 +cd8742940a0 +a +etccrondailyapt +width50 +deliveredto +src3dhttpwwwtributecanewsletter48imagesreelmaincontestc09 +httpwwwkeyspgpnet +xmozillastatus2 +localhost +twice +fa85215843cda1db4 +as +messageid +10 +proprietary +0700 +155747 +204506 +155009 +fastest +backward +aaaaaa +0200 +al +forkadminxentcom +2008110401 +age +textplain +i +127001 +cards +19412514545 +that +out +to +v1 +archivelatest575822 +singledrop +dvd +20080610 +tlsv1 +you +know +utc +meant +auth02nlegwnnet +easier +mailtoforkexamplecom +inreplyto +references +global +1309496247 +at +yyyylocalhostspamassassintaintorg +go +xspamlevel +id +imprononcable21 +listhelp +8a2c013a5023 +on +17 +same +023b813a4e9f +how +b +tnonsensefrom7080tnonsensefrom8090 +174549 +date +hrefhttpclickthruonlinecomclickq0b0niinq4zfrgyvagftmzctvxyhfr +marginal +tired +2010 +zzznetnoteinccom +timely +authenticationresults +googlecom +xuidl +envelopefrom +by +language3djavascript0d0d0dfunction +helpful +unsubscribe +5so5203809pwj6 +42 +2525 +much +0100 +not +height1td +with +from +gnome +spamassassintalklistssourceforgenet +mimeversion +enht +wsc +configuration +doj1ixqm4cb +rpmzzzlistfreshrpmsnet +spamassassin +first +xmozillastatus +listid +phenomena +news +available +20100511055911ga17757europaoffice +sender +iraq +message +seem +which +unwitting +helveticafont +air +dgxt +7bit +taggedabove10000 +be +listpost +20th +aenag6b0jci4a10 +attack +youd +into +quick +flash +her +123302 +were +xvirusscanned +version250cvs +usseal3bkp9h +sort +xloop +7 +mount +xauditid +funtionality +81168116 +really +many +according +8219575100 +ncsnfpmexvk +no +very +nffmxsmnmidar4beqwulbliszt +mailtorazorusersrequestlistssourceforgenetsubjectunsubscribe +intmx1corpredhatcom +e288a8 +when +xgreylist +httplistsfreshrpmsnetmailmanlistinforpmlist +muted +version325 +1000 +contenttype +xmozillakeys +xrcvirus +possible +like +other +of +myopia +log +and +xstatus +designs +debiankderequestlistsdebianorg +received +florence +free +lawsuits +phobos +112712 +084217 +hoping +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_0.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_0.txt new file mode 100644 index 00000000..b212e914 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_0.txt @@ -0,0 +1,282 @@ +sweet +width85 +moutngkundenserverde +please +for +postfix +1o32xq0003cgnu +encodingutf8 +lots +involved +12 +people +resentmessageid +clientip8219575100 +this +bhmwcfwzcjcosnbymvu4qxusb8rat28wpzn6vncotsy +00020 +cached +have +berra +your +reported +returnpath +1721652254 +cheers +is +learn +why +6219710167 +companies +00000000 +subject +4 +pregnantlooking +using +8exborderleft1p +listarchive +real +mov +2002 +xmailmanversion +rescanline +listunsubscribe +quotedprintable +disk +david +counterparts +httppackagesdebianorgunstablemainpythonflask +srchttpgservcnetzdnetcomclearoutboundgifappid2emid25136487nle440issue20020710 +standard +but +11 +091121 +jmlocalhost +security +typetext +201 +elsewhere +savviest +forkxentcom +penguins +recent +return +hda +7ex95b1ifjda0pe9rwylbliszt +if +unsubscription +tool +216136171252 +up +human +uses +so +080156 +a4fd116f70 +111752 +svenjoacgmxde +friend +ac90 +gnus +gya1 +thanks +50 +anthropogenic +re +assuming +tcpip +0b32f43f9b +afraid +result +134625 +set +dcsminingorg +required40 +it +xoriginaldate +esmtp +needed2 +listsdebianuserlisztdebianorg +20100512194713123modestasvainiuseu +db +a +deliveredto +v20mr325089fab431271372650326 +102809231923 +xmozillastatus2 +realization +localhost +2011 +localhostlocaldomain +subscribe +as +dogmaslashnullorg +96c805179a3143a8be1b1de74217c166csminingorg +size3d1img +spfpass +messageid +10874936 +continued +bzzzzzzz +phoboslabsnetnoteinccom +0700 +style3dmargi +incidents +whi +testsawlknownmailinglistnospamincreferences +suggestions +fgout1718googlecom +375 +contact +agent +they +flavors +080548 +forkadminxentcom +pgpsignature5 +2008110401 +1931201713 +open +sullivan +i +127001 +saou +19412514545 +that +to +charsetiso88591 +xlevel +you +web +bank +tribes +alias +most +mlsubscribertechcsminingorg +omniweb +was +resentdate +editor +week +inreplyto +ldapdb +references +hughppatcommediacom +pop3 +thread +gmailid128aa41a82d2c15c +id +port +driver +need +on +designates +wants +how +date +xhabeasswe7 +192733 +2010 +artists +me +googlecom +xuidl +envelopefrom +justin +by +checks +listmasterlistsdebianorg +helo +xmailinglist +will +famous +xbeenthere +back +historya +0100 +xamavisstatus +says +listsubscribe +with +from +rohit +1200 +nomore +mimeversion +0500 +heaven +gets +151621 +inc +guts +spamassassin +stopped +2 +httpslistmanexamplecommailmanlistinfoexmhusers +use +listid +fetchmail590 +sender +end +creating +precedence +mr +1984 +101428120 +depth +say +an +original +turk182chipwarenet +7bit +courts +ill +listpost +ranged +rights +latest +xvirusscanned +thousands +72bca1c38a +r +buddhism +81168116 +manual +httpwwwhabeascomreport +a25si15961158faa6820100518084353 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +parentage +xoriginalto +translation +httpthinkgeekcomsf +width375 +debianuserrequestlistsdebianorg +word +we +065925 +religions +wrongdoers +frommxmatcheshelodomain2 +reasons +group +contenttype +extremely +possible +descbc3sha +may +other +of +xstatus +record +received +233423 +r5e8c72pf0ycxet6sqc +wed +later +list +mailtoforkrequestxentcomsubjectsubscribe +pgpsignature +directory +broken +the +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_1.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_1.txt new file mode 100644 index 00000000..ae2b9c75 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_1.txt @@ -0,0 +1,283 @@ +9645519711031163664828javamailrootmonkey +661451366 +lists +trouble +oldreturnpath +for +postfix +encodingutf8 +maybe +find +544 +smtp +firewalls +bonkers +xmailscanner +years +resentmessageid +colspan2img +this +110636 +fraction +have +off +returnpath +imap +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +why +105203 +issuesbr +subject +protected +originated +qmqp +am +contentdisposition +listarchive +blogspot3 +2002 +bulk +tdfont +rpmlistadminfreshrpmsnet +rons +email +stare +usenetdoughgmaneorg +listunsubscribe +quotedprintable +but +11 +debian +co +jmlocalhost +onbr +security +degree +yyyylocalhostnetnoteinccom +online +daniel +j +systems +hosted +own +fallback +required53 +up +httpwwwaccucastcom +contended +roi +22 +20100508090848771mgbdebianyosemitenet +083834 +testsdnsfromrfcwhois +roundabout +run +exim +0400 +1287 +xmimeole +intended +esmtp +minutesfont +httpxentcompipermailfork +impact +listsdebianuserlisztdebianorg +two +failed +a +dont +deliveredto +khare +messageidreferencesmimeversioncontenttypecontentdisposition +wrote +became +localhost +useragent +m4l8s +as +dogmaslashnullorg +transit +criminal +build +airkvbgf3v46fcoh1nyafmw17 +training +pdt +messageid +matt +500 +m +10 +dog +221108 +particular +or +reimburse +garden +command +xrcspam +following +monique +contenttransferencoding +forkspamassassintaintorg +color666666 +nmh104 +chemistry +textplain +i +127001 +8bit +httpwwwnewsisfreecomclick48723998215 +out +jacobs +sending +sex +linkb +to +murphy +existing +supplied +8725d16f03 +xlevel +now +you +know +auth02nlegwnnet +113800pm +mailpz0f178googlecom +most +mlsubscribertechcsminingorg +help +version +long +references +there +mxgooglecom +found +change +6 +httpwwwisilocominfobetaisiloppchtm +doing +williams +yyyylocalhostspamassassintaintorg +go +id +jnlp +vanderkolk +listhelp +twofinger +processes +can +on +common +153745 +140510 +06 +designates +complete +date +kmail199 +take +eyes +signally +31 +april +who +2010 +authenticationresults +xuidl +envelopefrom +submitting +manager +by +i2laee79c571004070434pf2f6d34cq57be4fc100f60309mailcsminingorg +think +cs144080pphtvfi +unsubscribe +facearial +even +clarify +cascade +ago +e15si17739102fai7520100519034950 +new +management +much +0100 +not +httpfortytwochgpg92082481 +listsubscribe +disclose +with +from +cnofws +enht +stations +infjpbqfhyap +bindustry +errata +hits6828 +xcheckerversion +pass +81 +tnonsensefrom4050useragentuseragentmutt +details +listid +smiling +charsetutf8 +charsetusascii +fetchmail590 +v11 +size2 +end +testsalltrusted +its +apple +234344 +rpm +forged +listpost +russell +large +bgcolor000000img +rights +6416122236 +enough +192168110 +case +mailing +gmailid128afc0045de029b +httplistsdebianorgpan20100511215248csminingorg +intercourse +contains +xoriginalto +pcibiosbiosirq +3 +20020201150022b11472cshelsinkifi +no +dhersaaes256sha +1881416f03 +notinblnjabl15 +some +20020918 +0l0k003z6uku1090asmtp026maccom +contenttype +dpoboxcom +like +gecko20100317 +m8cs38513wfj +of +xhabeasswe5 +since +and +xstatus +received +wed +trademarks +hrefhttpwwwimakenewscomeletrachangecfmxmediaunspun2czzzunspunspamassassintaintorg2ctxtclick +goiubntfkszcurfbnwn9qz01xmfft2kidgplm +wish +v1410 +affects +the +ubuntuites diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_10.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_10.txt new file mode 100644 index 00000000..587c333c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_10.txt @@ -0,0 +1,279 @@ +httpwirelesskernelorgenusersdriversb43supportedchiptypes +webstuff +please +valigntop +trouble +xmailer +hdakecmtpl8rj7n45mxmu8qyzirsiabhro4dm +for +postfix +mailservice4imakenewscom +smtp +08 +c00112d0c59 +electric +inbox3448 +prevails +resentmessageid +this +in +certificate +htddw840abr +women +contenttypemultipartalternative +priority +returnpath +are +is +learn +hits56 +bit +subject +lesserpowered +produced +listarchive +2002 +331b31c6b5fab +bulk +micalgpgpsha1 +enig46ffd16c11dad5966d847622 +email +22sm2126438iwn020100405221301 +quotedprintable +lbertarchiveimagesdilbert2003482820710gif +ckloiber +interrogated +but +11 +0000 +72b2313a537a +what +remember +212173515 +forkxentcom +blue +xcmscore +launched +ist +unsubscription +1505 +own +required53 +up +so +disclaimer +httpwheniridemybikeunitedsituationde +mikemichaelmoorecom +hand +200209261527g8qfreg24218dogmaslashnullorg +hits121 +re +mjvc2067wvqk +assuming +18b4429409a +281726 +receivedspf +satalk +lockergnomecom +set +0400 +336 +esmtps +10300283774901tmdadeepeddyvirciocom +interviewed +override +required40 +325 +it +esmtp +seesaw +local +asiassa +entire +a +hrefhttpclickthruonlinecomclickq3doi9nibz4iutiayrvwlgwutnantpr +deliveredto +khare +0pxoff +isoiec +wrote +localhost +formally +useragent +as +razorcheck +490a93ed5c +pdt +has +messageid +pfont +signature +0700 +airkvbgf3v46fcoh1nyafmw117 +particular +or +seems +yourself +resentfrom +looking +vivendi +uswsflist2sourceforgenet +perl +xfelkmailscannerto +problems +forkadminxentcom +2008110401 +im +rescue +textplain +mimeole +i +depending +127001 +gaqgrsw11496 +classifierspam +work +wheres +that +out +0h3700ivfjmhs3mta7pltn13pbinet +to +singledrop +you +6621866216 +worrying +know +utc +103113 +around +chat +here +these +mlsubscribertechcsminingorg +recovery +easier +was +version +references +hash +at +1 +been +older +often +262621lenny4 +id +someone +m8cs29630wfo +port +listhelp +forkexamplecom +same +lawyers +unverified +government +my +gain +date +19317254 +score668 +wininsider +easy +115656 +gap +painted +rob +me +them +envelopefrom +by +client +886882 +helo +will +width75 +2525 +xbeenthere +0100 +eek +their +not +listsubscribe +score110 +with +from +means +books +mimeversion +4302010 +vcmfc +aanlktiloguohoz10mtb5iggjhseh0ohpvihesak7gcmailcsminingorg +session +pass +bukkake +xmozillastatus +listid +jmasonorg +charsetusascii +17wx8n00031d00 +sender +107181 +file +its +pda +be +listpost +considered +where +4bbca70f5090401pixelturecom +ron +time +104411 +version250cvs +egovernment +ns2egwnnet +nyu +ion +3d6e73479090304sricom +install +slrnhsjcc4bu7liampotooledipsyselfiporg +hits182 +debianuser +81168116 +concerned +delivery +rvp +8219575100 +46bff43f99 +123812 +no +archive +writes +does +enus +when +great +httpwwwgeocitiescomkindlyrat +634554 +z2saae88f271004301657m1cbc6552nf528f7446c7572ebmailcsminingorg +contenttype +xmozillakeys +width5tdtd +datwinkdaddy +like +ensuing +of +and +xstatus +releasing +received +device +repos +list +mailtoforkrequestxentcomsubjectsubscribe +phobos +month +pan20100507181232csminingorg +the +those +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_11.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_11.txt new file mode 100644 index 00000000..7520b83a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_11.txt @@ -0,0 +1,270 @@ +hrefhttpwwwfirststopwebsearchcomlockergnomeofferhtmlfirststop +subhellouseragentuseragentkmail +used +reference +notify +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +question +httpwwwoperacommail +httpuberbinnet +body +in +cached +have +needed +yyyyexamplecom +your +returnpath +xessionerrors +imap +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +why +melrto6wanadoofr +x8664 +benchmarks +1921683267 +useless +subject +less +contentdisposition +listarchive +included +2002 +xmailmanversion +heads +email +listunsubscribe +4be972907010109comcastnet +113225 +shortened +jmjmasonorg +0000 +below +jmlocalhost +jump +hoeppner +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +specifically +diet +ist +etcaptsourceslist +srchttpwwwzdnetcombgif +210016 +apr +gldudebianuser2mgmaneorg +required53 +xapparentlyto +etc +so +valuehtdiginput +interact +archivelatest575913 +size2table +book +re +cest +m8cs60639wfo +exim +before +eecsberkeleyedu +just +required40 +esmtp +egypt +listsdebianuserlisztdebianorg +g8ohbni01193 +a +dont +deliveredto +wednesday +resentsender +issuer +045935 +localhost +as +talk +spfpass +pdt +messageid +mapping +10 +href3dhttpclickthruonlin +dog +coming +unknown +unstable +0700 +old +103726 +or +resentfrom +httplistsdebianorghscb3af9f1doughgmaneorg +they +xrcspam +contenttransferencoding +1a01d3ef12 +2008110401 +im +de +1022329156 +textplain +invoked +i +127001 +httpwwwglenbrookpartnerscomweblogsjl +replybot +hear +to +123856 +79ee316f16 +9029110 +xlevel +intellectuals +20080610 +rogers +amavisdnew +small +f20cs84061wfi +0025 +font +most +these +hardware +g8a8l1i24928 +was +forwarding +metronom +hold +mxgooglecom +8510226115 +iplanet +at +cover +yyyylocalhostspamassassintaintorg +boundarynextpart000000001c24a394307a140 +id +ads +g984ch428341 +mangled +listhelp +issuing +on +squirm +transmitted +libperl4camlocamldev +2xive +error +underscores +date +shares +solaris +healthy +admitted +googlecom +xuidl +mach +mail +by +belongs +should +42 +wwwlindowscommyths +openprintingorg +reimaged +0100 +autolearnfailed +not +listsubscribe +times +with +ce3darialverdana +from +generated +0 +20 +overa +3d9ccf545070909freefr +person +0500 +20020930t0033140600 +linux +knowledge +both +pass +xmozillastatus +nested +use +1222 +listid +mtainterface +sent +mai +charsetusascii +fetchmail590 +report +app +dav39lk9vdnw3yqogam00002a9ahotmailcom +sender +precedence +seem +its +194842 +ffffff +an +discussion +listpost +supplies +073033 +release +xvirusscanned +anything +power +slrnhsjcc4bu7liampotooledipsyselfiporg +mailing +httpbugsdebianorgcgibinbugreportcgibug3d366797 +xgmanenntppostinghost +a0a8 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +thursday +81168116 +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +fri +xoriginalto +pounce +started +10143348 +iso88591qivborw0kggoaaaansuheugaaadaaaaawbamaaacllos0aaaamfbmveusk +3 +mike +no +add +archive +bebusinessbbr +testsemailattributioninreptoknownmailinglist +jmrpmjmasonorg +contenttype +required70 +like +of +log +and +record +barebones +received +200 +wed +red +list +mailtoforkrequestxentcomsubjectsubscribe +recorderabr +month +gmailid1285056bffc4e4c5 +thinks +the +cant diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_12.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_12.txt new file mode 100644 index 00000000..b7a9f897 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_12.txt @@ -0,0 +1,290 @@ +ditch +826930110 +0300 +spamassassindeveladminexamplesourceforgenet +application +please +theology +xmailer +nametopa +for +postfix +063530 +12730920611953974camelfamilypaciferacom +mailtodebianuserrequestlistsdebianorgsubjecthelp +network +services +smtp +test +working +mac +body +resentmessageid +clientip8219575100 +account5 +this +quicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +have +your +returnpath +imap +choose +is +why +knows +testsbayes002 +greetings +00000000 +eda37003 +iso88591qvdjlbzrnbhqxeiu9smsqcwukmnmjitkdnaaccbw2sh1fxqk3ijrtknzsol +text000000 +subject +theserversidecom +using +listarchive +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +color000000worldwide +fairly +bulk +email +r10cs66423wfa +listunsubscribe +volatile +fergal +screens +sure +0000 +c565dabb5c +debian +root +what +freegonefee +forkxentcom +gmailid1285fa5e445783b4 +applicationpgpsignature +mcdonalds +problem +fallback +up +stands +xacceptlanguage +thu +all +twoway +reliance +unreliability +thanks +publishes +6621867201 +90e5e13a4f72 +741258351 +tls10rsaaes256cbcsha132 +look +extranet +anam +receivedspf +exim +0400 +manuallybr +mailtoforkrequestxentcomsubjectunsubscribe +edt +else +it +sun +esmtp +two +partition +mailtoleavecustomers949326kmailryanairmailcom +a +width50 +deliveredto +xmozillastatus2 +pause +pages +resentsender +localhost +futureproofing +as +dogmaslashnullorg +inline +messageid +204709 +10 +coming +unknown +0700 +errors +10tb +midiplug +yet +umgs +vidal +color +03 +agent +garden +laptop +fffhomea +partner +iso88591qitzpxyncnmw1x62g9y2gufkejxo0a09niylkgpxnrhhecjmyuaujmnd +far +forkspamassassintaintorg +ordinary +zzzzlocalhost +archivelatest576879 +i +127001 +joint +committee +work +communications +that +out +to +20412720261 +alternative +ig +issue +dvd +20080610 +you +web +mpscph +amavisdnew +101421203 +mlsubscribertechcsminingorg +resentdate +confidentiality +inreplyto +believe +experience +at +contradict +id +meta +listhelp +boost +being +asked +board +ground +date +200932 +sse2 +69 +optionally +2010 +allow +hdatefromtosubjectmessageidreferencesmimeversion +testshtmlmessageldosubscriber +me +xuidl +58 +them +boundaryenigbb12e0cc4da2db4f9cf5ef72 +by +ipad +unsubscribe +obviously +xmailinglist +ldowhitelist5 +2525 +xbeenthere +autolearnfailed +shame +with +from +iusrsrcredhatbuildalsadriver090rc3include +means +mimeversion +gets +r16mr3596565fam621273464972509 +extras +105824 +173501 +xcheckerversion +pass +legitimately +choice +details +distributor +listid +british +hi +xaccountkey +sender +precedence +message +httpsourceforgenetdocmandisplaydocphpdocid9026groupid13487 +murdered +file +scrutiny +3331e13a6314 +evan +z14mr636586fag971274398034507 +tray +be +listpost +xpolicydweight +pnbspptable +videos +jaa25881 +get +055943 +time +bits +17xfeo0002td00 +barr +mailing +size1b1800br +81168116 +0976c13a4ed2 +catsmx2ucscedu +manual +moc +automatically +mirror +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +20020828100430378c3856matthiasrpmforgenet +0748213a4ff9 +claimed +keanuvisionyou +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +183657 +width100 +kde +com +ee38913a461f +no +drill +archive +having +zimbra +identify +when +contenttype +xmozillakeys +bjtxqup282nazlewsfzlbliszt +xrcvirus +c113d13a51be +of +01 +popularbatd +and +record +received +one +without +suite +transactions +wed +list +sonys +194913 +swe +freedexter +v1410 +gltranslatef00f +the +cent +2771816f1b diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_13.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_13.txt new file mode 100644 index 00000000..461a7d12 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_13.txt @@ -0,0 +1,275 @@ +1210 +namesignatureasc +trouble +qmail +broadening +xentcom +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +dot +pan0132 +support +smtp +wishful +alberta +people +aes256sha256 +any +clientip8219575100 +sounds +this +autolearnham +in +margin +501 +32227031 +your +practices +contacted +returnpath +operationsparadigm +iluglinuxie +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +execute +why +performances +23 +00000000 +114829 +potent +calling +said +examples +subject +given +using +isolnetsuxtechmonkeysnet +listarchive +kh9ngakjqem +cellspacing2 +2002 +xmailmanversion +netil +bulk +forth +explain +listunsubscribe +testsawl +0000 +4bf2e76b9070702studentulgacbe +debian +1921681100 +what +yyyylocalhostnetnoteinccom +parrot +d7mr9176665fgi751272854165513 +6c2ncovqgq03x4wi +spawn +running +usage +colorfffffftable +bgcoloreeeeee +our +supportedengines +u +re +hit +nor +exim +satalk +mnenhy076666 +governing +0400 +somewhere +xmimeole +mailtoforkrequestxentcomsubjectunsubscribe +edt +libboostdatetimedev +just +required40 +schampeolocalhost +it +esmtp +listsdebianuserlisztdebianorg +subscription +a +dont +deliveredto +xmozillastatus2 +resentsender +wrote +4242010 +localhost +as +pdt +messageid +151627 +10 +tony +compixelturesystemhookssystemhooks +text +ide +phoboslabsnetnoteinccom +0700 +160641 +or +color +contact +following +2007091301 +way +forkadminxentcom +2008110401 +im +bought +textplain +i +127001 +tabs +out +1921680101 +to +testsgmailldosubscriber +key +162608 +bcut +reverse +know +kgattinetechtargetcom +110507 +did +keeping +china +was +resentdate +fullupgrade +inreplyto +long +six +references +mxgooglecom +found +countries +response +at +xsender +debianuserlistsdebianorg +id +listhelp +can +on +hess +how +my +date +v149 +iron +69 +jalapeno +cvfnqqhiggoxky895ni2p06ys7q26btysl77b0nrg5xa +who +httpuseperlorgarticleplsid0209191443213 +justin +by +lisztdebianorg +even +width75 +ldowhitelist5 +2525 +back +software +listsubscribe +fish +with +from +rohit +dmzfirewall +adaware7 +hrefhttpclickthruonlinecomclickq3doi9nibz4yt04jzvwlgwutnantpr +mimeversion +giants +dinh +storybafont +predicting +mailtoforkspamassassintaintorg +incorrectly +xcheckerversion +listid +publishers +jmasonorg +charsetusascii +fetchmail590 +paired +xaccountkey +sender +precedence +suggestion +members +ie +its +20100524 +httpleesadevfarmcom +k2n1f5d398f1004061040t82b62056s7b582b73b234f060mailcsminingorg +e11652d0db9 +1022331136 +recall +x8664pclinuxgnu +cc +be +httpuseperlorgarticleplsid020913162209 +xpolicydweight +mailtoforkrequestxentcomsubjecthelp +were +hello +30 +xloop +form +httpequiv3dcontenttype +hits193 +httpdiveintomarkorgarchives20021002htmlrdfredux +argumentativeness +xoriginalto +about +wife +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +ppm +10143348 +httpthinkgeekcomsf +see +no +sat +rfe +router +200208221811laa21283maltesecat +downtown +gui +combosabr +fox +17eeb156a1e3b +contenttype +1032 +rose +of +summary +11fe0d0bb7b68ae0000042abf14bcce44036ee +log +received +pittsburg +windows +200 +195143 +wed +list +yy4xhzadbgnvbastflzlcmltawduifrydxn0ie5ldhdvcmsxoza5bgnvbastmlrlcm1zig9m +phobos +025223 +javadevbouncesmlsubscribertechcsminingorglistsapplecom +090125 +handle +broken +send +those +exposes diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_14.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_14.txt new file mode 100644 index 00000000..bf80be15 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_14.txt @@ -0,0 +1,283 @@ +est +20080517 +e3si1197745fga2920100505233753 +pulling +oldreturnpath +030510 +xmailer +for +organization +postfix +drwxrr +mailtodebianuserrequestlistsdebianorgsubjecthelp +encodingutf8 +maybe +smtp +12 +any +score109 +plant +verify +clientip8219575100 +this +in +move +have +your +players +returnpath +conclusions +readers +is +bat +source +uswsflist1bsourceforgenet +faster +height3d +among +gtk +httpwwwcandygeniuscom +00000000 +subject +4 +relay +am +contentdisposition +listarchive +d74e726e94477 +bulk +guide +email +careful +but +state +0000 +133801 +hei +degrees +debian +jmlocalhost +essential +15071 +glpopmatrix +etcinitdclamavdaemon +ist +if +good +apr +problem +description +httpoutoftouchblogspotcom +must +thu +all +tue +pbs +re +receivedspf +every +httpwwwtheregistercoukcontent426170html +336 +clean +edt +just +174559 +required40 +it +esmtp +razoradmin +protocolapplicationpgpsignature +124335 +local +convenient +a +dont +helodellbuntulocalnet +deliveredto +information +jon +rdnsdynamic01 +seeing +width160 +httpwwwblasercocomblogs +localhost +dogmaslashnullorg +spfpass +messageid +signature +judge +10 +ilug +hrefhttpclickthruonlinecomclickqe72ygqmvxo3akrdfvqs8cmb0qvser +spelt +0700 +config +or +germana +f160law15hotmailcom +g98gf1l25092 +064802 +contenttransferencoding +et +smalltalk +im +awhile +ma +textplain +i +127001 +changed +that +082813 +listsdebianorg +to +imho +modern +passing +fork +know +utc +scots +immutable +mlsubscribertechcsminingorg +was +7140abr +references +mxgooglecom +something +at +replyto +blog +cipher +id +over +port +listhelp +slightly +esmtpsa +can +on +related +same +designates +p26mr3985135wfa671271882229739 +trtdimg +thingsbr +how +deltacsmuozau +turned +104834 +date +g7mijaz19382 +temp +jalapeno +border0atd +68142198106 +mailtorpmzzzlistfreshrpmsnet +by +170146 +he +unsubscribe +client +wollen +ldowhitelist5 +2525 +0100 +little +unclosedbracket2206 +p +not +ignores +with +rpmlistfreshrpmsnet +from +still +rohit +1alnuq0007hv00 +mimeversion +heaven +200210010801g9181ek15588dogmaslashnullorg +651d213a43ea +style3dtextdecoratio +xmozillastatus +details +uid +use +listid +charsetutf8 +fetchmail590 +muddled +precedence +blocked +kb +anyone +disabled +its +errorsto +which +attempted +almost +say +an +cc +box +donnerstag +be +listpost +johnhallevergonet +where +better +enough +version250cvs +logmaneorg +ask +power +215 +o2g2fc5f091004111716oe078137dse05f0b82f4e519ccmailcsminingorg +recommended +xloop +nn1matxuah2 +xoriginalto +about +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +score101 +light +during +no +we +archive +gojomousanet +magnitude +digital +arialbflowchart20 +version325 +great +jmrpmjmasonorg +inside +contenttype +fontb +input +xrcvirus +datwinkdaddy +like +001414 +and +primary +xstatus +m8cs51580wfo +received +portability +pig +list +102919 +quantum +exmhusers +phone +wish +mailtoforkrequestxentcomsubjectsubscribe +created +m2h308d4701004251507y5ab5e093ye814f86e7e1d8c11mailcsminingorg +102047200 +the +maintain +outofprocess diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_15.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_15.txt new file mode 100644 index 00000000..17c03650 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_15.txt @@ -0,0 +1,263 @@ +sack +083317 +quality +pnbspp +trouble +oldreturnpath +read +able +hurry +for +postfix +lots +community +usabr +post +d6f9213a50f6 +132119 +httpfreshmeatnetprojectswebcppgui +autolearnno +admin +due +body +resentmessageid +account5 +reliable +your +returnpath +mailtodebianuserlistsdebianorg +is +httpdevelopersslashdotorgarticleplsid021112161219 +source +treats +why +halt +halfhearted +unplug +show +subject +using +am +hmimeversiondatemessageidsubjectfromtocontenttype +2002 +bulk +email +83d5813a51c7 +listunsubscribe +ends +7188806 +debian +9 +20020930203145570e6717matthiasegwnnet +jmlocalhost +url +heavily +themesinfo +oct +fallback +so +revolution +iguana +unix +lite +emacs211 +domain +alexander +u +native +1142 +moonwatcher +0400 +spamassassintalkadminlistssourceforgenet +just +it +united +esmtp +httpxentcompipermailfork +hrefhttpdigitalmedialockergnomecomlatest +a +dont +deliveredto +localhost +squareshaped +useragent +dogmaslashnullorg +configure +well +pdt +height3 +has +messageid +singh +chriss +signature +variant +unstable +text +0700 +notinsblxblspamhaus15 +old +httpxentcommailmanlistinfofork +flap +strong +interested +husband +maintaining +a40abr +gmailid12801d148efba944 +numerous +xrcspam +2007091301 +required50 +importance +127001 +that +out +ïòçáîéúáãéé +productive +aircraft +neg +185850 +buy +to +v1 +monday +71 +singledrop +now +fhoflnpfdhnl0u3noskroxgs1r4165spnxszqalhuuxkwfwgyrpoakpm9codumwmy +you +select +causing +80 +inreplyto +153940 +found +c1 +legal +taken +buttons +at +through +id +209sfnet +rfc +also +listhelp +need +can +on +garymcanadacom +designates +wants +how +wyf23 +after +rectangles +date +munnariozau +httpdocsyahoocominfoterms +jalapeno +2010 +cellsp +worthy +me +googlecom +xuidl +them +by +spare +gauntlets +diligent +actually +will +xbeenthere +back +their +anyway +6b30816f03 +a22481297391075contentsarticletitlecolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightnormalfontstylenormaltextdecorationunderline +not +listsubscribe +certification +with +from +assume +worldwritable +0500 +total20 +true +cuoco +linux +pass +use +listid +garticlefullstoryhover +put +jmasonorg +xaccountkey +sender +precedence +testsalltrusted +scoped +didnt +errorsto +say +notes +aug +hrefhttpwwwimakenewscomeletraupdatecfmxmediaunspun2czzzunspunspamassassintaintorgclick +into +412 +bigger +ronljohnsoncoxnet +xvirusscanned +sites +zzzzteana +more +output +case +datapower +4dec029409f +mincount +utf8q3vr9szxa5tvz2dbl2dyyrfv6k76a93l6g0a098gbpej2tkfikpap8d93f6v +html +joe +w +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +pinelnx444020906212058013901100000isolnetsuxtechmonkeysnet +vine +xoriginalto +about +no +archive +bomb +mp3s +value +does +srchttpcnetcombgif +when +group +bebusinessbbr +version325 +inside +contenttype +like +events +other +of +dialog +settings +record +received +one +wed +free +quietly +list +exmhusers +y20si5570588fah10220100524081209 +norte +the +score12 +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_16.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_16.txt new file mode 100644 index 00000000..3641ffdb --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_16.txt @@ -0,0 +1,277 @@ +hrefhttpclickthruonlinecomclickqa4hgtsq6p4mtc1olg463lvvi2rgydr +trouble +041718 +for +postfix +heading +services +aanlktiluy9zeq3g6jozedqwcynpau7hxoewytbt1kchmailcsminingorg +bs +12 +crashing +102br +g8n58oh19980 +titlecnet +clientip8219575100 +191543 +autolearnham +cached +nzst +switch +your +returnpath +featureabr +imap +190747 +are +is +mon +wait +martin +opera +b3xmqqrkciye0sgvv0l5qqnkxsuveigkrvbu6rdtbxwva3xplqu04qjzxn0htriacede354 +00000000 +17hywh0006nv00 +vander +market +subject +ancestral +difficulty +102234217 +qmqp +consider +listarchive +system +violating +7113422550 +2002 +bulk +200210010801g9181ak15561dogmaslashnullorg +email +rh +lake +but +0000 +below +inbox +listboxes +score6 +height6 +underline +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +101036720 +recent +closed +ist +unsubscription +background +ga +br +quinlanpathnamecom +apr +own +12921512853 +required53 +216136171252 +names +64422195 +185910 +normal +led +33aaf29409a +re +residence +receivedspf +httplistsdebianorg87fx1p7wwjfsfmerciadrilucastationmerciadriluca +void +0400 +095238 +it +noted +xoriginaldate +httpsourceforgenetmailarchivesforumphpforumspamassassindevel +esmtp +reading +a +deliveredto +mean +xmozillastatus2 +e17pfih0005yh01cpu59osdncom +localhost +as +mutt1520 +spfpass +well +pdt +has +messageid +10 +fraught +warning +certain +logged +archivelatest576487 +contact +backgroundhttpwwwlockergnomecomimagesissuetopright2gif +httplistsfreshrpmsnetpipermailrpmzzzlist +080055 +xrcspam +contenttransferencoding +textplain +until +127001 +22240 +that +out +d13si8606656fka220100412080008 +473 +192241 +hear +to +uninhabited +xlevel +now +scratch +know +child +did +neediness +craigdeersoftcom +planning +331vamm2 +here +mlsubscribertechcsminingorg +references +at +depends +debianuserlistsdebianorg +httpslistssourceforgenetlistslistinfospamassassintalk +id +debian3f +folder +5so1340930qyk3 +also +listhelp +esmtpsa +on +sagfsgfsgcom +murphywrongword101 +related +same +06 +sequence +check +intel +zzzzlocalhostnetnoteinccom +date +adam +rob +authenticationresults +me +xuidl +mail +commercial +by +then +smtpmailbouncedebianlaptopmlsubscribertechcsminingorglistsdebianorg +screen +client +listmasterlistsdebianorg +xmailinglist +120730 +mailin11applecom +xbeenthere +autolearnfailed +v1408089726 +listsubscribe +score110 +with +from +180642 +mimeversion +celejar +2 +linux +both +192546 +pass +xmozillastatus +use +listid +jmasonorg +02 +charsetusascii +fetchmail590 +done +741433120 +xaccountkey +stdoutfilenomsgbufamountread +sender +permitted +precedence +virtualincludebannertxt +its +errorsto +which +score7 +original +firmopinionholding +taggedabove10000 +pixelbuffer +paperweightdarwinnasagov +tough +be +penned +sune +get +removing +release +mailtoforkrequestxentcomsubjecthelp +hrefhttpclickthruonlinecomclickq5b5lowibkdutf1y12xcjjvdswizzcr +were +kept +unmount +c2a0swapbr +update +xloop +thursday +platform +81168116 +sgamma +basic +close +manual +updating +throw +about +herebap +8219575100 +150 +751189522 +width100 +kde +width375 +asking +images +lie +sat +archive +enfant +destination +075648 +version325 +162358 +theres +width3d40 +may +of +srchttpwwwtheserversidecomimagesj2eelogobluegif +received +3721f13a567b +one +windows +list +augdlistsapplecom +cve20101170 +the +httpradioweblogscom0106884 diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_17.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_17.txt new file mode 100644 index 00000000..701fcbd9 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_17.txt @@ -0,0 +1,277 @@ +rate +lights +pratice +ftp +trouble +xmailer +for +postfix +rpmlist +mailtodebianuserrequestlistsdebianorgsubjecthelp +firewalls +10204625 +post +046 +autolearnno +x +17230098 +across +years +etcautomaster +dkimsignature +this +in +needed +yyyyexamplecom +your +returnpath +imap +vm4010327 +are +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +testsbayes002 +subject +navigator +got +less +than +scenes +2002 +xmailmanversion +bulk +model +listunsubscribe +rzmta +connect +0667713a56dc +state +0000 +ot +jmlocalhost +0530 +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +url +impossible +would +remember +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +yyyylocalhostnetnoteinccom +bbb6a27456102 +if +project +gldudebianuser2mgmaneorg +gmailid12825cb607dd63da +2ba312940e2 +adiv +verifyok +up +so +razorusersexamplesourceforgenet +244b5106fe204cc6b65abb558954c536latencyzerocom +domain +archivelatest577066 +re +page +result +width3d300trtd +dcsminingorg +enabled +just +325 +99 +esmtp +forastero +listsdebianuserlisztdebianorg +m95r97u97tsp82qc +a +151708 +deliveredto +rv1919 +office +xmozillastatus2 +n17grpscdyahoocom +ghai +gmailid127e290c76a7952f +localhost +m2efrjy5m2gatxciokvlbliszt +2011 +as +build +oldschoolcompressionalgorithms +restricting +pdt +messageid +year +10 +skip +900 +krslwwezfwrfzbpucmndhmbabexeygodp0t4 +phoboslabsnetnoteinccom +0700 +old +color3d333333bdear +httpmaybeiotoeuropaorg +or +searched +looking +size2bfloyd +bootvmlinuz2632trunk686 +contact +ztyaoidgsvpxvd9pk3rv3k6ylm30a0e +exactly +contenttransferencoding +forkspamassassintaintorg +2008110401 +im +score3813 +mailtorazorusersexamplesourceforgenet +brand +httpwwwcaterwaulingcomblog +standardized +textplain +i +ps +files +that +127 +helouswsflist1sourceforgenet +listsdebianorg +to +809122912 +singledrop +base +xlevel +razorusersadminexamplesourceforgenet +ilugadminlinuxie +company +options +amavisdnew +k2q308d4701004140621ibf159f8dycc2bb534b1aeb7bdmailcsminingorg +share +intranet +mlsubscribertechcsminingorg +libamrnb +was +resentdate +inreplyto +41140153155 +references +mxgooglecom +at +tpwwwtwelvehorsescomwrapperdocviewjhtmld3dtescoiarea +id +154149 +can +on +guns +altsoftware +designates +how +seqset +date +a0 +2010 +allow +bugs +googlecom +052810 +defaultspcmsurround51device +oops2dat +by +unsubscribe +17str30000sj00 +listmasterlistsdebianorg +processed +actually +will +ldowhitelist5 +2525 +much +exmhworkersadminredhatcom +unless +stroking +not +listsubscribe +with +rpmlistfreshrpmsnet +from +260 +person +postscript +clicks +es +farmer +pass +xmozillastatus +l30mr1080942waf1341273840106932 +listid +0fw +a87d +redhats +done +xaccountkey +sender +permitted +precedence +landing +stephen +file +hrefhttpclickthruonlinecomclickq87izdoq8n4w11bhleplrq8kasp4er +errorsto +which +missed +valuelockergnomeinput +established +7bit +be +listpost +rmosunnmorenet +marcletde +youd +get +were +120400 +title +215 +more +fdisk +frequently +hello +type +81168116 +really +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +17inch +many +except +tm +8219575100 +width100 +hrefhttpwwwhandangocomplatformproductdetailjspplatformid1ampproducttype2ampsectionid0ampproductid30551ampcatalog1ampsiteid159 +threads +very +vzfejtuxyheekhcti1bxnbyo17iunt7pskfxk +when +great +unique +conspiracy +contenttype +xmozillakeys +versioned +footer +filed +enhancements +of +154027 +and +conflation +xstatus +received +one +srchttpartvcomcnet1dipg021502pointcamerajpg +list +phobos +display +s0832016219710167 +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_18.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_18.txt new file mode 100644 index 00000000..7c931a3b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_18.txt @@ -0,0 +1,276 @@ +blair +lists +down +application +forteanayahoogroupscom +trouble +oldreturnpath +for +postfix +httpwwwpycsnetworkbenchcategoriessports +marginheight3d8 +encodingutf8 +smtp +method +target3dblankhttpsdeveloperskypecomjirabrowsescl510a +nospamvuoreladk +circulation +body +clientip8219575100 +this +in +errorsbrdiv +certificate +have +yyyyexamplecom +returnpath +why +multipartsigned +ftpftpximiancompubximiangnomeredhat71i386gaimapplet05911ximian2i386rpm +makes +subject +aanlktil3iybhkej2xkvaoomq9wlcp0phw8zy2xm8lmailcsminingorg +qmqp +listarchive +469 +xmailmanversion +fontsize +email +previously +listunsubscribe +thought +standard +debian +root +ot +url +shared +would +reporter +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +usrsbinntpdate +ist +line +10873520 +good +running +ultraflat +095854 +own +fallback +infoznioncom +our +normal +60026000000 +domain +thanks +re +altabstract +nor +result +exim +servicescnauthsmtppsuedu +before +325 +us +rant +it +esmtp +xinjectedviagmane +watch +score71 +subscription +m8cs84211wfo +entire +a +khare +arsasha256 +information +xmozillastatus2 +resentsender +provides +localhost +though +05 +forward +as +dogmaslashnullorg +thing +frommxmatchesnotunvrhelodomain16 +scott +spfpass +pdt +messageid +signature +popescu +computing +warning +nice +or +main +equivalent +contact +they +xrcspam +far +contenttransferencoding +textplain +invoked +127001 +info +midrighttopmsg +that +out +bbh3ramc6ffk0mxe1qee6h123fotm6bqrek6tyegrvep2ik9n2klfltnypyeq0q +listsdebianorg +to +issue +least +172901 +you +256256 +bank +did +amavisdnew +28639 +markup +most +mlsubscribertechcsminingorg +hrefhttpclickthruonlinecomclickq145mraifzahzjyfu8zvqiztktr1yr +adjust0 +there +kick +clipeqheloip2 +at +1 +src3dhttpgservzdnetcomclearoutboundgifappid3d2emid3d25136482 +leenxcoza +id +morepatrioticthanthou +listhelp +on +17 +mailtospamassassintalkrequestlistssourceforgenetsubjectsubscribe +reports +designates +check +how +my +20910320317dialin1oshathenetnet +date +19317254 +2010 +adam +httpwww12hscomimagestesco1020offmincepiesgif +googlecom +xuidl +mail +envelopefrom +versionsslv3 +by +then +unsubscribe +unsigned +should +bst +ldowhitelist5 +2525 +wonder +0100 +not +delete +pc +with +from +stuff +still +wealth +rohit +selfregard +149 +sideeffects +next +profilename +httpseekerdicecomseekereplrelcode31 +httpslistmanspamassassintaintorgmailmanprivateexmhusers +linux +xcheckerversion +ldowhitelistnorealnamepgpsignature +xmozillastatus +could +listid +done +xaccountkey +pgdev83110lenny1mipseldeb +sender +permitted +precedence +file +its +which +face3dtimes +hack +percent +an +original +7bit +taggedabove10000 +aug +be +listpost +xpolicydweight +transition +get +connected +mailtoforkrequestxentcomsubjecthelp +starting +were +version250cvs +x11 +intelligently +rapidly +headericsminingorg +install +microsoft +some20 +1087 +between +cat +fri +spam +about +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +10143348 +displaying +17i7oe00023r00 +3 +mailtoexmhusersspamassassintaintorg +no +wired +aframelink +gnomedex +prototyping +some +tower +joystickcom +frustrating +contenttype +may +secure +archivelatest574307 +apollo +other +of +and +record +received +httpbumptiouslyblogspotcom +referencing +httpadrantsrantworkscom +eliascseucscedu +extraction +engine +list +days +testscapinithtmlmessage +cant diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_19.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_19.txt new file mode 100644 index 00000000..92583d4a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_19.txt @@ -0,0 +1,294 @@ +httpnewmarksdoorblogspotcom +newbie +used +toptionsgt +0300 +xmailer +for +postfix +2139621475 +mailtodebianuserrequestlistsdebianorgsubjecthelp +network +school +smtp +method +12 +cpe653144139wohresrrcom +dsa1842 +package +this +benefits +have +returnpath +imap +fonts +why +too +11fe0d0bb7cc2ae000000813744bc4c8fe301d +pearls +msklbajfiyrgr9pcxz7lvwau87ozczvnws +subject +155803 +using +private +tci +2213 +469 +bulk +rpmlistadminfreshrpmsnet +email +listunsubscribe +quotedprintable +david +but +sure +11 +jmlocalhost +gmailid12801d634d937683 +rssfeedsspamassassintaintorg +what +different +remember +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +httpninonanospeedblogspotcom +forkxentcom +up +xapparentlyto +so +our +do +debeers +policyabr +re +look +advanced +clave +0400 +0xbb +just +325 +macro +it +esmtp +two +local +mailman +a +deliveredto +everybody +giov +wrote +height +localhost +2011 +tools +as +dogmaslashnullorg +httpwwwtheregistercoukcontent426135html +spfpass +pdt +submittdform +messageid +uw +sentto2242572559141033992044zzzzspamassassintaintorgreturnsgroupsyahoocom +mutt1251i +steve +m +beginning +associate +005427 +0700 +products +linuxmodulesafipxver +resentfrom +krunner +dcc4e84cb8fc01ca8f8654c982ec8526 +perl +laptop +trying +gmail1 +im +gary +building +ma +zzzzlocalhost +jeremy +textplain +invoked +i +until +127001 +8bit +12si3898778fks2020100424042847 +2008 +that +uswsffw2sourceforgenet +cvgarkv6wxulwczxlyffz6yngldvhtzyaxtudctjej4sc +to +809122912 +green +preferably +20080610 +4bc634fe1020700studentulgacbe +you +web +utc +bank +202952 +formating +minutes +single +most +resentdate +week +inreplyto +long +mxgooglecom +0023122 +hours +los +at +rita +xspamlevel +id +davelocalhost +port +listhelp +can +on +17 +same +21012 +yyyylocalhostexamplecom +check +how +interest +deltacsmuozau +date +19317254 +f28mr13294898fgk701270638043781 +o4olgknc004323 +jalapeno +itself +allow +exmhworkers +them +ag +colspan3d4a +rocks +by +then +either +lisztdebianorg +initiates +53 +185605 +client +listmasterlistsdebianorg +helo +xmailinglist +new +cool +actually +will +via +guarantees +m7mr3342517qag3801274726689910 +ldowhitelist5 +2525 +ertzinger +0pt +management +0100 +xamavisstatus +their +yrp3o8zjo +not +with +from +diamond +still +generated +rohit +dmzfirewall +wood +class3dgmailquoteon +mimeversion +friedman +tiresome +harri +httpwwwmarciafeldmancomhtmlnewsshtml +spamassassin +dvds +both +definition +cssb +listid +make +answer +sent +charsetutf8 +fetchmail590 +xaccountkey +authenticated +friday +sender +end +message +anyone +replying +errorsto +tree +an +rpm +compiled +taggedabove10000 +x8664s +listmanspamassassintaintorg +be +supported +bhw9q1vzvgyjjgrlz1vxxiksmzd72gixi4pbzfwtt1bu +8f8ba13a4a34 +style3dtextdecorationnoneto +wukanj8t2lp8iafk2dqraqjsf8o9dz9u +better +101 +xvirusscanned +listened +192168110 +width430 +several +more +hello +cellpadding3d8trtdfont +81168116 +mirror +1124 +285326 +1022922178 +8219575100 +adrian +debianuserrequestlistsdebianorg +085355 +no +installed +leitl +when +group +srchttpartvcomcnet1dinlftgif +contenttype +xmozillakeys +of +ive +received +looms +hrefhttpclickthruonlinecomclickqc2oxxlquedqhem4sgvnxxfzciuinr +list +listsdebianvolatilelisztdebianorg +zoo +usrlibx11xorgconfd10synapticsconf +view +s0832016219710167 +the +width100img +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_2.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_2.txt new file mode 100644 index 00000000..8ffa17a9 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_2.txt @@ -0,0 +1,286 @@ +115104 +application +qmail +des +for +postfix +support +smtp +httpwwwshagmailcomunsubhistoryhtml +3height +genes +waldner +httpscriptingnewsuserlandcombackissues20020923when35654pm +href3dhttpwwwcomicscomwebmail +formatflowed +mac +onmouseover +httpdandaniellehomemindspringcom +resentmessageid +verify +clientip8219575100 +this +your +returnpath +is +source +shooting +episodes +nbspnbsp +luis +mailtospamassassintalkrequestexamplesourceforgenetsubjecthelp +gtk +00000000 +srchttpwwwzdnetcomproductsgraphicspid561958jpg +width121 +subject +try +godo +am +contentdisposition +httplistsdebianorg87zl0ccgerfsfturtlegmxde +2002 +bulk +dick +email +listunsubscribe +ends +sure +0000 +released +123507 +gocomimagesbtp2pgifccode3d1d74c2eapcode3def39755c +5021020020726133547028d0a10dogmaslashnullorg +security +hrefhttpclickthruonlinecomclickq1ay6zkilxxica1bbd5q0hzb17wnr +would +88135 +boot +ieyearecaayfakvjzfsacgkq7ro5m7lpzdj3oqceopwxwspcvusohxxnmdhdau +tmsgidgoodexchangeturicount01 +apr +problem +loader +fallback +going +httplistsdebianorg4bc79b4f2080500hardwarefreakcom +11a6613a58d2 +thu +do +pdf +200210050800g9580wk02532dogmaslashnullorg +startup +domain +0pxls +httpwwwlinuxiemailmanlistinfoilug +thanks +campaign +cest +warm +arises +receivedspf +185343 +0400 +dcsminingorg +colspan4img +before +mikes +left +understand +it +66187233200 +esmtp +10223513 +hypotheses +two +a +deliveredto +xmozillastatus2 +patchunifieddiffreferencesspamphrase0001xloop +resentsender +localhost +serif +2011 +useragent +as +inline +because +spfpass +pdt +c13mr4723586rvi861270530782703 +messageid +12811455196 +10 +isnt +computing +contends +0700 +httpiguanasuicidenet +134145 +suggestions +103709 +contact +groups +they +174614 +0200 +avbidderfortytwoch +includesysasoundh +171941 +blc4kclstjndy1wiiz1qp2r3blwzqnlwz63zdwd1vuxb2ipq4lsxqvdqrrlghxbx54m +textplain +i +127001 +8bit +ps +central +that +creation +to +completely +sep +permanent +lee +ratwaregeckobuild +quotupdatequotbr +did +fact +minutes +hdready +these +v +making +222400 +was +inreplyto +references +2002072010543231116qmailmailshellcom +through +id +wwii +apt +4bed4f8d2080007cyberspaceroadcom +checksum +httpslistssourceforgenetlistslistinforazorusers +port +listhelp +need +can +on +designates +happy +check +infringements +date +20020812fz464416600wwwdudexnet +reservation +allow +me +xuidl +mail +hrefhttpclickthruonlinecomclickq36wg7iii4fxkcfotmxvh3qainxq4r +by +listmasterlinuxie +tywkrkontaznlegxtfqypbsczkeae4zn8iqal4dcguuakit8 +should +xmailinglist +story +called +xbeenthere +back +2098516147 +not +listsubscribe +with +from +zzzzspamassassintaintorg +10223581 +departure +means +mimeversion +dr +enht +hits99 +spamassassin +first +gachallan +xcheckerversion +pass +5232c1c26e994 +xmozillastatus +src3dulgif +could +use +listid +clipeqfrommx31 +news +charsetusascii +hi +sender +end +precedence +storage +which +apple +operationfontbspan20 +say +27fa5341dd +testsawlinreptoknownmailinglistplingquery +cc +162605 +listpost +street +src3dhttpwwwzdnetcomgraphicsancho +youd +longer +bouncedebiansecuritymlsubscribertechcsminingorglistsdebianorg +theyre +mailtoforkrequestxentcomsubjecthelp +headericsminingorg +mailing +sgamma +patch +xauthorityanalysis +upgrade +manual +xoriginalto +about +8219575100 +see +nnfmp +no +smtpmailquicktimeusersbouncesmlsubscribertechcsminingorglistsapplecom +hspace6 +byte +archive +let +top +firefox +mailtorazorusersrequestlistssourceforgenetsubjectunsubscribe +alignmiddlefont +intmx1corpredhatcom +i486pclinuxgnu +group +theres +contenttype +practice +may +write +other +of +ive +mozilla +g918s1f18842 +xstatus +record +received +plan +free +list +creates +usually +access +perhaps +technical +the +maintain diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_20.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_20.txt new file mode 100644 index 00000000..2592c6a2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_20.txt @@ -0,0 +1,270 @@ +breaks +rate +tablep +135131 +zcvsnddetj3hwqj24t3oupcadnu7fnubpj0vljqzl4g7nuhqxoyvpekdyybjiub +economy +trouble +oldreturnpath +102298012 +for +postfix +drive +politics +xyahooprofile +support +sections +2131863320 +egwnnet +nbspa +moonbasezanshincom +emacand +towerboard +resentmessageid +this +in +content3dtexthtml +pocket +players +returnpath +imap +stays +1d1f02049c5d68fb4 +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +why +too +00000000 +mozilla50 +srchttpzdnetshoppercnetcomiceps12096258061201gif +subject +am +hence +cb4ed98bd07747faac7c3740bb63b29fapplecom +aggregator +listarchive +100045 +concluded +ancestor +xmailmanversion +akonadi +rpmlistadminfreshrpmsnet +fontsize +email +usenetdoughgmaneorg +listunsubscribe +canopener +jmjmasonorg +xorg0log +debian +jmlocalhost +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +what +httplistsdebianorg1252913082894381272137255091javamailrootmd01wowsynacorcom +yyyylocalhostnetnoteinccom +launched +size1btop +systems +if +background +problem +big +fallback +required53 +up +g81afvg18761 +our +spamtrap +thu +normal +a0mixer +bpbsaslquonix +minute +vncstartup +hrefhttpwwwnytimescom20020812businessmedia12bookhtmlhttpwwwnytimescom20020812businessmedia12bookhtmla +domain +re +f +postgrey131 +receivedspf +20100415153853f46e5f69celejarcsminingorg +0400 +vspace10 +just +it +replay +esmtp +listsdebianuserlisztdebianorg +local +os +a +dont +testsfourlaldosubscriber +deliveredto +xmozillastatus2 +16 +localhost +though +within +useragent +as +talk +thing +because +pdt +she +messageid +height8 +10 +photography +locked +divspan +gnulinux +additional +httpxentcommailmanlistinfofork +or +seems +scanning +leader +knew +lmzuwxio3atb0nktd8h1uahdblikdtkreftu +such +they +contenttransferencoding +required50 +2008110401 +open +revoke +textplain +i +127001 +httpwwwfrogstonenet +xpyzor +that +oef6zy1fvpa +dispatchtitle +aptget +to +match +singledrop +xlevel +code +you +modified +findings +around +various +tom +incompatible +border0abrtd +help +inreplyto +icedove +hash +discuss +10229220133 +go +id +engineer +investorbr30day +listhelp +esmtpsa +can +wants +check +worth +cost +date +19317254 +114243 +jalapeno +nyah +authenticationresults +spamassassintalkadminexamplesourceforgenet +xuidl +them +by +think +unsubscribe +164319 +style3dcolor +0100 +xamavisstatus +started20 +remained +autolearnfailed +alumni +not +listsubscribe +with +from +still +immediately +mimeversion +httplistsapplecommailmanoptionsaugdmlsubscribertech40csminingorg +give +spamassassin +dvds +pweve +life +fight +james +xmozillastatus +saa22715 +use +claims +merciadri +rather +sender +permitted +precedence +message +lennybrbrdefault +goldilocks +which +taggedabove10000 +discussion +be +listpost +where +get +logmaneorg +more +microsoft +wanted +nonvoting +005708 +20019 +gmailid1280567e9242a03d +bindinggenerator +no +we +archive +writes +mailfollowupto +gmbh +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +group +imposed +ignore +316 +contenttype +errorreceived +xrcvirus +willing +satelle +of +and +received +traction +free +list +mailtoforkrequestxentcomsubjectsubscribe +002249 +authority +technical +the +memory +cant +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_21.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_21.txt new file mode 100644 index 00000000..cfbacc3f --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_21.txt @@ -0,0 +1,254 @@ +terrible +rate +spokeswoman +enable +typesubmit +for +postfix +recipes +refuses +encodingutf8 +smtp +span +mjnk +kernel +body +archivelatest574186 +resentmessageid +tells +this +0l1a00c85x0u4bd6vms173015mailsrvcsnet +in +lta +htmlmessage1 +your +returnpath +mailtodebianuserlistsdebianorg +are +excuses +intervene +built +notifysuspend +show +subject +helonausicaa +qmqp +using +posts +listarchive +srchttpwwwcnetcominlinvhdgif +shell +2002 +xmailmanversion +forecast +email +requested +httpradioweblogscom0100463 +1024x768 +0000 +reconnect +contenttexthtml +purposebuilt +a22481297394458footerpublishercolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightnormalfontstylenormaltextdecorationnone +if +spamassassindevellistssourceforgenet +world +compliant +hotel +running +fallback +designed +maingmaneorg +our +thu +all +hrefhttpclickthruonlinecomclickq6b8g5zihecwprago280ywq4d0p6cr +6c4d016f21 +pgijp2cejugaaeezjzlbliszt +youre +f +marks +sendmail +dead +0400 +13 +edt +just +it +httpenwikipediaorgwikismartstandardsandimplementation +esmtp +902416 +camera +listsdebianuserlisztdebianorg +p9si4124506fkb320100430192113 +gmailid1284e4b5df252bb7 +lawrence +a +httpsecuritytrackerdebianorgtracker +moving +deliveredto +restructuring +namerobots +165656 +localhost +reinhart +2011 +as +dogmaslashnullorg +because +231 +5 +messageid +vspace1 +ton +signature +10 +recommendation +tv +hell +httpxentcommailmanlistinfofork +or +resentfrom +southeastern +contact +contenttransferencoding +2008110401 +ghnrg +userid +de +textplain +invoked +i +valignbottomimg +patrick +127001 +g93bupk26666 +cannot +to +sep +hjzpo95gxteabsc46izlbliszt +alternative +charsetiso88591 +sec +xlevel +you +overhead +know +256256 +most +was +mxgooglecom +found +transport +depends +been +debianuserlistsdebianorg +id +mailinglist +2440 +port +how +date +unravel +van +2010 +bug +xuidl +artist +envelopefrom +by +then +think +unsubscribe +gmexim +13220349890520020925163453magnesiumnet +xmailinglist +via +dc68713a538f +ldowhitelist5 +2525 +xbeenthere +xamavisstatus +software +11fe0d0bb7b68ae0000042ab554bd44e6cdd60 +not +listsubscribe +174102 +with +from +iso88591qz4ltpu6qolya6g9aw0krjlajcsgcicycxrcoxwmurhckrvhcedaur7own2 +desktopcopernicusdemoncouk +cnofws +clientip172541338 +mimeversion +gets +mymac0 +spamassassin +first +both +xcheckerversion +spot +address +15inch +could +boy +use +dcbe9294206 +charsetusascii +fetchmail590 +rather +lucamerciadristudentulgacbe +sender +dhmgmmdjvb47 +stephen +which +34df813a55d3 +almost +dir3dltrlta +022908 +original +experiencing +tough +be +listpost +httplistsdebianorg2010050707432040ecb684mihira +penned +choicemail +detailed +time +anything +hired +more +xloop +mailing +castle +81168116 +sgamma +confiscated +manual +homeabbr +xoriginalto +about +20000922 +8219575100 +1195 +stable +no +sat +61218821026951304845javamailrootabvsfo1acagent4 +m8grpscdyahoocom +download +contenttype +sid +mentioned +like +other +of +ii +and +sponsored +received +one +mailtoforkrequestxentcomsubjectsubscribe +httpkotiwelhocompmatilai +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_22.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_22.txt new file mode 100644 index 00000000..14099b06 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_22.txt @@ -0,0 +1,278 @@ +michael +writing +alt3dsylvia +papers +format +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +202pm +any +body +jpg +resentmessageid +autolearnham +in +httpslistmanspamassassintaintorgmailmanlistinfoexmhworkers +cached +your +gw01es3egwnnet +returnpath +imap +is +mon +source +faster +execute +why +19 +course +testsbayes002 +20985221194 +prices +subject +hard +iso88591qptbs5oo9toumhbdke23otci0a093cbil8dtogzk0bf2yq35wsewvyet2e +using +am +dslbased +listarchive +system +fingertips +2002 +xmailmanversion +bulk +hrefhttpwwweasywebeditorcomcreatewebpage000004htmeasy +forwarded +email +eridanus +model +listunsubscribe +jmjmasonorg +15 +state +0000 +debian +bgcolorccccccimg +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +similar +ist +c9825d053119 +stated +hmessageiddatefromuseragentmimeversiontosubjectreferences +fallback +telnet +so +dffbqidlud2dsxi4uytqs9z3udrrimg +all +lairxentcom +domain +re +strongly +reward +exim +0400 +mailtoforkrequestxentcomsubjectunsubscribe +edt +just +325 +us +it +xoriginaldate +esmtp +laws +a +dont +calls +moving +splitting +deliveredto +ieyearecaayfakvmyosacgkqvsrxyk4409xcwcgixo8aswa8db8m2sp4kv3c2l +khare +titlem +described +wrote +windowsreg +localhost +2011 +as +dogmaslashnullorg +spfpass +pdt +she +messageid +year +emacs222 +undeciphered +10 +6ookxih7dsyh4fguk5ams763nz5t0zys1yzeze9opjcxncb7rfig +walkiewicz +theo +sector +zzzzilugspamassassintaintorg +such +0200 +contenttransferencoding +usb +im +103203 +127001 +8bit +archivelatest574208 +that +g8tihhkl005444 +hrefhttpwwwlockergnomecomsubmithtmlsubmit +rpmzzzlistadminfreshrpmsnet +to +singledrop +xlevel +iny +you +utc +bank +xeec25lndts +was +symbol +long +there +happened +httplistsdebianorg20100414102012gf6054wastelandhomelinuxnet +kmail1132 +at +1 +jmsajmasonorg +th +debianuserlistsdebianorg +id +doesnt +pricy +height10br +port +listhelp +forkexamplecom +need +can +designates +happy +notice +b +date +zdnet +mo6541216221staembarqhsdnet +spamassassintalkadminexamplesourceforgenet +mail +simple +envelopefrom +theater +helped +manager +by +lisztdebianorg +publisher +client +acid +snip +mailtoexmhworkersexamplecom +will +via +xbeenthere +0100 +while +their +autolearnfailed +185717 +wont +not +listsubscribe +reboot +with +from +rohit +ddx9ygsej5amfx1uz63g58lvqjqvn +emssrv0 +mimeversion +tdtd +pet +give +spamassassin +fifty +2 +xcheckerversion +james +2czdonc9zoedud4dtj7dik3acol6myfqalzn16pg1v45lgfptesltvjm9f2kt1nedavkmve +use +listid +ape +fetchmail590 +spamproxyd +sender +end +precedence +07 +message +didnt +its +errorsto +apple +hot +turk182chipwarenet +7bit +taggedabove10000 +cc +listpost +c +shows +into +goes +were +script +special +guess +more +legs +microsoft +listmanager4084623190753220020820121844mothlightfastmailfmls1sendoutmailcom +125629 +form +shoe +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +size4tell +specialistabr +compressability +close +manual +fri +xoriginalto +capture +8219575100 +debianuserrequestlistsdebianorg +3 +sfnet +day +contenttype +xmozillakeys +helpunsubscribeupdate +g8ohvjf28951 +mailqy0f175googlecom +may +shania +since +away +01 +and +abrtshirt +record +received +decay +setup +one +bush +later +list +cif +mailtoforkrequestxentcomsubjectsubscribe +phobos +usually +the +only +freshrpms diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_23.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_23.txt new file mode 100644 index 00000000..64bf5fa2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_23.txt @@ -0,0 +1,270 @@ +hostyle +risc +302 +isnbspqqqqqqqqqqcnetnewslettersexamplecomp +butterflies +for +postfix +politics +mailtodebianuserrequestlistsdebianorgsubjecthelp +network +find +smtp +weight +noninfringing +href3dhttpwwwcomicscomwebmail +managed +cddb +g7fkd8717169 +resentmessageid +verify +account5 +this +minoltas +death +autolearnham +in +have +your +returnpath +imap +are +is +mon +northern +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +impress +versiontlsv1sslv3 +sends +00000000 +133517 +subject +already +qmqp +weekend +xmailmanversion +micalgpgpsha1 +email +httplistsdebianorgv2mb36d43c31005010313gc968713chc61e587759958e7dmailcsminingorg +11 +0000 +what +different +would +moreplicates +0108 +world +own +etc +so +081538pm +keep +lairxentcom +domain +re +present +cest +mailtoforkadminxentcom +httpwwwlifetimetvcommoviesindexhtml +analystsbrbintel +httplistsdebianorg4bec15064090508studentulgacbe +just +325 +facetony +it +esmtp +cable +listsdebianuserlisztdebianorg +a +sometimes +deliveredto +isoiec +localhost +httpusclickyahoocompt6ybbnxieaamvfiaa7gsolbtm +chirac +localhostlocaldomain +useragent +dogmaslashnullorg +because +blended +ldowhitelistratwaregeckobuild +pdt +has +messageid +enlt +sports +10 +0700 +xspamstatus +or +above +20985218210 +readonly +03 +width1br +2007091301 +gmail1 +color666666 +prize +pgpsignature5 +i +127001 +dumb +2008 +that +alternat +mentions +ote +to +tls +half +washington +singledrop +20080610 +license +web +utc +bank +findings +c2lnbi5jb20vcnbhmasga1uddwqeawifodadbgnvhsuefjaubggrbgefbqcdbayikwybbquh +department +5eb1e3ed6a +home +contenttypecontenttransferencoding +x1120081209 +um +practical +mlsubscribertechcsminingorg +123929 +190200 +was +resentdate +inreplyto +references +hold +mxgooglecom +6 +1 +replyto +go +xspamlevel +id +someone +007 +port +on +e20cs468587wfb +20100523232855gb26914penguincodegnomeorg +plugging +reflexive +date +jalapeno +exmhworkers +authenticationresults +score160 +googlecom +workaround +srchttpgservzdnetcomclearoutboundgifappid2emid25136482nle539issue20020716 +courierimap +by +then +unsubscribe +listmasterlistsdebianorg +gonzowaiderie +mta +xmailinglist +2525 +much +0100 +xamavisstatus +again +src +perform +not +listsubscribe +totalitarians +times +with +from +cnofws +spamassassintalklistssourceforgenet +mimeversion +size1 +supporter +spamassassin +2 +214054 +httplistsapplecommailmanlistinfox11users +could +listid +make +sent +charsetusascii +hrefhttpsupportmicrosoftcomdefaultaspxscidkbenusq321126httpsupportmicrosoftcomdefaultaspxscidkbenusq321126a +xaccountkey +tips +sender +end +precedence +commoditizing +dinosaurs +johnson +skeptic +its +errorsto +smx +httplistsdebianorg2010041516570240bbaa5babydosstargateorguk +aug +cc +listpost +smarthost02mailzennetuk +shows +knaana2xq8lpi6rtgd06yimcbrme45ih +starting +ignored +case +uncomment +17tvuk0001cs00 +81168116 +manual +sp +actual +erttd +many +antics +carrbfonttd +debianuserrequestlistsdebianorg +wmv +intmx1corpspamassassintaintorg +byte +184649 +190453 +group +known +contenttype +dnsfromrfcwhoisfourlafvgtmmultioddimprononcable1imprononcable2 +xrcvirus +suburban +of +minded +packages +and +10218114 +record +debiankderequestlistsdebianorg +received +one +suite +eugen +ttbomk +extremes +wed +list +antvilletreffen +32055 +norte +weld +valhalla +size1binteractive +sitenamewhatever +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_24.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_24.txt new file mode 100644 index 00000000..665d943d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_24.txt @@ -0,0 +1,285 @@ +17ui1g00011d00 +relay1applecom +used +e +namesignatureasc +copyright +trouble +oldreturnpath +for +postfix +puslingcom +post +this +in +have +041224 +ieyeabecaayfakvpfwsacgkqmrvqrkwzhmekqwcgpypb8xu7tcwsj0drznn7goe +off +returnpath +once +imap +ftpchdebianorg +testsldosubscriberldowhitelist +1ocfnr0007x6uv +focusclick +00000000 +french +subject +c9b11d289c04ce70215ea0435da +relay +shortly +worse +size +produced +listarchive +than +disperse +score49 +2461113164 +field +listunsubscribe +jmjmasonorg +0000 +egroupsew082 +20020719052006b12fa2940a1xentcom +jmlocalhost +jack +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +mplayer +synaptics +if +fallback +blocks +so +thu +keep +do +tue +metaphors +domain +run +cut +f +users +mnenhy076666 +imaginary +blow +132624 +copies +esmtps +required40 +325 +it +esmtp +mailtoexmhusersrequestredhatcomsubjectunsubscribe +listsdebianuserlisztdebianorg +os +a +subrange +dont +deliveredto +job +arsasha256 +resolvedipisnothelo05 +localhost +as +dogmaslashnullorg +because +size3d1img +spfpass +composer +has +messageid +popup +220233140121 +dkimneutral +user2injgi2dslmindspringcom +lower +addressed +notinsblxblspamhaus15 +comes +resentfrom +g99jpsl22137 +yusmallb +suggestions +strong +berating +score76 +such +trying +074730 +im +textplain +i +127001 +tomalformed117 +cannot +helouswsflist1sourceforgenet +listsdebianorg +grubgfxmode640x480 +to +monday +re2 +cache +generations +code +you +113115 +exchange +tdtr +15in +amavisdnew +landshark +v1078 +040947 +minutes +insults +alarms +suburb +rlfranksender6a38a9paradigmomegacom +inreplyto +version +references +there +p2g880dece01004110349x335b1dcdx88bdf3472c07fb1cmailcsminingorg +emssrv0spooldir +happened +at +1 +know20 +cipher +brbr +jmwebdevjmasonorg +debianuserlistsdebianorg +go +id +difficult +compatibility +doesnt +b2b352942bb +height10br +xcomplaintsto +5si910059fxm3220100408104230 +listhelp +forkexamplecom +on +cellpadding1 +designates +clientsservers +hypochondria +trtda +how +error +date +062457 +2010 +strange +rare +tooted +bells +authenticationresults +truly +xuidl +them +by +think +he +even +e2mr1813852fac1011274469166544 +listmasterlinuxie +listmasterlistsdebianorg +actually +will +002c01c22f674216c280f264a8c0sabeoie +xbeenthere +back +sophisticated +much +0100 +again +september +cd7d2d63c894 +not +listsubscribe +with +from +team +everythingbr +weber +mimeversion +enht +configuration +spamassassin +first +hoonting +1022364205 +xmozillastatus +listid +src3dhttpwwwtributecanewsletter48imagesvindieselgif +fetchmail590 +hi +sender +end +precedence +gwb17 +developers +httplistsdebianorgj2nd0bf7b0b1004301943o2dd0f30fwb811b71e1ef8682amailcsminingorg +material +7bit +taggedabove10000 +be +listpost +penned +mailtoforkrequestxentcomsubjecthelp +loss +time +xvirusscanned +multiple +road +s1753845 +case +7 +sendmailcf +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +stanhardwarefreakcom +pinelnx433020903132104011949100000watchermithralcom +videobased +keumri +xoriginalto +about +114258 +organizations +8219575100 +digeststyle +showing +691ae2020 +let +some +212716 +rtrxgoqk6c2kthvah1gditoowzrn6nbsxdvn4xubyz0wgvkquqew9vecpeellkv7fw9i +day +including +contenttype +234217 +other +of +155431 +and +brightly +sponsored +received +leadership +list +nv +hrefhttpclickthruonlinecomclickqe6qg8rqwyqdk1pmprxgugh6nqyvu4r +connections +cdale +nonstandard +directory +technical +the +never +those +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_25.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_25.txt new file mode 100644 index 00000000..a97b49a1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_25.txt @@ -0,0 +1,248 @@ +interesting +trouble +for +postfix +tapes +smtp +sideeffect +mac +body +verify +account5 +this +provided +have +amazon +returnpath +1b95d13a48c0 +is +mon +testsldosubscriberldowhitelist +listssecurityfocuscom +mozilla50 +subject +wednesdayi +soon +b6a0b2d0c78 +listarchive +xmailmanversion +11fe0d07b7c36ae000006674e24bd201b2e0b4 +applications +listunsubscribe +virtualincludesubscribesidetxt +thought +but +0000 +debian +jmlocalhost +helodomain +inbox +domainkeysignature +concreteeye +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +chipset +forkxentcom +front +systems +if +051929 +gldudebianuser2mgmaneorg +virtualincludesubscription2txt +so +xacceptlanguage +southwest +domain +howtos +re +host +music +receivedspf +result +exim +utf8qdwpreheun7x1kjxybf0zhlwsnsyjru4abtypxviyxqnqlrxwtcsj0a096gud +0400 +elias +mailtoforkrequestxentcomsubjectunsubscribe +edt +e580 +it +sun +xoriginaldate +esmtp +local +closer +os +a +deliveredto +regdwfcom +localhost +2011 +dogmaslashnullorg +1932521969 +pdt +hollywoods +messageid +unknown +0700 +old +202200 +wayside +bayes002 +httpxentcommailmanlistinfofork +or +seems +resentfrom +uswsflist2sourceforgenet +interested +g8nlixc03143 +each +2007091301 +contenttransferencoding +icsnewslettershdanchorgif +im +textplain +127001 +reporting +that +out +frame +frankly +newsx +isnbspqqqqqqqqqqzdnetspamassassintaintorgfontp +to +shot +now +you +spamassassintaintorg +utc +bank +most +here +week +inreplyto +long +travel +mxgooglecom +at +1 +replyto +xsender +shorter +id +folder +enemy +xcomplaintsto +listhelp +insist +can +on +ancient +pid +authors +how +date +pd2ml3sossvcprodshawca +supports +2010 +me +msncouk +by +then +rwrr +think +unsubscribe +lisztdebianorg +processed3ddestinationdconso +shfileutils +should +mta +xmailinglist +ldowhitelist5 +2525 +0100 +again +not +listsubscribe +with +hits78 +from +selling +5dd6f13a564f +rv100 +httpslistmanexamplecommailmanlistinfoexmhusers +linux +both +xmozillastatus +use +listid +jmasonorg +sent +oral +charsetusascii +fetchmail590 +couplings +kids +9222458232 +sender +precedence +httpsvcsafferonetrmphprjkeating +crelaxedrelaxed +its +which +cooperating +7bit +shape +xpolicydweight +hype +nations +get +supported +httpradioweblogscom0100945 +catchy +worlds +6416122236 +21550 +xvirusscanned +latter +legs +sort +emailed +required +81168116 +thinking +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +xoriginalto +many +marginheight0 +8219575100 +debianuserrequestlistsdebianorg +3 +sources +no +jokerunibech +top +17128113151 +white +policy +version325 +theres +contenttype +user +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +and +record +urlhttpwwwenglishupenneduafilreis50sschleslibhtml +received +device +href3dhttpclickthruonlinecomclickq3d1frvwqi58w5nhc5a01x +high +binge +shootout +list +o +phone +pinelnx44402100712315604199100000urgentrugacbe +the +memory +hed +xpriority diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_26.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_26.txt new file mode 100644 index 00000000..1fe0e948 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_26.txt @@ -0,0 +1,300 @@ +submitted +alltrusted +e +0300 +printer +9126mm +for +postfix +rpmlist +mailtodebianuserrequestlistsdebianorgsubjecthelp +101426910 +encodingutf8 +smtp +bts +verwendung +titlecnet +xmessageflag +2418 +this +in +19415019111 +women +have +your +returnpath +jul +imap +fierce +iluglinuxie +possibly +is +mon +d13si2423542fka220100430010606 +show +likely +subject +aaa19167 +produced +listarchive +than +fairly +469 +2002 +want +applications +054926 +height9a +grossi +connect +thought +subjects +0000 +unavailable +jmlocalhost +ehmadscientistcom +what +651page +chaos +25si1493039fxm4920100408164221 +hundreds +ist +if +world +apr +browser +fallback +mgraphicsanchordeskfrontpageheader2yourgif +httpwwwnewsisfreecomclick18381140215 +histoptionsnbuckets +limited +version240cvs +testsdnsfromrfcwhois +tue +2adrhss8cww8rvqkil6h63wmgzf7ha6zr8ufrnboupmaczl8atusjmbkl0hbtwmpwxgo +newsbfontbr +mesa +20100507070848362mspdebianorg +unnerving +re +miles +testsawlemailattributioninreptoknownmailinglist +sponsor +http13624411618000cgibinmarkcgi +receivedspf +r13mr2301595wak111271447853982 +crashes +edt +required40 +325 +h4y5qrrnsuguyq33kj2clin8behbrcuwc3zv4 +it +site +sun +esmtp +httpxentcompipermailfork +protocolapplicationpgpsignature +listsdebianuserlisztdebianorg +deliveredto +xmozillastatus2 +localhost +05 +hdds +useragent +as +dogmaslashnullorg +spfpass +11pt3b +messageid +10 +unknown +text +solved +notinsblxblspamhaus15 +told +shaneryaneducationgovie +pcmsurround71 +httpxentcommailmanlistinfofork +particular +0x015a4e35 +xrcspam +answers +pgpsignature5 +2008110401 +src3dhttpwwwzdnetcomgraphicscleargif +userid +service +textplain +i +127001 +outlook +xpyzor +whatever +work +srchttpwwwzdnetcomincludeadsjsrgroup2766 +postgresql +out +helouswsflist1sourceforgenet +180452 +to +sep +qmailldap103 +national +xlevel +you +spamassassintaintorg +qaa07539 +mp7e1h92d969a6qtpeog86of49fry6or +single +these +gmailid127d42d7d9120bf0 +was +resentdate +wrapwhat +085955 +zlinuxmanwowwaycom +164857 +links +there +change +titlezdnet +09bf69fe8b +height3d349 +af6decorecpp291 +cipher +debianuserlistsdebianorg +8d549279 +184351 +preinstalled +disaster +doesnt +19212399 +also +port +can +on +g8qfwjg25145 +link1 +appears +mailin13applecom +date +friends +gmailid128cadef90bf1112 +takes +pan20100511164828csminingorg +2010 +mail +video +by +hrefhttpclickthruonlinecomclickq89h28bqj1qvniv5fkumbxb6w5wwcr +remaining +lisztdebianorg +official +rsync +httpwwwgeocrawlercomredirsfphp3listrazorusers +ago +default +will +httpfreshmeatnetprojectscfg +back +0100 +att +specific +listsubscribe +with +from +still +wilt +125733 +mimeversion +0500 +accomplish +inc +amepkebldjjccdejhamiieajfiaaejwcseucscedu +fbiwinter +spamassassin +alone +163008 +nas +nbspbsome +both +xcheckerversion +could +listid +jmasonorg +charsetusascii +fetchmail590 +end +permitted +precedence +anuradha +unfortunately +yyyyspamassassintaintorg +score7 +contentdescription +211006 +7bit +taggedabove10000 +tolerable +be +listpost +ok +worthwhile +time +lnv106sbdfkide +prolific +more +mailing +stylecolor +merit +81168116 +really +sgamma +start +g93h04f17315 +ce89e27c8bbf3 +manual +183635 +fri +xoriginalto +netblocks +015538 +width5 +adrian +debianuserrequestlistsdebianorg +made +mailtoquicktimeapirequestlistsapplecomsubjecthelp +width3d100 +no +requests +divarchive +notinblnjabl15 +settler +colore4e4e4 +contenttype +xmozillakeys +80236232177 +10223132210 +mentioned +may +17216196 +convert +other +of +ive +states +received +one +high +right +waa23907 +list +beyond +phobos +windowmaker +200209092354g89ns8g04304lin12triumfca +3d +directory +the +5ewqfbn6k0j8 +radioed diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_27.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_27.txt new file mode 100644 index 00000000..f9c2d6db --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_27.txt @@ -0,0 +1,293 @@ +blu0smtp7 +valuehttpwwwlockergnomecomimageslgtoppergifinput +mgbyosemitenet +trouble +alink +dbrayfordcsminingorg +for +postfix +testsalltrusted18 +mailtodebianuserrequestlistsdebianorgsubjecthelp +support +smtp +false +question +20020204210703ijfy1068fep02appkolumbusfipihlajakotilo +post +test +20100506140413721bssiguanasuicidenet +httpwwwiscorgservicespubliclistsfirewallshtml +x +due +any +arsasha1 +body +cultures +clientip8219575100 +account5 +this +quicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +autolearnham +in +move +provided +tatankastortekcom +off +rootdevhda1 +returnpath +greg +imap +mailtodebianuserlistsdebianorg +are +is +testsldosubscriberldowhitelist +6219710167 +require +h17mr2281000fab451273377014844 +x8664 +fw +v2y8f23146b1004061031i5e410c8dj686a5625f24e0f43mailcsminingorg +show +subject +nobody +iso88591qptbs5oo9toumhbdke23otci0a093cbil8dtogzk0bf2yq35wsewvyet2e +am +lurking +para +469 +2002 +xmailmanversion +samu60csminingorg +email +listunsubscribe +rhughes +wrong +mia +root +jmlocalhost +wjqccl8tyur4kqjghsvsuhlxegzskxhwakjc1okjeiul0 +what +g8o80bc26569 +yyyylocalhostnetnoteinccom +xcmscore +httpwwwdigitalmediatreecomjimweblog +kerdylio +063229 +return +ist +if +good +rafael +world +br +g8o80cc26642 +fallback +srchttpwwwtheserversidecomimagesborlandgif +leptosomic +m8cs67090wfj +generate +so +absent +do +all +60026000000 +helorovervipulnet +domain +re +20771218155 +alton +hit +xegroupsreturn +connectivityfonttd +expresscnets +tls10dhersaaes256cbcsha132 +came +seconds +archivelatest576724 +it +151703 +esmtp +listsdebianuserlisztdebianorg +24 +namerestrict +163501 +deliveredto +rv1919 +xmozillastatus2 +resentsender +night +width160 +energy +insights +localhost +as +fed +thing +configure +spfpass +hrefhttpclickthruonlinecomclickq75whwtiku3zcagrokrjups7llz5zr +pdt +5 +has +messageid +rackspace +10 +lands +isnt +remind +xspamstatus +163225 +contain +9si6249810fks5620100411013312 +provide +far +07aa329413e +contenttransferencoding +framework +013506 +reason +im +building +textplain +i +127001 +leaving +071240 +that +uswsffw2sourceforgenet +sex +to +singledrop +jdk14x +tremendous +betas +103113 +did +edit +mlsubscribertechcsminingorg +references +eduardokalinowskicombr +10223589 +russians +httppatrickwebcomweblogcategoriesmyhobbies +tkerror +103396471712638camelamd1800 +id +listhelp +can +on +17 +xdebian +date +happens +190123 +jalapeno +2010 +hope +forgot +googlecom +artist +hugodebianfvwm +by +httpsecuritydebianorgpoolupdatesmainppostgresql83libecpgcompat383110lenny1i386deb +think +unsubscribe +inventor +rssfeedsjmasonorg +listmasterlistsdebianorg +expect +will +written +xbeenthere +0100 +prevents +unless +wont +listsubscribe +score110 +with +from +team +20 +suspect +ability +mimeversion +enht +recycled +express +httplistsapplecompipermailx11users +xcheckerversion +wifis +saa22715 +listid +make +64b +fetchmail590 +333 +sender +permitted +precedence +eas +eest +sweeping +developers +didnt +which +property +be +listpost +titlecreate +merely +large +porstacshelsinkifi +ok +theyre +mailtoforkrequestxentcomsubjecthelp +hz +connection +xvirusscanned +sites +anything +thousands +easytoread +leftwing +150911 +sr71 +xacceptlang +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +manual +none +httpwwwpaulkohlernet +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +contains +cisco +alt3dmore +no +archive +some +spending +gcc44 +visibly +contenttype +xmozillakeys +user +httpboingboingnet85528842 +of +and +received +7128494401050921271557302610javamailrootmd01wowsynacorcom +023043 +list +privs +3dph1 +fe734aa0f08144ebad86a8676e5cdd8aapplecom +directory +technical +debianlivelistsdebianorg +50005 +the +protectionsownwership +pf22hxseyrvfdg6iqrtxfu8pxestskwrkhd3i +never +those +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_28.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_28.txt new file mode 100644 index 00000000..5d1b1b0e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_28.txt @@ -0,0 +1,271 @@ +rate +43ac144155 +looks +4th +undefinedundefined +copyright +trouble +oldreturnpath +behind +for +postfix +hcm +politics +mailtodebianuserrequestlistsdebianorgsubjecthelp +maybe +183417 +lots +egwnnet +people +chris +body +20020829205404exzw15998imf13bisbellsouthnetadsl1572310msybellsouthnet +clientip8219575100 +height3d32920 +in +certificate +htmlmessage1 +bgcolore5e5e5img +returnpath +imap +mailtodebianuserlistsdebianorg +worry +is +mon +source +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +course +slow +subject +alloc +httpcvemitreorgcgibincvenamecginamecve20093876 +hostname +qmqp +posts +contentdisposition +hrefhttpclickthruonlinecomclickq9dkkcwqv0hoccbmdnuklvd75wkzzpr +g89jafm01952 +email +listunsubscribe +solid +but +161653 +9 +uncle +future +forkxentcom +3e +if +fallback +required53 +btwwpo9f291xkktzssqtutlxf9t3aheqqlttyyvbznq2rmdhgsqlxtt99xnsf3tapm +thu +keep +him +tue +divtdtr +institute +domain +orwant +thanks +previous +114147 +re +look +advanced +cut +effect +httplistsapplecommailmanlistinfojavadev +enter +152529 +uri +edt +seconds +just +it +sun +esmtp +testsawlknownmailinglistsignatureshortsparse +self +a +splitting +deliveredto +mean +information +src3dhttpwwwzdnetcomgraphicsanchordeskf +4bc02e709020904homenl +wrote +localhost +as +dogmaslashnullorg +thing +well +pdt +messageid +10 +john +dkimneutral +e20 +httpiguanasuicidenet +141141 +nice +popular +or +above +computer +belong +xrcspam +trying +haataja +hrhotmail +1204 +17cvgp00014l00 +finding +textplain +127001 +that +logic +selfesteem +090015 +190353 +cmd +to +singledrop +xlevel +now +utc +auth02nlegwnnet +jmilugjmasonorg +20100414173940ga17726acampbellorguk +bards +nextpart000042601c26ebd8df911e0 +references +change +hrefhttpnewscomcom12001120933954htmltagstnlhorizontalxprotxtnewsvisioncnet +yyyylocalhostspamassassintaintorg +added +id +lwv26awu +p14mr1394629fah471271513762089 +doesnt +port +listhelp +can +on +m8cs106909wfj +stan +mail8101 +detected +date +wrotebrblockquote +463 +order +by +think +unsubscribe +0m1180s +rssfeedsjmasonorg +comment +should +story +missing +2525 +xbeenthere +499 +0100 +multisearch +not +listsubscribe +20020909193637590bbb1whatexitorg +disclose +times +pc +with +from +still +1200 +bhdjzp4nl5m6c50bwczf7m1hnnfpbq2o5kky2lycksl4 +foeman +valentuathaorg +mimeversion +colspan +enht +bgcolorccccccfont +spamassassin +first +belphegorehughesfamilyorg +turn +life +lot +plain +listid +make +jmasonorg +charsetusascii +areas +report +width1td +sender +whole +redesigned +requires +afkkriqjqggea10 +001937 +7bit +aug +box +be +meet +184858 +18 +6416122236 +width100font +asset +headericsminingorg +xloop +uy5svrozctmu +nazis +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +card +81168116 +really +thinking +manual +pro +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +fri +xoriginalto +simply +af6decorecpp350 +jmdeadbeefjmasonorg +115318 +adrian +kde +see +srchttphomecnetcominlshrhcmostpopgif +antelivefreenet +no +hassles +archive +sfnet +top +k +needs +17128113151 +g715ra232199 +enus +when +day +sid +of +havent +01 +and +speak +received +c6e9a13a62d9 +list +cheating +rootlocalhost +the +maintain diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_29.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_29.txt new file mode 100644 index 00000000..684bf9ca --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_29.txt @@ -0,0 +1,278 @@ +rate +g7djqr416060 +particularly +boingboing +ratehard +v60026000000 +oldreturnpath +qmail +xmailer +for +postfix +smtp +formatflowed +104836 +account5 +height3d76 +returnpath +imap +iluglinuxie +are +is +mon +learn +his +too +testsbayes002 +greetings +likely +subject +try +qmqp +listarchive +valuehttpwwwsearchcomsearchtagexelectronicsdispatchsrchwebthe +adsl +shell +2002 +bulk +listunsubscribe +jmjmasonorg +whether +0000 +map +background3dhttpwwwcomi +201 +jack +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +regime +yyyylocalhostnetnoteinccom +fred +forkxentcom +felt +mailtoobjclanguagerequestlistsapplecomsubjectunsubscribe +testsawlknownmailinglistquotedemailtextreferences +going +httplistsapplecommailmanoptionsjavadevmlsubscribertech40csminingorg +huckleberry +so +ecomclickq3d78d5asidukhx8ilqr5vnmun5xsporr +keep +all +tue +lairxentcom +u +run +look +confined +page +exim +set +0400 +bird +esmtps +405388 +just +required40 +325 +left +it +esmtp +httpxentcompipermailfork +protocolapplicationpgpsignature +two +entire +a +cnet +msogenericfontfamilyroman3b +deliveredto +pages +wrote +darren +localhost +within +as +storyba +g6hlklj24842 +spfpass +training +pdt +has +messageid +optionsabr +saulcisupennedu +popescu +lugh +john +w39mr393328mue201271360368122 +dkimneutral +phoboslabsnetnoteinccom +0700 +height3d15f +tb0gzvvivaw1rozqpsu9gc +11fe0d07b7cf7ae000003b23ae4bdb81520458 +bolceri +xrcspam +0200 +way +textplain +127001 +8bit +files +shareholders +that +out +3d1e5c99 +to +machines +sep +3lxox7qdget +hrefhttpwwwgnomedexcomimg +charsetiso88591 +singledrop +key +20080610 +mimeversioncontenttypecontenttransferencoding +103113 +archivelatest575097 +hat +12111175112 +143825 +etccrontab +here +internet +inreplyto +sequences +there +mxgooglecom +transport +at +id +handled +port +listhelp +mailtospamassassintalkrequestlistssourceforgenetsubjectsubscribe +cellpadding1 +same +saa15363 +after +192959 +140715 +date +2002fontbr +servicesfonta +allow +them +archivelatest573879 +manager +by +boundaryexmh1581861840p +think +unsubscribe +classmates +g7se9iq22132 +ago +xmailinglist +new +ldowhitelist5 +xbeenthere +0pt +xamavisstatus +listsubscribe +exmhworkerslistmanredhatcom +with +from +stuff +fixed +books +mimeversion +priority3d67 +enht +next +2 +dvds +include +timeline +linux +xcheckerversion +pass +xmozillastatus +attachment +jmasonorg +sent +charsetutf8 +news +115827 +wwwdbsinfocom +hi +xaccountkey +sender +precedence +esecurity +position +message +mr +stephen +file +its +under +which +102232669 +say +mailfx0f47googlecom +4096 +7bit +listmanspamassassintaintorg +listpost +c +into +considered +1086639 +get +171731 +122431 +3d9b62478090205endeavorscom +configured +65194124178 +microsoft +howells +type +gig +bssiguanasuicidenet +141842 +urlhttpwwwstudentmontefioreulgacbemerciadri +intentionally +hd +81168116 +andor +discount +actual +xoriginalto +105 +8219575100 +ricklinuxmafiacom +3 +trate +lie +no +sat +rv11 +performance +50769 +group +theres +aforementioned +xmozillakeys +may +events +devselfast +of +since +audibility +and +xstatus +record +received +wouldnt +broken +the +freshrpms diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_3.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_3.txt new file mode 100644 index 00000000..6e7cb9ab --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_3.txt @@ -0,0 +1,262 @@ +rootlughtuathaorg +sunday +trouble +oldreturnpath +xmailer +nametopa +concede +for +postfix +g2icdeec9051005080532z810046edveb2a8ea4c51fa0d9mailcsminingorg +s +post +bodyhtml +begin +any +chris +deb +clientip8219575100 +account5 +this +in +certificate +malheurs +have +your +returnpath +1721652254 +communication +faster +knows +overregulation +spamassassindevel +throwing +f563 +bit +subject +pgp +given +qmqp +listarchive +posting +kilograms +garst +email +09 +secrets +24eecdfb60 +listunsubscribe +quotedprintable +jmjmasonorg +dsblorgskip0 +sure +0000 +debian +url +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +477594 +yyyylocalhostnetnoteinccom +daniel +image +fallback +required53 +nonsilly +clients +eat +normal +22 +all +wplayingbuttonmoreogifhttpwwwtributecanewsletter48imagesn +lowlevel +u +thanks +re +linuximage26323686 +host +look +loud +razorusers +dcsminingorg +required40 +11mbpsbrwireless +sun +esmtp +mailtoexmhusersrequestredhatcomsubjectunsubscribe +17a14c0002ww00 +serverlayout +dont +e2fsck +deliveredto +xmozillastatus2 +resentsender +pleased +localhost +bad +as +dogmaslashnullorg +g8ukr2f11316 +configure +5 +messageid +10 +tv +0700 +old +174727 +writers +contain +clamp +contact +ut +214223 +wyf22 +0200 +contenttransferencoding +dialling +textplain +i +until +127001 +that +java +1o8op70005igcl +listsdebianorg +hits00 +to +quote +66187233211 +issue +105549 +you +passing +utc +bank +replyby +userfriendly +mlsubscribertechcsminingorg +inreplyto +believe +found +at +id +httpskobblogspotcom +over +c28si5457288fka1420100501112250 +listhelp +on +17 +37441033577684munnariozau +codebases +being +related +same +designates +consistent +tr +b +date +alignleft +who +a8cc113a5297 +bolympus +things +solitaire +by +rwrr +unsubscribe +208971325 +listmasterlistsdebianorg +few +2525 +xbeenthere +0100 +not +listsubscribe +with +from +responding +mimeversion +o2r6d4219cc1004091025s676528f9s81b1926f2acdd799mailcsminingorg +6521443159 +spamassassin +091826 +pass +hrefhttpclickthruonlinecomclickq88ec5pqqkxb5kg4uyru358atsbbir +use +rarely +listid +charsetusascii +dispatches +precedence +101428120 +which +coins +an +ttyx +pmyimage +air +6ebzto5ezlavfbfm5ulbliszt +httpwwwfragmentizedcomjaykul +taggedabove10000 +aug +cc +be +street +get +her +mailtoforkrequestxentcomsubjecthelp +time +better +were +thousands +ask +xloop +i386redhatlinux +close +decided +g8ufio223184 +xoriginalto +fel +transmission +8219575100 +see +messnum +debianuserrequestlistsdebianorg +3 +shes +no +we +moved +slaves +archive +filetxt +ubuntu +policy +debiansid +version325 +78gb +contenttype +xrcvirus +of +002732 +01 +and +sponsored +received +one +ludjmpizoykalcfyq05lbliszt +pgpsignaturercvdindnswlmed +speed +breath +list +de915934cbf911d6a1d9003065f93d3atopsailorg +phone +abvsfo1acagent7 +created +phobos +v1410 +arguments +burdened +the +gcd +sol diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_30.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_30.txt new file mode 100644 index 00000000..170968be --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_30.txt @@ -0,0 +1,271 @@ +rate +163500 +please +oldreturnpath +read +xentcom +for +postfix +membership +gnupg +x +package +verify +clientip8219575100 +huge +sounds +this +0801 +7uwkiqypnebag9c2mc7lbliszt +committed +in +have +vanished +your +returnpath +once +choose +radiation +multipartsigned +testsbayes002 +argentsrv2com +00000000 +show +ehlo +server +subject +printers +121 +using +listarchive +ryan +excerpt +2002 +mailtorazorusersrequestexamplesourceforgenetsubjecthelp +09 +listunsubscribe +quotedprintable +decide +jmjmasonorg +ywh40 +0000 +debian +9 +shared +e6f3016f03 +style +ist +line +if +fourla01 +oct +purcheses +up +so +httpclickthruonlinecomclickq9etwqoqtmqjgrfwghhscn0u7nqud4r +httpsearchbetacpanorg +welltraveled +structures +49a9816f03 +lairxentcom +print +domain +kde443 +club +re +overpriced +sale +users +joelonsoftware +data +receivedspf +exim +before +112529 +301793 +mailtoforkrequestxentcomsubjectunsubscribe +edt +required40 +325 +it +esmtp +074527 +10223513 +893893 +os +a +href3dmailtooneincomelivingremovegroupsmsncom +147they146re +deliveredto +resentsender +httplistsdebianorg4bed2a894000902allumscom +wrote +071313 +localhost +though +as +marked +010602 +spfpass +pdt +5 +regular +messageid +beating +10 +height8td +limbonull +servicea +initially +notinsblxblspamhaus15 +quite +gnulinux +letters +resentfrom +looking +cios +each +contenttransferencoding +multipartalternative +im +jones +textplain +invoked +i +127001 +files +19412514545 +that +out +cannot +moutians +19216810053 +ace +to +tls +supplied +metapackage +8739yjfge7fsfmerciadrilucastationmerciadriluca +now +hmimeversioninreplytoreferencesfromdatemessageidsubjectto +20080610 +modern +td +egp +know +perfectly +bank +did +xspam +solution +mlsubscribertechcsminingorg +was +mailtoforkexamplecom +resentdate +references +namesubmittdform +87d3xrd8hufsfsmailinffhbrsde +debianuserlistsdebianorg +300 +remote +id +passwords +forkexamplecom +on +how +7412420024 +date +hits +uals +envelopefrom +too2e +by +then +filters +love +via +spain +xbeenthere +back +peoplesofta +0100 +their +bedford +nate +with +from +still +wood +20 +mimeversion +spamassassin +first +023030 +media +pass +listid +put +fetchmail590 +hi +sender +precedence +07 +ultimate +its +under +afkkriqjqggea10 +065214 +taggedabove10000 +reads +champion +goes +where +body8bits15 +mailtoforkrequestxentcomsubjecthelp +6416122236 +bits +xvirusscanned +several +install +i386redhatlinux +gecko20100411 +sametime +size4tell +81168116 +sgamma +gmailid1284c780487ba08a +start +manual +fri +many +engineering +copied +mistake +made +glettersummarylocationhover +no +9dd154ff43d8dd67cbc9e92a91156362 +iso88591qyjru4abtypxviyxqnqlrxwtcsj0a096gudsnwt0nxblkvuwjblkwvjr +spamd +when +chips +policy +contenttype +required70 +131245 +of +e9crit +20100503 +and +received +devices +without +list +testshtmlmessageimprononcable1 +nt +hrefbuzzimg +created +s0832016219710167 +the +send +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_31.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_31.txt new file mode 100644 index 00000000..a51c9d5e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_31.txt @@ -0,0 +1,286 @@ +target3dblank +oulook +10024 +httpwwwcompletereviewcomsaloonindexhtm +neugensliberoit +mailtodebiankdelistsdebianorg +desktop +xmailer +able +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +network +052632 +smtp +12 +working +51 +body +clientip172541337 +resentmessageid +registered +account5 +menu +in +cached +your +returnpath +zdnetbfontbr +hmessageid +bouncedebiankdemlsubscribertechcsminingorglistsdebianorg +are +is +httplistsdebianorg4bd9ce8690009hardwarefreakcom +task +testsbayes002 +subject +36864 +collection +57937 +using +am +2d20 +shouldnt +listarchive +2002 +xmailmanversion +12807050632050161270780727932javamailrootmd01wowsynacorcom +email +1993 +model +listunsubscribe +012023 +dmmpjsrc3dai0d0d0dfunction +jmjmasonorg +0000 +dropped +released +jmlocalhost +drunken +113426 +ed1fa16f03 +shared +would +httpwwwaskbjoernhansencomarchives20020924html +ist +if +unsubscription +christmas +required53 +1layqdjehgkkpalg6gvza +namedescription +messaging +our +normal +keep +do +bhl1wigmdthc6srkmsnehxql58zchewddyz3yw4nnava +lairxentcom +asap +httpwwwlinuxiemailmanlistinfoilug +u +hall +re +trick +pinebso444020822152438028231100000crankslacknet +exim +audio +e7a6a2d0d79 +hrefhttpclickthruonlinecomclickq42rf7cipkskxmmrb8pz3f74emirdr +dcsminingorg +edt +325 +it +esmtp +a +sometimes +deliveredto +resentsender +wrote +energy +102221 +localhost +as +dogmaslashnullorg +build +marked +23b4c27550a9a +airkvbgf3v46fcoh1nyafmw17 +messageid +10 +car +finally +ilug +notebooklaptop +mgbdebianyosemitenet +0700 +products +or +above +resentfrom +come +switching +doctors +contenttransferencoding +brdivdiv +required50 +2008110401 +im +stop +zzzzlocalhost +textplain +i +127001 +whatever +out +heck +to +srchttpshoppercnetcomiceps12086539401201gifa +charsetiso88591 +least +met +debug +c078416f03 +you +18si1915898fkq420100421110532 +utc +bank +did +around +fact +craigdeersoftcom +1031314884 +these +mlsubscribertechcsminingorg +was +there +found +596 +gw2panasascom +nothing +been +debianuserlistsdebianorg +law +id +libdvdcss +listhelp +driver +145856 +trtdimg +0xdeadbeefrequestpettingzoonet +highest +10br +date +threadindex +2556491791925261270748527351javamailrootmd01wowsynacorcom +2010 +of0d +me +xuidl +6c9353ed4c +by +indeed +helo +shafer +called +ldowhitelist5 +xamavisstatus +wont +environment +with +from +refer +assume +selling +cnofws +152931 +person +michlmayr +spamassassin +stopped +howls +linux +occurred +exit +listid +jmasonorg +charsetutf8 +done +merciadri +mating +didnt +leading +errorsto +which +pan20100425105933csminingorg +maddest +an +scientists +dated +taggedabove10000 +archivelatest573790 +quizzes +7262002 +be +xpolicydweight +blogabr +into +vp8 +worlds +resizes +sites +guess +objects +headericsminingorg +jerry +xloop +unpacking +10143320 +trunk +stylecolor +gecko20100411 +81168116 +upgrade +terms +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +fingers +225545 +xoriginalto +started +width5 +8219575100 +showing +no +target3dblankhttpvcspkgorgabr +intmx1corpspamassassintaintorg +argument +needs +enus +gettext +policy +day +contenttype +xmozillakeys +height13 +xrcvirus +possible +of +v17cs9479wfj +newsletters +xstatus +releasing +received +studying +wed +engine +earth +yyyyuseperlexamplecom +phobos +trash +comments +policies +httpslistmanredhatcommailmanlistinfoexmhusers +the +israelis +312 diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_32.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_32.txt new file mode 100644 index 00000000..386e67c5 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_32.txt @@ -0,0 +1,270 @@ +rate +pm +number +for +postfix +rpmlist +mailtodebianuserrequestlistsdebianorgsubjecthelp +20100508104207ga2878heimagjkdk +encodingutf8 +bulkscore0 +preventing +080407 +admin +reach +pdv5 +gods +8 +this +in +certificate +have +your +reported +returnpath +testsfourlahtmlmessage +imap +are +is +fffsecuritya +colorcc0000font +tipspointersetc +mozilla50 +subject +cheap +escribif3 +1006 +2002 +xmailmanversion +listunsubscribe +quotedprintable +connect +david +but +0800 +rssfeedsspamassassintaintorg +would +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +g7h70q622253 +another +devnewdevice +return +closed +ewy6 +apr +090249 +fallback +216136171252 +castileapplecom +151916 +so +xoriginatingip +lrwxrwxrwx +do +all +suggest +domain +book +buildercoma +had +re +testsawlemailattributioninreptoknownmailinglist +000000 +hrefhttpwwwgfx2swfcomgnomespecialboomer +widespread +f +37 +receivedspf +0400 +dcsminingorg +possibilities +tqcl8jfofrhhusr +it +esmtp +fix +listsdebianuserlisztdebianorg +24 +a +5megapixel +deliveredto +khare +localhost +bad +useragent +inline +thing +060658 +xqhkorhq7tr +instantly +5 +messageid +score69 +213527 +fullscreen +selected +whose +bootbr +124917 +contain +or +comes +39efa13a46d6 +171930 +contact +they +problems +arbitrary +firmware +forkadminxentcom +g7b9brj25638 +stop +textplain +i +127001 +work +files +flashbased +out +to +exact +charsetiso88591 +singledrop +g93llbu05318 +you +bank +amavisdnew +17kdru0001sk00 +cluster +tom +single +here +founding +change +something +1244f270b0654 +at +1 +through +xtlssubjectcgb25417s9 +debianuserlistsdebianorg +celebrity +id +offers +cols11 +collecting +listhelp +bascialy +alignabsmiddle +can +on +highwater +black +kyzeqoz8g1ojrmwopejnjelbhzgkmwopdebqwz5u4ic3d1c4cfnvmrqqpjzdgzyvv +punter +my +date +streams +ed7cc1c3b5 +genom +2010 +authenticationresults +xuidl +food +by +142253 +phicsanchordeskfrontpagebgif +returns +unsubscribe +hrefhttpcodeweaverscomproductscrossover +â +lisztdebianorg +httpwwwcrazytracycomblog +written +ldowhitelist5 +xbeenthere +0100 +xamavisstatus +declarations +listsubscribe +e20cs37224wfb +with +from +mimeversion +tdimg +1111 +first +could +use +listid +make +jmasonorg +charsetusascii +fetchmail590 +xaccountkey +trademarkets +regards +sender +permitted +priceperformance +mr +its +errorsto +an +125244 +modem +score7 +7bit +maccarthy +aug +345 +220617 +be +dinter +intmx1corpexamplecom +256962d1ef3 +aptrpmtuxfamilyorg +combo +time +better +were +version250cvs +multiple +sort +install +scrollbarhighlightcolor +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +really +132905 +novel +claimed +srchttphomecnetcominedsitngif +started +8219575100 +showing +142612 +nnfmp +writes +some +does +version325 +ignore +5245868 +contenttype +helpunsubscribeupdate +of +since +packages +and +primary +record +received +jenandtonic +one +without +zwwmg1kds60mpbhqzjuufeivz4rhlniutb1oy4mg65lab7ty8ttmstzaxw1ktfbgq6v +list +arcane +mailtoforkrequestxentcomsubjectsubscribe +124810 +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_33.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_33.txt new file mode 100644 index 00000000..727b819d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_33.txt @@ -0,0 +1,287 @@ +9p +1441 +103443 +624 +104421 +m8cs44860wfj +oldreturnpath +fashioned +read +mass +failure +postfix +encodingutf8 +httpwwwrolandtanglaocomcategoriessoapnxmlrpc +smtp +test +comicscomcomicsdilbertdailydilbertimagesdailydilbertheadermidbot +begin +alien +wasnt +resentmessageid +uswsflist1sourceforgenet +clientip8219575100 +account5 +httpwwwquicktopiccomboinghk9nshvkkrrxi +autolearnham +in +switched +your +slip +internal +returnpath +mailtodebianuserlistsdebianorg +point +fffsecuritya +too +bay +subject +networks20 +path +manuscripts +piles +using +469 +email +listunsubscribe +yes +fundamentalsbrbyahoo +jmjmasonorg +but +goto +11 +0000 +v15 +below +debian +security +be67e6c8431 +url +6541216221 +ehmadscientistcom +yyyylocalhostnetnoteinccom +width3d45 +online +radio +gldudebianuser2mgmaneorg +oct +fallback +required53 +xacceptlanguage +opensourceeeethzch +startup +lairxentcom +domain +pollack +090803 +primarily +had +pcie +re +cest +imprononcable1ldosubscriberldowhitelistmurphywrongword1 +uttered +dynnjablerr0 +recycling +httplistsapplecommailmanlistinfojavadev +every +61 +0400 +burst +idea +clean +required40 +cline +114910 +it +sun +xoriginaldate +demostrate +esmtp +bhwwap0brjmd4vhiueq8togmtcsztbrntweskbi1plk +141317 +24 +gmailid1280f785a65bb1c2 +quotedemailtextrcvdinmultihopdsbl +1237 +deliveredto +xmozillastatus2 +35 +disinterested +localhost +principle +as +80347440cc +butt +spfpass +well +offscreen +has +dateinpast12240992 +messageid +10 +height8td +gross +srchttphomecnetcombgif +b160f16f1e +organizing +spamassassintalkexamplesourceforgenet +httpxentcommailmanlistinfofork +or +comes +resentfrom +splitsabbr +fastest +a4rjf0nbhntpt6i8vq1c9cw17 +340976951424841270571770256javamailrootmd01wowsynacorcom +groups +000021 +they +aaaaaa +contenttransferencoding +framework +p05111a5bb9ae46820c6e66149496 +alttech +textplain +127001 +outlook +that +microsoftr +aaaaarpwkeg +to +purpose +ratwaregeckobuild +singledrop +now +you +web +know +utc +amavisdnew +sign +recently +resentdate +inreplyto +references +q2z951d34ac1004191247oa5db050v24437077ee2cbf9fmailcsminingorg +at +1 +id +modules +port +forkexamplecom +on +ceo +peltonen +date +friends +tex +dunno +2010 +strange +filetimeb59cf9f001c260c6 +625 +googlecom +by +20100507 +then +anger +unsubscribe +even +client +listmasterlistsdebianorg +helo +via +ldowhitelist5 +galilei +xbeenthere +gateway1messagingenginecom +back +0100 +xamavisstatus +little +src3dhttpwwwcnetcombgif +not +listsubscribe +102239978 +with +from +gasp +still +mailtodebiankderequestlistsdebianorgsubjecthelp +20 +mimeversion +heaven +wdd +outgoingsecurityfocuscom +af2sx6b92j091lhfioa9 +hotline +use +listid +fetchmail590 +xaccountkey +hdomainkeysignaturemimeversionreceiveddatereceivedmessageid +friday +sender +end +permitted +precedence +file +which +50abr +an +score7 +original +7bit +be +listpost +reads +account +hyderabad +bits +shortduration +version250cvs +finds +guess +install +rtorvivsnlnet +7 +repeatedly +benefit +bssiguanasuicidenet +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +none +pvb32 +ccg2sjwadlbr +spam +013346 +archive +writes +uptotheminute +some +tx +version325 +bizsmtp +contenttype +may +secure +apollo +other +of +eskso0ztqm4e +mozilla +01 +and +received +runtime +windows +wed +list +0031 +usually +s0832016219710167 +the +39b3016f22 +only +freshrpms diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_34.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_34.txt new file mode 100644 index 00000000..ff220b8e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_34.txt @@ -0,0 +1,273 @@ +b98dd16f6c +httpwwwalertnetorgthenewsnewsdeskn07347915 +210512 +execution +definedconfigsndusbmidimodule +postfix +experimental +ahhh +mailtodebianuserrequestlistsdebianorgsubjecthelp +find +lots +involved +084459 +clientip8219575100 +dkimsignature +400 +autolearnham +width563 +returnpath +imap +rssfeedsexamplecom +parodies +is +mon +among +testsbayes002 +00000000 +show +gecko20020802 +seen +avatar +gmailid128b92888512dbdf +subject +pgp +collection +produced +system +than +familiar +href3dhttpwwwtributecacontestfeardotcom080202detailsaspreferr +2002 +xmailmanversion +5101402002080109374100addec0jidmailnmcourtscom +rpmlistadminfreshrpmsnet +digest +late +yes +rq2d +11 +burgunderstrasse +mine +httpnewscomaucommonstorypage040575037762255e1376200html +wc +what +would +yyyylocalhostnetnoteinccom +forkxentcom +line +boundaryenigf8bc083208413a0c634acbf8 +12742049512021118camelfamilypaciferacom +higher +132732 +qy98 +6e78aa7beaaf01773dc69b0033c9f7d8 +18si798529fks520100419185917 +domain +thanks +attacked +melkkicshelsinkifi +had +re +width600font +widespread +64157321 +barbara +held +set +0400 +just +persons +esmtp +sheet +listsdebianuserlisztdebianorg +literature +tests2 +a +03beta +deliveredto +35 +wrote +kdepim +razorusersadminlistssourceforgenet +as +dogmaslashnullorg +102236577 +well +airkvbgf3v46fcoh1nyafmw17 +pdt +has +messageid +arent +103 +10 +noah +phoboslabsnetnoteinccom +httpxentcommailmanlistinfofork +seems +127280762829087camelpcstevenlan +contact +5446d13a4752 +they +grep +each +contenttransferencoding +forkspamassassintaintorg +required50 +way +reply +pgpsignature5 +textplain +invoked +i +127001 +outlook +8bit +001e680f15c08ca11a0485ba5202 +that +delspyes +out +whenever +thunderbird304 +120627 +gmt +pipeline +to +machines +809122912 +singledrop +probably +xbulkmail +presented +auth02nlegwnnet +gnomiesabr +bz1applecom +t6bulletmudyahoocom +221728 +copybrtdtrtbodytablebrbrbrbrtdtrtablebr +331vamm2 +mlsubscribertechcsminingorg +was +inreplyto +version +priority3d34 +legal +sndhdaintel +g990ahk08455 +at +1 +depends +iii +194222 +id +difficult +height10br +port +listhelp +can +implementation +on +lighter +naeojlmpjmjdfplhiojogehbdbaabmordiconnicholsoncom +designates +focus +driving +upgraded +mailgx0f216googlecom +raw +appears +date +tub +targetblankothersa +me +googlecom +xuidl +them +800dff5b3 +by +then +love +c650fc3c555b11df8dd3d033ee7ef46b02150157apbsaslquonixpoboxcom +rssfeedsjmasonorg +listmasterlistsdebianorg +xmailinglist +httpwwwcixcoukjimhweblogbloggerhtml +serv108segiulgacbe +ldowhitelist5 +2525 +xbeenthere +xamavisstatus +not +pc +with +from +invaliddate +e15si5258490fai2320100514020855 +042106 +0500 +tdimg +next +td6ru1bzk1ae +jobs +entrenched +pass +could +use +listid +put +200208071828045d5e4dcbkilroykamakiriadcom +xaccountkey +hsubjectfromtoinreplytoreferencescontenttypedatemessageidmimeversioncontenttransferencoding +sivaram +sender +precedence +deedsmisaculinknet +anyone +which +debians +an +cc +listpost +into +release +h +peoples +weather +changing +time +xvirusscanned +100049 +reducedifonttd +more +configure1353 +xloop +mailing +115134 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +about +httpwwwforbescomfinancemktguideappscompinfocompanytearsheetjhtml +httpsecuritydebianorgpoolupdatesmainppostgresql83libpq583110lenny1armeldeb +testsawlinreptoknownmailinglistpgpsignaturereferences +arial +no +archive +17128113151 +reportbug +including +contenttype +helpunsubscribeupdate +xrcvirus +width96 +of +and +received +vm8000100 +list +table +mailtodivxlistsdivxcom +obtain +minor +clumsy +the +172164831 +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_35.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_35.txt new file mode 100644 index 00000000..b3c70fe0 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_35.txt @@ -0,0 +1,291 @@ +rate +httpnewmarksdoorblogspotcom +rulenotspam +ratehard +xentcom +for +postfix +network +support +smtp +brdivdivbr +test +autolearnno +begin +httpenigmailmozdevorg +acceleration +years +clientip8219575100 +in +returnpath +053402 +bouncedebiankdemlsubscribertechcsminingorglistsdebianorg +are +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +his +oh +testsbayes002 +plane +23 +00000000 +subject +4 +qmqp +iso88591qxd86p3bwyaj25f726z5fbw7ddkm2el2423h25dudk7thshc +xmailmanversion +bulk +want +suites +base64 +listunsubscribe +width601img +duplicated +but +clickthruonlinecomclickq3da7fr2q8bwvdmi5xhlf5vlk7kcgfer +sure +0000 +would +212173515 +forkxentcom +xcmscore +winerseems +2si4205263fxm3520100409151515 +downloadsa +if +world +br +apr +oct +tag +fallback +required53 +messaging +so +influential +22 +httpclickthruonlinecomclickqe72ygqmvxons0rb0vqs8cmb0qvser +tue +1030507320321439camelheralddragonsdawnnet +deliverydate +5e16016f03 +highlight +users +61 +0400 +full +before +theresas +edt +just +esmtp +httpbidocom +893893 +24 +quotedemailtextreferencesreplywithquotes +entire +mx +a +contrary +habit +vi +wrote +localhost +within +useragent +decaying +as +dogmaslashnullorg +inline +because +well +pdt +messageid +10 +1910 +height8td +powered +told +old +or +resentfrom +hrefhttpwwwecobuildercomlockergnomecfmecobuilderabr +fgout1718googlecom +contact +vaa27638 +bitbitchmagnesiumnet +following +0200 +2007091301 +hrefhttplockergnomepricegrabbercomsearchgetprodphpmasterid533869harman +lan +contenttransferencoding +grubbr +utilities +growing +im +textplain +invoked +127001 +155208254214 +9b1243ca2c5 +4bce303150404coxnet +out +2096118386 +6pwrvo6d6qer +numbers +to +flaw +809122912 +met +xlevel +code +you +associated +polish +bank +hood +fools +colors +7ac06fc132 +simulated +missives +inreplyto +references +mxgooglecom +httpswwwinphoniccomrasprsourceforge1refcode1vs3390 +virus +c1 +b49d9ce08c39 +something +mediaabr +kmail1132 +1426113a6887 +xsender +blog +debianuserlistsdebianorg +rumor +id +doesnt +over +port +listhelp +debianspecific +canadian +designates +yyyylocalhostexamplecom +updated +check +intel +e6cda440c8 +after +date +happens +17021013354 +issues +sufficient +hope +084030 +forgot +simple +by +then +090023 +unsubscribe +dsa1813 +new +default +actually +will +ldowhitelist5 +2525 +xbeenthere +0100 +xamavisstatus +linuxx86 +adjusting +their +unless +not +listsubscribe +somebody +manufacturing +with +from +imprononcable1imprononcable2ldosubscriberldowhitelist +gnome +mimeversion +press +spamassassin +2 +mailtoforkspamassassintaintorg +resources +19si5248675fkr920100416193600 +spamassassinpmv +session +c0arpqdrad2fogilnrjkqdpptzhkrrugczviqsfyhk0fmoh3lqm44pm2j11t8e0t9gy +charsetusascii +xaccountkey +countrys +lucamerciadristudentulgacbe +sender +end +precedence +message +didnt +which +070737 +discussions +bhdhor6pysudjmg9dsm7oamqnh7dczpk9mdcndrdrs6g8 +listpost +noone +matter +millions +xvirusscanned +anything +titlefirststop +mdocabletv305 +nosend3d1td +moines +xloop +rlyvhihzwwgahibc3c0lbliszt +sgamma +manual +sp +firm +471 +csl +about +except +3 +width3d100 +no +archive +a225813917473867footerarchivecolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightboldfontstylenormaltextdecorationnone +some +libraryin +clarifications +reasons +gui +policy +contenttype +footer +xrcvirus +may +like +other +of +minded +and +sponsored +speak +received +without +p17si2535543fka4620100506103749 +list +comments +went +the +grokked +mosque +trigger diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_36.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_36.txt new file mode 100644 index 00000000..aec244a1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_36.txt @@ -0,0 +1,272 @@ +averages +rubidium +health +interesting +read +for +postfix +winning +3d8d5c3b2090202barreraorg +winner +smtp +area +the20 +this +hex +091415 +in +xkxlbfqbm7hg +065659 +have +returnpath +exmhworkersspamassassintaintorg +bouncedebiankdemlsubscribertechcsminingorglistsdebianorg +is +guy +source +backup +greetings +g8dekol07782 +said +subject +border3d1 +using +am +listarchive +than +srchttpviewatdmtcomaveviewaws05000012avedirectwi160hi60001 +n +2002 +bulk +want +email +arrested +listunsubscribe +but +20020805041438j42527100000mooncampusluthse +debian +jmlocalhost +qqqqqqqqqqzdnetspamassassintaintorg +howland +ist +testimony +good +ughhh +doublecapsword +fallback +216136171252 +process +tim +ldosubscriberldowhitelistrere +colorff3333bexclusive +compiler +u +httpblogsanityofsamcom +re +thingy +present +consequences +warm +ssl +youre +trick +price +receivedspf +102344 +ul +outside +0400 +hrefhttpclickthruonlinecomclickq35d6jtibizyije2zzknxltsypwc3ur +just +325 +fails +it +esmtp +1022324133 +subscription +a +deliveredto +xmozillastatus2 +archivelatest573554 +strigi +hrefhttpclickthruonlinecomclickq37emlqiqzfsldtpruq93kjz6ejcner +azeojxyrrgmea10 +localhost +administrations +4152010 +pdt +currently +messageid +slashdot +10 +trade +ranch +gnulinux +or +resentfrom +224754 +contact +color3d0000ff +belong +groups +such +0200 +2007091301 +21 +last +2008110401 +textplain +i +until +127001 +outlook +bhorzi2mwoxznc8me1exltlnnoy6cnddgwbyhyty5s +hrefhttpclickthruonlinecomclickq18whdzi987phkswbgtimeyr1n675ir +that +out +creation +acceptable +sending +to +sep +least +singledrop +expand +now +htmlfontcolorgreenhtmlfontcolorred +know +eg +most +mlsubscribertechcsminingorg +was +resentdate +inreplyto +20090614 +there +uname +change +global +kmail1132 +at +through +debianuserlistsdebianorg +added +go +songs +id +engineer +doesnt +listhelp +speedstream +processes +can +on +worries +installation +designates +yyyylocalhostexamplecom +date +slower +authenticationresults +googlecom +xuidl +gaining +simple +by +lisztdebianorg +flow +rssfeedsjmasonorg +border3d0font +xbeenthere +back +sharing +nautilus +lua +again +while +wont +212906450 +distinguishable +listsubscribe +with +from +092402 +samad +mimeversion +enht +tdtd +sinyal +first +both +beverley +xcheckerversion +xmozillastatus +listid +make +jmasonorg +charsetusascii +9b51d13a649c +lucamerciadristudentulgacbe +sender +permitted +precedence +control +50abr +say +an +cell +rpm +aug +181908 +listmanspamassassintaintorg +be +into +mailtodebiankderequestlistsdebianorgsubjectsubscribe +exmhuserslistmanredhatcom +bits +xvirusscanned +zzzzteana +allums +leg +1091516 +20769200243 +81168116 +sgamma +close +ours +wh +135019 +fri +hosting +score101 +see +debianuserrequestlistsdebianorg +a20 +productivity +stable +razor +we +accusing +mailtorazorusersrequestlistssourceforgenetsubjectunsubscribe +some +does +when +version325 +day +contenttype +challenges +of +and +cipherrc4md5 +received +speed +list +phone +mailtoforkrequestxentcomsubjectsubscribe +viruses +sylpheed +diskarbitration +cams +s0832016219710167 +memory +cant diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_37.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_37.txt new file mode 100644 index 00000000..6f0ee596 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_37.txt @@ -0,0 +1,298 @@ +httplistsdebianorgdebianuser200909msg00721html +ratehard +wwwmailshellcom +libslinet +for +postfix +012425 +mipsel +score41 +maybe +test +xmailscanner +78701 +disc +wasnt +resentmessageid +clientip8219575100 +this +operating +move +21br +myself +have +102233010 +returnpath +once +klabsbe +beautifullienet +channel +imap +socialists +iluglinuxie +are +is +141045 +guy +ofeglpgpchpacfljpailoeicecaamacarthyiolie +agree +too +100 +00000000 +replace +4581516f03 +caninitwithfilepath +11400 +subject +already +yuo +using +useragentoe +listarchive +albino +hrefhttpclickthruonlinecomclickqffsvovqms4myisimnzxrhbm0whther +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +fire +esq +2002 +email +listunsubscribe +hrefhttpclickthruonlinecomclickq59y53oienthmcognglpwbkmqztfr +0000 +root +best +mandrake +another +vol1 +applicationpgpsignature +line +xsessionerrors +if +belpic +kernel42926070322 +hmimeversionmessageiddatetofromsubjectcontenttype +prevent +xacceptlanguage +do +tue +asthma +lairxentcom +offending +berkeley +suggest +domain +rigorous +smokejo +pretty +pcie +re +iwvmjrwzhjlvtgdcwg8pxqhggs0cq7qtcjhd8 +look +youre +t3mr3646835fau161272924148788 +result +razorusers +prague +ldowhitelistmurphysexl2plingquery +edt +comfortable +just +516 +it +esmtp +abraham +listsdebianuserlisztdebianorg +os +catches +deliveredto +xmozillastatus2 +resentsender +wrote +localhost +as +formatted +thing +spfpass +pdt +has +messageid +locally +browns +10 +10011245 +beginning +unknown +unstable +ambience +photography +0700 +linked +specified +or +olpcpowerd +calculating +submit +resentfrom +indicate +distribute +problems +english +finish +contenttransferencoding +way +2008110401 +zzzzlocalhost +textplain +073949 +i +127001 +computerprinterbinding +that +alertsbr +alsalib +to +v1 +sep +singledrop +dynnjablskip0 +you +sharply +gmailid128d2a74ede5b55e +probably +httpwwwstudentmontefioreulgacbemerciadri +eg +vietnam +advice +20968120 +these +help +inreplyto +references +at +replyto +debianuserlistsdebianorg +id +treatmelikeanobject +also +listhelp +on +cellpadding1 +designates +integrate +stan +unverified +clock +6f1df414f4329ee27ada8e9b63a0c56dsquirrel1921681100 +error +mailtodebiantestingsecurityannouncelistsdebianorg +date +a0 +xml +expensive +supports +jalapeno +forfontp +issues +choke +saxon +convincing +xuidl +mail +by +either +unsubscribe +lisztdebianorg +listmasterlistsdebianorg +a3qpwe13e5nma10 +qyk5 +2525 +stoddard +0100 +xamavisstatus +httpslistmanspamassassintaintorgmailmanlistinfoexmhusers +software +wont +briefcase +not +schedule +hmessageiddatefromuseragentmimeversiontosubject +with +from +consistently +httpwwwpixiechildcomjournalindexshtml +mimeversion +enht +stylefontsize +first +2 +include +091706 +listid +traded +jmasonorg +done +hi +deadly +sender +unfortunately +message +seem +didnt +which +border3d0b +7bit +discussions +emacs +be +her +17o2rz0005ts00 +celebrate +dsimaccocoaviewref +vga +7 +mount +e53a816f72 +httplistsdebianorg4bedaed34020803cyberspaceroadcom +between +manual +valuelockergnomedigitinput +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +noelamac +fri +bb30f13a4a5b +xoriginalto +airline +3f4da348c53 +many +sufficiently +made +no +we +important +bernie +does +group +policy +version325 +contenttype +aeussern +study +may +d557ed124559 +of +xhabeasswe5 +and +xstatus +g8uku6k15698 +received +rabbit +limewire +virtuosostrigi +hrefhttpclickthruonlinecomclickqc2oxxlquedqhem4sgvnxxfzciuinr +list +deleted +homerperfectpresencecom +perhaps +arguments +the +freshrpms diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_38.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_38.txt new file mode 100644 index 00000000..5066443b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_38.txt @@ -0,0 +1,266 @@ +gmailid128031e1627510ba +assert +baxter +mailtodebiankdelistsdebianorg +hmimeversioninreplytoreferencesdatemessageidsubjectfromto +trouble +for +postfix +marginheight3d8 +network +find +smtp +26si9745315fxm7620100407005145 +gnupg +aes256sha256 +arsasha1 +solutions +verify +account5 +this +ba +autolearnham +in +margin +removal +have +returnpath +imap +is +uswsflist1bsourceforgenet +why +backup +testsbayes002 +00000000 +subject +pgp +relay +iso88591qptbs5oo9toumhbdke23otci0a093cbil8dtogzk0bf2yq35wsewvyet2e +produced +mailgw0f47googlecom +listarchive +vmware +81168116egwn +archivelatest575647 +a653e58a4d +20100428191714982mgbdebianyosemitenet +listunsubscribe +timid +zzzzexmhspamassassintaintorg +jmjmasonorg +n20grpscdyahoocom +but +sure +0000 +released +debian +url +zhao +height6 +wwwcomicscomcomicsdilbertdailydilbertimagescomicscomlogogif +0xe580b363 +certainly +facetahoma +heres +if +fourla01 +project +br +tumor +dramatica +required53 +httplistsdebianorg20100511055911ga17757europaoffice +going +reforming +kdestandard +maingmaneorg +thu +ctoryquot +all +lairxentcom +print +domain +mailtorazorusersrequestlistssourceforgenetsubjectsubscribe +r10cs60290wfa +re +cest +archivelatest573453 +ba8w1if4fi94v1d +93fileid +exim +clean +edt +seconds +required40 +esmtp +cable +impact +listsdebianuserlisztdebianorg +101005639034 +a +mozillas +deliveredto +job +wrote +localhost +because +pdt +messageid +181021 +signature +10 +fruitful +1br +81388138 +0700 +quite +a274101923387038footerpdfcolor000000backgroundcolortransparentfontfamilyverdanafontsizexxsmallfontweightboldfontstylenormaltextdecorationnone +httpxentcommailmanlistinfofork +width94 +felindapop2athenetnet +fontabba +they +xrcspam +0200 +far +contenttransferencoding +al +importance +textplain +mimeole +127001 +outlook +that +to +getting +hk7xso4vm1lmmuqscmwoj9tebbkzbgf4hen4rzmpr8vkvne98hppelri0cwhrf1qmu4vuw5csyeawyqlcj7acbcwmjhsawvildbirbqobzexiov2f8nfq7yatbdwrfifc4s5sbcrogsmku0eoereovahdfhhgiykv0ku9qxkqomywa5wjqqkfpimr2bbegc70wyo2ymzipjhhsvhhat99ezdefepy3hs5bklsk3z1rkjydngtcbbgh2 +now +october +you +204145 +technology +649118141 +did +amavisdnew +perhpas +most +describes +familiarity +modifications +inreplyto +version +long +200209011956g81ju6te002619oriondwfcom +references +there +mxgooglecom +button +stuck +2de4213a5b3e +at +points +debianuserlistsdebianorg +titlethe +xspamlevel +id +encryption +port +allowing +listhelp +need +autowhitelist +can +on +ac +how +z10si5121319fah7420100508144024 +gamasutra +date +kmail199 +mailtoexmhworkersrequestspamassassintaintorgsubjecthelp +permit +jalapeno +7646 +authenticationresults +depict +totoro +by +visit +httpwwwgeocrawlercomredirsfphp3listrazorusers +listmasterlistsdebianorg +picnic +new +dir3dltrlt +famous +cliff +ones +tablepbr +mutt14i +2525 +0100 +autolearnfailed +software +05042010 +not +listsubscribe +with +from +un +thailand +overview +first +115816 +turn +mailtoforkspamassassintaintorg +keywords +knowledge +xmozillastatus +could +disable +done +authenticated +precedence +ie +an +cc +targetblank +color0099333529fontfonttdtr +get +xvirusscanned +microsoft +mailing +cs78128057pphtvfi +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +transmission +av24166comexru +see +value990000input +no +dhersaaes256sha +iso88591qyjru4abtypxviyxqnqlrxwtcsj0a096gudsnwt0nxblkvuwjblkwvjr +17128113151 +when +conventions +lovingly +194108 +contenttype +xmozillakeys +xrcvirus +possible +smtpmailaugdbouncesmlsubscribertechcsminingorglistsapplecom +like +graphics +of +ive +xstatus +record +haze +received +list +gmailid12803d3209d148ae +proceeded +view +loripatelhotmailcom +the +cant diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_39.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_39.txt new file mode 100644 index 00000000..311e0e63 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_39.txt @@ -0,0 +1,296 @@ +hrefhttplockergnomepricegrabbercomsearchgetprodphpisbn0966103254get +rate +0300 +please +qmail +for +postfix +politics +icq +support +20100503134740005bssiguanasuicidenet +httpwwwcan4linuxorg +keyboards +taste +amd64 +your +returnpath +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +imap +mailtodebianuserlistsdebianorg +advertiseabrfont +165251 +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +gmailid128742ff92172087 +ghosts +testsldosubscriberldowhitelist +testsbayes002 +online31983206muudgykaughfdrr1bnewsletteronlinecom +subject +got +using +less +mailer +2002 +xmailmanversion +bulk +475 +listunsubscribe +quotedprintable +krammer +but +hspace5 +released +c5c8b16f16 +0800 +alignmiddleba +9 +175411 +111126 +would +opportunities +105033 +javatm +ist +if +good +detect +apr +problem +xrandr +angle +our +do +all +tue +archives +lairxentcom +domain +thanks +personalthe +derived +xproofpointdetails +re +worksbra +img +receivedspf +result +set +delayed +bird +mailtoforkrequestxentcomsubjectunsubscribe +required40 +us +rant +it +experimenting +esmtp +httpxentcompipermailfork +terroristes +two +torment +mx +a +dont +buried +deliveredto +resentsender +wrote +localhost +within +uncommented +as +dogmaslashnullorg +shout +has +messageid +10 +webmailspamcopnet +0700 +gnulinux +yet +wireless +httpcariadunixwiznet +score54 +or +doctype +contact +xrcspam +tainted +0200 +httpwwwwomanchildcombloghtml +open +144604 +textplain +invoked +127001 +151150 +desktopxab +that +out +frewicessucsbedu +66b6e16f16 +encourage +width3d85 +aidshiv +to +fall +permanent +supplied +filedeb +singledrop +xlevel +20080610 +utc +did +amavisdnew +auth02nlegwnnet +enables +recently +these +mlsubscribertechcsminingorg +1341476414 +version +six +potentially +mxgooglecom +change +documented +at +1 +through +33c3213a4737 +133810 +f13mr1383825fai981273092457506 +id +mailinglist +20000 +sutilbrbr +port +listhelp +on +being +60 +14 +nytimescom +date +tired +235733 +r2989529896 +caches +bug +solely +xuidl +them +mail +things +by +hiding +then +think +unsubscribe +listmasterlistsdebianorg +thats +story +42 +few +xbeenthere +little +cpu +autolearnfailed +gmail +feed +unless +not +listsubscribe +score110 +with +from +rohit +0 +v106 +043902pm +pass +xmozillastatus +listid +59hu +jmasonorg +charsetusascii +fetchmail590 +done +available +argotech +sender +precedence +its +errorsto +132312 +accelerator +score7 +7bit +qdns +aug +ontpageauthors442gif +be +listpost +street +xpolicydweight +4101213a6407 +trusting +get +her +src3dhttpwwwcnetcominlshsubfeatgif +width575 +045343 +were +title +headericsminingorg +straight +7 +bfar +card +81168116 +manual +terms +sm1 +regulation +httpclickthruonlinecomclickq89h2jbrpjirahzk5rhpcqnpvwaeyl9srr +0l0t00i73pl2h150amtaout23012netil +contains +xoriginalto +about +exim4 +10143348 +125604 +adrian +3 +arial +razor +we +intmx1corpspamassassintaintorg +past +identify +grads +contenttype +towards +required70 +xrcvirus +may +bottle +like +secure +might +of +since +ive +and +xstatus +received +one +ever +list +wish +the +httpwwwnewsisfreecomclick183997281440 +never +sh diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_4.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_4.txt new file mode 100644 index 00000000..9528dc14 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_4.txt @@ -0,0 +1,296 @@ +michael +looks +httpwwwtheshiftedlibrariancomcategoriespdas +r10cs25765wfa +launch +c2b9 +win32 +please +for +postfix +experimental +inventory +mailtodebianuserrequestlistsdebianorgsubjecthelp +dsa1871 +encodingutf8 +mailin12applecom +span +etcsysconfignetworkscripts +formatflowed +2si9362550fks1220100503102310 +module +17230098 +ciphers +myself +returnpath +imap +worry +are +is +h2mr17697830ann751270621908541 +testsldosubscriberldowhitelist +classtitlenbspgnomestuffptdtrtablep +4bd1e83c3010603pupmunivfcomtefr +173705 +said +subject +trace +defaults +galactic +contentdisposition +than +n +xmailmanversion +bulk +want +killing +hrefhttpwwworaclecomgosrc1393043ampact9check +gfx +email +09 +listunsubscribe +thought +20100506091306917dididebiancknoworg +181811 +but +0000 +jmlocalhost +security +hrefhttpupdateslockergnomecomlatest +different +gmailid127d4404807ca52b +yyyylocalhostnetnoteinccom +jaa22837 +subscribers +if +apr +oct +fallback +process +uses +our +spamtrap +must +minute +all +cs +startup +102113 +webmail +features +re +123 +750 +receivedspf +exim +mailtoforkrequestxentcomsubjectunsubscribe +edt +it +esmtp +202051 +xinjectedviagmane +a +609 +deliveredto +messageidreferencesmimeversioncontenttypecontentdisposition +maintainer +create +httpwwwntknet20020927jesuslargejpg +wrote +receive +localhost +asusmode6 +webjunkies +as +modulesp +highresolution +compaq +spfpass +instantly +pdt +messageid +pfont +signature +10 +despair +unknown +0700 +xspamstatus +divdiv +agent +xrcspam +trying +0200 +whats +contenttransferencoding +forkadminxentcom +19fca41723 +pgpsignature5 +2008110401 +de +culture +textplain +127001 +11fe0d0ab7c42ae000006518e54bcb20d5ca85 +iesubericnet +abandonment +pint +to +testscapinitldosubscriber +xlevel +mailtoexmhusersrequestspamassassintaintorgsubjecthelp +now +you +td +ettled +bank +did +amavisdnew +110236 +minutes +250ms +score39 +size3d1balso +mlsubscribertechcsminingorg +zeroed +references +found +at +through +emacs23250 +attach +authentication +color3d000000downloadsfonta +grandchildren +fatal +id +folder +handled +bhmsg0z8rtvqetjll7zlzc1voaioj4jtbncf0amfj9u +port +listhelp +127214956143802camellocalhost +on +common +being +kde442 +readersbfontbr +afford +date +y20mr5015654fau781273047034832 +friends +lack +jalapeno +sores +iqecbaebcaagbqjl7qcaaaojehnws3jeoi3pv5shwfc9d44t7bpqvde7wzivacm +bug +17cvti0007rs00 +googlecom +xuidl +cab +urbaniak +by +then +worked +unsubscribe +nowrapfont +ago +new +xbeenthere +0100 +xamavisstatus +cause +not +listsubscribe +href3dmailtofirewalruscsminingorgfirewalruscsminingorgagt +with +from +still +glvertex3f00f +311 +mimeversion +0500 +separate +inc +mulberry220b1 +anymore +include +life +both +xcheckerversion +200420 +194351 +5a9a02941ce +use +listid +sender +precedence +xmlendifif +position +g8girhc05769 +heartbeat1messagingenginecom +errorsto +almost +azfxj001pgpzmkjsundscbwb5o6ka4 +an +boundarynextpart000000501c264127545c1d0 +aug +be +listpost +merely +m09 +into +breakup +script +0909090909noscript +headericsminingorg +online320224c4ddw5l6ay9s9ksrr1newsletteronlinecom +xloop +debianuser +comparing +stylecolor +ride +g976wqk21300 +scripts +upgrade +14si22934720muo220100413060027 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +contains +federation +4813b144 +about +started +simply +gmailid12884051554c7c34 +8219575100 +continues +se7enorg +dsa1863 +no +moved +add +6e51c7bc36a345c2ad7533b49394124acsminingorg +enus +saremsgidlong450893 +1000 +height82 +contenttype +xmozillakeys +xrcvirus +smtpmailaugdbouncesmlsubscribertechcsminingorglistsapplecom +filed +of +modestasvainiuseu +rfsrzuftkuysiydf6zjkuokdsue6vdmpbnru +xstatus +record +received +113933 +windows +list +phobos +haymarketedacuk +javadevbouncesmlsubscribertechcsminingorglistsapplecom +knownmailinglistrcvdinmultihopdsbl +the +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_40.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_40.txt new file mode 100644 index 00000000..0ca18e8e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_40.txt @@ -0,0 +1,280 @@ +46rom +score119 +ratehard +oldreturnpath +xmailer +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +devel +hatched +aol +resentmessageid +this +paul +ba +yyyyexamplecom +your +returnpath +imap +cheers +is +uswsflist1bsourceforgenet +testsldosubscriberldowhitelist +among +plugins +burn +024820 +subject +try +collection +linuxdebianuser +qmqp +consider +robin +dave +listarchive +bulk +three +001517475c8a468ca20483bd259e +listunsubscribe +laaljusticegcca +4be972907010109comcastnet +jmjmasonorg +0000 +desert +different +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +yyyylocalhostnetnoteinccom +fontfamily +captured +testsfourlagmailhtmlmessage +own +going +hen +xacceptlanguage +74ef813a5136 +newsletter +do +all +helorovervipulnet +had +recycling +result +exim +bnextgeneration +181838 +0400 +idea +tokenizertokenizertokenizeheaders +examine +edt +required40 +it +esmtp +rewardsfontb +subscription +a +3dbea4305f7 +deliveredto +220901 +office +information +xmozillastatus2 +235342 +blackberryaebr +google +afaik +113700 +localhost +slacknet +2011 +localhostlocaldomain +useragent +as +build +offensive +has +messageid +unknown +0700 +xspamstatus +or +2001 +perl +contact +december +secondary +xrcspam +mwsfud +0200 +contenttransferencoding +rsescommmclickthrough233170622702155823333143ubn6sdjvj8dmr08myuluac +avbidderfortytwoch +1b619212d30 +im +stop +importance +textplain +httpwwwauracomcomteledyn +127001 +testing +avoid +frolovsigmaisprasru +that +michaelashcsminingorg +thunderbird304 +hrefhttpclickthruonlinecomclickq8bsbgbqy0yjeu6yseyieeldh4b1rr +to +singledrop +you +fork +manipulating +103113 +size2you +tokenizertokenize +amavisdnew +solution +here +mlsubscribertechcsminingorg +httpwwwheldincontemptcom +mxgooglecom +personal +at +through +added +go +id +15da916f6b +imprononcable21 +also +port +listhelp +on +complaints +06 +ilugsocial +intel +how +zzzzlocalhostnetnoteinccom +outweighs +dates +date +class3dthmfgmisctext +7600 +xyuw2b2gxmiph0h2amwirldskjfnge1vknfb18ye1ewnxweuyj95sa0gwfgmfszd8 +045557 +mailtorpmzzzlistfreshrpmsnet +authenticationresults +me +googlecom +xuidl +xhost +kiener +by +whilst +tall +even +84c7 +ext3 +utility +ipv620014b7820002 +client +shakespear +thats +new +bst +link +few +ldowhitelist5 +xbeenthere +back +200207271701saa23172lughtuathaorg +not +listsubscribe +sides +with +from +repositories +money +clamsmtp +mimeversion +spamassassin +first +both +failing +copy +listid +make +charsetusascii +fetchmail590 +approach +lucamerciadristudentulgacbe +sender +precedence +tried +shoddy +didnt +its +errorsto +6217143253 +89c9613a55cc +apple +an +7bit +m8cs47341wfo +be +listpost +17ysox00048e00 +c +peaks +gmailid127e948f3756dccb +partitions +mailtoforkrequestxentcomsubjecthelp +special +more +latter +headericsminingorg +mailtobitbitchmagnesiumnet +1241854 +xloop +spamassassintalk +between +81168116 +26302686 +about +10143348 +grokster +aware +lines +kde +see +demand +geneva +17128113151 +xmailscannerclean +jmrpmjmasonorg +contenttype +hrefhttpclickthruonlinecomclickq11l8cii2ap8sfpx5xa0eixbhh7rrr +study +xrcvirus +mailqy0f175googlecom +may +of +intranets +mozilla +imposes +received +xpe +increase +wed +obsequious +list +o +6445128110 +sept +wouldnt +the +send +xpriority +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_41.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_41.txt new file mode 100644 index 00000000..26988720 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_41.txt @@ -0,0 +1,291 @@ +rate +sunday +please +oldreturnpath +for +organization +postfix +pan0132 +support +smtp +officer +hereptdtrtablebr +x +due +any +chris +20100523102039gb2494kinakutalocal +8938112 +plant +clientip8219575100 +this +in +returnpath +imap +cheers +are +mon +esrs +agree +too +00000000 +subject +4 +gmailid128939d4a3e158e7 +relay +listarchive +xmailmanversion +increasing +smtpmailquicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +listunsubscribe +odd +yes +104557 +chaste +6img +0000 +debian +jmlocalhost +url +similar +what +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +owned +forkxentcom +blue +good +httpwwwfingerfarmcomblog +up +so +our +ave +345b25ffb04e47b89d07a99a9e33c0d4applecom +20020720184341e23917iesubericnet +t +newsletter +play +keep +width468 +tue +2222 +domain +book +re +httpastonvillablogfootballcomdrkeene +20020829 +f +users +hrefhttplockergnomepricegrabbercomsearchgetprodphpmasterid568577pioneer +result +held +exim +articles +0400 +xc5a +seconds +required40 +us +it +score +esmtp +893893 +deserve +deliveredto +xmozillastatus2 +google +localhost +httpnewsbbccouk1hi2210091stm +2011 +as +dogmaslashnullorg +build +spfpass +231 +5 +messageid +zzzzlocalhostspamassassintaintorg +signature +10 +signed +durnik +hrefhttpwwworaclecomebusinessnetworkshowiseminarhtml1293857watcha +httpwwwbondedsendercom +text +feet +phoboslabsnetnoteinccom +0700 +plugin1 +cupa1b0ee0oa04b8yrwlbliszt +xrcspam +far +contenttransferencoding +2008110401 +im +userid +textplain +xfce4notesplugin +127001 +avoid +creeps +programs +that +out +rpmzzzlistadminfreshrpmsnet +httpradioweblogscom0105617 +aptget +to +fall +singledrop +20080610 +incworldcom +know +demon +did +nononsense +boundary22773175alt +331vamm2 +most +v +mlsubscribertechcsminingorg +030059 +there +mxgooglecom +experience +doing +at +090051 +been +debianuserlistsdebianorg +added +mix +id +anders +doesnt +listhelp +on +being +black +installation +check +debiankdelistsdebianorg +abdullah +hide +ll +dates +date +xhabeasswe7 +4bbff50c8010308homenl +jalapeno +sense +110327 +ben +httpwwwtimesonlinecoukprintfriendly014335108300html +mail +besides +manager +by +auszug +even +gmexim +helo1921681241 +listmasterlistsdebianorg +variable +will +tm7mr0zkeqs2 +ldowhitelist5 +2525 +241a +0100 +bz3applecom +autolearnfailed +agenda +9286142199 +not +listsubscribe +with +from +m8cs15448wfj +translate +mimeversion +pyknic +spamassassin +2 +both +use +listid +restricted +jmasonorg +charsetusascii +fetchmail590 +done +xaccountkey +rather +sender +permitted +precedence +its +slackmoehrlemecom +errorsto +which +ronan +apple +an +taggedabove10000 +ca131503e1 +be +listpost +sj1slb03gw2 +dissent +bits +version250cvs +more +case +headers +215956 +4f5d32d0c6b +xloop +xhabeasswe6 +mechanism +mark +platform +liberal +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +fri +xoriginalto +about +investors +aanlktilqtvstnerew1lmnlsyto13zpxsygphtejgivmailcsminingorg +mailtoexmhusersspamassassintaintorg +no +console +installed +router +let +c452 +131656 +textalign +some +fs +does +digital +when +version325 +define +theres +contenttype +xmozillakeys +might +of +summary +and +received +setup +oath +right +transactions +list +g96805k15046 +fontimg +unquoted +search +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_42.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_42.txt new file mode 100644 index 00000000..8d7c7362 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_42.txt @@ -0,0 +1,262 @@ +hrefhttpclickthruonlinecomclickqdadt8jqupqnel9djbjsmvkpv3dxsr +conventional +namesignatureasc +me20 +xmailer +for +organization +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +pan0132 +maybe +jmuseperljmasonorg +popularity +fastgrowing +leaders +operations +ch20 +this +in +httpedtechdevorgrights +bccd +3d1tdtr +returnpath +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +is +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +101431608 +multipartsigned +testsbayes002 +powell +subject +relay +using +40 +camaleón +listarchive +110228 +fairly +included +2002 +ideal +email +taxing +rh +but +1002627 +valign3dtoptd +102231867 +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +mailtodebianlaptoprequestlistsdebianorgsubjecthelp +boot +if +unsubscription +srchttpwwwzdnetcombgif +captured +fallback +required53 +216136171252 +20021002t0934000800 +normal +keep +all +pdf +26 +domain +httpwwwlinuxiemailmanlistinfoilug +had +psychoblogger +re +destructive +run +height19 +data +disrupt +set +pardon +transportation +crashes +edt +it +esmtp +a +deliveredto +office +200208210251g7l2pqkb001805turingpoliceccvtedu +localhost +2011 +useragent +bouncedebiannewsmlsubscribertechcsminingorglistsdebianorg +as +dogmaslashnullorg +classic +libxfcegui44 +httpwww1tcnetnejpfmuratalinuxdown +has +messageid +score69 +10 +220233140121 +warning +xspamstatus +resentfrom +zclient5011ga2696rhel4 +doubleact +et +t14mr402106fan851273255343636 +reason +last +2008110401 +im +mailtorazorusersexamplesourceforgenet +cyclical +importance +textplain +invoked +i +127001 +files +cannot +hour +helouswsflist1sourceforgenet +hes +listsdebianorg +to +murphy +soft +singledrop +lshaped +exciting +now +startling +utc +mckeownb +amavisdnew +inbound +minutes +80 +definitionsmain1005180171 +alarms +vertigo +hardware +mlsubscribertechcsminingorg +help +was +version +there +found +happened +191129 +experience +hopes +at +1 +replyto +been +id +listhelp +processes +on +nameq +being +designates +tr +vision +after +date +norfolk +bug +062526 +2002081111113628aab804matthiashaasebennewitzcom +googlecom +mail +by +status +ross +20624818377dslteksavvycom +via +link +2525 +xbeenthere +0100 +their +cause +smtpwowsynacorcom +not +listsubscribe +with +from +rohit +17217132188 +size1 +first +2 +8024132206ucnombresttdes +msgkeys +pass +hacksawhacksaworg +5gay1e0094dcmzy02gaz5b +address +use +charsetusascii +sender +precedence +anyone +bphfcskucivcaxpav8obvwo75rvkd6yvgwomafa2srb97hee33qicgmwpzuyl4uhp +stephen +its +errorsto +arialfont20 +214638 +81258125submit +24e1616f03 +development +be +listpost +street +xpolicydweight +get +paying +time +script +latter +xloop +type +required +valignbottom +responsive +decided +xoriginalto +simply +10143348 +8219575100 +2495 +no +sat +sfnet +troopersmarry +144328 +uninstall +sony +some +magnitude +enus +reasons +083105 +saremsgidlong450893 +contenttype +xmozillakeys +might +of +and +20d5e13a4ff5 +record +received +bush +cellpadding3d2 +list +121303 +1087506 +the +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_43.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_43.txt new file mode 100644 index 00000000..02a60c5c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_43.txt @@ -0,0 +1,303 @@ +pumpanddump +allowed +c2b9 +economy +xmailer +for +postfix +politics +manila +m3grpscdyahoocom +june +involved +autolearnno +superuseronly +any +mord +bye +window +area +account5 +this +autolearnham +in +cached +have +gsurveyresultslinkhover +your +off +returnpath +mailtodebiankderequestlistsdebianorgsubjectunsubscribe +mailtodebianuserlistsdebianorg +is +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +too +course +testsbayes002 +tartiest +makes +23 +coinstallable20 +logging +said +subject +hard +4 +am +contentdisposition +listarchive +xmailmanversion +bulk +17hyuf0000ob00 +email +listunsubscribe +but +0000 +debian +best +url +hrefhttpclickthruonlinecomclickq019oxdi2jhs5wf8rmvurmpgpxz6rlr +would +yyyylocalhostnetnoteinccom +ist +if +spamassassindevellistssourceforgenet +arabia +082901 +limited +problembr +thu +do +all +u +had +name3dtitle +1a04813a457c +re +hacking +users +61 +before +xmimeole +3b1a52d0dc5 +required40 +it +esmtp +165314 +httpxentcompipermailfork +a +dont +deliveredto +khare +tagged +bythinkgeek +squeaky +localhost +thmfginactivetext +dogmaslashnullorg +talk +inline +because +overcome +pdt +5 +messageid +g7jhl5z02133 +gaa23883 +10 +c3mr11346361wfj171273035702604 +0700 +120837 +xspamstatus +or +us25000 +resentfrom +computer +groups +they +xrcspam +0200 +2007091301 +114013 +way +nmh104 +indisecouk +im +service +brand +textplain +invoked +i +127001 +that +yep +possibility +to +809122912 +contractual +singledrop +xlevel +93000 +fwd +20080610 +technology +did +amavisdnew +v +mlsubscribertechcsminingorg +ikifi +was +editor +inreplyto +long +doit +references +something +archivelatest575026 +at +replyto +xsender +yyyylocalhostspamassassintaintorg +bgcolorfffffftd +debianuserlistsdebianorg +remote +httpslistssourceforgenetlistslistinfospamassassintalk +id +meta +consultant +httpslistssourceforgenetlistslistinforazorusers +doesnt +also +nolayer +port +listhelp +hassle +hrefhttplockergnomepricegrabbercomsearchgetprodphpmasterid555517musicmatch +on +being +yyyylocalhostexamplecom +click +my +zzzzlocalhostnetnoteinccom +error +after +date +happens +122 +rolled +cybersecurity +mailtoexmhworkersrequestspamassassintaintorgsubjecthelp +a0 +jalapeno +issues +note +001619 +me +googlecom +xuidl +workaround +by +think +lisztdebianorg +official +xegroupsfrom +xmailinglist +permits +cool +ldowhitelist5 +2525 +receiving +laura +xamavisstatus +not +varlogxorg0log +listsubscribe +with +from +200209302251315fe5aa07matthiasegwnnet +brown +20 +uj2znt5wei5t +cnofws +1alnuq0007hv00 +mimeversion +enht +color000000tech +cwgdated102779151394b180deepeddycom +25456 +august +spamassassin +first +media +flag +xcheckerversion +choice +ohio +make +put +beneficiary +charsetusascii +1027711160103586camelavalon +sender +precedence +cdd980289cb141329f52b72c6d375dddattnet +its +be +listpost +get +matter +were +latest +kinda +081c +ignored +ceaseanddesist +more +vga +headericsminingorg +xloop +r +investigated +mailing +valignbottom +81168116 +pushbr +pin +frommxmatchesnothelodomain0 +duncan +listening +patelpp +spam +dpkg +xoriginalto +masterlistsapplecom +face3darialhelveticasansserifguinness +quicktime +no +0h35003jgmyx9gmta7pltn13pbinet +archive +124911 +20020918 +when +great +day +voyage +film +1000 +contenttype +xmozillakeys +targetblankwine +xrcvirus +may +gecko20100317 +other +of +and +record +received +list +definitely +solitaireabr +the +those +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_44.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_44.txt new file mode 100644 index 00000000..57caf7bd --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_44.txt @@ -0,0 +1,282 @@ +jvmplatformversion +rate +093650 +score119 +hmimeversioninreplytoreferencesdatemessageidsubjectfromto +commandline +register +xmailer +listsdebiantestingsecurityannouncelisztdebianorg +for +postfix +rpmlist +smtp +18si7794034wfa1220100516172542 +post +725524d7d8601394183f59bd344bee82squirrelwebmailapplecom +working +people +2418 +across +account5 +autolearnham +in +32a6813a4a14 +returnpath +imap +are +mon +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +martin +g8tiw2f01446 +23 +subject +try +got +qmqp +provider +system +score49 +xmailmanversion +ymmv +listunsubscribe +but +0000 +lpr +jmlocalhost +security +messed +would +080 +200756 +certainly +blue +ist +rootdownload +0d1cc15d864e2 +fourla01 +apr +required53 +216136171252 +up +so +q11mr2461275ago901274680109300 +i686 +suggest +xhabeasswe1 +thanks +re +httplistsapplecommailmanlistinfojavadev +receivedspf +result +exim +set +testsforgedrcvdtrailinreptoknownmailinglist +dcsminingorg +came +border0tdtr +xmimeole +cans +required40 +325 +it +esmtp +24 +2125442165 +a +deliveredto +information +exmh +wrote +localhost +spamassassindevelexamplesourceforgenet +as +dogmaslashnullorg +psuedocode +httpradioweblogscom0110837 +nl +messageid +billwstoddardcom +srchttphomecnetcombgif +unknown +dkimneutral +vanwoerkom +phoboslabsnetnoteinccom +0700 +4dfa2c44e +or +hrefhttpclickthruonlinecomclickqa9ilbqqpuxjlucjrqz2jqzv91zvmlr +gmailid1280ab09584f6020 +resentfrom +strong +aptcache +dsl206191151102evergonet +contenttransferencoding +goodman +hits9733 +divided +textplain +invoked +i +127001 +httpmaddogweblogscom +work +helvetica +files +that +schemes +to +v1 +sep +charsetiso88591 +singledrop +xlevel +technology +know +utc +probably +113723 +1ofaib0006mdol +amavisdnew +httpstreamrippersourceforgenet +testsawltnonsensefrom4050 +hardware +mlsubscribertechcsminingorg +help +resentdate +httpwwwsammydmancom +references +found +block +at +been +older +debianuserlistsdebianorg +go +id +g9181kk15695 +port +listhelp +on +designates +tr +remove +click +interest +date +take +frozen +jalapeno +takes +customizable +authenticationresults +googlecom +xuidl +435368043088 +71141641 +by +112834 +worked +obviously +even +checks +silence +thats +butterworth +actually +36b2 +sound +2525 +again +qtj +their +autolearnfailed +installing +listsubscribe +environment +times +with +from +stuff +1022350193 +cdptpaomtalbmailrrcom +07132001 +intro +mimeversion +august +altavista +2 +greeting +make +answer +jmasonorg +02 +charsetusascii +permitted +precedence +132118 +values +earliest +johnson +its +errorsto +which +american +an +7bit +090147 +taggedabove10000 +073615 +height3d7 +shows +mailtodebiankderequestlistsdebianorgsubjectsubscribe +fine +bits +more +heart +mx1spamassassintaintorg +130 +economics +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +eliminates +inconvenient +sgamma +manual +sm1 +fri +transmission +8219575100 +see +made +no +add +having +harder +17128113151 +fond +digital +policy +great +kre +contenttype +xmozillakeys +months +may +like +other +of +ive +01 +and +baskinrobbins +received +without +windows +posted +phobos +1087687 +cases +alyon153176230w86202abowanadoofr +sylpheed +handle +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_45.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_45.txt new file mode 100644 index 00000000..67ede1b9 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_45.txt @@ -0,0 +1,267 @@ +dos +211626 +lists +pm +ratehard +486432 +zumblog +for +postfix +20100521152315ga5433googlecom +business +155609 +adding +125534 +score109 +resentmessageid +this +in +move +benefits +priority +returnpath +imap +retpoc59dto +is +opera +reviewed +built +ict +mozilla50 +clamdscan +subject +defaults +using +xoperatingsystem +469 +familiar +rbthomaspoboxcom +nextserver +ana +pluckersitescooper +httpwwwdebianorg +previously +listunsubscribe +quotedprintable +gbbfr7ir9eoa20yy +muchlittle +ltmodem +archivelatest576226 +uk +ot +gibsona +height6 +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +another +online +j +accucast +ist +running +5teuz38dsvikvdikzathw5eqwoaiviyueerjhew1t9yun7rrjsqtd6dzzreowvqlr +fallback +going +h4mr788869fas561273277074209 +razorusersexamplesourceforgenet +bragging +thu +22 +traumatize +domain +native +re +host +look +f +users +receivedspf +0400 +fonttdtrtr +clean +yorks +edt +164137 +required40 +325 +themws +esmtp +partition +a +trustworthy +helodellbuntulocalnet +deliveredto +xmozillastatus2 +resentsender +exmh +wrote +localhost +as +dogmaslashnullorg +164121 +because +dtable +messageid +nonecompaq +10 +dealing +decompression +told +xspamstatus +yet +httpxentcommailmanlistinfofork +ireland +command +193457 +2007091301 +contenttransferencoding +answers +1921685425 +im +zzzzlocalhost +textplain +mimeole +i +127001 +reporting +java +cannot +listsdebianorg +montanaro +to +singledrop +toward +you +did +specification +sign +yv7vdjsur +archivelatest574275 +downstairs +resentdate +imbrwqkyopigawec0sszt6u32fh1niqwlzn2k +heed +references +there +mxgooglecom +vs +c1 +020dd40c52 +at +debianuserlistsdebianorg +go +id +src3dhttpwwwzdnetcomgraphicsanchordeskfrontpagebgif +operation +xcomplaintsto +listhelp +need +on +acn +designates +intminline1msgidminmsgmaxlineminline2maxmsgminms +wants +worth +date +jalapeno +cvfnqqhiggoxky895ni2p06ys7q26btysl77b0nrg5xa +itself +c0e8813a5529 +authenticationresults +googlecom +xuidl +addressbook +attemps +by +then +ba7b73ee17 +comment +listmasterlistsdebianorg +should +via +link +hasnt +gif +curious +xbeenthere +explicitly +kencoargoluxcom +not +score110 +with +from +mimeversion +enht +giants +membermount +inc +obscures +bypass +listid +httpcaffeinediaryblogspotcom +fetchmail590 +bent +hi +stephenson +sender +precedence +say +an +latin +7bit +harvesting +taggedabove10000 +development +listpost +downloaded +httpwwwgeekstylecouk +into +bigger +18 +collect +weather +better +bits +were +version250cvs +x11 +logmaneorg +more +xloop +debianuser +goofy +between +81168116 +decided +eirikur +eonauthrelay2 +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +aoki +according +see +quicktime +moved +add +archive +sfnet +mailfollowupto +gt +policy +version325 +xmozillakeys +clickherelink +ts01050sligoindigoie +of +and +record +received +created +raid1 +httpwwwjibberjabbacom +httplistsdebianorg1710565045925434770heavisideinvalid +technical +valigntoptd +search +the +width3d200 +only +authfail diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_46.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_46.txt new file mode 100644 index 00000000..c5b853cd --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_46.txt @@ -0,0 +1,300 @@ +rojinsky +nmh104dev +bgcoloreeeeeeimg +trouble +for +postfix +20100521152315ga5433googlecom +width3d3020 +encodingutf8 +charsetiso885915 +2410200153 +harley +gnupg +autolearnno +httpquantumtunnelscom +spiess +years +this +operating +in +returnpath +once +imap +mailtodebianuserlistsdebianorg +are +is +why +multipartsigned +testsbayes002 +nike +62 +seen +said +examples +subject +using +httpgeigersundayblogspotcom +contentdisposition +than +score698 +email +anand +listunsubscribe +throughout +but +whether +1471454022 +co +httplistsdebianorg4bbeb3a17050609coxnet +jmlocalhost +best +strolled +204942 +what +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +dsa20511 +teemu +radio +specially +ist +background +sakiko +apr +versions +highly +fallback +up +guerin +per +backups +do +tue +head +uu3hilnuoae +tahoma +06232000 +u +pretty +re +cest +users +result +68116120 +1931228222 +7d9162d0db1 +ease +delayed +required40 +us +it +esmtp +httpxentcompipermailfork +listsdebianuserlisztdebianorg +hanchemathntnuno +111529 +a +iso88591qq3b2h0a096265 +deliveredto +distro +khare +arsasha256 +information +xmozillastatus2 +resentsender +seeing +width160 +height +localhost +sees +as +dogmaslashnullorg +thing +fileshare +spfpass +96000hz +messageid +pay +10 +unknown +networknbsp +0700 +quite +7twy +121819 +resentfrom +profundities +mouse +they +xrcspam +2007091301 +contenttransferencoding +importance +textplain +i +127001 +185716 +outlook +testsgmailhtmlmessage +uae +that +out +neil +buy +to +half +charsetiso88591 +095618 +razorusersadminexamplesourceforgenet +20080610 +tlsv1 +westkacuedu +know +lifepreserving +104435 +planning +recently +most +fat +mlsubscribertechcsminingorg +easier +was +resentdate +inreplyto +references +there +nothing +at +points +been +xspamlevel +id +209sfnet +someone +need +can +on +common +tasks +installation +designates +consolebr +m8cs105855wfo +click +my +aaaaaqaaaua +nvidia +error +httpwwwtheregistercoukcontent3926249html +otherland +date +11fe0d0bb7c81ae0000050dc3b4bbf816b5daa +who +légaux +g86k17q07388 +me +xuidl +by +worked +3d6e09845030708startechgroupcouk +rssfeedsjmasonorg +comment +isbn +listmasterlistsdebianorg +helo +231403 +bz3applecom +says +autolearnfailed +listsubscribe +environment +with +from +plingquery139 +stuff +rohit +741258247 +20 +stature +dalnet +shellsubscriptionslockergnomecom +silly +httpslistmanspamassassintaintorgmailmanprivateexmhusers +linux +use +mode +listid +8d96985e292dfbd97625453689c +charsetusascii +fetchmail590 +sender +precedence +vm7090206 +an +7ed4016f03 +7bit +ill +lcomemailsz3d468x60ord3d +cc +yahoo +where +get +125 +kennedy +mailtoforkrequestxentcomsubjecthelp +better +unusable +allums +ja +fundamental +xloop +html +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +w +81168116 +httpwwwzimbracomproductsdesktopfeatureshtml +manual +k2gb59a0661005052317jf0fa9ec4n2399a8ed8d64e1damailcsminingorg +about +10143348 +8219575100 +jmdeadbeefjmasonorg +continues +see +saturday +no +western +past +c76104234217hsd1wacomcastnet +notinblnjabl15 +sony +some +download +g86hvgc18460 +does +when +policy +contenttype +xmozillakeys +xrcvirus +duman2kvrkwjgsiimqzswe5qnwptuvv +20125312211811520011108 +gecko20100317 +javascript +jeffrey +e9crit +etch +and +record +received +one +without +1022321204 +052430 +asl +wed +mithlondarda +list +irection +105941 +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_47.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_47.txt new file mode 100644 index 00000000..d80e59cc --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_47.txt @@ -0,0 +1,270 @@ +used +10024 +application +trouble +oldreturnpath +typesubmit +for +postfix +maybe +g12cf1323284 +harmony +body +young +resentmessageid +verify +account5 +this +in +cached +27af0440f0 +amd64 +have +dsun358adbvdesignanalogcom +your +off +returnpath +is +175105 +multipartsigned +testsbayes002 +fruit +show +subject +pgp +try +2002 +jmhall +mailtorazorusersrequestexamplesourceforgenetsubjecthelp +bulk +rpmlistadminfreshrpmsnet +email +0lvebg1nddvg2ykg010var +dmail200212 +listunsubscribe +connect +misconduct +11 +0000 +topmargin0 +sashimi +jmlocalhost +another +yyyylocalhostnetnoteinccom +adaptor +if +good +browser +running +fallback +required53 +going +notification +thu +helomd01wowsynacorcom +tue +leramilerctrorg +lairxentcom +domain +thanks +re +importing +youre +result +exim +clean +spamassassintalkadminlistssourceforgenet +edt +just +325 +it +esmtp +httpxentcompipermailfork +listsdebianuserlisztdebianorg +friendfontab +marginwidth0 +a +dont +deliveredto +xmozillastatus2 +resentsender +localhost +2011 +linuximage26325br +as +spfpass +pdt +reminder +gnomecredits +alpinedeb20010042715341008360localhost +messageid +588432 +year +10 +messageidreferencestoxmailer +unknown +glen +testsblogspotldosubscriber +0700 +notinsblxblspamhaus15 +sven +kent +strip +httpxentcommailmanlistinfofork +or +above +25110 +klicken +problems +2007091301 +contenttransferencoding +required50 +quarter +way +color666666 +virtusertable +im +mailtorazorusersexamplesourceforgenet +service +textplain +127001 +8bit +2008 +files +that +virtual +listsdebianorg +to +v1 +nil +sep +freebies +1110 +match +charsetiso88591 +now +tlsv1 +dav23pvtyw4zlvuv6xp00009131hotmailcom +salon +ah +smtpmailbouncedebiannewsmlsubscribertechcsminingorglistsdebianorg +exchange +know +did +around +inreplyto +version +vs +something +newslettersa +at +through +lughtuathaorg +id +structure +listhelp +forkexamplecom +can +083353 +201518 +my +date +g +takes +esmtpa +2010 +reached +googlecom +102993570829981camelshiva +bgcolorffcc00 +207171180101amazoncom +manager +by +ipad +64 +deloptes +gmexim +listmasterlistsdebianorg +expect +should +xmailinglist +qyk5 +will +0100 +software +yf27ki6cco2fsfprotonpathnamecom +httpwwwhawaiistoriescom +listsubscribe +b059a4412f +with +rpmlistfreshrpmsnet +from +rohit +width1 +mimeversion +enht +gets +addicted +spamassassin +exclusive +e2e1616f7d +media +linux +lot +xmozillastatus +listid +charsetusascii +fetchmail590 +xaccountkey +regards +sender +precedence +skqaxylrpdga6wbyxgvlbliszt +4e0705992da50cfb4 +skynet +its +which +irish +7bit +listpost +dinter +topic +nets +repair2e22 +6416122236 +always +xvirusscanned +thurman +legs +install +122815 +30 +mailing +nusrat +between +allrecursive +about +class3d +requests +overwhelming +entrepreneurs +contenttype +xmozillakeys +gecko20100317 +other +since +and +xstatus +record +7xd8urcbazpaopcvhu8lbliszt +received +pig +list +811105 +hrefhttpclickthruonlinecomclickq08ifoqirjfbeeakm5g3fercpgy9ir +tires +rewriteengine +3d +thinks +the +d diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_48.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_48.txt new file mode 100644 index 00000000..1af51fea --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_48.txt @@ -0,0 +1,245 @@ +significantly +xygmxtrusted +oldreturnpath +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +backgroundhttptechupdatezdnetcomtechupdateibg232850gif +smtp +bs +n2 +working +hrnlocp881doughgmaneorg +resentmessageid +name +account5 +in +501 +spacer +have +your +returnpath +bearer +165526 +mailtodebianuserlistsdebianorg +192146 +is +httpwwwportigalcom +built +1712811332 +subject +4 +contentdisposition +than +2002 +bulk +want +09 +listunsubscribe +but +0000 +9 +welchpanasascom +gocomimagesbtp2pgifccode3d1d74c2eapcode3def39755c +role +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +url +dp6h4t9mu9xxvruoyzekynp57impiz2hlpumcuwnnnmpa5edmmfuba4gz8gpm21xgtxfo +similar +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +rtl8168d8111d +hair +style +good +telling +freebie +so +must +134552 +testsldowhitelist +tue +lairxentcom +re +users +receivedspf +colm +cropped +before +hrefhttpclickthruonlinecomclickqad4iaqzebsic0nbiqhaihsszky0pr +esmtp +reading +listsdebianuserlisztdebianorg +os +movie +newer +deliveredto +hrefhttpclickthruonlinecomclickq65k82zidptigiqcgwd70vof0m8p9zr +xmozillastatus2 +localhost +useragent +as +c36f +spfpass +hrefhttpwwwgnometomescomtome001828htmlinternet +pdt +messageid +spanked +bhb8rm7q0doza3sljid4fvbjeqe1kdpsrj0jepxe7kum +10 +unrealistic +noelamaccsminingorg +old +gnulinux +config +man +writers +or +awfully +safe +0200 +contenttransferencoding +forkspamassassintaintorg +way +092213 +12231148143 +mailtorazorusersexamplesourceforgenet +comic +stop +textplain +127001 +l20mr2017097qak671271734562360 +work +that +floating +helouswsflist1sourceforgenet +gmt +to +getting +relay2applecom +wind +xlevel +now +4bc6fab06080406studentulgacbe +october +freshrpmsnet +know +utc +single +021009 +rlcglj6m4qii +resentdate +183740 +believe +references +mxgooglecom +doing +isiloxc +at +iii +yyyylocalhostspamassassintaintorg +gmailpem +debianuserlistsdebianorg +20897132119 +id +httpradiontwizardsnetcategoriespda +on +mailtoexmhworkersspamassassintaintorg +same +mar +socalled +click +space +board +d0a5acd2c8bd +date +mo6541216221staembarqhsdnet +165236 +explanation +envelopefrom +by +en +then +either +obviously +thats +default +actually +will +2525 +xbeenthere +2098516147 +trusted +not +listsubscribe +with +from +enht +quotedemailtextreferencesspamphrase0001 +mail63opentransfercom +first +2 +lot +xmozillastatus +use +make +charsetutf8 +02 +fetchmail590 +sender +precedence +165155 +2fnsvxbs3dfasgh7slbliszt +22bf513a5a7b +file +resqlplperl8383110lenny1mipsdeb +graph +be +listpost +into +get +relevant +starting +better +whitehead +xvirusscanned +logmaneorg +ns2egwnnet +microsoft +experiences +mailing +passive +bfonttd +140047 +executives +translation +promise +morning +8219575100 +see +demand +debianuserrequestlistsdebianorg +sat +sfnet +dhersaaes256sha +performance +does +1000 +contenttype +xmozillakeys +craft +capactior +like +of +sourceloopaes +states +and +received +red +list +distruption +holding +pursuing +comments +clumsy +the +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_49.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_49.txt new file mode 100644 index 00000000..bf8ac1f3 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_49.txt @@ -0,0 +1,294 @@ +michael +roundup +namesignatureasc +application +valigntop +copyright +desktop +mass +xentcom +for +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +gnupg +people +amacatergalacticdemoncouk +module +jfoster81747verizonnet +clientip8219575100 +account5 +123003 +escribic3b3 +in +3d6b46ab162571ed4e92clocalhost +based +switch +copying +priority +returnpath +once +imap +is +mon +mailtosecproghelpsecurityfocuscom +multipartsigned +subject +already +system +concluded +2002 +xmailmanversion +rpmlistadminfreshrpmsnet +email +listunsubscribe +join +yes +0000 +jmlocalhost +helodomain +role +urgent +similar +forkxentcom +ist +if +fourla01 +apr +povão +up +085946 +frames +tue +26 +obvious +httpwwwlinuxiemailmanlistinfoilug +yf265whg1aifsfprotonpathnamecom +somehow +abvsfo1acmta5cnetcom +imagined +0400 +clean +031128 +mailtoforkrequestxentcomsubjectunsubscribe +edt +058 +sun +esmtp +listsdebianuserlisztdebianorg +two +a +helodellbuntulocalnet +deliveredto +wednesday +wrote +localhost +tools +dogmaslashnullorg +talk +div +spfpass +well +pdt +has +messageid +goal +quiet +popup +locally +newslettertitle +10 +height8td +ilug +172204166 +old +blockdevice +strong +processing +disks +055927 +they +0200 +2007091301 +172541336 +contenttransferencoding +101143am +required50 +21 +2008110401 +tnonsensefrom9293tnonsensefrom9394 +textplain +127001 +a8f2fd042a9a +agreed +that +out +java +listsdebianorg +hear +to +0h35ebw1tf +sep +shot +match +touchscreens +66187233211 +000801c26287f29604600200a8c0jmhall +singledrop +code +passing +exchange +utc +probably +did +maximum +most +190200 +resentdate +e20cs407967wfb +inreplyto +references +mxgooglecom +sequencing +experience +1 +debianuserlistsdebianorg +mix +id +209sfnet +positions +engineer +inpmrfci01 +folder +port +flemmis +distributed +forkexamplecom +esmtpsa +on +being +machine +ceo +same +humm +designates +subscriptionslockergnomecom +how +debiankdelistsdebianorg +14 +after +explode +date +who +2010 +mailtorpmzzzlistfreshrpmsnet +mailtorpmzzzlistrequestfreshrpmsnetsubjecthelp +googlecom +newly +180438 +order +envelopefrom +helped +by +en +then +think +unsubscribe +005146 +ctrentcom +however +expect +xmailinglist +default +actually +called +ldowhitelist5 +2525 +wonder +xbeenthere +0100 +wont +not +listsubscribe +times +with +from +cnofws +clicks +20020720051950ge28205linuxmafiacom +anymore +spamassassin +5712a16f8b +2 +tulsa +hardly +xmozillastatus +hrefhttpwwwlockergnomecomsubmithtmlsuggest +styletextdecorationnonemore +use +httpfreshmeatnetusersdougedoug +listid +put +fetchmail590 +size2 +rather +listsdebiankdelisztdebianorg +sender +permitted +envisager +precedence +mm +lawn +file +its +gillmor +agentpm +irish +an +4bcdd3f0a +atom +056 +taggedabove10000 +be +listpost +get +vlc +socialadminlinuxie +hrefhttpclickthruonlinecomclickq20poifirzzenfabiotrpp7us2qf9r +case +room +type +mailing +mark +81168116 +xauthorityanalysis +btomeraider +psybill +xoriginalto +dotancohencsminingorg +10143348 +no +writes +when +policy +version325 +theres +xmozillakeys +helpunsubscribeupdate +4bbb0a8a9010407coxnet +httptopicacomub1dcmwb2yxqn +like +of +settings +packages +xstatus +received +classen +ever +augdlistsapplecom +phobos +200209070351g873pc613144pcp02138704pcsreston01vacomcastnet +nates20 +the +004513 +send +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_5.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_5.txt new file mode 100644 index 00000000..af9b2875 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_5.txt @@ -0,0 +1,259 @@ +xev +please +oldreturnpath +behind +for +postfix +etcnetworkinterfaces +lots +smtp +involved +classtipnewstitle +formatflowed +arsasha1 +window +verify +href3dhttpgroupsmsncomoneincomelivingone +account5 +dkimsignature +this +in +certificate +cached +reported +returnpath +mailtodebianuserlistsdebianorg +is +uswsflist1bsourceforgenet +diagrams +003234 +moderator +indices +mini +server +subject +qmqp +floppy +than +hmimeversiondatemessageidsubjectfromtocontenttype +camerab +lected +2002 +bulk +listunsubscribe +quotedprintable +state +werent +0000 +jmlocalhost +best +spend +201 +would +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +yyyylocalhostnetnoteinccom +094052am +applicationpgpsignature +ist +unsubscription +apr +required53 +mtn +113618 +normal +gnus +2e +144728 +26 +101937 +class6jm +native +somehow +re +data +exim +set +dcsminingorg +clean +just +325 +xoriginaldate +esmtp +cable +listsdebianuserlisztdebianorg +24 +deliveredto +pages +chanted +resentsender +wrote +seeing +localhost +as +spfpass +messageid +signature +popescu +10 +finally +john +exmhworkersadminspamassassintaintorg +0700 +xspamstatus +httpxentcommailmanlistinfofork +or +resentfrom +public +come +2007091301 +forkadminxentcom +2008110401 +g8okn0c20652 +textplain +i +127001 +hrefhttpnlcomcomservleturlloginemailqqqqqqqqqqzdnetexamplecombrandzdnetfree +work +zzzzexamplecom +checker +servers +1oerkd0004w6ag +rpmzzzlistadminfreshrpmsnet +to +completely +murphy +singledrop +20080610 +imho +b36mr626059muo561271387385372 +know +utc +listmanredhatcom +single +httpbroadscapeventurescomweblogdfme +differently +234 +these +hrefh2 +mlsubscribertechcsminingorg +was +resentdate +happened +nothing +4965513a510c +at +softlink +often +171303 +linuxaccesible +id +bgcolor000000 +httpslistssourceforgenetlistslistinforazorusers +modules +listhelp +can +on +designates +yyyylocalhostexamplecom +integrate +acroread +kyzeqoz8g1ojrmwopejnjelbhzgkmwopdebqwz5u4ic3d1c4cfnvmrqqpjzdgzyvv +my +date +jalapeno +182250 +authenticationresults +supposed +truly +xuidl +ashersky +by +lisztdebianorg +screen +malay +client +helo +ldowhitelist5 +0100 +promises +p +autolearnfailed +software +wont +not +listsubscribe +fees +times +with +from +still +rohit +20 +classtitlenbspgnomecoreptdtrtablep +subscribed +gets +quotedemailtextreferencesspamphrase0001 +beta +spamassassin +use +listid +3156 +charsetutf8 +charsetusascii +xaccountkey +sender +precedence +07 +yyyyspamassassintaintorg +prevail +110241 +7bit +taggedabove10000 +listpost +fep02appkolumbusfi +mccormac +get +matter +155430 +enough +xvirusscanned +marry +update +install +microsoft +listmanager4084623190753220020820121844mothlightfastmailfmls1sendoutmailcom +7 +050517 +accused +uhhhhhhh +81168116 +theme +manual +spam +xoriginalto +example +biological +032035 +no +archive +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +httpslistssourceforgenetlistslistinfospamassassindevel +policy +version325 +theres +contenttype +xmozillakeys +hrefhttpnlcomcomservleturlloginemailqqqqqqqqqqzdnetexamplecombrandzdnetmanage +xrcvirus +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +other +of +and +xstatus +received +one +125135 +wed +directory +the +cds diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_50.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_50.txt new file mode 100644 index 00000000..1828a5a0 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_50.txt @@ -0,0 +1,275 @@ +rate +dangerous +xmailer +for +postfix +network +question +020838 +autolearnno +any +city +penny +resentmessageid +account5 +certificate +smoke +plate +132934 +your +internal +reported +returnpath +imap +mailtodebianuserlistsdebianorg +choose +is +utf8qcicycxrcoxwmurhckrvhcedaur7own20a099cqewexrexa7tjlyn9gscdhb +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +baa19534 +repository +httpwwwimakenewscom +size3d1a +0xb7eb7ec8avparameters +subject +4 +already +fluid +qmqp +listarchive +469 +2002 +xmailmanversion +bonus +want +listunsubscribe +disk +jmjmasonorg +width599 +0000 +004348 +jmlocalhost +essential +082051 +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +setting +172740 +ist +if +good +problem +4bc714707040501studentulgacbe +up +bgcoloreeeeee +maingmaneorg +hrefhttpclickthruonlinecomclickq16tcji4cfvq9diqgdblruabdxru4r +so +expanded +tue +domain +re +highlight +receivedspf +helodarwinctymecom +61 +route3f +required40 +164233 +it +esmtp +947dd47c67 +frederik +yyyydogmaslashnullorg +os +a +movie +archivelatest1295 +moving +deliveredto +008 +seals +resentsender +wrote +localhost +as +pdt +httpjeremyzawodnycomblogarchives000203html +messageid +500 +reminders +lugh +dog +testsfvgtmmultiodd +0700 +xspamstatus +yet +or +trying +0200 +2007091301 +ieyeabecaayfakvpw3qacgkqdnbfk86fc0rdwcghxga5lciwc9vpymzpe5jiqjk +season +contenttransferencoding +2008110401 +serves +textplain +invoked +i +127001 +info +19412514545 +ha +out +suggestionaspan +udev +opened +hear +to +become +recreate +charsetiso88591 +generations +20080610 +you +associated +web +utc +did +hat +selfnextserver +135146 +webnotenet +mlsubscribertechcsminingorg +help +resentdate +week +inreplyto +believe +references +mxgooglecom +6 +at +replyto +debianuserlistsdebianorg +size2font +id +wikipedia +deepening +also +mccarthy +listhelp +hrefhttpwwwhandybackupcomimg +elzyx2xe5gatedatbofhit +on +languages +v511 +filesystems +machine +multipartmixed +designates +complete +check +intel +how +newscom +after +date +19317254 +d1dee3ec9e +jalapeno +iso88591qcolin20nevin +allow +me +googlecom +xuidl +ve +by +then +think +virtools +120140 +lisztdebianorg +stream +7412582175 +will +inhouse +peoplesofta +0100 +listsubscribe +with +rpmlistfreshrpmsnet +from +still +cnofws +couple +tdtd +spamassassin +clicking +xmozillastatus +copy +use +listid +make +fetchmail590 +done +sender +unfortunately +developers +ultimate +didnt +errorsto +an +evolution +taggedabove10000 +sorry +cc +be +10227137143 +help0d +mailtodebiankderequestlistsdebianorgsubjectsubscribe +get +existence +smtpdsaslsecurityoptions +form +width36 +widen +erl561mailspamassassindnspmline374did001 +81168116 +spam +ifnerr +s40abr +8219575100 +3 +dreamweaver +major +notinblnjabl15 +093613 +xgreylist +3a8ef13a553e +version325 +contenttype +xmozillakeys +constitutionally +xrcvirus +like +officials +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +other +of +and +nametitleimage +xstatus +record +received +exhaustion +one +plan +k3b +exmhusers +cif +xen +arguments +search +the +httpwwwlifetimetvcomreallifehealthfeaturesdrugfreehtml +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_51.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_51.txt new file mode 100644 index 00000000..6e4acfe4 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_51.txt @@ -0,0 +1,297 @@ +michael +gelernt +species +httpwwwnewsisfreecomclick38688976215 +shower +alternatives +namesignatureasc +application +trouble +xentcom +for +postfix +bhow +hometheater +smtp +1227 +satisfy +uonlinecomclickq3d05gdulig8pctun3efnodtxi8fiiur +begin +formatflowed +any +chris +markallumscom +this +andreimpopescucsminingorg +returnpath +jul +imap +132920 +is +151149 +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +his +caffeinecsclubuwaterlooca +testsbayes002 +00000000 +ebay +myuqf0mcocy +text000000 +subject +custom +using +system +m8cs73574wfj +c0bfc1bf7c068 +2002 +xmailmanversion +bulk +itifontp +email +listunsubscribe +border +connect +192818 +0000 +debian +jmlocalhost +tobias +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +deed +mandrake +remember +202745 +radio +br +running +problem +20020910020728ofei9751sccrmhc01attbicomintellistation +2482 +up +etc +messaging +bgcolor3dccccccimg +do +outoftown +cu88l76nlcarhvwe6lbliszt +lairxentcom +ptdtd +had +re +trick +page +09747lenny2 +0400 +esmtps +edt +just +m8cs29094wfj +esmtp +protocolapplicationpgpsignature +reading +fix +listsdebianuserlisztdebianorg +inches +subscription +a +shrivels +deliveredto +wrote +hicsanchordeskfrontpagebgif +t2si11383950faa7820100511070717 +localhost +useragent +as +spfpass +barreraorg +currently +messageid +year +score69 +di +10 +petitbourgeois +computing +walking +phoboslabsnetnoteinccom +ranch +gnulinux +contain +strong +blockquotedivbrdoing +2dda329418e +trying +0200 +gmail1 +exactly +al +required50 +21 +forkadminxentcom +im +textplain +i +127001 +aikctkhd0fzma10 +that +hits188 +hfromtosubjectdateuseragentreferencesinreplytomimeversion +readmetxt +hrefhttpclickthruonlinecomclickq0eeotiiqzharlshv4r8djx98lbk4r +to +sep +66187233211 +singledrop +dvd +code +20080610 +20021007103831642c1bbbhostingj2solutionsnet +know +utc +forteanaowneryahoogroupscom +did +talkabout +company +mlsubscribertechcsminingorg +resentdate +inreplyto +mxgooglecom +at +been +39 +id +xcomplaintsto +port +listhelp +can +on +designates +focus +my +b83a03f6e9 +gamasutra +date +friends +jalapeno +tired +eldred +authenticationresults +sourceforge +spamassassintalkadminexamplesourceforgenet +replytypeoriginal +xuidl +them +somep +envelopefrom +by +either +lisztdebianorg +client +listmasterlistsdebianorg +thats +should +via +width75 +xbeenthere +back +0100 +again +p +kaa82784 +not +listsubscribe +surprising +with +from +spamassassintalklistssourceforgenet +0500 +enht +harri +clamav +spamassassin +true +ventilation +address +could +use +mode +charsetutf8 +charsetusascii +end +precedence +message +stt008linuxsite +under +yle3dmargin +7bit +sw +be +listpost +attached +brbfont +looked +fairchild +mailtoforkrequestxentcomsubjecthelp +power +0a1ba26a93978 +install +arise +html +really +start +4bbae46a6060104wanadoofr +mbr +fri +xoriginalto +masterlistsapplecom +impression +105 +many +edward +mistake +debianuserrequestlistsdebianorg +3 +mike +snap +no +razor +we +add +archive +sfnet +important +mailtorpmlistrequestfreshrpmsnetsubjectunsubscribe +winnick +antipiracybabr +version325 +lennybackports +contenttype +xmozillakeys +places +may +secure +smtpmailjavadevbouncesmlsubscribertechcsminingorglistsapplecom +nfrwalso +other +since +and +194651 +received +setup +size3 +bliss +o +therapist +m8cs75626wfj +mailtoforkrequestxentcomsubjectsubscribe +comments +accented +directory +sell +the +httplistsdebianorgdebianuser +rv099 +memory +cant +width100img +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_52.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_52.txt new file mode 100644 index 00000000..119aa756 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_52.txt @@ -0,0 +1,286 @@ +static +used +myzme65geyo +8xfq7v3mvciqog10eajna70kcewlgrnykllfbaeedjdtarc9 +httplistsdebianorg2010051021595631e1142babydosstargateorguk +pm +xmailer +for +mailtodebianuserrequestlistsdebianorgsubjecthelp +services +smtp +post +httptiltedwisdomcomgashtml +206248157232 +any +78701 +reso +years +allows +account5 +in +suppor +altspecial +have +needed +networks +color999999balso +returnpath +imap +mailtodebianuserlistsdebianorg +choose +are +is +why +martin +8369116f6d +score11 +target3d +among +00000000 +subject +given +qmqp +song +using +private +emc +contentdisposition +flist +donate +bulk +efc0a12c8ba2 +posting +001043 +rpmlistadminfreshrpmsnet +entering +listunsubscribe +quotedprintable +score2636 +11 +wrong +jmlocalhost +d982b16f16 +helodomain +alt3dnews +httplistsfreshrpmsnetmailmanlistinforpmzzzlist +httpnewscomaucommonstorypage040575037762255e1376200html +what +would +20100417073348ga2861kelgar0x539de +counterpressure +background +fallback +thu +manpower +keep +32836194125148311030699168squirrelspamassassintaintorg +g6vhzd203399 +hit +neighbour +peter +ready +receivedspf +0400 +binsh +esmtps +required40 +325 +it +sun +esmtp +httpxentcompipermailfork +items +cd3sf3f +200208280042154bca2588matthiasrpmforgenet +term +subscription +a +appearances +036 +hang +deliveredto +xmozillastatus2 +sitemesh +145847 +wrote +localhost +useragent +as +dogmaslashnullorg +well +pdt +messageid +588432 +062723 +signature +10 +lugh +unstable +0700 +quite +132229216135 +loftware +uswsflist2sourceforgenet +2001 +a40abr +j35j31gicdbpuyb4lh9wzmvrovnr2xdjcvoqu3meqbcxjehrvu1olvv390t9m +contenttransferencoding +2008110401 +secproghelpsecurityfocuscom +textplain +i +127001 +duck +that +out +thunderbird304 +dark +creation +to +sep +120442 +20080610 +ilugadminlinuxie +utc +small +syntax +albertwhiteirelandsuncom +unpeaceably +here +k29si3575023fkk1520100420151340 +mlsubscribertechcsminingorg +help +coreabr +was +resentdate +version +references +there +through +listuserneopittstateedu +085616 +id +someone +investorbr30day +doesnt +wellknown +also +need +on +hrefhttpwwwlockergnomecomrecommendhtmlrecommend +click +raw +date +httpftpczdebianorgdebianpoolmaineeglibclibc621026amd64deb +mailtoexmhworkersrequestspamassassintaintorgsubjecthelp +belloutput +cvfnqqhiggoxky895ni2p06ys7q26btysl77b0nrg5xa +2si2934000fks1220100423144735 +authenticationresults +googlecom +xuidl +by +q2w880dece01004180920m4f3f9906u46872de28d1d722fmailcsminingorg +k28mr299794mui251271352434848 +unsubscribe +even +214 +bob +should +tel16503272600 +new +via +width75 +fffff +ldowhitelist5 +covered +2525 +consensus +promises +again +preferences +not +with +from +books +widt +executing +1alnuq0007hv00 +mimeversion +tdtd +095320 +celejar +2 +flag +both +xcheckerversion +definition +pass +uid +use +listid +make +02 +news +charsetusascii +available +xaccountkey +171338 +sender +permitted +precedence +skills +a05200a01b9c80b70e2c220910320317 +its +errorsto +say +20020824 +be +mailtoforkrequestxentcomsubjecthelp +matter +xvirusscanned +version250cvs +13fonttd +height3d8720 +xloop +7 +10142633 +mailing +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +manual +anas +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +132030 +fri +gibson +xoriginalto +about +wartime +hrefhttpzdnetshoppercnetcomshoppingresellers0708514119946796htmltagstcrmpdppr9946796 +proc +see +httpwwwineedmorecowbellcomblog +no +archive +width3d65nbsptdtrtr +gnus513 +enus +day +including +contenttype +httpwwwnewsisfreecomclick28418828215 +of +sponsored +received +usable +wed +list +architects +comments +view +directory +the +volunteers +grokked +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_53.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_53.txt new file mode 100644 index 00000000..90235bee --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_53.txt @@ -0,0 +1,262 @@ +200210060800g9680ak15248dogmaslashnullorg +mason +side +fscking +debianuserspanishlistsdebianorg +read +xmailer +for +postfix +ss +program +s +tlcb +6416883170 +constantly +any +resentmessageid +verify +uswsflist1sourceforgenet +clientip8219575100 +account5 +this +in +cached +transitionalen +your +returnpath +mailtodebianuserlistsdebianorg +possibly +is +mon +source +60028001106 +feature +23 +subject +listarchive +3dffffef +mdodating201 +nnetwvvuvfu +2002 +bulk +research +connect +anthony +1126 +jmlocalhost +neue +j2se +what +would +nameliberalism +interactions +greeted +apr +mms +required53 +vivekkhera +uses +postfixlughtuathaorg +crash +published +re +size3d2july +host +idilbertemailcomemailsz3d468x60ptile3d1ord3d +constitutional +325 +it +sun +esmtp +xinjectedviagmane +893893 +two +local +20020906134417ga16820cthulhugergca +a +dont +deliveredto +83 +khare +localhost +useragent +as +inline +1931201712 +spfpass +well +pdt +messageid +score69 +170136 +10 +xgpgkeyserver +changes +0700 +or +ientertainment +submit +ireland +aanlktin9u3dmww3x0et8i9crpfwphp7lefe9ltoptljmailcsminingorg +urlhttpwwwfeinsteinsnetharlan +001517447a467934870485859c9f +2007091301 +contenttransferencoding +textplain +ilk +i +127001 +lw15fdlaw15hotmailmsncom +listsdebiansecuritylisztdebianorg +helvetica +that +out +rpmzzzlistadminfreshrpmsnet +to +sep +031744 +glossy +you +modern +know +utc +tdtr +bank +banner +did +home +cult +was +there +yourselfa +mxgooglecom +countries +aka +at +replyto +older +mp3oggs +id +engineer +nolayer +listhelp +driver +can +on +rwxrxrx +reports +my +cost +date +chpkjshnkjsgn0fugtweuqlyv3mot4fcbi0ighe0imhe0iigeuejgkyejqgaetqiaptmoqga +jalapeno +pci +sales +xsieve +xuidl +envelopefrom +by +then +unsubscribe +0100 +their +not +listsubscribe +reboot +with +from +still +widt +selling +investigations +href3dhttpwwwtwelvehorsescommmclickthrough2 +mimeversion +ricochetusrlocaletcricochet +tech +stopped +first +true +xcheckerversion +nonwicked +bouncedebianlaptopmlsubscribertechcsminingorglistsdebianorg +3d6538748010204barreraorg +use +picture +httpwwwjayredingcom +sent +fetchmail590 +xaccountkey +httppatrickwebcomweblogcategoriesinternettechnology +sender +precedence +07 +bx2gnexpracrab8ss5alaawqeoptsicuwumfvtwbi9ke1suxp1cxex7k8pvvxnzfk +message +1984 +its +an +score7 +randomly +taggedabove10000 +discussions +be +ok +222427 +removing +theyre +xurl +version250cvs +114629 +recommended +mount +mailing +1310sansserif +gecko20100411 +between +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +thursday +platform +81168116 +start +facility +n19grpscdyahoocom +updating +g7mbihl25223 +frommxmatchesnothelodomain0 +indepth +xoriginalto +many +score101 +httpthinkgeekcomsf +presses +no +ebooksabr +sat +important +top +20020823094833abdf4c44eargotech +979be8feccf611d6817e000393a46deaalumnicaltechedu +notinblnjabl15 +value +when +chips +saremsgidlong450893 +idt +contenttype +helpunsubscribeupdate +8220here +may +of +ive +height60 +and +cipherrc4md5 +record +bprevious +received +today +wed +list +richard +authority diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_54.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_54.txt new file mode 100644 index 00000000..a08a6aa8 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_54.txt @@ -0,0 +1,285 @@ +snow +boingboing +srchttpwwwcnetcomidpsmrbgif +copyright +oldreturnpath +335 +for +organization +postfix +c28si5988633fka4420100501192704 +smtp +httpwwwshagmailcomunsubhistoryhtml +pointing +08 +working +people +any +clientip8219575100 +dkimsignature +quicktimeapibouncesmlsubscribertechcsminingorglistsapplecom +in +have +needed +yyyyexamplecom +your +returnpath +point +is +mon +uswsflist1bsourceforgenet +hindi +101431608 +barrichello +150638 +explorer +subject +hard +silent +17k9nm0005yw00 +52523 +2002 +listunsubscribe +state +0800 +jmlocalhost +auth +url +essential +what +mest +different +would +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +httpgroupsgooglecomgroupsq22whitewaterfarting22 +ist +5so913933qyk3 +quinlanpathnamecom +problem +versions +smime +distros +quotes +so +4bd596d85050002heardname +re +asians +users +receivedspf +width1tdtr +set +g8igudc07204 +0400 +before +partialrfc2369 +just +required40 +rsa +undisclosed +us +it +esmtp +option +191128 +entire +a +deliveredto +pages +unticking +sworn +create +wrote +aislesorted +ionel +matched +localhost +tools +dogmaslashnullorg +thing +5 +she +messageid +200209051326422938347386mailmanlairxentcom +mapping +10 +certain +0700 +or +seems +5206 +p3plsmtpa0105prodphx3secureserver +althot +contact +come +groups +affect +they +nosend1br +provide +2007091301 +contenttransferencoding +usb +2008110401 +im +size2july +mimelite +textplain +i +127001 +w12si12208346fah8720100517015948 +borders +that +out +invite +estación +saving +to +sep +xlevel +know +did +amavisdnew +auth02nlegwnnet +behalf +mlsubscribertechcsminingorg +190200 +was +inreplyto +references +21078 +found +smtpsendmyrealboxcom +been +debianuserlistsdebianorg +074606 +id +doesnt +listhelp +need +on +6b9e313a4747 +garymcanadacom +being +player +tr +language +005601 +after +anaconda +date +2010042018310663af750dabydosstargateorguk +zdnet +recalibrate +2010 +tend +me +kcvpixelbufferwidthkey +mail +132821 +things +by +visit +think +either +motherboard +geege +ldowhitelist5 +xbeenthere +smokestacks +much +1986839 +not +listsubscribe +with +from +still +fixed +means +w3c +mimeversion +unsubscribeoracleeblastcoma +spamassassin +realtime +2 +httpslistmanspamassassintaintorgmailmanprivateexmhusers +tastes +linux +both +irrational +podium +httptechupdatezdnetcomtechupdatestoriesmain014179286454000html +could +use +listid +put +charsetusascii +fetchmail590 +regards +013518 +sender +precedence +crelaxedrelaxed +file +14b +errorsto +powerpc +employment +193906 +an +modem +rpm +taggedabove10000 +cc +listpost +2dfuvxrfyy0u +xpolicydweight +plans +informationb +6416122236 +src3dhttpwwwciscocomaucomvpsgtccdadisplayimage0img24700 +xvirusscanned +part +192168110 +more +iso88591qbgskvlvm9zsejlgaaaac3rstlmzs96sgzles95lgbtq30a096qaaai0surb +xloop +7 +mailing +mark +81168116 +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +29 +mbr +actual +062918 +xoriginalto +responsibilities +udell4 +see +4464cbcf14d57c0ea0525985629 +notinblnjabl15 +some +occasions +httplistsfreshrpmsnetmailmanlistinforpmlist +policy +contenttype +v7mr3307366qaq1871271949435099 +xrcvirus +18si6022688far020100524143041 +like +other +of +jealous +r2yb36d43c31004300127q8adf79dan4640c1148027b04emailcsminingorg +and +record +received +xsaeximscanned +list +reminiscent +usually +chairman +href3dhttpclickthruonlinecomclickq3d703vnjidp +the +memory diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_55.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_55.txt new file mode 100644 index 00000000..47fe6600 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_55.txt @@ -0,0 +1,284 @@ +analogous +e +please +oldreturnpath +xmailer +able +for +organization +postfix +smtp +question +monitor +community +feel +mailtodnsswapdigestlistsironcladnetau +working +spread +resentmessageid +area +clientip8219575100 +tells +in +certificate +180241 +women +based +priority +returnpath +imap +5cab713a4f5a +why +a52block +hrefhttpswwwsandhillscomsecuresmartcomputingcpufreetrial3aspsourcecp491bonuscomputer +subject +printers +qmqp +using +listarchive +system +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +n +ac5b713a526b +2002 +xmailmanversion +bulk +1259 +lib32ncurses5 +listunsubscribe +join +httpsciscocomauaucomciscologin11516use100htmlfont20 +0800 +5143 +degrees +future +would +if +compensate +fallback +up +2500 +so +afcc513a4ef4 +msergeantstartechgroupcouk +clickq3d42rf7cipkgjavfr3ppz3f74emirdr +lairxentcom +gmailid128a8b054ff1dede +httpwwwlinuxiemailmanlistinfoilug +hrefhttpclickthruonlinecomclickqfdbxdmqgor1rekypqoiti64q9t8pr +guaranteeing +figures +look +srchttpwwwhandangocomimgsthechampion2002featuredsoftwarebargif +0400 +p5qmvtoxhkuhjsbigvdffkyfqjppsao0wiam +just +httpsourceforgenetmailarchivesforumphpforumspamassassindevel +esmtp +17hwir00006r00 +listsdebianuserlisztdebianorg +a +usrsrclinux +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +deliveredto +sadev +job +xmozillastatus2 +helogrunt2ihugconz +create +wrote +localhost +2011 +dogmaslashnullorg +targetmail +pdt +messageid +hrefhttpwwwzdnetcomtechupdatefiltersmrc014175602044300html +20pre1 +10 +unknown +9cfb827e4120b +certain +0700 +colspan2font +140149 +men +sbin +133311 +walkiewicz +contact +come +such +they +xrcspam +2007091301 +directories +contenttransferencoding +textplain +i +127001 +outlook +qqqqqqqqqqregexamplecom +whatsoever +hopefully +19412517218 +124208 +ij607wfeygk +listsdebianorg +to +sep +fall +bnetworkingbbr +singledrop +20080610 +npr +bank +130238 +ximn +lightning +actodc +g82h3bl27025 +pumped +063235 +operate +these +talking +internet +inreplyto +version +references +mxgooglecom +messy +vs +happened +experience +at +1 +through +10300286476462tmdadeepeddyvirciocom +been +yyyylocalhostspamassassintaintorg +debianuserlistsdebianorg +id +disaster +on +eizldtfkopobhhstrvobgep771vpmhcc0ksxstlheqqr1kuptqnvhx0iummdsoyagoq +being +related +designates +stan +attacker +my +after +date +a5093c44db8a7241ad5f67dfa8e +authenticationresults +googlecom +them +mail +by +946a726a41419 +unsubscribe +lisztdebianorg +new +will +10216186138 +much +little +mx1examplecom +hackers +score96 +autolearnfailed +listsubscribe +times +with +from +refer +href3dmailtomaddu +mimeversion +couple +enht +implement +spamassassin +include +xcheckerversion +unretained +uid +use +dealloc +charsetutf8 +faceverdana +charsetusascii +fetchmail590 +xaccountkey +permitted +precedence +httpwwwfurnarinet +33000 +valuehttpwwwlockergnomecominput +ie +errorsto +which +fast +apple +nullmailer +score7 +7bit +taggedabove10000 +m8cs138636wfo +be +listpost +374 +get +krb5 +texthtml +time +better +072859 +million +chandler +xvirusscanned +dicecomb +xloop +busy +forgive +81168116 +pin +updating +bugtraq +many +see +newsphere +mailtoexmhusersspamassassintaintorg +images +no +sat +having +8c44e4289 +nameh1b +17128113151 +ubuntu +some +enus +group +contenttype +xmozillakeys +may +like +1301494218 +of +ive +and +xstatus +received +one +28 +hrefhttpclickthruonlinecomclickqa9ilbqqpuxnpyb8eoz2jqzv91zvmlr +sentto2242572601671038790646jmjmasonorgreturnsgroupsyahoocom +cases +chairman +access +directory +the +archivelatest573746 +xpriority diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_56.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_56.txt new file mode 100644 index 00000000..fb4387bd --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_56.txt @@ -0,0 +1,297 @@ +forgotten +plugin +laughing +216163192220 +significantly +species +4cc202d0c28 +copyright +format +for +postfix +impressive +mailtodebianuserrequestlistsdebianorgsubjecthelp +network +support +smtp +ol +test +drivers +feel +body +g8deioh01540 +clientip8219575100 +this +predicts +in +cached +cdt +alignrightfont +mailtodebianlaptoprequestlistsdebianorgsubjectunsubscribe +fingerprint +returnpath +imap +are +cvsrootspamassassinspamassassinmanifestv +is +mon +txqueuelen0 +132804 +trtdfont +00000000 +subject +referencesspamphrase0001tolocalparteqreal +less +listarchive +unpissup +worsening +2002 +6416612219 +email +listunsubscribe +exist +resubmit +but +kaa20920 +0000 +kde4 +httpradioweblogscom0108992 +similar +domainkeysignature +integration +would +yyyylocalhostnetnoteinccom +certainly +chose +273 +ist +if +good +world +httpwwwscreenitcommovies2002thesumofallfearshtml +required53 +fu3d +3015 +hrefhttpclickthruonlinecomclickqf7n0qnqlgzfyw3rqhlrzigir6imoer +httpwwwlifetimetvcommoviesinfomove3002html +our +u23mr11814098ybh1501273682315890 +do +him +re +frankenstein +f +receivedspf +interface +testsforgedrcvdtrailforfreeinvalidmsgid +0400 +dcsminingorg +idea +0xbb +124126 +anadebianorg +edt +just +255 +nightmare +required40 +esmtp +listsdebianuserlisztdebianorg +cars +a +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +helary +deliveredto +arsasha256 +xmozillastatus2 +resentsender +wrote +httpusclickyahoocompt6ybbnxieaamvfiaa7gsolbtm +dogmaslashnullorg +store +pdt +5 +messageid +year +10 +feeding +0700 +g7mjsyz21645 +sven +testsawlknownmailinglistnospamincreferences +splitsabbr +2436b21aa2203eee039e2cbb458 +fgout1718googlecom +perl +contact +such +httplistsfreshrpmsnetpipermailrpmzzzlist +they +problems +forkspamassassintaintorg +friday20 +forkadminxentcom +userid +comic +textplain +i +127001 +increased +19412514545 +out +net +srchttpwwwzdnetcomincludeadsjsrgroup2560 +20020920104055201c2986matthiasrpmforgenet +httpmembersaolcominossencekenkeyhtml +listsdebianorg +saving +to +sep +interactive +getting +singledrop +drop +you +2k7jwtavhdec2lrbpr +egp +utc +bank +bkk8ochrhed7okz122ztw3tybmfv9hhhqgj8tolgoxj49tf0fo0mdt4dv76kbewab0 +long +15so316802fxm6 +photograph +references +mxgooglecom +backwardscompatibility +nothing +at +debianuserlistsdebianorg +go +id +doesnt +port +listhelp +can +on +same +board +interest +appears +14 +gotten +therefore +date +usual +happens +april +representative +who +2010 +xuidl +resolution +122442 +cnetcombgif +by +then +think +lisztdebianorg +gmexim +xmailinglist +new +gif +ones +xbeenthere +0pt +0100 +gb23ilrr471946 +xamavisstatus +185717 +park +not +listsubscribe +exmhworkerslistmanredhatcom +with +from +101501320 +still +0quot +subscribed +enht +jobs +095905 +nosend1tdtrtbodytable +hrefhttpwwwgnomedexcomour +pass +address +swap +use +listid +words +charsetusascii +fetchmail590 +xaccountkey +sender +end +permitted +090110 +class +errorsto +echoed +81258125submit +0aaf1440cc +codec +7bit +cc +be +into +considered +pmatilai +quick +score104 +mailtoforkrequestxentcomsubjecthelp +1921681183 +6416122236 +starting +17s8t20003io00 +time +xfuhafi +version250cvs +215 +more +xloop +worthless +spamassassintalk +mailing +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +81168116 +anas +29 +fri +gmailid127f8745bb10e852 +8219575100 +sticks +see +width180 +204 +add +archive +sigjflqfragj4pcpl2ep06lgs +contenttype +xmozillakeys +user +towards +xrcvirus +securities +pecondonmesanetworksnet +of +record +received +020030 +period +height1brtd +list +71ypdaowdqvdn9pettingzoonet +month +mad +the +only diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_57.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_57.txt new file mode 100644 index 00000000..849ef238 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_57.txt @@ -0,0 +1,262 @@ +forgotten +namesignatureasc +for +postfix +smtp +jmuseperljmasonorg +xmailscanner +v2012 +any +hrefhttpclickthruonlinecomclickq1ejquli4jwhrb77cpqesb8le27qdrr +years +account5 +dkimsignature +this +keyboards +73 +autolearnham +cdt +returnpath +petrol +mailtodebianuserlistsdebianorg +point +are +mon +uswsflist1bsourceforgenet +bouncedebianusermlsubscribertechcsminingorglistsdebianorg +oceanfree +likely +request +subject +mohr +qmqp +am +contentdisposition +listarchive +system +than +2002 +featureprods +email +immune +but +0000 +828fb2940eb +mine +visible +smtpmailbouncedebianusermlsubscribertechcsminingorglistsdebianorg +080 +yyyylocalhostnetnoteinccom +tuesday +generation +plugin2 +ist +if +fourla01 +smtpq4gnmailissas9143net +libgnomekbdcommon +oct +own +fallback +required53 +up +all +dictionary +hackedbox +size1b +had +hit +run +leopard +receivedspf +61 +httpgo2emicrosoft2ecom3flinkid3d9729707 +edt +else +required40 +it +sun +xoriginaldate +esmtp +listsdebianuserlisztdebianorg +confusing +a +kamal +dont +mailin +deliveredto +information +resentsender +hrefhttpclickthruonlinecomclickq96pjnwqtdhdsjmy1ujuf1uaoeg0srr +rdnsdynamic01 +localhost +though +qp +05 +useragent +as +p9si1842418fkb320100429152508 +thing +091425 +1022324148 +messageid +10 +availablebr +ilug +encrypt +solved +sven +string +weblogs +computer +perl +contact +they +2007091301 +contenttransferencoding +framework +usb +zzzzexmhexamplecom +im +textplain +i +127001 +work +out +to +match +getting +singledrop +serverside +20080610 +associated +fork +filter +colspan3d2font +mlsubscribertechcsminingorg +resentdate +inreplyto +version +long +references +mailmsnjabber +signatureshortdensespamphrase0102useragentthebat +at +replyto +been +yyyylocalhostspamassassintaintorg +cipher +debianuserlistsdebianorg +added +dsa1787 +id +nolayer +listhelp +on +sequence +shown +discover +check +aaaaaqaaaua +error +date +67c5716f03 +debianmultimedia +jalapeno +who +esmtpa +issues +servicesfonta +bug +adam +easytouse +l +mailtorpmzzzlistrequestfreshrpmsnetsubjecthelp +by +exmhusersadminredhatcom +then +doubt +unsubscribe +ldowhitelist +new +will +g79il0b19672 +ldowhitelist5 +0100 +xamavisstatus +listsubscribe +certification +with +from +refer +debianlenny +ability +mimeversion +114250 +mailtoforkspamassassintaintorg +life +packging +xmozillastatus +use +listid +available +xaccountkey +sender +precedence +economical +message +stephen +its +errorsto +an +happening +httpwwwiamerickacom +taggedabove10000 +cc +be +230402 +brbfont +large +8bff916f20 +texthtml +geek +brbri +30 +httpwebpagesmarlboroedudebockweblogindexhtml +xloop +rh8 +tvdspaceratio2219 +httpwwwteledyncom +mailing +xgmanenntppostinghost +81168116 +manual +decided +xoriginalto +160925 +mailtorpmlistrequestfreshrpmsnetsubjectsubscribe +many +ff +no +bord +ptwqs6fjpjna0nbbxe1lbliszt +enus +when +known +policy +mailtoaugdlistsapplecom +contenttype +xrcvirus +other +of +and +record +received +without +fetchmail +taa26129 +darbar +sylpheed +the +httpwwwgeocrawlercomredirsfphp3listspamassassintalk +parts diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_58.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_58.txt new file mode 100644 index 00000000..dece52e1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_58.txt @@ -0,0 +1,277 @@ +ftp +read +xmailer +javaspecific +for +postfix +valuedetailquote +icq +1022366204 +support +smtp +community +admin +begin +mailwebnotenet +jpg +resentmessageid +dkimsignature +this +door +httphomefilternetnljanwolsheimer2xalthtml +httpwwwchimpswithkeyboardscomblog +in +margin +cached +returnpath +rust +bfontabr +are +mon +source +faster +testsldosubscriberldowhitelist +195343 +00000000 +myfileselect +market +subject +try +qmqp +listarchive +than +c911f16f6a +68116126 +listunsubscribe +yes +standard +but +eastrmmtao107coxnet +sure +httpwwwugnncom201004macmalwarealert +background3dhttpwwwcomi +offutt +lori +forkxentcom +proper +ist +if +apr +011102 +fallback +quotes +dreams +so +e67fe3ea26 +normal +gletterpost +xoriginatingip +all +dilemma +101735 +filesystem +g14eg5a26662 +domain +theft +httpwwwlinuxiemailmanlistinfoilug +whrkisaix8jwiekli0gk0glbzhjhaqrqtgj18 +demos +correct +4bec15064090508studentulgacbe +users +0400 +y4mr1555525faf511274452903404 +edt +325 +it +sunited +esmtp +blockquotedivbrdivbrdivdivdivi +protocolapplicationpgpsignature +listsdebianuserlisztdebianorg +pack +a +deliveredto +xmozillastatus2 +wrote +spearhead +google +distinction +localhost +20315833245dyniinetnetau +103700 +2011 +as +dogmaslashnullorg +inline +thing +141832 +pdt +5 +messageid +keating +selected +43094764257 +organizing +0700 +mechanics +care +kent +or +interested +03 +total +ignoring +problems +2007091301 +framework +2008110401 +gary +fieldengineer76228205248 +textplain +mimeole +i +127001 +httpconferencesoreillynetcomcsmacosx2002viewesess3281 +22240 +that +out +producing +v546 +to +least +108 +20080610 +you +hughescr +2010042717180153fff689celejarcsminingorg +1014118827 +name3dnptrail2111121 +k9si18235928fad2920100519074529 +2283 +aggressive +bz1applecom +102809235184 +59m +331vamm2 +boxes +kevin +httpwwwcaextremeorg +resentdate +074320 +believe +references +vs +11fe0d07b7cf7ae000003b237a4beab92b4b78 +discuss +experience +at +1 +id +someone +pan20100429194506csminingorg +2440 +disaster +doesnt +port +archivelatest32193 +need +can +reflex +on +ppl +same +installation +06 +wall +mailtojavadevrequestlistsapplecomsubjectsubscribe +how +date +19317254 +hrefhttpwwwhandangocomplatformproductdetailjspplatformid1ampproducttype2ampsectionid0ampproductid43645ampcatalog1ampsiteid159 +vobcopy +gray +jalapeno +2010 +occurs +me +ameless2gif +googlecom +mail +he +thats +xmailinglist +curious +xbeenthere +much +xamavisstatus +listsubscribe +with +from +still +rohit +httpwwwozzienetblog20020925htmla65 +cnofws +mimeversion +enht +tested +r2g81c921f31004260456z3c6f41ddg86e45cdae1257104mailcsminingorg +systemoutprintlnlist +xmozillastatus +use +listid +jmasonorg +fetchmail590 +sender +precedence +stephen +file +its +which +fully +taggedabove10000 +13380006681398081270566209388javamailrootmd01wowsynacorcom +be +contrib +isn39t +her +fine +partitions +gamix +xvirusscanned +anything +x11 +farquhar +width430 +special +more +81168116 +sgamma +monitors +xoriginalto +aanlktilqtvstnerew1lmnlsyto13zpxsygphtejgivmailcsminingorg +triad +see +3 +104904 +no +archive +some +fear +0600 +contenttype +helpunsubscribeupdate +203338 +of +and +asks +received +setup +wed +list +28 +access +technical +the +cant +mdt diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_59.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_59.txt new file mode 100644 index 00000000..61709459 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_59.txt @@ -0,0 +1,281 @@ +looks +lists +pm +10872235 +trouble +classtitlenbspgnomeaudioptdtrtablep +oldreturnpath +ways +for +postfix +width483 +network +find +question +203541 +formatflowed +205628 +verify +name +clientip8219575100 +allows +account5 +till +in +design +have +your +returnpath +once +are +is +1beae43f9b +why +bgcolorffffff +course +testsbayes002 +145129 +23 +105613 +said +subject +pgp +hostname +052334 +relay +using +2002 +xmailmanversion +micalgpgpsha1 +recipe +email +listunsubscribe +border +11 +0000 +land +topmargin0 +jmlocalhost +81228122 +campaigning +proved +17w6ex0003ng00 +what +201 +different +44 +another +yyyylocalhostnetnoteinccom +online +fc695557d42cccad92de8ff27964ade9 +applicationpgpsignature +accucast +if +20100504063239783tchateletfreefr +a34bc16f03 +running +restore +gmailid12821dc56c4f5fb4 +do +roper +complained +collapsed +re +present +httpwwwhumorisdeadcomblog +afraid +data +every +opportunity +capturing +before +laa31421 +it +altm +esmtp +httpxentcompipermailfork +os +a +deliveredto +didunfortunately +resentsender +google +localhost +05 +as +dogmaslashnullorg +because +well +height3 +has +regular +messageid +marker +warning +mgbdebianyosemitenet +powered +overheard +yet +kaddressbook +d4si1809556fac9420100521074142 +resentfrom +public +hrefhttpwwwecobuildercomlockergnomecfmecobuilderabr +contact +2007091301 +required50 +tomas +im +section +educations +textplain +127001 +shareholders +httplistsapplecommailmanoptionsx11usersmlsubscribertech40csminingorg +buy +isnbspqqqqqqqqqqzdnetspamassassintaintorgfontp +to +interactive +6129 +charsetiso88591 +singledrop +now +tlsv1 +you +amavisdnew +pv9f9f9f9f9f0vl4fc93h +solution +here +links +references +happened +second +cooker +debianuserlistsdebianorg +id +listhelp +driver +can +tasks +stalag +yyyylocalhostexamplecom +o13si7770220fah4120100525101217 +systemerroremit +how +appears +date +cmdline +a0 +69 +jalapeno +who +googlecom +xuidl +envelopefrom +contact20 +manager +by +then +think +ipad +unsubscribe +m9v4k8os +listmasterlinuxie +listmasterlistsdebianorg +link +ldowhitelist5 +2525 +0100 +xamavisstatus +again +grublegacy +tracked +ft +89551f9 +not +listsubscribe +score110 +with +from +defined +enht +spec +hits6745 +clamav +spamassassin +r2g81c921f31004260456z3c6f41ddg86e45cdae1257104mailcsminingorg +turn +include +assumptions +1022364205 +linux +xcheckerversion +pass +could +make +charsetutf8 +fetchmail590 +available +sender +invariably +permitted +mckelvey +yyyyspamassassintaintorg +crelaxedrelaxed +an +cell +original +io +db09716f1a +a601116f56 +cc +box +be +listpost +fmsmtp06dlancineticde +into +where +schampeoheskethcom +version250cvs +farquhar +kind +hdomainkeysignaturemimeversionreceivedreceivedinreplyto +objects +sort +bssiguanasuicidenet +gecko20100411 +mailtodebianuserrequestlistsdebianorgsubjectunsubscribe +sgamma +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +xoriginalto +outlive +alinkcc0000 +8219575100 +word +made +intuitive +rv11 +enus +theres +along +contenttype +xmozillakeys +xsaslcyrusserverfirst +input +lennyvolatile +may +of +and +record +received +avittaa +list +cvs +sylpheed +sell +the +mobo +xpriority +never +d diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_6.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_6.txt new file mode 100644 index 00000000..15017fed --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_6.txt @@ -0,0 +1,278 @@ +rate +widely +mailtodebiankdelistsdebianorg +kellyclowerscsminingorg +berserk +180611 +for +postfix +encodingutf8 +smtp +6621867196 +hrefhttpclickthruonlinecomclickqf2ozvtquwprrj3gv0pvvdi5z7ednr +hrefnews42htmltopback +managed +17230098 +pappygiyiucefqroeyvlso0erk6qzzncogjluykxu6mk0y6macoadxecaqothnryq +resentmessageid +clientip8219575100 +account5 +dkimsignature +this +411 +in +charts +firewall +your +returnpath +obliges +point +are +is +versiontlsv1sslv3 +score108 +19 +feature +width145font +00000000 +1712811332 +subject +displ +system +20100504024346ga7799chunchems20nix +t5cejhoavnwojfoxhvs653llidasxhhti8p0pl3xwk7pt2rillv7ladaqbid4fjy24 +apc +2002 +bulk +ideal +three +walked +initial +forwarded +listunsubscribe +but +width599 +whether +0800 +wrong +debian +jmlocalhost +0530 +plus +domainkeysignature +what +another +ha13sm1786896ibb1520100510132526 +launched +fourla01 +project +apr +fallback +2482 +so +bgcolor3dfffffffont +classsidebar +thu +xoriginatingip +all +him +hand +print +ptdtd +alexander +090041 +mailtorazorusersrequestlistssourceforgenetsubjectsubscribe +cannes +verrall +re +214424 +wylieblog +postgrey131 +just +required40 +rant +it +100919am +xoriginaldate +eeeeee +esmtp +size2926td +runs +daves +owenpermafrostnet +24 +subscription +partition +a +deliveredto +job +mean +xmozillastatus2 +idbb7c321c +localhost +daa21056 +useragent +as +dogmaslashnullorg +xvrscore +spfpass +pdt +has +messageid +10 +underpowered +ilug +unknown +dkimneutral +084410 +gnulinux +discuss2 +protests +or +1vd9ng3fpaztbsvyf2f2ibdsttd92aelkpifjbrwyua7k3q3funinrznaphf +xrcspam +trying +0200 +reason +contacts +score149 +textplain +i +that +2829313a5536 +to +v1 +piii +charsetiso88591 +singledrop +xpgpkey2 +utc +bank +19972002 +recently +most +inreplyto +believe +references +found +79d9413a5693 +meddling +discuss +at +1 +law +id +doesnt +jabber +listhelp +theyll +issuing +on +035150 +743282 +cellpadding1 +readersbfontbr +designates +after +date +hrefhttpclickthruonlinecomclickqf2ozvtquwwqrqaoa0pvvdi5z7ednr +editionbr +463 +mailtorpmzzzlistfreshrpmsnet +authenticationresults +me +googlecom +by +en +client +helo +xmailinglist +qne6u037n3cab8cppz8lbliszt +gif +ldowhitelist5 +xbeenthere +206160226 +much +httpslistmanspamassassintaintorgmailmanlistinfoexmhusers +not +listsubscribe +with +from +still +routed +unregistered +subscribed +mimeversion +213922 +give +143933 +properly +v106 +life +channelonline +pass +xmozillastatus +listid +fetchmail590 +sender +precedence +unfortunately +johnson +class +crelaxedrelaxed +1049 +taggedabove10000 +cc +be +listpost +waste +sune +into +20100512112158209howlandprisscom +10s +18 +better +anything +ignored +ids +parc +output +hp +xloop +groundbfontbr +r +html +manual +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +chef +xoriginalto +dotancohencsminingorg +see +3 +no +moved +sfnet +score000 +xgreylist +group +version325 +contenttype +xmozillakeys +filed +might +of +nows +havent +and +record +sponsored +speak +received +isnbspqqqqqqqqqqzdnetspamassassintaintorgp +p05111a4eb9869cb2eb2a66149496 +list +gmailid12898ad365e23d18 +phobos +display +width209 +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_7.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_7.txt new file mode 100644 index 00000000..739a5c54 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_7.txt @@ -0,0 +1,275 @@ +192332 +used +pm +stating +xmailer +hurry +for +postfix +s +encodingutf8 +find +quotnquot +n0nbn0nbus +further +08 +people +x +production +dkimsignature +this +d651043963eda6fb4 +width399 +reg +httpwwwinteralianetindexphp +based +have +returnpath +rssfeedsexamplecom +iluglinuxie +893893112 +is +n39grpscdyahoocom +005120 +ddos +n28grpscdyahoocom +bit +subject +4 +try +enig31caaed0fb049d5bd7604594 +listarchive +smtpmailbouncedebiankdemlsubscribertechcsminingorglistsdebianorg +included +shell +2002 +xmailmanversion +want +listunsubscribe +jmjmasonorg +but +state +security +urgent +inbox +bhnua2m5uhyg9hexlryyoknpchy7xzf4o3qpooc7nyqq +212173515 +yyyylocalhostnetnoteinccom +ist +if +unsubscription +background +lcd +good +happen +required53 +hits2877 +ithe0d +httplistsapplecommailmanoptionsjavadevmlsubscribertech40csminingorg +hrefhttpclickthruonlinecomclickqb1yjxjq0mipripyur9ifb0nlnlr +thu +b4zgvdnr0dadzfzxw6lbliszt +kioslave +tue +isle +domain +pse +re +supermicro +host +04232010 +receivedspf +result +61 +0400 +exmh2113882517p +just +required40 +325 +it +vietnamera +esmtp +option +ps2 +a +deloptesyahoocom +deliveredto +unusual +clearly +responsibility +localhost +lenny +useragent +dogmaslashnullorg +4152010 +pdt +96000hz +messageid +gecko20090706 +10 +phoboslabsnetnoteinccom +0700 +nowgettime +xspamstatus +or +hints +broadcasts +gmailid128739f44994a175 +lexicon +divdiv +mouse +such +they +2007091301 +519015741132491272296881284javamailrootmd01wowsynacorcom +im +o3ji1oav002872 +ma +textplain +i +127001 +outlook +testsgmailhtmlmessage +to +factor +enig91847f0ca91f3bcc9384f202 +base +xlevel +now +20080610 +steady +you +utc +keeping +auth02nlegwnnet +here +mlsubscribertechcsminingorg +was +resentdate +week +long +references +alb +something +at +through +httpadvogatoorgpersonraphdiaryhtmlstart252 +id +meta +209sfnet +215415 +pressing +mcrypt +113242 +political +listhelp +robinrkrahlde +on +17 +twells +000e0cd5d0120026320486684691 +reports +06 +altctrlf1 +optimistic +my +date +forwardedby +mailtoexmhworkersrequestspamassassintaintorgsubjecthelp +receipt +beat +sales +2009 +authenticationresults +xuidl +20020917172627a1dbdc44dargotech +commercial +envelopefrom +by +garymteledyncom +unsubscribe +snip +xmailinglist +1075a29409c +actually +link +0100 +xamavisstatus +little +again +h95f875ldeafyifzeo2d60ft +autolearnfailed +hatred +reduction +with +from +maker +20 +mimeversion +tdtd +favour +tested +spamassassin +realtime +mailtoforkspamassassintaintorg +linux +xcheckerversion +pass +could +sent +charsetutf8 +charsetusascii +xaccountkey +copyfight +sender +end +permitted +precedence +whole +didnt +101428120 +errorsto +which +place +score7 +locales +taggedabove10000 +debate +discussion +be +listpost +nations +get +worlds +xvirusscanned +version250cvs +attractive +207202171254 +153853 +about +avoided +hosting +href3dhttpwwwjhstuckeycom1080jpeghttp +8219575100 +talkboards +no +having +requests +felicityklugenet +arranged +contenttype +xmozillakeys +15000 +m86ed5m86me7we75ew0k3 +input +may +write +of +received +one +wed +free +list +hoping +probability +the diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_8.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_8.txt new file mode 100644 index 00000000..f757119a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_8.txt @@ -0,0 +1,296 @@ +rate +nonfreely +debiankde +c2sbx +0300 +mailtodebiankdelistsdebianorg +444 +debianlaptoprequestlistsdebianorg +xentcom +for +postfix +program +network +smtp +4f98c13a46a1 +account5 +tomwhoreslacknet +in +belly +returnpath +zdnetbfontbr +mailtodebianuserlistsdebianorg +is +psyche +101431608 +testsldosubscriberldowhitelist +testsbayes002 +sorj9ngao0pkmwygpzccytwsmkmis0je42mo3ku7lnve24rnjbo7zznxyhv5k5ktnltqmi9w +00000000 +crossfirewall +useless +subject +qmqp +205902 +xenigmailversion +applying +width588 +httpwwwnewmarscomfirstwords +81168116egwn +included +028 +bears +email +194235 +listunsubscribe +disk +but +colspan7 +0000 +0800 +9 +domainkeysignature +what +giving +certainly +initialization +ist +line +highly +fallback +required53 +thu +aenrup4estpardfz5g9lbliszt +all +startup +103719 +domain +10877330 +pinelnx433020905160120022237100000watchermithralcom +providers +behaving +archivelatest577506 +re +assuming +kuznetsovs +87db7 +look +page +articles +rea +before +mailtoforkrequestxentcomsubjectunsubscribe +enabled +gddrescue +required40 +slight +it +score +rodbegbie +esmtp +172541337 +fix +blackcombpanasascom +os +mailtospamassassintalkrequestlistssourceforgenetsubjectunsubscribe +deliveredto +xmozillastatus2 +resentsender +localhost +earlier +2011 +as +cambridge +rising +spfpass +8542716f03 +231 +has +messageid +plug +10 +dies +xspamstatus +colts +pudge +httpxentcommailmanlistinfofork +or +4bed20469000006coxnet +command +they +trying +streamlining +required50 +clears +2008110401 +ghnrg +textplain +i +until +127001 +aikctkhd0fzma10 +that +orgdebianuserrequestlistsdebianorgabr +to +sep +114133 +element +809122912 +singledrop +fwd +20080610 +tlsv1 +pinelnx433020909155345032400100000hydrogenleitlorg +xenvelopeto +simpson +was +resentdate +inreplyto +version +sequences +references +found +c1 +at +a25si4724118faa1620100523143748 +id +over +listhelp +can +on +appears +after +cost +date +sans +jalapeno +easy +jim +2010 +me +googlecom +xuidl +defaultspcmsurround51device +them +helped +powerbr +holders +by +exmhusersadminredhatcom +bogus +then +think +either +unsubscribe +fort +lisztdebianorg +even +214 +listmasterlistsdebianorg +new +actually +will +mailin11applecom +majority +missing +2525 +xbeenthere +0100 +153536 +c18mr2917281fat481273916694875 +not +listsubscribe +etbssiguanasuicidenetagtspan +with +from +diamond +team +mimeversion +next +stations +first +actions +ignorant +netdiscover +aligncenter +families +pass +xmozillastatus +use +listid +11fe0d0bb7b68ae0000042ab8c4bd2f195dbdd +news +0l2x00j8pn7x00h0amtaout20012netil +available +sender +15000000000 +an +scrutiny +kahlessaddixnet +taggedabove10000 +x8664pclinuxgnu +xbrightmailtracker +box +development +shows +roman +64286735 +titlesoftware +time +151335 +latest +xvirusscanned +214046 +more +mdocabletv305 +update +ns2egwnnet +mount +colorcc00003650fonttd +hin +3ware +william +httpwwwmejlgaarddkblog +ichat +3 +during +no +very +sat +console +mailfollowupto +scandinavia +some +value +does +when +leavecustomers949326kmailryanairmailcom +reasons +version325 +great +sansserif +contenttype +xmozillakeys +may +like +secure +of +ive +log +packages +and +color000000downloadsfonta +devfb0 +xstatus +record +received +searchnetworkingliststechtargetcom +list +days +mid2003 +forteanaunsubscribeegroupscom +mailtonunomagalhaeseuipppt +arguments +the +never +those diff --git a/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_9.txt b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_9.txt new file mode 100644 index 00000000..2a0ccf9d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/NormalTrainingEmail_9.txt @@ -0,0 +1,300 @@ +101850 +fashion +182848 +trouble +121157 +qmail +xmailer +unread +for +postfix +mailtodebianuserrequestlistsdebianorgsubjecthelp +smtp +drumming +post +works +08 +californias +autolearnno +v2012 +x +any +resentmessageid +huge +dkimsignature +73 +autolearnham +in +off +returnpath +imap +are +mon +sansserifb1995br +agree +eastrmmtao105coxnet +plane +00000000 +imprononcable2ldosubscriberldowhitelistmurphywrongword1 +subject +already +ctible +qmqp +provider +caspian +listarchive +bulk +rpmlistadminfreshrpmsnet +email +listunsubscribe +quo +lets +11 +debian +96de40a2655bb0e854e962782aa +height6 +another +front +image +profit +if +center +gldudebianuser2mgmaneorg +tool +big +required53 +lexgatd +up +srchttpwwwcnetcominlinvrhclosegif +triptpolers +cellpadding3 +dadada +xacceptlanguage +thu +22 +do +60026000000 +hand +domain +kicked +re +assuming +figures +run +price +users +receivedspf +result +acter20 +enabled +required40 +325 +it +reminds +esmtp +150904 +superior +schiltz +two +a +deliveredto +wednesday +resentsender +wrote +localhost +earlier +05 +forward +as +dogmaslashnullorg +prompt +spfpass +pdt +5 +has +messageid +industry +especially +0700 +184517 +above +herself +strong +contact +agent +null +xrcspam +2007091301 +whats +contenttransferencoding +2163418188 +userid +p0cgsr1rbjkanlhhy3lbliszt +20100508085900025lisireiszcsminingorg +eventually +textplain +8d5f943f99 +127001 +charset3dusascii +5173916f16 +xoriginalnewsgroups +ops +4996 +tester +httpwwwugnncom201004macusenet +daly +listsdebianorg +4bbfba3b20009webde +to +sep +getting +xlevel +masking +you +expert +120328 +web +edit +87r5m4d0w8fsfmerciadrilucaeeeworkgroup +help +190200 +was +resentdate +week +version +personal +at +1 +attach +testsbigfontctypejusthtmldearsomebodyforgedrcvdtrail +been +xspamlevel +id +snug +largest +aes128sha128 +colspa +xcomplaintsto +port +listhelp +mug +need +insist +on +height50 +f3gawjbe6papgcz7fzlbliszt +complete +my +zzzzlocalhostnetnoteinccom +n12mr206682fad351271790911462 +date +live +frozen +httpsexamplesourceforgenetlistslistinforazorusers +l +authenticationresults +xuidl +mail +by +ass +father +emaillifetimetvcom +then +cameras +versatile +should +ghostscriptdivdivi +picked +helps +httpfreshmeatnetprojectswolfpack +called +xbeenthere +dd45613a54bf +0100 +2000 +hrefhttpclickthruonlinecomclickqcfebmqlw2hw8nmhvjaylio7r3fxer +empire +not +listsubscribe +httpfgvgiovannablogspotcom +character +with +from +httpwwwmemoriafotonet +un +023241 +behaviour +src3dhttpsccommunitiesmsncomimgcgif +ldowhitelistnorealname +pass +oaa07112 +use +listid +91184323 +charsetusascii +dolly +report +regards +header +5img +sender +precedence +102809238414 +anyone +errorsto +152735 +an +cc +be +listpost +directorys +dartmouth +fine +bouncedebiansecuritymlsubscribertechcsminingorglistsdebianorg +rights +bits +xvirusscanned +script +unreliable +more +case +moines +xloop +wanted +highligh +ptfont +camp +divblockquote +enriching +81168116 +manual +updating +mailtodebianuserrequestlistsdebianorgsubjectsubscribe +cccccctrtd +193351 +alguien +qb +debianuserrequestlistsdebianorg +3 +no +dhersaaes256sha +let +125717 +contenttype +xrcvirus +write +jimdouglasmaccom +of +httpusclickyahoocomsrpzmcktmeaamvfiaa7gsolbtm +received +high +posted +phobos +month +140536 +hoping +width3d561 +cnetbfontbr +the +send +hong diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_0.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_0.txt new file mode 100644 index 00000000..d41a6819 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_0.txt @@ -0,0 +1,366 @@ +rootlughtuathaorg +used +xasgorigsubj +down +homes +32224988 +read +for +organization +narrative +mattersnbsp +xbarracudaspamstatus +bordercolor3d111111 +c11197ted +hotels +usabr +yosyxworldnetattnet +lefebvre +fringilla +sa154 +targetnew +body +name +this +10th +in +alignrightfont +your +returnpath +height3d5td +breakdown +are +is +guy +thirdparty +his +workflow +namesitename +srchttp65112138121mailblockimagesmblogotransgif +benchmarks +pst +prices +uhqhzcdc4ze4xpcylejuqpze1afy9escjsnwozqy2naibembmdldjvwjwxxvzj7ttdlpurbiypg +approximate +subject +try +soon +e9vywlwkrwv3mlwsvlq0jrs4oxssmilzfrbbsbqm1vhjtc7b6ubaktdruospmy7aowkzmjdryo +effects +hrefhttp3255tsajalircniciud22of9awer648b56e5z460f54ampsieehokefy56643156563099499332159industry +system +organ +fairly +elder +included +6hx1xjapiaqemec9lwd8gxabnfffka4opezgp7r8grabnqp4s1uqpe57zjhrrrz +email +yes +e15mr2610440fgi231272296699829 +jmjmasonorg +standard +15 +contenttexthtml +would +professionals +proper +style +line +if +deliver +rdnsnone +143134 +face3darial +corrections +00005e83686900001971000014a2mailcareernetnl +so +xbarracudaconnect +250 +ushered +3c2ffont3e3cfont +all +georgenetnitronet +hand +rustic +served +u +attended +workers +pretty +settling +alignrighttend +campaign +xmailerpresent +data +page +every +61 +runnersup +0400 +rolandapiggee3zfhotmailcom +independence +115pecul97rly +mint +it +sun +esmtp +nbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +24 +172122 +determined +colspan3d22222 +mx +representing +dont +situation +ih7cq0lbpdn8ngzcptio8uhiephd9dkmmslbwmieqzuag6co8ksl2ol7tcnvlxtoh1q +moving +xmozillastatus2 +hibodycsminingorg +35 +berlin +though +2011 +gallowglass +has +teaching +messageid +350 +m +10 +xasgtag +list20 +mime +especially +align3dcenter +tda +particular +or +195554 +public +013425 +092036 +50000000 +hrefhttpwwwwebcredit2002comcid2020ampmid3300img +48 +175109 +a1a2cdb6c9e4b5c6a1a2b9a4b3ccd3c3b5c6b5c4bfaab7a2ba +xbarracudaurl +exactly +contenttransferencoding +regardsmrs +width10nbsptd +insphere +de +daily +textplain +127001 +color3dcc3300span +outlook +bsmtpd +diversity +girls +cards +4fontfontbblockquote +unload +cannot +quarantinelevel10000 +to +220430 +sep +name3dhowsoon +miss +sucked +charsetiso88591 +cts +singledrop +embassy +value3dsubmit +colspan3d5 +td +web +euoaiwu3ruoyazlgwuuhoaz5ouniaoqfir1nxs7keohamyxq43ocn6ukstaxtokednplh4qcqwh +32 +jun +veronica +various +tlo2txg1x5yahza9lzp5s921knveopcxovdj8fcjessd5e7ey4cmee8q4yy5cprl83ppue +rev +similarly +recognizes +valueoregonoregonoption +version +visiting +vps +aton +name3dgenerator +personal +block +205234 +steal +lxnlcmlmiibzaxplptigy29sb3i9izawmdawmd5bdxrvlwluc3rhbgxhdglvbia8l2zvbnq +at +1 +htmlimageratio02 +300 +id +ratio +height3d330 +dxb +on +17 +costbenefit +hrefhttpxui67bixyeqawcndaicqzugovyburjetoimyyij1da342067njogozqreoix +htmlmessage +060026 +click +space +ink +date +valueoh +inspiring +bsfsc0tg035a +96 +091632 +international +245 +authenticationresults +forever +xx286646anonpenetfi +googlecom +order +hrefhttp69f32brownchooseru +peace +by +facearial +however +antispam +050 +spain +6e5bb6f +0100 +unter +2000 +p +wont +reduction +not +style3dborderright +xbarracudabblip +with +from +preserved +means +stronga +money +mimeversion +size1 +phy115ic115 +2 +aligncenter +racing +both +transmissions +xmozillastatus +02 +fetchmail590 +areas +claims +available +xaccountkey +ten +regards +stiffen +permitted +chetnik +message +12245142206clientattbicom +639388152lsandialmoonglobalcom +its +agecroft +1904 +19216818251 +titlefor +n9sjpdum010186 +020 +five +pr0nsubject +aug +be +listpost +c +stronger +account +linuxmidrangecom +texthtml +rights +rules +grantsfontfontfontbh3center +fonttd +road +flowu +several +more +evros +6a7d743d45 +safety +type +form +html +valignbottom +size3d1 +style3dfont +height28 +chronicles +decided +mxsqv8aoco5wbh25dgzirdsbl9wjhgoqdad0az2noqc48gzycs14om1vvoqce59nnkfonj6 +padding5px +contains +fri +rememberance +about +m0620212mailcsminingorg +0900 +transaction +3 +bgcolor3d660000font +imre +sf8azjlsf0fif0v9o72ekbxjbdud987n30ulqn1b6r026xlllraagc7iqy81ajdouy36cq1 +naacpcrf013812 +i3ookxilckivwcrlavp0qtjerwvfcapj4tm6tz2gnka5wn1uobehqk0fti6pdhnbabugxcxofsc4 +when +chips +contenttype +xmozillakeys +presidnt +johannesburg +elaborate +may +assist +of +states +100elights +csminingorg +401 +and +received +spiritually +windows +métier +right +width3d91font +sa392hl +pennies +dear +deleted +table +cases +leave +directory +classstyle14font +094353 +associates +died +those +advertising +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_1.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_1.txt new file mode 100644 index 00000000..723876c1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_1.txt @@ -0,0 +1,384 @@ +ikotoik +contracts +used +xasgorigsubj +pre +w8it4ejlkqswliuj61vq81znhuxhuqfda5ii9s10nttzgdbvcjcgvzsoo5b1fjk56e9kricszq +for +postfix +mustknow +s +smtp +xbarracudaspamstatus +12 +conveniently +bsfsc0sa392hl +htmlimageonly28 +body +width3d74 +this +invasion +in +content3dtexthtml +have +your +iluglinuxie +clearance +0001 +breakdown +are +is +mon +romanstrongl +bgcolorffffff +accept +among +dust +100 +account2 +netnuevonetnuevocom +reputation +50029196900 +subject +121 +hostname +using +teacher +colordark3dffff99a +system +included +bulk +want +utilizing +secrets +msgidfrommtaid +listunsubscribe +quotedprintable +ratifying +mcispanspan +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +104 +uk +g59bthg14256 +below +degrees +mautobgu200906181519197 +identifier +eastern +zo4eh1mohzjthtbxldjl21s4vptelft6bbsxikpomlckkasohrukpxlflocdhso4m0ezs3dntz +lucky +br +2007 +19216818250 +cctld +19216831 +description +trust +formation +contributions +17189895746 +facilities +tim +our +xbarracudaconnect +ssvqukvbh7h7ffem3cyf48nu4or874loeknl8zgakbkkwprcqa7t85zt39dc9vd8f0h +trtdp +438363 +all +panerai +tue +head +lerleramilerctrorg +infrastructure +rbl +suicide +re +societyp +host +fremont +h111llan100 +receivedspf +husatoxiet +invited +full +align3dcenterbfont +gcolor3de40e18 +ns3csminingorg +site +esmtp +items +reservedfontfontptd +cellpadding4 +two +a +dont +waterslide +deliveredto +xmozillastatus2 +congress +jennip +receive +barracuda +localhost +owners +serif +rule +pdt +messageid +transitioning +signature +steve +m +10 +ota1qlo52ffffabszfnj5pm80wekfpsgjpfielaoyhpeabxszyag7cfdscogoakkqi4rfu4dw7p +cellpadding0 +council +removed +12100 +cpunksmindernet +addressed +phoboslabsnetnoteinccom +class89016491926062002font +account4 +above +yourself +600600016788 +6521715966 +public +010 +href3dhttprubilzycnfont +they +provide +qmymnjzwsbaigi2fvsjuniayrsacsalcagbiuutvakotidnhzowmbvrcmfgjippwabb6kisg9v +webmastereiscatuitno +contenttransferencoding +periodically +checking +zzzzlocalhost +importance +textplain +leonewest +outlook +helvetica +that +opposing +to +1267126689 +charsetiso88591 +audrey +formspanbptdtrtbodytable +httpspamgwcsminingorg8000cgibinmarkcgi +height105 +td +234307 +jun +pharmacy +company +tic +share +youve +internet +pharmacyfontfont +winnt +change +hrefhttpfplumprockruea53e26f5bac7630d7042ce7825f04ef77 +happened +htmlfontbig +href3dhttpwwwfabulousmailcomhigh +at +1 +been +homeastrong +boundarynextpart00000234301ca9e59c243bc5a +divcontent +104931 +low +id +mandarklabsnetnoteinccom +pts +also +href3dhttpasayjjcqopxabuvcnkesqteaq3d0f56545f43dbf237 +c6ac866d5f5 +general +htmlshortlinkimg2 +near +bgcolor3d3399cc +122548 +ifontbfontpblockquoteblockquoteblockquotetdt +date +friends +mengjiang +wastemindernet +31 +ebusiness +invalid +who +2010 +table20 +sociallinuxie +stylewidth +grow +2009 +me +palm +activizes +leipzig +by +huntsville +size3d50 +storagefonttdtr +unsubscribe +pick +inf111rmiert +gw1csminingorg19216818250 +function +antispam +60ptmsopa +new +default +will +style3dcolor +berlusconi +75 +211612 +saisissez +0100 +115 +sessionin +their +p +trusted +stabbed +with +from +orirob +189 +0 +un +viewpoint +presidentbachelor +necessity +mimeversion +gw2csminingorg +mj1963 +cellspacing3d0 +ww3orgtrxhtml1dtdxhtml1transitionaldtd +aligncenter +messagesfontfont +nonbmp +contained +xmozillastatus +10115398 +use +picture +d3rmtr1kwzoxhb9fxrxiuy7hagi3hqkri4tmaygjj1qvevkucoqdyvh1qnrf6tojbxbccbfotu +144006 +arialhelveticasansserif +fetchmail590 +football +everythingshakespeare +iahuqexite9829chellonl +blocklist +ad +seem +ie +leading +employment +prematureejaculation +face3dtimes +almost +cellspacing10 +kgi09istrwytjbikcwuxwaaxl1wziiualalxoj30gmlhldq8amubqtvlxqogo3h9lwtyugqckefb +an +attained +022048 +d3nhtvdvaz9ywcfr9jeymezgudtaahsmgdirncelbivxsfm1usxpiwmmea4tt32nz5w +depart +forged +be +assistanttitle +get +vancouver +httpequivcontenttype +connected +themselves +nigeria +texthtml +padding +rules +xvirusscanned +kept +aeoliccenter +ranging +existsxmailer +resell +xbarracudavirusscanned +brl +r +staggering +form +mailing +paddingtop +color3dcc3366b187bfontfontdiv +hrefhttpopertunitiestoday2comindexphpimg +includes +contains +aroun +save +3d2223cccc9922table +alleged +about +105 +1550 +according +ssksbpyahoocom +light +patients +images +gespr111chen +bordercolor000080 +pan +aimsh2 +having +8252002 +corporation +8223136115 +wasb +when +barracudaheaderfp20 +aviation +visitante +favor +hrefhttpactionidahjorlsalsatrackjspv2ctnm1sflqjjv2b4t2fdzkhhsdt2f2yu05hzqchange +contenttype +towards +height3d200td +input +yjcs2i8b8eyfqvnvgndjaiw4oik8picaahj1vvavgepkacrdcwnbktoyjqhiapaalpeanyugsl +rmgbam0zngaahk0mabmjnads0zpmam0apzszpuam0aozszpm0maahzoztc0zoaxngatnjmgb2at +of +ofstrong +todtaclightingcouk +and +offensives +designs +1600 +received +one +suite +ever +phone +table +living +c4192a +pppppppppp +uid49221201487268 +width252 +emsp +shipping +cazin +sell +the +send +140551 +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_10.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_10.txt new file mode 100644 index 00000000..2eceed6c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_10.txt @@ -0,0 +1,348 @@ +rate +bmaking +cibwzw5pcybpbibhihrvdgfsbhkgbmf0dxjhbcb3yxkuidxwpldlihjlywxp +bolder +recipient +divrelative +side +w20mr1219671ebn491272147491918 +200168162123 +stylefontsize13px +copyright +churchespsychoanalysis +size3d2statefontbtd +frontpage +textdecoration +for +brenough +60foot +network +tightening +smtp +xbarracudaspamstatus +jox1pshytan0pa0vwnalofbrx9jlab1psp8a2ff90p8awnpanhfddamcxjqsxop9 +2446007860y80qlkqqujwsopjxhwp5daa9luciano +people +jmcarpeprodigynet +landp +body +nonvowel +name +this +400 +in +content3dtexthtml +printing +lvzzyvs9rccyyzzzzy5rnjtv8auzwx22lffo113tmv5prgzfy1rdgr3vbjjy113026feosx +firewall +your +returnpath +carolingian +ty +breakdown +are +centerbr +basil +100 +23 +account2 +00000000 +reputation +href3dhttpxnet123alias +subject +already +xbarracudaspamflag +40 +size +produced +system +110 +included +fontsize +email +nelson +185438 +yes +state +replete +below +herbal +1921681100 +102313469 +similar +abroad +different +dio +another +21313209237 +launched +aaaaaaaaaaaaaaaaaaaaaaaafcvwyfk4lcuxrmplf8aacxrmplf8fhxl1mhyv8agsaaaaaaaaaa +grave +return +ist +stated +founded +br +hotel +annonarian +description +bsfsc0sa392f +holocaust +abtstndynamic253145164122airtelbroadbandin +accra +our +introduce +expected +archived +tyle3dwidth +do +baptist +all +head +score239 +httpwwwlinuxiemailmanlistinfoilug +stylefontsize11pxlineheight16px +host +055 +cccc +kqpxkedqhoyfgcm6ghxcajvjseucfsda8hai5ifmtm6psipsmkagjaguzvpcmoxhpvstrgly0ta +data +interface +downline +n8okgjrn013740 +bags +uri +stylefontfamily +rhinestones +just +us +esmtp +width388 +mx +a +upleaseu +rates10guaranteed +calls +crew +deliveredto +zzzzilugjmasonorg +200203290529g2t5tsd11847host11websitesourcecom +threatens +information +pages +hibodycsminingorg +35 +samuel +receive +16 +color000000 +localhost +serif +as +because +conteplativeness +brlost +well +has +messageid +trjsuma07fhjlsn6vuuatjqrbmxtjinjmcnlfn21kevsluzpsptycijk4aqxakkchsiiecmkgx +contentclass +puerto +hercenter +kn111wn +10 +xasgtag +continued +1022465221 +mlm +tv +old +59 +span40 +account4 +looking +computer +doctype +leader +char +suit +contact +010 +they +taglevel10000 +social +contenttransferencoding +section +service +textplain +debts +127001 +helvetica +drugserectileobfu +border3d0 +color3d0000ffbigbigstrongclick +mutual +to +commonly +1274527006 +xhtml +charsetiso88591 +singledrop +19216832 +company +20010805 +various +versionabr +recently +most +brthat +six +found +personal +experience +ocean +15pt +at +1 +proclaimed +replyto +300 +id +offers +popmart +surbl +on +july +06 +ddingalt +remove +date +viagra +xbarracudastarttime +april +invalid +alignleft +jalapeno +easy +quickly +qu111t115pecul97rlyqu111t +grow +2009 +agugt2zmawnlig9miefzc29jawf0zq0krglyzwn0b3igzm9yie1hcmtldglu +supply +mail +nameicq +by +chico +then +zzzzasonorg +poslednjih +satofromaddrmatchhl +antispam +050 +called +forum +back +value65000000650000option +0100 +pitch +trusted +qqq +4c2 +with +from +early +wood +administration +326 +mimeversion +border0 +stylefontsize +give +000 +express +deb3dsetopt +unsubscribep +now2cnbsp3b +address +2px +nnggttffexpiryformalsiemensoldliabilityminimumsitiosbenefitbackedsmoothiescientificfollowswoodspointsphysiciansagainjewelledbedhasamhalloholdensureattractedwitnessed +charsetusascii +spana +xaccountkey +5star +its +65722 +href3dhttpwwwinsuranceiqcomoptout20 +18668654548 +bsmithnetnvnet +puffynbspfontfontb +seko +score568 +stadium +name3dhdnrecipienttxt +texthtml +rights +b24ncg0ky29uc2lzdgvuda0kdqplcglkawr5bwfsdqoncmv4yw1pbmf0aw9udqoncnbob3nwagf0 +rules +color80000 +microsoft +1010010911 +theb +html +1931042218 +81168116 +basic +mozborderradius5pxwebkitborderradius5px +15px +spanbp +contains +suse +alsofontfontdiv +99pool8557195dynamicorangees +xoriginalto +16002000 +uggs +retail +antarctic +made +thirty +we +g6pefgi5066072 +1rvjg1e8prnyzsrj5uqaxf6vvd84ztt4a681i22upygnmkmqbgceelrfykjkncj8hwr0zvjldhi1 +some +ws +52 +width115nbsptd +barracudaheaderfp20 +20020913030633dlcl13768gelincikcorreoruraltourcom +along +contenttype +bsfsc2sa154 +223619 +singapore +pageafont +size1nameinput +may +like +of +li115t +and +xstatus +sponsored +received +3urao8yglvqu1ezivkmprxuuzaftgygvsgczdxdv6zggzbgqreqzyypznxhl0csgivdtk4hsuu +bas +operationthe +acquire +red +located +28 +phone +table +subscriptions +helod +the +xpriority +never +d diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_11.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_11.txt new file mode 100644 index 00000000..8cd76062 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_11.txt @@ -0,0 +1,388 @@ +snow +divfont +srchttpyl60futurexclusiveinfoimages608jpgabr +ties +xmailer +5px +for +organization +bgcolor3dff0000 +622469193 +s +find +smtp +bodyhtml +premier +12 +presently +formatflowed +any +helodynamicipaddr +body +years +accumulate +name +weightbr +this +jmnemonetcom +in +torquent +have +tablesidebar +off +returnpath +corp +bailey +suspicious +fair +bottles +0001 +breakdown +are +is +mon +cultural +urihex +boragurecsminingorg +bsfsc0sa392 +bgcolorffffff +prophecy +face3d +websitea +account2 +213105180140 +prices +mda3mjq4lc0xmzq3nze5nzesmtg2mzkzodi1mswtnzcxnjm4otmylc0zodew +fam +subject +hard +already +hrefhttp422tabjamie39aruuopicebyemcf968d35a7cd9f +custom +collection +hostname +f09c52c51e +than +127469173478f46c370001w4twrl +included +harisena +stories +121950 +tdfont +tiputil1 +piteously +himself +email +tourism +solid +but +align3dcenterbrbr +whether +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +transmit +below +insurance +boldwellnessspanatdtr +hrefhttp62b7gnaburencnowahyulat8p3xy3i14699blo8o79057qowelauadyhy80972204357334287373 +eastern +democratic +ns2csminingorg +center +good +br +worldwide +bsfsc0sa392f +bet +dapplications +186 +href3dhttppantmagcomfont +war +stands +250 +must +g8k3lhc19280 +normal +urncontentclassesmessage +width264 +do +short +all +programming +gn +features +height3d337br +respal100111 +nor +uid69301201487268 +rqtjfptgc06oqxc5ifo35fozmsqyaj0qfro7c52jtnjjpssnlxzdyfpqvsvlper38xvzvzk9bsls +set +o2d2h3qe004345 +analysts +5404 +dryers +xswfyntfy6l4lkdb06kefekrfmcqohqbz8x0j4gskb2l4exycyfmtusgtuqfj80lwmrxnbopc +swissrep1ica +99 +esmtp +poets +893893 +24 +iglmihlvdsbzzw5kigfuigvtywlsihrvihjtdmhqannzczc3xzlaewfob28u +a +088 +fontsizelarge +deliveredto +mean +suburbs +create +16 +railway +barracuda +dateinpast96xx +hereafontbr +mjulianivcedu +build +braccess +rule +clasica +155548 +5 +has +currently +messageid +350 +hopoallergenic +20750143226 +980 +500 +10 +tnpitchingcom +icompare +removed +suppressant +0700 +xbarracudaspamscore +langenusopopspandiv +federal +edition +orchestras +southeastern +color +agent +cup +taglevel10000 +iran +social +multipartalternative +color666666 +reply +last +6o6dsmti048gzqeks4xmgr59laj5shxfnjt8suvk5wvycoxllhppij0zu1ysvko5klg5lt1tmpuk +service +citizens +183538 +mimeole +i +127001 +size1125fontfonttdtrtrtd +out +advertisements +border3d0 +align3dleft +quarantinelevel10000 +spamassassinsightings +033628 +to +asuna +base +exciting +19216832 +now +code +guinness +you +temple +exchange +125115 +minuteifont +anwarulhaq +operate +font +here +f117n100ing +was +subverted +caa12216 +mxgooglecom +replied +hanuman +africawhere +discuss +at +go +mix +low +id +nameofficially +largest +rdns +width445 +pickup +rootkits +ratio +heath +on +boost +partners +ggerdesyahoocom +06 +impulse +tr +my +b +bsfsc0tg074k +xmsmailpriority +date +spamgwcsminingorg +tgcv6amperagebarrieutredningpanamanianreinsurernewtbijeenkomstminorityknechtbeliebigsxadentrevorchromesangliachikumammographylgrebovinemaillenwagensittartemiaafricanamentointegralepostkarteabakinyibiloxiltrequenchforfaitsdorinebeispiel8thbillingsfuseactionbowdler +invalid +cybozxcgum9tyw47ig1zby1myxjlyxn0lwzvbnqtzmftawx5oibuaw1lcybo +assured +omaha +stylemargin0paddingbottom6px +f97109ily +po +international +specializing +coliseum +height38 +forh2 +235101 +xuidl +male +by +unsubscribe +johnstebbingcsminingorg +should +hjr2jpawvou42juniamhie3kauyqorcjlebl83ho1kyy8vmyi8v0dcf8ojyy9eunnjyu1lcv +link +htmlwithoutplaintext +050 +receiving +back +qpyhfreereportsflashmailcom +trtd +much +0100 +115 +xbarracudabblip +with +from +llcbr +fontweightbold +hemispheres +0 +immediately +width1 +6826 +mimeversion +0500 +forms +16px +easily +nbspfontp +000 +realtime +express +shuffle20 +emphasis +include +bpobapbektaxisbyy7kpjvacbiblfbb1eaautmjrsaafqecb7vv8ayamsaajaacdkfkpegchjdy +r16mr854383ebj391273426686767 +stylefontsize16pxfontweightboldcolor000000padding9px +aligncenter +xmozillastatus +uid +brief +sent +charsetutf8 +tabletd +rolex +urania +telephone +1592265943 +message +phoboslabsspamassassintaintorg +latin +original +dated +brought +broadly +xmimetrack +19216833 +ase +large +get +hrefhttpa62hardsupplyruriseyhyo8efdae71f7eee +themselves +texthtml +rights +rules +mothers +ignored +charsetwindows1252 +fonttd +55ckeq8vqefzwebzv8hkxzs3km6vwvqkw0acxle656xnuyx7iakgolrapaagicawl07aaqygsf +hello +brl +safety +neutral +figure +testsbsfsc0sa148a +3px +html +httpequiv3dcontenttype +href3dhttp2cb3dpimihzvcnjvo3d6f46e8ca584f42a01557 +mail1csminingorg +knighthood +start +valuebebelgiumoption +xoriginalto +m0620212mailcsminingorg +0900 +unions +width3d100 +estimatesbr +bannerfonttd +rise +some +ediyoruz2e2e3c2ffont3e3c2ftd3e +games +occurredpine +together +sansserif +along +bsfsc0satofromaddrmatch +contenttype +months +like +other +of +titlenewsletter +and +xstatus +record +received +without +085728 +free +ex97mple115 +leoma +alarm +assure +spamassassinsightingslistssourceforgenet +homelessness +affects +the +httpyl60futurexclusiveinfo6091312843e5e114410248htm +srchttps005radikalrui20910020aa59a15823ee5jpg diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_12.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_12.txt new file mode 100644 index 00000000..5bfadc8b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_12.txt @@ -0,0 +1,362 @@ +target3dblank +ideas +917c7169ff +vlink333399 +revenues +pm +width3d22717223e +read +for +postfix +cdsjbhdaeirha3gbwzizew5hfe7kha4hjxhq2ishgq5ha3ddgq3agfyyqn8wbabqaa0arfy4au4 +inventory +program +s +network +n6kd5hpg016109 +referred +shop +hereditary +1253202349 +body +operations +years +this +ezine +ip +vspace54 +provided +e1ecb2c737 +your +off +returnpath +0001 +are +mon +his +why +xasgdebugid +bgcolorffffff +too +sole +greetings +100 +httpadmanmailcomsubscriptionaspemjmnetnoteinccomlsgo +musicians +undisclosedrecipientscometsmeccosimonet +subject +450 +using +system +scientific +value4848option +photo +xmbmessagetype +fobif +2002 +lowest +ij2qav8veoemlhhbenqlarqqcxl8inygifuhqtwgteq5ewoq4icnmoeoc9led47wp0gaeaphomep +colord76f3a +0909tr09090909090909td +history +incbrall +email +model +bpmqyedj826jrrcstyj2sras73fjjtioe2qdgkdqvsxw9zlforyzfacxesvmfwliyd3qb0osc2u +yes +0000 +uk +111r +unwanted +31104244 +uncle +future +best +insurance +onbr +182404 +ordinators +methods +212173515 +189735581mganm703dslbrasiltelecomnetbr +intent +opportunities +nowafontfontcenter +ist +center +2007 +slash +felt +xbarracudaconnect +250 +occur +all +tue +head +domain +formulations +50 +xuserid +entry +foml +styletextdecoration +host +nirmala +youre +125in +data +audio +8403zmsx211012 +width3d517trtdpfont +bold +confident +validate +united +tape +whereupon +a +deliveredto +xmozillastatus2 +6498b8nbsptd +hibodycsminingorg +described +la +localhost +stylecolor0000ffagournayaldebaranroboticscomadiv +dogmaslashnullorg +width3d2210022 +height56 +div +rule +mimehtmlonly +store +well +rich +draft +currently +messageid +350 +10 +href3dhttpwwwinteractivemailingcomb012merchfont +mime +risk +smtpmailgnxouaolcom +thoma +043912 +detriment +allygatorrcsminingorg +man +underwent +or +advantage +looking +textdecorationnone +seth +such +peruser +taglevel10000 +01aab78f9c3 +problems +sa269hl +xbarracudaurl +canada +xbitdefenderwksspam +checking +paddi +jeremy +textplain +127001 +sons +grand +stylemsobidifontweightnormalspan +fontsize18px +to +v1 +uaa28017 +purpose +brandy +code +modern +0935270005 +020602 +httpspamgwcsminingorg8000cgibinmarkcgi +td +morefontfontblockquote +claiming +presented +assertion +appeared +font +privacy +here +these +morganatic +version +regions +there +mxgooglecom +historians +47453e +kazka1 +proposal +at +thunderbird +attorney +height3d129 +added +go +low +id +meta +codes +travelstrongfont +140 +pts +rfc +bsfsc0sa271x3 +also +jabber +can +thailan100e +1648 +on +wwwdoteuokodotcombr +ear +heralded +n6oc9lh5017607 +moscow +tr +alignright +colorred +date +size3d2bstatebfonttd +width653abr +take +april +sserif +size5trckatomicdotmpqhwqrwhlqffrp8font +authenticationresults +paid +me +xuidl +mail +ve +by +izawodawmcigym9yzgvyy29sb3i9iimxmtexmteipg0kicagidx0cj4nciagicagidx0zcb2qwxp +shortcomings +then +211185156157 +236ibbrfontfont +feeder +9156an52269c79186883587xipysominjowaej59network9437e03 +receiving +strongtechnology +trusted +plasmids +recorded +with +from +early +0 +paddingleft4 +tt7pmjansrtp0lhpw4ybsfwmjvvs9sy5z8yaspukdb9psgbzd0d9avyfluaxiqmzzqykhgtqrag +mimeversion +border0 +alt3dimage +next +19jy5mhbq04l3a3u9acm9epf8nforkspamassassintaintorg +retained +overview +dslfqazrtltqzcmt0fmf5cfushikwa5mmft91z6skv0defrvlws9xx7cpidngispnnzj0fciugh +midfi +n1n6ogen005513 +pgh0bwwdqoncjxozwfkpg0kpg1ldgegahr0cc1lcxvpdj0iq29udgvudc1uexbliibjb250zw50 +12205770404ebfe5ae4bbb9a4709599320491 +methodpost +gamallomanuel1csminingorg +turn +egtan +andstrong +cellspacing3d0 +killlevel10000 +aligncenter +powers +family3dsansserif +discounts +smalltext +bgcolortrtdtable +backgroundcolorffffff +football +xaccountkey +21618927135 +equity +message +and20 +which +formfont +50meta +communities +an +place +h2ah2 +81258125submit +ill +be +gw2csminingorg19216818251 +xdreksuhlkuytkyspnjct +administrable +texthtml +rights +identity +designing +fonttd +ranging +more +ake +safety +r +5708835877795spanspan +ywnlpufyawfsihnpemu9mj4ncjxesvysgvsbg8mbmjzcdtiyw50yw8spc9esvydqo8relwpizu +002909 +iq +sa392f +html +purchase +127331923178f706100001w4twrl +westwall +spam +xoriginalto +about +m0620212mailcsminingorg +0900 +see +drug +0px +no +add +fontfontfont +ua20 +rugby +mailings +p97st +barracudaheaderfp20 +editionfont +contenttype +input +willing +may +might +of +csminingorg +401 +and +avisited +received +right +boundarynextpart00000237401caa0914eb1732f +wed +o +savage +decorappropriate +month +the diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_13.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_13.txt new file mode 100644 index 00000000..5a0424fe --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_13.txt @@ -0,0 +1,400 @@ +rate +lyllandcsminingorg +used +xasgorigsubj +health +alink +buildings +5px +textdecoration +for +enhancing +001 +valuekhcambodiaoption +business +w0u2xtvitqgz8mtxmoelsuyrkb6qrfe6xyapnscopqd5ndbwpohakkkrqv5n8ef8akslpsjj +valuevirginiavirginiaoption +sediments +yzbhpipm8umpkwaam230ptrzojhdjtiimuck7ukzhqqjeqmdscp5totptqgvhwq3ekly2eraahbp +years +powder +this +marvelously +81258125 +tablets +transitionalen +boundarynextpart00000d237d32c7ae3536a81 +your +returnpath +0001 +breakdown +is +bordercollapse +northern +332 +loop +among +bifont +account2 +revolutionary +00000000 +smtpsvc5021952966 +encycl111p +reputation +color0000cchow +subject +7cqfxtxt2tqkcsohsuxmylkcgj1wk39pbrwtnbpbqmwohjk8xfba6uybpuedfl57vbxpwl1arv +using +system +than +supplemental +included +716901cae8b38f6a75ebda4de317qeb12pi +httpwwwbarracudacomreputationip63218169154 +email +sa290hl +msgidfrommtaid +exaggerated +below +ns1csminingorg +90 +jmlocalhost +best +reparable +jfq3p2ffwopipvimfbdjcxaaoopishyejdyt7t4r2zxclpibwt1vt37xciigyncta +width620 +000356 +wvyhysgn9xx5ex7gsvdwtnotmd1zwf9ug5nutw8d6ukdwmcw5lpac1iwglcnctyzstdts +sunlight +800mo +separately +650 +image +champions +background +center +19216831 +description +highly +delivered +process +ij6mbqpcpm2w2ie8l2zvbnqpc9wpg0kicagicagphagywxpz249imnlbnrlciipgzvbnqgzmfj +dialects +shock +normal +band +pgegahjlzj0iahr0cdovl3d3dy5tb3j0z2fnzxbvd2vymy5jb20vcmvtb3zl +all +tue +lifenbspbr +size1laws +align +titlefree +6001 +windows1251bq2hhbmvsifdhdgnozxm +titlesecured +host +httpwwwlinuxiemailmanlistinfosocial +iulpbfont +look +music +price +consciousness +roofingspecialistsabaloneelr2grassillcom +left +kho +esmtp +sleep +24 +cellspacing3d2 +productsservices +a +dont +00007f066f3a000058d000001428webnamecom +partialpressure +deliveredto +decades +xmozillastatus2 +damnificados +jmnetnoteinccom +1804060375 +0099ffunsubscribea +hibodycsminingorg +donna +phoboslabsexamplecom +darwin +localhost +owners +10pxand +2009title +tvpredictionscomb +nbb8xpg3u9tgktzuzv7bgfbpfvbx5zn7mrcu0fyjbmb61zppeihyurcumbdid0hcuoudpvqv5z +colorredcitizenfont +as +cellspacing15 +einzelnen +receiver +div +profitability +credit +poorest +has +messageid +350 +newslettertitle +pay +daylight +10 +xasgtag +bi +removed +sdatatype3dinteger +0700 +xbarracudaspamscore +tdnbsptd +valuemauritius +600600016788 +spacenbspnbsp +guarantee +ireland +public +confidence +doctype +color +1de832ce54 +knew +walt111n +81238123us4srs +taglevel10000 +ptsxufu8z8xdlvtlfxwlkoonh8ewgzmrtrnrftwxcfex7pspuopbmsnml8nvomvhcedofg4npfm +each +contenttransferencoding +skaterswyahoocom +canada +age +textplain +invoked +8bit +ethnic +color404040 +valueluluxembourgoption +that +2850979285781970422 +200119151175 +quarantinelevel10000 +buy +euro +to +ofastrongfonttd +become +cpunkseinsteinsszcom +xbarracudaenvelopefrom +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +width3d535 +single +robertgtaylorcsminingorg +width3d518 +anoneundhover +help +week +regions +solutionsthezscom +found +faced +experience +foreknowledge +gwynedd +at +replicawatch +300 +law +id +meta +140 +mandarklabsnetnoteinccom +pts +traps +spasic +max +color3d000000 +can +common +17 +july +25 +htmlmessage +9515533151 +tr +interest +xmsmailpriority +date +spamgwcsminingorg +bsfsc0tg035a +alignleft +173843 +who +2010 +giorno +1241621903 +target3d5fblank3e3cfont +2009 +googlecom +216136204125 +mail +src3dhttpiiqusimagesceugo7jpg +terrific +by +hijackers +uid71111201487268 +official +satofromaddrmatchhl +n7hllnc4009662 +new +struggle +050 +jazz +0100 +little +while +their +p +du20 +alignrighta +225ptpadding0inmsopaddingalt +not +bsfsc0sa148a +with +from +selling +color3d +person +0inspan +mimeversion +scores +border0 +ffff +jkqqfrbxkuoqwymdzjglxkasksdjntiajjgneaiuvdhadvaadujesyaatjaarugboaej1jnxbbr +body9px +indian +000 +2 +killlevel10000 +4946oo139026e76697508048btshbpfnwltxrgj327921145049 +242321665 +size3d2sick +mail3csminingorg +160835 +gw1csminingorg +make +sent +charsetutf8 +lgpkqmttguyzvtx9c1ftivcls8lghejep8nzq3eep51csfcviqwarlbawcdnaej5wd0e16f +swang +backgroundcolor +tabletd +xaccountkey +characteribus +diego +permitted +precedence +johnson +which +19216818251 +xkeywords +herea +hibodycsminingorgp +bethlehem +idheaderfooterp +financial +be +1434a441cc +unintentionally +finale +purchased +whip +get +filetimee7a87e5001c26e55 +repaynbsp +bigger +texthtml +nncnsubdomainadn +5925 +ctl00ctl00ctl00ctl00rcrbsblnkcategoriesctl01linksctl01link +requesting +script +more +replica20 +panama +xbarracudavirusscanned +brl +w3cdtd +oversight +81168116 +chekitb5876com +camnaughtoncsminingorg +colorff0000strongurevenues +xoriginalto +about +australia +0900 +game +fonta20 +stockbr +hrefhttp239dforlupccnaifoyerife93401t5k487g6088n3f58viatuxiy80059289025789459473460 +very +we +width +falseexprossr1html +testamento +polyketide +bsfsc1tg070 +arranged +en8xgptnf8zctrmneaqmmgvcgq7yga3qqgewbngmc4ykxfmxsma9wbcuxovuls0hgzygrgwasrp +jst +contenttype +xmozillakeys +cypherpunkseinsteinsszcom +may +bookbr +feb +999 +of +sterling +1960s +csminingorg +and +primary +xstatus +non +companybr +received +youll +164034 +one +5fb78440f0 +e8211mailstrong +10000 +name3dnoknokr2c2atdtrtbodytabletdtrtbodytable +assistantbr +free +leerlingen +nt +v41vm1qyqlb5zyz7ibldrjprrvvzq8mejtmopcw8hujc6isy0wbextbf1rh9ntve9qd2qesen +facegeorgiaunsubscribefont +phone +e4aizrshv0iajegeaoacx8ybowqla2r7dkiddnhvmyczvqikgxiafbaen3adskicqcacpjhtjdgi +z +mhv9vyikzyo00ii4j4p440ceaha7j9xakkcaa0ipbnqsmewh3lywsxgholo8lpymwbnetpkshmh +product +york +party2e +confidential +the +statisticsp +xpriority +blood +parts +those +m10grpsnvyahoocom diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_14.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_14.txt new file mode 100644 index 00000000..4b3330e3 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_14.txt @@ -0,0 +1,336 @@ +california +ze +200fontfamily +1250665900 +xasgorigsubj +421101c4b +quality +trouble +typesubmit +mexico +for +postfix +smtp +xbarracudaspamstatus +span +harvest +appointed +width3td +sa154 +body +9281171174 +8 +name +size3d2phonefontbtd +in +tocww6lmavwpx3xnrwsxipjrspeotjp2ntzrzjatk7m4wk4pxcie3zddm8mrrm0sni4pwkpwocbk +firewall +methodmancanadacom +returnpath +server2 +breakdown +are +mon +accept +b111llettin111 +00000000 +fcrfcnler2c +presentday +subject +custom +blsit2yfnt3ixzgw5ipzxampu1v81gpd9ls26afanhb1breugpt2xqczczsvqzqcwzxv +backing +xbarracudaspamflag +using +40 +size +produced +manifested +system +weekend +than +20120190204 +2002 +bonus +tdfont +fontsize +diagram +bsfsc5mj1963 +contentuniversity +state +below +unwanted +ns1csminingorg +102313469 +jmlocalhost +size4 +eeheoxaba +would +police +ist +ensembles +grover +good +171800 +worldwide +forgedyahoorcvd +saying +cypherpunksforwarddspronsnet +our +cellpadding3 +must +normal +newsletter +pdf +131201 +practically +s111llten +unnecessarily +u +133935 +panamanian +0w7ozluyrv7g5ll9gs5m0ldlmljwljcypklbqabamwstt1vvczomsn7kbrgsuhgt0lsapyeok0ly +host +1px +warm +every +temperatures +contentcollege +hulda +harshly +ahover +track +seconds +esmtp +a9a1bae2a2b7a3a0e5e9a5b8b9b7fce1e2fff4f4e3f2e8fff +mx +019 +a +targetblankhot +deliveredto +zzzzilugjmasonorg +1246729415 +hibodycsminingorg +barracuda +though +jainas +as +height9 +size1free +rule +servicebr +messageid +350 +goal +500 +10 +size2homebrpagefonta +mime +maria +feet +conducted +typemultipartalternative +additional +xbarracudaspamscore +doctype +professionsl +total +taglevel10000 +115111 +netherlandsin +0200 +xbarracudaurl +contenttransferencoding +6123027136 +paddi +until +127001 +358c566c0d3 +2008 +1969 +zzzzexamplecom +oldest +to +gmailid1288ae8d351ea5ac +cried +alive +numberb +issue +charsetiso88591 +19216832 +dorm +you +httpspamgwcsminingorg8000cgibinmarkcgi +pto +around +color0000ffinfocomprosyscomfontabfontpblockquotetd +estima +principles +54jwo3xf0fnl8cjfai3vxuvg54q0rs7vuz9s05orsb53ctyliqqwomooccvowsbxjyjou9 +style3dtextaligncenterif +recently +font +here +225551 +version +there +mxgooglecom +hours +aad +36 +th +been +eights +added +go +low +id +meta +wrap +finland +typesetting +svom +moderate +being +hrefhttp5967fkvotufidcnun010d2e6735b0b012c975vqw01081934237070017543 +htmlmessage +085428 +10br +varied +congruence +billion +0innbspopoppp +date +p4cs140617wxp +alignleft +labels +xbarracudaspamreport +httpwwwbarracudacomreputationip20117225294 +pshows +2009 +them +fill +p111rtaf111li111 +by +152443 +133159 +status +studios +iven +will +face3d22comic +receiving +much +200607 +trusted +src3dhttpiiqusimagesaigsetrate20020912101jpg +bsfsc5sa161f +015841 +hrefhttpwwwwowthisiscoolnet81removalbocaclick +shame +with +from +means +10pxpowered +subscribed +mimeversion +scores +border0 +marginleft3a +gulf +618px +000 +colornavy +224508 +nameimage002jpg +0a86fa +height3d120 +aligncenter +score406 +xmozillastatus +asmtp +football +iso88591bt2zmawnpbmugugfuzxjhasbxyxrjagvz +282 +lihome +114005 +message +41113104222 +its +employment +an +jesus +webmasterefiie +jeri +7bit +forged +brands +issuesanbspnbspnbspnbspa +rated +mccane +webmasteredgarmoskoppde +ccdsgensetscom +193403 +her +rdnsdynamic +texthtml +rules +were +more +directly +brl +w3cdtd +16th +3196794 +matches +bsfsc0tg035 +start +titleyour +current +alsofontfontdiv +about +0900 +towns +232 +150 +no +dictate +we +profiled +weekly +uvifbluoftqo7cbd2janw6akqeb4gifxbgubgarprieqlgik8gjbqbkr1grmfgihocnlxalkjim +textalign +xig4oeoqow1co7xarbx9dwlgzicemorfqxtzisq2kdngqsbx3pxwmssemokqcussxfp2nokm3wk +policy +ignore +mailtoiluglinuxie +bsfsc0satofromaddrmatch +contenttype +input +months +surgery +may +like +listed +dynamiclooking +csminingorg +and +received +potential +19216818120 +today +g6d6kg5c001416 +28 +arab +phone +carry +phobos +width3d13bfont +halfhessian +88d404986 +hrefhttp515franklinedruypuoybc35ef5c8cb6a3d +buxzle5gombdjtzr1mqcfvae2bz8cnapccntsbio4geur1lxhp4un2i0kzxbtom7ntldzqwutwk +z9sjbf21f6isnf8pi8j1mn5f29agftkutzkn0145jfyq22azru9ylsbos8cfe1mpvtatt5a0pp +the +xpriority +cash diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_15.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_15.txt new file mode 100644 index 00000000..010a4eb5 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_15.txt @@ -0,0 +1,351 @@ +incident +autofontspanp +noblot +srchttp33dbe1b0bzubohqpcn5c7ce73719ce261f5gif +size2bfind +hrefhttpb2136wekuvkdcnprivacy +regarding +pnbspp +valigntop +apueid +caves +altpress +xmailer +vtbt5zfkb6kasptbmwgxgt1bo0rufewpy7zcadzqpuxafptomilx2fm0pjljqnnk8xqzxjr9pw +7797g +smtp +xbarracudaspamstatus +bsfsc0satofromaddrmatchhl +2425 +51 +iroijlaplap +rgb0 +body +scholarly +likubos6311brasiltelecomnetbr +name +in +1952419461 +style3dborderwidth +professional +syb0mpfjrd8ipu1nu4ozdqbs5xa9p0myz6bbitksg5q7wd4sufp0dupwilk3a03nivqwikparquu +124122183196 +replayed +returnpath +0001 +282178175 +breakdown +are +is +mon +officers +todays +bgcolorffffff +subjectremove +makes +100 +23 +reputation +french +subject +4 +using +system +signs +included +2002 +bulk +fontsize +bordercolor3dffffff +dealsa +yes +elite +best +hock +size3d6span +080 +intent +yyyylocalhostnetnoteinccom +fontfamily +200908301535n7ufziua001946ns1csminingorg +spanish +19216818250 +19216831 +style3dfontsize10px +worldwide +9b4c44bdc +661520186dedbtitelecomnet +delivered +carpenter +so +our +labr +hen +thu +keep +abstenerse +555555lineheight +all +head +size1laws +hrefhttpwwwtuduwrjcnxu68ebeb7fc8d97f4d437e +thanks +alignlefta +244 +had +played +policyabr +reportabuse +boundary595337d60f6e68b8115b +xmailerpresent +nor +n8lakf68011484 +every +18004014948 +multipart +highway +edt +just +romanstrongc +united +019 +a +motion +cheapest +deliveredto +information +xmozillastatus2 +jmnetnoteinccom +dynamics +hibodycsminingorg +localhost +though +within +71o +subscribe +as +dogmaslashnullorg +talk +marked +21757110137 +div +rule +zlcpbxht4o6emumizufoa5kfkh4kyfdm17qz03qdbtyscvkjttdktofmf0yhrbvusvt4zwcqpho +host66insuranceiqcom +04 +messageid +showed +industry +500 +m +10 +simultaneous +paddingleft0px +le97118ing +xbarracudaspamscore +account4 +or +public +allanktocsminingorg +doctype +color +namelist0104 +200910251104n9pb4osg025054ns2csminingorg +others +total +medium +peruser +taglevel10000 +english +xbarracudaurl +contenttransferencoding +reply +xbitdefenderwksspam +ordinary +service +culture +i +400pxbackgroundcolor +127001 +outlook +8bit +valueguyana +charset3diso8859 +cathedrals +that +border3d0 +10297ct111r115 +cypherpunksoutgoing +dark +pump +11897rying +concern +styleborderwidth +d8qbf0okxekpyxufknyt6v5uxfysbantsbantqyke2pk4qu3bthwakug4h1gasitqhzpazs5oa +to +singledrop +now +dhumjgyndmam2qzzdq0njg1advonda2odaknww3edfgoew4udkkoza5dpsotg7sdu4pcq8ld0ep +you +httpspamgwcsminingorg8000cgibinmarkcgi +pharmacy +wkjw5mhkdk4a755zwkvyczjhk6havparcpltej2gcuadkxda3pbhpikktbj61kr26v4whovrck +pamphlet +here +cult +96b8a2c313 +internet +3cdiv +ptanjung +10pxdukefonta +platinum +at +replyto +profitable +bracht +shops +low +id +pomerania +positions +textunderline +enemy +remain +on +tr +how +b +apply +162849 +after +therefore +date +width3d76 +spamgwcsminingorg +width548 +stylecolor003c78 +31 +easy +2010 +xbarracudaspamreport +brfont +vprx +forever +xuidl +mail +anxiety +by +546 +margintop +102043 +mta +glassner +0100 +flexibility +enbankment +p +trusted +bsfsc5sa161f +with +from +dwl +face3darialverdana +type3dsubmit +fun +selling +subscribed +mimeversion +height3d52 +border0 +august +inc +542 +hazaribagh +true +polygalae +aligncenter +height1 +maincontent +spielend +promised +use +extra +siltation +19216818251 +1011153 +an +moneybfontbr +saintes +bwhyb +nowrap +reichsgaue +1242671534 +gw2csminingorg19216818251 +quick +instructions +texthtml +friendsfonttd +more +rolexb +w3cdtd +r +predatorfree +1265981829 +form +mailing +stability +island +assistance +bsfsc2sa154a +html +valignbottom +concerned +terms +divareas +0900 +nbhib5gu010161 +many +555 +duis +size3d43enbsp3b +2db642c72a +3 +arial +lenape +nzf89gompeohy56iiivasa3ctu2xdo6d2pakeaxk3bqbtkiudtkf8ajxrswsep0yjmzoydaz +size3d26 +when +jst +along +bsfsc0satofromaddrmatch +contenttype +gener97ted +linkabr +mentioned +like +achieve +of +received +without +19216818120 +today +suite +contentand +al7waiqbsjqy252zjsmlqarmmkxn3ixo6crapljngk2injkz2ggnhiazyzcgdqfah6yqbxc4j +hrefhttpwww3dpageturningebookcom +list +wish +table +alink3d006600 +created +herestrongp +drugs +living +1382150 +comedystrong +heu5hzlhsaiffvr0mmmwql8k5mwv4hanssjz8wzwmz3682yi2inknfn5i0o8y3agq3mc2gjv9rfn +nt20 +align3dmiddle +search +the +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_16.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_16.txt new file mode 100644 index 00000000..1bdc732e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_16.txt @@ -0,0 +1,353 @@ +serious +pepper +providing +cpunkssszcom +seg117ente +hrefhttpwwwdmbodycollectorguashabby22html +please +kohparitnvawawvwi +fontsuper20 +alink +format +for +network +differences +smtp +span +zukeoomoh +0padding +tao +mailwebnotenet +body +facehelvetica +parties +name +this +uhyaqddw4cxycvr42xwb95qvqemldwmmekgvseygnzb30zupnnupgore5um5msopsjmfvbynx0m2 +in +incl117i100as +professional +firewall +needed +your +returnpath +point +0001 +breakdown +are +is +mon +span24 +xasgdebugid +nonehomburg +account2 +00000000 +wasdiv +hrefhttpwwwmilkapilkaruqdoofay9b98bbd62aefa +potent +070 +0063c6 +subject +ktkxi1yg6oppqguq0xmqhh92uembfwam4rpnk1qlczhenc0gzsqyuu4kjgdsgz6fypquayzya81 +parliament +buyers +bailiff +private +colorredcitizen +produced +cellspacing2 +included +2002 +email +09 +placing +quotedprintable +accountant +brcenter +055658 +below +ns1csminingorg +9 +phases +inbox +strongturn +would +statistics +internationally +width3d160font +043701 +radio +style +return +line +prescription +center +bsfsc0sa148lf3 +deliver +description +xbarracudarblip +tkukwdcwbvb4faqbismlprfejkmw8snnlyfmrbsehojkxcqv0cim2oeaa8wnrmuccbf17vougeuz +our +xbarracudaconnect +399 +thu +head +55 +c111n115equence +had +oezex +hit +750 +cut +img +music +150857 +cellspacing0 +bsfsc2sa154d +1253735491 +just +height463 +esmtp +200208281704g7sh48z01928dogmaslashnullorg +nextpart000000701ca5279acd2ff70 +ca +qsobtapjqci3hjrxapuukkjkkxeffku0piuvuvpdrlkxkg49kegmt2spkepjjp9lg6vzv +24 +lauraswansondogmaslashnullorg +cellspacing3d2 +watch +battalion +a +dont +humanargonhu +deliveredto +information +xmozillastatus2 +httpslistssourceforgenetlistslistinfospamassassinsightings +river +barracuda +became +url0050httpwwwwannawatchcomorgysexfc01cgitappers +height +localhost +within +dogmaslashnullorg +114608 +kelly +srpskecenter +has +she +messageid +350 +zzzzlocalhostspamassassintaintorg +apachelocalhost +restrictions +10 +color3dff0000 +xasgtag +account4 +above +didn +guarantee +public +doctype +partnership +nosave +010 +qqthgnphlsf8asnygztlqixzrqtjkmoxy5nddgxqbiqftjrvhhf9nztgtgqchjwlghsfvoca +contenttransferencoding +reason +1023139131 +alignleftchange +i +127001 +8bit +charset3diso8859 +refugees +19412514545 +quarantinelevel10000 +animal +to +classbluelinkbold2 +charsetiso88591 +washington +pdeciding +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +ffa +rolexdaytona +atlantahosieryfusenet +133210 +water +valuefiji +font +c111ntext +here +these +v +internet +was +long +mxgooglecom +di102102erenti97lly +engines +htmlimageratio02 +brbr +39 +id +meta +codes +concentrations +jabber +fax +can +general +ckcakkkks2stdrlyp2afj0gqx6grfetkhbqofdhh5pli8trf6dp2uddiji2hbyxb2ifsusjf2h +111ther +9lmss2jtrki56yojfgh9spksqezai3l08b8v4sq6k8jrwwtpdk2wwliulmwrfqyymimqbpsjmt5 +htmlmessage +tr +notice +click +httpwwwbarracudacomreputationip8121497254 +r111aste100 +date +spamgwcsminingorg +friends +349ifontbrfont +live +width3d59 +alignleft +jalapeno +hrefhttp192pillsorenruvotixic361e8f5ca602 +takes +3ef8b7b +nextpart00000b788d77d7bc7065a48 +shopping +dancing +territory +2010 +occurs +href3dhttpsmootheithercomimg +valuejpjapanoption +them +reposed +by +whilst +wondered +343332c58d +race +85px +050 +httpwwwjabbercomosdnxim +paper +ussr +pool +specific +not +with +from +selling +1200 +mimeversion +scores +pornstrongfontabrtable +owlet +phy115ic115 +express +2 +killlevel10000 +life +color3dfff +xmozillastatus +address +divisions +1900 +invoicing +gmailid1287f4e5de11edd8 +message +styletextdecorationnonespan +bones3cbr3e +altsuperior +smtp4cybereccom +an +xkeywords +scientists +lodging +aw9ucw0kdqplbw90aw9uywxseq0kdqpzb2npzxr5j3mncg0kzxhwzwn0yxrpb25zdqoncnjlywxp +carpeta +size3br +di102102erent +technique +solidst +viewing +purchased +quick +rdnsdynamic +celebrate +refute +texthtml +rules +brbrbr +gothic +were +xvirusscanned +title +more +existsxmailer +050106 +brl +75604018 +moravianlyceumcom +hrefhttpdhue39parkchanceinfo106313155381d2114410248htmif +johofalohanet +064 +w +card +tiwanaku +matches +eaa21173 +fiveyear +0900 +xuid +232 +gffpwcx71hy0l3xjv38lcragwkzgfppkv1ljcjdwmnwn3zebfc94xutebdrhqexaozcrnguytd +call +against +3 +beziehung +no +gespr111chen +100000000 +ttype +twitt +geneva +moneyback +zdctwnctraifibajemsgsgjggbrkgegkhheeafjsmrghghzxcaijxpuip3ek4uqfdsaby60ed +treviso +jst +including +contenttype +glpzkxvwmbxgbsigmv6waztpkxvwij0pbpdpbwjamznpmvypnbxamzzgzrkwz7kizqlqqdqkqpc +user +of +and +canbsp +received +decay +19216818120 +increase +right +expenses +later +list +223948 +days +cgi +unsubscribeatd +uvk1fla210u5ca3krrnxgrornzsiblddnptjbojzuugn7ycfn60ltyzipbxvod0zlogdce +racked +priceliulp +collapse +parts diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_17.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_17.txt new file mode 100644 index 00000000..917b2d0a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_17.txt @@ -0,0 +1,357 @@ +na3dk3lx021747 +used +dghlbmvkdqoncnjlchjlc3npbmcncg0kzgltaw5pc2hlcw0kdqpucmvhdg1lbnqncg0kz2vuzxrp +se +lists +vorbis +hmuprzd +dynamicadsl84223220162clientitiscaliit +textdecoration +for +unsubscribea +smtp +xbarracudaspamstatus +membership +rodents +feel +width3d95 +constantly +city +body +uppertitle +color3dff0000freefontbfontp +years +wi +name +this +brabout +committed +in +fraction +cdt +brnbspnbspnbsp +transitionalen +removal +have +off +returnpath +worry +breakdown +are +is +his +xasgdebugid +050020 +19 +hydroxyl +h5countriesh5 +hrefhttp0afpillsorenruepeqoladf01cdd12fd4f0click +070 +twemlow +plays +colorff6600gtfontbfonttd +said +subject +1b43616845 +relay +am +size +system +than +2002 +770px +email +b4000b +desperate +hey8aiang5aitb4qwakbeaijlfc0ohcb8x65dsbiiff6awf5rwkam1smgagilurxvudohkccnpgk +but +9 +shipment +commentatd +contenttexthtml +evolves +enhanced +style +image +flags +httpwwwbarracudacomreputationip2127520558 +xbownrlfmyomadaucn8k0wwacnofetlpuiujtrc55rts7lholwgznrghtwbkesselso5wxosms +good +world +delivered +visite100 +etc +face3darial +uvowegisud +xbarracudaconnect +limited +tion20 +normal +keep +yugoslav +print +rbl +dich +suicide +alignleftmassachusettsp +colorffffff +htmlhead +every +1311805735 +0400 +v7zqqky28rffel9lj6lurxsmqmlwzfkmr2ov1octuxshzodlvwgmqrztgdsxqk8qk6e3kiipmfax +064147 +before +retirement +rottnest +edt +advertisement +hrefhttpae7d6ef0f427cd5fellgatherruunsubscribea +us +it +esmtp +dont +deceive +acaangauadaaaaabadhcsu0ebgxkuevhiff1ywxpdhkaaaaabwaaaaaaaqea4adkfkb2jlagsa +deliveredto +information +aspect +bsfsc0sa119a +hibodycsminingorg +receive +s113117eeze +humans +localhost +5501 +height153 +westrong +5 +currently +67ze67768ae7mjqbunk5zbgrnojsp7bligc4ukbiaac3ufneugdxogtdogdce7dx7tvq7duu +messageid +pwuekpersonalro +pcs +kn111wn +10 +color3dff0000 +xasgtag +midfield +align3dcenter +alink3d663366 +xbarracudaspamscore +or +federal +public +15274145157 +such +taglevel10000 +contenttransferencoding +xbitdefenderwksspam +im +checking +jeremy +textplain +mimeole +127001 +kingdom +bsmtpd +files +prso +that +out +fictional +quarantinelevel10000 +prix +to +steep +charsetiso88591 +singledrop +now +78174104140 +colspan3d5 +you +211327 +3ak1ky3ehpyphfrzvmimvrfj1hnic4g9oagcjkvfkgahigumd6lapmu0zxmmjpz1pu5dzvfdl6 +td +32 +hrefhttptsproductinfocomcasp679736c65a20642aaeb0a112 +oxx5jmwebs4dwecxxs5nan0rvib5m8cbehjtvkru3f4ks9xzb7jtoa0m1iaa4hdtcclicb4nn +hospital +forb +bgcolor3dffffff +tracks +version +civilian +24th +points +id +meta +over +destac +stabilization +on +25 +qgjqqifkxpdcyhbcwpdoe0xeky5k3g7pcb4ycu8xiakcqifsxxibabowhkbpx3igxndegl08ajb4 +indiv +mbfontfont +properties +p111117rrez +xmsmailpriority +date +gmailid128bb599a95e11cd +shares +spamgwcsminingorg +onmouseoverwindowstatusbigger +e1955f2acc3814ceepimac6d4c3wwwfacebookcom +xbarracudastarttime +31 +border0atd +172334 +dvdrfontbfontli +l +third +width592 +authenticationresults +ray +intimate +suspicions +xuidl +them +mail +2009br +by +even +sa081a +biographical +ngbrmtfg5v7isqalzwdij7hpjpyxajhwtwvbdoyv76d7mjf9wh5clvdw3okr6dqistbzhex6h1 +csbookswileycom20 +ones +foreign +value65000000650000option +september +p +httpeinsteinsszcomcdr +software +namepresident +trusted +addr +genericphar +unionleader20 +with +from +unlikely +fun +hu9vf3fap5amymvthj9krgle712kebhenodkdkj8k6w1010u633hmvnng6bg0fst4k6azdg8mz +thyroid +money +updatesfonta +smtpmailjacquelinsteffenhagen8hlhotmailcom +mimeversion +vigor +stylefontsize +abnegatebr +000 +first +nsmeilleursitescom +2 +killlevel10000 +cellpadding3d1 +xmozillastatus +address +httpwwwbarracudacomreputationip12412014391 +details +alimony +use +charsetusascii +color0063c6click +namekeywords +xaccountkey +sender +todaybr +position +message +10px +which +19216818251 +stylemarginbottom16pxhello +developments +loc +an +percei118ing +xkeywords +webmasterefiie +7bit +citizensb +be +c +virginia +roman +optin +texthtml +jonathan +loss +were +nuevamente +fatherand +0nbsp +romance +existsxmailer +microsoft +religion +staggering +ebenezer +malaysia +3px +html +start +none +nw +29 +softwareb +emails +m0620212mailcsminingorg +score1277 +king +mariners +becoming +spinosae +24232154203 +call +0px +no +subdivided +700 +14these +some +r92aoxjjpeoz4bdxnbuuvkzpsavwtuclm6najxcg5bhxfvzini9kv39tjyxjglbrtvykdgiofgq +nbspbr +electoral +stylestyle +barracudaheaderfp20 +great +bsfsc0satofromaddrmatch +contenttype +stylecolor000000view +des34newsahotmailcom +may +like +mjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjiymjlwaarcadraqgdasia +of +csminingorg +and +received +one +sa392hl +1266000236 +free +phone +table +went +leave +view +8899nbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +confidential +the +xpriority +never +titleterms diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_18.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_18.txt new file mode 100644 index 00000000..aca07c4a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_18.txt @@ -0,0 +1,369 @@ +incident +jesti +used +divfont +htmlimageonly32 +xasgorigsubj +species +abiwurcnqofeb3dffe0c9534c34a11fac231627 +please +zs48l3nwyw4pc9mb250pjwvyj48l3adqo8ccbhbglnbj0ibgvmdcipgi +2003 +read +xmailer +th111115e +for +aaaaaaaaaaaaaaaaaaaaaaaaaabyulglkswtn3vpi0wpyqklzapo7pdj9znthtcy3t6o9dnv +network +educationin +services +smtp +xbarracudaspamstatus +subdivisions +vanish +glassmolding +body +policya +taciti +osocyla +successfully +this +400 +in +margin +cdt +fuller +have +needed +your +returnpath +are +is +mon +xasgdebugid +companies +00000000 +location +subject +vlink0033cc +nobody +concurrent +relay +using +produced +real +union +n +httpwwwbarracudacomreputationip19066150124 +messenger +aligncentera +httpsciscientificdirectnetcasp83999041248fb1d7ff79fc36 +yes +24325 +standard +state +debt +released +substantial +1002627 +below +9 +security +shui +matsudaira +underline +image +hospitales +good +replacement +height3d21 +browser +problem +up +namedescription +names +standards +22 +do +all +tue +scored +1889625141 +spouse +127227827278f7e0870001w4twrl +0099ffabout +chatbrbr +f +xmailerpresent +c7b3d1671b +vorfeld +whitelisted +12696442904c369b800001w4twrl +pwvk4icljsbiwvujidyeaojne5xvnfehsiy5aozidj86s1syrlj2gjigmyrap5cuf4bx30pof +oecd +boundaries +esmtp +option +dont +deliveredto +xmozillastatus2 +scotfree +jmnetnoteinccom +barracuda +vent +localhost +silver +05 +qing +gaming +rich +has +messageid +year +10 +xasgtag +cellpadding0 +marker +3839lko100z326ibjmmoaxyso25306d0c91pcguymuleya +xbarracudaspamscore +contain +sister +divides +public +doctype +hilt111n +223639 +contact +total +varies +010 +each +fleet +italy +postings +backgroundcolorb00836 +contenttransferencoding +o4gnvud2024304 +133928 +ctspresumed +aimed +open +textplain +mimeole +i +outlook +manhattan +that +out +specials +quarantinelevel10000 +midlands +stylemargin0px +282193981 +d97te +to +v1 +nnhtmlwithouttp +color880000please +fx4 +19216832 +now +code +size2save +value7070option +httpspamgwcsminingorg8000cgibinmarkcgi +probably +share +klgdfjhbv9amqgcquxygrngdghnqgokvgehmimzeoitj2obcbyqghpjkafooj06xpgloqivjcq +font +most +here +was +inreplyto +f117ente +dslb088074028166poolsarcoripnet +clkmp2sqwsop0qftstcawchtjmmwrwk33vt61oqdlg0a8w0gip7zyyvdpnzdftgkrrxnofs0q +size3in +thecenter +htmlfontbig +at +replyto +low +id +north +pts +rdns +hobby +can +on +25 +mind +tr +complete +xmsmailpriority +rankings +enraged +date +vol +deep +greatest +cellpadding5 +progress +ol203154fibertelcomar +2010 +hope +dutch +shots +slobodanu +xuidl +supply +peace +final +by +width4 +loved +unsubscribe +antispam +b5ec019dc13d626f7c3199db2d74attcanadanet +should +xmailinglist +story +will +050 +75 +ph +foreign +trtd +adderallbr +xmk31sivc4gvaqys57vq3yzpvo31gdkuzpr2c5jvpymseo1ru6nigwo9q9nwbk6qxbvque1bnhmc +trusted +not +i102 +with +from +0 +ancestorb +style3dwi +dacia +size3d3corporate +mimeversion +65lso +give +taking +210 +000 +express +cellspacing3d0 +pharmaceutical +sing +tongue +exile +millionfont +xmozillastatus +address +durée +prosper +gw1csminingorg +recognised +charsetusascii +ccsun37ccntuedutw +xaccountkey +thousand +message +didnt +its +control +q0t7tu2p2sabbyzbbh6iyxn0fphrns8ytxxqcrsisesnmjc3972xopocqbxhgyxr +an +7bit +be +sblxblspamhaus +get +competition +texthtml +rights +archtop +3000 +time +xvirusscanned +fonttd +special +existsxmailer +microsoft +paddingtop +vient3dgap +html +matches +really +longitudinal +none +color3dcc3366b164bfontfontdiv +o3r2ygtn006857 +100estr117ya +emails +spam +xoriginalto +about +0900 +umuioxoywa +brjmnetnoteinccom +hu6b7u0ickg1 +jeaatiry +excess +see +daa17258 +hail +call +facearialbia +3 +images +bilv2ltkkobhygowmmokyfedl6wjqdjqvi7wkymhvspy5qi3u107guteiylypkonpmzuffyyn9c +ha2gz61qrmddzdjhthpvihmpxfciqvjjvtmlgooxryauuvdrgdqynqvfthi6sk0535wgcz4c9cvh +sat +17pxstrongbr +undervalued +ulcontent +201129 +factoring +lm8li7zgn8f9rxtivpvds2fetsqgaepnh6a9bhevnympkeh97o7hti4knttqltjqwiydqqafshp +52 +dilute +202011 +e2wcmajqrdgdvym1wdwob8mjatcbyxtsasi8mvduvbrhhzqitqmcspqzafhjsdrq1lz +jst +contenttype +65000 +reservedp +bsfsc2sa154 +ukemoxavu +alignrightfreedom +llc +may +like +of +england +since +colspan3a +401 +and +received +windows +19216818120 +tube +skeptical +solicit +list +611px +nominee +accessible +url3d0022httpinternetemail +hoping +helod +access +k5cs246737ibd +assortment +bgcolor666666 +the +xpriority +never +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_19.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_19.txt new file mode 100644 index 00000000..4a9efbdc --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_19.txt @@ -0,0 +1,356 @@ +096 +postal +providing +readi +pulling +200434 +xmailer +for +permanently +s +support +smtp +standing +stylecolor3974b4newsletterijuypocomspan +10pxprincefonta +bodyhtml +bsfsc0sa392hl +tracker +xbarracudaheaderalert +body +policya +prove +type3dtextcss +years +fashionnbspthe +in +o11lr7nz000092 +provided +transitionalen +malden +firewall +your +returnpath +jul +0001 +satfamsbl +breakdown +are +molecular +namegenerator +yw1pbmf0aw9udqoncktvbglvcg91bg9zdqoncnjhzglvz3jhcggncg0kmthgluzeryyjodixmq0k +bgcolorffffff +nameprogid +commitments +00000000 +scarcity +213105180140 +effort +subsidiary +show +o1oj3v5b000913 +subject +cutthroat +compliancebr +170648 +charlton +less +consider +size +produced +trtable +recognition +facing +av +mobutu +avenue +httpwwww3orgtrhtml4loosedtd +thought +yes +but +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +below +meantime +9 +text3d000000 +httpgoldrockerscom +phillipm +par +fontfamily +style +hunter +styleheight +xbownrlfmyomadaucn8k0wwacnofetlpuiujtrc55rts7lholwgznrghtwbkesselso5wxosms +highrise +alto +2007 +apr +description +palo +axymu26kzwibababafupmhdtux1mhsqbbssop06nt26gk2uxo50umfvuz80vx00n4tzfv3uj3hu +our +xbarracudaconnect +zero +do +7149194185 +in100ische +welles +doodamanswbellnet +rest +26 +served +rbl +6318616221 +had +refinancing +img +every +res117lta100111s +81118111 +trees +it +esmtp +sleep +fitnessatd +jlennon909aolcom +893893 +fm +pr0n +color0000ffclick +ff80ff +margo +a +proud +deliveredto +clubs +hibodycsminingorg +barracuda +localhost +serif +as +dogmaslashnullorg +yield +rule +bgu1aqaceqmhmrieqvfhcsitbtkbkrshsuijwvlr8dmkyufygpjduxvjcztxjqyworkdbyy1 +profitability +mimehtmlonly +has +ciopcom +messageid +contentclass +350 +65925578 +10 +trade +mime +removed +yuan +5a1ebbfa9aaa9ada6bbeca9beeca3a7a1adf6e389f6e2e882 +alignleftthesep +linked +giveaway +xbarracudaspamscore +popular +or +yourself +maoidceqkc9thdazjjs1nismyyigyxhd3d3ajqwhwt1e1ntjaqdcpntgwdyvffw9gjjfaugx +public +bgcolor3d5492c3 +informationnbspnbspnbsp +0200 +contenttransferencoding +hrefhttpwwwglobalcybercollectivecomwsbfreetrialofferhtmclick +way +im +de +i +127001 +m0620212mail1csminingorg +lineheight13px +division +quarantinelevel10000 +xmimeautoconverted +lineheight150 +alignleftsome +163334 +to +charsetiso88591 +lsmtp +you +emjmnetnoteinccomem +httpspamgwcsminingorg8000cgibinmarkcgi +xanaxlbr +td +squeaks +web +alerttitle +jun +hqpronsnet +sign +operate +consultation +1023115130 +here +these +color3d990033malefontbtd +mxgooglecom +82 +6 +uid57441201487268 +name3dgenerator +larger20 +at +been +brbr +jeffgpronet +low +id +pts +jabber +need +on +canadian +same +25 +2bturns +htmlmessage +tr +government +105335 +emledem +date +spamgwcsminingorg +194305 +departments +incorrect +12th +2009 +5503 +order +mail +video +by +en +3energy +übrigens +back +trtd +little +sa148ahl +tactezlaa20 +remained +href3dmailtobbyaletesiaolcom +width3d534 +specific +with +from +generated +early +width1 +santa +denali99aolcom +eclipsed +border0 +mj1963 +alliance +o0re1cax006312 +first +emphasis +singer +aligncenter +tdemailbody +d4faf8c83e1 +mail3csminingorg +james +address +fetchmail590 +xaccountkey +sender +te +eest +message +eh75bw9ategtt2k9815zcrsqkugtnn6v0gbums5k0urgizzfn170jm7hrtae1bobmmfvquwhd +200905171242n4hcgb7n015410ns2csminingorg +httpwwwnettempscomcareerdev +american +communities +7bit +spidmax3d1026 +htmlshortlinkimg3 +be +waste +lm7dlyvv +size3d4to +100224 +converters +swr +l3300 +where +sportb +always +time +creole +part +kept +mi +href3dhttprdyahoocomdirhttp611 +w3cdtd +axjlywwdqo8aw1nihnyyz1odhrwoi8vz3jvdxbzlnlhag9vlmnvbs9ncm91cc96agvuc2hpmdev +20020719162521805035 +html +simultaneously +merchant +programi +town +81168116 +pursuit +paddingleft +xoriginalto +m0620212mailcsminingorg +aligncenternbspp +investors +thank +size1city +images +width3d100 +025239 +0px +we +participatte +qaehulah +does +name3dcity +sansserif +contenttype +xmozillakeys +wwkarboswmlfbkugywt2oyaxgwaqzogjw2yfmcgacbdqxoqxnqkrbekxsn8ie8zqkdw37ji6hs +agreements +input +pageafont +achieve +of +dynamiclooking +received +19216818120 +10000 +sa392hl +telecasts +52week +beyond +arab +dear +wish +phobos +bannerflextop +month +thanhnaclightingcouk +addresses +view +copymydvd +the +190009 +dofontfontblockquote +send +parts +height552 +d diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_2.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_2.txt new file mode 100644 index 00000000..8313e934 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_2.txt @@ -0,0 +1,364 @@ +inevitable +neunet +used +widely +tbody +xasgorigsubj +submarines +please +copyright +uci +bgcolorcccccc +able +buildings +for +monitored +replica +network +class3dbold4panda +support +smtp +msofareastfontfamilytimes +aligncentermiltonp +involved +satisfaction +people +children +reserves +2001fonthr +win +body +osocyla +617399 +this +b4aa82 +brbigger +in +als +ickes +prefigure +size3d2fontstrongbr +have +your +off +returnpath +nov +0001 +breakdown +are +is +mailmiestosk +selfmotivated +his +smh +feature +s0ft +estate +among +president +epsons +00000000 +prices +reputation +pressure +bit +subject +width3d30 +hereafontfont +relay +xbarracudaspamflag +me9vbiyuafmb3fwipmjymnvlgecahkspmon4ihqlqbelxsodskupaodczkgjygo3u571njasrgn +system +mt +organ +size2br +included +721 +tiger +avenue +fontsize +minos +abuse +bgcolor3deee8f615td +yes +adapting +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +0000 +054044 +url +axzwajkqayugb2rgalmgqjblbd25dgacdwbeb7gt2slbflaagvwc1ejfaxozzzzzaurrayleacbk7 +hrefhttp0b8b9xbapobazcnqb7d50344902d57383a04684dq4830232601834252602452 +ns2csminingorg +front +width550a +injections +131130 +zenith +19216818250 +rdnsnone +inquiry +geology +voted +our +a78twmb4j55oefz9klutsnpactdcgufbibb2p19ktxdt9pwp59jrycrtjiypwoj7wad4tzq +normal +do +10pt +tue +nonremovable +50 +facts +stylecolorce0000contact +empcorreoonolabcom +receivedspf +whitelisted +nonof +cqfehwwyeciarzatjaw4ahehiq61sboocbfllqlashcmmpaggifdjhpzabtagdoudqqwooacaba +stylefontfamily +left +it +m0089lla +crackmicecrackmicecom +lifont +esmtp +efforts +two +local +a +deliveredto +information +smtp1admanmailcom +hibodycsminingorg +raymond +century +personaly +feltokru +messageid +concealed +12398877622da0000b0000w4twrl +focused +owns +10 +xasgtag +sirius3dsmuggling +andojapancom +removed +raytitle +0700 +nice +xbarracudaspamscore +bits168 +lor3dff0000bvigoral +letters +processing +coastoption +andp +fats +demandsbr +cdo +color3d0000ff +010 +they +millionusd +contenttransferencoding +multipartalternative +legend +textplain +mimeole +127001 +outtheyll +verticalalign +out +n4py43roqji9hti9qpddhfdahencxzt78muujyyrnyp9ugiioapszaknfk0w6rdofteaijwnfl +styletextalign +quarantinelevel10000 +g2t5ans15641 +specializes +prix +to +cmvvbndlyi5jb20ncg0kww91igfyzsb3zwxjb21licencg0kotc1n0fxrgoz +xhtml +ig +charsetiso88591 +least +dvd +now +fontfamilyarial +isa +xbarracudaenvelopefrom +you +know +polish +ordi +someones +around +bsfsc0satofromdomainmatchhl +webnotenet +here +n1iejasc014383 +127272240578f661e50001w4twrl +was +change +button +name3dgenerator +msgidfrommtaheader2 +at +size3d2 +replyto +id +meta +efontfont +offers +dollars +3skglkm3qpsh1wbg21otxbxtx91wqx4y91kon0w2epuaz6dap5ot0uwcrptqyawortymt +pts +on +being +width35font +htmlmessage +tr +alignright +complete +click +after +cost +heavy +wabfontbrbrf +date +bsfsc5mj3865 +xbarracudastarttime +border0atd +xbarracudaspamreport +width3d141 +sales +note +ray +paid +dashed +mail +communicationp +redox +clymerb +87192220226 +by +ztmyx0aeqaaaup9beplqfqfxovhkzl0sab63maaaaaav329ju54zjwmi8p5f4f7lkttem8bcuc +unsubscribe +barrmumford +102914 +wxg2fumgb7gpmoufq1ruwvs5vpyyq3plvacdofvvvusmxly9tnckbo9lbvwq7machip6002 +httpspamgwcsminingorg8000cgimodmarkcgi +helo +will +via +int +sbfontbr +tholins +back +c111mpelling +7667403577451363125 +again +their +stylefontweight +criticized +zealand +not +with +from +still +mimeversion +0500 +bsfsc0sa275bhl +august +bgcolorff0000 +174556 +life +1021349204 +aaadaabptflnufvtierjr0luquwgq0fnrvjbicagicagicagae9mwu1qvvmgt1busunbtcbdty4s +use +make +3br +end +positionabsolute +marketingsubject +message +which +accredited +face3dtimes +say +an +kccf9dvqk4dsuqe81llwmn3wb6yqulntxfjj2qozgwlp1mblggjkynvvmihzaj8acgmrcohb1nis +c +ontime +videos +account +vancouver +httpequivcontenttype +texthtml +starting +telecomitaliait +ls0tls0tls0tls0tls0tdqpzb3ugyxjlihjly2vpdmluzyb0aglzig1lc3nhz2ugymvjyxvz +time +nova +uid35281201487268 +arising +onyour +directly +brl +81fontbspanfont +httpequiv3dcontenttype +surrounded +intentionally +purchase +deals +tr20 +81168116 +matches +097 +metal +karesava +kdbmkiqni7yvikeedxe0ppupgwxk7putvdcqljiitovr8lsqxxgoaxrah51bhzcmwpk0bjic +clause +publication +0900 +simply +thank +many +compulsory +against +arial +0px +no +webmasteredgecpacom +very +moralesrlyceumcom +mailings +some +127341333778f43e780001w4twrl +japanese +bsfsc1tg070 +jst +including +contenttype +xmozillakeys +vvkbjjjbarbwz0gh5bayh0rayvf0fgpbmugog3ddtrjbcrnpltssyj5zoooplfjupqldf8axz5 +user +america +singapore +of +elitedish +csminingorg +and +fff +received +plan +sa392hl +anytimeb +list +color3366cc +phone +bgcolor3d000000 +95nbspspancontribute +europe +the +xpriority +centerfontp diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_20.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_20.txt new file mode 100644 index 00000000..fb2d43bb --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_20.txt @@ -0,0 +1,332 @@ +e +writing +strongsite +name3dzip +trouble +xmailer +5px +for +postfix +unsubscribea +network +school +smtp +conditions +lxa648hf +schaefer +body +this +400 +in +mailtospamassassinsightingsexamplesourceforgenet +letterspacing +1978 +your +align3dcenter20 +returnpath +imap +dk828c65f4fvat6c1vkmf1ckj746v2mmlnk1m6akfzdkgwi5oc0rcpgy0gmmajkyioxc9cauyr +is +mon +b2b +hereatd +23 +newsh2 +00000000 +d4cdcd +xbarracudabwlip +said +market +subject +pgp +furchterlichste +custom +size1emailb +kxvdcoajql2hwyr2qbeewkatccbemzolayrqoaiyyztzmpnw1iioqiaxwmknoeti7ga7ugbbtga +77 +definitional +using +real +src3dhttpemmansell +bfont +srchttpa02fcvkunrorfcnlogogif +history +con +quotedprintable +cmmsysaolcom +throughout +but +shamed +natalensis +insurance +barbary +uasbr +binder +underline +another +httpwwwfacebookcomeditaccountphp +border3d5 +ist +br +description +delivered +xbarracudarblip +our +size16 +t +size3d5bguaranteedb +iwujbynyc4knng1drgg6prrcwesc1ltod1oizswe4mgbkahcqntng71rqscjfadidkyem6osphz +short +all +head +distribution +26 +gb241w808214 +pretty +host +sale +img +price +page +tanzania +owbxsjhx0cpz3xyv41tmm4rzifc8aaqudvdxvfj3u2zlfdtzbdkpxjtdem47osxlbj68mluva +itemheader +before +9fc7response +bold +stylefontfamily +left +peaceful +esmtp +24 +local +a +deliveredto +xmozillastatus2 +173118 +receive +railway +barracuda +century +localhost +decree +g6n1qwqv025043 +as +marked +forces +wristwatches +rule +href3dhttpwwwa +pdt +messageid +newslettertitle +10 +xasgtag +27a +cellpadding0 +color808000inbspnbspnbspifontfont +painless +unknown +b1fgx1awjrvxvthsufnh4e2wvhhdqewv1jat3dgj2vz2pkcmvfnvj5aiwbxrndqayuqou5g +g4db54e30128 +warning +201005120653562352qmailwwwbeamtelecom +findemubr +or +questions +doctype +char +others +010 +peruser +provide +cents3dmejor +contenttransferencoding +color3dff0000models +service +textplain +127001 +94304 +outlook +8bit +helvetica +115944 +text3d444444 +to +charsetiso88591 +singledrop +separated +ubohunyw +060256 +td +lord +middlefont +department +home +1023115130 +here +these +poster +225551 +altshop +otx50sedyumiqcnyviwuf3d398eb616cbbca9ce86pjigjja3d26597304777 +found +6 +htmlfontbig +at +wraps +often +sa148a +low +id +offers +popmart +sa154hl +pts +also +135453 +on +17 +waterloo +related +xoriginalarrivaltime +g8jgs8ft008051 +shown +htmlmessage +principeoption +tr +calculate +complete +p111117rrez +apply +stylefontfamilyarial +xmsmailpriority +color000080 +date +wodaxugu +spamgwcsminingorg +pcetlsbzyxzlzcbmcm9tihvybd0omdaymilodhrwoi8vaw50zxjuzxquzs1t +ciagicagicagicagicagicagidquli4uliakntasmdawicsnciagicagicag +wastemindernet +worried +12px +f111rmatting +marketing +xbarracudaspamreport +marital +authenticationresults +reservedtd +g2v55kt24035 +dbrxoye608622f3a1c3222237190179 +operator +by +srchttp20912615962spacergif +love +ls9xjyovhjzjz7mms3lhv5546umqqp5u4rqq9qm93ooxeszb95mastsg7udcj9mum4cqv4pmrk +yill01hotmailcom +gw1csminingorg19216818250 +new +story +few +trtd +0100 +software +stopping +with +from +1255468845 +faqfontfontafonttd +ji1xwakuxhddaoodynafugzhysqxgmbwavdaqsedlsrmagmejtbcqgvorahlrlnnmgdjfecaqc32 +person +mimeversion +0500 +tdimg +press +haringey +2 +killlevel10000 +actions +livnptntp5bxhoed5i9jwurtrjqo2i3fkxjlxm1dk0rqrejsewijbdtnnduynkbclww03gjxr +militant +xmozillastatus +gw1csminingorg +charsetutf8 +hi +size2 +xaccountkey +blocklist +whole +colorff0000namebr +five +heat +160 +amirah +lipresentationsli +development +be +noel8rcygromjygpkju2nzg5okneruzhselku1rvvldywvpjzgvmz2hpann0dxz3ehl6gooe +va3wzsdrzuabs29artsvcferkdzbn9dlfldydwogoo65yzdqk3j5fm4dtp60o7mul5hbksdbd +unbounded +get +httpequivcontenttype +1967 +texthtml +18 +rules +loss +time +anything +piazza +mexicooption +cg0kdqpnyw5hz2vtzw50dqoncnzpbmnyaxn0aw5ldqoncmlmb3nmyw1pzgunpc9tuefopgoncmfs +cardinal +more +4ucphlhghapz0f8kbhqyjwdwds69v0h1pftnuhmzyjhqxuyfsfwbns8acgvpl8 +microsoft +consolidation +000b11a65aad8587c2b33bb78ec3ubrjup +162942 +3px +html +inspected +81168116 +colorffffff09090909reg5 +color336699br +b248l3rkpg0kicagicagicagicagphrkihdpzhropsi0miins4yntalpc90 +stopvipeprojectwebcom +spam +according +populationa +call +asking +arial +class3dtitle149tdtd +no +we +military +does +hrefhttpcooa14jnuaixagcnybuoymuhep1890426e73a9fb81yevqbufjdy1550214841 +former +jst +nbspnbsp20 +contenttype +dac7f1635c +may +listed +cerificates +csminingorg +ii +and +record +received +19216818120 +wed +list +28 +table +dgljdqoncmfzc29jawf0zwqncg0kcmvjb21tzw5kyxrpb25zdqoncm9jy2fzaw9uywxseq0kdqo4 +access +fee +the +xpriority +parts diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_21.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_21.txt new file mode 100644 index 00000000..63036e23 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_21.txt @@ -0,0 +1,354 @@ +face3dariosoixxxx +irs +ielovevstkvudqogdqo9pt09pt09pt09pt09pt09pt09pt09pt09pt09pt09 +number +xmailer +for +c4194a +business +smtp +xbarracudaspamstatus +june +facearialshell +usbrbrclick20 +wi +court +bv8adfbkd9z8kyzu38mdj05obktieln8wf3xaogwakqyd68a3w3fadx0ohquwsx +name +005306 +ip +in +reg +sizablebr +based +22e +have +your +returnpath +062140 +mello +are +is +o0a1jnjk029348 +versiontlsv1sslv3 +bgcolorffffff +width3d349 +width3d150 +100 +revolutionary +00000000 +a3hfawjxjkbmiazv8rdjmtcvobwkaczotu5cmyidj2btp8zafqyzqbygeig1dw7xxeaczqqaha +show +reputation +n9e9rdga028827 +subject +relay +using +corrinneanalisamsbordersgroupinccom +imperious +usd176m +frances +included +l5f903kp6lzwh8eneootn9yfwq1b1pm7op5fqpdpj5zm9hfxatscqtjzoag7nbvzvbd1 +2002 +lowest +bulk +ibkbskwscdxxhgpgpb4fa0tyruuzffqfx9e2eljbxfivbzimgfme4bvhhwso4jegiy1w0yolpkn +want +three +jsamsnetcentcom +ttnetnettr +email +secrets +yes +pcebbd3dab9e3b6abd6d0c9bdb9c5d5f2b5c4c5b7cdfed5d5 +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +land +below +n6s95v91017981 +symantec +pop +best +insurance +shui +width150 +northwest +li +hundreds +ns2csminingorg +danso +size5149149149149149click +tolerance +own +up +process +115ur10297ce +face3darial +war +162609 +cellpadding3 +whopping +xbarracudaconnect +t +play +all +head +publicly +50000 +6zjcdvgbjfrdvwmvocub06rz2tr4eq5u6hundapu0q2ecmaqqs1twpbqnbvblqooyircl +rbl +agency +01302003 +displaced +size3d2fontnbspdiv +zm7q6wegxrhqp1ec1j7tef4c6d3o7lqjrroxxnvsur7kz40ckh7hvjwywq4uw27psqhmeu5qqcaw +features +sale +img +price +msgidfrommtaheader +ll117via +cellspacing0 +full +ahover +confirmationfontstrongdivdivbodyhtml +14pt +track +width3d60 +us +alignleftoleo +esmtp +titlenewslettertitle +a +dont +targetblankhot +calls +deliveredto +subglacial +xmozillastatus2 +hulled +hibodycsminingorg +responsibility +receive +barracuda +localhost +120924 +as +internets +harris +forces +reasonable +div +pdt +xravantivirus +has +messageid +pfont +ap +year +10 +mime +900 +old +linked +16pt +popular +183334 +or +above +judgmentsbr +public +cpunkshqpronsnet +come +0825070004 +010 +taglevel10000 +trying +quebec +contenttransferencoding +includin +ma +service +127001 +zrmjks9x5kvtv39pbwtwf3pr0ciw00ffpd48fec4rtvhvu7g8pdcx8myiakeiysrlzj8w0agun +lost +work +adet +out +programme +correspondence +cypherpunksoutgoing +meab +married +albany +to +sep +fall +code +1014118715 +you +qb1in +web +demgegenuber +company +select +je +182410 +arialcolor8c3256textdecorationnone +font +celpulsomercedeslinenet +6 +something +1 +size3d2 +brsuch +replicawatch +yyyylocalhostspamassassintaintorg +msif +brbr +bnisch +hero20 +id +meta +286 +procedures +mandarklabsnetnoteinccom +pts +over +can +25 +remove +click +charges +3141 +famvirb +eate +party +date +re102lect115 +31 +classaltnormalbr +alignleft +cellpadding5 +easy +121633 +xbarracudaspamreport +aaaaaaaaavqaaahdaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaadhcsu0efbdm +lsb36 +181246 +l +idl7569s971t24u0 +xuidl +surroundingsh2 +mail +by +clothing +riskthis +fort +pick +distinct +struggle +via +voiceless +size3d224223e3cb3eor +aligncentertrouble +p +12564618246d7b00f60000w4twrl +farbmagazin +style3dborderright +romanstrongviagra +enormous +with +from +0 +stronga +size3d7 +mimeversion +size1 +fontsize20px +heaven +stylefontsize +gw2csminingorg +altclick +currency +could +alimony +picture +gw1csminingorg +sent +words +xaccountkey +sender +end +message +file +which +19216818251 +easierdiv +ruled +be +brbfont +attracting +gw2csminingorg19216818251 +route +httpequivcontenttype +texthtml +fake +script +ratify +ranging +title +special +hrefhttp935aa0sqopacincn568358x3y06i26j143229qjelixumefexyphibody +qualitybfontbr +xbarracudavirusscanned +hungarians +h97s +testsbsfsc0sa148a +mechanism +html +httpequiv3dcontenttype +boosters +beneficial +m14zltsfh1trwlcm3zkcloy6gcaimqehucrtxjr4drcf2e0ioahc6uobeo27vcyflultrq3tgbbj +molt23aolcom +81168116 +basic +spam +about +m0620212mailcsminingorg +example +0900 +towns +width5 +see +saturday +predators +no +possession +wmv +we +having +j5skhbsvxntk9kw1xdxl9vzmdoawprbg1ub2n0dxz3ehl6e3x9fn9xeaagibagqeawqfbgch +insurancetitle +pfp1j1kxdgegin0yj2xgw9uiarhqldlgfvjxsgggbkhsca8liaigm1dkfairawhamwuweubcdnew +baxnjmjnads0uazrmgbzawxioqjzqbnlmgcpy1oezp2axnaaqadadswkzs7qaemypsmmm0nlmgc +pbxf1ex1vmz2hpamtsbw5vynn0dxz3ehl6e3xaaawdaqaceqmrad8a7p7dzolf2zel9us0 +including +contenttype +xmozillakeys +settlements +possible +may +like +other +of +csminingorg +and +primary +received +free +deleted +8734icum5941fzrj1099dvnh4450zrld0601vnia4052smew9113qiau3562mewq9923ibvn21775 +send +parts diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_22.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_22.txt new file mode 100644 index 00000000..a9989adf --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_22.txt @@ -0,0 +1,340 @@ +xasgorigsubj +please +163548 +valigntop +trouble +compete +abbian111 +xmailer +reserved +for +intelligent +program +network +smtp +2020 +cznxuykhqdg8msorzb67cpf5zf5tbbohk98ce9daamgwl1oazxtnxpods5nmabgm1gqg6inmemz +xmailerinfo +any +xmlnshttpwwww3org1999xhtml +2001fonthr +across +ip +052 +in +based +professional +0725 +your +gafioc4f0gmdscaqvgfmp5zviyg0uaaidqrzgidcfdzgwgguycgkqebgkfyutqpn2lfvtpdjda +01ca0953edcc5f900ac2995dxpyeeueb +0001 +is +100 +23 +customize +helodynamicdialin +subject +try +custom +relay +included +2002 +pop4nxwyzy6waa9bmaaaex5t9yj8p6hj1mknfaprassxxny3vs9plbvq8nsvidm1x8umxxaoinm +size1bcomplete +284 +late +email +abuse +con +yes +but +sure +transmit +below +93139103196 +ns1csminingorg +9 +kings +best +security +align3dcenterfont20 +contenttexthtml +underline +would +1991 +partern +statistics +radio +blue +ns2csminingorg +fontfamily +style +chanting +image +width3d465 +19216831 +problem +description +rdnsnone +own +delivered +our +cellpadding3 +philippines +xbarracudaconnect +xacceptlanguage +4h70akibujkhenxgzotimsmgjokldq7ocgc1hparxkylhpbfobx0ol4702ktcy5djdzd5bzxvzo +normal +all +colore77d2d +head +leroy +barred +26 +rbl +54pt +youre +n51jiin9008982 +data +ul +clientip8316119359 +bold +us +stylefontsize15pxpadding3px +it +persons +esmtp +value3dinfofairlaneannuitycom +timbwcvuzbyaunlbqewqvs1siks22flsqro2fbj9bvukwjgzwx7cm2lujk5vvwc4uen2q3f +ownercypherpunkseinsteinsszcom +chaque +cjjggeefmaajbkinrquwaarfewrbrhxptuhfh728vyfbvjhrxxxwr1qqimbpghegwe2eaccl75y +dont +deliveredto +comercio +increasingly +xmozillastatus2 +manufacture +hibodycsminingorg +as +build +g7r4gez25436 +textdecorationnoneupdate +because +rule +mimehtmlonly +has +currently +messageid +locally +downs +500 +10 +xasgtag +availablebr +mime +ccc +hsgmw3ctxukbenz0otazavjtabxsnid5bmpg8vuw2anlkjgaawwtqadlovw3sq0m1jkda7mrgnww +0700 +men +spnews +public +121707 +interested +brwere +taglevel10000 +contenttransferencoding +textplain +i +bsmtpd +grand +work +helvetica +mmc +declared +to +stylepadding +4less +awarded +code +174402 +you +tdtr +jun +width3d447nbsp +keeping +1093808 +svog +font +1023115130 +flurry +zzzzjmasonorg +extracts +help +inreplyto +version +there +bytes +engines +at +1 +pantsfontali +low +id +73jaadx8lqmqt8otrrsomsjjzk5kwghjsdu4rxsettgfrypticzurwvj1dsxmxpy5o1c +chiefs +mandarklabsnetnoteinccom +need +on +brazilian +8f12a2c322 +httpwwwdormwarsnet +climate +began +tr +click +b +relaydubt31nwcgroupcom +sanction +date +spamgwcsminingorg +viagra +hrefhttpwwwtuduwrjcnazoon68ebeb7fc8d97f4d437e +021 +alignleft +n17mr6108986fab231273577581701 +easy +xbarracudaspamreport +table20 +stylewidth +1eitlxvlay2zpas6htg4dak4yzfjbfmo7vfi66uufdmsf9u18yxwvtgzkqlbonpl7kxvblx42trh +formula +12th +authenticationresults +cease +naturalization +opening +mail +by +wholesale +visit +he +unsubscribe +facearial +mrhealthbtamailnetcn +will +link +050 +yellow +receiving +back +trtd +0c5deu9p8koxhhtw25wqaakqxyq8cr08tzsgvdlwjd7v3skrraxbcft8u0cing14pwnf7msop +8px +their +p +yaol +addr +moment +bsfsc0sa148a +with +from +marshallese +invaliddate +type3dsubmit +1995 +money +mundo +virtue +href3d22httpnewsmoneycentralmsnco +mj1963 +first +cellspacing3d0 +killlevel10000 +width3d40bfont +deeps +definition +sa275hl +2px +sent +styletextdecorationunderline +words +fetchmail590 +colspan6 +available +xaccountkey +otonysucal +boundaryc04e5c24b3ae4ba8a82d2aa0500e2d84 +ad +members +message +16410024360 +almost +iabmkv7c2dlm8y78yzcg5xnfdbe4dgegd0rev7dnt7mrebab1zohtujxqzq2izkz6zkd3rq7qa +dedication +an +latin +herea +7bit +carpeta +financial +be +500000 +longer +brwebsite +sought +ikozi1950lexiconpartnerscouk +matter +texthtml +height1nbsptd +charsetwindows1252 +repeat +y0ilbyele0idlcaraa3hb8qod32eqr6jncu3wfaavhpzwasemfchuspdnslscxikqi5zwgbgvfo +httpequiv3dcontenttype +card +brby +uncensored +realize +subscriptionsstrong +height3d30 +rat117re +many +iciafontfonti20 +arial +difference +0px +no +moved +name3dstate +size3how +columbia +valuemt +1264068587 +jst +contenttype +xmozillakeys +slsfyasny3fwbfgrbhgoi6eacunmkxnjszozqatfnzrmgbsam0mam0agam0lfac5qdqvlz9 +input +willing +like +of +ive +csminingorg +and +avisited +received +19216818120 +industrybr +located +list +wish +table +sa392f2 +earn +hrbthk0ickg3 +australian +view +psiwiibjzwxsc3bhy2luzz0imyigy2vsbhbhzgrpbmc9ijeipg0kicagicag +never +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_23.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_23.txt new file mode 100644 index 00000000..197af4b1 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_23.txt @@ -0,0 +1,337 @@ +sansserif20 +please +134nbspnbspnbsp +ahiithemailcom +for +resale +smtp +xbarracudaspamstatus +june +ijlqle +12 +reach +newspaper +aligncenterfont +bgcolor999999 +valuegwguineabissauoption +the20 +instead +in +pond +connects +firewall +your +off +returnpath +breakdown +are +is +mon +xasgdebugid +versiontlsv1sslv3 +60029002180 +makes +23 +00000000 +200911110851nab8p9hr027976ns1csminingorg +reputation +ripoff +subject +xbarracudaspamflag +using +thereafterthe +accordingly +fontsize16pxadobe +swsyxkrrjvmo9wd3r9escl3x6mio3qyx1jpfdx6mldlmmumuhkhlzloecy56ubwhgjw7fj8yrme +winter +late +n672brv2024954 +hrefhttp48cmilealsoruuyhecaa29f5e14f84d5ba +but +0000 +below +lonely +apathy +devalue +would +radio +students +if +tax +project +br +97re +exmyeqwmdawmdbemdawmdawmdawmdawmdawmdawmdawmdawmdawmaq0lcw0odraodhaudg4ofbqo +description +mx1csminingorg +criteria +up +replenishment +mx1freebsdorg +2827262524232221201f1e1d1c1b1a19181716151413121110 +our +per +xbarracudaconnect +250 +keep +22 +size2flinstonesfonttd +friend +all +head +archives +problemsaccessed +print +boipelobeldencsminingorg +alexander +thosep +deliverydate +herebfontap +doquotbr +ratesfontbtd +3564443f9e +htmlhead +multipart +hrefhttpe43pharmraimund20arunymotuhiuu7feb9cabc4 +confident +gmailid1287febd66e45370 +saw +it +site +esmtp +httpxentcompipermailfork +two +a +deliveredto +adhere +create +tear +color000000 +localhost +4ovttcjuy8hauk8vsuc2dwztziwixuuznyoa3sbnopztjlbmkycrsmbwocvzjw76bl9ssig1vu +build +height3d433 +because +ricathe +div +mimehtmlonly +messageid +350 +10 +coming +unknown +esr +o18nishx010565 +old +125117 +xbarracudaspamscore +114625 +sells +yourself +guarantee +textdecorationnone +size3d5lendersfontbnbspfontspan +total +010 +peruser +soapelecohclick +contenttransferencoding +multipartalternative +de +threatening +bledisloe +mimeole +i +127001 +helvetica +quarantinelevel10000 +to +v1 +surcharges +issue +singledrop +103926 +code +know +interestingspanbbspan +returned +compatible +height3d181 +department +privacy +these +size2wsiwyg +211217214 +version +brother +believe +hours +at +id +sa154hl +fazz2lxxjfbdlqmfvgi4ldy09h1vvu06l91vrlp5jllcxtdbsceofdqbduesshgfczxw9vsken +mandarklabsnetnoteinccom +pts +n4gnsmkx016065 +can +on +languages +player +htmlmessage +discover +apply +n7j7bcyb009184 +date +spamgwcsminingorg +xbarracudastarttime +jalapeno +who +xbarracudaspamreport +po +2009 +height3d40 +code7 +xuidl +mail +cryp +by +columns +visit +unsubscribe +johnstebbingcsminingorg +g533nl726103 +helo +new +will +justifynon +men39s +050 +75 +customer +back +1258102535 +0100 +strategy +003341 +illinois +p +light3dffffff +urlnumericsubdomain +29th +not +classbodya +with +from +dwl +width1 +e102102ect +namens +subscribed +mimeversion +scores +size1 +gw2csminingorg +color3dff0000how +resorts +media +alignrightsent +manuals +address +resistivity +listid +bsamplesb +gw1csminingorg +8122812220020902revision +words +fetchmail590 +hi +tenth +sa275bhl +xaccountkey +characteribus +permitted +aukqymbshrj39kzzuwzule6n1bgzrhhzd2m8h3qqnmuktdqulwfy1kyjecoppfdvyx8vadgf3u +valuewebmasterefiie +anyone +teachertube +taxonomists +width3d77 +fully +7bit +estore +be +meet +hype +records +get +nobodylocalhost +href3d22http3a2f2fwww2ewldinfo2ecom2fdownload2femail2fnewees2ezip223edownload +dpusanet +bfontlitdtrtbody +more +case +xbarracudavirusscanned +w3cdtd +r +18yrs +0pxdiv +html +mail1csminingorg +81168116 +liberal +delivery +about +tour +0900 +150 +width100 +metrop +pgnrcomrhtmlc3d1113074r3d1112136t3d1225499824l3d1d3d89042084u +0px +we +having +name3dstate +let +mj3762 +moneyback +ou +face3darialbviagrabfontp +45004550 +sansserif +contenttype +xmozillakeys +cotés +candidate +bbeing +070341 +may +align3dcentertable +of +size3d2fo +5000 +brdefeat +since +401 +and +received +cpunkslocalhost +19216818120 +d116716018 +cellpadding3d2 +fast91gipac20westernbargecom +eats +proposals +033621 +nt +table +helouser +cases +href3dhttpbjpoidfecnfont +3d +view +cellspacing3d3 +the +presentation +decline +cant +s6txp0gmbiymxaywthi0pkrgrsm9zwvzm9udd48l3adqogicagica8ccbhbglnbj0iy2vudgvy diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_24.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_24.txt new file mode 100644 index 00000000..d417148d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_24.txt @@ -0,0 +1,342 @@ +michael +used +reference +preading +okar +copyright +interesting +number +format +for +postfix +2page +regulated +smtp +june +divmuchdiv +further +readingsstrongap +any +xmlnshttpwwww3org1999xhtml +score908 +mailwebnotenet +wash5 +body +years +name +this +in +based +color0000ffnbspclick +transitionalen +removal +reliable +have +your +returnpath +ranunculaceae +0001 +breakdown +are +is +mon +4kwmgtvudeggouephewojfksyuijbecazlkq5caqbcs0pfnt5k2coxxutpmsz0affomehdqw6emk +uefopjwvrk9ovd48rk9ovcbmywnlpsjcb29rbwfuie9szcbtdhlszsigc2l6zt0inyipfnq +awygew91igrpzcbub3qgyxnrigzvcib0aglzlcbvcib3axnoihrvigjligv4 +xasgdebugid +s117cces +100 +bsfsc5mj3761 +00000000 +india +size3d4spa +bsfsc0sa290rn +xxxxxxxxjp +subject +municipalities +try +got +415 +listarchive +lowest +history +interaction +quotedprintable +sbr +181453 +marchette +sure +wrong +ns1csminingorg +jmcheetahnet +security +rowspan2 +colorcc0000freefontb +091622txtdogmaslashnullorg +online +levels +tuesday +good +19216818250 +improvements +19216831 +exchangeeliteadvcom +description +intervals +process +10pxunsubscribefonta +etc +face3darial +theastrongfonttd +war +our +coast +all +readersdigest +rest +204045 +loaded +had +utrecht +deliverydate +host +img +ready +unlist4meyahoocom +href3dhttp6767ydeceyercnyenaqaop3d6767b54ba85639caff829z +style3dcolor666666 +technically +hrefhttp4d4cbfxqufojehcnquaoxejybq7427c64373r88411234g4uxadon911817498129208230405632 +edt +1999brswansea +sun +esmtp +c2vudgf0aw9udqoncm1lzglhc3rpbmfsdqoncm1lcgvyawrpbmuncg0kyxnzb2npyxrlza0kdqph +titlenewslettertitle +laserfax +local +a +roberts +hibodycsminingorg +receive +color000000 +energy +airborne +as +dogmaslashnullorg +forces +voice +rule +mimehtmlonly +messageid +350 +72 +newslettertitle +vienna +10 +theories +xasgtag +mime +verdana +whose +045703 +yet +or +categorized +doctype +1052 +total +west +contenttransferencoding +growing +035e67c20d4a4732e4a02ca23db0txpuyp +i +127001 +standardizing +styletextalign +align3dleft +touristoriented +xbarracudabrltag +to +hrefhttpqysn80jnuaixagcnxoqufoluxy1890426e73a9fb81qmusijh1550214841 +sep +charsetiso88591 +typetextcss +code +httpspamgwcsminingorg8000cgibinmarkcgi +td +web +russian +defjakxfyceavqjrf4tmkkb83hecqli5kq4wcat5ndub62gzt4y6ecsfwqn6riwantzs +hqpronsnet +font +le3ddisplayblock +022520 +abelard +href3dhttpwwwblurgreatcomfont +version +change +6 +personal +roll +doing +gb1hcl808717 +invoiced +replyto +pacific +wolf +go +id +sa154hl +pts +also +can +on +being +same +71ce443f9b +likewise +notice +language +san +consolidated +after +date +bonetitle +xbarracudastarttime +bsfsc0tg035a +1439 +xuidl +morefonttdtr +trailer +mail +raised +by +think +bgcolor3ddddddd +centera +682169198 +airnav +strategy +their +printed +trusted +reduction +28003200 +not +with +from +20 +anderson +mimeversion +border0 +content +press +resigned +colorffcc33 +placed +killlevel10000 +aligncenter +u7mr11500141wee511272631068543 +boundaryboundaryid57z4opjiwztvbinvv279zq +nlix7mqwl1r2lmfopudnzkgufs37nxsf9lotzrer6zkxsqbvdukvbqxndmsb3pqoa6gja +xmozillastatus +make +news +approach +hubert +v11 +lesser +available +size2 +xaccountkey +sketch +blocklist +message +fired +sanctions +xkeywords +air +offer +herea +461ca2c54a +forged +f64law4hotmailcom +19216833 +debate +targetblank +sa011 +be +sa011c +i0uiywe9xdhemunohecfkun4cmxfkokxt5txdk2mfvr6osgdjgewzjihrcv3cdorejjvxb2fb +httprecquyfmyjcom +neki +instructions +release +texthtml +18 +possess +commons +raa04566 +21font +power +cypherpunkssszcom +microsoft +xbarracudavirusscanned +12pximg +colspan3d7 +xloop +required +assistance +html +81168116 +relatively +potency +terms +contains +fri +bibliography +spam +k117nstmarkt +0900 +king +150 +see +height375 +n9sefysq021224 +made +896px +133mubabstnbstnmacodslattnet +no +approved +kiev +top +pmunoznetnowcl +deepens +conservation +does +013 +hrefmailtobestoffersdishnetdslnetsubjectremovebestoffersdishnetdslneta +barracudaheaderfp20 +contenttype +bdear +fbemdawmdberdawmdawmeqwmdawmdawmdawmdawmdawmdawmdawmdawmdazwaarcabwafad +may +valuemgmadagascaroption +secure +other +of +england +away +and +received +youll +potential +emailp +australian +view +sell +the +width3d200 +those +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_25.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_25.txt new file mode 100644 index 00000000..e587a71a --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_25.txt @@ -0,0 +1,346 @@ +55265313 +nbsp +used +e +beautiful +dangerous +httphnvzatsehsyrcom +pnbspp +sown +quartiere +format +rzbyisgmwxqcytabagafp3aexnczu5apzxobjlagjainnaakckburkcdtamvemvnaihqafjqbt +for +postfix +tdtrtable +services +smtp +june +body +name +this +ip +in +mta21bigpongcom +henry +eaa +returnpath +breakdown +are +is +mon +rd +selfassured +bgcolorffffff +g8 +accept +k2cs40471iba +23 +bsfsc5mj3761 +subject +topoftheline +using +am +produced +scientific +c1c3a440cc +included +lowest +appp701106td +strongmiss +href3dhttpdiscountshackcomehtmclick +quotedprintable +shbig505413body104 +inland +3r3109uznbspnbspif +hrefhttpbac70nihosslcnyxaavyaq36895as587hr8z052078190etyluiixag7060351295602543767160uuhedekypc51u0jt2z60nw0a692640einamolas27426626062224502832 +but +optout +benzodiazepine +octoberthe +best +hxcqkbrhsqcpai8wmqaxetbkeapvgab0xf8iqovqany1gf3oaxjwitmuazmtvqvogvfka4lkaiz +xenical +undertake +182d7168a1 +text3d000000 +contenttexthtml +101041 +excessive +part121076313585444161028690797657 +t97ke +glance +smtpserver1cflrrcom +hardenp +lsj1fixjalw1gab5jntlqsturk41pxbxhwjaduvcbewz9rfc2gflpj2ruqhuotm8f48vxya526b +2007 +19216818250 +19216831 +description +rdnsnone +delivered +names +1867 +cypherpunksforwarddspronsnet +aligncenterlondonp +notification +name3dbtnsubmit +our +cannonschart +ave +250 +bgcolor3dwhite +autoprotectionextensions +do +all +tue +align +26 +u +rbl +host +img +nor +166 +0400 +journalism +httpwwwbarracudacomreputationip66118651 +selectedmmoption +it +esmtp +lyrics +partition +a +94 +deliveredto +protein +office +information +xmozillastatus2 +httpslistssourceforgenetlistslistinfospamassassinsightings +bebayb +mondayfridayopopspanstrongp +nosend3d1 +localhost +forward +because +rule +sslnbspshopping +mimehtmlonly +messageid +year +pcs +institutions +10 +xasgtag +trade +mime +unknown +perfect +account4 +popular +or +icagicagica8ynidqogicagicagicagicagicagicagidwvzm9udd48l2i +dhcp +frank +2001 +peruser +critical +taglevel10000 +problems +xbarracudaurl +holder +ofh5 +zzzzlocalhost +gallaudet +127001 +outlook +charset3diso8859 +115umm97rize +helvetica +valuecyprus +that +out +border3d0 +face3dverdana +buy +to +xhtml +singledrop +ksdiz6cwz29mu09zczhsodn8b8gl4cpslc5wndy39dg77cvslc0kcoshssf9wkmdj +now +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +fedex +rgif +issued +18012980122 +2010p +here +inreplyto +change +eputqgejnj +valigntopibfont +pop3 +policyap +at +go +low +id +wrapping +333a +external +abandon +htmlmessage +transmitted +tr +government +maya +turned +rowspan4 +date +spamgwcsminingorg +eshop +capital +takes +who +2010 +xbarracudaspamreport +allow +abr +easytouse +2009 +desktops +chargefontfontblockquote +xuidl +computers +house +4400 +fill +flawed +by +rates +tall +satofromaddrmatchhl +checks +height3d104 +nextpart0000023823983c1f9dfb9df87 +centera +will +conducting +bordercolor111111 +sheets +trtd +bsfsc5mj3661 +produ +afomaafomaclubinternetfr +nehru +aabdb3b5cmlnahqgkgmpide5otggsgv3bgv0dc1qywnryxjkienvbxbhbnkaagrlc2maaaaaaaaa +wfzky1fgmdhlxfrh +c111ncerne +with +from +means +disclosures +neglecting +20 +money +mimeversion +0500 +size1 +border0 +entity +roxus +bsfsc0sa335 +mail3csminingorg +whirligigs +xmozillastatus +ludendorff +1255176789 +use +1900 +put +sent +words +computerfont +sender +class +asia +alagny10115237abowanadoofr +which +control +19216818251 +mitten +property +afterthought +1618 +forged +partija +size2statenbsp +nations +auctionfontbfont +httpequivcontenttype +texthtml +jonathan +trucks +srchttpc2fbpoxokuwcnspacergif +xbarracudavirusscanned +directly +30 +brl +w3cdtd +form +freebr +html +httpequiv3dcontenttype +identified +mail1csminingorg +81168116 +confiscated +partit111 +xoriginalto +m0620212mailcsminingorg +pratique +width100 +against +antarctic +ff +made +gespr111chen +very +we +add +width +namelist0203 +threaten +12440203452da100a60000w4twrl +evening +firmness +when +barracudaheaderfp20 +including +reed +contenttype +vacate +cypherpunkseinsteinsszcom +122916 +may +like +of +dynamiclooking +states +csminingorg +and +avisited +received +19216818120 +increase +xaa29507 +sentence +free +table +cage +europe +the +planbfont diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_26.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_26.txt new file mode 100644 index 00000000..14814d3c --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_26.txt @@ -0,0 +1,357 @@ +down +scottsdale +please +for +titlewuhuqoqaozuk +smtp +xbarracudaspamstatus +060259 +appointed +readingsstrongap +any +4jj8xc5jj6ihxa9bvcg7g8scbrkipfu5yahdwzxleqy1qdey6kcsm4chqb1ru0usog0u7redyd5q +mailwebnotenet +body +searcher +074304 +aligncenterbillionaire +area +name +this +052 +in +203001 +extraastrongfonttd +plate +firewall +href3dhttp31ada4fdekemoscnejm3d51990ef85a6ab65c10b3oyj3d3 +have +needed +ecke +your +returnpath +alignleftstatesp +imap +carolingian +gmailid128ba18fd504d3c8 +0001 +are +is +xasgdebugid +bgcolorffffff +managing +seminar +width3d600 +200910211139n9lbdlav015616ns2csminingorg +size3d1a +subject +custom +size3d2namefontbtd +relay +using +inline20 +contentdisposition +scientific +htmlfontlowcontrast +68144 +124384838211ce00120000w4twrl +pattern +late +himself +email +diagram +nrolled +join +yes +below +athletics +br20 +security +diocese +value7979option +532ed16a39 +contenttexthtml +cccccc +style +image +line +srchttpmta4rle2rainroleszcomkfcblaqatlifteiceaaaooa +br +cypherpunksdspronsnet +captured +align3dmiddlebfont +highly +tag +17189895746 +up +going +fontfont +so +our +xbarracudaconnect +thu +normal +friendship +short +head +magnifying +u +dispute +width3d15nbsptd +color3dffffffagefonttd +fffwebsitesatd +sale +htmlhead +type3dhidden +0400 +idea +ahover +edt +intended +bulb +o35ifnt8003983 +esmtp +boundarynextpart0000023d201c9e57a88ad7350 +sc +francebrfonttd +a +marvin +deliveredto +information +concentration +jmnetnoteinccom +32229347 +accepted +233 +tenbagger +color000000 +dqp2b2x1bwv0cmljdqoncmrpc2nvdmvyzwqncg0kcgfydgljaxbhbnrzdqoncm5vbmnhbgnpzmll +barracuda +localhost +within +toepassing +18774518336 +mimehtmlonly +host66insuranceiqcom +pdt +currently +messageid +iibmsu5lpsijrkywmdawiibwtelosz0ii0zgmdawmcigquxjtks9iingrjaw +locally +10 +xasgtag +een +downbfontbfontli +tda +125490006564b3003a0000w4twrl +xbarracudaspamscore +federal +above +letters +har +color +010 +d983d16f03 +peruser +1255176789033900390000w4twrl +taglevel10000 +xbarracudaurl +borderleft +mimeole +127001 +8bit +helvetica +files +jiggetsbars +that +cellpadding10 +ns4csminingorg +gmt +to +legislature +charsetiso88591 +least +met +washington +pgh0bwwdqo8ym9ket4ncjxmb250ignvbg9ypsjmzmzmzmyipnnretwvzm9u +waddler +httpspamgwcsminingorg8000cgibinmarkcgi +technology +td +ontsize +unconventional +around +mimwangrahs0ahghacwgaazaaf0abv3aagcqad9wbns5c8akehy5el7wawkwctsglgrqaitqnik5 +nonlocal +various +j05pnmmba8nmyjhgoilh40km6a3uy7lphrng7kq2sigbpsp91dts6lvyql81wg142uu3win +here +windows20071031 +quvfl3od6drxlv5kvd9ozji5xucxpya3e6ny7burqextdqap8ajlpyuqlirxy6e7 +response +constructed +htmlfontbig +through +effie +centerof +id +meta +positions +pts +rdns +over +heath +also +q +bccnkad3nwobfijmq729o1onnvojzsk9ramu75bhbtapqyha9kd7dapk6oqutguniwikflv +on +cancer +banta +25 +populations +tr +julissa +appears +date +mail2csminingorg +xbarracudastarttime +31 +invalid +tdhr +rob +href3dhttpqrdspellyourcomzwz +substations +fnm4ala6acj4uul1g6k7n7vzmhrgk0iyhm5ovyfmxr4ja3egcqzwkpkfoqzcmwswyyzqmw9gxq5q +scheme +video +by +132734 +however +fontweightbolda +9fde3fefef5e1e2ede99eada0b7aaabeae2edebb9a8a9a2af +struggle +library +ones +style3dcolor +xbeenthere +0100 +not +remnant +with +from +bgcolorffffffdiv +unsolicited +0 +mimeversion +ia0ku2f2zsb0aglzig9uigegzglzaybhcyb3zwxsigp1c3qgaw4gy2fzzsbp +border0 +categories +inc +000 +fifty +anyonecanedit +include +killlevel10000 +life +kst +copy +btreatment +simplified +done +available +xaccountkey +permitted +publishing +message +occasionally +its +color3dcc3366b350bfontfontdiv +place +air +7bit +jalotat +discussion +homeowner +where +icmgmi4gswygew91igrpzcbub3qsignvbnrpbnvligfkdmvydglzaw5nig9y +get +httpequivcontenttype +measured +valuehawaii +edge +municipal +anything +a01 +185744 +special +profits +thediv +microsoft +xbarracudavirusscanned +w3cdtd +form +testsbsfsc0sa148a +html +stylecolor +wide +height454 +inner +billing +relatively +upda +november +asianetcoth +ptsize3d14 +fryiuagygfii2pay +brbrthe +tibetan +16r5pmvbfkw80dztjz6g8bnf5iazvtjfgq42t7f9ogjt0qgi1vyhkrgwrwr5rkkt8mv4gzerocel +muscle +bsfsc2sa031 +n111117vel +spam +xoriginalto +example +except +000044124742000066f100007dd5jerrysdodgecom +transaction +0px +no +cartier +weekly +undervalued +past +speech +some +lbhrkgkkufgv6gmkxqryblgt1wpkownywfiju9mdquoobggrvj3m2h1lilxrvcexrs0ukaeopcu +does +marketplace +contenttype +align3dcenter3e3ccenter3e +feb +other +of +testsbsfsc0sa218 +since +ive +away +and +primary +212947 +offline +received +without +19216818120 +leadership +accounts +list +smtpsvc +wish +table +went +china20 +2006 +york +technological +the +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_27.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_27.txt new file mode 100644 index 00000000..d5f851d9 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_27.txt @@ -0,0 +1,343 @@ +textdecorationnoneforward +ideas +size3d22622 +2100000 +side +rolling +format +for +postfix +st +smtp +sattriyah5 +size3d2thank +102134077 +hrefhttpwww185wiildaccesscomdownload +any +managed +body +area +this +oregon +instead +in +pjwvcd4ncjxwigfsawdupsjszwz0ij48yj48zm9udcbzaxplpsi0ij48c3bh +comoros +transitionalen +namegeneratorhead +needed +your +returnpath +nov +suspicious +0001 +breakdown +is +mon +zfqm06mneuxjnnrtyjelltbhsiwxg5zuqdkmpambr3fsospgau9gqifuxt8uqyzav6u7fz2gmxr +xasgdebugid +officers +60029002180 +account2 +00000000 +valuesasaudi +subject +o2d82tw3019774 +custom +using +kbqcc2ztau6niyu61cexdq2dmexybjnkewuhg2ewxzanisgspasoakhjqu8vqcznufjmnihgmyls +system +212745 +surface +2002 +research +commission +ia0kicagicagicagicagicagicagicagicagicagicagag91cnmupc9gt05u +aligncentera +fontsize +solid +yes +below +ns1csminingorg +future +alt3drendezvous +url +officialsin +contenttexthtml +obliged +hundreds +confided +image +ist +if +tax +good +br +madame +palo +up +seek +differencethe +xbarracudaconnect +t +newsletter +keep +22 +led +tue +head +styleborderbottom +centernumbers +cladists +vouchersfontatd +attended +50 +native +height3d217td +ratesfontbtd +1px +name3dsubmit +price +borderright +multipart +hrefhttpd3979f118a6fzyhuhscn +full +ahover +edt +just +understand +it +site +sa290rn +width565 +sun +esmtp +centerin +24 +two +a +deliveredto +receive +barracuda +regimental +inline +because +rule +color3dcc3366b199bfontfontdiv +messageid +xasgtag +mime +width674 +text +0700 +appeal +aekawacsminingorg +or +public +char +daytitle +jyh5ax3y3jmclflse6hvwr1dzzduhwrzdlm9vrwat8y1egzg7tq6wbgxegmnhhcbchm +010 +taglevel10000 +trimg +netscape +anb +093629 +eyesstrong +contenttransferencoding +193120211219 +xbitdefenderwksspam +installationfontp +lions +importance +width445td +127001 +hrefhttp4b1asteadquietruvybilozaucdab4e98736c +8bit +charset3dusascii +border3d0 +href3dhttpwwwthezscom0003 +ns4csminingorg +wounded +quarantinelevel10000 +remv0615cbkru +to +stylepadding +gates +existing +took +singledrop +brleads +approximately +you +httpwwwqvescomtrimsociallinuxie7c297c134077 +disapproval +technology +around +pdepriver2420521191rnocrescharterpipelinenet +size3d3 +hedotuot +planning +font +here +colspan3d3 +blcmp +payment +change +historians +6 +016d67e76d4c5225e4b57ab08de4jeramx +thomas +at +shorter +id +rightmargin10 +titlenews +need +surbl +width3d1 +on +17 +31108086 +height3d361 +06 +defaming +tr +interest +regnetdichotomizedbolsasdicamentsrepairedefficacerellenadoremarkableremixespiosresiduesvorschriftenrendererrendirrenzyvriendschapreprepitiendoweigertreprocessedgracchusresolveudirectrixresentmentsanrioreservadadirigidadisallowsubroutinesdiscardingrestrictiverevenusretrasowacooldiscussrfcdjustwalgreenrindisheveler +14 +turned +date +viagra +021 +tank +charged +procurement +shopping +2010 +xuidl +promotion +flawed +untilb +by +gw1csminingorg19216818250 +antispam +203 +masks +8500 +nugent +185844 +181845 +with +type3dtext +from +kunt +mimeversion +scores +tdimg +width3d435 +administratively +000 +promotional +srchttps2redeyeemailcom80storestoregraph54509597042gif +023515 +killlevel10000 +pass +xmozillastatus +booka +oaipuses +sa275bhl +xaccountkey +titlezapp +sender +junk +yyyyspamassassintaintorg +uprising +mohsin +influence +its +19216818251 +almost +hrefhttp994emedgaspard24aruuaiwiaaq1c3c22a5e86 +bsfsc5mj2022 +37d60f6e688115b15863026nmzodjeteh6411613266 +ohrt90feqflw3i5ikdmlphzp4wbljuqs0bjksaf54qeuwsepoa3awysrzzwvo1qybc3lnlst +lespaulbhotmailcom +account +get +excitecom +msright +colspan2 +texthtml +rules +changing +edge +booksfontfontfontblockquote +smokestack +flpeie0trabxljq6spit5iphgib7pgqt2wcptgvhl2ajcspnalw5cdsrdvtty93t10n +httprdyahoocomhttp80717188asianaid818932 +xbarracudavirusscanned +brl +2136 +xloop +7 +jaski +n8aav48kjjkfymwdmyfbz58vsunf8zzvsmefprxtjef4sur7d2f8addwnszzq19p7f5 +awq249neoplusadsltpnetpl +jeeg +hrefhttp0314bgeyozzncncuzubegiz812661sl3w5jt762p05h6rqwenasocite32476721001939642710161logusobe9849580118z859765k98l136kizyqu423651015299591373164click +html +size3d2computer +moss +160832 +federate +tr20 +mail1csminingorg +m0620212mailcsminingorg +colorccccccfontnbsp +95nbspadobe +0900 +aware +aligncenterthis +messnum +ogxkqpzfz3auda8fnlimnpzuip8awn73eudesfq7vzhzmti3gfsuhgacjetu3b5gdvxkfitcdsh +size1shipping +3 +no +we +nupep +a117100it111r +geneva +nbspwe +score791 +10877021 +contenttype +xmozillakeys +may +10997p +migration +might +106160185195 +other +of +and +conservative +hrefhttp04bd9axagonoqcnoyvyw1634943328913ug79n7g83uubofih79579335184697095905104 +received +entitled +bespanbspan +parkway +otdpxu7xbxrbhqlacvf6bwfvndwtw3uilt5ukjprlorumk8ptn1lsbelubeywxyuuubvvkdtt7iu +althea +wish +table +recession +needp +month +helod +view +oqoaenjl8quavghuhkp8awpxrtuifubd4x64watgctzkeusvxefmu5b9r3rrdi8qku7h0opx3r +the +those +only +sol diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_28.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_28.txt new file mode 100644 index 00000000..92558dd4 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_28.txt @@ -0,0 +1,354 @@ +74xaappit6nfcwcfh9ltmqfwl2meitkayjczjg1vqddwcqvjb4i61xv8nuoo0jefsm1ojrr +11597me +abbe +xmailer +for +varsity +222103 +stock +askedfontfontblockquote +knptpv5fmyyk01jewwz61sf0nbvqa3brkeczfuxo2q8qgzm2qip6dbqj19oxqszzpwkmr +network +msgiddollars +xbarracudaspamstatus +usabr +construction +friction +networkp +any +12422172182by +xmlnshttpwwww3org1999xhtml +strongall +valuepgpapua +body +years +area +d0b4d7d6c2a5a1a2bed3cad2bacdb9a4b3ccd5d5c3f7a1a3p +this +in +6135221168 +professional +transitionalen +have +your +returnpath +quotefree +suspicious +is +hems +his +why +versiontlsv1sslv3 +tfoq4ifn70qefughsd5x6vf0ijxfxkrnbwkoydfwrx8nqht51pqtvx7apoakfxodu0swhyd1pij +companies +100 +00000000 +influenced +subject +121 +62dvklb4ecgdvl6l9kbxl7zyuyfrgulq7prcxjyfuktowyfrmjqp96hw6noxfqx2q75f8arze +o126x70f008326 +using +size +listarchive +than +373 +included +quotedprintable +but +bsfsc0satfamsbl +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +11 +below +ns1csminingorg +poojafernscsminingorg +security +url +duba +afrikaner +would +strongsave +colorff6600bgratitudes +nonepayment +intent +online +if +br +19216818250 +big +our +pyoli4ml2acigqpajnrd13xtaox4jlzv1y2w13uvlczsgalg1361dx5fqici2qwiwzqn2yrn2dky +xbarracudaconnect +250 +normal +all +tue +empires +print +re +ffffffreviewsabr +055 +1px +xmailerpresent +htmlhead +borderright +cellspacing0 +0400 +full +emaem +inci100entes +edt +stylefontfamily +fontweightboldsaleatd +width3d60 +understand +ns3csminingorg +esmtp +whereupon +dont +re102lect97nce +award +courage +deliveredto +icagicagica8relwigfsawdupwnlbnrlcj48qj48rk9ovcanciagicagicag +congress +height +localhost +strongthis +within +serif +harris +4px +div +has +regular +messageid +silva +10 +xasgtag +cellpadding0 +industrial +wisdom +perfect +divspan +account4 +deathtospamdeathtospamdeathtospam +115qu97re115 +public +london +color +010 +they +ljye5dbxrginsih5nrhmwdycogownpuz4iw4s6tjosa2nskykaybqigajxoukwsgpeakwi +taglevel10000 +xbarracudaurl +contenttransferencoding +multipartalternative +e1955f2acc3814ceesjajebtac6d4c3wwwfacebookcom +144119 +service +zzzzlocalhost +importance +127001 +kingdom +committee +work +styletextalign +crude +searsheatsystems +xbarracudabrltag +to +getting +charsetiso88591 +000459 +httpspamgwcsminingorg8000cgibinmarkcgi +ejthmqeruikawyazfigfeewhjpjziebsigqkagsficbacgznceyyaqs0ggfz0hw1abibafw0sc +teens +color404040contactfontanbspnbspnbsp +bgcolor3dffffff +fontweight +home +font +here +c111n115t97ncy +there +name3dgenerator +legal +consumer +msgidfrommtaheader2 +xsender +added +go +1862 +sa148a +import +id +confluence +286 +width3d4 +pts +need +coworkers +transfer +transferred +border0fontbfont +htmlmessage +tr +060026 +b +n111t +townsp +xmsmailpriority +altblocked +date +instant +mail2csminingorg +styletextdecorationnonehereafontbrp +xbarracudastarttime +engaged +who +largemorespanafontbp +2009 +hunderten +xuidl +hrefhttp8642fsponevohcnjp11d2ff2653f3bb19d8a9aqk008199133291771685656 +them +mail +14195 +scheme +wenbspwill +by +rates +split +gw1csminingorg19216818250 +client +antispam +isbn +cool +htmlwithoutplaintext +trtd +while +their +p +software +28003200 +not +pc +with +from +anderson +mimeversion +evidence +jayerkvtwcpochdfluxpdpxdcgqnld5kjtn1dpaynlpjfe5vtgoqfmjcola8sqyc6tprx0j03x +stylecolorffffff +height329 +first +awes111me +media +killlevel10000 +linux +both +nav4 +use +gens +xaccountkey +e118en +message +00 +ascended +asia +and20 +mailtosocialrequestlinuxiesubjectsubscribe +10px +pbayes +its +under +accredited +labyrinthine +ffffff +3ccenter3e3ctable +sincerest +c +roman +theyre +rights +uid44501201487268 +rules +31103662 +million +quarries +kind +282165498 +more +tonsure +denied +hello +brl +religion +avail +mailing +z8o3a4rjwmuttp4zjftr22rnttrgvkby4fxtwoigbioimztcmjorzsyht5ip4u35wo1o1wjf6rw +81168116 +pmwuqffx60lkn31tahvyjba57u5voettahwppshzaebwfmtddjog4pq1uyqdmcabv05czbxue6 +ro +xoriginalto +align3dcenterb +0900 +engineering +width100 +light +nextpartvxklypzzzdwwhofz5e1rz +athletic +vessels +barracudaheaderfp3043 +very +we +earthquake +anda +gmailid1288723aca21d4db +rugby +color3d000000save +monthly +k +mailings +g4s9x9e32410 +mail1insuranceiqcom +turkishresident +value +undertaking +trusteddealer +when +fulgencio +jcjreemsncom +day +contenttype +america +freeatd +offices +sonjamaajonl +may +listed +g6pgtwc05929 +of +dynamiclooking +divnewsletter +and +indigenous +percentages +nbebfve3023203 +received +one +color404040subscribefontafonttd +19216818120 +name3dnoknokr2c2atdtrtbodytabletdtrtbodytable +free +classmsobodytext2 +ga9dbgsi3mfygvzzkneo8ppzjjgt5duge75hfpjvy18wkpungcslwglpcfl0wo9ueqqmqkf1 +accessible +chesnels +faceverdanaarialhelveticacant +c0wrg06dkmudwwp0qtisrsag0lsrnmbtokuxpkwx61tabymwywq4fxlntb7eqpixvkh1u1yiica +the +parts +those +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_29.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_29.txt new file mode 100644 index 00000000..b57ec8a2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_29.txt @@ -0,0 +1,350 @@ +h2big +pmitra +divfont +xasgorigsubj +1980s +crfontfontfontcentertd +xmailer +for +postfix +d97n +network +premier +nbspa +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c09jpg +contentious +huge +this +death +ip +instead +in +113uiz +transitionalen +have +heard +returnpath +460250211 +0001 +strongspan +why +rvbpulrtifrprefzieforcbhrvqgu1rbulrfra0kt04gwu9vuibst0feifrp +smtpd32606 +makes +23 +account2 +smtpsvc5021952966 +reputation +bsfsc0sa290rn +fewer +subject +4 +given +targetblankprivacy +using +system +httpwwwbarracudacomreputationip1872387149 +included +2002 +sweepstakes +previously +solid +busta +thought +inanamayacaksiniz2e3cbr3eherseye +but +state +whether +below +future +best +undertake +url +replyb +contenttexthtml +mitchell +suitefontfont +switche100 +v30m +ds5izsbszwz0ihrozsbzdw0gb2ygjdgwig1pbgxpb24gihvzigrvbgxhcnmgaw4gy2fzacwg +forkxentcom +fontfamily +ist +if +center +19216831 +problem +description +oct +bsfsc0sa392f +mileage +rdnsnone +neither +tag +up +marginleft +xbarracudaconnect +archived +southwest +xoriginatingip +do +tue +head +fit +spinetail +club +deliverydate +entry +cut +img +consecutive +set +yyhstsb0j1gyzu7cjty8544zeelplnaaukfpxjptvdcx2bax7zn5ubwv7jihd7bd83scvuu4g +13 +16200 +dollar +esmtp +sheet +893893 +24 +sec117re100 +local +determined +arrangements +a +coldfusion +width50 +deliveredto +xmozillastatus2 +brings +grasp +jmnetnoteinccom +hibodycsminingorg +accepted +river +as +dogmaslashnullorg +remains +618011096fontp +store +credit +pdt +has +messageid +pfont +apprentice +350 +objections +diploma +500 +10 +mime +vocalese +align3dcenter +tda +men +xbarracudaspamscore +color +come +1256211881 +they +whats +a8bfa0f1eafbfcfbedeee8a8a3b9aca9baf7fcf5f4988a8588 +xbarracudaurl +exactly +contenttransferencoding +193120211219 +reply +mail1001 +brand +textplain +127001 +8bit +avoid +21021494112 +19412514545 +that +army +middle +saving +to +sep +bantalcsminingorg +alternative +steep +charsetiso88591 +maxlength15 +browsera +code +cellpadding3d0 +you +journaltitle +ifpoliteru +pto +company +around +bgcolor3dffffff +tradition +woman +planning +these +n0l3mfzobjttdvgojg4x482zvglnpvfc3hbtfjlqxxvjf9tudpfvkoijlbracy20xp0jusobfmt +internet +was +repair +version +brother +dir +replyto +xsender +id +dollars +collecting +ratio +can +external +facearialstreetnbspaddressfonttdcentercentertd +cerro +brwhy +1phrmd0hktytoo4e5fautdctafqat36dwnmj2iozz3sqavnkdaggxzlafw04eaxyeeupcyxpkmp +tr +focus +chambers +government +complete +233603 +relaydubt31nwcgroupcom +date +dominant +2010 +offering +thath5 +rendered +xuidl +simple +male +okidata +by +leveling +il +assigned +he +however +posuere +new +link +epcsvqlscamunqlsmnw1ca1s8a2pwaae8ayhfwl3zwugfqntlqw9uanssameaazkamx4aticamx +style3dcolor +xbeenthere +0100 +again +september +p +lifetime +bordertop +with +from +celmer +team +0 +bcc +texas +taa03135 +mimeversion +separate +give +pieces +006a83e56e0c5852c1e85bb85ae3humcbb +value4040option +both +pass +238 +address +could +bring +gw1csminingorg +charsetutf8 +faceverdana +british +scappaticci +xaccountkey +position +members +cols3d1 +anyone +esc110329738836411025989487446468inrovingcom +p0olqpgtmbgo9ifnp8ay08doxkrgyo0we2kolker0oxgubikaeok0o4x0fnf7uooaizlgssm +almost +b3vnacbvcmrlcnmgdg8gyxqgbgvhc3qgz2v0ig15ig1vbmv5igjhy2snjy4n +an +025105 +7bit +19216833 +financial +excerpts +be +into +roman +get +rdnsdynamic +02eb78a7e3f +texthtml +12hours +constantinople +guess +httpwwwgeocrawlercomredirsfphp3listwebmaketalk +1932110245 +080122 +shuffling +romanband20 +sa +html +platform +wh +xk4nv7cqsidb1bh0mrvefur1mnxk4t6q9qnoq7cefzdlmetwop7bv3fli6n5jthwzmf0trfz +many +025201 +150 +types +call +1860 +class3d +color3d22ff0000223esanayi +content3dmshtml +occurredpine +resultsfontfontblockquote +jst +day +contenttype +xmozillakeys +east +listed +might +of +srchttpwwwprizeinthebagnetimagesrvmovieoceans11c07jpg +havent +england +dynamiclooking +and +cheapcializ +received +lightweight +windows +19216818120 +suomen +slacklnecom +days +deleted +phone +saa05985 +table +corps +earn +sa154a +the +classmsonormal +cant +xpriority +parts +those +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_3.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_3.txt new file mode 100644 index 00000000..78f77cf0 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_3.txt @@ -0,0 +1,386 @@ +xasgorigsubj +species +cellspacing1 +please +xmailer +idaho +format +textdecoration +for +ct +replica +tapes +s +find +business +09a +treyrefillsru +smtp +9px +people +cmes +aligncenterfont +jmnetcomukcouk +body +years +area +jeffersonfullscale +name +this +size2a +in +pushing31csminingorg +hrefhttp7deskonibutcnazosifimci576h4o93gt36976o763698a +your +returnpath +manner +düsseldorf +0001 +breakdown +is +radiation +100 +account2 +pricesfontip +symbolism +arrang +aawhnoaaaaaaaaagqaokcxlams9bksist4xva6ammsp5ryulfzbwveb0hhoksbkkn4hyuvltjdqo +200912111649nbbgn4oq019048ns1csminingorg +subject +hibodycsminingorgfont +custom +collection +using +am +system +mailer +anybr +bfont +testshtmlimageonly16 +included +emthatem +research +targetblanknewsletterssubscriptionshtrdwbrcomastrong +base64 +email +david +httpwwww3orgtrhtml4loosedtd +yes +but +state +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +below +102313469 +apostolic +argud +contenttexthtml +what +different +excessive +koi8rbig1hdhrlcib3agvyzsb5b3ugyxjllia +pplayerabilityp +0102 +nanotechnology +style +ist +description +132635 +bsfsc0sa392f +up +going +height314 +our +xbarracudaconnect +do +tue +head +hand +src3dhttpiiqusimagesindy120020610101ajpg +50000 +domain +had +targetblankmacon +img +f +bvlgarirolexdior +uybf0qfq9xfsbhbwxlwjjcg9erevdk6tyolwyt1mk5wtypmp1rc5mpowl6g0ntmgmezhpwiyz +harmlessly +partei +title123turnkeycomtitle +krmv2vy3zvvxtd7hguwn0kcfaomkc0jx0yszzmqojqd7mjpwmq6qujgwna9xgzf9y6j0p23 +novak +exceeds +whitelisted +alignmiddlea +it +knowingly +111601 +united +esmtp +sc +rgb207 +893893 +onl +a +websites +deliveredto +jdecfhkdx4bdsodeq3q6zeckfyglxof5y6x6rzuyp6jfyqgwzozsfzywokfk7o6xlhqr +information +xmozillastatus2 +alt3dpmg +35 +barracuda +localhost +2009title +ihn0ewxlpsjtyxjnaw4tdg9woiawchg7ig1hcmdpbi1ib3r0b206idbwecigywxpz249imnlbnrl +fed +well +5 +she +messageid +350 +10 +reunion +atd +moode +especially +emotion +resortfontp +topmargin3d6 +14720000 +xbarracudaspamscore +16pt +httpxentcommailmanlistinfofork +or +letters +public +questions +textdecorationnonemobilea +010 +taglevel10000 +provide +far +english +quarter +reply +httpequiv3d22contenttype22 +service +127001 +out +xmras +styletextalign +markers +middle +albany +to +v1 +quote +pd0vpvt7n5dndpbzxvkxyvf7n0u2bwuxsl2yl1fovuvvb6j66vgf6dp0zlomoynasilucn +charsetiso88591 +singledrop +expand +g8pkxec12677 +key +19216832 +code +cellpadding3d0 +you +jnj75oyotr76axpajstbsdkaxqeyui75dtqen2p2yd2ep7k3mrirxhb096fb2gdum7fw1ewa +td +know +color3d666600afontfonttdtrtbodyta +carbonate +around +idx0000t138 +chat +maximum +httpwwwdigitalrivercom +360 +eppsnetnomadcom +these +internet +version +believe +uid106971201487268 +84h97nk +variety +at +aycw2a8e4ojqiurts7uurgs5ojxuiot1ojj4yafibkg5p27nv92d1pxkom0crublmlzuhmz74p2 +law +id +3c2ftd3e3c2ftr3e3ctr3e3ctd +height3d63 +freddie +rdns +also +fax +can +surbl +on +designates +height3d114br +education +600px +government +how +lightsensitive +detected +023c33c46c3c5555b5a10ab57ad8alpasn +date +strongperiod +021 +jydrbmv3jycgaxqgd291bgruj3qgd29yay4gsm9kesb0b3rhbgx5iglnbm9y +hrefhttp8ad48d3d6ef183pointdoesru +xbarracudastarttime +31 +bsfsc0tg035a +7molnv5sp5eyhbcqajuzagvmdghi6cgczazjkuy2ddv7vyygkp0pkusrofqyudkwtsa1jsiblmp +value3djerryliversaigannuitycom +uizygfxb9khus3e3y6xpfmbciw93efuskn3161mthfr7og6ocyncqjj1k3sogpb6vp2ujiek +o1mmtksn028028 +2010 +xbarracudaspamreport +folders +2009 +mail +commercial +helped +hrefhttpku12thestrongstoreinfo13820250f3c745790671htm +manager +by +yisr2rhyetl3bkk23x8ax4ib3x6v44w89c768jjyya8oyux6ezxztm37m5xcasyaseyr9a +he +unsubscribe +selfrule +even +status +hrefhttpdrbluehornetcomphase2survey1surveyhtmcidjtveeqactionupdateeemailhibodycsminingorgmh4e73c3d93c4d7a41990b4238b47fedd3unsubscribe +antispam +sporting +expect +will +width3d247 +few +hazardous +receiving +trtd +nehru +p +cause +addr +hrefhttpdrbluehornetcomphase2survey1surveyhtmcidjtveeqactionupdateeemailhibodycsminingorgmh83e256ca7e16ed7a9d39f54c68e290a9unsubscribeafontp +xbarracudabblip +with +from +instrumental +dwl +still +0 +mimeversion +scores +border0 +dancers +valid +haringey +titleinstitute +valuerussia +levitra +2 +mailnetnoteinccom +properly +killlevel10000 +panels +region +001429 +both +xmozillastatus +address +copy +brief +charsetutf8 +fetchmail590 +prevention +permitted +grab +which +032304 +latin +place +altthe +attributed +7bit +pr0nsubject +cc +dévaluation +poems +httpequivcontenttype +winfrey +optin +texthtml +size3d1v +rules +261 +hlduejgyhqbe3ldumrqjxmda9tb28rqhlduejgyhqbe3ldumrqjxmda9tb28rqhlduejgyhqbe3l +fonttd +quam +hv77a40ickgq +several +xbarracudavirusscanned +brl +w3cdtd +neutral +130stop +color000000bfont +form +sa392f +html +inner +bordercolorlight3dffffff +discount +includes +fri +emails +spam +simply +ind +settlement +no +we +approved +content3dmshtml +devoted +when +barracudaheaderfp20 +sansserif +contenttype +xmozillakeys +husbands +href3dhttp981e9evqificancnjql3df06301bc2f23f8cc117c885j +cardinalate +may +other +of +style3dpositionabsolute +and +received +setup +one +bgcolor3d2200000022 +por +aw5npsiwiibjzwxsugfkzgluzz0imcigym9yzgvypsiwij4ncjxuqk9ewt4ncjxuuj4ncjxu +spent +width214td +casey +days +28 +table +q8a3myp80fkuhy8sl9p2awxrpp7u5rs4pfvfhiw7s2petj9hfvfubkf5owdscx7c +surveys +152318 +hrefhttpwwwmedcenterexpectcom9747hibody +the +df6bf16223 +xpriority +parts +administrative +boosting diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_30.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_30.txt new file mode 100644 index 00000000..b3db2615 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_30.txt @@ -0,0 +1,338 @@ +s111me +xasgorigsubj +please +trouble +format +1950 +for +le115115 +unsubscribea +school +smtp +xbarracudaspamstatus +sa154 +door +ip +in +guinea +mortgage +firewall +practices +returnpath +jul +breakdown +are +is +o1gdkwiu010677 +guys +xasgdebugid +god +companies +o5oaghf1mev7xlkwcmuahy2eu4xzenlwbcvzcoh4bby3d1xlbgjwdtuw37y9v78a +account2 +00000000 +size3d1a +stars +mailtoilugrequestlinuxiesubjectunsubscribe +135 +subject +width3d538 +already +smd9ypi0yswud8txk2tqxpvazlcxtyv724oprtii5r1m91a2wlnclvug5rgutxulhilqlrnr +system +contentmshtml +landowners +included +imo +email +yes +but +solve +below +jump +url +impossible +suuurreeee +501px +online +fontfamily +line +if +center +good +srchttpwwwselebrettyru10gif +19216818250 +problem +rdnsnone +own +ypsb32lsgpzrgcqb19twa0ir3mi2n0aqlubpvumt0chcgjr39khksjvy0ylxltbgn2el +face3darial +our +newsletter +asgard +minute +head +stoney +look +tellc +trial +successfontfontbblockquote +0400 +multipart +144332 +edt +score +united +esmtp +bunsubscribeb +heathernetnvnet +new3e +rhythms +a +2165419436 +loading +9xo1xkkklngljyu42aayodyrtzfg6cvpgnhcchfdrv9jxpwfapyb5ul1v1l8yzlldme35tww +deliveredto +20pxfontsize13px +12557011680b2a006f0000w4twrl +xmozillastatus2 +bsfsc0sa119a +align3dcenterfont +barracuda +employer +section1 +muller +earlier +1115682 +contactthe +score060 +div +rule +mimehtmlonly +store +size18px +has +messageid +350 +locally +hrefhttpabsexyhumbleruojujip19d78fcd9fe1 +10 +industrial +storeyed +old +englishspeaking +determining +hrefhttpa2drugselliot90aruuzosuehid2b77876fd +doctype +main +color +010 +medium +peruser +taglevel10000 +qualifybrbr +exhrlsvrgemsbarbadoscom +xbarracudaurl +multipartalternative +photoelectric +borderleft +zzzzlocalhost +textplain +127001 +promotions +that +out +styletextalign +helouswsflist1sourceforgenet +producing +stylepadding0 +concern +to +naf +become +fall +charsetiso88591 +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +0c5022c65d +weightl0ss +083548 +width570 +small +boundarynextpart2rfkindysadvnqw3nerasdfcharsetgb2312 +1865 +n51jifp4022489 +moneybr +srchttpi42tinypiccomatlrg3jpg +these +help +internet +score458 +inreplyto +094622 +ucoa +second +msgidfrommtaheader2 +at +1 +replyto +been +added +go +low +acuoyloagiol +id +meta +ip2416321087adsl2staticversatelnl +on +c117ri111sit +sequence +tr +omission +date +fraud +erected +g +xbarracudastarttime +april +who +width112 +orion +2010 +hrefhttp72cfnodoruccndudoky3c7351c62216edb9eaf5efcejjhoql274231831104886586939978zmhvbgojprkjo +style3dfontsize11px +hope +third +2009 +xuidl +mail +fill +usa +by +600600016825 +georgetown +facearial +selfappraisal +gw1csminingorg19216818250 +listmasterlinuxie +thats +style3dcolor +few +receiving +trtd +0100 +lived +trusted +with +from +dwl +uid131321201487268 +0 +color3d +administration +mimeversion +evidence +width478 +petersburg +august +inc +policyfontafonttd +first +combine +rowspan3d3 +c2131715213bjarenet +xmozillastatus +could +use +put +rolex +size2 +decimus +funds +message +00 +unzaecsmjfv646svxeqthz2mesv9toyqjkvzjnx97aqplatpdeivzehepnpvnirtxvtfj82byx +financi +to423xfit32ulhcezzzrrhvq6sjdr7bslvxwksejee6kagxsccv2xt7s6jy5gyuvuq1ukw5lwruf +album +under +control +aligncenterhurricane +five +offer +forged +aug +19216833 +be +grew +013046 +aolcom +get +dcom +psa +125 +padding +rules +always +luba +marry +microsoft +receivefontb +région +assistance +html +lender +acl146neoplusadsltpnetpl +greater +close +288 +none +meters +act +contains +hrefhttpwwwbufunoceruulutoa69602a8b9c +xoriginalto +0900 +3198870 +predators +a20 +3 +during +no +we +important +battles +366 +chf +textalign +mail1insuranceiqcom +japanese +anastos +o3q8nxjs010416 +093 +jst +film +sty +contenttype +may +like +won +of +csminingorg +record +received +19216818120 +free +list +28 +table +complex +rootlocalhost +practiced +heaviest +search +the +xpriority +color3dff0000hostingvccomputersiefontafont +those diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_31.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_31.txt new file mode 100644 index 00000000..f16a4ed3 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_31.txt @@ -0,0 +1,342 @@ +alt3dplease +colordfdfc1 +xasgorigsubj +health +colorbluevolunteersfont +papers +rzbyisgmwxqcytabagafp3aexnczu5apzxobjlagjainnaakckburkcdtamvemvnaihqafjqbt +for +postfix +jury +itifont +width3d +business +services +xbarracudaspamstatus +width3d95 +people +li71 +lead +aligncenterfont +vextedit +recognized +wasnt +area +this +in +provided +transitionalen +removal +your +returnpath +hrefhttpfdthedrickerectruywafodabif05fecf0efa1cb3 +are +is +mon +xasgdebugid +versiontlsv1sslv3 +19 +accept +dyahoocouk +account2 +070 +location +text000000 +subject +hrefhttpc2fbpoxokuwcnemqxaku2743e57decfd4aef63302dudyalnhibodycsminingorgyxjxipi1422834888982689422018 +custom +independent +scategorytype +originally +size +system +scientific +included +lowest +fco1ncktcbdr70x2y5pelijcy9qhizua4igau9iaqfl1j5pwnbacgbueuu2shjntfp20uarmy0z +history +tdfont +portland +email +placing +solid +320 +yes +15 +state +uk +below +94304tdtrtabletdtrtablebodyhtml +correspondencebr +200341 +forkxentcom +h5as +fontfamily +tiputil1corptipreleasecom +205638 +founded +br +apr +congo +mx1csminingorg +xbarracudarblip +so +our +thep +thu +appear +tue +nzetodc1dqonckzyzwvgdwxsvgv4da0kdqpqyw5lba0kdqpzdwjjyxjpbmfsdqoncmx5bxboywrl +burtonp +hand +i6illcsxhi0zxzz775m8b82mbss036v1cr6f7gfqetwpo1fqfx0wbjpopw4fuk6l29g8q62hxt +book +sciscientificdirectnet +played +value75000000750000option +fairings +1area +multipart +ahover +regulates +llyovalyaolcom +it +site +united +size3d2phonefonttd +esmtp +reading +inp +a +nonmarxist +omega +job +information +xmozillastatus2 +shocked +create +hibodycsminingorg +9psofyd3p9ld7fgar7vt7kh2hd6fzqxyak70yh9h3en2upswacvu9psofyd3p9ld7fgar7vt7 +localhost +as +inline +neprestano +001635 +rule +load +messageid +bantaboobycsminingorg +0909tr0909090909090909tr090909090909090909td +500 +10 +xasgtag +cellpadding0 +mime +iezjtkfoq0lbtcbgukvfre9nicencj09pt09pt09pt09pt09pt09pt09pt09 +0700 +hrefhttptss3applycallcome25mlfeacxeblcniwaiaztiezcwllasidaaedneoxeiaweooeiimg +yet +hypertension +4924161298765444333 +public +greece +backgr +engelbertstr +doctype +tournament +contact +total +z6ssftzyozdl71ntxep79dt0mddz7c4odj4pd48lq7y8wp7czv6cuz8dcwmny8tuyprv +hrefhttpec4d1diwohevcn53j515k08687iw1sejfp90r5ywoazihanhibodyhyxoyso +010 +taglevel10000 +81128112 +xbarracudaurl +nonehttpwwwfacebookcomninboxreadmessagephpt189327550468mid307afe1fdf8ad7a7f1c6c0617661bf8bnmfacebook +rnqjrmlh0n9ddgx9f8at1x0ycmtngpeb9nbgpvho53xv52xf8acwv7yjyf1hy8zibj +reason +hrefhttp81154geniusliftru +ctspresumed +textplain +i +kioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkioqkio8qli +committee +emwriterem +quarantinelevel10000 +width134 +robert +to +sep +000f01cabb9238c3dd20949e1d7d897b7b084a8844evcpwb +dirttrack +museum +lliq4okelextvwvss5egz1yjcij9uvvzl69lirwkdopnboozxbicf8qzc1nj5azknjwl3dmcqv +you +laude +401en +company +alchemy +lfctfen +department +youve +here +help +personal +ht +at +en1argerpenis +id +also +need +boost +caunt +asked +htmlmessage +zulu +drugstores +date +mail2csminingorg +021 +lottery +31 +representative +corrosion +who +maeda +receipt +reservation +engineers +xuidl +order +mail +spanp +by +234142 +wholesale +g6nhhphy090509 +improve +however +4odoeusrgdlsepbpxrwata3zbobmokqdqtgn7urawydjy2auz61cbltus2ptmprclcwayma94fp +bbsayencsminingorg +fujiwara +receiving +trusted +f97cebook +not +listsubscribe +ms4ymi4xmdevaw50zxjlc3qvaw5kzxg1mju0nc5odg0ipjxgt05uia0kicag +pc +with +from +2qgaldjtzwmmxxiruqixnvvamnfx4x8ookunviscinzfbgd0oaolrgqdtft3p79dxtw6cgqo4op +mimeversion +preaching +src3dspacergif +luciano +000060c62ff6000068bc00001cd1alabamacojp +x1b427zpeiyo2dyrng6zwndx9wq9wzk4url3slx2224vuydrn1mfhenuxln6hs5b9onl +2 +fontsize11px +media +discounts +magnetophon +mail3csminingorg +xmozillastatus +address +nature +use +bgcolor3dffffcc +charsetusascii +backgroundcolor +available +namekeywords +tits +21618927135 +sender +end +permitted +members +influence +pbayes +a82uruejr2cjwd8na11hwrf5gst7ku3nzb3eglnfw4x3hlhhvtuurjkd00fgy4sifuz7ifzp8a +which +almost +vb3ygfwnugvasb3k084wy0g5jxnsk0ban7dev93iu1smxz57y4gk1tpih2ekb3opq6cgn7rn +an +ung0ciqjb53pppoepx6mgwkm5j4wlxgm5nzs5pyrdblhhewiudrgulgslbvluxmhrlmvuo2xhh +webmasterefiie +herea +contrast +h97ven39t +be +alu1lnth3yahur64x95cp9eobvgdrcxunped84df3awrrvz7oblvvkih3vvqzktoy2sst1jjqu +viewing +into +nations +083815 +mutuelle +cou +fine +predict +texthtml +rights +padding +stylebackgroundcolorfbb034 +xvirusscanned +archbishop +more +abstracting +microsoft +1010010911 +w3cdtd +16th +ruffled +typically +assistance +html +contains +save +xoriginalto +soldadura +thisbr +150 +types +forest +width5td +pan +color838285br +sansseriftotal +mailings +does +digital +qtimydot +great +film +contenttype +boys +representation +may +661784619 +officials +of +csminingorg +and +hyam +received +setup +opp +19216818120 +124744 +qualité +table +cases +living +helod +addresses +ufkky6fwrxmbx8gp1npxihpws3jcuxodehifqpaxjjjx6ds1zzjpqzkodjn1quzwc5zd34q0pujf +the +202555 +1300 +parts +those +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_32.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_32.txt new file mode 100644 index 00000000..388bb0c2 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_32.txt @@ -0,0 +1,352 @@ +boggs +size3d15brbrbrpto +ideas +width85 +size5a +enable +121001 +moreover +please +height129 +chief +for +hereafontbfontp +spfneutral +network +winner +smtp +xbarracudaspamstatus +shop +literacystrong +di102102erence115 +people +1193 +xmlnshttpwwww3org1999xhtml +b32si3112301ana20071101095540 +body +name +this +death +homecbipccom +in +oxidational +smoke +based +firewall +your +returnpath +jul +exercised +0001 +is +mon +investements +rd +stylecolor5f800dclick +xasgdebugid +00breplica20 +19 +inconsistent +1859 +213105180140 +nonmed +reputation +subject +parker +relay +wto +using +40 +produced +href3dhttplykoqljunpujuwcnajk3d +1030135594011692caramailcom +included +email +islamism +solid +shbig505413body104 +throughout +andnbsp +emdynasties +future +mia50hotmailcom +jmlocalhost +insurance +text3d000000 +marque +intent +cjqqtutw2mmqfgkik4bieaeqgq4emrhvtaafopbpybo60bjuhydyiaaqrbzuw5jvkdg5uu1bcd +typeimagexicon +online +8220stiffers8221abr +ns2csminingorg +fontfamily +style +httpwwwbozombocomhugetitsindexphpaffid1534 +image +ist +center +big +designed +duep +our +german +xbarracudaconnect +snowman +newsletter +do +w6radvfxp5fzvbqv83x8f4jspmdb53ahjmwcpwdpwkvterbanvxpelkmwvlnmgepv9 +head +colorwhite +domain +younger +january +wantap +tropical +sale +cut +lam5xylwexfhsnzv1jsuxxjrzvx7ih0ujkwq66ldjyu2ppds42hs7vjlqogeyeavuyjkposas9tv +data +full +safelyubr +seconds +vector +033023 +employeescelebrations +235829 +esmtp +tests +a +re102lect97nce +intense +deliveredto +office +hibodycsminingorg +35 +localhost +adler +walk +family3d +indicating +build +because +3197816 +8226nbspduration +has +messageid +350 +10 +xasgtag +unknown +oaa11786 +0700 +appeal +excl117si111ns +xbarracudaspamscore +account4 +srchttpwww3videoprofessorcomimagesjohncdjpgaptdtrtablebr +public +doctype +peruser +xbarracudaurl +contenttransferencoding +bsfsc0sa081a +multipartalternative +21 +bartholomieu +building +mimeole +i +127001 +committee +reject +aligncenterp +divmsonormal +2008 +accession +aedes +athelinakarimuecplazanet +centerplanning +quarantinelevel10000 +rowspan3d2 +20020474168 +to +v1 +xhtml +rel97ti118ely +charsetiso88591 +base +zv70f5naaprp0vdkm4u7rulxsu6wr6abz88n4avotpbxjjuh3y0oadlzmxxssn5fjq5fp +now +code +you +ywnvldo6kvwufoc7ptkbaut7d8vzv7v0rx0xx90nuqoipaysro4f3f1rqoqfoplfwm580nd6n +around +historical +aggressive +fgdksxsi51egdznt9qsbua11lbvpn93qv1rwqzgxflrs1pvssn1vl5drcy44hs97f0le3z +windsor +share +font +symbol +brfrom +zyb0agugz2ftymxpbmcgdhlwzswgaxqgdg9vaybtzsbzzxzlcmfsihdlzwtz +6 +uc +againnbsp +htmlfontbig +second +15pt +at +approval +replyto +been +httpwwwbarracudacomreputationip8511596156 +oxijionuk +law +id +ads +dollars +structure +ukrainian +also +pbbenefitsbp +idautonumber3 +machine +same +httpwwwadvbizcncontactus +size3d1nbspsans +xmsmailpriority +after +47 +date +member +color3d0000cc +xbarracudastarttime +heraldic +03xui5yi5loebl1ast4plfmoottvlewbw8ppluvllunekrhjfltr9ivl6dkyqfjxamsk0mk2kwnh +takes +xbarracudaspamreport +giorno +international +n8f7tgt9010451 +them +by +split +size3d50 +jocks +httpspamgwcsminingorg8000cgimodmarkcgi +6170 +arsafont +nonepark +style3dbackgrou +0100 +their +p +with +from +hegemony +un +proof +healthstrongfontbr +22adv22 +mimeversion +border0 +nevertheless +august +styletextdecorationnone +2862 +faceverdanabr +postulate +address +59nbspnbspnbspnbsp +charsetutf8 +fetchmail590 +xaccountkey +regards +eest +message +king100111ms +its +implement3dwiels +link3dffffff +american +an +offer +7bit +forged +ayblock +decorated +be +looked +business09 +republic +libno +isikybu3d53130353067unsubscribeanbspnbsp +latest +mysteri111117s +sandra +average +1010010911 +xbarracudavirusscanned +1000s +guaranteed +html +passe +w +81168116 +matches +vjimosz +trtr +41 +emails +donfelipenetnowcl +xoriginalto +destroyed +nzdqpcaaapxcsaixk94aqccddfcgot0lanwefakuzcq6aegcahdal4mv7gziq4iubpurmzhawwaw +truruspamsubj +value3d304704 +ne +150 +olive +samson +d3997utres +3 +no +we +161126 +important +top +leisure +1015 +erlaubt +performance +does +including +contenttype +xmozillakeys +may +barbaracom +of +ii +401 +and +2009td +fulltime +received +19216818120 +increase +defining +cellpadding3d2 +free +month +11323631212 +discriminationthe +view +we0ffer +confidential +the +c1kpxzmh1irrs9603jephocjfpnnlcgs3vydpukf7nngpabu5ry6ztxyzy28nll2ynntmvvj2xgw +2611 +llars +advertising diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_33.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_33.txt new file mode 100644 index 00000000..0c671f5e --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_33.txt @@ -0,0 +1,339 @@ +psi0iibjb2xvcj0iizawmdbgriis8yxtbdqrq5zraku7kybf6pc9mb250pjwvcd4nciagicag +borb +size4human +2iudjoyinlotgv24tssq8awkxegagjiywnsfwiyz7vu9tiklaapkffuqalyx4qadi4hgqvaf2dr +recipient +tbody +xasgorigsubj +valigntop +153 +quinn +read +5px +for +network +smtp +xbarracudaspamstatus +55344fontpbrbra +usabr +bsfsc0sa392hl +package +due +sa154 +body +seaside +this +size2a +based +transitionalen +firewall +your +pilgrims +gafioc4f0gmdscaqvgfmp5zviyg0uaaidqrzgidcfdzgwgguycgkqebgkfyutqpn2lfvtpdjda +returnpath +jul +are +is +xasgdebugid +account2 +00000000 +reputation +market +subject +eu +121 +custom +using +l111111115ely +system +mov +included +zip +ideal +history +ust +09 +hrefhttpwwwdivlcomindexaspl2cid78img +475 +quotedprintable +border +1265059627 +throughout +below +reconnect +housing +colorff6600bgratitudes +48995 +hrefhttp2f78cokecoccnjzomibuwix1a0c18717b09911579aafbrpjhlwnlnhibodycsminingorgiykijile0607054147055103466697previous +if +495pxborder +br +christmas +tool +description +trust +rdnsnone +delivered +bottom +up +c111n111zcan +etc +european +size3d2emailfontbtd +blepcenterbodyhtml +normal +100irigir +all +lerlerctrorg +head +mid +rbl +size3d2fontnbspdiv +policyabr +facetimes +commander +055322 +img +nor +htmlhead +lettres +68 +166 +outside +5ebn6onq8vp09fb3pn69oadambaairaxeapwdcooopgjs9s0ds00nuaymxsaqtio2eomkp +esmtp +24 +cars +a +dont +deserve +deliveredto +xmozillastatus2 +aspect +hibodycsminingorg +le9vas6ankazxzixzaybjkj2yhyknkzncxfmwmmzcsewif2mmrcnplzwkq0xk8grhhnsxrooeow +barracuda +5aemjcuo4ycinbto21legc5oblqed61tuzs3ef5ruzmkjikbb4pfvcvhv8alk5pvykuxconqrll +cellpad +feelings +as +build +hospitals +messageid +350 +e11si4648635fga820100426084457 +altitudes +10 +xasgtag +bordercolorlightffffcc +cellpadding0 +slocum +yours +removed +linked +k111nk117rrenz +xbarracudaspamscore +bits168 +sister +or +width3d403font +charity +public +doctype +2001 +20px +safe +peruser +taglevel10000 +2adid +each +english +contenttransferencoding +age +section +zzzzlocalhost +127001 +outlook +avoid +httpemaservertrafficmagnetnettrafficmagnetwwwr100002304448423bpoq88r31zff +work +54 +that +specials +ns4csminingorg +searcher3c2ffont3e3c2fb3e +q8ybghgosa1ee1b0jx7seywab6i4di8tqdo0dr54samangdagglfdrhgwwaw3ojvrqaiocubghlo +bfonttdtrtbodytable +to +v1 +aabvvqeo2iixjjet04joamroak5fwoliqfbtpmx2n3h2awfrk8acdddpqcuaseibdtahroax +xhtml +httpwwwbarracudacomreputationip8913422138 +national +code +you +dominate +httpspamgwcsminingorg8000cgibinmarkcgi +td +web +qdzxwcfex81waapnfaj95fzxabr0crk9fqecwdz7ymvnhnp8a88v5rjqfyeiopofn3 +tdtr +32 +hidden3dtrue20 +various +planning +href3dhttp2005811533rl000306jgnmzmhtml +privacy +here +pounds +alnvj5flnt9xapudxpajflhl2knt9ov9xv3bl063gcfmf86fq8a9zqx8aipf86fq +version +foundation +pfc6bgnmolbcrghkjcvky0nerdjxmruahjgqnjkoow85fszkdzk1mk4qkhka0brwoklcmtvcqj +taken +attorney +17th +canem +bracht +lnf8apofn3lnf8aautzozuxmvwa8n5rjr5r8apvja40bcloqlzxwcfet81waaxzg +id +pickup +ratio +800758220309 +also +zzzzlocalhostjmasonorg +on +indivisible +government +click +date +spamgwcsminingorg +scottish +live +xbarracudastarttime +margintop10px +strongthe +size3d3grown +xbarracudaspamreport +noreply +height166ap +incfontuspanfont +me +xuidl +mail +content02 +manager +by +en +aligncenterbidp +however +dramatically +director +will +link +0100 +p +alignrighta +inderjitsinghsainicsminingorg +trusted +with +from +nvspqehnbdc6tbsdyfrb8rzanj1ztuk3ncylneu0guuaqu9chixrl5faaaayl4jk6oadlrntkj +type3dsubmit +height368 +height100 +mimeversion +border0 +programmable +august +2 +122436267 +linux +65007 +n9p0mjjd008936 +charsetutf8 +words +fetchmail590 +1100 +mating +message +wim +under +rjuaqhot0jfnridvjh3j3faerdzrhyyez5naopcnkdr3p8by4pjvzer3qwkkkqpccomqxoch3z7e +evolution +g6n8qop12861 +be +viewing +8cida7kgiivng2diwyzapfkmm44tiqnzngfy9aso0cswusg0mvhzrjusutwws4lpfqjvpiakgu5 +befa62c495 +h +texthtml +color000000if +riverp +xvirusscanned +power +special +more +microsoft +msdouglasnetokcom +064 +size3d1 +camp +cadeau +mail1csminingorg +really +2036916370 +digit +ielnn4tjfextmw5akqtwyeskbzebjqap9emrqwiegaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +ptsize10 +emails +xoriginalto +m0620212mailcsminingorg +tour +narrow +skylands +fonta20 +sansserif5250fontdivtdtd +111102 +aye072 +describing +2zmuipfgzpwuphhjtn4pksy3zup8oassiitq2vo4mx71pfkwghiuhlcqs2rnokkwbbzjimcmxff +made +very +sat +name3demail +93129105218 +top +titus +webmasterekuhnlede +past +bgcolorwhite +ptdtr +pnbsp +opakkzzaiw0wtvdlwsbd1xbeuqgcsmcgef8k8tkdo3bgjgjud6h3r3u4xwbrph7sdsg6sixt8k +barracudaheaderfp20 +200102 +policy +rgb103 +contenttype +instrument +charset3diso88591 +write +of +csminingorg +divas +and +received +one +high +right +wholesa +list +table +fkli8p4mztx9ho3lbuzpgcnxcnvp6dyhvain3mmdlkdaqtyjxwk88rlslvsllgmvjef0bbkn +score081 +dgltzsb0byb0ywtligfkdmfudgfnzsbvzibmywxsaw5nigludgvyzxn0ihjh +search +the +send +parts diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_34.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_34.txt new file mode 100644 index 00000000..bcab7c13 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_34.txt @@ -0,0 +1,332 @@ +eclipticcomparison +bxcbwpxvtlhqrhai5i4oky764kodisgbweavpqutrbgwgocd1nolbyyjf5vtj5fp3qry58we8zpt +used +helodynamicipaddr2 +size3d2emailfonttd +xasgorigsubj +poker +bmake +textdecoration +for +postfix +msdtc +083805 +paclitaxel +conditions +question +vriicuor46g1kzhd7dsbzlkq0eger0aqpbngmikboxide6im6gg6zeamkcnozkxj0sn7wuz6jkn +helodynamicipaddr +bgcolor10517fterrys +body +itspantd +this +instead +classtthis +your +returnpath +1260277451 +0001 +breakdown +is +score1893 +learn +xasgdebugid +valign3dbottom +altnewsletter +100 +23 +account2 +plays +subject +wondersharediv +parliament +custom +xbarracudaspamflag +equivalence +mt +abolitionism +included +size1a +aligncentera +sign20 +reaping +but +ava +permission +postcard +what +underline +would +remember +6v2ctpn3d5i9qvmos8sy3tbdvkej4wupvkq5aimhpz2udysty6po36v2csdm6vpsuzhzgjbov +intent +luma56csminingorg +opportunities +online +specifically +101012089 +xghtmtggc +5a7c82c39c +commentary +if +laccompagnement +19216831 +tag +hotmailcom +namedescription +0011 +face3darial +bsfsc0sa424 +our +xbarracudaconnect +250 +do +all +tue +v35takv2xnqubzfeuyuy0khknt45wk3drcqsm56oomelcjwzcmtcjbfc5h1wsluwjldi8vz4 +book +lerleramilerctrorg +60744zrtrot7406616waahoooqot1725317ebxeicxxtozrtrd2finestscorcom +classifieds +align3dcenterall +interface +msgidfrommtaheader +045244 +whitelisted +cellspacing0 +0400 +digits +before +it +esmtp +items +ca +commerce +17b3076f0ae +determined +improvem +a +technology22 +sometimes +deliveredto +href3dhttp24e67ydeceyercniuq +job +information +xmozillastatus2 +attention +century +localhost +as +build +rule +href3dhttpwwwdiskeepercomdiskeeperhomet +has +messageid +10 +trade +cellpadding0 +tda +tri +or +comes +tonersnbspfontp +pcp01978751pcsaubrnh01micomcastnet +public +questions +color +nextpart000000701ca57d5af6060a0 +signalservicethezscom +010 +architecture +size1grants +u18 +contenttransferencoding +de +ticket +titled +indirectly +outlook +8bit +programs +border3d0 +xbarracudabrltag +styleborderwidth +to +width304 +xhtml +charsetiso88591 +now +xbarracudaenvelopefrom +you +dysmeromorph +ilugadminlinuxie +h2intramural +gmsorlyuko5xvaj2qpve9t2nvjrh6aytmad755j9kxafdpluwrpyq2njfjp5pxi4ra2pxcdfr +31106356 +h2septemberh2 +centercnncenter +mebr +documents +101220 +colspan9nbsptd +mxgooglecom +repeaters +name3dgenerator +44yesonlineisbestcom +msgidfrommtaheader2 +at +153448 +srchttpimg188imageshackusimg188518jivezezyialeekjpg +id +meta +forests +pts +nak54cfi001394 +fontsize12px +on +cloud +being +education +authors +click +b +language +individual +xmsmailpriority +date +src3dhttpiiqusimagestba0729200207293gif +inhabitants +spandivtdtrtbodytabletdtrtbodytabletable +xbarracudastarttime +appreciate +stracona +allow +abr +jordan +092605 +artists +2009 +banking +me +valign3dtop +mail +girl +final +920c815e5f +by +align3dmiddlea +antispam +hate +back +0100 +imagine +empire +tributaries +sides +times +paddingright +with +from +early +jackson +73w8l +texas +mimeversion +border0 +fontsize20px +bsfsc0sa275bhl +522px +14523923587 +000 +2 +include +htmllinkpushhere +james +httpwiorynivhexliqsnet +make +sent +charsetutf8 +135211 +words +size2 +xaccountkey +sender +message +control +server1 +an +partnershipfontfontfontblockquote +subjugating +pantry +7bit +zhdkgyxh2tua2gihruwwm3yta6iblmabzwq0cycpnxf0kxbqmeeeyjgsiuwqkrbugcnjk9dovha +be +movement +youd +into +2051586244 +quick +size3d2your +texthtml +padding +4em +were +xvirusscanned +average +width780 +figure +form +testsbsfsc0sa148a +us62 +html +size3d1 +httphgsaqeduxurobcomaae494e46ec68b71f45b720e723294525d4 +alignrightcontainedp +spam +xoriginalto +timesp +anddiv +m0620212mailcsminingorg +194759 +simply +hibodycsminingorgbr +belowfontdiv +planovima +3 +arial +made +0px +srchttpecuyuxbilozcnspacergif +no +we +color000000bpoor +width +known +film +regiments +1000 +contenttype +xmozillakeys +sold +uid35541201487268 +104307 +of +since +and +record +201005050521o455lxor011731gw2csminingorg +received +one +without +red +200600 +table +felonious +titlecheck +went +view +york +stylepaddingright +wisconsin +m10grpsnvyahoocom +sackville diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_35.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_35.txt new file mode 100644 index 00000000..332bfecb --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_35.txt @@ -0,0 +1,335 @@ +widely +opinions +productsabr +se +xmailer +chief +program +s +tdtrtable +smtp +valuelatvia +june +span +041 +width3d95 +002824 +investing +body +du +challenge +name +jya0hherjwnuugyecyogsceobdegdyinwyqayvx7cltsbeilnvxyjo0sick83aleubrbaxt +interestednbspin +have +rkyipg0kicagicagicagia0kicagicagicagidxkaxygywxpz249injpz2h0 +returnpath +introduction +cellspacing3d +clearance +87zlo9y88ahtyaaaaaaaaaaaj9lzix2irmj0pb9q2yz7zh5fjcnaak28srp0sw7kz9evsbiyu +are +mon +m56wwhdwchq4kjrct6jjdewmvntpfjvvfaa6p1rm7y8g7mfvx7f5p6zw4ob6gilhnfeq +200208250144caa18619webnotenet +color3d0000ff3eeasy +xasgdebugid +s0ft +among +rqzxzlkirhhqsikvo6msxolvan0lf1x0uqqwarrgldcekrqargrzoowcxwskt58yg2nlfndltru +100 +judgementschild +he7vt1pojbdfrvgbr9sh9fkd9dajdd94sd7mmedi33d16807pcscp5bbsyip0penhp1rul0y +subject +custom +hostname +noneuropean +h111pef117lly +than +1711 +included +want +uid121371201487268 +aligncentera +eprint +c111111kie +abuse +con +bgcolor +lake +15 +hrefhttp58ebb9dbupotxpcn +0800 +best +0530 +degree +would +although +bafter +online +proper +hundreds +pa +etc +250 +thu +all +50 +played +img +noekaaqbpdwktrqjirknqs45nnk7auijsklpsvx5xjljswn5ckl3wx4jhmmlizjjmqmfyaajrv +users +ready +nor +vlgt1utrxrxvry8zki0fegknhw7azsjksya4k1ixlwdouwpzdzg7pij8wvywsslqnri8wg5ftbq +width18nbsptd +srchttpwwwebonylust4freecommailerimagesebonylust01jpg +ff0000brfontfont +13 +uri +115pecul97rly +tha +commuting +us +hrefhttpwwwcompagniehumaineorgautumn91html +manynbspcompanies20 +esmtp +24 +ac0tnzrkuaz10264ptxqpu068ueyrdv1qjwxms3ubx2gocuk1k2dtmd0fiomvqkvx61v49bsjb6 +a +boundarynextpart0000023e47e62046639db9805 +width3d2230022 +deliveredto +jmnetnoteinccom +tuigif +hibodycsminingorg +super +barracuda +localhost +as +bfontp +textdecorationunderline +rivers +messageid +pfont +n6gm2ryx024578 +pcs +10 +xasgtag +trade +cellpadding0 +mime +refund +operas +hrefhttpwwwmilkapilkaruuiqxjs9b98bbd62aefa +201004271524o3rfoole017373gw1csminingorg +burgeoning +yet +xbarracudaspamscore +strength +or +letters +doctype +650px +style3ddisplay +xbarracudaurl +contenttransferencoding +userid +michigan +comic +i +127001 +delineatedsaalvincentdeniervergangenheitdesconfianzavikingaircraftwsxvict2edishonorabledestinedrecruitmentgiuliogoogleplexvinjettvisbygirouxvitaminavolckerdisfranchisevolviendovoorheesrepopulatecarmadinrepositorieswdgacceptedwcfarvatowhmacerbategreenvillecarnotrimbaudaquitaniaanthropometricsricordi +charset3diso8859 +hammer +that +square +to +v1 +charsetiso88591 +senate +you +know +worldnbspnbspnbsp +defenses +hrefhttpd5d01qutuhixcnsod47524c20ffbc695a1ac0c8iz2431271099774536068075 +recently +cult +aqebaqaaaaaaaaabaaidbaugbwgjcgsqaaeeaqmcbaifbwyibqmmmweaahedbcesmqvbuwet +was +caro +woodworking +coastline +c1 +color3dyellowbcenter +size1just +face3darialthese +at +through +older +id +140 +pts +stylecolor336699homefonta +sandbanks +d55072c1c0 +0019 +remain +churches +need +on +external +htmlmessage +width0 +xmsmailpriority +thinganonymously +date +spamgwcsminingorg +member +outfielders +shopping +2010 +1eitlxvlay2zpas6htg4dak4yzfjbfmo7vfi66uufdmsf9u18yxwvtgzkqlbonpl7kxvblx42trh +authenticationresults +notified +8pt +valuable +plataforma +xuidl +them +finn +by +zzzzasonorg +epicenter +size4a +unsubscribe +new +privately +spain +paper +0100 +september +p +remained +125744 +tdbfont +trusted +not +with +from +preg117ntar +successesquotbrbr +20 +mimeversion +scores +give +542 +wojciechowski +first +alone +buanqavo +aligncenter +antivirus +messagesfontfont +xmozillastatus +choice +could +serie +sent +beneficiary +charsetusascii +jody +xaccountkey +hrefhttpwwwlewdozedcnjkiruqxaciojcosyexifqmicaben0691431 +ttpwwwwannaberichnetmortgageclick +spliced +message +anyone +speaks +19216818251 +property +citythe +forged +aug +cannibis +color3d22ff0000223e3cb3ehediyeli2e2e3c2fb3e3c2ffont3e3c2fu3e3c2ffont3e3c2ftd3e +color3de0e0e0 +httpequivcontenttype +relevant +debtors +282179628 +surrounding +instructions +texthtml +wsuhjlsanzi3v0yzdkcginqxvgur0pxiq4errirspozszozxqyi5opkm0alrszooam0zpkm0al +oztnotoifalfiheqikaeqbiakzimta12automated00onlinecom +million +title +size3d22222bfont +manufacturer +microsoft +xbarracudavirusscanned +bsfsc5sa269hl +w3cdtd +7 +97 +guaranteed +html +style3dfont +81168116 +illustrated +aspx +act +innovative +emails +7xsvh7fikzyanuaavswd6sagf3tlrfl1jlqnqgnhs96xq9b8h1l9otlknpjktxeiiinposiddi +capture +family +sa335 +msoborderbottomalt +rgb255 +httpwwwmailcomesandgoescomvirilityaffiliate +limit +hutchinson +imre +no +sfnet +important +altapproved +vizepekey +digital +centerwascenter +name3dcity +barracudaheaderfp20 +height3d435 +day +sansserif +1000 +contenttype +extremely +z32qt8o5phnv5oflgvmzyomtnnnvjyofkbdvepyghsmun2qjaodwk1qe15qmsthdbdhwkivxx +volume +enjoyed +1941711618customerteliacom +listed +111136 +of +dynamiclooking +and +1374 +avisited +mexican +icemannetnvnet +received +jj +one +19216818120 +wed +alt3d8003271460br +table +confidential +vba26jejbtngef7xfjcwkuwyi8pfseych6hr3z0g0uoveeikrjdoo3ucurs8jg0suu6r5mk5xk +died +send +advertising +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_36.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_36.txt new file mode 100644 index 00000000..9ad61983 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_36.txt @@ -0,0 +1,323 @@ +zbvkacssdgj5mkhcwsy17atgl1jyiaoyuewaj8vvfhnkypfvf7todr6d7fen7gtphnyt4lz +dictatorship +widely +orh5 +fifa +orwig +read +typesubmit +ways +format +for +color3d333333quot +postfix +network +school +smtp +xbarracudaspamstatus +false +weight +y3tfxrvaiqiiiyhze3aampz497tal4bwckpw8daopxe074baokooosgshfe81tn3t +defeated +body +height3d1 +years +registered +this +redistribution +in +content3dtexthtml +margin +164 +transitionalen +1978 +your +returnpath +once +0001 +breakdown +are +is +commitments +src3dhttpiiqusimagesfairlane200207182a03gif +account2 +00000000 +nagve +070 +pressure +subject +soon +styletd +relay +using +40 +am +size +shouldnt +xjqxsr3mamwbl8rsditvbvsrj8mvfp8uid0sjmsat0kdzfioaaach5bafkagcall0bogctaf0a +mailer +2002 +lowest +want +tg9hbnmsigfuzcbnb3jlisa8l0irxzlbib3axroigxlc3mgdghhbibwzxjm +deletedyou +field +email +height115px +thought +ehjmcg6aklicgiiuim5wonqirr7eztdofjri8udi8ljkzsb2wbdasikjdaqmf40nf7ghhce +unwanted +ns1csminingorg +unavailable +future +best +pc9cpnrozsbyawdodcbwzxjzb24goikncjxhighyzwy9imh0dha6ly93d3cuag90zgf0ztjuaxrl +eleven +ceremony +j +closed +if +mayfair +neither +glgl3v1hwcr38ahv8ayvqlmtmx1cg7q2f9vstcwhplvc7bg4fnbg2sybpuer9ptpkrvdgwmh +up +directmarkeudoramailcom +qibaryhe2476superkabelde +xbarracudarblip +xbarracudafingerprintfound +valuevavatican +thu +autoprotectionextensions +minute +head +marginwidth3d0 +sgo +lerleramilerctrorg +theirp +peter +denis +22pxfont +img +f +colorffffff +xqqajuz1sfsjazabecqweejvwqcoawqahqaa49vb8jawahoq8iuyeqiekhohjwgiya6ika0cwml +comfortable +ysbizxr0zxigaw52zxn0bwvudcb3axroigegzmfzdgvyihjldhvybibhbmqg +it +tami +esmtp +implied +24 +a +highratio +k2biztld +deliveredto +office +xarrde7yy5px61oseod2oaan3fllmp5uapzmk5pc5aortldrgjrrsz5pxawuf8ahf8ayj19wbs +9qkjheaszucmktjhf9s966rzeuajiirqsmrio4nugfyn8arwyuaqowynkkkygwmuwamykqmotxo +localhost +dogmaslashnullorg +persistent +rule +arialbrowse +012a12a22e6b4242c4a36cb82ad2nsnnbl +width3d650 +messageid +year +playhouse +valuebdbangladeshoption +10 +john +gonzaga +fraudulent +nominated +balance +or +country +supplier +guarantee +lithium +public +others +total +they +contenttransferencoding +pt09pt09pt09dqpcrsbbie1jtexjt05bsvjfiexjs0ugt1rirvjtifdjvehj +thisb +decision +reply +titlehttp6623113363indexhtmlida12b +909 +3cp3e3cb3e3cfont +theh2 +127001 +8bit +bsmtpd +cfcfcfpadding20px +helvetica +ebook +that +desperately +upon +styleborderwidth +smtpmaillkelliwqnpdnrwnet +to +v1 +rel97ti118ely +name3dcontactname +washington +singledrop +national +005217 +you +letter +web +planning +font +most +here +internet +week +there +mxgooglecom +6 +at +030355 +htmlimageratio02 +go +sa148a +low +155413 +id +tenderness +d6f0hfueot84xtcxzdcczchosx2v3aukamx7p0xpv7vfv8a09ncpsy8rijih2g38mwzs +mandarklabsnetnoteinccom +pts +1146 +leather +htmlmessage +tr +wants +check +gain +xmsmailpriority +date +border3d10 +mail2csminingorg +spamgwcsminingorg +friends +230 +millarem +valuesmsan +110515 +2010 +everything +2009 +xuidl +envelopefrom +bi111graphy +by +then +love +gw1csminingorg19216818250 +moneybli +antispam +new +centera +75 +xbeenthere +says +their +p +trusted +not +forced +with +from +refer +apologies +width1 +20 +mimeversion +komeis +000 +aligncenter +itbr +xxsmall +pgrkhkampq9a6qkqoycxdeuce55lf6jc2jharedseornvijdwrxss9pdoujebcua1bwrbkc8ss +sent +charsetutf8 +fetchmail590 +042142 +xaccountkey +staff +which +hendrick +smtp4cybereccom +xkeywords +offer +herea +alllifecom +instinct +catholic +get +size3d63enbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3bnbsp3b3c2ffont3e +7197122240 +texthtml +srchttpdou2findflorcomfg56foiluzenoqiuceobnawirzexklqsrrexqooeehnuooeczbelfoc +padding +rules +million +sobom +special +more +1010010911 +esophageal +annually +xbarracudavirusscanned +brl +hrefhttpa03vkunrorfcnucydodyh94hg50z49913x91v375i8qiamaa900799114670527935932 +uid134271201487268 +81168116 +reply20 +lbiejxrayqe3ebdaawwxrmarfb4rukmdhdbchs6yufhmfzygaamccebairte4ayhudmauggywogf +height3d30 +spam +airline +m0620212mailcsminingorg +started +countrycode +sareadult2 +topquality +3 +dfdfdf +very +namewarezcdshtml +white +including +011858 +1000 +contenttype +east +may +like +li4uli4uli4umtegbw9udghzihbhc3nlzcb0agvuigl0igx1y2tpbhkgy2ft +of +01 +and +193a30 +received +cpunkslocalhost +youll +today +valueflorida +wed +free +table +tzkhf0dphzcabyhrdrepbwh0ewqhgsqkcxqgsmfnbebqhuhadrcbuwj1nabirtyjgwfabeslsazg +level +renova +amounts +the +send +cash diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_37.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_37.txt new file mode 100644 index 00000000..4faa8eba --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_37.txt @@ -0,0 +1,382 @@ +ofb +795eb3 +helodynamicipaddr2 +xasgorigsubj +health +en118ir111nment +please +bair +xmailer +onlybfont +for +organization +network +zprwzpbcadptcee9cojd2qzddwindzfn34pdjdaxtgqlfqfsqtzzfg3rdknetbzfkakmckdxjn +m2rjk0iue6ejttgq2ggawh0c7hjte7t0yrce6jusatgce5ggcdpt3pongsrbuerhgtgkesgawxa +smtp +194610 +bodyhtml +further +9px +questionsbr +bsfsc0sa392hl +size1no +xmlnshttpwwww3org1999xhtml +city +body +height3d1 +name +this +400 +trustworthyfontdiv +portions +in +fragmentation +ci8vls0dqo8l3njcmlwdd4ncg0kpc9ib2r5pg0kpc9odg1spg0k +printing +g8p28hc04819 +muslims +size3d5font +finalist +returnpath +jumpstartbr +0001 +breakdown +are +is +mon +127226628578f6cc820001w4twrl +task +sacramento +accept +force +president +213105180140 +prices +reputation +xbarracudabwlip +subject +minus +custom +httpjhmphebcyxehtanet +got +relay +065559 +xbarracudaspamflag +v60029002180 +system +2002 +bulk +killing +late +width3d28 +quotedprintable +yes +11 +ns1csminingorg +url +enforces +designate +jack +212173515 +online +hundreds +repr +df8696c +worldwide +description +rdnsnone +going +quickest +xbarracudarblip +vbanvfakvoagpualteaivfai5h0jmuaiyxsjklajkiagfvaivvakj4amj7akvoadahajv3aj11al +placep +population +our +xbarracudaconnect +thu +jmart63573attnet +head +vorstellte +facecomic +aa7fncl8tommwdbt2whfq3oeooopm4ua4oakkan3hnfepbucmf9mkwakephfiremph6gjp8a +50 +had +nahjerqx018166 +burning +announced +trick +opportunity +cellspacing0 +skt4dbr5ztcs92wvrlahpcyhd6ubvtazx0qzyj2mtylwylaczwhcortchzq +push +edt +just +255 +left +sun +esmtp +a +buried +discounttime +deliveredto +office +n46ewmod025893 +style3dfontfamilyarial +hibodycsminingorg +seeing +advance +situations +localhost +dishh2 +as +marked +cambridge +div +rule +mimehtmlonly +meanwhile +5 +messageid +akashanetpl +29995 +10 +mime +feet +brmy +phoboslabsnetnoteinccom +xbarracudaspamscore +contain +or +htmlimageonly16 +trackedbr +public +dhcp +char +contact +total +peruser +they +taglevel10000 +istanbul +contenttransferencoding +color003366 +stop +jeremy +debts +127001 +oz +outlook +8bit +href3dhttpnyavekepcnimg +lemony +work +charset3diso8859 +prayers +out +named +venture +quarantinelevel10000 +n4i7qerg014308 +styleborderwidth +to +holidays +117tilizzare +quote +facilement +1267126689 +exact +lsmtp +now +county +you +exchange +advice +operate +revision +barent +k89jglpfeh66xxc6fdlgetayicojf7bewmmg9exjgclagznbehfvkxfcfee2b13voma22xar2x +name3dgenerator +second +at +lth +been +nhc +id +sir +over +listhelp +levelbfont +surbl +external +width550 +fromexcessbase642 +tr +color0000ffbthe +xmsmailpriority +date +inhabitants +waiter +border0td +supports +fontbp +who +bronze +5087726408156400551715001108381 +treatments +h2disruptive +height38 +2009 +authenticationresults +yk0reyyfickojejevmaoyvjeskxonoiv6yzqopmkzkcjypmwasa2hmuong40icq6iggmjjrfgno +xuidl +moosomin +mail +by +bsagicagicagp3n1ymply3q9bw9yzwluzm8ipmhlcmu8l2epc9wpg0kpc9k +r111ugh +acquisitionsfontbrbrbdivlidiv +sean +he +yearsprecisely +pick +satofromaddrmatchhl +mo +new +will +kiffened +6hshjppdzpojntn2qjac1gaccbkc15pxpiphjfimqbevmb3encsoafifpjxpmbivuzo4ueg +sound +iqeu3pat8lwp6xnz2kjf8kcaetacrg3ad2f8lwqljg32uumatka9f9qpstjaqzf61bgkb +marginbottom0pxmargintop0px +again +alyssalt08yourip21com +not +xbarracudabblip +81168112 +with +from +dwl +homepage +width1 +20 +1200 +97dd +mimeversion +0500 +aligncenterdouglashome +next +polo +stevep +turn +kst +3dcenterfont +xmozillastatus +81 +address +details +rasa +2px +nnggttffexpiryformalsiemensoldliabilityminimumsitiosbenefitbackedsmoothiescientificfollowswoodspointsphysiciansagainjewelledbedhasamhalloholdensureattractedwitnessed +british +words +xaccountkey +282 +sender +width3d250img +ad +unfortunately +message +10px +its +tree +mitsubishi +19216818251 +round +an +caribbean +equipment +size1who +targetblank +be +c +141111 +httpequivcontenttype +rdnsdynamic +matter +sta +rules +starting +better +xvirusscanned +sites +romance +several +more +e6859de083 +microsoft +directly +w3cdtd +r +class3dstyle2font +size1your +color3dffffff +type +required +bsfsc2sa154a +html +httpternetblahcom8080 +mail1csminingorg +none +contains +emails +xoriginalto +styleborder0px +m0620212mailcsminingorg +revolutionb +irc5v7m4rud5jdlvv7h012jw2dmfvmxrp23klmienynaq8yxksvqozidcs5fw51m1j3zwcx5 +thank +lines +senior +olor +against +3 +arial +images +no +ele11897ted +alignleftp +doubtless +some +fear +062215 +barracudaheaderfp20 +necrotic +sansserif +xmozillakeys +style3dpaddingright +months +10pxthisfonta +listed +of +100elights +csminingorg +increasedecrease +distance +fontp +cleveland +received +one +bespanbspan +sa392hl +align3d22center22table +posted +table +technical +thinggoesherecomremoveaspclick +the +boundarynextpart000000701ca5881378b1e10 +scriptures +xpriority +d +advertising diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_38.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_38.txt new file mode 100644 index 00000000..0fe36f45 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_38.txt @@ -0,0 +1,351 @@ +b2tjzhjvbs8pgltzybzcmm9ahr0cdovl3d3dy5nyxrlmtiwlmnvbs9ib29ry2ryb20vy2qz +2day +providing +524px +10pxcomfonta +textdecoration +for +postfix +program +zzvjznhtirfrna2ujyp503cayznofw0smolng9qwbbrsnps5llybbc5safkt4wcfxsck1pqa1rc +find +xbarracudaspamstatus +htsac6hguhptrip68u48cjyaygfokg7h0pm5jj0tlh29vrtc48kdvztwkcqwx9pek2y6 +srchttpetne104attendbuzzinfoimages3395jpgabr +sourcefontbr +x +due +ui9caa6gaaaykaaguaaaygaaaahizjjtrkm1z48gogzqaaaaaaaaaazxvdk6oabniaadkaaaabka +website +aada8aba9bbb9eca4bdbee8b9a5bfadeee8a4ada6adf18a9f +mailwebnotenet +body +eyeballing +area +name +this +ezine +emwac +in +content3dtexthtml +certificate +jefferson +businessfontp +transitionalen +firewall +returnpath +v60027412600 +0001 +breakdown +is +arbitrarily +his +xasgdebugid +too +wwwn9gl0xwiz1p0rl8qiirrhp9szpo3lfevxncfdx8vvkgcfvg4aqmmrq2fj0utuldyncvsvbl +scared +fori +friendafonttd +successfully20 +hellenistic +account2 +00000000 +seminar +11px +size3d1a +222624 +market +subject +billions +using +less +narbkrmw006300 +cellspacing2 +four +ryan +aligncentera +1571119187 +width715 +state +obligated +below +richmond +pria +xc1m3hwapl2h3jnm3x8d3rcyzwq9s5kqbstgsryo5fdeywnrfmdlkgkkbl3kmo6bpqn1awq3b +obliged +style +j +image +flags +debianid +pan3d22222 +description +rdnsnone +407 +process +nurst +hoses +our +xbarracudaconnect +54e7d16858 +thu +standards +do +head +cd +alink66cc66 +height3d337br +host +multipart +prestaties +centerblack +282166630 +align3dcenterbfont +edt +ywr91642908075547052401100531077373 +sheds +us +united +esmtp +rlyyk04aolmdcom +boring +width3d50font +a +situation +contrary +brbrbrbrbrbrbrbr +river +seeing +width728px +1mzszoafupm03njmgbatnnzszoafmkztc0maah5pm03njmgb2am03ngaafzszpm0ualmkzsuual +localhost +build +align3dleftfont +rule +bsfsc2sa154hl +mimehtmlonly +well +scrolls +messageid +350 +month20 +formbr +10 +xasgtag +removeeuropeaninternetcom +unknown +align3dcenter +removed +phoboslabsnetnoteinccom +v117lnerable +xbarracudaspamscore +172642 +contain +or +47ft8k0lz924ec8mmf6xnph14h8qldux0oykffjx5z4d2qc8jb71gkukrhcjbtgtjd8j1rp +6822 +computer +6521715966 +seth +200910251104n9pb4osg025054ns2csminingorg +peruser +taglevel10000 +each +far +rosedale +contenttransferencoding +signhtmltagsupperandlowercasenmd08100101 +hrefhttp6bdrefillmarten81aruiolijivotud057d9ca250d2 +contacts +ac0261616a +277725 +service +uid69781201487268 +cxvaeu7fis1yaxonhtkjw6y5cvijka2qvw5vteu3x2podknqcqwbtzmiihdnjdiop6p8aw4tp +8bit +division +quarantinelevel10000 +to +satellite +getting +lee +miss +charsetiso88591 +cellpadding3d0 +you +policiesp +reservedptd +je +department +viewap +was +alignleftthree +miniplantas +younbsp +version +hrefhttpc9bbthedrickerectruatiawyokieb7a50ff27 +bytes +legal +1980816894 +continental +at +1 +replyto +been +remote +id +pts +color3dff0000yoursfonti +rdns +disaster +nonemaking +verifyno +betterbr +colourful3dreduction +color3d000000 +on +107040100 +speciesb +none7dah +black +htmlmessage +bsfsc5sa011 +click +lies +relate +date +live +take +height414 +smtpmaillikubos6311brasiltelecomnetbr +internetall +progress +income +offering +trades +valuable +them +valueukraine +by +unsubscribe +facearial +zzzzlocalhostexamplecom +improve +regulatory +drydio5hnhpxljletldqgp3kxyufyakqzaekda05q2qwgu4jxvj6irtv4o0xv7oacmxysrlvc +height80 +cousin +1257841461244d004b0000w4twrl +0100 +6pg5mf7ehe4idbtwoevilsmailcom +association +5434655660web7602mailinyahoocom +with +from +applicantsbr +mimeversion +border0 +1144124187dynamichinetnet +lis +hrefhttpdrbluehornetcomct27833172825285769m1133771572ae07dadf1297fbb0452048e91119d515img +thoughts +2 +pass +xmozillastatus +listid +charsetutf8 +conference +hi +available +xaccountkey +mating +ad +message +ultimate +19216818251 +shall +1269644290 +an +caribbean +31108047 +pairings +213251 +valuetntunisiaoption +lc0yntc5mju4odqsltewotaxmti3mtmslte4mtq4otmxmjksmza0otewmtg0 +into +p111tete +suomenlinna +uxv5vrxporvvuvgz9mthptp3fnwluoev7fwaxq9vsu5pjjbycuzsxoqaqaqaqaqaqa5pryq +httpequivcontenttype +13px +unit +texthtml +rights +rules +welcome +enough +latest +kept +road +xbarracudavirusscanned +directly +brl +w3cdtd +7 +224552 +windows1251bw1nqqu1dia +shippingfontibtd +alerts +html +repaybrb +really +municipality +start +lus +discount +todayaspanfontp +bsfsc2sa031 +about +example +005401c1e7060ecc13c0147ba8c0xg395local +alink0033cc +000044124742000066f100007dd5jerrysdodgecom +images +0px +no +sat +architect +having +steadily +013 +when +group +jst +define +sansserif +llyralocalhost +contenttype +xmozillakeys +faa +href3dhttpc87hucojojcnwyxypj3deb101e14f0be4a1ffdf44791 +may +feb +of +hrefhttp254118gsd423150mcomadulttoyhtmlclick +record +received +without +19216818120 +increase +herby +titleuwouonovaqiv +alignright20thcenturyp +contentdefined +table +valign3dbottom20 +rhitta +xgmailreceived +kklilbwkgqf8xajdxcnf8jhq3p3wcq1woopjepilvt1uvia4uf8jdqvp15dxaaoo +parts +never +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_39.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_39.txt new file mode 100644 index 00000000..2437236d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_39.txt @@ -0,0 +1,329 @@ +abstraction +recipient +brbrmy +regardless +please +for +htmlimageonly24 +services +smtp +titlealbany +june +people +series +51 +any +3dhttpviviacomuamhtml2nvilk4ix8moreap +city +body +policya +across +in +n3d0 +n51ldpv5007728 +professional +atdtr +ez +returnpath +point +breakdown +mon +energizers +bgcolorffffff +19 +estate +081502 +clncgggzeasfjkdvgvakp1jbzzl8kqc9zvckocpjbydwspyujigw0r0j6dp50kvgvv0omuoydg7 +reputation +text000000 +subject +zezujaep +cipheredhdssdescbc3sha +germany +xbarracudaspamflag +using +produced +ty5iuks5jqy6y86srt9tnom1ixhmpslt7tbexsctltmroof7uvxgg0vhsuf5zczjelrh2 +system +laa29642 +bsome +072701 +fontsize +email +democraciesh2the +msgidfrommtaid +quotedprintable +mansueto +yes +hing +but +041311 +size2recipients +9 +best +beastiemcwaolcom +agencies +url +vaa13010 +paddingbottom +pydyebzgbpvvqpb1bkm3vbaxg321ig4ayjghebwm4rnxplfxi2u1ej1bnlaqvmocltmtjeqkcvr +emailnbsp3b +renderings +image +19216831 +villa +conflict +our +xbarracudaconnect +excellent +newsletter +26 +book +had +campaign +host +sale +colorffffff +page +every +9a0a4440a8 +n8e7t7hq015760 +before +align3dcenterbfont +left +itsuaopdsumaafnspas0g05gr1frzqzgp3asplsncrpsqlnoo9op2kmvixtfoxrqbkkshlm5g +n9hatorb022732 +esmtp +1000725 +watch +a +davis +xmozillastatus2 +ht3d10 +attention +spring +unifying +localhost +rebel +muycjbpw7sqq7arjkdar1agv6uejdjhwlu7u465pd0ohsgvyptwdyl0mtlh9xusgg5akqjbf +biggest +sees +inline +because +rule +messageid +10 +dog +vormkrijgers +unknown +stylefontsize12pxcolor000000fontfamilyarial +perfect +certain +0700 +typemultipartalternative +murmured +xbarracudaspamscore +above +sells +600600016788 +guarantee +ring +processing +dhcp +doctype +runaway +they +numerous +contenttransferencoding +al +way +xbitdefenderwksspam +1265981829761db09a0001w4twrl +building +multipartmixedboundary +lost +helvetica +carolina +ayour +styletextalign +ns4csminingorg +quarantinelevel10000 +brresidency +archduke +styleborderwidth +to +v1 +charsetiso88591 +brbrit +singledrop +fasooption +code +cellpadding3d0 +you +httpspamgwcsminingorg8000cgibinmarkcgi +exchange +advancedspanbp +department +vile +font +bgjk1cb0wi5xrq9qav4wngqzcy0vasajbluwxlqabdnwvo2pbkdlabwqogtsxs5fazggcbvaasmq +most +here +these +discoveries +armies +rokovodstvu +help +internet +believe +there +legal +orsonhacksawmachinecom +second +at +189015170212xddynamicctbcnetsupercombr +id +north +4175fontp +bsfsc5tg242d +rdns +over +on +general +centerjoannes +height50 +machine +size3for +player +cpmnbsp +border3d +date +mail2csminingorg +96 +thisisagreatfreepornmovieframesetupcom +2010 +international +hope +mlgaytrdxk5kjd84qb0mgwwa7ix5pxxgfrobczg1zdpgo2n9xobaaxmxg6zdpfylqaolfzhigd +width561 +ceremonia +authenticationresults +xuidl +mail +disease +by +selfrule +httpspamgwcsminingorg8000cgimodmarkcgi +gw1csminingorg19216818250 +bluecat +turmoilcenter +size3d15 +xbeenthere +szczecin +115 +little +trusted +parallels +not +with +from +mimeversion +scores +lnh4eg1hdgnolm5ldc9ozwfydc5nawyiihdpzhropsixnsigagvpz2h0psix +inc +o1cihrlq011620 +aligncenter +life +wf +pass +xmozillastatus +address +copy +gw1csminingorg +faceverdana +conference +words +charsetusascii +areas +done +jan +blast +captain +gmailid1289c3a90444d06f +sender +permitted +tried +value3ddrogersinsbuyercom +disabled +smaller +xkeywords +herea +chu +sorry +catholic +be +httpwwwadclickwspcfmo283spk007 +targettop +get +18441 +texthtml +rights +arising +minimum +alignrightanzusp +consisted +microsoft +xbarracudavirusscanned +7 +neutral +vanessasoarecsminingorg +mailnbspat +stylecolor +ant3z3swpiwvlesdxqbtanfx8jjxhyxf9jlawswzqrfeaxzfytyfbp3rjjp3h3xrprmuaxpsmn +name3dwebsite +nanticoke +norbr +100estr117ya +15120997190 +converted +0900 +many +marginheight0 +150 +5638e5ddae35font +size1andfont +6118820325 +width +postcards +height146img +downtown +014035 +gtfrom +great +style3dtextal +mx2csminingorg +jst +along +contenttype +xmozillakeys +america +inning +dario +assist +of +mostly +csminingorg +401 +avisited +designs +received +potential +bgcolor3dd +parish +fee +n111rm97lly +stylepaddingright +associates +xpriority +parts +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_4.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_4.txt new file mode 100644 index 00000000..a191fc07 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_4.txt @@ -0,0 +1,346 @@ +dghlbmvkdqoncnjlchjlc3npbmcncg0kzgltaw5pc2hlcw0kdqpucmvhdg1lbnqncg0kz2vuzxrp +recipient +titledo +astonishment +method3dpost +pm +sansserif20 +please +gbgm +constant +number +xmailer +lagos +for +postfix +business +services +smtp +colonies +span +method +xwebtvsignature +city +body +twenty +name +allows +fontcenter +this +170725 +in +based +delay +1906517166 +returnpath +are +is +mon +communication +groom +xasgdebugid +23 +account2 +00000000 +elections +213105180140 +reputation +xbarracudabwlip +subject +emrs +xbarracudaspamflag +using +private +system +operates +included +2002 +experts +fontsize +brthe +email +24002800 +hoop +petiti111n +agreement +below +what +planet +valuecvcape +64105237170 +briefly +color3d000000shipping +moyennes +online +commissioning +ns2csminingorg +style +return +troops +flames +br +usage +delivered +auto000000434609apperetocom +etc +european +war +christ +villain +thu +normal +led +all +alt31509jpg +60026000000 +smtpmailpaulthrivementoringcom +suggest +uid121431201487268 +pinp +149nbsp1998 +result +ul +3980f4416e +166 +godly +0400 +xmuorgmfr24d04o0lmo8h6brvclnvjca6ksdlixqgkhwhhqbp2vjwobm2cd6bi4lptfbny58wmn +fontweightboldsaleatd +left +it +esmtp +1000725 +thefonta +893893 +24 +100000 +adjusts +a +passed +deliveredto +information +12021 +hibodycsminingorg +night +became +height +71o +tools +div +rule +5 +messageid +350 +5joay7kykazfvg5k5pfomss0nmtjaqym2nnrhaiysekotzqkq8e4pntixygyxumz4gopxp2sg9tv +sharp +prevailed +theories +xasgtag +virgin +zwlyihnlcxvlbmnlig9uihrozsbsaxn0lcbpbibhbnkgd2f5ig90agvyihro +dreamed +175403 +or +rick +public +i5m9o9vr2wtp6tfwn0war9ouzsueli4jtwbs9virlz7bo7ess79kwzgyfdsrb6e7v61oqt +cdo +th3d98 +uid119331201487268 +010 +enjoyingbr +service +eventually +127001 +8bit +transcaucasian +servers +ns4csminingorg +wounded +quarantinelevel10000 +w6ua7cov2qraa80u4juaztc2ryyxyhvjh61e42tzw2hgppjq5bqnirj73bnllblxbqpcxslxq7 +to +took +charsetiso88591 +base +torch +now +modified +facearialnbspnbspnbspnbsp +utc +hampshirenew +company +fontweight +these +f117n100ing +sec111n100a +week +inreplyto +titlebold11px +friendatdtrtablebody +ccfl +49 +boyffriend +rgpujrqayobny449klerjxipxqcdamdptwuecigaqg9dsk4qojqswcdb5q0wboe1adn5j4ogna +addition +412222002733161846763r2d2 +annals +msgidfrommtaheader2 +bilbao +1 +through +replyto +points +been +instructionfontp +added +go +id +towering +rdns +acs +aligncenteralso +mar +tr +my +lies +b +23064401c23a7883301c506b01a8c0insuranceiqcom +charges +facebook +san +fm9krnmbjknpthnrmukdzjjvdqalnm4pyzk3b61rvychhgj0pcd1oa06kjgcyqkx6kvjqaucue +date +g +xbarracudastarttime +31 +border0atd +adenophora +2010 +typewriterbspan +style3dtextalign +xuidl +bizarre +bsuckstarts +fill +simple +lasernetpueblavirtualcommx +by +pstrongto +satofromaddrmatchhl +dprasworldnetattnet +hgqqyulyeclisn2y4eqfeyeexa3ok3qothfm8asjhgwo3njjcyiicnaecmbhjaihcsoo9ldjjo6u +valign3dtop3e3c2ftd3e3c2ftr3e3c2ftbody3e3c2ftable3e3cbr3e +virtually +trtd +university +p +height15td +not +xbarracudabblip +with +from +fontweightbold +type3dsubmit +1995 +x6falds46gdqals8vf3uwwy0evqe6nduxjpamhr6viuccni7go3hwgrhlvmi4nubueon8skavqp +width1 +10pxpowered +mimeversion +border0 +believes +give +bgcolorff0000 +000 +highyield +eating +south +include +aligncenter +session +fight +xmozillastatus +romanstrongcialis +gw1csminingorg +charsetutf8 +3b +available +jan +size2 +xaccountkey +absolutely +gmailid12830badb3615247 +altsuperior +an +targetblank +marketing09 +be +guwaazyu1fnriwopaszgdocroqeefgikjostpu8jatrmrdnapeecbdzibpayzlieemayuhbgg9o7 +where +impinging +deceased +xvirusscanned +title +power +xrnaw +tackle +º +brl +wanted +color3dffffff +pppa28resalekansascity14r7102dialinxnet +html +stylecolor +llengua +bsfsc5mj1840 +m111re +href3dhttpwwwwidxixidcnqvazifiaeqpq3d +aligncentersomep +recommendationustronguem +1884823157 +advancefee2 +xoriginalto +150 +ageinhibit +a20 +3 +word +no +very +illegal +add +mindful +jq25nyahoocom +59tmkulcnitzqske9ilqdfm0wvgds4yy6nikyklgoifkcnaqjmyucvniaiwarhgpwkajebksa +c3azwxsuk1jcwxghvjglnnf2nqswjgcbatjjrv5sae81sd7ipbalwd6844xu2j5gzubqhekt7d +does +stylestyle +lose +insertion +sty +along +bsfsc0satofromaddrmatch +contenttype +xmozillakeys +singapore +like +other +since +and +received +mega +bordercolor32336b +jednom +wed +free +697px +table +helod +leave +europe +collapse +the +send +parts +210234 +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_5.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_5.txt new file mode 100644 index 00000000..fead8364 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_5.txt @@ -0,0 +1,380 @@ +legala +california +1999fontbtdtr +xasgorigsubj +please +brto +for +charset3dutf8 +smtp +conditions +xbarracudaspamstatus +shop +entitlement +precisely +presently +schizophrenic +mailwebnotenet +anytime +years +young +name +g34si5943649wej920100516113019 +this +missingheaders +in +transitionalen +tocww6lmavwpx3xnrwsxipjrspeotjp2ntzrzjatk7m4wk4pxcie3zddm8mrrm0sni4pwkpwocbk +have +your +returnpath +suspicious +server2 +are +mon +source +his +built +100 +00000000 +elections +styletextaligncenterspan +subject +121 +706px +originally +size +dgggbgl0dgxlignvc3qgdg8gew91libjigzvbgxvd2vkdqp0agugc2ltcgxl +julnbsp2009fontstrongppnbspp +aligncentergobi +ryan +included +lowest +y2u9irzqt6lf6sigc2l6zt0incigy29sb3i9iimwmdawrkyiprncpmg8l2zvbnqpgzvbnqgzmfj +09 +ana +lbnyepl2uuqszsm4omqfpznhfas0uswcehrqlla1tzhedouwkuzc849x0ufy8d32b1wuqchzppot +bia +a96suajsyqg8vibg1e4njea4hayssqab7kkd8aoya7hapnjf2d3zxqxgvpjfv12ggh7jn696anta +stylecolor747474nbspcopy +appliances +243e22c414 +border +concentrate +yes +centerfo +jmjmasonorg +but +centerfont +value4444option +robbing +brbrfont +115c97ttering +212173515 +hundreds +redmond +timbila +style +ist +systems +if +br +3193991 +20082010 +delivered +population +clients +size3d2emailfontbtd +xbarracudaconnect +smtpmailysauadyco3604eurotelcz +all +httpdnbjackierxru58931d5277aa5b481dd1abe600a4010d +him +head +tahoma +tellus +26 +centernumbers +domain +50 +ny +sale +6yh2htacfeet1sq9qvf0umsoqrfdzyexafexdz6a5l63ihvdwakmpscpzzdijlody +receivedspf +articles +valuegoodgood +align3dcenterbfont +envex +confident +it +site +esmtp +option +215638 +estimates +marginwidth0 +boycott +a +dont +nowrapnbsptd +deliveredto +jmnetnoteinccom +became +andy567487yahoocom +humans +as +arrive +mimehtmlonly +credit +has +messageid +10 +mime +sa191f1 +especially +hrefhttp7a50tabjamie39aruyrasyye1d8f0a6daac4 +friendsrelatives +2112483279 +4ljj2ndu9p16ms9wtwog4r8mnkjwsq8whj9gs3dxnpdsfz61ll42ev8azbxmazvwbn +174553 +1010112520 +xbarracudaspamscore +popular +890 +listing +weinthal +6521715966 +cbyi +doctype +cdo +total +wings +west +they +taglevel10000 +deflexed +italy +contenttransferencoding +way +contacts +akron +outlook +8bit +bsmtpd +swiss +igfsd2f5cybmcmvlisa8ysbocmvmpsjodhrwoi8vbxlmcmvlcgf5c2l0zs41 +that +branislav +styletextalign +afontfont +1245743696 +quarantinelevel10000 +performers +residential +48br +to +sep +miss +charsetiso88591 +commitment +you +associated +httpspamgwcsminingorg8000cgibinmarkcgi +td +charset3dwindows +around +bgcolor3dffffff +share +home +oscar +font +robertgtaylorcsminingorg +here +width600 +version +webcam +mxgooglecom +factory +change +class3dtextbthere +at +1 +size3d2 +th +brbr +low +id +meta +width582 +offers +bgcolor000000 +pts +rdns +dimkhh0xuaguxufqf3flzw37dc6xownquow7h7e0ddlnwnmn09rcnkhhpgxlzisfdr7xa3ewern +over +cpunkswastemindernet +can +on +external +altbranded +hrefhttpa02fxqufojehcnibenumki63n8g2722955g455044li3wropozaoz61102176139027379476click +asked +reflecting +htmlmessage +tr +wants +complete +how +appears +courtroyal +xmsmailpriority +24720 +date +viagra +31 +coral +hope +o4pbpe6s012000 +2009 +authenticationresults +replytypeoriginal +42e0 +xuidl +mail +envelopefrom +3b9de4bf1 +by +da29d2c632 +chouer +wholesale +d97367fab23 +official +new +actually +xbeenthere +receiving +sa148ahl +p +not +with +from +texfq1rjt05tdqonckltu1vfuw0kdqpgzwrlcmf0aw9udqoncm1lzglhc3rpbmfsdqoncmfudgli +0 +person +mimeversion +icagicagicagicagicanciagicagicagicagicagicagidxmb250ignvbg9y +16px +000 +100rag111ns +first +aligncenter +arbitration +2009strong +3fb0e0profile +xmozillastatus +x21q7sczk2kwehgnhbnxxgy56uxrkihtt5fm14cmsbbuysr0yxzqo8sxwdslgpkonyxiynqlhdbn +electro +charsetutf8 +fetchmail590 +report +ten +hrefhttp8afdb0nzucocadcnyiqezy85nv28m876397hbg468typypi654021682483656937023 +sender +message +10px +19216818251 +place +offer +7bit +be +depiction +c +into +get +httpequivcontenttype +texthtml +editing +rules +033626 +were +xvirusscanned +anything +hrefhttpgvrlogduckrud80c870d65fview +ranging +special +more +existsxmailer +microsoft +anywhere +30 +r +mountain +2c4af43f99 +202911 +testsbsfsc0sa148a +assistance +html +httpequiv3dcontenttype +1719 +kosovohowever +bsfsc5mj1840 +concerned +firm +current +muscle +174900 +xoriginalto +0900 +1268570875079de6ff0001w4twrl +hrefhttpcbbdrugsiain81aruubaqytoh730b8a03d7a2c9 +1264975042 +value3d304704 +150 +3 +href3dhttpb93d7csorowxfcn +no +we +nupep +qzuveohejkkargromusouko5ukesminkcfkglc9cnuiw9bajugaskigacqag0hlict6hmgkenncb +lowered +iiypqralqvqqibs7vylilormpzn7dil5g4klu4czoboi5tcookiiephagrlhslufudec7ucel6r +japanese +stylestyle +treviso +great +jst +316 +1000 +contenttype +blending +hazardousi +filled +fictitious +feb +10pxfewfonta +secure +hill +other +of +states +received +servicemen +potential +19216818120 +wed +free +list +leerlingen +wish +table +14ytxrwst2agavnupbxjlogntdmwati4kboamkjdpstumvvq9tqlak24zetbmlk6jtxj3ob +helod +viruses +pr111pertie115 +view +derivadas +truffles +the +xpriority +those +width3d550 diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_6.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_6.txt new file mode 100644 index 00000000..fbbdde8b --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_6.txt @@ -0,0 +1,388 @@ +nbsp +menna +used +widely +xasgorigsubj +184323 +trouble +abbian111 +titleget +865cd7a196d +for +hrefhttp46franklinedrunujya715914adf8click +network +lots +smtp +212035 +hotels +rmb +further +191108 +appointed +people +any +rebuilt +mailwebnotenet +zurich +width29 +8 +area +this +content3dtexthtml +diversifieds +have +your +returnpath +jul +choose +digestive +breakdown +is +nights +stylecolor5f800dclick +xasgdebugid +god +companies +bgcolorffffff +19 +121406 +60029002180 +account2 +11px +bluea0 +reputation +062044 +server +abandoned +subject +custom +border3d1 +width579 +classstyle2a +n +75one +2002 +glycosides +other39s +countryifont +email +iervie5vdcbjagfuz2ugaxqgaw4gyw55ihdhes4gsxqgd29ya3mgzxhjzwvk +quotedprintable +alignmiddle +border +value3dinsuranceiq +yes +31108178 +15 +httpwwww3orgtrxhtml1dtdxhtml1transitionaldtd +topmargin0 +below +ns1csminingorg +text3d000000 +cam20 +080 +wb1wozrf3e79rhrzuwcyprs7gdrw99liuj37xaaoyxf3u79rhrzu5gtlsyn +online +style +if +pa +bgcolor3de0e0e0nbsptd +headertopbackgroundcolord3d3d3bordertop3px +popfriendly +br +o8avpi0baknr6oqyv3jpayhkhrf7b2x7v3ol72feupuw5gunl2tcwhcwspjmdlviepye9ihpvh +19216818250 +19216831 +bsfsc0sa392f +tolerance +big +rdnsnone +up +4bdb3f212cd6d47dtradalnet +fontfont +2500 +introduce +rg1ct58hiaswa4fsxsw71nkcxvxzbsypnwbinu6swkzlnameu9ck4qzrhctyxqygrjiqak4aylw +zero +noshade +t +normal +22 +band +26 +282177138 +had +facts +host +pleasespan +bvlgarirolexdior +laugh +titleonline +cellspacing0 +0400 +multipart +idea +hemoglobin +align3dcenterbfont +030253 +edt +just +8365980d93b +esmtp +24 +a +dont +calls +n5hmps1s016489 +deliveredto +styleverticalaligntop +smtpmailtimesharecarriedtrd2finestscorcom +hibodycsminingorg +barracuda +fornbsp +localhost +arialb +5 +messageid +10 +xasgtag +mime +addressa +biased +unknown +council +covering +alink3d663366 +1ehttcwkeaecjklonnghpe4uc2xhumt3u2ta0q0qe6sgi48va2pjjjfapjjjjtapz +icagicagicagie1vcnrnywdlpc9gt05upjwvqj48l0rjvj48l1repjwvvfi +fontspanfont +specified +sister +or +article +rites +n9sdvpg0029958 +taglevel10000 +0200 +81128112 +contenttransferencoding +multipartalternative +way +4069 +formerly +open +uid46431201487268 +1023139131 +eventually +television +i +127001 +unconscious +xmras +division +border3d0 +fungi +aircraft +parks +n91mgltr022871 +saving +to +singledrop +19216832 +titledoes +nicole +code +you +participation +httpspamgwcsminingorg8000cgibinmarkcgi +td +src3dhttpiiqusimages4s200201223agif +bank +claiming +presented +chapter +michel +advise +was +thrush +version +size2owed +mxgooglecom +found +at +1 +hrefhttpwwwcokodageruysygexyfymekjceb5d662abcb7 +52ft +added +go +id +meta +offers +seiner +pts +10112 +entertainment +rfc +1988 +on +ac +black +tr +how +language +sa424 +donahue +date +charged +stylecolor003c78 +width3d43 +xbarracudastarttime +takes +issues +ministere +boiling +width34a +offering +westminster +safetyfont +xuidl +9338145197ip71fastwebnetit +mail +colonizationem +valentine +coastnbspbrand +by +s8a7tr2svg2gvu6d9zwd2nqcpgakkkzqu6p8a1iuu2nrwcsx6igd65x7qqutnxo1fokxms +blacknbspnbsp +verdanawhen +pick +antispam +lang3d0bstaff +wwwdatalocalhost +nknwdz0v8awzantlwcsf2dexclj4qr6bzm8b4ud1mnkn8az0oahuy8akn9nxsj +february +v55045221200 +style3dcolor +httpinfolinkhrcxjmhtml4ynulwkhpp +saisissez +0pt +0100 +color666666span +their +p +idcontent +style3dborderright +hrefhttpbc3f6c59brownchooseruabout +81168112 +with +from +unsolicited +team +bar +person +b250igzhy2u9irzqt6lf6sigy29sb3i9iingrjawrkyiihnpemu9ijyipqazm6ztxmehpc9mb250 +fromlocalnovowel +parting +70 +taking +000 +sa331 +express +first +life +yestastyru +hrefhttpb8gjxdjfharuyzaqaqih532d4ae155c +sent +2822 +sa275bhl +xaccountkey +staff +webmastereditincom +anyone +themnbspfont +1258025765 +errorsto +pmiles +requires +round +an +prison +re97cti111n115 +aug +suggests +natural +be +get +church +rdnsdynamic +impinging +texthtml +rights +stylebackgroundcolorffffffbordertop0px +webmasterelectrowebcom +rules +nbspptd +were +bfonta +filetime71fd20e001c23159 +more +1388464 +sort +xbarracudavirusscanned +src3dhttpimg245imageshackusimg24510487176jpg +l6ffpaiyvyrm3jmq205hkscasrbhsrca4k989tvaupyg0toqwbivzop2zinhlejydnkv2ajni +testsbsfsc0sa148a +html +httpequiv3dcontenttype +deals +stylebdiv +clallam +ribe +metal +fri +3cu3e +about +ypjk +0900 +alink0033cc +174713 +wsysi62820e90e66c599723319 +3 +104904 +images +sources +no +hinged +we +major +12440203452da100a60000w4twrl +military +092607 +756j0fk913z +hibody +does +52 +policy +jst +day +bsfsc0satofromaddrmatch +contenttype +xmozillakeys +4327 +srchttpwwwahahealthcomassocimagese6e6dsamtote14gif +other +of +controlling +incurred +and +sponsored +received +today +phone +pragmatic +addresses +nmhxlvfapflvecuutfacyopamuajrrrqauuuuafjs0uaiaarmn0hoaztqnli6vpseucmm03vk2 +europe +21doctype +sa031 +the diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_7.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_7.txt new file mode 100644 index 00000000..893d526d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_7.txt @@ -0,0 +1,384 @@ +width3d670 +colorffffffpublish +recipient +tune +application +en118ir111nment +rue +xmailer +able +for +postfix +lunchboxes +smtp +xbarracudaspamstatus +bsfsc0satofromaddrmatchhl +involved +resultsbr +works +2084010766 +body +chksnsiubhxgsr9ewp8mmxiefxvyaro28hartmfcxccdbfhaapwmpfbeaulbbgeqemajfj1qbloe +approve +prove +c3 +name +grace +this +400 +lives +clickable +in +81258125 +transitionalen +your +off +returnpath +jul +tel +034856 +breakdown +are +is +031c16b24b3e3426a3b12dd05ae2hfbrnu +source +xasgdebugid +bgcolorffffff +00000000 +hrefhttponlinedesignradixwebnetdapper91html +070 +width31 +subject +xbarracudaspamflag +using +ehances +system +prior +included +2002 +ffffffweb +diseases +fontsize +height284a +httpwwwlinuxiepipermaililug +104 +cybercare +touch +below +103543 +future +height282 +would +psrykkemmnd46e2xrw2yg2o11bpztww5caekw2gzhovvz6svmp3sqngzhez0krs8h +yyyylocalhostnetnoteinccom +teach +levels +0102 +ns2csminingorg +20010803114020704271a6051rovdb001rovingcom +ist +if +xxsmallif +madame +size2ifont +scale +filetimef7b6512001c1f874 +62016 +alignleftprinting +clients +httpwwwbarracudacomreputationip1156921310 +our +ave +xbarracudaconnect +minute +do +all +subscriber +26 +unparseable +browse +17231331 +had +deliverydate +score152 +look +410 +cuirassiers +testimonies +aqeaf4m2bepsv6bpknrikyccfkpmjbtqdqpgraj9djil2w9m8yquzraielgrbniblrmsvffyb +0400 +bold +uri +edt +stylefontfamily +esmtp +24 +3928288 +charsetkoi8r +dont +kindly +deliveredto +xmozillastatus2 +create +localhost +moodenhancing +as +225pt +bfontp +bii +because +div +rule +size3d3onef +37077 +messageid +concealed +10 +localhostcsminingorg +enrolled +align3dcenter +stir +council +e0 +phoboslabsnetnoteinccom +cas +nmo4wogfc02abk4huhfqfniqsjaofuvkrzekngti9keymh96rvcx0ilgvudc1wht5xm80qgom8 +or +statement +allanktocsminingorg +doctype +fats +cognizance +such +they +nict +trampoline +xbarracudaurl +contenttransferencoding +hereabr +multipartalternative +21 +color3dff0000models +webbed +formerly +rudinkin +borderleft +textplain +8bit +charset3diso8859 +zzzzexamplecom +out +src3dhttp365sunshinehuicomcreditfixemaillinegifbr +49d10a687b034multipartmimeboundary +mutual +h5portuguese +to +ensure +xhtml +charsetiso88591 +least +typetextcss +code +you +httpspamgwcsminingorg8000cgibinmarkcgi +td +niue +teens +around +namegcblnqjpg +201002041038o14ac1vg001318ns1csminingorg +font +these +195679309258527jyadrzdfoydwsvo7816220961 +ghetto +rhdjprwg2hbydhttaridywcvofarumakak4ppt0jbwsktjdkuarg2hhzz9quqn2jyppvpifspla +24th +color3dff0000199fontfontdivtd +microsofts +nothing +8025 +htmlfontbig +thomas +at +been +u1rssunuifbssvzbq1k8l2zvbnqpgjypg0kicagicagicagifnwzwnpywwg +low +id +meta +breplica20 +3189319 +rdns +blvd +also +url0022httpinternetemail +asserted +can +being +htmlmessage +h5unit +hrefhttpurqpotentfigurecom +properties +tr +simplify +government +click +b +xmsmailpriority +beaches +heavy +href3dhttp60f667tewetemcnekoigetaci3d9478u2 +date +recordbreaking +eradicated +h2ofh2 +spamgwcsminingorg +windows1251bsgvybwvzifdhdgnozxm +who +seven +sciences +xbarracudaspamreport +authenticationresults +xuidl +order +mail +male +by +rates +origin +mourning +phillips +visit +width4 +unsubscribe +even +checks +raa09761 +antispam +should +060000 +paints +050 +called +altmainhead +p +software +trusted +l3nwyw4t1g2aaraqekqvrlepncm86zirmg7tmnqpc669lj0wcikar6lkvppc9mb250pjxmb250 +paddingright +with +from +bgcolor3d2223ebedcb22 +grande +0 +999999 +20 +softwarefontbtd +mimeversion +addres +nowp +gw2csminingorg +es +fontsize11px +exclusive +enjoy +026 +18px +size2lock +xmozillastatus +use +sent +ily +words +size2 +xaccountkey +282 +yuan20 +sender +precedence +n7uja4av011200 +200449 +message +morissette +455px +cuckoomany +19216818251 +an +place +herea +9ydntazxsknlelol +size3d4 +cc +targetblank +be +5d6f01ca57db22b9f3005288ba27aclightingcouk +detailed +get +xvirusscanned +alignrightofp +srchttpc2fbpoxokuwcnspacergif +033014 +several +straight +participate +30 +w3cdtd +neutral +color3dffffff +gwendolynmay +classnormal +html +inner +w +81168116 +really +none +29 +contains +fri +showcases +5simx +marginheight0 +engineering +kabini +seid1zxst3evhytxkzu4gw49cv4jquptr2st6lithm2hc9qzm7izle3e99l9oluglkplmz5bqgf +hibodycsminingorgbr +5638e5ddae35font +width100 +3 +drug +velit +no +alleviated +89448p +transform +width +ttype +yunnan +800 +entrepreneurs +hibody +great +inside +contenttype +agreements +081135 +inquiries +might +of +since +dynamiclooking +and +received +one +high +today +ms +free +wish +table +href3d22http3a2f2f2022e1012e1632e343a812fultimatehgh5frun2f22 +minor +view +the +insatiable +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_8.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_8.txt new file mode 100644 index 00000000..50eb4d3d --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_8.txt @@ -0,0 +1,370 @@ +neunet +harding +link242424 +widely +divfont +0300 +beautiful +serial +mission +serviceb +herebbr +romanstrongv +format +mother +for +st +trump +network +systematicsh5 +services +smtp +xbarracudaspamstatus +spiritual +adding +bsfsc0sa392hl +portion +body +nts7antzgbftq9b8qrp86rbasxcrlnvwlpccwikzs1satgasigb2aksigb1fjrqatfffac0ulf +operations +name +this +tercera +death +in +cdt +only20 +your +c4 +wblrbl7f3udxrsutzoskkkkacumlvacthv8atj6osuzrjpi7ythv2p8a6osgd5oooopl +returnpath +classcopyimg +height3d5td +breakdown +is +mon +gasoline +his +hindi +stylecolor5f800dclick +cwaaaaaqaaaaaqaaakaaaajaaaaaadhcsu0ehg1vukwgb3zlcnjpzgvzaaaabaaaaaa4qklnbbog +colspan3 +918 +021946 +built +displays +100 +o2d0t531004925 +direct20 +reputation +seen +location +subject +4 +using +system +gafontbr +texts +tdfont +bonuses +disk +yes +jmjmasonorg +below +102313469 +youfontbr +contenttexthtml +naj93ium021972 +wilmitdekalbnet +police +si +style +image +exceed +texteaebec +19216818250 +happen +tool +width800a +own +xbarracudarblip +fontfont +population +our +cellpadding3 +limited +mps +friend +eloquence +all +advertise +interrupted +head +100ivers111 +vlink3dc0c0c0 +phd +mattei +cash09 +raa26647 +9175 +pthe +ready +ke +receivedspf +size3d2mean +nangabadan +cellspacing0 +idea +trimming +bold +just +advertisement +site +esmtp +basisi +reading +impact +superior +term +a +crew +deliveredto +takeover +barracuda +localhost +build +because +div +apartment +24px +has +messageid +350 +g7nm2tz09890 +10 +44d1516f03 +xasgtag +mailtonogold4me666excitecomsubjectremove +psychological +re115ult +spnews +residents +public +doctype +recreational +contact +hrefhttpclicktopspecialofferscomcgibintplid1075060763006 +peruser +contenttransferencoding +way +puta +ctspresumed +mimeole +127001 +eq1rjtentjp8ackpinwjamvnkjp8kkklivkifeshl88aisf4uf8actawrj7zx1ztci +fu +helvetica +backtd +3189699 +out +classasotvlinkimg +lirefinance +cannot +width3d590 +aircraft +360ptspan +height3d219 +to +v1 +sep +interactive +ensure +sottolinea +you +td +village +department +lobbying +youfontfontdivbodyhtml +here +making +pjwvrk9ovd48rk9ovcbzdhlszt0irk9ovc1tsvpfoia2chqiia0kicagicag +help +repair +loan +version +visiting +nonecenterspan +found +1246002203 +pop3 +relationship +shootingb +continental +width371 +th +consisting +nonindustrialised +low +id +meta +north +over +poetry +also +profile +on +15pxpadding22px +patternlength +alfa +06 +payperview +b +successful +xmsmailpriority +dri +date +spamgwcsminingorg +member +take +xbarracudastarttime +alignleft +bordercolor3dcc3333 +hrefhttpwwwfurbidizcnautepqzi58182qfaxqumuutomigek58182unsubscribehibodycsminingorgunsubscribe +qnxmdrostcom +note +2009 +mail +respectfully +male +by +andcenter +unsubscribe +studios +hrefhttprealmedtechcomimg +111p111sici +o11cjord022711 +o0qf1zho018748 +curious +filenameecp10ccgif +software +boundaryboundaryofdocument +reservedbra +not +81168112 +with +from +189 +0 +alignleftchouth +o126x7ic008323 +mimeversion +stylefontsize +vowels +august +width3d478 +10px98ment2k2003br +andstrong +2071911661 +killlevel10000 +winds +aligncenter +fontispanspan +xmozillastatus +lang3d0nbspnbspnbsp +address +charsetutf8 +phpmailersourceforgenet +boundarynextpart00000233be1e640405ba1e005 +ibuthousands +message +which +accredited +control +an +020 +suc +37d60f6e688115b15863026nmzodjeteh6411613266 +turns +longer +roman +account +rollout +rdnsdynamic +size3onefontspan +pulitzera +name3dhdnrecipienttxt +ctl00ctl00ctl00ctl00rcrbsblnkcategoriesctl01linksctl01link +better +were +httpwwwinsurancemailnet +websitefontdiv +axis +more +denied +xbarracudavirusscanned +height3d2 +w3cdtd +form +hancock +activate +httpequiv3dcontenttype +sheffield +clarity +matches +value5151option +visuvondfyserieskathevolontariapparatchiksvoltagegammaltvotidc2vrijwilligerswatersportsganasarchipelagowc1xwdgtvuelingganginggastroair2waterwvaluewhujs2fwashappetitwarriorswallstwarshipwidelyusedwatermanwcsstorewatersolublewatertable2fmarketinggbvbmjwaymakerwebsupportweavinggearbeitet +bankruptcy +spam +0900 +according +encylopedia +see +ziyaret +srchttpmta33annfilesfourscombp5reloowuanicecxiriaeltfwaevxnadeeakbtoeineiioceeoqxeaaaiqceanakwie +3457158148 +bgcolor3d660000font +during +3several +no +very +we +213671963 +595af2c2d5 +having +e5fbfd +content3dmshtml +textalign +ywwgywr2zxj0axnlbwvudhmgd2hpy2ggyxjlihzpzxdlzcbiesb0ag91c2fu +including +width132 +contenttype +65553480 +httpwwwfsfginccom +may +events +of +csminingorg +and +640 +fontfamilytahoma +centeramtrak +received +one +today +sentence +dear +phone +02brz9re2cymif3f5v50wczwaq4nndk99gqezhzi7leeiurmtsoa5zfnwdwnhpkexqdy4 +table +marking +hrefhttpiwaydrugru954ad68b35a101bb70a22f6012923422 +product +onesu +the +alt3dfeatured +parts +yearfontfont +those +signal +only diff --git a/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_9.txt b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_9.txt new file mode 100644 index 00000000..6b557cc6 --- /dev/null +++ b/Figaro/src/test/resources/BookData/Training/SpamTrainingEmail_9.txt @@ -0,0 +1,349 @@ +bjkgnbxng1vdahiniqc0rawgbheojhu1c83xmommxytwaydjsstiqzyugrqyrlavatu5ys5oab +xasgorigsubj +homes +health +please +read +number +5px +textdecoration +for +postfix +program +referred +smtp +mailwebnotenet +xbarracudaheaderalert +body +hrefhttp2bmedeven33bruedinaec9d601f22763c7 +court +8 +area +name +the20 +this +401k +in +hr +firewall +returnpath +nov +point +breakdown +are +is +wait +nowatd +19 +colspan3 +value6363option +boldpaddingtop +makes +size3d22322 +account2 +00000000 +show +reputation +bit +aligncenteragep +explorer +subject +try +soon +cellpadding3d3 +xbarracudaspamflag +using +32bit +me9vbiyuafmb3fwipmjymnvlgecahkspmon4ihqlqbelxsodskupaodczkgjygo3u571njasrgn +system +than +concluded +2002 +want +winter +history +vic +email +quotedprintable +sansseriffontsize13pxcolor4ddb5e +11 +0000 +below +ns1csminingorg +challenging +future +width670 +contenttexthtml +causes +marginbottom +verdanaif +styleheight +image +1252059779 +world +br +annual +19216818250 +up +asp +etc +so +our +normal +urncontentclassesmessage +stone +webmasteregayzzncom +tue +head +26 +print +growth +dddddd +host +092114 +ul +height3d39font +msgidfrommtaheader +cellspacing0 +multipart +valign3dmiddle +subtracted +just +us +it +ore +united +84h97nks +aligncenternbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbspnbsp +a +112th +information +213620 +create +hibodycsminingorg +localhost +snicket +as +build +rule +mimehtmlonly +has +messageid +ywlsaw5nlca8l2zvbnqpc9mb250pjxmb250igzhy2u9imfyawfsiibzaxpl +pay +500 +10 +mime +width3d618 +0pxacentertd +council +efocu +hasb +0700 +target3dblankun +verh111117100ing +or +3351 +questions +sa218 +groups +peruser +hrefhttpct2consumertodaynetcgibin14floyhe10cczme0d40er0acchevy +following +contenttransferencoding +multipartalternative +facearialhelveticafont +cordelia +nonefederala +service +ppformcentertdtrtbodytablediv +zzzzlocalhost +127001 +8bit +monstercouk +g6ncmjxu073580 +that +styletextalign +200208020404faa00665webnotenet +to +quote +miss +charsetiso88591 +ngdbrxm0msqhfbwii7pnhllplhmcmthxamedfieklbimam2wik0ectyh2qvpz4qwbicggjbne +you +sa392c +nonechristiana +httpspamgwcsminingorg8000cgibinmarkcgi +h3font +32 +sign +operate +wh53afxjx3hdvi8r9hevnxt1vpvi8sncs6jpuszijk09xrnblb0kqm3rekcacacae9vqo7xv1g +font +realism +these +hbpnul563pib6gguwaaaaaaaaaaaaaaz9ze6ir27d0njiz7lypyjxlfhao8xk0xad3aiepydcfb9 +res +version +believe +payment +found +ecological +margin10in +tep9vjqhfynfv4or874loic45kkgsfat7w2fqj7t9lc5vd8fs47rwem6kubyanpy +at +through +leboutillier +helenaoption +go +id +meta +doesnt +over +ulfontb +on +width3d2270022 +sw9 +bgcolor0c7a65nbsptd +date +spamgwcsminingorg +18662377397font +31 +april +who +marketing +territory +impervious +2009 +xuidl +valign3dtop +by +1934 +unsubscribe +checks +however +cellspacing2trtd +margintop +antispam +004459 +new +111p111sici +titlelibrary +awdodg1hcmdpbj0imcigbwfyz2luagvpz2h0psiwiibtyxjnaw53awr0ad0i +style3dcolor +few +back +not +listsubscribe +with +from +0 +accelerating +0500 +stylefontsize +give +inc +000 +mj1963 +first +cellspacing3d0 +killlevel10000 +actions +james +xmozillastatus +wail +size3d20 +iiiii +width152 +repaid +nashua +apfont +xaccountkey +end +permitted +message +ie +pbayes +its +employment +property +ffffff +latin +percei118ing +htmltagexisttbody +paddingbottom5px +36284000 +heat +absolute +development +ew91cibzcgvha2vyczwvyj4sighpz2hsawdodgluzyb0agugdgv4dcbhcybpdcbyzwfkcyeg +stylemargin +train +rdnsdynamic +marketers +1002264587 +padding +xvirusscanned +ikip +several +1955 +noticeafont20 +form +fbi +064 +html +210568221 +gone +href +between +mail1csminingorg +householder +jimnetnogginscom +hyvezj9h2fxs4yab2fxs4ydhv7op2aadhv7op2hyvezj9gahyvezj9h2fxs4yab2fxs4ydhv +spam +thank +hermanos +c3ktdg8tcmvtzw1izxigzg9tywluig5hbwugzm9yihrozsbzyw1lihbyawnl +excess +brbeing +arial +images +width3d100 +athletic +vessels +we +808080br +content3dmshtml +366 +nameindexr3c3 +111214 +mainefontbr +some +hrefhttp77hardsupplyruevifaawi13418c9c4eca38e +utnhsljx3rvs0cwvekpnropbppmcq2etzylzlwgzgsyjyvplbhvk2pbfz0ekcnbzurxj56ynvr +contenttype +xmozillakeys +debating +owing +dario +may +inquiries +graphics +won +other +of +since +dynamiclooking +hrefhttp254118gsd423150mcomadulttoyhtmlclick +401 +received +19216818120 +free +name3dphone +hoy +helod +handle +syntactical +1973 +only +himalayan diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AbstractionTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AbstractionTest.scala index 4c238c0d..159d4673 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AbstractionTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AbstractionTest.scala @@ -23,6 +23,7 @@ import com.cra.figaro.language._ import com.cra.figaro.library.atomic.continuous._ import com.cra.figaro.library.compound._ import com.cra.figaro.test.tags.NonDeterministic +import com.cra.figaro.algorithm.factored.factors.factory.Factory class AbstractionTest extends WordSpec with Matchers { "A regular discretization" should { @@ -46,7 +47,7 @@ class AbstractionTest extends WordSpec with Matchers { u.addPragma(a) assert(u.pragmas contains a) } - + "not have a pragma after adding and removing it" in { Universe.createNew() val u = Uniform(0.0, 1.0) @@ -85,7 +86,7 @@ class AbstractionTest extends WordSpec with Matchers { "Computing the values of an abstract element" when { "given an atomic element using a regular discretization scheme" should { - "produce an sequence of the correct size of uniformly distributed values" taggedAs(NonDeterministic) in { + "produce an sequence of the correct size of uniformly distributed values" taggedAs (NonDeterministic) in { Universe.createNew() val u = Uniform(0.0, 1.0) u.addPragma(Abstraction(100)(AbstractionScheme.RegularDiscretization)) @@ -141,14 +142,15 @@ class AbstractionTest extends WordSpec with Matchers { val uniform = Uniform(0.0, max) uniform.addPragma(Abstraction(numBins)(AbstractionScheme.RegularDiscretization)) Values()(uniform) - val factors = Factory.make(uniform) + Variable(uniform) + val factors = Factory.makeFactorsForElement(uniform) factors.size should equal(1) val factor = factors(0) val variable = Variable(uniform) factor.variables should equal(List(variable)) - + factor.getIndices.foldLeft(0)((sum, _) => sum + 1) should equal(numBins) - for { indices <- factor.getIndices} { + for { indices <- factor.getIndices } { factor.get(indices) should be(1.0 / max +- 0.000001) // constant density of Uniform(0, max) } } @@ -165,7 +167,9 @@ class AbstractionTest extends WordSpec with Matchers { val apply = Apply(uniform, (d: Double) => d * d) apply.addPragma(Abstraction(numBinsApply)(AbstractionScheme.RegularDiscretization)) Values()(apply) - val factors = Factory.make(apply) + Variable(apply) + Variable(uniform) + val factors = Factory.makeFactorsForElement(apply) factors.size should equal(1) val factor = factors(0) val uniformVariable = Variable(uniform) @@ -204,12 +208,12 @@ class AbstractionTest extends WordSpec with Matchers { val apply = Apply(uniform1, uniform2, (d1: Double, d2: Double) => d1 * d2) apply.addPragma(Abstraction(numBinsApply)(AbstractionScheme.RegularDiscretization)) Values()(apply) - val factors = Factory.make(apply) - factors.size should equal(1) - val factor = factors(0) val uniform1Variable = Variable(uniform1) val uniform2Variable = Variable(uniform2) val applyVariable = Variable(apply) + val factors = Factory.makeFactorsForElement(apply) + factors.size should equal(1) + val factor = factors(0) factor.variables should equal(List(uniform1Variable, uniform2Variable, applyVariable)) // No longer true for sparse factors // factor.getIndices.foldLeft(0)((sum, _) => sum + 1) should equal(numBinsUniform * numBinsUniform * numBinsApply) @@ -249,13 +253,13 @@ class AbstractionTest extends WordSpec with Matchers { val apply = Apply(uniform1, uniform2, uniform3, (d1: Double, d2: Double, d3: Double) => d1 * d2 * d3) apply.addPragma(Abstraction(numBinsApply)(AbstractionScheme.RegularDiscretization)) Values()(apply) - val factors = Factory.make(apply) - factors.size should equal(1) - val factor = factors(0) val uniform1Variable = Variable(uniform1) val uniform2Variable = Variable(uniform2) val uniform3Variable = Variable(uniform3) val applyVariable = Variable(apply) + val factors = Factory.makeFactorsForElement(apply) + factors.size should equal(1) + val factor = factors(0) factor.variables should equal(List(uniform1Variable, uniform2Variable, uniform3Variable, applyVariable)) // No longer true for sparse factors // factor.getIndices.foldLeft(0)((sum, _) => sum + 1) should equal(numBinsUniform * numBinsUniform * numBinsUniform * numBinsApply) @@ -296,15 +300,15 @@ class AbstractionTest extends WordSpec with Matchers { val chain = Chain(flip, (b: Boolean) => if (b) uniform1; else uniform2) chain.addPragma(Abstraction(numBinsChain)(AbstractionScheme.RegularDiscretization)) Values()(chain) - val factors = Factory.make(chain) - factors.size should equal(3) // 1 for selection variable; 2 for the conditional selectors - val List(factor1, factor2, factor3) = factors - val selectorVar = factor1.variables(1) val flipVariable = Variable(flip) val uniform1Variable = Variable(uniform1) val uniform2Variable = Variable(uniform2) val chainVariable = Variable(chain) - factor1.variables should equal(List(flipVariable, selectorVar, chainVariable)) + val factors = Factory.makeFactorsForElement(chain) + factors.size should equal(3) // 1 for selection variable; 2 for the conditional selectors + val List(factor1, factor2, factor3) = factors + val selectorVar = factor1.variables(2) + factor1.variables should equal(List(flipVariable, chainVariable, selectorVar)) factor2.variables should equal(List(selectorVar, uniform1Variable)) factor2.getIndices.foldLeft(0)((sum, _) => sum + 1) should equal(2 * numBinsChain * numBinsUniform) factor3.variables should equal(List(selectorVar, uniform2Variable)) @@ -313,6 +317,7 @@ class AbstractionTest extends WordSpec with Matchers { val uniform1Values: List[Double] = uniform1Variable.range.map(_.value) val uniform2Values: List[Double] = uniform2Variable.range.map(_.value) val chainValues: List[Double] = chainVariable.range.map(_.value) + val selectorValues: List[List[Any]] = selectorVar.range.map(_.value.asInstanceOf[List[Any]]) def closest(chainValue: Double, uniformValue: Double): Boolean = { def minDiff: Double = (Double.MaxValue /: chainValues)((d1: Double, d2: Double) => d1 min math.abs(uniformValue - d2)) @@ -327,17 +332,20 @@ class AbstractionTest extends WordSpec with Matchers { j <- 0 until numBinsUniform k <- 0 until numBinsChain } { - val selectorBin = i * numBinsChain + k + val selectorBin = selectorValues.indexWhere(l => { + l(0).asInstanceOf[Regular[Boolean]].value == flipValues(i) && l(1).asInstanceOf[Regular[Double]].value == chainValues(k) + }) if (check1(flipValues(i), uniform1Values(j), chainValues(k))) { factor2.get(List(selectorBin, j)) should equal(1.0) } - else { factor1.get(List(selectorBin, j)) should equal(0.0) } + else { factor2.get(List(selectorBin, j)) should equal(0.0) } } for { i <- 0 to 1 j <- 0 until numBinsUniform k <- 0 until numBinsChain } { - val selectorBin = i * numBinsChain + k - + val selectorBin = selectorValues.indexWhere(l => { + l(0).asInstanceOf[Regular[Boolean]].value == flipValues(i) && l(1).asInstanceOf[Regular[Double]].value == chainValues(k) + }) if (check2(flipValues(i), uniform2Values(j), chainValues(k))) { factor3.get(List(selectorBin, j)) should equal(1.0) } else { factor3.get(List(selectorBin, j)) should equal(0.0) } } @@ -346,7 +354,7 @@ class AbstractionTest extends WordSpec with Matchers { } "Running variable elimination on a model with multiple discretized elements" should { - "produce the correct answer" taggedAs(NonDeterministic) in { + "produce the correct answer" taggedAs (NonDeterministic) in { Universe.createNew() val flip = Flip(0.5) val uniform1 = Uniform(0.0, 1.0) @@ -367,7 +375,7 @@ class AbstractionTest extends WordSpec with Matchers { * Uniform2 will result in (2,3) with expected weight 2.5. * Therefore flip should be around 0.4 for true. */ - ve.probability(flip, (b: Boolean) => b) should be(0.4 +- 0.02) + ve.probability(flip)(b => b) should be(0.4 +- 0.02) ve.kill } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AlgorithmTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AlgorithmTest.scala index 3e0fc767..afe1d356 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AlgorithmTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/AlgorithmTest.scala @@ -22,6 +22,8 @@ import com.cra.figaro.language._ import com.cra.figaro.library.atomic.continuous._ import com.cra.figaro.library.compound._ import scala.collection.mutable.Map +import com.cra.figaro.library.atomic.discrete.FromRange +import com.cra.figaro.library.atomic.discrete.Binomial class AlgorithmTest extends WordSpec with Matchers { "An algorithm" should { @@ -29,10 +31,10 @@ class AlgorithmTest extends WordSpec with Matchers { Universe.createNew() val c = Flip(0.3) val a = new SimpleAlgorithm(c) - an [AlgorithmInactiveException] should be thrownBy { a.distribution(c) } - an [AlgorithmInactiveException] should be thrownBy { a.expectation(c, (b: Boolean) => 1.0) } - an [AlgorithmInactiveException] should be thrownBy { a.probability(c, (b: Boolean) => true) } - an [AlgorithmInactiveException] should be thrownBy { a.probability(c, true) } + an[AlgorithmInactiveException] should be thrownBy { a.distribution(c) } + an[AlgorithmInactiveException] should be thrownBy { a.expectation(c, (b: Boolean) => 1.0) } + an[AlgorithmInactiveException] should be thrownBy { a.probability(c)(b => true) } + an[AlgorithmInactiveException] should be thrownBy { a.probability(c, true) } } "allow queries after starting, stopping, and resuming" in { @@ -42,17 +44,17 @@ class AlgorithmTest extends WordSpec with Matchers { a.start() a.distribution(c) a.expectation(c, (b: Boolean) => 1.0) - a.probability(c, (b: Boolean) => true) + a.probability(c)(b => true) a.probability(c, true) a.stop() a.distribution(c) - a.expectation(c, (b: Boolean) => 1.0) - a.probability(c, (b: Boolean) => true) + a.expectation(c)(b => 1.0) + a.probability(c)(b => true) a.probability(c, true) a.resume() a.distribution(c) - a.expectation(c, (b: Boolean) => 1.0) - a.probability(c, (b: Boolean) => true) + a.expectation(c)(b => 1.0) + a.probability(c)(b => true) a.probability(c, true) } @@ -62,10 +64,10 @@ class AlgorithmTest extends WordSpec with Matchers { val a = new SimpleAlgorithm(c) a.start() a.kill() - an [AlgorithmInactiveException] should be thrownBy { a.distribution(c) } - an [AlgorithmInactiveException] should be thrownBy { a.expectation(c, (b: Boolean) => 1.0) } - an [AlgorithmInactiveException] should be thrownBy { a.probability(c, (b: Boolean) => true) } - an [AlgorithmInactiveException] should be thrownBy { a.probability(c, true) } + an[AlgorithmInactiveException] should be thrownBy { a.distribution(c) } + an[AlgorithmInactiveException] should be thrownBy { a.expectation(c, (b: Boolean) => 1.0) } + an[AlgorithmInactiveException] should be thrownBy { a.probability(c)(b => true) } + an[AlgorithmInactiveException] should be thrownBy { a.probability(c, true) } } "not allow start after starting" in { @@ -73,16 +75,16 @@ class AlgorithmTest extends WordSpec with Matchers { val c = Flip(0.3) val a = new SimpleAlgorithm(c) a.start() - an [AlgorithmActiveException] should be thrownBy { a.start() } + an[AlgorithmActiveException] should be thrownBy { a.start() } } "not allow stop, resume, or kill before starting" in { Universe.createNew() val c = Flip(0.3) val a = new SimpleAlgorithm(c) - an [AlgorithmInactiveException] should be thrownBy { a.stop() } - an [AlgorithmInactiveException] should be thrownBy { a.resume() } - an [AlgorithmInactiveException] should be thrownBy { a.kill() } + an[AlgorithmInactiveException] should be thrownBy { a.stop() } + an[AlgorithmInactiveException] should be thrownBy { a.resume() } + an[AlgorithmInactiveException] should be thrownBy { a.kill() } } "allow start after starting and killing" in { @@ -99,7 +101,7 @@ class AlgorithmTest extends WordSpec with Matchers { val f = Flip(0.3) val a = new SimpleAlgorithm(f) a.start() - a.probability(f, (b: Boolean) => b) should equal(0.3) + a.probability(f)(b => b) should equal(0.3) } "compute the probability of a value" in { @@ -127,7 +129,7 @@ class AlgorithmTest extends WordSpec with Matchers { val a = new SimpleAnytime(c) a.start() a.stop() - val x = a.expectation(c, (b: Boolean) => -1.0) + val x = a.expectation(c)(b => -1.0) a.expectation(c, (b: Boolean) => -1.0) should equal(x) a.kill() } @@ -140,9 +142,33 @@ class AlgorithmTest extends WordSpec with Matchers { a.stop() a.resume() val x = a.expectation(c, (b: Boolean) => -1.0) - a.expectation(c, (b: Boolean) => -1.0) should be > (x) + a.expectation(c)(b => -1.0) should be > (x) a.kill() } + + "block on stop, kill or queries" in { + Universe.createNew() + val num = 400 + def makeModel(depth: Int, elems: List[Element[_]]): List[Element[_]] = { + if (depth > 0) { + val e = Chain(FromRange(2, 10), (i: Int) => Binomial(i, 0.2)) + makeModel(depth - 1, elems :+ e) + } else elems + } + val l = makeModel(num, List()) + + for { _ <- 0 until 20 } { + val alg = Importance(l: _*) + alg.start + Thread.sleep(1000) + alg.stop + val init = Universe.universe.activeElements.size + alg.kill + val after = Universe.universe.activeElements.size + after should equal (400*2) + } + + } } "A one time sampler" should { @@ -173,10 +199,10 @@ class AlgorithmTest extends WordSpec with Matchers { val myFlip = Flip(0.8) val s = new SimpleWeighted(myFlip) s.start() - s.probability(myFlip, (b: Boolean) => b) should be(1.0 / 3 +- 0.001) + s.probability(myFlip)(b => b) should be(1.0 / 3 +- 0.001) } } - + "A probability of query algorithm" should { "return the correct mean of an element" in { Universe.createNew() @@ -184,7 +210,7 @@ class AlgorithmTest extends WordSpec with Matchers { val n = Normal(u, 1) val imp = Importance(100000, n) imp.start() - imp.mean(n) should be (0.5 +- 0.01) + imp.mean(n) should be(0.5 +- 0.01) } "return the correct variance of an element" in { @@ -193,10 +219,10 @@ class AlgorithmTest extends WordSpec with Matchers { val n = Normal(u, 1) val imp = Importance(100000, n) imp.start() - imp.variance(n) should be (2.0 +- 0.05) + imp.variance(n) should be(2.0 +- 0.05) imp.kill() } - + "return the correct element representing the posterior probability distribution of an element" in { val u1 = Universe.createNew() val x = Flip(0.6) @@ -209,7 +235,7 @@ class AlgorithmTest extends WordSpec with Matchers { alg1.kill() val alg2 = VariableElimination(z) alg2.start() - alg2.probability(z, true) should be (0.75 +- 0.00000001) + alg2.probability(z, true) should be(0.75 +- 0.00000001) alg2.kill() } } @@ -217,7 +243,7 @@ class AlgorithmTest extends WordSpec with Matchers { class SimpleAlgorithm(val f: AtomicFlip) extends ProbQueryAlgorithm { lazy val universe = Universe.universe lazy val queryTargets = List(f) - + val p = f.prob def doStart() = () @@ -245,7 +271,7 @@ class AlgorithmTest extends WordSpec with Matchers { class SimpleAnytime(targets: Element[_]*) extends AnytimeProbQuery { lazy val universe = Universe.universe lazy val queryTargets = targets.toList - + var count = -1000000000 override def initialize() = count = 0 diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/BPTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/BPTest.scala index 366a11ec..6cbb2d96 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/BPTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/BPTest.scala @@ -26,6 +26,7 @@ import com.cra.figaro.library.atomic.continuous.{Uniform => CUniform} import com.cra.figaro.library.compound.IntSelector import com.cra.figaro.algorithm.lazyfactored.LazyValues import com.cra.figaro.algorithm.UnsupportedAlgorithmException +import com.cra.figaro.algorithm.factored.factors.factory.Factory class BPTest extends WordSpec with Matchers { @@ -39,7 +40,8 @@ class BPTest extends WordSpec with Matchers { val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) val semiring = SumProductSemiring() LazyValues(Universe.universe).expandAll(Universe.universe.activeElements.toSet.map((elem: Element[_]) => ((elem, Integer.MAX_VALUE)))) - val factors = Universe.universe.activeElements flatMap (Factory.make(_)) + Universe.universe.activeElements.foreach(Variable(_)) + val factors = Universe.universe.activeElements flatMap (Factory.makeFactorsForElement(_)) val graph = new BasicFactorGraph(factors, semiring) val fn = graph.adjacencyList.filter(p => { p._1 match { case fn: FactorNode => true; case _ => false; } }) val vn = graph.adjacencyList.filter(p => { p._1 match { case vn: VariableNode => true; case _ => false; } }) @@ -55,7 +57,8 @@ class BPTest extends WordSpec with Matchers { val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) val semiring = SumProductSemiring() LazyValues(Universe.universe).expandAll(Universe.universe.activeElements.toSet.map((elem: Element[_]) => ((elem, Integer.MAX_VALUE)))) - val factors = Universe.universe.activeElements flatMap (Factory.make(_)) + Universe.universe.activeElements.foreach(Variable(_)) + val factors = Universe.universe.activeElements flatMap (Factory.makeFactorsForElement(_)) val graph = new BasicFactorGraph(factors, semiring) val fn = graph.adjacencyList.filter(p => { p._1 match { case fn: FactorNode => true; case _ => false; } }) val vn = graph.adjacencyList.filter(p => { p._1 match { case vn: VariableNode => true; case _ => false; } }) @@ -116,9 +119,9 @@ class BPTest extends WordSpec with Matchers { val tol = 0.000001 bp.probability(e2, (i: Int) => i == 0) should be(e2_0 +- tol) - bp.probability(e2, (i: Int) => i == 1) should be(e2_1 +- tol) + bp.probability(e2)(_ == 1) should be(e2_1 +- tol) bp.probability(e2, (i: Int) => i == 2) should be(e2_2 +- tol) - bp.probability(e2, (i: Int) => i == 3) should be(e2_3 +- tol) + bp.probability(e2)(_ == 3) should be(e2_3 +- tol) } "with no conditions or constraints produce the correct result" in { @@ -187,7 +190,7 @@ class BPTest extends WordSpec with Matchers { val tolerance = 0.0000001 val algorithm = BeliefPropagation(10, f)(u1) algorithm.start() - algorithm.probability(f, (b: Boolean) => b) should be(0.6 +- globalTol) + algorithm.probability(f)(b => b) should be(0.6 +- globalTol) algorithm.kill() } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactorTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactorTest.scala index 00731b05..7c528d47 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactorTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactorTest.scala @@ -22,26 +22,15 @@ import com.cra.figaro.algorithm.lazyfactored.LazyValues import com.cra.figaro.algorithm.lazyfactored.Regular import com.cra.figaro.algorithm.lazyfactored.ValueSet import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler -import com.cra.figaro.language.Apply -import com.cra.figaro.language.Apply3 -import com.cra.figaro.language.Apply4 -import com.cra.figaro.language.Apply5 -import com.cra.figaro.language.CachingChain -import com.cra.figaro.language.Chain -import com.cra.figaro.language.Condition -import com.cra.figaro.language.Constant -import com.cra.figaro.language.Dist -import com.cra.figaro.language.Flip -import com.cra.figaro.language.Inject -import com.cra.figaro.language.Name.stringToName -import com.cra.figaro.language.NamedEvidence -import com.cra.figaro.language.Reference.stringToReference -import com.cra.figaro.language.Select -import com.cra.figaro.language.Universe +import com.cra.figaro.language._ import com.cra.figaro.library.atomic.continuous.Normal import com.cra.figaro.library.atomic.continuous.Uniform import com.cra.figaro.library.compound.CPD import com.cra.figaro.algorithm.factored.ParticleGenerator +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.factored.VariableElimination +import scala.collection.mutable.ListBuffer +import com.cra.figaro.algorithm.factored.factors.factory.Factory class FactorTest extends WordSpec with Matchers with PrivateMethodTester { @@ -68,20 +57,20 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { Universe.createNew() val e1 = Flip(0.2) Values()(e1) - val v1 = Variable(e1).id + val v1 = Variable(e1).id Variable.clearCache LazyValues.clear(Universe.universe) Values()(e1) val v2 = Variable(e1).id v1 should equal(v2) } - + "always be equal to a variable with the same id" in { Universe.createNew() val e1 = Flip(0.2) Values()(e1) val v1 = Variable(e1) - val v2 = new Variable(ValueSet.withStar(Set[Boolean]())) {override val id = v1.id} + val v2 = new Variable(ValueSet.withStar(Set[Boolean]())) { override val id = v1.id } v1 == v2 should equal(true) } @@ -112,40 +101,43 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v2 = Variable(e2) val v3 = Variable(e3) val v4 = Variable(e4) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) val indices = List(1, 0, 2, 1) f.set(indices, 0.2) f.set(indices, 0.3) f.get(indices) should equal(0.3) } - + + /* "get updated set of factors for an element when the factors have been updated" in { Universe.createNew() val v1 = Flip(0.5) Values()(v1) - val f1 = Factory.make(v1)(0) - val f1mod = f1.mapTo((d: Double) => 2.0*d) + val f1 = Factory.makeFactorsForElement(v1)(0) + val f1mod = f1.mapTo((d: Double) => 2.0 * d) Factory.updateFactor(v1, List(f1mod)) - Factory.make(v1)(0).get(List(0)) should equal(f1mod.get(List(0))) - } - -// "have the first index List be all zeros" in { -// Universe.createNew() -// val e1 = Flip(0.1) -// val e2 = Constant(8) -// val e3 = Select(0.2 -> "a", 0.3 -> "b", 0.5 -> "c") -// val e4 = Flip(0.7) -// Values()(e1) -// Values()(e2) -// Values()(e3) -// Values()(e4) -// val v1 = Variable(e1) -// val v2 = Variable(e2) -// val v3 = Variable(e3) -// val v4 = Variable(e4) -// val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) -// f.firstIndices should equal(List(0, 0, 0, 0)) -// } + Factory.makeFactorsForElement(v1)(0).get(List(0)) should equal(f1mod.get(List(0))) + } + * + */ + + // "have the first index List be all zeros" in { + // Universe.createNew() + // val e1 = Flip(0.1) + // val e2 = Constant(8) + // val e3 = Select(0.2 -> "a", 0.3 -> "b", 0.5 -> "c") + // val e4 = Flip(0.7) + // Values()(e1) + // Values()(e2) + // Values()(e3) + // Values()(e4) + // val v1 = Variable(e1) + // val v2 = Variable(e2) + // val v3 = Variable(e3) + // val v4 = Variable(e4) + // val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4)) + // f.firstIndices should equal(List(0, 0, 0, 0)) + // } "have the next index List carry and add correctly" in { Universe.createNew() @@ -161,7 +153,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v2 = Variable(e2) val v3 = Variable(e3) val v4 = Variable(e4) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) val ia = List(1, 0, 1, 1) val indices = f.getIndices val ar = indices.nextIndices(ia).get @@ -182,7 +174,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v2 = Variable(e2) val v3 = Variable(e3) val v4 = Variable(e4) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) val ia = List(1, 0, 2, 1) val indices = f.getIndices indices.nextIndices(ia) should equal(None) @@ -208,8 +200,8 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v4 = Variable(e4) val v5 = Variable(e5) val v6 = Variable(e6) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) val unionVars = PrivateMethod[(List[Variable[_]], List[Variable[_]], List[Int], List[Int])]('unionVars) val (parents, output, indexMap1, indexMap2) = f invokePrivate unionVars(g) val union = parents ::: output @@ -233,8 +225,8 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v2 = Variable(e2) val v3 = Variable(e3) val v4 = Variable(e4) - val f = Factory.simpleMake[Double](List(v1, v2, v3)) - val g = Factory.simpleMake[Double](List(v4, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3), List()) + val g = Factory.defaultFactor[Double](List(v4, v3), List()) f.set(List(0, 0, 0), 0.0) f.set(List(1, 0, 0), 0.1) f.set(List(2, 0, 0), 0.2) @@ -274,7 +266,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v1 = Variable(e1) val v2 = Variable(e2) val v3 = Variable(e3) - val f = Factory.simpleMake[Double](List(v1, v2, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3), List()) f.set(List(0, 0, 0), 0.0) f.set(List(1, 0, 0), 0.1) f.set(List(2, 0, 0), 0.2) @@ -299,7 +291,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v1 = Variable(e1) val v2 = Variable(e2) val v3 = Variable(e3) - val f = Factory.simpleMake[Double](List(v1, v2)) + val f = Factory.defaultFactor[Double](List(v1, v2), List()) f.set(List(0, 0), 0.0) f.set(List(1, 0), 0.2) f.set(List(2, 0), 0.4) @@ -319,7 +311,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { Values()(e2) val v1 = Variable(e1) val v2 = Variable(e2) - val f = Factory.simpleMake[Double](List(v1, v2, v1)) + val f = Factory.defaultFactor[Double](List(v1, v2, v1), List()) f.set(List(0, 0, 0), 0.1) f.set(List(1, 0, 0), 0.2) f.set(List(0, 1, 0), 0.3) @@ -347,15 +339,15 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v1 = Variable(e1) val v2 = Variable(e2) val v3 = Variable(e3) - val f = Factory.simpleMake[Double](List(v1, v2, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3), List()) f.set(List(0, 0, 0), 0.6) f.set(List(1, 0, 0), 0.1) f.set(List(2, 0, 0), 0.2) f.set(List(0, 0, 1), 0.3) f.set(List(1, 0, 1), 0.4) f.set(List(2, 0, 1), 0.5) - val g = f.recordArgMax(v3.asInstanceOf[Variable[Any]], - (x: Double, y: Double) => x < y) + val g = f.recordArgMax(v3.asInstanceOf[Variable[Any]], + (x: Double, y: Double) => x < y) g.variables should equal(List(v1, v2)) g.get(List(0, 0)) should equal(true) g.get(List(1, 0)) should equal(false) @@ -375,14 +367,14 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v1 = Variable(e1) val v2 = Variable(e2) val v3 = Variable(e3) - val f = Factory.simpleMake[Double](List(v1, v2, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3), List()) f.set(List(0, 0, 0), 0.0) f.set(List(1, 0, 0), 0.1) f.set(List(2, 0, 0), 0.2) f.set(List(0, 0, 1), 0.3) f.set(List(1, 0, 1), 0.4) f.set(List(2, 0, 1), 0.5) - val g = f.marginalizeTo(SumProductSemiring().asInstanceOf[Semiring[Double]], v3) + val g = f.marginalizeTo(v3) g.variables should equal(List(v3)) val p1 = 0.0 + 0.1 + 0.2 val p2 = 0.3 + 0.4 + 0.5 @@ -404,7 +396,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v1 = Variable(e1) val v2 = Variable(e2) val v3 = Variable(e3) - val f = Factory.simpleMake[Double](List(v1, v2, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3), List()) f.set(List(0, 0, 0), 0.0) f.set(List(1, 0, 0), 0.05) f.set(List(2, 0, 0), 0.1) @@ -417,7 +409,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { f.set(List(0, 1, 1), 0.15) f.set(List(1, 1, 1), 0.2) f.set(List(2, 1, 1), 0.25) - val g = f.marginalizeTo(SumProductSemiring().asInstanceOf[Semiring[Double]], v1, v3) + val g = f.marginalizeTo(v1, v3) g.variables should equal(List(v1, v3)) g.get(List(0, 0)) should be(0.0 +- 0.000001) g.get(List(1, 0)) should be(0.1 +- 0.000001) @@ -437,7 +429,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { Values()(e2) val v1 = Variable(e1) val v2 = Variable(e2) - val f = Factory.simpleMake[Double](List(v1, v2, v2)) + val f = Factory.defaultFactor[Double](List(v1, v2, v2), List()) f.set(List(0, 0, 0), 0.06) f.set(List(0, 0, 1), 0.25) f.set(List(0, 1, 0), 0.44) @@ -482,7 +474,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { Universe.createNew() val v1 = Constant(7) Values()(v1) - val List(factor) = Factory.make(v1) + val List(factor) = Factory.makeFactorsForElement(v1) factor.get(List(0)) should equal(1.0) } } @@ -493,7 +485,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { Universe.createNew() val v1 = Flip(0.3) Values()(v1) - val List(factor) = Factory.make(v1) + val List(factor) = Factory.makeFactorsForElement(v1) factor.get(List(0)) should equal(0.3) factor.get(List(1)) should equal(0.7) } @@ -506,7 +498,9 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v1 = Select(0.2 -> 0.1, 0.8 -> 0.3) val v2 = Flip(v1) Values()(v2) - val List(factor) = Factory.make(v2) + Variable(v1) + Variable(v2) + val List(factor) = Factory.makeFactorsForElement(v2) val vals = Variable(v1).range val i1 = vals.indexOf(Regular(0.1)) val i2 = vals.toList.indexOf(Regular(0.3)) @@ -522,7 +516,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { Universe.createNew() val v1 = Select(0.2 -> 1, 0.3 -> 0, 0.1 -> 2, 0.05 -> 5, 0.35 -> 4) Values()(v1) - val List(factor) = Factory.make(v1) + val List(factor) = Factory.makeFactorsForElement(v1) val vals = Variable(v1).range val i1 = vals.indexOf(Regular(1)) val i0 = vals.indexOf(Regular(0)) @@ -546,7 +540,8 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val c3 = Constant(0.5) val v3 = Select(v1 -> 1, v2 -> 2, c1 -> 4, c2 -> 5, c3 -> 3) Values()(v3) - val List(factor) = Factory.make(v3) + Universe.universe.activeElements.foreach(Variable(_)) + val List(factor) = Factory.makeFactorsForElement(v3) val v1Vals = Variable(v1).range val v2Vals = Variable(v2).range val v3Vals = Variable(v3).range @@ -597,20 +592,33 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v3FalseIndex = v3Vals.indexOf(Regular(false)) val v1Index = v3.outcomes.indexOf(v1) val v2Index = v3.outcomes.indexOf(v2) - val selectFactor :: outcomeFactors = Factory.make(v3) - outcomeFactors.size should equal(2) - val v1Factor = outcomeFactors(v1Index) - val v2Factor = outcomeFactors(v2Index) + Universe.universe.activeElements.foreach(Variable(_)) + val selectFactor :: outcomeFactors = Factory.makeFactorsForElement(v3) + outcomeFactors.size should equal(3) + val v1Factor = outcomeFactors(1+v1Index) + val v2Factor = outcomeFactors(1+v2Index) + val pairFactor = outcomeFactors(0) + val pairRange: List[List[Any]] = pairFactor.output.head.range.map(_.value.asInstanceOf[List[Any]]) selectFactor.get(List(v1Index)) should equal(0.3) selectFactor.get(List(v2Index)) should equal(0.7) - v1Factor.get(List(v1Index, v1TrueIndex, v3TrueIndex)) should equal(1.0) - v1Factor.get(List(v1Index, v1FalseIndex, v3TrueIndex)) should equal(0.0) - v1Factor.get(List(v1Index, v1TrueIndex, v3FalseIndex)) should equal(0.0) - v1Factor.get(List(v1Index, v1FalseIndex, v3FalseIndex)) should equal(1.0) - for { i <- 0 to 1; j <- 0 to 1 } v1Factor.get(List(v2Index, i, j)) should equal(1.0) - v2Factor.get(List(v2Index, 0, v3FalseIndex)) should equal(1.0) - v2Factor.get(List(v2Index, 0, v3TrueIndex)) should equal(0.0) - for { i <- 0 to 1} v2Factor.get(List(v1Index, 0, i)) should equal(1.0) + val index1 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == v1Index && p(1).asInstanceOf[Regular[Boolean]].value == true) + val index2 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == v1Index && p(1).asInstanceOf[Regular[Boolean]].value == false) + v1Factor.get(List(index1, v3TrueIndex)) should equal(1.0) + v1Factor.get(List(index2, v3TrueIndex)) should equal(0.0) + v1Factor.get(List(index1, v3FalseIndex)) should equal(0.0) + v1Factor.get(List(index2, v3FalseIndex)) should equal(1.0) + for { i <- 0 to 1; j <- 0 to 1 } { + val ind = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == v2Index && p(1).asInstanceOf[Regular[Boolean]].value == v3Vals(i).value) + v1Factor.get(List(ind, j)) should equal(1.0) + } + val v2index1 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == v2Index && p(1).asInstanceOf[Regular[Boolean]].value == true) + val v2index2 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == v2Index && p(1).asInstanceOf[Regular[Boolean]].value == false) + v2Factor.get(List(v2index1, v3FalseIndex)) should equal(1.0) + v2Factor.get(List(v2index1, v3TrueIndex)) should equal(0.0) + for { i <- 0 to 1 } { + val ind = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == v1Index && p(1).asInstanceOf[Regular[Boolean]].value == v3Vals(i).value) + v2Factor.get(List(ind, i)) should equal(1.0) + } } } @@ -639,10 +647,13 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v3t = v3Vals.indexOf(Regular(true)) val v5f = v5Vals.indexOf(Regular(false)) val v5t = v5Vals.indexOf(Regular(true)) - val selectFactor :: outcomeFactors = Factory.make(v5) - outcomeFactors.size should equal(2) - val v1Factor = outcomeFactors(v3Index) - val v2Factor = outcomeFactors(v4Index) + Universe.universe.activeElements.foreach(Variable(_)) + val selectFactor :: outcomeFactors = Factory.makeFactorsForElement(v5) + outcomeFactors.size should equal(3) + val v1Factor = outcomeFactors(1+v3Index) + val v2Factor = outcomeFactors(1+v4Index) + val pairFactor = outcomeFactors(0) + val pairRange: List[List[Any]] = pairFactor.output.head.range.map(_.value.asInstanceOf[List[Any]]) selectFactor.get(List(v102, v204, 0)) should be(0.2 / 0.6 +- 0.0001) selectFactor.get(List(v102, v204, 1)) should be(0.4 / 0.6 +- 0.0001) selectFactor.get(List(v102, v206, 0)) should be(0.2 / 0.8 +- 0.0001) @@ -651,132 +662,152 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { selectFactor.get(List(v108, v204, 1)) should be(0.4 / 1.2 +- 0.0001) selectFactor.get(List(v108, v206, 0)) should be(0.8 / 1.4 +- 0.0001) selectFactor.get(List(v108, v206, 1)) should be(0.6 / 1.4 +- 0.0001) - v1Factor.get(List(0, v3t, v5t)) should equal(1.0) - v1Factor.get(List(0, v3f, v5t)) should equal(0.0) - v1Factor.get(List(0, v3t, v5f)) should equal(0.0) - v1Factor.get(List(0, v3f, v5f)) should equal(1.0) - for { i <- 0 to 1; j <- 0 to 1 } v1Factor.get(List(1, i, j)) should equal(1.0) - v2Factor.get(List(1, 0, v5f)) should equal(1.0) - v2Factor.get(List(1, 0, v5t)) should equal(0.0) - for { i <- 0 to 0; j <- 0 to 1 } v2Factor.get(List(0, i, j)) should equal(1.0) + val v1index1 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == 0 && p(1).asInstanceOf[Regular[Boolean]].value == true) + val v1index2 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == 0 && p(1).asInstanceOf[Regular[Boolean]].value == false) + v1Factor.get(List(v1index1, v5t)) should equal(1.0) + v1Factor.get(List(v1index2, v5t)) should equal(0.0) + v1Factor.get(List(v1index1, v5f)) should equal(0.0) + v1Factor.get(List(v1index2, v5f)) should equal(1.0) + for { i <- 0 to 1; j <- 0 to 1 } { + val ind = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == 1 && p(1).asInstanceOf[Regular[Boolean]].value == v5Vals(i).value) + v1Factor.get(List(ind, j)) should equal(1.0) + } + val v2index1 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == 1 && p(1).asInstanceOf[Regular[Boolean]].value == v5Vals(0).value) + //val v2index2 = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == 1 && p(1).asInstanceOf[Regular[Boolean]].value == fals) + v2Factor.get(List(v2index1, v5f)) should equal(1.0) + v2Factor.get(List(v2index1, v5t)) should equal(0.0) + for { i <- 0 to 0; j <- 0 to 1 } { + val ind = pairRange.indexWhere(p => p(0).asInstanceOf[Regular[Int]].value == 0 && p(1).asInstanceOf[Regular[Boolean]].value == v5Vals(i).value) + v2Factor.get(List(ind, j)) should equal(1.0) + } } } "given an atomic not in the factor" should { "automatically sample the element" in { - Universe.createNew() + Universe.createNew() val v1 = Normal(0.0, 1.0) Values()(v1) - val factor = Factory.make(v1) - factor(0).size should equal(ParticleGenerator.defaultArgSamples) - factor(0).get(List(0)) should equal(1.0/ParticleGenerator.defaultArgSamples) + val factor = Factory.makeFactorsForElement(v1) + factor(0).size should equal(ParticleGenerator.defaultNumSamplesFromAtomics) + factor(0).get(List(0)) should equal(1.0 / ParticleGenerator.defaultNumSamplesFromAtomics) + } + + "correctly create factors for continuous elements through chains" in { + val uni = Universe.createNew() + val elem = If(Flip(0.3), Uniform(0.0, 1.0), Uniform(1.0, 2.0)) + ParticleGenerator(uni) + val alg = VariableElimination(elem) + alg.start() + alg.distribution(elem).toList.size should be(14) + } } - -// "given a chain" should { -// "produce a conditional selector for each parent value" in { -// Universe.createNew() -// val v1 = Flip(0.2) -// val v2 = Select(0.1 -> 1, 0.9 -> 2) -// val v3 = Constant(3) -// val v4 = Chain(v1, (b: Boolean) => if (b) v2; else v3) -// Values()(v4) -// val v1Vals = Variable(v1).range -// val v2Vals = Variable(v2).range -// val v4Vals = Variable(v4).range -// val v1t = v1Vals indexOf Regular(true) -// val v1f = v1Vals indexOf Regular(false) -// val v21 = v2Vals indexOf Regular(1) -// val v22 = v2Vals indexOf Regular(2) -// val v41 = v4Vals indexOf Regular(1) -// val v42 = v4Vals indexOf Regular(2) -// val v43 = v4Vals indexOf Regular(3) -// -// val factor = Factory.make(v4) -// val List(v4Factor) = Factory.combineFactors(factor, SumProductSemiring, true) -// -// v4Factor.get(List(v1t, v21, 0, v41)) should equal(1.0) -// v4Factor.get(List(v1t, v22, 0, v41)) should equal(0.0) -// v4Factor.get(List(v1t, v21, 0, v42)) should equal(0.0) -// v4Factor.get(List(v1t, v22, 0, v42)) should equal(1.0) -// v4Factor.get(List(v1t, v21, 0, v43)) should equal(0.0) -// v4Factor.get(List(v1t, v22, 0, v43)) should equal(0.0) -// v4Factor.get(List(v1f, v21, 0, v41)) should equal(0.0) -// v4Factor.get(List(v1f, v22, 0, v41)) should equal(0.0) -// v4Factor.get(List(v1f, v21, 0, v42)) should equal(0.0) -// v4Factor.get(List(v1f, v22, 0, v42)) should equal(0.0) -// v4Factor.get(List(v1f, v21, 0, v43)) should equal(1.0) -// v4Factor.get(List(v1f, v22, 0, v43)) should equal(1.0) -// -// } -// -// "produce a conditional selector for each non-temporary parent value" in { -// Universe.createNew() -// val v1 = Flip(0.2) -// val v4 = Chain(v1, (b: Boolean) => if (b) Select(0.1 -> 1, 0.9 -> 2); else Constant(3)) -// Values()(v4) -// val v1Vals = Variable(v1).range -// val v4Vals = Variable(v4).range -// -// val v1t = v1Vals indexOf Regular(true) -// val v1f = v1Vals indexOf Regular(false) -// val v41 = v4Vals indexOf Regular(1) -// val v42 = v4Vals indexOf Regular(2) -// val v43 = v4Vals indexOf Regular(3) -// -// val factor = Factory.make(v4) -// val List(v4Factor) = Factory.combineFactors(factor, SumProductSemiring, true) -// -// v4Factor.get(List(v1t, v41)) should equal(0.1) -// v4Factor.get(List(v1t, v42)) should equal(0.9) -// v4Factor.get(List(v1t, v43)) should equal(0.0) -// v4Factor.get(List(v1f, v41)) should equal(0.0) -// v4Factor.get(List(v1f, v42)) should equal(0.0) -// v4Factor.get(List(v1f, v43)) should equal(1.0) -// } -// } - -// "given a CPD with one argument" should { -// "produce a single factor with a case for each parent value" in { -// Universe.createNew() -// val v1 = Flip(0.2) -// -// val v2 = CPD(v1, false -> Flip(0.1), true -> Flip(0.7)) -// Values()(v2) -// -// val v1Vals = Variable(v1).range -// val v2Vals = Variable(v2).range -// -// val v1t = v1Vals indexOf Regular(true) -// val v1f = v1Vals indexOf Regular(false) -// val v2t = v2Vals indexOf Regular(true) -// val v2f = v2Vals indexOf Regular(false) -// val v3t = 0 -// val v3f = 1 -// val v4t = 0 -// val v4f = 1 -// -// val factor = Factory.make(v2) -// val List(v2Factor) = Factory.combineFactors(factor, SumProductSemiring, true) -// -// v2Factor.get(List(v1t, v3t, v4t, v2t)) should equal(1.0) -// v2Factor.get(List(v1t, v3t, v4f, v2t)) should equal(1.0) -// v2Factor.get(List(v1t, v3f, v4t, v2t)) should equal(0.0) -// v2Factor.get(List(v1t, v3f, v4f, v2t)) should equal(0.0) -// v2Factor.get(List(v1t, v3t, v4t, v2f)) should equal(0.0) -// v2Factor.get(List(v1t, v3t, v4f, v2f)) should equal(0.0) -// v2Factor.get(List(v1t, v3f, v4t, v2f)) should equal(1.0) -// v2Factor.get(List(v1t, v3f, v4f, v2f)) should equal(1.0) -// v2Factor.get(List(v1f, v3t, v4t, v2t)) should equal(1.0) -// v2Factor.get(List(v1f, v3t, v4f, v2t)) should equal(0.0) -// v2Factor.get(List(v1f, v3f, v4t, v2t)) should equal(1.0) -// v2Factor.get(List(v1f, v3f, v4f, v2t)) should equal(0.0) -// v2Factor.get(List(v1f, v3t, v4t, v2f)) should equal(0.0) -// v2Factor.get(List(v1f, v3t, v4f, v2f)) should equal(1.0) -// v2Factor.get(List(v1f, v3f, v4t, v2f)) should equal(0.0) -// v2Factor.get(List(v1f, v3f, v4f, v2f)) should equal(1.0) -// } -// } + + // "given a chain" should { + // "produce a conditional selector for each parent value" in { + // Universe.createNew() + // val v1 = Flip(0.2) + // val v2 = Select(0.1 -> 1, 0.9 -> 2) + // val v3 = Constant(3) + // val v4 = Chain(v1, (b: Boolean) => if (b) v2; else v3) + // Values()(v4) + // val v1Vals = Variable(v1).range + // val v2Vals = Variable(v2).range + // val v4Vals = Variable(v4).range + // val v1t = v1Vals indexOf Regular(true) + // val v1f = v1Vals indexOf Regular(false) + // val v21 = v2Vals indexOf Regular(1) + // val v22 = v2Vals indexOf Regular(2) + // val v41 = v4Vals indexOf Regular(1) + // val v42 = v4Vals indexOf Regular(2) + // val v43 = v4Vals indexOf Regular(3) + // + // val factor = Factory.makeFactorsForElement(v4) + // val List(v4Factor) = Factory.combineFactors(factor, SumProductSemiring, true) + // + // v4Factor.get(List(v1t, v21, 0, v41)) should equal(1.0) + // v4Factor.get(List(v1t, v22, 0, v41)) should equal(0.0) + // v4Factor.get(List(v1t, v21, 0, v42)) should equal(0.0) + // v4Factor.get(List(v1t, v22, 0, v42)) should equal(1.0) + // v4Factor.get(List(v1t, v21, 0, v43)) should equal(0.0) + // v4Factor.get(List(v1t, v22, 0, v43)) should equal(0.0) + // v4Factor.get(List(v1f, v21, 0, v41)) should equal(0.0) + // v4Factor.get(List(v1f, v22, 0, v41)) should equal(0.0) + // v4Factor.get(List(v1f, v21, 0, v42)) should equal(0.0) + // v4Factor.get(List(v1f, v22, 0, v42)) should equal(0.0) + // v4Factor.get(List(v1f, v21, 0, v43)) should equal(1.0) + // v4Factor.get(List(v1f, v22, 0, v43)) should equal(1.0) + // + // } + // + // "produce a conditional selector for each non-temporary parent value" in { + // Universe.createNew() + // val v1 = Flip(0.2) + // val v4 = Chain(v1, (b: Boolean) => if (b) Select(0.1 -> 1, 0.9 -> 2); else Constant(3)) + // Values()(v4) + // val v1Vals = Variable(v1).range + // val v4Vals = Variable(v4).range + // + // val v1t = v1Vals indexOf Regular(true) + // val v1f = v1Vals indexOf Regular(false) + // val v41 = v4Vals indexOf Regular(1) + // val v42 = v4Vals indexOf Regular(2) + // val v43 = v4Vals indexOf Regular(3) + // + // val factor = Factory.makeFactorsForElement(v4) + // val List(v4Factor) = Factory.combineFactors(factor, SumProductSemiring, true) + // + // v4Factor.get(List(v1t, v41)) should equal(0.1) + // v4Factor.get(List(v1t, v42)) should equal(0.9) + // v4Factor.get(List(v1t, v43)) should equal(0.0) + // v4Factor.get(List(v1f, v41)) should equal(0.0) + // v4Factor.get(List(v1f, v42)) should equal(0.0) + // v4Factor.get(List(v1f, v43)) should equal(1.0) + // } + // } + + // "given a CPD with one argument" should { + // "produce a single factor with a case for each parent value" in { + // Universe.createNew() + // val v1 = Flip(0.2) + // + // val v2 = CPD(v1, false -> Flip(0.1), true -> Flip(0.7)) + // Values()(v2) + // + // val v1Vals = Variable(v1).range + // val v2Vals = Variable(v2).range + // + // val v1t = v1Vals indexOf Regular(true) + // val v1f = v1Vals indexOf Regular(false) + // val v2t = v2Vals indexOf Regular(true) + // val v2f = v2Vals indexOf Regular(false) + // val v3t = 0 + // val v3f = 1 + // val v4t = 0 + // val v4f = 1 + // + // val factor = Factory.makeFactorsForElement(v2) + // val List(v2Factor) = Factory.combineFactors(factor, SumProductSemiring, true) + // + // v2Factor.get(List(v1t, v3t, v4t, v2t)) should equal(1.0) + // v2Factor.get(List(v1t, v3t, v4f, v2t)) should equal(1.0) + // v2Factor.get(List(v1t, v3f, v4t, v2t)) should equal(0.0) + // v2Factor.get(List(v1t, v3f, v4f, v2t)) should equal(0.0) + // v2Factor.get(List(v1t, v3t, v4t, v2f)) should equal(0.0) + // v2Factor.get(List(v1t, v3t, v4f, v2f)) should equal(0.0) + // v2Factor.get(List(v1t, v3f, v4t, v2f)) should equal(1.0) + // v2Factor.get(List(v1t, v3f, v4f, v2f)) should equal(1.0) + // v2Factor.get(List(v1f, v3t, v4t, v2t)) should equal(1.0) + // v2Factor.get(List(v1f, v3t, v4f, v2t)) should equal(0.0) + // v2Factor.get(List(v1f, v3f, v4t, v2t)) should equal(1.0) + // v2Factor.get(List(v1f, v3f, v4f, v2t)) should equal(0.0) + // v2Factor.get(List(v1f, v3t, v4t, v2f)) should equal(0.0) + // v2Factor.get(List(v1f, v3t, v4f, v2f)) should equal(1.0) + // v2Factor.get(List(v1f, v3f, v4t, v2f)) should equal(0.0) + // v2Factor.get(List(v1f, v3f, v4f, v2f)) should equal(1.0) + // } + // } "given an apply of one argument" should { "produce a factor that matches the argument to the result via the function" in { @@ -791,7 +822,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v13 = v1Vals indexOf Regular(3) val v20 = v2Vals indexOf Regular(0) val v21 = v2Vals indexOf Regular(1) - val List(factor) = Factory.make(v2) + val List(factor) = Factory.makeFactorsForElement(v2) factor.contains(List(v11, v20)) should equal(false) factor.get(List(v11, v21)) should equal(1.0) factor.get(List(v12, v20)) should equal(1.0) @@ -820,7 +851,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v30 = v3Vals indexOf Regular(0) val v31 = v3Vals indexOf Regular(1) val v32 = v3Vals indexOf Regular(2) - val List(factor) = Factory.make(v3) + val List(factor) = Factory.makeFactorsForElement(v3) factor.contains(List(v11, v22, v30)) should equal(false) factor.get(List(v11, v22, v31)) should equal(1.0) factor.contains(List(v11, v22, v32)) should equal(false) @@ -864,7 +895,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v40 = v4Vals indexOf Regular(0) val v41 = v4Vals indexOf Regular(1) val v42 = v4Vals indexOf Regular(2) - val List(factor) = Factory.make(v4) + val List(factor) = Factory.makeFactorsForElement(v4) factor.contains(List(v11, v21, v31, v40)) should equal(false) factor.get(List(v11, v21, v31, v41)) should equal(1.0) factor.contains(List(v11, v21, v31, v42)) should equal(false) @@ -913,7 +944,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v50 = v5Vals indexOf Regular(0) val v51 = v5Vals indexOf Regular(1) val v52 = v5Vals indexOf Regular(2) - val List(factor) = Factory.make(v5) + val List(factor) = Factory.makeFactorsForElement(v5) factor.contains(List(v11, v21, v31, v4false, v50)) should equal(false) factor.get(List(v11, v21, v31, v4false, v51)) should equal(1.0) factor.contains(List(v11, v21, v31, v4false, v52)) should equal(false) @@ -985,7 +1016,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v60 = v6Vals indexOf Regular(0) val v61 = v6Vals indexOf Regular(1) val v62 = v6Vals indexOf Regular(2) - val List(factor) = Factory.make(v6) + val List(factor) = Factory.makeFactorsForElement(v6) factor.contains(List(v11, v21, v31, v4false, v5false, v60)) should equal(false) factor.get(List(v11, v21, v31, v4false, v5false, v61)) should equal(1.0) @@ -1035,7 +1066,8 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v2 = Select(0.5 -> 4, 0.5 -> 5) val v3 = Inject(v1, v2) Values()(v3) - val List(factor) = Factory.make(v3) + Universe.universe.activeElements.foreach(Variable(_)) + val List(factor) = Factory.makeFactorsForElement(v3) val v1Vals = Variable(v1).range val v2Vals = Variable(v2).range @@ -1103,7 +1135,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { v1.setCondition((i: Int) => i != 2) v1.setConstraint(((i: Int) => i.toDouble)) Values()(v1) - val List(condFactor, constrFactor, _) = Factory.make(v1) + val List(condFactor, constrFactor, _) = Factory.makeFactorsForElement(v1) val v1Vals = Variable(v1).range val v11 = v1Vals indexOf Regular(1) val v12 = v1Vals indexOf Regular(2) @@ -1123,7 +1155,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val f = Flip(0.5) val lv = LazyValues() lv.expandAll(Set((f, -1))) - val factors = Factory.make(f) + val factors = Factory.makeFactorsForElement(f) factors should be(empty) } } @@ -1142,8 +1174,10 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val a = CachingChain(x, y, (x: Boolean, y: Int) => if (x || y < 2) u1; else u2)("a", dependentUniverse) Values(dependentUniverse)(a) val evidence = List(NamedEvidence("a", Condition((d: Double) => d < 0.5))) + Universe.universe.activeElements.foreach(Variable(_)) + dependentUniverse.activeElements.foreach(Variable(_)) val factor = - Factory.makeDependentFactor(Universe.universe, dependentUniverse, () => ProbEvidenceSampler.computeProbEvidence(20000, evidence)(dependentUniverse)) + Factory.makeDependentFactor(Variable.cc, Universe.universe, dependentUniverse, () => ProbEvidenceSampler.computeProbEvidence(20000, evidence)(dependentUniverse)) val xVar = Variable(x) val yVar = Variable(y) val variables = factor.variables @@ -1247,7 +1281,7 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v13 = v1Vals indexOf Regular(3) val v20 = v2Vals indexOf Regular(0) val v21 = v2Vals indexOf Regular(1) - val List(factor) = Factory.make(v2) + val List(factor) = Factory.makeFactorsForElement(v2) factor.get(List(v10, v20)) should equal(1.0) factor.contains(List(v10, v21)) should equal(false) factor.get(List(v11, v20)) should equal(1.0) @@ -1280,9 +1314,12 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { val v40 = v4Vals indexOf Regular(0) val v41 = v4Vals indexOf Regular(1) val v42 = v4Vals indexOf Regular(2) - - val factor = Factory.make(v4) - val List(v4Factor) = Factory.combineFactors(factor, SumProductSemiring().asInstanceOf[Semiring[Double]], true) + Variable(v1) + Variable(v2) + Variable(v3) + Variable(v4) + val factor = Factory.makeFactorsForElement(v4) + val List(v4Factor) = combineFactors(factor, SumProductSemiring().asInstanceOf[Semiring[Double]], true) v4Factor.get(List(v10, v40, v30, 0)) should equal(0.0) v4Factor.get(List(v10, v40, v31, 0)) should equal(0.0) @@ -1305,4 +1342,115 @@ class FactorTest extends WordSpec with Matchers with PrivateMethodTester { } } } + + val maxElementCount = 6 + val maxSize = 500 + val newFactors = ListBuffer[Factor[Double]]() + val tempFactors = ListBuffer[Factor[Double]]() + + /** + * Combines a set of factors into a single larger factor. This method is used when a factor has + * been decomposed into many dependent Factors and a single Factor is required. + */ + def combineFactors(oldFactors: List[Factor[Double]], semiring: Semiring[Double], removeTemporaries: Boolean): List[Factor[Double]] = { + newFactors.clear + tempFactors.clear + + for (factor <- oldFactors) { + if (factor.hasStar) { + newFactors += factor + } else { + tempFactors += factor + } + } + + var nextFactor = tempFactors.head + + for (factor <- tempFactors.tail) { + val commonVariables = factor.variables.toSet & nextFactor.variables.toSet + + if (commonVariables.size > 0) { + val newVariables = factor.variables.toSet -- nextFactor.variables.toSet + val potentialSize = calculateSize(nextFactor.size, newVariables) + if ((nextFactor.numVars + newVariables.size) < maxElementCount + && potentialSize < maxSize) { + nextFactor = factor.product(nextFactor) + } else { + if (removeTemporaries) { + newFactors ++= reduceFactor(nextFactor, semiring, maxElementCount) + } else { + newFactors += nextFactor + } + nextFactor = factor + } + } else { + newFactors += nextFactor + nextFactor = factor + } + } + + if (nextFactor.numVars > 0) { + if (removeTemporaries) { + newFactors ++= reduceFactor(nextFactor, semiring, maxElementCount) + } else { + newFactors += nextFactor + } + } + newFactors.toList + } + + val variableSet = scala.collection.mutable.Set[Variable[_]]() + val nextFactors = ListBuffer[Factor[Double]]() + + private def reduceFactor(factor: Factor[Double], semiring: Semiring[Double], maxElementCount: Int): List[Factor[Double]] = { + variableSet.clear + + var resultFactor = Factory.unit[Double](semiring).product(factor) + + (variableSet /: List(factor))(_ ++= _.variables.asInstanceOf[List[Variable[_]]]) + for (variable <- variableSet.filter { _.isInstanceOf[InternalVariable[_]] }) { + resultFactor = resultFactor.sumOver(variable) + variableSet.remove(variable) + } + + var elementCount = variableSet count (v => !isTemporary(v)) + + var tempCount = 0; + + for { variable <- variableSet } { + if (isTemporary(variable) && elementCount <= maxElementCount) { + nextFactors.clear + nextFactors ++= Factory.concreteFactors(Variable.cc, variable.asInstanceOf[ElementVariable[_]].element, false) + (variableSet /: nextFactors)(_ ++= _.variables.asInstanceOf[List[ElementVariable[_]]]) + elementCount = variableSet count (v => !isTemporary(v)) + + for (nextFactor <- nextFactors) { + resultFactor = resultFactor.product(nextFactor) + } + tempCount += 1 + } + } + + if (tempCount > 0 && elementCount <= maxElementCount) { + for { variable <- variableSet } { + if (isTemporary(variable)) { + resultFactor = resultFactor.sumOver(variable) + } + } + + } + List(resultFactor) + } + + private def calculateSize(currentSize: Int, variables: Set[Variable[_]]) = { + (currentSize /: variables)(_ * _.size) + } + private def isTemporary[_T](variable: Variable[_]): Boolean = { + variable match { + case e: ElementVariable[_] => e.element.isTemporary + case i: InternalVariable[_] => true + case _ => false + } + } + } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactoredAlgorithmTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactoredAlgorithmTest.scala new file mode 100644 index 00000000..dfa82e51 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/FactoredAlgorithmTest.scala @@ -0,0 +1,52 @@ +/* + * FactoredAlgorithmTest.scala + * Variable elimination tests. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.algorithm.factored + +import org.scalatest.Matchers +import org.scalatest.{ WordSpec, PrivateMethodTester } +import math.log +import com.cra.figaro.algorithm._ +import com.cra.figaro.algorithm.factored._ +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.library.atomic.continuous._ +import com.cra.figaro.util._ +import com.cra.figaro.test._ +import scala.collection.mutable.Map +import com.cra.figaro.test.tags.Performance +import com.cra.figaro.test.tags.NonDeterministic + +class FactoredAlgorithmTest extends WordSpec with Matchers { + "A Factored Algorithm" when { + "getting needed elements" should { + "return factors originating from the targets and evidence" in { + Universe.createNew() + + val teacherSkill = Flip(0.6) + val studentAbility = Flip(0.7) + val pass = Chain(studentAbility, (ability: Boolean) => + if (ability) If(teacherSkill, Flip(0.9), Flip(0.7)) + else If(teacherSkill, Flip(0.6), Flip(0.3))) + pass.observe(false) + val ve = VariableElimination(teacherSkill) + val (neededElements, _) = ve.getNeededElements(List(teacherSkill), Int.MaxValue ) + neededElements.size should equal(9) + } + } + } + + +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/GibbsTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/GibbsTest.scala new file mode 100644 index 00000000..e1920620 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/GibbsTest.scala @@ -0,0 +1,248 @@ +/* + * GibbsTest.scala + * Gibbs sampler tests. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 4, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.algorithm.factored + +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored.{ LazyValues, ValueSet } +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.compound.If +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.factored.gibbs.WalkSAT +import com.cra.figaro.algorithm.factored.gibbs.StateNotFoundException +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler +import com.cra.figaro.language.Element.toBooleanElement + +class GibbsTest extends WordSpec with Matchers { + "A Gibbs sampler" should { + "block Apply elements" in { + Universe.createNew + val u1 = Uniform(1, 2, 3) + val u2 = Uniform(4, 5, 6) + val a = Apply[Int, Int, Int](u1, u2, _ + _) + val alg = Gibbs(1, a) + alg.initialize() + val blocks = alg.createBlocks() + blocks.map(_.toSet) should contain theSameElementsAs (List( + Set(Variable(u1), Variable(a)), + Set(Variable(u2), Variable(a)))) + } + + "block Chain elements" in { + Universe.createNew + val f = Flip(0.3) + val u1 = Uniform(1, 2, 3) + val u2 = Uniform(2, 3, 4) + val c = If(f, u1, u2) + val alg = Gibbs(1, c) + alg.initialize() + val icv = alg.variables.find(_.isInstanceOf[InternalChainVariable[_]]).get + val blocks = alg.createBlocks() + blocks.map(_.toSet) should contain theSameElementsAs (List( + Set(Variable(f), icv), + Set(Variable(u1), Variable(c), icv), + Set(Variable(u2), Variable(c), icv))) + } + + "with an unconstrained model produce the correct result" in { + Universe.createNew + val f = Flip(0.3) + val s1 = Select(0.1 -> 1, 0.4 -> 2, 0.5 -> 3) + val s2 = Select(0.7 -> 2, 0.1 -> 3, 0.2 -> 4) + val c = If(f, s1, s2) + // 0.3 * 0.4 + 0.7 * 0.7 = 0.61 + test[Int](c, _ == 2, 0.61) + } + + "with a constrained model produce the correct result" in { + Universe.createNew + val f = Flip(0.3) + val s1 = Select(0.1 -> 1, 0.4 -> 2, 0.5 -> 3) + val s2 = Select(0.7 -> 2, 0.1 -> 3, 0.2 -> 4) + val c = If(f, s1, s2) + c.addConstraint(identity) + // 0.61 * 2 / (0.3*0.1*1 + 0.3*0.4*2 + 0.3*0.5*3 + 0.7*0.7*2 + 0.7*0.1*3 + 0.7*0.2*4) + test[Int](c, _ == 2, 0.4939) + } + + "with a constraint on a Chain produce the correct result for the parent" in { + Universe.createNew + val f = Flip(0.3) + val c = If(f, Flip(0.8), Constant(false)) + c.addConstraint(b => if (b) 2.0 else 1.0) + // (0.3 * 0.8 * 2) / (0.3 * 0.8 * 2 + 0.3 * 0.2 + 0.7) + test[Boolean](c, identity, 0.3871) + } + + "with a constraint on a Chain result correctly constrain the Chain but not the parent" in { + Universe.createNew + val f = Flip(0.3) + val r1 = Flip(0.8) + r1.addConstraint(b => if (b) 2.0 else 1.0) + val c = If(f, r1, Constant(false)) + test[Boolean](f, identity, 0.3) + // r1 true with probability (0.8 * 2) / (0.8 * 2 + 0.2) = 0.8889 + // 0.3 * 0.8889 + test[Boolean](c, identity, 0.2667) + } + + "with an element used multiple times use the same value each time" in { + Universe.createNew + val f = Flip(0.3) + val e = f === f + test[Boolean](e, identity, 1.0) + } + + "not underflow" in { + Universe.createNew() + val x = Flip(0.99) + for (i <- 0 until 10) { + x.addConstraint((b: Boolean) => if (b) 1e-100; else 1e-120) + } + test[Boolean](x, identity, 1.0) + } + } + + "WalkSAT" should { + "return the correct state on a conditioned model" in { + def chainMapper(chain: Chain[_, _]): Set[Variable[_]] = LazyValues(chain.universe).getMap(chain).values.map(Variable(_)).toSet + + Universe.createNew() + val x = Flip(0.5) + val y = Flip(0.5) + val z = Flip(0.5) + val result = (x || !y) && z && !((x !== y) !== z) + // Satisfied by x = true, y = false, z = true + result.observe(true) + val factors = makeFactors() + val variables = factors.flatMap(_.variables).toSet + val state = WalkSAT(factors, variables, SumProductSemiring(), chainMapper) + List((x, true), (y, false), (z, true)).foreach(pair => { + val (elem, value) = pair + val variable = Variable(elem) + variable.range(state(variable)).value should be(value) + }) + } + + "fail on a contradictory model" in { + def chainMapper(chain: Chain[_, _]): Set[Variable[_]] = LazyValues(chain.universe).getMap(chain).values.map(Variable(_)).toSet + + Universe.createNew() + val x = Flip(0.5) + val y = Flip(0.5) + val result = (x !== y) && (x === y) + result.observe(true) + val factors = makeFactors() + val variables = factors.flatMap(_.variables).toSet + a[StateNotFoundException] should be thrownBy (WalkSAT(factors, variables, SumProductSemiring(), chainMapper)) + } + } + + "A default block sampler" should { + "produce sub-factors on initialization" in { + Universe.createNew + val v1 = new Variable(ValueSet.withoutStar(Set(0, 1, 2, 3))) + val v2 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val semiring = LogSumProductSemiring() + val factor = new SparseFactor[Double](List(v1), List(v2), semiring) + factor.set(List(0, 0), 1.0) + factor.set(List(1, 0), 2.0) + factor.set(List(1, 1), 3.0) + factor.set(List(2, 0), Double.NegativeInfinity) + factor.set(List(2, 1), 4.0) + val sampler = BlockSampler.default(List(v2), List(factor)) + val List(mbLookupFactor) = sampler.mbLookupFactors + mbLookupFactor.get(List(0)).get(List(0)) should be(1.0) + mbLookupFactor.get(List(1)).get(List(0)) should be(2.0) + mbLookupFactor.get(List(1)).get(List(1)) should be(3.0) + mbLookupFactor.get(List(2)).contents.get(List(0)) should be(empty) + mbLookupFactor.get(List(2)).get(List(1)) should be(4.0) + mbLookupFactor.contents.get(List(3)) should be(empty) + } + + "normalize and make un-logarithmic a factor" in { + Universe.createNew + val v1 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val v2 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val semiring = LogSumProductSemiring() + val factor = new SparseFactor[Double](List(v1), List(v2), semiring) + factor.set(List(0, 0), -5000) + factor.set(List(1, 0), -5001) + factor.set(List(1, 1), -5002) + val sampler = BlockSampler.default(List(), List()) + val normalized = sampler.normalizeFactor(factor) + val sum = 1 + math.exp(1) + math.exp(2) + val tol = 1e-9 + normalized.get(List(0, 0)) should be((math.exp(2) / sum) +- tol) + normalized.get(List(1, 0)) should be((math.exp(1) / sum) +- tol) + normalized.get(List(1, 1)) should be((1 / sum) +- tol) + } + + "compute a sampling factor" in { + Universe.createNew + val v1 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val v2 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val v3 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val semiring = LogSumProductSemiring() + val f12 = new SparseFactor[Double](List(v1), List(v2), semiring) + val f32 = new SparseFactor[Double](List(v3), List(v2), semiring) + f12.set(List(0, 0), math.log(0.3)) + f12.set(List(0, 1), math.log(0.7)) + f12.set(List(1, 0), math.log(0.8)) + f12.set(List(1, 1), math.log(0.2)) + f32.set(List(0, 0), math.log(0.25)) + f32.set(List(0, 1), math.log(0.75)) + f32.set(List(1, 0), math.log(0.1)) + f32.set(List(1, 1), math.log(0.9)) + val sampler = BlockSampler.default(List(v2), List(f12, f32)) + val currentSamples = collection.mutable.Map[Variable[_], Int](v1 -> 0, v2 -> 1, v3 -> 1) + val samplingFactor = sampler.computeSamplingFactor(currentSamples) + val tol = 1e-9 + samplingFactor.get(List(0)) should be((0.3 * 0.1) / (0.3 * 0.1 + 0.7 * 0.9) +- tol) + samplingFactor.get(List(1)) should be((0.7 * 0.9) / (0.3 * 0.1 + 0.7 * 0.9) +- tol) + } + + "cache recent sampling factors" in { + Universe.createNew + val v1 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val v2 = new Variable(ValueSet.withoutStar(Set(0, 1))) + val semiring = LogSumProductSemiring() + val factor = new SparseFactor[Double](List(v1), List(v2), semiring) + factor.set(List(0, 0), math.log(0.3)) + factor.set(List(0, 1), math.log(0.7)) + factor.set(List(1, 0), math.log(0.8)) + factor.set(List(1, 1), math.log(0.2)) + val sampler = BlockSampler.default(List(v2), List(factor)) + val currentSamples = collection.mutable.Map[Variable[_], Int](v1 -> 0, v2 -> 1) + val samplingFactor1 = sampler.getSamplingFactor(currentSamples) + val samplingFactor2 = sampler.getSamplingFactor(currentSamples) + samplingFactor1 should be theSameInstanceAs samplingFactor2 + } + } + + def makeFactors(): List[Factor[Double]] = { + LazyValues(Universe.universe).expandAll(Universe.universe.activeElements.toSet.map((elem: Element[_]) => ((elem, Integer.MAX_VALUE)))) + Universe.universe.activeElements.foreach(Variable(_)) + Universe.universe.activeElements flatMap (Factory.makeFactorsForElement(_)) + } + + def test[T](target: Element[T], predicate: T => Boolean, prob: Double, tol: Double = 0.025) { + val algorithm = Gibbs(100000, target) + algorithm.start() + algorithm.probability(target, predicate) should be(prob +- tol) + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/SemiringTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/SemiringTest.scala index 3adfe584..761f41cf 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/SemiringTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/SemiringTest.scala @@ -13,7 +13,7 @@ package com.cra.figaro.test.algorithm.factored -import scala.collection.Map +import scala.collection.immutable.Map import scala.collection.Seq import scala.collection.mutable @@ -135,25 +135,24 @@ class SemiringTest extends WordSpec with Matchers with PrivateMethodTester { "handle zero values in weighted multiplication without crashing" in { val numberOfParameters = 1 - val param = Dirichlet(1, 1, 1) - val parameterMap = mutable.Map.empty[Parameter[_], Seq[Double]] - parameterMap += param -> Seq(0.0, 0.0, 0.0) + val param: Parameter[_] = Dirichlet(1, 1, 1) + val parameterMap = Map.empty[Parameter[_], Seq[Double]] + (param -> Seq(0.0, 0.0, 0.0)) - val semiring = new SufficientStatisticsSemiring(parameterMap.toMap) + val semiring = new SufficientStatisticsSemiring(parameterMap) - val allZeros = semiring.sum((0.50, mutable.Map(param -> Seq(0.0, 0.0, 0.0))), (0.50, mutable.Map(param -> Seq(0.0, 0.0, 0.0)))) + val allZeros = semiring.sum((0.50, Map(param -> Seq(0.0, 0.0, 0.0))), (0.50, Map(param -> Seq(0.0, 0.0, 0.0)))) allZeros._2(param)(0) should be(0.0 +- 0.001) allZeros._2(param)(1) should be(0.0 +- 0.001) allZeros._2(param)(2) should be(0.0 +- 0.001) - val leftZeros = semiring.sum((0.50, mutable.Map(param -> Seq(0.0, 0.0, 0.0))), (0.50, mutable.Map(param -> Seq(0.2, 0.6, 0.4)))) + val leftZeros = semiring.sum((0.50, Map(param -> Seq(0.0, 0.0, 0.0))), (0.50, Map(param -> Seq(0.2, 0.6, 0.4)))) leftZeros._2(param)(0) should be(0.1 +- 0.001) leftZeros._2(param)(1) should be(0.3 +- 0.001) leftZeros._2(param)(2) should be(0.2 +- 0.001) - val rightZeros = semiring.sum((0.50, mutable.Map(param -> Seq(0.3, 0.4, 0.5))), (0.50, mutable.Map(param -> Seq(0.0, 0.0, 0.0)))) + val rightZeros = semiring.sum((0.50, Map(param -> Seq(0.3, 0.4, 0.5))), (0.50, Map(param -> Seq(0.0, 0.0, 0.0)))) rightZeros._2(param)(0) should be(0.15 +- 0.001) rightZeros._2(param)(1) should be(0.2 +- 0.001) @@ -168,7 +167,7 @@ class SemiringTest extends WordSpec with Matchers with PrivateMethodTester { val semiring = SufficientStatisticsSemiring(parameterMap.toMap) - val result = semiring.product((0.50, mutable.Map(param -> Seq(0.3, 0.4, 0.5))), (0.50, mutable.Map(param -> Seq(0.8, 0.432, 0.0)))) + val result = semiring.product((0.50, Map(param -> Seq(0.3, 0.4, 0.5))), (0.50, Map(param -> Seq(0.8, 0.432, 0.0)))) result._1 should equal(0.25) //The simple product is used in the multiplication step. result._2(param)(0) should be(1.1 +- 0.001) @@ -185,7 +184,7 @@ class SemiringTest extends WordSpec with Matchers with PrivateMethodTester { val semiring = SufficientStatisticsSemiring(parameterMap.toMap) - val result = semiring.sum((0.50, mutable.Map(param -> Seq(0.3, 0.4, 0.5))), (0.50, mutable.Map(param -> Seq(0.8, 0.432, 0.0)))) + val result = semiring.sum((0.50, Map(param -> Seq(0.3, 0.4, 0.5))), (0.50, Map(param -> Seq(0.8, 0.432, 0.0)))) result._1 should be(1.0 +- 0.001) //(.5*.3 + .5*.8)/1 = 0.15 + .4 = .55 result._2(param)(0) should be(0.55 +- 0.001) @@ -246,7 +245,7 @@ class SemiringTest extends WordSpec with Matchers with PrivateMethodTester { p += 1 } - def randomParameterMap(): mutable.Map[Parameter[_], Seq[Double]] = + def randomParameterMap(): Map[Parameter[_], Seq[Double]] = { val paramMap = mutable.Map.empty[Parameter[_], Seq[Double]] @@ -256,7 +255,7 @@ class SemiringTest extends WordSpec with Matchers with PrivateMethodTester { (paramMap(p))(index).+(random.nextDouble()) } } - paramMap + paramMap.toMap } def create(): (Double, Map[Parameter[_], Seq[Double]]) = diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/VETest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/VETest.scala index b2ed3474..8447ad37 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/VETest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/VETest.scala @@ -1,13 +1,13 @@ /* - * VETest.scala + * VETest.scala * Variable elimination tests. - * + * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jan 1, 2009 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -28,6 +28,8 @@ import com.cra.figaro.test._ import scala.collection.mutable.Map import com.cra.figaro.test.tags.Performance import com.cra.figaro.test.tags.NonDeterministic +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredMPEVE class VETest extends WordSpec with Matchers { "A VEGraph" when { @@ -46,8 +48,8 @@ class VETest extends WordSpec with Matchers { val v4 = Variable(e4) val v5 = Variable(e5) val v6 = Variable(e6) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) val af = AbstractFactor(f.variables) val ag = AbstractFactor(g.variables) val graph = new VEGraph(List(f, g)) @@ -82,8 +84,8 @@ class VETest extends WordSpec with Matchers { val v4 = Variable(e4) val v5 = Variable(e5) val v6 = Variable(e6) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) val af = AbstractFactor(f.variables) val ag = AbstractFactor(g.variables) VEGraph.cost(List(af, ag)) should equal(18) // 2*1*3*2 + 1*3*1*2 @@ -114,9 +116,9 @@ class VETest extends WordSpec with Matchers { val v5 = Variable(e5) val v6 = Variable(e6) val v7 = Variable(e7) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) - val h = Factory.simpleMake[Double](List(v1, v7)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) + val h = Factory.defaultFactor[Double](List(v1, v7), List()) val graph1 = new VEGraph(List(f, g, h)) val score = graph1.score(v3) score should equal(-10) // 2*1*2*1*2 - (2*1*3*2 + 1*3*1*2) @@ -141,11 +143,11 @@ class VETest extends WordSpec with Matchers { val v5 = Variable(e5) val v6 = Variable(e6) val v7 = Variable(e7) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) - val h = Factory.simpleMake[Double](List(v1, v7)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) + val h = Factory.defaultFactor[Double](List(v1, v7), List()) val graph1 = new VEGraph(List(f, g, h)) - val graph2 = graph1.eliminate(v3) + val (graph2, _) = graph1.eliminate(v3) val VariableInfo(v1Factors, v1Neighbors) = graph2.info(v1) v1Factors.size should equal(2) // h and the new factor assert(v1Factors exists ((factor: AbstractFactor) => factor.variables.size == 5)) // all except v3 and v7 @@ -167,11 +169,11 @@ class VETest extends WordSpec with Matchers { val v5 = Variable(e5) val v6 = Variable(e6) val v7 = Variable(e7) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) - val h = Factory.simpleMake[Double](List(v1, v7)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) + val h = Factory.defaultFactor[Double](List(v1, v7), List()) val graph1 = new VEGraph(List(f, g, h)) - val graph2 = graph1.eliminate(v3) + val (graph2, _) = graph1.eliminate(v3) val VariableInfo(v1Factors, v1Neighbors) = graph2.info(v1) v1Neighbors should not contain (v3) } @@ -206,10 +208,10 @@ class VETest extends WordSpec with Matchers { val v6 = Variable(e6) val v7 = Variable(e7) val v8 = Variable(e8) - val f = Factory.simpleMake[Double](List(v1, v2, v3, v4)) - val g = Factory.simpleMake[Double](List(v5, v3, v2, v6)) - val h = Factory.simpleMake[Double](List(v1, v7)) - val i = Factory.simpleMake[Double](List(v8, v1, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3, v4), List()) + val g = Factory.defaultFactor[Double](List(v5, v3, v2, v6), List()) + val h = Factory.defaultFactor[Double](List(v1, v7), List()) + val i = Factory.defaultFactor[Double](List(v8, v1, v3), List()) val order = VariableElimination.eliminationOrder(List(f, g, h, i), Set(v5, v8))._2 assert(order == List(v3, v4, v1, v6, v7, v2) || order == List(v3, v4, v1, v7, v6, v2) || @@ -228,7 +230,7 @@ class VETest extends WordSpec with Matchers { def make(numVars: Int): Traversable[Factor[Double]] = { val universe = new Universe val a: List[Variable[_]] = List.tabulate(numVars)(i => Variable(Flip(0.3)("", universe))) - for { i <- 0 to numVars - 2 } yield Factory.simpleMake[Double](List(a(i), a(i + 1))) + for { i <- 0 to numVars - 2 } yield Factory.defaultFactor[Double](List(a(i), a(i + 1)), List()) } val factors1 = make(small) val factors2 = make(large) @@ -306,7 +308,7 @@ class VETest extends WordSpec with Matchers { val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) test(f, (b: Boolean) => b, 0.6) } - + "on a different universe from the current universe, produce the correct result" in { val u1 = Universe.createNew() val u = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) @@ -315,7 +317,7 @@ class VETest extends WordSpec with Matchers { val tolerance = 0.0000001 val algorithm = VariableElimination(f)(u1) algorithm.start() - algorithm.probability(f, (b: Boolean) => b) should be(0.6 +- tolerance) + algorithm.probability(f)(b => b) should be(0.6 +- tolerance) algorithm.kill() } @@ -381,7 +383,22 @@ class VETest extends WordSpec with Matchers { ve.probability(y, true) should be(((0.1 * 0.2 + 0.9 * 0.2) / (0.1 * 0.2 + 0.9 * 0.2 + 0.9 * 0.8)) +- 0.0000000001) ve.kill } - + + "with a very wide model produce the correct result" in { + Universe.createNew() + var root = Flip(0.5) + + val rand = new scala.util.Random(System.currentTimeMillis) + for (_ <- 0 until 1000) { + val v = If(root, Flip(0.5), Flip(0.5)) + if (rand.nextBoolean) { + v.observe(true) + } else { + v.observe(false) + } + } + test(root, (r: Boolean) => r == true, 0.50) + } } "MPEVariableElimination" should { @@ -398,7 +415,7 @@ class VETest extends WordSpec with Matchers { // p(e1=F,e2=T,e3=T) = 0.25 * 0.9 * 0.4 = .09 // p(e1=F,e2=F,e3=F) = 0.25 * 0.1 * 0.6 = .015 // MPE: e1=T,e2=F,e3=F,e4=T - val alg = MPEVariableElimination() + val alg = MPEVariableElimination() alg.start alg.mostLikelyValue(e1) should equal(true) alg.mostLikelyValue(e2) should equal(false) @@ -407,7 +424,7 @@ class VETest extends WordSpec with Matchers { alg.kill } } - + def test[T](target: Element[T], predicate: T => Boolean, prob: Double) { val tolerance = 0.0000001 val algorithm = VariableElimination(target) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/FactorPerformanceTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/FactorPerformanceTest.scala index 087e0261..1bc5afce 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/FactorPerformanceTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/FactorPerformanceTest.scala @@ -23,6 +23,7 @@ import com.cra.figaro.util._ import com.cra.figaro.algorithm.lazyfactored.LazyValues import com.cra.figaro.language.Element.toIntElement import scala.reflect.runtime.universe +import com.cra.figaro.algorithm.factored.factors.factory.Factory class FactorPerformanceTest extends WordSpec with Matchers with PrivateMethodTester { @@ -40,9 +41,9 @@ class FactorPerformanceTest extends WordSpec with Matchers with PrivateMethodTes val p2v = Variable(p2) val sumv = Variable(sum) - val p1Factor = Factory.make(p1).head - val p2Factor = Factory.make(p2).head - val sumSparseFactor = Factory.make(sum).head + val p1Factor = Factory.makeFactorsForElement(p1).head + val p2Factor = Factory.makeFactorsForElement(p2).head + val sumSparseFactor = Factory.makeFactorsForElement(sum).head val sumBasicFactor = new BasicFactor[Double](sumSparseFactor.parents, sumSparseFactor.output) val arg1Indices = sumSparseFactor.parents(0).range.zipWithIndex @@ -81,7 +82,7 @@ class FactorPerformanceTest extends WordSpec with Matchers with PrivateMethodTes val p2v = Variable(p2) val sumv = Variable(sum) - val sumSparseFactor = Factory.make(sum).head + val sumSparseFactor = Factory.makeFactorsForElement(sum).head val sumBasicFactor = new BasicFactor[Double](sumSparseFactor.parents, sumSparseFactor.output) val arg1Indices = sumSparseFactor.parents(0).range.zipWithIndex val arg2Indices = sumSparseFactor.parents(1).range.zipWithIndex diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/SparseFactorTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/SparseFactorTest.scala index b253583f..d1934e8b 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/SparseFactorTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/SparseFactorTest.scala @@ -22,26 +22,12 @@ import com.cra.figaro.algorithm.lazyfactored.LazyValues import com.cra.figaro.algorithm.lazyfactored.Regular import com.cra.figaro.algorithm.lazyfactored.ValueSet import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler -import com.cra.figaro.language.Apply -import com.cra.figaro.language.Apply3 -import com.cra.figaro.language.Apply4 -import com.cra.figaro.language.Apply5 -import com.cra.figaro.language.CachingChain -import com.cra.figaro.language.Chain -import com.cra.figaro.language.Condition -import com.cra.figaro.language.Constant -import com.cra.figaro.language.Dist -import com.cra.figaro.language.Flip -import com.cra.figaro.language.Inject -import com.cra.figaro.language.Name.stringToName -import com.cra.figaro.language.NamedEvidence -import com.cra.figaro.language.Reference.stringToReference -import com.cra.figaro.language.Select -import com.cra.figaro.language.Universe +import com.cra.figaro.language._ import com.cra.figaro.library.atomic.continuous.Normal import com.cra.figaro.library.atomic.continuous.Uniform import com.cra.figaro.library.compound.CPD import com.cra.figaro.algorithm.factored.ParticleGenerator +import com.cra.figaro.algorithm.factored.factors.factory.Factory class SparseFactorTest extends WordSpec with Matchers with PrivateMethodTester { @@ -111,7 +97,7 @@ class SparseFactorTest extends WordSpec with Matchers with PrivateMethodTester { val v2 = Variable(e2) val v3 = Variable(e3) val v4 = Variable(e4) - val f = Factory.simpleMake[Double](List(v1, v2, v3)) + val f = Factory.defaultFactor[Double](List(v1, v2, v3), List()) val g = new SparseFactor[Double](List(v4), List(v3)) f.set(List(0, 0, 0), 0.0) f.set(List(1, 0, 0), 0.1) @@ -159,7 +145,7 @@ class SparseFactorTest extends WordSpec with Matchers with PrivateMethodTester { val v3 = Variable(e3) val v4 = Variable(e4) val f = new SparseFactor[Double](List(v1, v2), List(v3)) - val g = Factory.simpleMake[Double](List(v4, v3)) + val g = Factory.defaultFactor[Double](List(v4, v3), List()) f.set(List(0, 0, 0), 0.0) f.set(List(1, 0, 0), 0.1) f.set(List(2, 0, 0), 0.2) @@ -197,8 +183,8 @@ class SparseFactorTest extends WordSpec with Matchers with PrivateMethodTester { Values()(f) Values()(e) val vf = Variable(f) - val f1 = Factory.make(f) - val f2 = Factory.make(e) + val f1 = Factory.makeFactorsForElement(f) + val f2 = Factory.makeFactorsForElement(e) val result21 = f2(0).product(f1(0)/*, SumProductSemiring*/) val sum21 = result21.sumOver(vf/*, SumProductSemiring*/) @@ -358,7 +344,7 @@ class SparseFactorTest extends WordSpec with Matchers with PrivateMethodTester { f.set(List(0, 0, 1), 0.3) f.set(List(1, 0, 1), 0.4) f.set(List(2, 0, 1), 0.5) - val g = f.marginalizeTo(SumProductSemiring(), v3) + val g = f.marginalizeTo(v3) g.variables should equal(List(v3)) val p1 = 0.0 + 0.1 + 0.2 val p2 = 0.3 + 0.4 + 0.5 @@ -393,7 +379,7 @@ class SparseFactorTest extends WordSpec with Matchers with PrivateMethodTester { f.set(List(0, 1, 1), 0.15) f.set(List(1, 1, 1), 0.2) f.set(List(2, 1, 1), 0.25) - val g = f.marginalizeTo(SumProductSemiring(), v1, v3) + val g = f.marginalizeTo(v1, v3) g.variables should equal(List(v1, v3)) g.get(List(0, 0)) should be(0.0 +- 0.000001) g.get(List(1, 0)) should be(0.1 +- 0.000001) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/factory/ChainFactorTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/factory/ChainFactorTest.scala index e6ce6e17..0261c518 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/factory/ChainFactorTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/factored/factors/factory/ChainFactorTest.scala @@ -18,34 +18,35 @@ import org.scalatest.PrivateMethodTester import org.scalatest.WordSpec import com.cra.figaro.algorithm.Values import com.cra.figaro.algorithm.factored.factors.{ SumProductSemiring, Semiring, Variable } -import com.cra.figaro.algorithm.lazyfactored.LazyValues -import com.cra.figaro.algorithm.lazyfactored.Regular -import com.cra.figaro.algorithm.lazyfactored.ValueSet +import com.cra.figaro.algorithm.lazyfactored._ import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler -import com.cra.figaro.language.Apply -import com.cra.figaro.language.Apply3 -import com.cra.figaro.language.Apply4 -import com.cra.figaro.language.Apply5 -import com.cra.figaro.language.CachingChain -import com.cra.figaro.language.Chain -import com.cra.figaro.language.Condition -import com.cra.figaro.language.Constant -import com.cra.figaro.language.Dist -import com.cra.figaro.language.Flip -import com.cra.figaro.language.Inject -import com.cra.figaro.language.Name.stringToName -import com.cra.figaro.language.NamedEvidence -import com.cra.figaro.language.Reference.stringToReference -import com.cra.figaro.language.Select -import com.cra.figaro.language.Universe +import com.cra.figaro.language._ import com.cra.figaro.library.atomic.continuous.Normal import com.cra.figaro.library.atomic.continuous.Uniform import com.cra.figaro.library.compound.CPD import com.cra.figaro.algorithm.factored.ParticleGenerator -import com.cra.figaro.algorithm.factored.factors.Factory +import com.cra.figaro.algorithm.lazyfactored.Extended +import com.cra.figaro.algorithm.factored.factors.factory.Factory class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { + def findIndex[T, U](range: List[List[Any]], select: T, chain: U): Int = { + val i = range.indexWhere(p => p(0).asInstanceOf[Regular[T]].value == select && p(1).asInstanceOf[Regular[U]].value == chain) + i + } + + def pairEqual[T, U](pair: List[Any], i0: Extended[T], i1: Extended[U]) = { + pair(0).asInstanceOf[Regular[T]] == i0 && pair(1).asInstanceOf[Regular[U]] == i1 + } + + def selectorEqual[T, U](pair: List[Any], i0: Extended[T]) = { + pair(0).asInstanceOf[Regular[T]] == i0 + } + + def chainEqual[T, U](pair: List[Any], i1: Extended[U]) = { + pair(1).asInstanceOf[Regular[U]] == i1 + } + "Making factors from an element" when { "given a chain" should { @@ -67,10 +68,29 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { val v42 = v4Vals indexOf Regular(2) val v43 = v4Vals indexOf Regular(3) - val factor = Factory.make(v4) + Variable(v1) + Variable(v2) + Variable(v3) + Variable(v4) + val factor = Factory.makeFactorsForElement(v4) val v4Factor = factor(0) - - v4Factor.get(List(v1t, 0, v41)) should equal(1.0) + val pairRange: List[List[Any]] = v4Factor.output.head.range.map(_.value.asInstanceOf[List[Any]]) + + for { + i <- v1Vals + j <- v4Vals + k <- v4Factor.output.head.range + } { + val v1ind = v1Vals indexOf (i) + val v4ind = v4Vals indexOf (j) + val tempind = v4Factor.output.head.range indexOf (k) + val p = v4Factor.get(List(v1ind, v4ind, tempind)) + if (pairEqual(k.value.asInstanceOf[List[Any]], i, j)) { + p should equal(1.0) + } else p should equal(0.0) + } + /* + v4Factor.get(List(v1t, 0, v41)) should equal(1.0) v4Factor.get(List(v1t, 0, v42)) should equal(0.0) v4Factor.get(List(v1t, 0, v43)) should equal(0.0) v4Factor.get(List(v1t, 1, v41)) should equal(0.0) @@ -88,8 +108,12 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { v4Factor.get(List(v1f, 5, v41)) should equal(0.0) v4Factor.get(List(v1f, 5, v42)) should equal(0.0) v4Factor.get(List(v1f, 5, v43)) should equal(1.0) + * + */ } + + "produce a conditional selector for each dependent element" in { Universe.createNew() val v1 = Flip(0.2) @@ -108,9 +132,27 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { val v42 = v4Vals indexOf Regular(2) val v43 = v4Vals indexOf Regular(3) - val factor = Factory.make(v4) - val v2Factor = factor(1) - + Variable(v1) + Variable(v2) + Variable(v3) + Variable(v4) + val factor = Factory.makeFactorsForElement(v4) + + for { + k <- 0 until v1Vals.size + i <- 0 until factor(k+1).parents.head.size + j <- 0 until (if (v1Vals(k).value) v2Vals.size else 1) + } { + val f = factor(k+1) + val p = f.get(List(i, j)) + val vals = if (v1Vals(k).value) v2Vals else Variable(v3).range + if (selectorEqual(f.parents.head.range(i).value.asInstanceOf[List[Any]], v1Vals(k))) { + if (chainEqual(f.parents.head.range(i).value.asInstanceOf[List[Any]], vals(j))) p should equal(1.0) + else p should equal(0.0) + } else p should equal(1.0) + } + + /* v2Factor.get(List(0, v21)) should equal(1.0) v2Factor.get(List(0, v22)) should equal(0.0) v2Factor.get(List(1, v21)) should equal(0.0) @@ -126,6 +168,7 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { v2Factor.get(List(5, v21)) should equal(1.0) v2Factor.get(List(5, v22)) should equal(1.0) + val v3Factor = factor(2) v3Factor.get(List(3, 0)) should equal(0.0) @@ -136,10 +179,14 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { v3Factor.get(List(0, 0)) should equal(1.0) v3Factor.get(List(1, 0)) should equal(1.0) v3Factor.get(List(2, 0)) should equal(1.0) + * + */ } } + /* CPDs are chains, these tests are not needed */ + /* "given a CPD with one argument" should { "produce a selector factor over the parent variables" in { Universe.createNew() @@ -160,7 +207,7 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { val v4t = 0 val v4f = 1 - val factor = Factory.make(v2) + val factor = Factory.makeFactorsForElement(v2) val v2Factor = factor(0) v2Factor.get(List(v1t, 0, v2t)) should equal(1.0) @@ -200,7 +247,7 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { val v4t = 0 val v4f = 1 - val factor = Factory.make(v2) + val factor = Factory.makeFactorsForElement(v2) val v3Factor = factor(1) v3Factor.get(List(0, v3t)) should equal(1.0) @@ -229,6 +276,9 @@ class ChainFactorTest extends WordSpec with Matchers with PrivateMethodTester { } } + * + * + */ } - + } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/filtering/ParParticleFilterTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/filtering/ParParticleFilterTest.scala new file mode 100644 index 00000000..3cd6c653 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/filtering/ParParticleFilterTest.scala @@ -0,0 +1,381 @@ +/* + * ParParticleFilterTest.scala + * Parallel particle filter tests. + * + * Created By: Lee Kellogg (lkellogg@cra.com) + * Creation Date: Jun 2, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.algorithm.filtering + +import org.scalatest.Matchers +import org.scalatest.{ WordSpec, PrivateMethodTester } +import com.cra.figaro.algorithm.filtering._ +import com.cra.figaro.language._ +import com.cra.figaro.language.Universe._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.test._ +import scala.language.existentials +import com.cra.figaro.test.tags.Performance +import com.cra.figaro.test.tags.NonDeterministic + +class ParParticleFilterTest extends WordSpec with PrivateMethodTester with Matchers { + + val numThreads = 8 + + "A parallel particle filter" when { + "constructing a belief state from a set of weighted samples" should { + "contain a sample with fraction proportional to its weight" in { + val numParticles = 10000 + val genWithStates = () => { + val staticSnapshot = new Snapshot + val universe = new Universe() + val f1 = Flip(0.2)("f1", universe) + val f2 = Flip(0.7)("f2", universe) + f1.value = true + f2.value = false + val dynamicSnapshot1 = new Snapshot + dynamicSnapshot1.store(universe) + val state1 = new State(dynamicSnapshot1, staticSnapshot) + f1.value = false + f2.value = false + val dynamicSnapshot2 = new Snapshot + dynamicSnapshot2.store(universe) + val state2 = new State(dynamicSnapshot2, staticSnapshot) + (universe, state1, state2) + } + val gen = () => genWithStates()._1 + val pf = ParticleFilter.par(() => gen(), (u: Universe) => new Universe(), numParticles, numThreads) + val pfClass = pf.getClass.getSuperclass + val (_, state1, state2) = genWithStates() + val weightedParticles: Seq[ParticleFilter.WeightedParticle] = List((0.4, state1), (0.6, state2)) + + pf.updateBeliefState(weightedParticles) + (pf.beliefState count (_ == state1)).toDouble / numParticles should be(0.4 +- 0.01) + } + } + + "constructing the initial belief state" should { + "contain a state with fraction proportional to its probability" taggedAs(NonDeterministic) in { + val numParticles = 20000 + val gen = () => { + val universe = new Universe() + val f1 = Flip(0.2)("f1", universe) + val i2 = If(f1, Flip(0.3)(Name.default, universe), Flip(0.6)(Name.default, universe))("f2", universe) + i2.observe(true) + universe + } + val qf1True = 0.2 * 0.3 + val qf1False = 0.8 * 0.6 + val pf1True = qf1True / (qf1True + qf1False) + val pf = ParticleFilter.par(gen, (u: Universe) => new Universe(), numParticles, numThreads) + pf.start() + pf.currentProbability("f1", true) should be(pf1True +- 0.01) + } + } + + "constructing a new belief state with no evidence" should { + "contain a state with fraction proportional to its expected probability under the initial belief state" in { + try { + val numParticles = 20000 + val gen = () => { + val universe1 = new Universe() + val f1 = Flip(0.2)("f", universe1) + universe1 + } + def trans(u: Universe): Universe = { + val universe2 = new Universe() + val f2 = If(u.get[Boolean]("f"), Flip(0.8)(Name.default, universe2), Flip(0.3)(Name.default, universe2))("f", universe2) + universe2 + } + val pf = ParticleFilter.par(gen, trans, numParticles, numThreads) + pf.start() + pf.advanceTime(List()) + val p = 0.2 * 0.8 + 0.8 * 0.3 + pf.currentProbability("f", true) should be(p +- 0.01) + } catch { + case e: java.util.NoSuchElementException => e.printStackTrace() + } + } + } + + "constructing a new belief state with evidence" should { + "contain a state with fraction proportional to its expected probability under the initial belief state" + + "conditioned on the evidence" in { + val numParticles = 50000 + val gen = () => { + val universe1 = new Universe() + val f11 = Flip(0.2)("f1", universe1) + universe1 + } + def trans(u: Universe): Universe = { + val universe2 = new Universe() + val f12 = If(u.get[Boolean]("f1"), Flip(0.8)(Name.default, universe2), Flip(0.3)(Name.default, universe2))("f1", universe2) + val f22 = If(f12, Flip(0.6)(Name.default, universe2), Flip(0.1)(Name.default, universe2))("f2", universe2) + universe2 + } + val pf = ParticleFilter.par(gen, trans, numParticles, numThreads) + pf.start() + pf.advanceTime(List(NamedEvidence("f2", Observation(true)))) + val qf1True = (0.2 * 0.8 + 0.8 * 0.3) * 0.6 + val qf1False = (0.2 * 0.2 + 0.8 * 0.7) * 0.1 + val pf1True = qf1True / (qf1True + qf1False) + pf.currentProbability("f1", true) should be(pf1True +- 0.01) + } + + "correctly estimate static variables" in { + val numParticles = 100000 + val staticGen = () => { + val static = new Universe() + val x = Flip(0.2)("x", static) + static + } + val gen = () => { + val universe2 = new Universe() + universe2 + } + def trans(static: Universe, previous: Universe): Universe = { + val universe3 = new Universe() + val y = If(static.get[Boolean]("x"), Flip(0.8)(Name.default, universe3), Flip(0.1)(Name.default, universe3))("y", universe3) + universe3 + } + val pf = ParticleFilter.par(staticGen, gen, trans(_, _), numParticles, numThreads) + pf.start() + pf.advanceTime(List(NamedEvidence("y", Observation(true)))) + val qxTrue = 0.2 * 0.8 + val qxFalse = 0.8 * 0.1 + val pxTrue = qxTrue / (qxTrue + qxFalse) + pf.currentProbability("x", true) should be(pxTrue +- 0.01) + } + } + + "iterating over two time steps with evidence on each step" should { + "contain a state with the correct probability" in { + val numParticles = 100000 + val gen = () => { + val universe1 = new Universe() + Flip(0.2)("f1", universe1) + universe1 + } + def trans(previous: Universe): Universe = { + val universe2 = new Universe() + val previousF1 = previous.get("f1").asInstanceOf[Element[Boolean]] + val f1 = If(previousF1, Flip(0.8)(Name.default, universe2), Flip(0.3)(Name.default, universe2))("f1", universe2) + val f2 = If(f1, Flip(0.6)(Name.default, universe2), Flip(0.1)(Name.default, universe2))("f2", universe2) + universe2 + } + val pf = ParticleFilter.par(gen, trans, numParticles, numThreads) + pf.start() + pf.advanceTime(List(NamedEvidence("f2", Observation(true)))) + val result1 = pf.currentProbability("f1", true) + pf.advanceTime(List(NamedEvidence("f2", Observation(false)))) + val result2 = pf.currentProbability("f1", true) + val pF11T = (0.2 * 0.8 + 0.8 * 0.3) * 0.6 + val pF11F = (0.2 * 0.2 + 0.8 * 0.7) * 0.1 + val answer1 = pF11T / (pF11T + pF11F) + val pF10T_F11T_F21T_F12T_F22F = 0.2 * 0.8 * 0.6 * 0.8 * 0.4 + val pF10T_F11T_F21T_F12F_F22F = 0.2 * 0.8 * 0.6 * 0.2 * 0.9 + val pF10T_F11F_F21T_F12T_F22F = 0.2 * 0.2 * 0.1 * 0.3 * 0.4 + val pF10T_F11F_F21T_F12F_F22F = 0.2 * 0.2 * 0.1 * 0.7 * 0.9 + val pF10F_F11T_F21T_F12T_F22F = 0.8 * 0.3 * 0.6 * 0.8 * 0.4 + val pF10F_F11T_F21T_F12F_F22F = 0.8 * 0.3 * 0.6 * 0.2 * 0.9 + val pF10F_F11F_F21T_F12T_F22F = 0.8 * 0.7 * 0.1 * 0.3 * 0.4 + val pF10F_F11F_F21T_F12F_F22F = 0.8 * 0.7 * 0.1 * 0.7 * 0.9 + val pF12T = pF10T_F11T_F21T_F12T_F22F + pF10T_F11F_F21T_F12T_F22F + pF10F_F11T_F21T_F12T_F22F + pF10F_F11F_F21T_F12T_F22F + val pF12F = pF10T_F11T_F21T_F12F_F22F + pF10T_F11F_F21T_F12F_F22F + pF10F_F11T_F21T_F12F_F22F + pF10F_F11F_F21T_F12F_F22F + result1 should be(answer1 +- 0.01) + val answer2 = pF12T / (pF12T + pF12F) + result2 should be(answer2 +- 0.01) + } + + "correctly estimate static variables" in { + val numParticles = 100000 + val staticGen = () => { + val static = new Universe() + val x = Flip(0.2)("x", static) + static + } + val initialGen = () => { + val initial = new Universe() + val y = Flip(0.3)("y", initial) + initial + } + def trans(static: Universe, previous: Universe): Universe = { + val universe3 = new Universe() + val y = If(static.get[Boolean]("x"), Flip(0.8)(Name.default, universe3), previous.get[Boolean]("y"))("y", universe3) + universe3 + } + val pf = ParticleFilter.par(staticGen, initialGen, trans(_, _), numParticles, numThreads) + pf.start() + pf.advanceTime(List(NamedEvidence("y", Observation(true)))) + val pXY0TT = 0.2 * 0.3 + val pXY0TF = 0.2 * 0.7 + val pXY0FT = 0.8 * 0.3 + val pXY0FF = 0.8 * 0.7 + val pYtTGivenXYtMinusTT = 0.8 + val pYtFGivenXYtMinusTT = 0.2 + val pYtTGivenXYtMinusTF = 0.8 + val pYtFGivenXYtMinusTF = 0.2 + val pYtTGivenXYtMinusFT = 1.0 + val pYtFGivenXYtMinusFT = 0.0 + val pYtTGivenXYtMinusFF = 0.0 + val pYtFGivenXYtMinusFF = 1.0 + val pY2TGivenXY0TT = pYtTGivenXYtMinusTT * pYtTGivenXYtMinusTT + pYtFGivenXYtMinusTT * pYtTGivenXYtMinusTF + val pY2TGivenXY0TF = pYtTGivenXYtMinusTF * pYtTGivenXYtMinusTT + pYtFGivenXYtMinusTF * pYtTGivenXYtMinusTF + val pY2TGivenXY0FT = pYtTGivenXYtMinusFT * pYtTGivenXYtMinusFT + pYtFGivenXYtMinusFT * pYtTGivenXYtMinusFF + val pY2TGivenXY0FF = pYtTGivenXYtMinusFF * pYtTGivenXYtMinusFT + pYtFGivenXYtMinusFF * pYtTGivenXYtMinusFF + val pXY2TT = pXY0TT * pY2TGivenXY0TT + pXY0TF * pY2TGivenXY0TF + val pXY2FT = pXY0FT * pY2TGivenXY0FT + pXY0FF * pY2TGivenXY0FF + val pxTrue = pXY2TT / (pXY2TT + pXY2FT) + pf.currentProbability("x", true) should be(pxTrue +- 0.01) + } + } + + "iterating over two time steps with evidence on each step" should { + "contain a the correct distribution over an element" in { + val numParticles = 100000 + val gen = () => { + val universe1 = new Universe() + Flip(0.2)("f1", universe1) + universe1 + } + def trans(previous: Universe): Universe = { + val universe2 = new Universe() + val previousF1 = previous.get("f1").asInstanceOf[Element[Boolean]] + val f1 = If(previousF1, Flip(0.8)(Name.default, universe2), Flip(0.3)(Name.default, universe2))("f1", universe2) + val f2 = If(f1, Flip(0.6)(Name.default, universe2), Flip(0.1)(Name.default, universe2))("f2", universe2) + universe2 + } + val pf = ParticleFilter.par(gen, trans, numParticles, numThreads) + pf.start() + pf.advanceTime(List(NamedEvidence("f2", Observation(true)))) + pf.advanceTime(List(NamedEvidence("f2", Observation(false)))) + val qf1TrueTime1 = (0.2 * 0.8 + 0.8 * 0.3) * 0.6 + val qf1FalseTime1 = (0.2 * 0.2 + 0.8 * 0.7) * 0.1 + val pf1TrueTime1 = qf1TrueTime1 / (qf1TrueTime1 + qf1FalseTime1) + val pf1FalseTime1 = 1 - pf1TrueTime1 + val qf1TrueTime2 = (pf1TrueTime1 * 0.8 + pf1FalseTime1 * 0.3) * 0.4 + val qf1FalseTime2 = (pf1TrueTime1 * 0.2 + pf1FalseTime1 * 0.7) * 0.9 + val pf1TrueTime2 = qf1TrueTime2 / (qf1TrueTime2 + qf1FalseTime2) + val d = pf.currentDistribution("f1").asInstanceOf[Stream[(Double, Boolean)]] + d.size should equal(2) + if (d(0)._2 == true) { + d(0)._2 should equal(true) + d(1)._2 should equal(false) + d(0)._1 should be(pf1TrueTime2 +- 0.01) + d(1)._1 should be((1.0 - pf1TrueTime2) +- 0.01) + } else { + d(1)._2 should equal(true) + d(0)._2 should equal(false) + d(1)._1 should be(pf1TrueTime2 +- 0.01) + d(0)._1 should be((1.0 - pf1TrueTime2) +- 0.01) + } + } + } + + "iterating over many time steps" should { + "not suffer from memory leaks" taggedAs (Performance) in { + val numParticles = 1000 + val numSteps = 1000 + val gen = () => { + val universe1 = new Universe() + Constant(Array.fill(1000)(0))("f1", universe1) + universe1 + } + def trans(previous: Universe): Universe = { + val universe2 = new Universe() + val previousF1 = previous.get[Array[Int]]("f1") + Apply(previousF1, (a: Array[Int]) => a.map(_ + 1))("f1", universe2) + universe2 + } + val pf = ParticleFilter.par(gen, trans, numParticles, numThreads) + pf.start() + for { i <- 1 to numSteps } { + pf.advanceTime(List()) + } + } + } + + "splitting up the particle indices over threads" should { + + val numParticles = 1000 + val numThreads = 7 + val pf = ParticleFilter.par(() => new Universe(), (u) => new Universe(), numParticles, numThreads) + val calculateIndices = PrivateMethod[Seq[(Int, Int)]]('calculateIndices) + val indices = pf invokePrivate calculateIndices(numParticles, numThreads) + + "assign work to each thread" in { + indices.length should equal(numThreads) + } + + val lengths = indices map { case (start, end) => end - start + 1 } + + "assign all the indices" in { + lengths.sum should equal(numParticles) + } + + "assign the indices evenly" in { + (lengths.max - lengths.min) should (equal(0) or equal(1)) + } + } + + "creating and running algorithm over multiple threads" should { + + val numParticles = 1000 + val numThreads = 8 + + var staticCalledCount = 0 + val static = () => { + staticCalledCount += 1 + new Universe() + } + + var initCalledCount = 0 + val initial = () => { + initCalledCount += 1 + new Universe() + } + + var transCalledCount = 0 + val transition = (u1: Universe, u2: Universe) => { + transCalledCount += 1 + new Universe() + } + val pf = ParticleFilter.par(static, initial, transition, numParticles, numThreads) + + "start with an empty belief state" in { + pf.beliefState.toList.count(_ == null) should equal(numParticles) + } + + "initialize universes for each thread when algorithm starts" in { + staticCalledCount should equal(0) + initCalledCount should equal(0) + transCalledCount should equal(0) + pf.run() + staticCalledCount should equal(numThreads) + initCalledCount should equal(numThreads) + transCalledCount should equal(0) + } + + "call transition function for each thread every time step" in { + val numSteps = 1000 + for (_ <- 1 to numSteps) { + pf.advanceTime() + } + staticCalledCount should equal(numThreads) + initCalledCount should equal(numThreads) + transCalledCount should equal(numSteps * numThreads) + } + + "update belief state with particles from all threads" in { + pf.beliefState.length should equal(numParticles) + pf.beliefState.toList.count(_ == null) should equal(0) + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/lazyfactored/LazyValuesTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/lazyfactored/LazyValuesTest.scala index a0be8248..ce232719 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/lazyfactored/LazyValuesTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/lazyfactored/LazyValuesTest.scala @@ -94,7 +94,7 @@ class LazyValuesTest extends WordSpec with Matchers { values(elem1, 1) values(elem1, 1) values(elem1, 0) - a should equal (1) + a should equal (2) } "use the old result when called twice on the same universe" in { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/EMWithMHTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/EMWithMHTest.scala index 26069794..8fe42c07 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/EMWithMHTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/EMWithMHTest.scala @@ -58,7 +58,7 @@ class EMWithMHTest extends WordSpec with PrivateMethodTester with Matchers { f.observe(false) } - val algorithm = EMWithBP(terminationCriteria, 10, b)(universe) + val algorithm = EMWithMH(terminationCriteria, 10000, b)(universe) algorithm.start val result = b.MAPValue diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/OnlineEMTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/OnlineEMTest.scala index f1d48104..205e31fa 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/OnlineEMTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/OnlineEMTest.scala @@ -29,7 +29,239 @@ import scala.math.abs import java.io._ import com.cra.figaro.test.tags.NonDeterministic -class OnlineEMTest extends WordSpec with PrivateMethodTester with Matchers { +class OnlineEMTest extends WordSpec with PrivateMethodTester with Matchers { + + "Online expectation maximization with BP" when + { + + "used to estimate a Beta parameter" should + { + + "detect bias after a large enough number of trials" in + { + val initial = Universe.createNew + val b = Beta(2, 2) + def transition = () => { + val next = Universe.createNew + val f = Flip(b)("f",next) + next + } + val algorithm = EMWithBP.online(transition, b)(initial) + algorithm.start() + for (i <- 1 to 7) { + algorithm.update(List(NamedEvidence("f", Observation(true)))) + } + + for (i <- 1 to 3) { + algorithm.update(List(NamedEvidence("f", Observation(false)))) + } + + + val result = b.MAPValue + algorithm.kill + result should be(0.6666 +- 0.01) + + } + + "take the prior concentration parameters into account" in + { + val initial = Universe.createNew + val b = Beta(3.0, 7.0) + def transition = () => { + val next = Universe.createNew + val f = Flip(b)("f",next) + next + } + val algorithm = EMWithBP.online(transition, b)(initial) + algorithm.start() + for (i <- 1 to 7) { + algorithm.update(List(NamedEvidence("f", Observation(true)))) + } + for (i <- 1 to 3) { + algorithm.update(List(NamedEvidence("f", Observation(false)))) + } + val result = b.MAPValue + algorithm.kill + result should be(0.50 +- 0.01) + } + + "learn the bias from observations of binomial elements" in { + val initial = Universe.createNew + val b = Beta(2, 2) + def transition = () => { + val next = Universe.createNew + val f = Binomial(5,b)("binomial",next) + next + } + val algorithm = EMWithBP.online(transition, b)(initial) + algorithm.start() + algorithm.update(List(NamedEvidence("binomial", Observation(4)))) + algorithm.update(List(NamedEvidence("binomial", Observation(3)))) + val result = b.MAPValue + algorithm.kill + result should be(0.6666 +- 0.01) + } + + "correctly use a uniform prior" in { + val initial = Universe.createNew + val b = Beta(1, 1) + def transition = () => { + val next = Universe.createNew + val f = Binomial(5,b)("binomial",next) + next + } + val algorithm = EMWithBP.online(transition, b)(initial) + algorithm.start() + algorithm.update(List(NamedEvidence("binomial", Observation(4)))) + algorithm.update(List(NamedEvidence("binomial", Observation(3)))) + val result = b.MAPValue + algorithm.kill + result should be(0.7 +- 0.01) + } + + } + + "used to estimate a Dirichlet parameter with two concentration parameters" should + { + + "detect bias after a large enough number of trials" in + { + val initial = Universe.createNew + val d = Dirichlet(2, 2) + def transition = () => { + val next = Universe.createNew + val s = Select(d, true, false)("s",next) + next + } + val algorithm = EMWithBP.online(transition, d)(initial) + algorithm.start() + for (i <- 1 to 7) { + algorithm.update(List(NamedEvidence("s", Observation(true)))) + } + for (i <- 1 to 3) { + algorithm.update(List(NamedEvidence("s", Observation(false)))) + } + val result = d.MAPValue + algorithm.kill + result(0) should be(0.6666 +- 0.01) + + } + + "take the prior concentration parameters into account" in + { + val initial = Universe.createNew + val d = Dirichlet(3, 7) + def transition = () => { + val next = Universe.createNew + val s = Select(d, true, false)("s",next) + next + } + val algorithm = EMWithBP.online(transition, d)(initial) + algorithm.start() + for (i <- 1 to 7) { + algorithm.update(List(NamedEvidence("s", Observation(true)))) + } + for (i <- 1 to 3) { + algorithm.update(List(NamedEvidence("s", Observation(false)))) + } + val result = d.MAPValue + algorithm.kill + result(0) should be(0.50 +- 0.01) + + } + + } + + "used to estimate multiple parameters" should + { + + "leave parameters having no observations unchanged" in + { + val initial = Universe.createNew + val d = Dirichlet(2.0, 4.0, 2.0) + val b = Beta(2.0, 2.0) + val outcomes = List(1, 2, 3) + + def transition = () => { + val next = Universe.createNew + val s = Select(d, outcomes:_*)("s",next) + next + } + val algorithm = EMWithBP.online(transition,d,b)(initial) + algorithm.start() + for (i <- 1 to 4) { + algorithm.update(List(NamedEvidence("s", Observation(1)))) + } + + for (i <- 1 to 2) { + algorithm.update(List(NamedEvidence("s", Observation(2)))) + } + + for (i <- 1 to 4) { + algorithm.update(List(NamedEvidence("s", Observation(3)))) + } + + + + val result = d.MAPValue + + result(0) should be(0.33 +- 0.01) + result(1) should be(0.33 +- 0.01) + result(2) should be(0.33 +- 0.01) + + val betaResult = b.MAPValue + betaResult should be(0.5) + algorithm.kill + } + + "correctly estimate all parameters with observations" in + { + val initial = Universe.createNew + val d = Dirichlet(2.0, 3.0, 2.0) + val b = Beta(3.0, 7.0) + val outcomes = List(1, 2, 3) + + def transition = () => { + val next = Universe.createNew + val s = Select(d, outcomes:_*)("s",next) + val f = Flip(b)("f",next) + next + } + val algorithm = EMWithBP.online(transition,d,b)(initial) + algorithm.start() + for (i <- 1 to 4) { + algorithm.update(List(NamedEvidence("s", Observation(1)))) + } + + for (i <- 1 to 2) { + algorithm.update(List(NamedEvidence("s", Observation(2)))) + } + + for (i <- 1 to 4) { + algorithm.update(List(NamedEvidence("s", Observation(3)))) + } + + for (i <- 1 to 7) { + algorithm.update(List(NamedEvidence("f", Observation(true)))) + } + + for (i <- 1 to 3) { + algorithm.update(List(NamedEvidence("f", Observation(false)))) + } + + val result = d.MAPValue + + result(0) should be(0.35 +- 0.01) + result(1) should be(0.28 +- 0.01) + result(2) should be(0.36 +- 0.01) + + val betaResult = b.MAPValue + betaResult should be(0.5 +- 0.01) + algorithm.kill + } + } + } + "Online expectation maximization" when { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/SufficientStatisticsVariableEliminationTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/SufficientStatisticsVariableEliminationTest.scala index 6a05ec09..b1870b08 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/SufficientStatisticsVariableEliminationTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/learning/SufficientStatisticsVariableEliminationTest.scala @@ -56,7 +56,7 @@ class SufficientStatisticsVariableEliminationTest extends WordSpec with PrivateM val one = (1.0, emptyMap) val algorithm = SufficientStatisticsVariableElimination(paramMap) - val elements = List(f1,f2,p1,f3) + val (elements, _) = algorithm.getNeededElements(List(f1,f2,p1,f3), Int.MaxValue) val values = Values(universe) values(f1) values(f2) @@ -101,7 +101,7 @@ class SufficientStatisticsVariableEliminationTest extends WordSpec with PrivateM val one = (1.0, emptyMap) val algorithm = SufficientStatisticsVariableElimination(paramMap) - val elements = List(f1,f2,f3,p1) + val (elements, _) = algorithm.getNeededElements(List(f1,f2,p1,f3), Int.MaxValue) val values = Values(universe) values(f1) values(f2) @@ -147,7 +147,7 @@ class SufficientStatisticsVariableEliminationTest extends WordSpec with PrivateM val one = (1.0, emptyMap) val algorithm = SufficientStatisticsVariableElimination(paramMap) - val elements = List(f1,f2,f3,p1) + val (elements, _) = algorithm.getNeededElements(List(f1,f2,p1,f3), Int.MaxValue) val values = Values(universe) values(f1) values(f2) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/AnnealingTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/AnnealingTest.scala index 09112119..d64498e8 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/AnnealingTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/AnnealingTest.scala @@ -170,7 +170,7 @@ class AnnealingTest extends WordSpec with Matchers with PrivateMethodTester { def buildComplicatedModel(): Universe = { val u = Universe.createNew() - val flips = Seq.fill(10000)(Math.pow(0.01, Random.nextDouble)).map(SwitchingFlip(_)) + val flips = Seq.fill(1000)(Math.pow(0.01, Random.nextDouble)).map(SwitchingFlip(_)) val ifs = for ((f, i) <- flips.zipWithIndex) yield { val fi = If(f, SwitchingFlip(Math.pow(0.01, Random.nextDouble)), Constant(false))(i.toString(), u) if (Random.nextDouble > 0.80) fi.observe(Random.nextBoolean()) @@ -185,18 +185,18 @@ class AnnealingTest extends WordSpec with Matchers with PrivateMethodTester { if (annealer.isActive) annealer.resume else annealer.start Thread.sleep(100) annealer.stop() - for (i <- 1 to 10000) { + for (i <- 1 to 1000) { if (annealer.isActive) annealer.resume else annealer.start Thread.sleep(25) annealer.stop() - for (i <- 0 until 10000) { + for (i <- 0 until 1000) { annealer.mostLikelyValue(u.getElementByReference[Boolean](i.toString())) } } annealer.kill } catch { case knf: Exception => fail("running algorithm should not produce exceptions.") - case _ => + case _: Throwable => () } } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ElementSamplerTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ElementSamplerTest.scala index 1a1ed9e0..f3412a2a 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ElementSamplerTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ElementSamplerTest.scala @@ -259,6 +259,7 @@ class ElementSamplerTest extends WordSpec with Matchers with PrivateMethodTester val es = ElementSampler(target) es.start Thread.sleep(1000) + es.stop val predProb = es.probability(target, predicate) predProb should be(prob +- epsilon) es.kill diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ImportanceTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ImportanceTest.scala index 696178a2..7acb88eb 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ImportanceTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ImportanceTest.scala @@ -31,19 +31,20 @@ import com.cra.figaro.test.tags.NonDeterministic import scala.language.reflectiveCalls import org.scalatest.Matchers import org.scalatest.{ PrivateMethodTester, WordSpec } +import scala.collection.mutable.Set + class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { "Sampling a value of a single element" should { "reject sampling process if condition violated" in { Universe.createNew() - val target = Flip(0.7) + val target = Flip(1.0) target.observe(false) val numTrials = 100000 val tolerance = 0.01 val imp = Importance(target) - val state = Importance.State() - an[RuntimeException] should be thrownBy { imp.sampleOne(state, target, Some(true)) } + an[RuntimeException] should be thrownBy { imp.lw.traverse(List((target, None, None)), List(), 0.0, Set()) } } @@ -54,9 +55,8 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { val numTrials = 100000 val tolerance = 0.01 val imp = Importance(target) - val state = Importance.State() - val value = imp.sampleOne(state, target, Some(false)) - value should equal(false) + imp.lw.traverse(List((target, Some(false), None)), List(), 0.0, Set()) + target.value should equal(false) } "for a Constant return the constant with probability 1" in { @@ -211,8 +211,7 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { Apply(B, (b: List[List[Boolean]]) => b.head) }) val alg = Importance(1, c) - val state = Importance.State() - alg.sampleOne(state, c, None) + alg.lw.computeWeight(List(c)) c.value.asInstanceOf[List[Boolean]].head should be(true || false) } } @@ -327,7 +326,7 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { // Uniform(0,1) is beta(1,1) // Result is beta(1 + 16,1 + 4) // Expectation is (alpha) / (alpha + beta) = 17/22 - alg.expectation(b, (d: Double) => d) should be((17.0 / 22.0) +- 0.02) + alg.expectation(b)(d => d) should be((17.0 / 22.0) +- 0.02) val time1 = System.currentTimeMillis() // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample @@ -429,7 +428,7 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { // uniform(0,1) is beta(1,1) // Result is beta(1 + 1600,1 + 400) // Expectation is (alpha) / (alpha + beta) = 1601/2003 - alg.expectation(beta, (d: Double) => d) should be((1601.0 / 2003.0) +- 0.02) + alg.expectation(beta)(d => d) should be((1601.0 / 2003.0) +- 0.02) val time1 = System.currentTimeMillis() // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample @@ -475,6 +474,8 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { i.kill() } + /* These tests are no longer valid. Since there is a hidden dependency, we can't support this */ + /* "resample elements inside class defined in a chain" in { Universe.createNew() class temp { @@ -500,6 +501,8 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { //alg.probability(b, true) should be (0.9 +- .01) } + * + */ "not suffer from stack overflow with small probability of success" taggedAs (Performance) in { Universe.createNew() @@ -616,6 +619,44 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { } } + "Sampling the posterior" should { + "produce the correct answer for marginals" in { + Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) + a.setConstraint((i: Int) => i.toDouble) + // U(true) = \int_{0.2}^{1.0} (0.3 + 2 * 0.7) p = 0.85 * 0.96 + // U(false) = \int_{0.2}^(1.0) (2 * (1-p)) = 0.64 + val u1 = 0.85 * 0.96 + val u2 = 0.64 + val pos = Importance.sampleJointPosterior(f) + val probTrue = (pos.take(1000).toList.map(t => t(0).asInstanceOf[Boolean])).count(p => p) + probTrue.toDouble / 1000.0 should be(u1 / (u1 + u2) +- .01) + } + + "produce the correct answer for joint" in { + Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) + a.setConstraint((i: Int) => i.toDouble) + val pair = ^^(f,a) + val alg = Importance(20000, pair) + alg.start() + + val pos = Importance.sampleJointPosterior(f, a) + val samples = pos.take(5000).toList.map(t => (t(0).asInstanceOf[Boolean], t(1).asInstanceOf[Int])) + + val probTrueTwo = samples.count(p => p._1 == true && p._2 == 2).toDouble/5000.0 should be(alg.probability(pair, (true, 2)) +- .01) + val probTrueOne = samples.count(p => p._1 == true && p._2 == 1).toDouble/5000.0 should be(alg.probability(pair, (true, 1)) +- .01) + val probFalseTwo = samples.count(p => p._1 == false && p._2 == 2).toDouble/5000.0 should be(alg.probability(pair, (false, 2)) +- .01) + val probFalseOne = samples.count(p => p._1 == false && p._2 == 1).toDouble/5000.0 should be(alg.probability(pair, (false, 1)) +- .01) + alg.kill() + } + + } + } def weightedSampleTest[T](target: Element[T], predicate: T => Boolean, prob: Double) { @@ -633,9 +674,8 @@ class ImportanceTest extends WordSpec with Matchers with PrivateMethodTester { def attempt(): (Double, T) = { try { - val state = Importance.State() - val value = imp.sampleOne(state, target, None) - (state.weight, value.asInstanceOf[T]) + val weight = imp.lw.computeWeight(List(target)) + (weight, target.value.asInstanceOf[T]) } catch { case Importance.Reject => attempt() } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/LikelihoodWeighterTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/LikelihoodWeighterTest.scala new file mode 100644 index 00000000..de451562 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/LikelihoodWeighterTest.scala @@ -0,0 +1,109 @@ +/* + * LikelihoodWeighterTest.scala + * Importance sampling tests. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.algorithm.sampling + +import org.scalatest._ +import org.scalatest.Matchers +import com.cra.figaro.algorithm._ +import com.cra.figaro.algorithm.sampling.Importance.Reject +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous._ +import com.cra.figaro.library.atomic._ +import com.cra.figaro.library.atomic.discrete.Binomial +import com.cra.figaro.library.compound._ +import com.cra.figaro.test._ +import com.cra.figaro.util.logSum +import JSci.maths.statistics._ +import com.cra.figaro.test.tags.Performance +import com.cra.figaro.test.tags.NonDeterministic +import scala.language.reflectiveCalls +import org.scalatest.Matchers +import org.scalatest.{ PrivateMethodTester, WordSpec } +import com.cra.figaro.algorithm.sampling.LikelihoodWeighter +import com.cra.figaro.library.cache.NoCache +class LikelihoodWeighterTest extends WordSpec with Matchers with PrivateMethodTester { + + // Note, many test cases are covered under the Importance sampling tests + val normalizer = 1.0 / math.sqrt(2.0 * math.Pi) + + "Running likelihood weighting" should { + "Correctly build dependencies" in { + Universe.createNew() + val result1 = Flip(0.7) + val result2 = Flip(0.5) + + val c1 = If(Flip(0.1), Flip(0.2), result1) + val c2 = If(Flip(0.1), Flip(0.2), result2) + val c3 = If(Flip(0.5), c1, c2) + + val lw = new LikelihoodWeighter(Universe.universe, new NoCache(Universe.universe)) + for { _ <- 0 until 500 } lw.traverse(List(), Universe.universe.activeElements, 0.0, scala.collection.mutable.Set()) + lw.dependencies.contains(result1) should equal(true) + lw.dependencies.contains(result2) should equal(true) + } + + "Undo the weight if a result element was sampled before its parent when the parent has an observation" in { + Universe.createNew() + val result1 = Normal(0, 1) + val result2 = Normal(1, 1) + val c = Chain(Constant(true), (b: Boolean) => if (b) result1 else result2) + result1.addConstraint((d: Double) => Normal.density(1.0, 1.0, 1.0)(d)) + c.observe(0.0) + + val lw = new LikelihoodWeighter(Universe.universe, new NoCache(Universe.universe)) + val weight = lw.traverse(List(), List(result1, result2, c), 0.0, scala.collection.mutable.Set()) + val correct = result1.density(0.0) * Normal.density(1.0, 1, 1)(0) + math.exp(weight) should be(correct +- .00001) + } + + "Not have invalid states for a chain if sampled in the wrong order" in { + Universe.createNew() + val result1 = Normal(0, 1) + val result2 = Normal(1, 1) + val c = Chain(Constant(true), (b: Boolean) => if (b) result1 else result2) + c.observe(0.0) + val f = result1 ++ Constant(0.0) + val lw = new LikelihoodWeighter(Universe.universe, new NoCache(Universe.universe)) + val weight = lw.traverse(List(), List(f, result1, result2, c), 0.0, scala.collection.mutable.Set()) + f.value should equal(0.0) + } + + "Not have invalid states for a dist if sampled in the wrong order" in { + Universe.createNew() + val result1 = Normal(0, 1) + val result2 = Normal(1, 1) + val c = Dist(1.0 -> result1, 0.0 -> result2) + c.observe(0.0) + val f = result1 ++ Constant(0.0) + val lw = new LikelihoodWeighter(Universe.universe, new NoCache(Universe.universe)) + val weight = lw.traverse(List(), List(f, result1, result2, c), 0.0, scala.collection.mutable.Set()) + f.value should equal(0.0) + } + + "Not overflow the stack" in { + def next(count: Int): Element[Boolean] = { + if (count == 0) Constant(true) + else Chain(Flip(0.5), (b: Boolean) => next(count - 1)) + } + Universe.createNew() + val start = next(2000) + val lw = new LikelihoodWeighter(Universe.universe, new NoCache(Universe.universe)) + lw.computeWeight(List(start)) + //an[StackOverflowError] should not be thrownBy { lw.computeWeight(List(start)) } + } + + } + +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/MHTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/MHTest.scala index bab3bc92..2d9f4ab7 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/MHTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/MHTest.scala @@ -32,7 +32,7 @@ import com.cra.figaro.test.tags.Performance class MHTest extends WordSpec with Matchers with PrivateMethodTester { "Anytime MetropolisHastings" should { - + "for a constant produce the constant with probability 1" in { Universe.createNew() val c = Constant(1) @@ -257,13 +257,13 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { try { mh.start() mh.stop() - mh.probability(f1, (b: Boolean) => b) should be(p1 / (p1 + p2) +- tolerance) + mh.probability(f1)(b => b) should be(p1 / (p1 + p2) +- tolerance) } finally { mh.kill() } } - "with a condition on the result of an If in which both consequences are random, correctly condition the test" taggedAs(NonDeterministic) in { + "with a condition on the result of an If in which both consequences are random, correctly condition the test" taggedAs (NonDeterministic) in { val numSamples = 100000 val tolerance = 0.01 Universe.createNew() @@ -276,7 +276,7 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { alg.stop() val p1 = 0.2 * 0.7 val p2 = 0.8 * 0.4 - alg.probability(elem1, (b: Boolean) => b) should be(p1 / (p1 + p2) +- tolerance) + alg.probability(elem1)(b => b) should be(p1 / (p1 + p2) +- tolerance) } finally { alg.kill() } @@ -293,7 +293,7 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { try { alg.start() alg.stop() - alg.probability(elem2, (d: Double) => 1.4 < d && d < 1.6) should be(1.0 +- tolerance) + alg.probability(elem2)(d => 1.4 < d && d < 1.6) should be(1.0 +- tolerance) } finally { alg.kill() } @@ -336,8 +336,8 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { } "Testing a Metropolis-Hastings algorithm" should { - - "produce correct results with a typed scheme" in { + + "produce correct results with a typed scheme" in { val numSamples = 200000 val tolerance = 0.01 Universe.createNew() @@ -361,28 +361,27 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { try { mh.start() mh.stop() - mh.probability(f1, (b: Boolean) => b) should be(p1 / (p1 + p2) +- tolerance) + mh.probability(f1)(b => b) should be(p1 / (p1 + p2) +- tolerance) } finally { mh.kill() } } - - //Need better use cases for these two schemes. + + //Need better use cases for these two schemes. "produce correct results with a switching scheme" in { Universe.createNew() val s1 = Select(0.4 -> 1, 0.6 -> 4) val s2 = Select(0.2 -> 2, 0.8 -> 3) s1.setConstraint((i: Int) => if (i == 1) 1; else 0.25) val p1 = Predicate(s1, (i: Int) => i == 4) - val scheme = SwitchScheme(() => (s1,s2),Some(FinalScheme(() => Universe.universe.randomStochasticElement()))) + val scheme = SwitchScheme(() => (s1, s2), Some(FinalScheme(() => Universe.universe.randomStochasticElement()))) val alg = MetropolisHastings(scheme) s1.value = 1 val (_, scores, _) = alg.test(10000, List(p1), List()) // probability of s1 changing to 4 is 0.5 (prob. s1 proposed) * 0.6 (prob 4 is chosen if s1 proposed) * 0.25 scores(p1) should be(0.075 +- 0.01) } - - + "produce values satisfying a predicate the correct amount of the time without conditions or constraints" in { Universe.createNew() val s1 = Select(0.4 -> 1, 0.6 -> 4) @@ -421,7 +420,7 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { } "Fixed bugs" should { - "propose temporary elements when non-temporary elements are non-stochastic" taggedAs(NonDeterministic) in { + "propose temporary elements when non-temporary elements are non-stochastic" taggedAs (NonDeterministic) in { Universe.createNew() val c1 = NonCachingChain(Constant(0), (i: Int) => Flip(0.7)) val a1 = If(c1, Constant(1), Constant(0)) @@ -430,6 +429,47 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { alg.start() alg.probability(a1, 1) should be(0.7 +- 0.01) } + + "not record invalid states" taggedAs (NonDeterministic) in { + Universe.createNew() + val c1 = Flip(0.9) + val a1 = If(c1, com.cra.figaro.library.atomic.discrete.Uniform(1, 2), Select(0.999 -> 2, 0.001 -> 3)) + a1.observe(3) + val alg = MetropolisHastings(10000, ProposalScheme.default, a1, c1) + alg.start() + alg.distribution(c1).toList.exists(s => s._2 == true) should be(false) + } + "correctly update elements in order" taggedAs (NonDeterministic) in { + // bug test for issue #453 + + Universe.createNew() + val Market = Select(0.5 -> 0, 0.3 -> 1, 0.2 -> 2) + val Survey = CPD(Constant(true), Market, + (false, 0) -> Constant(-1), (false, 1) -> Constant(-1), (false, 2) -> Constant(-1), + (true, 0) -> Select(0.6 -> 0, 0.3 -> 1, 0.1 -> 2), + (true, 1) -> Select(0.3 -> 0, 0.4 -> 1, 0.3 -> 2), + (true, 2) -> Select(0.1 -> 0, 0.4 -> 1, 0.5 -> 2)) + val Found = Chain(Survey, (i: Int) => com.cra.figaro.library.atomic.discrete.Uniform(true, false)) + def Value_fcn(f: Boolean, m: Int): Double = { + if (f) { + m match { + case 0 => -7.0 + case 1 => 5.0 + case 2 => 20.0 + } + } else { + 0.0 + } + } + val Value = Apply(Found, Market, Value_fcn) + val Z = ^^(Found, Value) + + val alg = MetropolisHastings(20000, ProposalScheme.default, Z) + alg.start() + + val a = alg.distribution(Z).toList + alg.distribution(Z).toList.exists(s => s._2._1 == false && s._2._2 != 0.0) should be(false) + } } def newState(mh: MetropolisHastings): State = { @@ -439,6 +479,44 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { method.invoke(mh).asInstanceOf[State] } + "Sampling the posterior" should { + "produce the correct answer for marginals" in { + Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) + a.setConstraint((i: Int) => i.toDouble) + // U(true) = \int_{0.2}^{1.0} (0.3 + 2 * 0.7) p = 0.85 * 0.96 + // U(false) = \int_{0.2}^(1.0) (2 * (1-p)) = 0.64 + val u1 = 0.85 * 0.96 + val u2 = 0.64 + val pos = MetropolisHastings.sampleJointPosterior(f) + val probTrue = (pos.take(1000).toList.map(t => t(0).asInstanceOf[Boolean])).count(p => p) + probTrue.toDouble / 1000.0 should be(u1 / (u1 + u2) +- .01) + } + + "produce the correct answer for joint" in { + Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) + a.setConstraint((i: Int) => i.toDouble) + val pair = ^^(f, a) + val alg = Importance(20000, pair) + alg.start() + + val pos = MetropolisHastings.sampleJointPosterior(f, a) + val samples = pos.take(5000).toList.map(t => (t(0).asInstanceOf[Boolean], t(1).asInstanceOf[Int])) + + val probTrueTwo = samples.count(p => p._1 == true && p._2 == 2).toDouble / 5000.0 should be(alg.probability(pair, (true, 2)) +- .01) + val probTrueOne = samples.count(p => p._1 == true && p._2 == 1).toDouble / 5000.0 should be(alg.probability(pair, (true, 1)) +- .01) + val probFalseTwo = samples.count(p => p._1 == false && p._2 == 2).toDouble / 5000.0 should be(alg.probability(pair, (false, 2)) +- .01) + val probFalseOne = samples.count(p => p._1 == false && p._2 == 1).toDouble / 5000.0 should be(alg.probability(pair, (false, 1)) +- .01) + alg.kill() + } + + } + def getDissatisfied(mh: MetropolisHastings): Set[Element[_]] = { val cls = mh.getClass().getSuperclass() val method = cls.getDeclaredMethod("getDissatisfied") @@ -508,7 +586,7 @@ class MHTest extends WordSpec with Matchers with PrivateMethodTester { mh.start() Thread.sleep(1000) mh.stop() -// println(mh.getSampleCount.toString + " samples") + // println(mh.getSampleCount.toString + " samples") mh.probability(target, predicate) should be(prob +- tolerance) } finally { mh.kill() diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ParImportanceTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ParImportanceTest.scala new file mode 100644 index 00000000..bc373dda --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/sampling/ParImportanceTest.scala @@ -0,0 +1,453 @@ +/* + * ParImportanceTest.scala + * ParImportance sampling tests. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.algorithm.sampling + +import org.scalatest._ +import org.scalatest.Matchers +import com.cra.figaro.algorithm._ +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous._ +import com.cra.figaro.library.atomic._ +import com.cra.figaro.library.atomic.discrete.Binomial +import com.cra.figaro.library.compound._ +import com.cra.figaro.test._ +import JSci.maths.statistics._ +import com.cra.figaro.test.tags.Performance +import com.cra.figaro.test.tags.NonDeterministic +import org.scalatest.Matchers +import org.scalatest.{ PrivateMethodTester, WordSpec } +import com.cra.figaro.language.Name.stringToName +import com.cra.figaro.language.Reference.stringToReference +class ParImportanceTest extends WordSpec with Matchers with PrivateMethodTester { + + val numThreads = 3 + + "Producing a weighted sample of an element in a universe" should { + "with no conditions or constraints produce the same result as sampling the element individually" in { + val gen: Function0[Universe] = () => { + val universe = Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u)("f", universe) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2)(Name.default, universe), Constant(2)(Name.default, universe)) + universe + } + weightedSampleTest(gen, "f", (b: Boolean) => b, 0.6) + } + + "with a condition on a dependent element produce the result with the correct probability" in { + val gen = () => { + val universe = Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u)("f", universe) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2)(Name.default, universe), Constant(2)(Name.default, universe)) + a.setCondition((i: Int) => i == 2) + universe + } + // U(true) = \int_{0.2}^{1.0) 0.7 p = 0.35 * 0.96 + // U(false) = \int_{0.2}^{1.0) (1-p) + val u1 = 0.35 * 0.96 + val u2 = 0.32 + weightedSampleTest(gen, "f", (b: Boolean) => b, u1 / (u1 + u2)) + } + + "with a constraint on a dependent element produce the result with the correct probability" in { + val gen = () => { + val universe = Universe.createNew() + val u = Uniform(0.2, 1.0) + val f = Flip(u)("f", universe) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2)(Name.default, universe), Constant(2)(Name.default, universe)) + a.setConstraint((i: Int) => i.toDouble) + universe + } + // U(true) = \int_{0.2}^{1.0} (0.3 + 2 * 0.7) p = 0.85 * 0.96 + // U(false) = \int_{0.2}^(1.0) (2 * (1-p)) = 0.64 + val u1 = 0.85 * 0.96 + val u2 = 0.64 + weightedSampleTest(gen, "f", (b: Boolean) => b, u1 / (u1 + u2)) + } + + "with an element that uses another element multiple times, " + + "always produce the same value for the different uses" in { + val gen = () => { + val universe = Universe.createNew() + val f = Flip(0.5) + val e = new Eq("e", f, f, universe) + universe + } + weightedSampleTest(gen, "e", (b: Boolean) => b, 1.0) + } + + "with a constraint on an element that is used multiple times, only factor in the constraint once" in { + val gen = () => { + val universe = Universe.createNew() + val f1 = Flip(0.5) + val f2 = Flip(0.3) + val e1 = f1 === f1 + val e2 = f1 === f2 + val d = Dist(0.5 -> e1, 0.5 -> e2)("d", universe) + f1.setConstraint((b: Boolean) => if (b) 3.0; else 2.0) + universe + } + // Probability that f1 is true = 0.6 + // Probability that e1 is true = 1.0 + // Probability that e2 is true = 0.6 * 0.3 + 0.4 * 0.7 = 0.46 + // Probability that d is true = 0.5 * 1 + 0.5 * 0.46 = 0.73 + weightedSampleTest(gen, "d", (b: Boolean) => b, 0.73) + } + + "with an observation on a compound flip, terminate quickly and produce the correct result" taggedAs (NonDeterministic) in { + // Tests the likelihood weighting implementation for compound flip + val gen = () => { + val universe = new Universe + val b = Uniform(0.0, 1.0)("b", universe) + for (_ <- 1 to 16) { Flip(b)("", universe).observe(true) } + for (_ <- 1 to 4) { Flip(b)("", universe).observe(false) } + universe + } + val alg = Importance.par(gen, numThreads, "b") + alg.start() + Thread.sleep(100) + val time0 = System.currentTimeMillis() + alg.stop() + // Uniform(0,1) is beta(1,1) + // Result is beta(1 + 16,1 + 4) + // Expectation is (alpha) / (alpha + beta) = 17/22 + val exp = alg.expectation("b", (d: Double) => d) + val time1 = System.currentTimeMillis() + // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous + // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample + alg.shutdown + (time1 - time0) should be <= (500L) + exp should be((17.0 / 22.0) +- 0.02) + } + + "with an observation on a parameterized flip, terminate quickly and produce the correct result" taggedAs (NonDeterministic) in { + // Tests the likelihood weighting implementation for compound flip + val gen = () => { + val universe = new Universe + val b = Beta(2.0, 5.0)("b", universe) + for (_ <- 1 to 16) { Flip(b)("", universe).observe(true) } + for (_ <- 1 to 4) { Flip(b)("", universe).observe(false) } + universe + } + val alg = Importance.par(gen, numThreads, "b") + alg.start() + Thread.sleep(100) + val time0 = System.currentTimeMillis() + alg.stop() + // Result is beta(2 + 16,5 + 4) + // Expectation is (alpha) / (alpha + beta) = 18/27 + val exp = alg.expectation("b")((d: Double) => d) + val time1 = System.currentTimeMillis() + // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous + // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample + alg.shutdown + (time1 - time0) should be <= (500L) + exp should be((18.0 / 27.0) +- 0.02) + } + + "with an observation on a parameterized binomial, terminate quickly and produce the correct result" in { + // Tests the likelihood weighting implementation for chain + val gen = () => { + val universe = new Universe + val beta = Beta(2.0, 5.0)("beta", universe) + val bin = Binomial(2000, beta)("", universe) + bin.observe(1600) + universe + } + val alg = Importance.par(gen, numThreads, "beta") + alg.start() + Thread.sleep(1000) + val time0 = System.currentTimeMillis() + alg.stop() + // Result is beta(2 + 1600,5 + 400) + // Expectation is (alpha) / (alpha + beta) = 1602/2007 + val exp = alg.expectation("beta")((d: Double) => d) + val time1 = System.currentTimeMillis() + // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous + // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample + alg.shutdown + (time1 - time0) should be <= (500L) + exp should be((1602.0 / 2007.0) +- 0.02) + } + + "with an observation on a chain, terminate quickly and produce the correct result" in { + // Tests the likelihood weighting implementation for chain + val gen = () => { + val universe = new Universe + val beta = Uniform(0.0, 1.0)("beta", universe) + val bin = Binomial(2000, beta)("", universe) + bin.observe(1600) + universe + } + val alg = Importance.par(gen, numThreads, "beta") + alg.start() + Thread.sleep(1000) + val time0 = System.currentTimeMillis() + alg.stop() + // uniform(0,1) is beta(1,1) + // Result is beta(1 + 1600,1 + 400) + // Expectation is (alpha) / (alpha + beta) = 1601/2003 + val exp = alg.expectation("beta")((d: Double) => d) + val time1 = System.currentTimeMillis() + // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous + // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample + alg.shutdown + (time1 - time0) should be <= (500L) + exp should be((1601.0 / 2003.0) +- 0.02) + } + + "with an observation on a dist, terminate quickly and produce the correct result" in { + // Tests the likelihood weighting implementation for dist + val gen = () => { + val universe = new Universe + val beta = Beta(2.0, 5.0)("beta", universe) + val dist = Dist(0.5 -> Constant(1000)(Name.default, universe), 0.5 -> Binomial(2000, beta)(Name.default, universe))("", universe) + dist.observe(1600) // forces it to choose bin, and observation should propagate to it + universe + } + val alg = Importance.par(gen, numThreads, "beta") + alg.start() + Thread.sleep(1000) + val time0 = System.currentTimeMillis() + alg.stop() + // Result is beta(2 + 1600,5 + 400) + // Expectation is (alpha) / (alpha + beta) = 1602/2007 + val exp = alg.expectation("beta")((d: Double) => d) + val time1 = System.currentTimeMillis() + // If likelihood weighting is working, stopping and querying the algorithm should be almost instantaneous + // If likelihood weighting is not working, stopping and querying the algorithm requires waiting for a non-rejected sample + alg.shutdown + (time1 - time0) should be <= (500L) + exp should be((1602.0 / 2007.0) +- 0.02) + } + } + + "Running ParImportance sampling" should { + "produce the correct answer each time when run twice with different conditions" in { + def gen(obs: Double*) = () => { + val universe = Universe.createNew() + val s = Select(0.5 -> 0.3, 0.5 -> 0.6) + val f = Flip(s)("f", universe) + for (o <- obs) { + s.observe(o) + } + universe + } + + val i1 = Importance.par(gen(0.3), numThreads, 20000, "f") + i1.start() + i1.probability("f", true) should be(0.3 +- 0.01) + i1.kill() + + val i2 = Importance.par(gen(0.3, 0.6), numThreads, 20000, "f") + i2.start() + i2.probability("f", true) should be(0.6 +- 0.01) + i2.kill() + } + + /* Test is not valid + "resample elements inside class defined in a chain" in { + val gen = () => { + val universe = Universe.createNew() + class temp { + val t1 = Flip(0.9) + } + val a = CachingChain(Constant(0), (i: Int) => Constant(new temp)) + val b = Apply(a, (t: temp) => t.t1.value)("b", universe) + universe + } + val alg = Importance.par(gen, numThreads, 10000, "b") + alg.start + alg.probability("b", true) should be(0.9 +- .01) + alg.kill + } + * + */ + + "not suffer from stack overflow with small probability of success" taggedAs (Performance) in { + val gen = () => { + val universe = Universe.createNew() + val f = Flip(0.000001)("f", universe) + f.observe(true) + universe + } + val i = Importance.par(gen, numThreads, 1, "f") + i.start + } + + "not suffer from memory leaks" taggedAs (Performance) in { + val gen = () => { + val universe = Universe.createNew() + val c = NonCachingChain(Uniform(0.2, 1.0), (d: Double) => Flip(d)(Name.default, universe))("c", universe) + universe + } + val i = Importance.par(gen, numThreads, 1000000, "c") + i.start + } + } + + "Computing probability of evidence using ParImportance sampling" when { + + "given a vanilla model with one condition" should { + "return the probability the condition is satisfied" in { + val gen = () => { + val universe = Universe.createNew() + val f = Flip(0.7)("f", universe) + universe + } + probEvidenceTest(gen, 0.7, List(NamedEvidence("f", Observation(true)))) + } + } + + "given a vanilla model with two independent conditions" should { + "return the probability both conditions are satisfied" in { + val gen = () => { + val universe = Universe.createNew() + val f1 = Flip(0.7)("f1", universe) + val f2 = Flip(0.4)("f2", universe) + universe + } + val prob = 0.7 * 0.4 + val evidence = List(NamedEvidence("f1", Observation(true)), NamedEvidence("f2", Observation(true))) + + val alg = Importance.par(gen, numThreads, 10000) + alg.start() + alg.probabilityOfEvidence(evidence) should be(prob +- 0.01) + } + } + + "given a vanilla mode with two dependent conditions" should { + "return the probability both conditions are jointly satisfied" in { + val gen = () => { + val universe = Universe.createNew() + val d = Select(0.2 -> 0.6, 0.8 -> 0.9) + val f1 = Flip(d)("f1", universe) + val f2 = Flip(d)("f2", universe) + universe + } + probEvidenceTest(gen, 0.2 * 0.6 * 0.6 + 0.8 * 0.9 * 0.9, List(NamedEvidence("f1", Observation(true)), NamedEvidence("f2", Observation(true)))) + } + } + + "given a vanilla model with two dependent conditions and a constraint" should { + "return the probability both conditions are satisfied, taking into account the constraint" in { + val gen = () => { + val universe = Universe.createNew() + val d = Select(0.5 -> 0.6, 0.5 -> 0.9)("d", universe) + d.setConstraint((d: Double) => if (d > 0.7) 0.8; else 0.2) + val f1 = Flip(d)("f1", universe) + val f2 = Flip(d)("f2", universe) + universe + } + probEvidenceTest(gen, 0.2 * 0.6 * 0.6 + 0.8 * 0.9 * 0.9, List(NamedEvidence("f1", Observation(true)), NamedEvidence("f2", Observation(true)))) + } + } + + "given a simple dist with a condition on the result" should { + "return the expectation over the clauses of the probability the result satisfies the condition" in { + val gen = () => { + val universe = Universe.createNew() + val d = Dist(0.3 -> Flip(0.6), 0.7 -> Flip(0.9))("d", universe) + universe + } + probEvidenceTest(gen, 0.3 * 0.6 + 0.7 * 0.9, List(NamedEvidence("d", Observation(true)))) + } + } + + "given a complex dist with a condition on the result" should { + "return the expectation over the clauses of the probability the result satisfies the condition" in { + val gen = () => { + val universe = Universe.createNew() + val p1 = Select(0.2 -> 0.4, 0.8 -> 0.6) + val p2 = Constant(0.4) + val d = Dist(p1 -> Flip(0.6), p2 -> Flip(0.9))("d", universe) + universe + } + probEvidenceTest(gen, 0.2 * (0.5 * 0.6 + 0.5 * 0.9) + 0.8 * (0.6 * 0.6 + 0.4 * 0.9), List(NamedEvidence("d", Observation(true)))) + } + } + + "given a continuous uniform with a condition" should { + "return the uniform probability of the condition" in { + val gen = () => { + val universe = Universe.createNew() + val u = Uniform(0.0, 1.0)("u", universe) + universe + } + val condition = (d: Double) => d < 0.4 + probEvidenceTest(gen, 0.4, List(NamedEvidence("u", Condition(condition)))) + } + } + + "given a caching chain with a condition on the result" should { + "return the expectation over the parent of the probability the result satisfies the condition" in { + val gen = () => { + val universe = Universe.createNew() + val p1 = Select(0.4 -> 0.3, 0.6 -> 0.9) + val c = CachingChain(p1, (d: Double) => if (d < 0.4) Flip(0.3)(Name.default, universe); else Flip(0.8)(Name.default, universe))("c", universe) + universe + } + probEvidenceTest(gen, 0.4 * 0.3 + 0.6 * 0.8, List(NamedEvidence("c", Observation(true)))) + } + } + + "given a non-caching chain with a condition on the result" should { + + "return the expectation over the parent of the probability the result satisfies the condition" in { + val gen = () => { + val universe = Universe.createNew() + val p1 = Uniform(0.0, 1.0) + val c = NonCachingChain(p1, (d: Double) => if (d < 0.4) Flip(0.3)(Name.default, universe); else Flip(0.8)(Name.default, universe))("c", universe) + universe + } + probEvidenceTest(gen, 0.4 * 0.3 + 0.6 * 0.8, List(NamedEvidence("c", Observation(true)))) + } + } + + "given a chain of two arguments whose result is a different element with a condition on the result" should { + "return the correct probability of evidence in the result" in { + val gen = () => { + val universe = Universe.createNew() + val x = Constant(false) + val y = Constant(false) + val u1 = Uniform(0.0, 1.0) + val u2 = Uniform(0.0, 2.0) + val a = CachingChain(x, y, (x: Boolean, y: Boolean) => if (x || y) u1; else u2)("a", universe) + universe + } + def condition(d: Double) = d < 0.5 + probEvidenceTest(gen, 0.25, List(NamedEvidence("a", Condition(condition)))) + } + } + + } + + def weightedSampleTest[T](gen: Function0[Universe], target: Reference[T], predicate: T => Boolean, prob: Double) { + val numTrials = 100000 + val tolerance = 0.01 + val algorithm = Importance.par(gen, numThreads, numTrials, target) + algorithm.start() + algorithm.probability(target, predicate) should be(prob +- tolerance) + } + + def probEvidenceTest(gen: Function0[Universe], prob: Double, evidence: List[NamedEvidence[_]]) { + val alg = Importance.par(gen, numThreads, 10000) + alg.start() + alg.probabilityOfEvidence(evidence) should be(prob +- 0.01) + } + +} + diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/ExpandTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/ExpandTest.scala similarity index 69% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/ExpandTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/ExpandTest.scala index 954b24a9..346fec04 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/ExpandTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/ExpandTest.scala @@ -10,14 +10,15 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured +package com.cra.figaro.test.algorithm.structured -import org.scalatest.{WordSpec, Matchers} -import com.cra.figaro.experimental.structured.ComponentCollection -import com.cra.figaro.experimental.structured.Problem +import org.scalatest.{ WordSpec, Matchers } +import com.cra.figaro.algorithm.structured.ComponentCollection import com.cra.figaro.language._ import com.cra.figaro.library.atomic.discrete.Uniform import com.cra.figaro.library.collection.MakeArray +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.language.Name.stringToName class ExpandTest extends WordSpec with Matchers { "Expanding a chain" should { @@ -34,8 +35,8 @@ class ExpandTest extends WordSpec with Matchers { val c4 = cc(e4) c4.expand(false) - c4.subproblems.size should equal (1) - c4.subproblems(false).target should equal (e3) + c4.subproblems.size should equal(1) + c4.subproblems(false).target should equal(e3) } "when repeating a parent value, not create a new expansion" in { @@ -52,7 +53,7 @@ class ExpandTest extends WordSpec with Matchers { c4.expand(false) c4.expand(false) - cc.expansions.size should equal (1) + cc.expansions.size should equal(1) } "at one parent value, not add the parent" in { @@ -68,7 +69,7 @@ class ExpandTest extends WordSpec with Matchers { val c4 = cc(e4) c4.expand(false) - cc.contains(e1) should equal (false) + cc.contains(e1) should equal(false) } "for a full expansion with an added parent, add subproblems for all the parent values" in { @@ -87,9 +88,9 @@ class ExpandTest extends WordSpec with Matchers { c1.generateRange() c4.expand() - c4.subproblems.size should equal (2) - c4.subproblems(true).target should equal (e2) - c4.subproblems(false).target should equal (e3) + c4.subproblems.size should equal(2) + c4.subproblems(true).target should equal(e2) + c4.subproblems(false).target should equal(e3) } "for a full expansion with an unadded parent, not expand anything or add the parent" in { @@ -105,8 +106,8 @@ class ExpandTest extends WordSpec with Matchers { val c4 = cc(e4) c4.expand() - c4.subproblems.size should equal (0) - cc.contains(e1) should equal (false) + c4.subproblems.size should equal(0) + cc.contains(e1) should equal(false) } "if the result elements are already in the collection, not create a new component but point to the existing one in the subproblem" in { @@ -129,11 +130,67 @@ class ExpandTest extends WordSpec with Matchers { c4.expand() c4.generateRange() - c4.range.hasStar should equal (false) - c4.range.regularValues should equal (Set(1, 2, 3)) + c4.range.hasStar should equal(false) + c4.range.regularValues should equal(Set(1, 2, 3)) } } + def process(comp: ProblemComponent[_]) = { + comp.generateRange(Int.MaxValue ) + comp.makeNonConstraintFactors(false) + comp.makeConstraintFactors(Lower) + } + + "Raising a subproblem" should { + "add non-constraint factors to the chain" in { + Universe.createNew() + val cc = new ComponentCollection + // raising should use the old chain method + val pr = new Problem(cc) + val e1 = Flip(0.2) + def f(b: Boolean) = if (b) Uniform(1,2) else Uniform(3,4) + val e4 = Chain(e1, f) + pr.add(e4) + pr.add(e1) + val c4 = cc(e4) + process(cc(e1)) + c4.expand() + c4.subproblems.foreach(f => process(cc(f._2.target))) + process(c4) + c4.raise(Lower) + c4.nonConstraintFactors.size should be (5) + val tf = cc(c4.subproblems(true).target).nonConstraintFactors(0) + val ff = cc(c4.subproblems(false).target).nonConstraintFactors(0) + val tfVar = c4.nonConstraintFactors.exists(p => p.output.exists(_ == tf.variables.head) || p.parents.exists(_ == tf.variables.head)) + val ffVar = c4.nonConstraintFactors.exists(p => p.output.exists(_ == ff.variables.head) || p.parents.exists(_ == ff.variables.head)) + tfVar should be (false) + ffVar should be (false) + } + + "add constraint factors to the chain" in { + Universe.createNew() + val cc = new ComponentCollection + // raising should use the old chain method + val pr = new Problem(cc) + val u1 = Uniform(1,2) + val u2 = Uniform(3,4) + val e1 = Flip(0.2) + def f(b: Boolean) = if (b) u1 else u2 + val e4 = Chain(e1, f) + u1.observe(1) + pr.add(e4) + pr.add(e1) + val c4 = cc(e4) + process(cc(e1)) + c4.expand() + c4.subproblems.foreach(f => process(cc(f._2.target))) + process(c4) + c4.raise(Lower) + c4.constraintFactors(Lower).size should be (1) + } + + } + "Expanding a MakeArray" should { "at one count, add all the previously unadded items to the problem" in { Universe.createNew() @@ -150,8 +207,8 @@ class ExpandTest extends WordSpec with Matchers { val c5 = cc(e5) c5.expand(2) - cc.contains(e1) should equal (true) - cc(e1).problem should equal (pr) + cc.contains(e1) should equal(true) + cc(e1).problem should equal(pr) } "leave previously added items in their original problem" in { @@ -170,8 +227,8 @@ class ExpandTest extends WordSpec with Matchers { val c5 = cc(e5) c5.expand(2) - cc(e0).problem should equal (pr1) - cc(e1).problem should equal (pr2) + cc(e0).problem should equal(pr1) + cc(e1).problem should equal(pr2) } "for a full expansion with an added numItems, add the items for the maximum parent value" in { @@ -191,9 +248,9 @@ class ExpandTest extends WordSpec with Matchers { c4.generateRange() c5.expand() - cc.contains(e0) should equal (true) - cc.contains(e1) should equal (true) - cc.contains(e2) should equal (true) + cc.contains(e0) should equal(true) + cc.contains(e1) should equal(true) + cc.contains(e2) should equal(true) } "for a full expansion with an unadded numItems, not expand anything or add numItems" in { @@ -210,10 +267,10 @@ class ExpandTest extends WordSpec with Matchers { val c5 = cc(e5) c5.expand() - cc.contains(e0) should equal (false) - cc.contains(e1) should equal (false) - cc.contains(e2) should equal (false) - cc.contains(e4) should equal (false) + cc.contains(e0) should equal(false) + cc.contains(e1) should equal(false) + cc.contains(e2) should equal(false) + cc.contains(e4) should equal(false) } } @@ -229,7 +286,7 @@ class ExpandTest extends WordSpec with Matchers { c1.expand(2) c1.expand(4) - c1.maxExpanded should equal (4) + c1.maxExpanded should equal(4) } "add all and only the previously unadded items to the problem" in { @@ -244,8 +301,8 @@ class ExpandTest extends WordSpec with Matchers { c1.expand(4) val n2 = cc.components.size - n1 should equal (3) // 1 for the MakeArray and 2 for the items - n2 should equal (5) + n1 should equal(3) // 1 for the MakeArray and 2 for the items + n2 should equal(5) } } @@ -260,7 +317,7 @@ class ExpandTest extends WordSpec with Matchers { c1.expand(4) c1.expand(2) - c1.maxExpanded should equal (4) + c1.maxExpanded should equal(4) } "add no items to the problem" in { @@ -275,8 +332,8 @@ class ExpandTest extends WordSpec with Matchers { c1.expand(2) val n2 = cc.components.size - n1 should equal (5) - n2 should equal (5) + n1 should equal(5) + n2 should equal(5) } } @@ -294,7 +351,7 @@ class ExpandTest extends WordSpec with Matchers { c1.generateRange() c2.expand() - c2.maxExpanded should equal (4) + c2.maxExpanded should equal(4) } "add all the elements to the collection" in { @@ -310,7 +367,7 @@ class ExpandTest extends WordSpec with Matchers { c1.generateRange() c2.expand() - cc.components.size should equal (6) // e1, e2, and the four items + cc.components.size should equal(6) // e1, e2, and the four items } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/FactorMakerTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/FactorMakerTest.scala similarity index 93% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/FactorMakerTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/FactorMakerTest.scala index 1f857968..20533134 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/FactorMakerTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/FactorMakerTest.scala @@ -10,13 +10,15 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured +package com.cra.figaro.test.algorithm.structured -import org.scalatest.{WordSpec, Matchers} +import org.scalatest.{Matchers, WordSpec} import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured._ -import com.cra.figaro.algorithm.lazyfactored.{Star, Regular, ValueSet} -import ValueSet.{withoutStar, withStar} +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.lazyfactored.{Regular, Star, ValueSet} +import ValueSet.{withStar, withoutStar} import com.cra.figaro.library.atomic.continuous.Beta import com.cra.figaro.library.atomic.continuous.Dirichlet import com.cra.figaro.algorithm.factored.ParticleGenerator @@ -29,7 +31,8 @@ import com.cra.figaro.util.HashMultiSet import com.cra.figaro.library.atomic.discrete.FromRange import com.cra.figaro.library.collection.MakeArray import com.cra.figaro.library.compound.FoldLeft -import com.cra.figaro.experimental.structured.factory.Factory.{makeTupleVarAndFactor, makeConditionalSelector} +import com.cra.figaro.algorithm.factored.factors.factory.Factory.{makeConditionalSelector, makeTupleVarAndFactor} +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredVE class FactorMakerTest extends WordSpec with Matchers { "Making a tuple variable and factor for a set of variables" should { @@ -47,7 +50,7 @@ class FactorMakerTest extends WordSpec with Matchers { c2.generateRange() val v1 = c1.variable val v2 = c2.variable - val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, v1, v2) + val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, None, v1, v2) val vars = tupleFactor.variables vars.size should equal (3) @@ -75,7 +78,7 @@ class FactorMakerTest extends WordSpec with Matchers { c3.generateRange() val v1 = c1.variable val v3 = c3.variable - val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, v1, v3) + val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, None, v1, v3) val vs = tupleVar.valueSet vs.hasStar should equal (false) @@ -112,7 +115,7 @@ class FactorMakerTest extends WordSpec with Matchers { c3.generateRange() val v1 = c1.variable val v3 = c3.variable - val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, v1, v3) + val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, None, v1, v3) tupleFactor.contents.size should equal (4) val v1IndexT = v1.range.indexOf(Regular(true)) @@ -161,8 +164,8 @@ class FactorMakerTest extends WordSpec with Matchers { val v1 = c1.variable val v2 = c2.variable val v3 = c3.variable - val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, v1, v2) - val selector = makeConditionalSelector(tupleVar, Regular(true), v3) + val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, None, v1, v2) + val selector = makeConditionalSelector(tupleVar, Regular(true), v3, Set()) selector.variables should equal (List(tupleVar, v3)) } @@ -186,8 +189,8 @@ class FactorMakerTest extends WordSpec with Matchers { val v1 = c1.variable val v2 = c2.variable val v3 = c3.variable - val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, v1, v2) - val selector = makeConditionalSelector(tupleVar, Regular(true), v3) + val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, None, v1, v2) + val selector = makeConditionalSelector(tupleVar, Regular(true), v3, Set()) val ctIndexT2 = tupleVar.range.indexOf(Regular(List(Regular(true), Regular(2)))) val ctIndexF2 = tupleVar.range.indexOf(Regular(List(Regular(false), Regular(2)))) @@ -234,14 +237,14 @@ class FactorMakerTest extends WordSpec with Matchers { val v1 = c1.variable val v2 = c2.variable val v3 = c3.variable - val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, v1, v2) + val (tupleVar, tupleFactor) = makeTupleVarAndFactor(cc, None, v1, v2) val c1IndexStar = v1.range.indexWhere(!_.isRegular) val c1Star = v1.range(c1IndexStar) val c3IndexStar = v1.range.indexWhere(!_.isRegular) val c3Star = v3.range(c3IndexStar) - val selectorF = makeConditionalSelector(tupleVar, Regular(false), v3) - val selectorStar = makeConditionalSelector(tupleVar, c1Star, v3) + val selectorF = makeConditionalSelector(tupleVar, Regular(false), v3, Set()) + val selectorStar = makeConditionalSelector(tupleVar, c1Star, v3, Set()) val ctIndexStar2 = tupleVar.range.indexOf(Regular(List(c1Star, Regular(2)))) val ctIndexF2 = tupleVar.range.indexOf(Regular(List(Regular(false), Regular(2)))) val ctIndexStar3 = tupleVar.range.indexOf(Regular(List(c1Star, Regular(3)))) @@ -466,7 +469,7 @@ class FactorMakerTest extends WordSpec with Matchers { val List(factor) = c2.nonConstraintFactors factor.variables should equal (List(c1.variable, c2.variable)) - factor.size should equal (ParticleGenerator.defaultTotalSamples * 2) + factor.size should equal (ParticleGenerator.defaultMaxNumSamplesAtChain * 2) for { (p, index) <- c1.variable.range.zipWithIndex } { factor.get(List(index, 0)) should equal (p.value) factor.get(List(index, 1)) should equal (1 - p.value) @@ -729,7 +732,7 @@ class FactorMakerTest extends WordSpec with Matchers { val List(factor) = c2.nonConstraintFactors factor.variables should equal (List(c1.variable, c2.variable)) - factor.size should equal (ParticleGenerator.defaultTotalSamples * 2) + factor.size should equal (ParticleGenerator.defaultMaxNumSamplesAtChain * 2) for { (xprobs, i) <- c1.variable.range.zipWithIndex j <- 0 until c2.variable.range.size @@ -832,10 +835,11 @@ class FactorMakerTest extends WordSpec with Matchers { val index = c2.variable.range.indexOf(Regular(n)) factor.get(List(index)) should equal (0.0) } + } } - } - "given a parameterized binomial with a fixed number of trials with an added success probability, when the parameterized flag = false" should { + + "given a parameterized binomial with a fixed number of trials with an added success probability, when the parameterized flag = false" should { "use the chain decomposition, conditioning on the values of the parameter" in { Universe.createNew() val cc = new ComponentCollection @@ -855,14 +859,14 @@ class FactorMakerTest extends WordSpec with Matchers { val List(var1, var2, tupleVar) = tupleFactor.variables var1 should equal (c1.variable) var2 should equal (c2.variable) - factors.size should equal (ParticleGenerator.defaultTotalSamples) + factors.size should equal (ParticleGenerator.defaultMaxNumSamplesAtChain) val vars = factors(0).variables vars.size should equal (2) vars(0) should equal (tupleVar) } } - "given a parameterized binomial with a fixed number of trials with an unadded success probability, when the parameterized flag = false" should { + "given a parameterized binomial with a fixed number of trials with an unadded success probability, when the parameterized flag = false" should { "create a simple factor with probability 1 for * and probability 0 for the outcomes" in { Universe.createNew() val cc = new ComponentCollection @@ -900,7 +904,7 @@ class FactorMakerTest extends WordSpec with Matchers { val List(factor) = c1.nonConstraintFactors factor.variables should equal (List(c1.variable)) - factor.size should equal (ParticleGenerator.defaultTotalSamples) + factor.size should equal (ParticleGenerator.defaultMaxNumSamplesAtChain) } } @@ -1036,7 +1040,7 @@ class FactorMakerTest extends WordSpec with Matchers { } } - "given a chain with parent without *" should { + "given a chain with parent without * using the old chain method" should { "produce a conditional selector for each parent value" in { Universe.createNew() val cc = new ComponentCollection @@ -1208,7 +1212,7 @@ class FactorMakerTest extends WordSpec with Matchers { } } - "given a chain with parent with *" should { + "given a chain with parent with * using the old chain method" should { "produce an appropriate conditional selector for the * parent value" in { Universe.createNew() val cc = new ComponentCollection @@ -1244,6 +1248,276 @@ class FactorMakerTest extends WordSpec with Matchers { } } + "given a chain with no globals, no *, and the same values for each parent value, using the new chain method" should { + "produce a single factor connecting parent and child" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val pr = new Problem(cc) + val v1 = Select(0.3 -> 1, 0.2 -> 2, 0.5 -> 3) + val v2 = Chain(v1, (i: Int) => Flip(i / 10.0)) + pr.add(v1) + pr.add(v2) + val c1 = cc(v1) + val c2 = cc(v2) + c1.generateRange() + c2.expand() + + val pr1 = cc.expansions(v2.chainFunction, 1) + val subV1 = pr1.target + val subC1 = cc(subV1) + subC1.generateRange() + subC1.makeNonConstraintFactors() + pr1.solve(new ConstantStrategy(marginalVariableElimination)) + + val pr2 = cc.expansions(v2.chainFunction, 2) + val subV2 = pr2.target + val subC2 = cc(subV2) + subC2.generateRange() + subC2.makeNonConstraintFactors() + pr2.solve(new ConstantStrategy(marginalVariableElimination)) + val pr3 = cc.expansions(v2.chainFunction, 3) + val subV3 = pr3.target + val subC3 = cc(subV3) + subC3.generateRange() + subC3.makeNonConstraintFactors() + pr3.solve(new ConstantStrategy(marginalVariableElimination)) + + c2.generateRange() + c2.makeNonConstraintFactors() + + val List(factor) = c2.nonConstraintFactors + factor.variables should equal (List(c1.variable, c2.variable)) + val v1Vals = c1.variable.range + val v2Vals = c2.variable.range + val v11 = v1Vals indexOf Regular(1) + val v12 = v1Vals indexOf Regular(2) + val v13 = v1Vals indexOf Regular(3) + val v2f = v2Vals indexOf Regular(false) + val v2t = v2Vals indexOf Regular(true) + factor.size should equal (6) + factor.get(List(v11,v2f)) should be (0.9 +- 0.0001) + factor.get(List(v11,v2t)) should be (0.1 +- 0.0001) + factor.get(List(v12,v2f)) should be (0.8 +- 0.0001) + factor.get(List(v12,v2t)) should be (0.2 +- 0.0001) + factor.get(List(v13,v2f)) should be (0.7 +- 0.0001) + factor.get(List(v13,v2t)) should be (0.3 +- 0.0001) + } + } + + "given a chain with no globals, no *, different values for each parent value, using new chain method" should { + "produce a factor mapping the parent to the child with the union of the values" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val pr = new Problem(cc) + val v1 = Flip(0.5) + val v2 = Chain(v1, (b: Boolean) => if (b) Select(0.1 -> 1, 0.9 -> 2) else Select(0.2 -> 2, 0.8 -> 3)) + pr.add(v1) + pr.add(v2) + val c1 = cc(v1) + val c2 = cc(v2) + c1.generateRange() + c2.expand() + + val prf = cc.expansions(v2.chainFunction, false) + val subVf = prf.target + val subCf = cc(subVf) + subCf.generateRange() + subCf.makeNonConstraintFactors() + prf.solve(new ConstantStrategy(marginalVariableElimination)) + val prt = cc.expansions(v2.chainFunction, true) + val subVt = prt.target + val subCt = cc(subVt) + subCt.generateRange() + subCt.makeNonConstraintFactors() + prt.solve(new ConstantStrategy(marginalVariableElimination)) + + c2.generateRange() + c2.makeNonConstraintFactors() + + val List(factor) = c2.nonConstraintFactors + factor.variables should equal (List(c1.variable, c2.variable)) + val v1Vals = c1.variable.range + val v2Vals = c2.variable.range + val v1f = v1Vals indexOf Regular(false) + val v1t = v1Vals indexOf Regular(true) + val v21 = v2Vals indexOf Regular(1) + val v22 = v2Vals indexOf Regular(2) + val v23 = v2Vals indexOf Regular(3) + factor.size should equal (6) + factor.get(List(v1f,v21)) should be (0.0 +- 0.0001) + factor.get(List(v1f,v22)) should be (0.2 +- 0.0001) + factor.get(List(v1f,v23)) should be (0.8 +- 0.0001) + factor.get(List(v1t,v21)) should be (0.1 +- 0.0001) + factor.get(List(v1t,v22)) should be (0.9 +- 0.0001) + factor.get(List(v1t,v23)) should be (0.0 +- 0.0001) + + } + } + + "given a chain with no globals, * in subproblem, using new chain method" should { + "produce a factor mapping parent to child including *" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val pr = new Problem(cc) + val v1 = Select(0.3 -> 1, 0.2 -> 2, 0.5 -> 3) + val v2 = Chain(v1, (i: Int) => Flip(i / 10.0)) + pr.add(v1) + pr.add(v2) + val c1 = cc(v1) + val c2 = cc(v2) + c1.generateRange() + c2.expand() + + val pr1 = cc.expansions(v2.chainFunction, 1) + val subV1 = pr1.target + val subC1 = cc(subV1) + subC1.generateRange() + subC1.makeNonConstraintFactors() + pr1.solve(new ConstantStrategy(marginalVariableElimination)) + val pr2 = cc.expansions(v2.chainFunction, 2) + val subV2 = pr2.target + val subC2 = cc(subV2) + subC2.generateRange() + subC2.makeNonConstraintFactors() + pr2.solve(new ConstantStrategy(marginalVariableElimination)) + val pr3 = cc.expansions(v2.chainFunction, 3) + // no range generation or factor creation + pr3.solve(new ConstantStrategy(marginalVariableElimination)) + + c2.generateRange() + c2.makeNonConstraintFactors() + + val List(factor) = c2.nonConstraintFactors + factor.variables should equal (List(c1.variable, c2.variable)) + val v1Vals = c1.variable.range + val v2Vals = c2.variable.range + val v11 = v1Vals indexOf Regular(1) + val v12 = v1Vals indexOf Regular(2) + val v13 = v1Vals indexOf Regular(3) + val v2f = v2Vals indexOf Regular(false) + val v2t = v2Vals indexOf Regular(true) + val v2Star = v2Vals indexWhere (!_.isRegular) + factor.size should equal (9) + factor.get(List(v11,v2f)) should be (0.9 +- 0.0001) + factor.get(List(v11,v2t)) should be (0.1 +- 0.0001) + factor.get(List(v11,v2Star)) should be (0.0 +- 0.0001) + factor.get(List(v12,v2f)) should be (0.8 +- 0.0001) + factor.get(List(v12,v2t)) should be (0.2 +- 0.0001) + factor.get(List(v12,v2Star)) should be (0.0 +- 0.0001) + factor.get(List(v13,v2f)) should be (0.0 +- 0.0001) + factor.get(List(v13,v2t)) should be (0.0 +- 0.0001) + factor.get(List(v13,v2Star)) should be (1.0 +- 0.0001) + + } + } + + "given a chain with no globals, * in parent values, using new chain method" should { + "produce a factor mapping parent to child including *" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val pr = new Problem(cc) + val vt = Constant(true) + val vf = Constant(false) + val v1 = Uniform(vt, vf) + val v2 = Chain(v1, (b: Boolean) => if (b) Select(0.1 -> 1, 0.9 -> 2) else Select(0.2 -> 1, 0.8 -> 2)) + pr.add(vt) + pr.add(vf) + pr.add(v1) + pr.add(v2) + val ct = cc(vt) + val cf = cc(vf) + val c1 = cc(v1) + val c2 = cc(v2) + ct.generateRange() + ct.makeNonConstraintFactors() + // do not generate range for cf + c1.generateRange() // will include true and * + c1.makeNonConstraintFactors() + c2.expand() + + val prt = cc.expansions(v2.chainFunction, true) + val subVt = prt.target + val subCt = cc(subVt) + subCt.generateRange() + subCt.makeNonConstraintFactors() + prt.solve(new ConstantStrategy(marginalVariableElimination)) + + c2.generateRange() + c2.makeNonConstraintFactors() + + val List(factor) = c2.nonConstraintFactors + factor.variables should equal (List(c1.variable, c2.variable)) + val v1Vals = c1.variable.range + val v2Vals = c2.variable.range + val v1Star = v1Vals.indexWhere(!_.isRegular) + val v1t = v1Vals indexOf Regular(true) + val v21 = v2Vals indexOf Regular(1) + val v22 = v2Vals indexOf Regular(2) + val v2Star = v2Vals.indexWhere(!_.isRegular) + factor.size should equal (6) + factor.get(List(v1Star,v21)) should be (0.0 +- 0.0001) + factor.get(List(v1Star,v22)) should be (0.0 +- 0.0001) + factor.get(List(v1Star,v2Star)) should be (1.0 +- 0.0001) + factor.get(List(v1t,v21)) should be (0.1 +- 0.0001) + factor.get(List(v1t,v22)) should be (0.9 +- 0.0001) + factor.get(List(v1t,v2Star)) should be (0.0 +- 0.0001) + } + } + + "given a chain with globals" should { + "not use the new method even when the new method is being used" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val pr = new Problem(cc) + val v1 = Constant(1) + val v2 = Flip(0.5) + val v3 = Chain(v2, (b: Boolean) => if (b) Apply(v1, (i: Int) => i) else Constant(2)) + pr.add(v1) + pr.add(v2) + pr.add(v3) + val c1 = cc(v1) + val c2 = cc(v2) + val c3 = cc(v3) + c1.generateRange() + c2.generateRange() + c3.expand() + val subPf = cc.expansions(v3.chainFunction, false) + val vPf = subPf.target + val cPf = cc(vPf) + cPf.generateRange() + cPf.makeNonConstraintFactors() + subPf.solve(new ConstantStrategy(marginalVariableElimination)) + val subPt = cc.expansions(v3.chainFunction, true) + val vPt = subPt.target + val cPt = cc(vPt) + cPt.generateRange() + cPt.makeNonConstraintFactors() + subPt.solve(new ConstantStrategy(marginalVariableElimination)) + c3.generateRange() + c3.makeNonConstraintFactors() + + c3.nonConstraintFactors.length should be > (1) + } + } + + "given a chain in which the outcome is a global, using the new method" should { + "produce the right result" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val pr = new Problem(cc) + val v1 = Constant(1) + val v2 = Flip(0.5) + val v3 = Chain(v2, (b: Boolean) => if (b) v1 else Constant(2)) + StructuredVE.probability(v3, 1) should equal (0.5) + } + } + "given an apply of one argument without *" should { "produce a sparse factor that matches the argument to the result via the function" in { Universe.createNew() @@ -3588,7 +3862,7 @@ class FactorMakerTest extends WordSpec with Matchers { factor1L.get(List(e2Index2, 0)) should equal (1.0) factor2L.size should equal (2) factor2L.get(List(e2IndexStar, 0)) should equal (0.3 +- .0001) - factor2L.get(List(e2Index2, 0)) should equal (0.3) + factor2L.get(List(e2Index2, 0)) should equal (0.3 +- .0001) val List(factor1U) = c11.constraintUpper val List(factor2U) = c12.constraintUpper factor1U.variables should equal (List(c2.variable, c11.variable)) @@ -3598,7 +3872,7 @@ class FactorMakerTest extends WordSpec with Matchers { factor1U.get(List(e2Index2, 0)) should equal (1.0) factor2U.size should equal (2) factor2U.get(List(e2IndexStar, 0)) should equal (1.0) - factor2U.get(List(e2Index2, 0)) should equal (0.3) + factor2U.get(List(e2Index2, 0)) should equal (0.3 +- .0001) } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/ProblemTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/ProblemTest.scala similarity index 89% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/ProblemTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/ProblemTest.scala index cc8c9458..231ff612 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/ProblemTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/ProblemTest.scala @@ -10,13 +10,14 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured +package com.cra.figaro.test.algorithm.structured import org.scalatest.{WordSpec, Matchers} -import com.cra.figaro.experimental.structured._ import com.cra.figaro.language._ import com.cra.figaro.library.compound.If import com.cra.figaro.library.collection.MakeArray +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.language.Name.stringToName class ProblemTest extends WordSpec with Matchers { "Adding an element to a problem" should { @@ -65,15 +66,6 @@ class ProblemTest extends WordSpec with Matchers { c1.maxExpanded should equal (0) } - "when adding an element twice, throw an exception" in { - Universe.createNew() - val cc = new ComponentCollection - val pr = new Problem(cc) - val e1 = Flip(0.1) - pr.add(e1) - - an [Exception] should be thrownBy pr.add(e1) - } } "Checking for the presence of a component" should { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/RangeTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/RangeTest.scala similarity index 99% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/RangeTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/RangeTest.scala index 5d11925d..b2a014ef 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/RangeTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/RangeTest.scala @@ -10,13 +10,12 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured +package com.cra.figaro.test.algorithm.structured import org.scalatest.{WordSpec, Matchers} import com.cra.figaro.language._ import com.cra.figaro.library.atomic.discrete.Uniform import com.cra.figaro.library.compound.If -import com.cra.figaro.experimental.structured._ import com.cra.figaro.algorithm.lazyfactored.ValueSet._ import com.cra.figaro.util.MultiSet import com.cra.figaro.algorithm.factored.ParticleGenerator @@ -28,6 +27,8 @@ import com.cra.figaro.library.atomic.continuous.Dirichlet import com.cra.figaro.library.atomic.discrete.Binomial import com.cra.figaro.library.compound.FoldLeft import com.cra.figaro.library.compound.IntSelector +import com.cra.figaro.algorithm.structured._ + class RangeTest extends WordSpec with Matchers { "Setting the range of a component" should { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/BPSolverTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/BPSolverTest.scala new file mode 100644 index 00000000..df963fab --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/BPSolverTest.scala @@ -0,0 +1,1020 @@ +/* + * BPSolverTest.scala + * Test of a belief propagation problem solver. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.test.algorithm.structured.solver + +import org.scalatest.{ WordSpec, Matchers } +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.factors.Factor +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.factored.factors.SumProductSemiring +import com.cra.figaro.algorithm.lazyfactored.Regular +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.solver._ + +class BPSolverTest extends WordSpec with Matchers { + + "Making a tuple factor for the BP solver" should { + "create a factor whose variables are the targets plus a tuple variable" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Flip(0.5) + val e2 = Constant(1) + val pr = new Problem(cc) + pr.add(e1) + pr.add(e2) + val c1 = cc(e1) + val c2 = cc(e2) + c1.generateRange() + c2.generateRange() + val v1 = c1.variable + val v2 = c2.variable + val bp = new BPSolver(pr, Set(), Set(v1, v2), List(), 100, SumProductSemiring()) + + val vars = bp.tupleFactor.variables + vars.size should equal(3) + vars.contains(v1) should equal(true) + vars.contains(v2) should equal(true) + vars.contains(bp.tupleVar) should equal(true) + } + + "create a variable whose range is all the tuples of the targets, without * when the targets do not have *" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Flip(0.5) + val e2 = Constant(1) + val pr = new Problem(cc) + pr.add(e1) + pr.add(e2) + val c1 = cc(e1) + val c2 = cc(e2) + c1.generateRange() + c2.generateRange() + val v1 = c1.variable + val v2 = c2.variable + val bp = new BPSolver(pr, Set(), Set(v1, v2), List(), 100, SumProductSemiring()) + + val vs = bp.tupleVar.valueSet + vs.hasStar should equal(false) + if (bp.tupleFactor.variables(0) == v1) { + vs.regularValues should equal(Set(List(Regular(true), Regular(1)), List(Regular(false), Regular(1)))) + } else { + vs.regularValues should equal(Set(List(Regular(1), Regular(true)), List(Regular(1), Regular(false)))) + } + } + + "create a variable whose range is all the tuples of the targets, with * when the targets have *" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Flip(0.5) + val e21 = Constant(1) + val e22 = Constant(2) + val e3 = Dist(0.4 -> e21, 0.6 -> e22) + val pr = new Problem(cc) + pr.add(e1) + pr.add(e21) + pr.add(e3) + val c1 = cc(e1) + val c21 = cc(e21) + val c3 = cc(e3) + c1.generateRange() + c21.generateRange() + c3.generateRange() + val v1 = c1.variable + val v3 = c3.variable + val v3IndexStar = v3.range.indexWhere(!_.isRegular) + val v3Star = v3.range(v3IndexStar) + val bp = new BPSolver(pr, Set(), Set(v1, v3), List(), 100, SumProductSemiring()) + + val vs = bp.tupleVar.valueSet + vs.hasStar should equal(false) + if (bp.tupleFactor.variables(0) == v1) { + vs.regularValues should equal(Set(List(Regular(true), Regular(1)), List(Regular(false), Regular(1)), + List(Regular(true), v3Star), List(Regular(false), v3Star))) + } else { + vs.regularValues should equal(Set(List(Regular(1), Regular(true)), List(Regular(1), Regular(false)), + List(v3Star, Regular(true)), List(v3Star, Regular(false)))) + } + } + + "create a sparse factor in which regular values are mapped to the corresponding tuple and * values are mapped to *" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Flip(0.5) + val e21 = Constant(1) + val e22 = Constant(2) + val e3 = Dist(0.4 -> e21, 0.6 -> e22) + val pr = new Problem(cc) + pr.add(e1) + pr.add(e21) + pr.add(e3) + val c1 = cc(e1) + val c21 = cc(e21) + val c3 = cc(e3) + c1.generateRange() + c21.generateRange() + c3.generateRange() + val v1 = c1.variable + val v3 = c3.variable + val bp = new BPSolver(pr, Set(), Set(v1, v3), List(), 100, SumProductSemiring()) + + val factor = bp.tupleFactor + val vt = bp.tupleVar + factor.contents.size should equal(4) + val v1IndexT = v1.range.indexOf(Regular(true)) + val v1IndexF = v1.range.indexOf(Regular(false)) + val v3Index1 = v3.range.indexOf(Regular(1)) + val v3IndexStar = v3.range.indexWhere(!_.isRegular) + val v3Star = v3.range(v3IndexStar) + if (factor.variables(0) == v1) { + val vtIndexT1 = vt.range.indexOf(Regular(List(Regular(true), Regular(1)))) + val vtIndexF1 = vt.range.indexOf(Regular(List(Regular(false), Regular(1)))) + val vtIndexTStar = vt.range.indexOf(Regular(List(Regular(true), v3Star))) + val vtIndexFStar = vt.range.indexOf(Regular(List(Regular(false), v3Star))) + factor.get(List(v1IndexT, v3Index1, vtIndexT1)) should equal(1.0) + factor.get(List(v1IndexF, v3Index1, vtIndexF1)) should equal(1.0) + factor.get(List(v1IndexT, v3IndexStar, vtIndexTStar)) should equal(1.0) + factor.get(List(v1IndexF, v3IndexStar, vtIndexFStar)) should equal(1.0) + } else { + val vtIndex1T = vt.range.indexOf(Regular(List(Regular(1), Regular(true)))) + val vtIndex1F = vt.range.indexOf(Regular(List(Regular(1), Regular(false)))) + val vtIndexStarT = vt.range.indexOf(Regular(List(v3IndexStar, Regular(true)))) + val vtIndexStarF = vt.range.indexOf(Regular(List(v3IndexStar, Regular(false)))) + factor.get(List(v3Index1, v1IndexT, vtIndex1T)) should equal(1.0) + factor.get(List(v3Index1, v1IndexF, vtIndex1F)) should equal(1.0) + factor.get(List(v3IndexStar, v1IndexT, vtIndexStarT)) should equal(1.0) + factor.get(List(v3IndexStar, v1IndexF, vtIndexStarF)) should equal(1.0) + } + } + } + + "Running marginalBeliefPropagation without *" when { + + "given a flat model with no conditions or constraints" should { + "produce the correct result over a single element" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + val pr = new Problem(cc, List(e2)) + pr.add(e1) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c2)) + pr.solved should equal(true) + val result = multiplyAll(pr.solution) + result.variables should equal(List(c2.variable)) + result.size should equal(2) + val c2IndexT = c2.variable.range.indexOf(Regular(true)) + val c2IndexF = c2.variable.range.indexOf(Regular(false)) + result.get(List(c2IndexT)) should be(0.6 +- 0.00000001) + result.get(List(c2IndexF)) should be(0.4 +- 0.00000001) + } + + "produce the correct result over multiple elements" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + val pr = new Problem(cc, List(e2, e3)) + pr.add(e1) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c2, c3)) + val result = multiplyAll(pr.solution) + result.variables.size should equal(2) + val c2IndexT = c2.variable.range.indexOf(Regular(true)) + val c2IndexF = c2.variable.range.indexOf(Regular(false)) + val c3IndexT = c3.variable.range.indexOf(Regular(true)) + val c3IndexF = c3.variable.range.indexOf(Regular(false)) + result.size should equal(4) + val var0 = result.variables(0) + val var1 = result.variables(1) + if (var0 == c2.variable) { + var1 should equal(c3.variable) + // Note the answers are incorrect, but since the model is loopy now we can't guarantee the answer. This check is to ensure + // that any subsequent changes to BP that change this value should be noted + result.get(List(c2IndexT, c3IndexT)) should equal(0.36 +- 0.00001) // should be 0.6 + result.get(List(c2IndexT, c3IndexF)) should equal(0.24 +- 0.00001) // should be 0 + result.get(List(c2IndexF, c3IndexT)) should equal(0.24 +- 0.00001) // 0 + result.get(List(c2IndexF, c3IndexF)) should equal(0.16 +- 0.00001) // .16 + } else { + var0 should equal(c3.variable) + var1 should equal(c2.variable) + // Note the answers are incorrect, but since the model is loopy now we can't guarantee the answer. This check is to ensure + // that any subsequent changes to BP that change this value should be noted + result.get(List(c3IndexT, c2IndexT)) should equal(0.36 +- 0.00001) // should be 0.6 + result.get(List(c3IndexT, c2IndexF)) should equal(0.24 +- 0.00001) // should be 0 + result.get(List(c3IndexF, c2IndexT)) should equal(0.24 +- 0.00001) // 0 + result.get(List(c3IndexF, c2IndexF)) should equal(0.16 +- 0.00001) // .16 + } + } + } + + "given a condition on a dependent element" should { + "produce the result with the correct probability" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c1)) + val result = multiplyAll(pr.solution) + val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) + val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) + val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) + val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) + result.size should equal(4) + val x3 = 0.25 * 0.3 + val x5 = 0.25 * 0.5 + val x7 = 0.25 * 0.7 + val x9 = 0.25 * 0.9 + val z = x3 + x5 + x7 + x9 + result.get(List(c1Index3)) should be((x3 / z) +- 0.000000001) + result.get(List(c1Index5)) should be((x5 / z) +- 0.000000001) + result.get(List(c1Index7)) should be((x7 / z) +- 0.000000001) + result.get(List(c1Index9)) should be((x9 / z) +- 0.000000001) + } + } + + "given a constraint on a dependent element" should { + "produce the result with the correct probability" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.addConstraint((b: Boolean) => if (b) 0.5 else 0.2) + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c1)) + val result = multiplyAll(pr.solution) + val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) + val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) + val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) + val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) + result.size should equal(4) + val x3 = 0.25 * (0.3 * 0.5 + 0.7 * 0.2) + val x5 = 0.25 * (0.5 * 0.5 + 0.5 * 0.2) + val x7 = 0.25 * (0.7 * 0.5 + 0.3 * 0.2) + val x9 = 0.25 * (0.9 * 0.5 + 0.1 * 0.2) + val z = x3 + x5 + x7 + x9 + result.get(List(c1Index3)) should be(x3 / z +- 0.000000001) + result.get(List(c1Index5)) should be(x5 / z +- 0.000000001) + result.get(List(c1Index7)) should be(x7 / z +- 0.000000001) + result.get(List(c1Index9)) should be(x9 / z +- 0.000000001) + } + } + + "given two constraints on a dependent element" should { + "produce the result with the correct probability" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.addConstraint((b: Boolean) => if (b) 0.5 else 0.2) + e3.addConstraint((b: Boolean) => if (b) 0.4 else 0.1) + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c1)) + val result = multiplyAll(pr.solution) + val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) + val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) + val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) + val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) + result.size should equal(4) + val x3 = 0.25 * (0.3 * 0.5 * 0.4 + 0.7 * 0.2 * 0.1) + val x5 = 0.25 * (0.5 * 0.5 * 0.4 + 0.5 * 0.2 * 0.1) + val x7 = 0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1) + val x9 = 0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1) + val z = x3 + x5 + x7 + x9 + result.get(List(c1Index3)) should be(x3 / z +- 0.000000001) + result.get(List(c1Index5)) should be(x5 / z +- 0.000000001) + result.get(List(c1Index7)) should be(x7 / z +- 0.000000001) + result.get(List(c1Index9)) should be(x9 / z +- 0.000000001) + } + } + + "given constraints on two dependent elements" should { + "produce the result with the correct probability" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e2.addConstraint((b: Boolean) => if (b) 0.5 else 0.2) + e3.addConstraint((b: Boolean) => if (b) 0.4 else 0.1) + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c1)) + val result = multiplyAll(pr.solution) + val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) + val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) + val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) + val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) + result.size should equal(4) + val x3 = 0.25 * (0.3 * 0.5 * 0.4 + 0.7 * 0.2 * 0.1) + val x5 = 0.25 * (0.5 * 0.5 * 0.4 + 0.5 * 0.2 * 0.1) + val x7 = 0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1) + val x9 = 0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1) + val z = x3 + x5 + x7 + x9 + result.get(List(c1Index3)) should be(x3 / z +- 0.000000001) + result.get(List(c1Index5)) should be(x5 / z +- 0.000000001) + result.get(List(c1Index7)) should be(x7 / z +- 0.000000001) + result.get(List(c1Index9)) should be(x9 / z +- 0.000000001) + } + } + + "given a contingent condition on an element" should { + "produce the result with the correct probability" in { + val universe = Universe.createNew() + val cc = new ComponentCollection + val ec1 = new EC1 + val ec2 = new EC1 + val e11 = Flip(0.6)("e1", ec1) + val e12 = Flip(0.3)("e1", ec2) + val e2 = Select(0.8 -> ec1, 0.2 -> ec2)("e2", universe) + universe.assertEvidence("e2.e1", Observation(true)) + val pr = new Problem(cc, List(e2)) + pr.add(e11) + pr.add(e12) + val c11 = cc(e11) + val c12 = cc(e12) + val c2 = cc(e2) + c11.generateRange() + c12.generateRange() + c2.generateRange() + c11.makeNonConstraintFactors() + c12.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c11.makeConstraintFactors() + c12.makeConstraintFactors() + c2.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + pr.globals should equal(Set(c2)) + val result = multiplyAll(pr.solution) + val c2Index1 = c2.variable.range.indexOf(Regular(ec1)) + val c2Index2 = c2.variable.range.indexOf(Regular(ec2)) + result.size should equal(2) + val x1 = (0.8 * 0.6) + val x2 = (0.2 * 0.3) + val z = x1 + x2 + result.get(List(c2Index1)) should be((x1 / z) +- 0.000000001) + result.get(List(c2Index2)) should be((x2 / z) +- 0.000000001) + } + } + + "with an element that uses another element multiple times, " + + "always produce the same value for the different uses" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Flip(0.5) + val e2 = Apply(e1, e1, (b1: Boolean, b2: Boolean) => b1 == b2) + val pr = new Problem(cc, List(e2)) + pr.add(e1) + val c1 = cc(e1) + val c2 = cc(e2) + c1.generateRange() + c2.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + val result = multiplyAll(pr.solution) + val c2IndexT = c2.variable.range.indexOf(Regular(true)) + val c2IndexF = c2.variable.range.indexOf(Regular(false)) + result.get(List(c2IndexT)) should be(1.0 +- 0.000000001) + result.get(List(c2IndexF)) should be(0.0 +- 0.000000001) + } + + "with a constraint on an element that is used multiple times, only factor in the constraint once" in { + Universe.createNew() + val cc = new ComponentCollection + val f1 = Flip(0.5) + val f2 = Flip(0.3) + val e1 = Apply(f1, f1, (b1: Boolean, b2: Boolean) => b1 == b2) + val e2 = Apply(f1, f2, (b1: Boolean, b2: Boolean) => b1 == b2) + val d = Dist(0.5 -> e1, 0.5 -> e2) + f1.setConstraint((b: Boolean) => if (b) 3.0; else 2.0) + + val pr = new Problem(cc, List(d)) + pr.add(f1) + pr.add(f2) + pr.add(e1) + pr.add(e2) + val cf1 = cc(f1) + val cf2 = cc(f2) + val ce1 = cc(e1) + val ce2 = cc(e2) + val cd = cc(d) + cf1.generateRange() + cf2.generateRange() + ce1.generateRange() + ce2.generateRange() + cd.generateRange() + cf1.makeNonConstraintFactors() + cf2.makeNonConstraintFactors() + ce1.makeNonConstraintFactors() + ce2.makeNonConstraintFactors() + cd.makeNonConstraintFactors() + cf1.makeConstraintFactors() + cf2.makeConstraintFactors() + ce1.makeConstraintFactors() + ce2.makeConstraintFactors() + cd.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + // Probability that f1 is true = 0.6 + // Probability that e1 is true = 1.0 + // Probability that e2 is true = 0.6 * 0.3 + 0.4 * 0.7 = 0.46 + // Probability that d is true = 0.5 * 1 + 0.5 * 0.46 = 0.73 + val result = multiplyAll(pr.solution) + val dIndexT = cd.variable.range.indexOf(Regular(true)) + val dIndexF = cd.variable.range.indexOf(Regular(false)) + val pT = result.get(List(dIndexT)) + val pF = result.get(List(dIndexF)) + (pT / (pT + pF)) should be(0.73 +- 0.000000001) + } + + "with elements that are not used by the query or evidence, produce the correct result" in { + val u1 = Universe.createNew() + val cc = new ComponentCollection + val u = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val f = Flip(u) + val a = If(f, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) + val pr = new Problem(cc, List(f)) + pr.add(u) + pr.add(a) + val cu = cc(u) + val cf = cc(f) + val ca = cc(a) + cu.generateRange() + cf.generateRange() + ca.expand() + ca.generateRange() + cu.makeNonConstraintFactors() + cf.makeNonConstraintFactors() + ca.makeNonConstraintFactors() + cu.makeConstraintFactors() + cf.makeConstraintFactors() + ca.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + val result = multiplyAll(pr.solution) + val fIndexT = cf.variable.range.indexOf(Regular(true)) + val fIndexF = cf.variable.range.indexOf(Regular(false)) + val pT = result.get(List(fIndexT)) + val pF = result.get(List(fIndexF)) + (pT / (pT + pF)) should be(0.6 +- 0.01) + } + + "with a model using chain and no conditions or constraints, when the outcomes are at the top level, produce the correct answer" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + val cc = new ComponentCollection + val pr = new Problem(cc, List(e4)) + pr.add(e1) + pr.add(e2) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.expand() + c4.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + val result = multiplyAll(pr.solution) + val c4Index1 = c4.variable.range.indexOf(Regular(1)) + result.get(List(c4Index1)) should be((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) + } + + "with a model using chain and no conditions or constraints, when the outcomes are nested, produce the correct answer" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + val cc = new ComponentCollection + val pr = new Problem(cc, List(e4)) + pr.add(e1) + val c1 = cc(e1) + val c4 = cc(e4) + c1.generateRange() + c4.expand() + val c2 = cc(e2) + val c3 = cc(e3) + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + val result = multiplyAll(pr.solution) + val c4Index1 = c4.variable.range.indexOf(Regular(1)) + result.get(List(c4Index1)) should be((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) + } + + "with a model using chain and a condition on the result, when the outcomes are at the top level, correctly condition the parent" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e4.observe(1) + + val cc = new ComponentCollection + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.expand() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + val result = multiplyAll(pr.solution) + val c1IndexT = c1.variable.range.indexOf(Regular(true)) + val c1IndexF = c1.variable.range.indexOf(Regular(false)) + val pT = result.get(List(c1IndexT)) + val pF = result.get(List(c1IndexF)) + (pT / (pT + pF)) should be((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) + } + + "with a model using chain and a condition on the result, when the outcomes are nested, correctly condition the parent" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e4.observe(1) + + val cc = new ComponentCollection + val pr = new Problem(cc, List(e1)) + pr.add(e4) + val c1 = cc(e1) + val c4 = cc(e4) + c1.generateRange() + c4.expand() + val c2 = cc(e2) + val c3 = cc(e3) + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + val result = multiplyAll(pr.solution) + val c1IndexT = c1.variable.range.indexOf(Regular(true)) + val c1IndexF = c1.variable.range.indexOf(Regular(false)) + val pT = result.get(List(c1IndexT)) + val pF = result.get(List(c1IndexF)) + (pT / (pT + pF)) should be((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) + } + + "with a model using chain and a condition on one of the outcome elements, when the outcomes are at the top level, correctly condition the result" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e2.observe(1) + val cc = new ComponentCollection + val pr = new Problem(cc, List(e4)) + pr.add(e1) + pr.add(e2) + pr.add(e3) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.expand() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + val result = multiplyAll(pr.solution) + val c4Index1 = c4.variable.range.indexOf(Regular(1)) + val c4Index2 = c4.variable.range.indexOf(Regular(2)) + val c4Index3 = c4.variable.range.indexOf(Regular(3)) + val p1 = result.get(List(c4Index1)) + val p2 = result.get(List(c4Index2)) + val p3 = result.get(List(c4Index3)) + (p1 / (p1 + p2 + p3)) should be((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) + } + + "with a model using chain and a condition on one of the outcome elements, when the outcomes are at the top level, " + + "not change the belief about the parent" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e2.observe(1) + val cc = new ComponentCollection + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.expand() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + val result = multiplyAll(pr.solution) + val c1IndexT = c1.variable.range.indexOf(Regular(true)) + val c1IndexF = c1.variable.range.indexOf(Regular(false)) + val pT = result.get(List(c1IndexT)) + val pF = result.get(List(c1IndexF)) + (pT / (pT + pF)) should be(0.3 +- 0.01) + } + + "with a model using chain and a condition on one of the outcome elements, when the outcomes are nested, correctly condition the result" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e2.observe(1) + + val cc = new ComponentCollection + val pr = new Problem(cc, List(e4)) + pr.add(e1) + val c1 = cc(e1) + val c4 = cc(e4) + c1.generateRange() + c4.expand() + val c2 = cc(e2) + val c3 = cc(e3) + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + val result = multiplyAll(pr.solution) + val c4Index1 = c4.variable.range.indexOf(Regular(1)) + val c4Index2 = c4.variable.range.indexOf(Regular(2)) + val c4Index3 = c4.variable.range.indexOf(Regular(3)) + val p1 = result.get(List(c4Index1)) + val p2 = result.get(List(c4Index2)) + val p3 = result.get(List(c4Index3)) + (p1 / (p1 + p2 + p3)) should be((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) + } + + "with a model using chain and a condition on one of the outcome elements, when the outcomes are nested, " + + "not change the belief about the parent" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e2.observe(1) + + val cc = new ComponentCollection + val pr = new Problem(cc, List(e1)) + pr.add(e4) + val c1 = cc(e1) + val c4 = cc(e4) + c1.generateRange() + c4.expand() + val c2 = cc(e2) + val c3 = cc(e3) + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalBeliefPropagation())) + + val result = multiplyAll(pr.solution) + val c1IndexT = c1.variable.range.indexOf(Regular(true)) + val c1IndexF = c1.variable.range.indexOf(Regular(false)) + val pT = result.get(List(c1IndexT)) + val pF = result.get(List(c1IndexF)) + (pT / (pT + pF)) should be(0.3 +- 0.01) + } + + } + + "Running MPE VariableElimination" when { + "given a target" should { + "produce the most likely factor over the target" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.75 -> 0.2, 0.25 -> 0.3) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + val pr = new Problem(cc, List(e1)) + pr.add(e1) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // p(e1=.2,e2=T,e3=F,e4=F) = 0.75 * 0.2 * 0.8 = .12 + // p(e1=.2,e2=F,e3=T,e4=F) = 0.75 * 0.8 * 0.2 = .12 + // p(e1=.3,e2=T,e3=F,e4=F) = 0.25 * 0.3 * 0.7 = .0525 + // p(e1=.3,e2=F,e3=T,e4=F) = 0.25 * 0.7 * 0.3 = .0525 + // MPE: e1=.2,e2=F,e3=F,e4=T + // If we leave e1 un-eliminated, we should end up with a factor that has e1=.2 at .48 and e1=.3 at .1225 + // However, since BP normalizes according to a MaxProduct semiring, the values are not normalized, so we look at the ratio + pr.solve(new ConstantStrategy(mpeBeliefPropagation(20))) + val f = pr.solution reduceLeft (_.product(_)) + f.numVars should equal(1) + if (f.get(List(0)) > f.get(List(1))) { + f.get(List(0)) / f.get(List(1)) should be(0.48 / 0.1225 +- 0.000001) + } else { + f.get(List(1)) / f.get(List(0)) should be(0.48 / 0.1225 +- 0.000001) + } + } + } + + "given a flat model" should { + "produce the correct most likely values for all elements with no conditions or constraints" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.75 -> 0.2, 0.25 -> 0.3) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + val pr = new Problem(cc, List()) + pr.add(e1) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // p(e1=.2,e2=T,e3=F,e4=F) = 0.75 * 0.2 * 0.8 = .12 + // p(e1=.2,e2=F,e3=T,e4=F) = 0.75 * 0.8 * 0.2 = .12 + // p(e1=.3,e2=T,e3=F,e4=F) = 0.25 * 0.3 * 0.7 = .0525 + // p(e1=.3,e2=F,e3=T,e4=F) = 0.25 * 0.7 * 0.3 = .0525 + // MPE: e1=.2,e2=F,e3=F,e4=T + pr.solve(new ConstantStrategy(mpeBeliefPropagation(20))) + pr.recordingFactors(c1.variable).get(List()).asInstanceOf[Double] should be(0.2 +- .0000001) + pr.recordingFactors(c2.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c3.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c4.variable).get(List()).asInstanceOf[Boolean] should be(true) + } + + "produce the correct most likely values for all elements with conditions and constraints" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.5 -> 0.2, 0.5 -> 0.3) + e1.addConstraint((d: Double) => if (d < 0.25) 3.0 else 1.0) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + e4.observe(true) + val pr = new Problem(cc, List()) + pr.add(e1) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // MPE: e1=.2,e2=F,e3=F,e4=T + pr.solve(new ConstantStrategy(mpeBeliefPropagation(20))) + pr.recordingFactors(c1.variable).get(List()).asInstanceOf[Double] should be(0.2 +- .0000001) + pr.recordingFactors(c2.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c3.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c4.variable).get(List()).asInstanceOf[Boolean] should be(true) + } + } + } + + def multiplyAll(factors: List[Factor[Double]]): Factor[Double] = factors.foldLeft(Factory.unit(SumProductSemiring()))(_.product(_)) + + class EC1 extends ElementCollection {} +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/BPSolverTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/GibbsSolverTest.scala similarity index 68% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/BPSolverTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/GibbsSolverTest.scala index f04a6035..c00ceb91 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/BPSolverTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/GibbsSolverTest.scala @@ -1,168 +1,182 @@ /* - * BPSolverTest.scala - * Test of a belief propagation problem solver. + * GibbsSolverTest.scala + * Test of a Gibbs sampling problem solver. * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 21, 2015 * * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured.solver +package com.cra.figaro.test.algorithm.structured.solver import org.scalatest.{WordSpec, Matchers} +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored.Regular +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured._ -import solver._ -import com.cra.figaro.library.compound._ import com.cra.figaro.library.atomic.discrete.Uniform -import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.experimental.structured.factory.Factory -import com.cra.figaro.algorithm.factored.factors.SumProductSemiring -import com.cra.figaro.algorithm.lazyfactored.Regular -import com.cra.figaro.algorithm.factored.factors.Variable +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.solver._ -class BPSolverTest extends WordSpec with Matchers { - "Making a tuple factor for the BP solver" should { - "create a factor whose variables are the targets plus a tuple variable" in { - Universe.createNew() - val cc = new ComponentCollection - val e1 = Flip(0.5) - val e2 = Constant(1) - val pr = new Problem(cc) - pr.add(e1) - pr.add(e2) - val c1 = cc(e1) - val c2 = cc(e2) - c1.generateRange() - c2.generateRange() - val v1 = c1.variable - val v2 = c2.variable - val bp = new BPSolver(pr, Set(), Set(v1, v2), List(), 100) - - val vars = bp.tupleFactor.variables - vars.size should equal (3) - vars.contains(v1) should equal (true) - vars.contains(v2) should equal (true) - vars.contains(bp.tupleVar) should equal (true) - } - - "create a variable whose range is all the tuples of the targets, without * when the targets do not have *" in { - Universe.createNew() - val cc = new ComponentCollection - val e1 = Flip(0.5) - val e2 = Constant(1) - val pr = new Problem(cc) - pr.add(e1) - pr.add(e2) - val c1 = cc(e1) - val c2 = cc(e2) - c1.generateRange() - c2.generateRange() - val v1 = c1.variable - val v2 = c2.variable - val bp = new BPSolver(pr, Set(), Set(v1, v2), List(), 100) - - val vs = bp.tupleVar.valueSet - vs.hasStar should equal (false) - if (bp.tupleFactor.variables(0) == v1) { - vs.regularValues should equal (Set(List(Regular(true), Regular(1)), List(Regular(false), Regular(1)))) - } else { - vs.regularValues should equal (Set(List(Regular(1), Regular(true)), List(Regular(1), Regular(false)))) +class GibbsSolverTest extends WordSpec with Matchers { + "Running a Gibbs solver" should { + "correctly produce blocks" when { + "given a problem that uses Apply" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Uniform(1, 2, 3) + val e2 = Uniform(4, 5, 6) + val e3 = Apply[Int, Int, Int](e1, e2, _ + _) + val pr = new Problem(cc, List(e1, e2, e3)) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c2.generateRange() + c3.generateRange() + // Make factors so variables have known parents + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + val v1 = c1.variable + val v2 = c2.variable + val v3 = c3.variable + + val solver = new GibbsSolver(pr, Set(), Set(v2 , v3), pr.components.flatMap(_.nonConstraintFactors), 1, 0, 1, BlockSampler.default) + // Call initialize to set solver.variables so createBlocks may be called + solver.initialize() + solver.createBlocks().map(_.toSet) should contain theSameElementsAs List(Set(v1, v3), Set(v2, v3)) } - } - "create a variable whose range is all the tuples of the targets, with * when the targets have *" in { - Universe.createNew() - val cc = new ComponentCollection - val e1 = Flip(0.5) - val e21 = Constant(1) - val e22 = Constant(2) - val e3 = Dist(0.4 -> e21, 0.6 -> e22) - val pr = new Problem(cc) - pr.add(e1) - pr.add(e21) - pr.add(e3) - val c1 = cc(e1) - val c21 = cc(e21) - val c3 = cc(e3) - c1.generateRange() - c21.generateRange() - c3.generateRange() - val v1 = c1.variable - val v3 = c3.variable - val v3IndexStar = v3.range.indexWhere(!_.isRegular) - val v3Star = v3.range(v3IndexStar) - val bp = new BPSolver(pr, Set(), Set(v1, v3), List(), 100) - - val vs = bp.tupleVar.valueSet - vs.hasStar should equal (false) - if (bp.tupleFactor.variables(0) == v1) { - vs.regularValues should equal (Set(List(Regular(true), Regular(1)), List(Regular(false), Regular(1)), - List(Regular(true), v3Star), List(Regular(false), v3Star))) - } else { - vs.regularValues should equal (Set(List(Regular(1), Regular(true)), List(Regular(1), Regular(false)), - List(v3Star, Regular(true)), List(v3Star, Regular(false)))) + "given a problem that uses Chain" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Flip(0.3) + val e2 = Uniform(1, 2, 3) + val e3 = Uniform(4, 5, 6) + val e4 = If(e1, e2, e3) + val pr = new Problem(cc, List(e1, e2, e3, e4)) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.expand() + c4.generateRange() + // Make factors so variables have known parents + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + val v1 = c1.variable + val v2 = c2.variable + val v3 = c3.variable + val v4 = c4.variable + + val solver = new GibbsSolver(pr, Set(), Set(v4), pr.components.flatMap(_.nonConstraintFactors), 1, 0, 1, BlockSampler.default) + // Call initialize to set solver.variables so createBlocks may be called + solver.initialize() + val v5 = (solver.variables -- Set(v1, v2, v3, v4)).head + solver.createBlocks().map(_.toSet) should contain theSameElementsAs List(Set(v1, v5), Set(v2, v4, v5), Set(v3, v4, v5)) } - } - "create a sparse factor in which regular values are mapped to the corresponding tuple and * values are mapped to *" in { - Universe.createNew() - val cc = new ComponentCollection - val e1 = Flip(0.5) - val e21 = Constant(1) - val e22 = Constant(2) - val e3 = Dist(0.4 -> e21, 0.6 -> e22) - val pr = new Problem(cc) - pr.add(e1) - pr.add(e21) - pr.add(e3) - val c1 = cc(e1) - val c21 = cc(e21) - val c3 = cc(e3) - c1.generateRange() - c21.generateRange() - c3.generateRange() - val v1 = c1.variable - val v3 = c3.variable - val bp = new BPSolver(pr, Set(), Set(v1, v3), List(), 100) - - val factor = bp.tupleFactor - val vt = bp.tupleVar - factor.contents.size should equal (4) - val v1IndexT = v1.range.indexOf(Regular(true)) - val v1IndexF = v1.range.indexOf(Regular(false)) - val v3Index1 = v3.range.indexOf(Regular(1)) - val v3IndexStar = v3.range.indexWhere(!_.isRegular) - val v3Star = v3.range(v3IndexStar) - if (factor.variables(0) == v1) { - val vtIndexT1 = vt.range.indexOf(Regular(List(Regular(true), Regular(1)))) - val vtIndexF1 = vt.range.indexOf(Regular(List(Regular(false), Regular(1)))) - val vtIndexTStar = vt.range.indexOf(Regular(List(Regular(true), v3Star))) - val vtIndexFStar = vt.range.indexOf(Regular(List(Regular(false), v3Star))) - factor.get(List(v1IndexT, v3Index1, vtIndexT1)) should equal (1.0) - factor.get(List(v1IndexF, v3Index1, vtIndexF1)) should equal (1.0) - factor.get(List(v1IndexT, v3IndexStar, vtIndexTStar)) should equal (1.0) - factor.get(List(v1IndexF, v3IndexStar, vtIndexFStar)) should equal (1.0) - } else { - val vtIndex1T = vt.range.indexOf(Regular(List(Regular(1), Regular(true)))) - val vtIndex1F = vt.range.indexOf(Regular(List(Regular(1), Regular(false)))) - val vtIndexStarT = vt.range.indexOf(Regular(List(v3IndexStar, Regular(true)))) - val vtIndexStarF = vt.range.indexOf(Regular(List(v3IndexStar, Regular(false)))) - factor.get(List(v3Index1, v1IndexT, vtIndex1T)) should equal (1.0) - factor.get(List(v3Index1, v1IndexF, vtIndex1F)) should equal (1.0) - factor.get(List(v3IndexStar, v1IndexT, vtIndexStarT)) should equal (1.0) - factor.get(List(v3IndexStar, v1IndexF, vtIndexStarF)) should equal (1.0) + "given a Chain with compact factors" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val e1 = Flip(0.4) + val e2 = Chain(e1, (b: Boolean) => if(b) Uniform(1, 2, 3, 4) else Uniform(3, 4, 5)) + val e3 = Apply(e2, (i: Int) => i * 2) + val pr = new Problem(cc, List(e3)) + pr.add(e1) + pr.add(e2) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c1.makeNonConstraintFactors() + c2.expand() + for((_, spr) <- c2.subproblems) { + val target = cc(spr.target) + target.generateRange() + target.makeNonConstraintFactors() + spr.solve(new ConstantStrategy(marginalVariableElimination)) + } + c2.generateRange() + c2.makeNonConstraintFactors() + c3.generateRange() + c3.makeNonConstraintFactors() + val v1 = c1.variable + val v2 = c2.variable + val v3 = c3.variable + + val solver = new GibbsSolver(pr, Set(), Set(v3), pr.components.flatMap(_.nonConstraintFactors), 1, 0, 1, BlockSampler.default) + // Call initialize to set solver.variables so createBlocks may be called + solver.initialize() + // Chain should not be blocked with parent, but should still be blocked with its deterministic children + solver.createBlocks().map(_.toSet) should contain theSameElementsAs List(Set(v1), Set(v2, v3)) } } } - "Running beliefPropagation without *" when { - + // Largely the same as VESolver tests, but modified to include tolerance and replacing Dist where needed + "Running Gibbs without *" when { "given a flat model with no conditions or constraints" should { + "produce the correct result with a compact Chain factor" in { + Universe.createNew() + val cc = new ComponentCollection + cc.useSingleChainFactor = true + val e1 = Flip(0.4) + val e2 = Chain(e1, (b: Boolean) => if(b) Uniform(1, 2) else Uniform(2, 3)) + val e3 = Apply(e2, (i: Int) => i * 2) + val pr = new Problem(cc, List(e3)) + pr.add(e1) + pr.add(e2) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + c1.generateRange() + c1.makeNonConstraintFactors() + c2.expand() + for((_, spr) <- c2.subproblems) { + val target = cc(spr.target) + target.generateRange() + target.makeNonConstraintFactors() + spr.solve(new ConstantStrategy(marginalVariableElimination)) + } + c2.generateRange() + c2.makeNonConstraintFactors() + c3.generateRange() + c3.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) + + val result = multiplyAll(pr.solution) + result.variables should equal (List(c3.variable)) + result.size should equal (3) + val c3Index2 = c3.variable.range.indexOf(Regular(2)) + val c3Index4 = c3.variable.range.indexOf(Regular(4)) + val c3Index6 = c3.variable.range.indexOf(Regular(6)) + result.get(List(c3Index2)) should be (0.2 +- tol) + result.get(List(c3Index4)) should be (0.5 +- tol) + result.get(List(c3Index6)) should be (0.3 +- tol) + } + "produce the correct result over a single element" in { Universe.createNew() val cc = new ComponentCollection @@ -184,7 +198,7 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c2)) pr.solved should equal (true) @@ -193,8 +207,8 @@ class BPSolverTest extends WordSpec with Matchers { result.size should equal (2) val c2IndexT = c2.variable.range.indexOf(Regular(true)) val c2IndexF = c2.variable.range.indexOf(Regular(false)) - result.get(List(c2IndexT)) should be (0.6 +- 0.00000001) - result.get(List(c2IndexF)) should be (0.4 +- 0.00000001) + result.get(List(c2IndexT)) should be (0.6 +- tol) + result.get(List(c2IndexF)) should be (0.4 +- tol) } "produce the correct result over multiple elements" in { @@ -217,7 +231,7 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c2, c3)) val result = multiplyAll(pr.solution) @@ -231,17 +245,17 @@ class BPSolverTest extends WordSpec with Matchers { val var1 = result.variables(1) if (var0 == c2.variable) { var1 should equal (c3.variable) - result.get(List(c2IndexT, c3IndexT)) should equal (0.6) - result.get(List(c2IndexT, c3IndexF)) should equal (0.0) - result.get(List(c2IndexF, c3IndexT)) should equal (0.0) - result.get(List(c2IndexF, c3IndexF)) should equal (0.4) + result.get(List(c2IndexT, c3IndexT)) should be (0.6 +- tol) + result.get(List(c2IndexT, c3IndexF)) should be (0.0 +- tol) + result.get(List(c2IndexF, c3IndexT)) should be (0.0 +- tol) + result.get(List(c2IndexF, c3IndexF)) should be (0.4 +- tol) } else { var0 should equal (c3.variable) var1 should equal (c2.variable) - result.get(List(c3IndexT, c2IndexT)) should equal (0.6) - result.get(List(c3IndexT, c2IndexF)) should equal (0.0) - result.get(List(c3IndexF, c2IndexT)) should equal (0.0) - result.get(List(c3IndexF, c2IndexF)) should equal (0.4) + result.get(List(c3IndexT, c2IndexT)) should be (0.6 +- tol) + result.get(List(c3IndexT, c2IndexF)) should be (0.0 +- tol) + result.get(List(c3IndexF, c2IndexT)) should be (0.0 +- tol) + result.get(List(c3IndexF, c2IndexF)) should be (0.4 +- tol) } } } @@ -269,7 +283,7 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -283,10 +297,10 @@ class BPSolverTest extends WordSpec with Matchers { val x7 = 0.25 * 0.7 val x9 = 0.25 * 0.9 val z = x3 + x5 + x7 + x9 - result.get(List(c1Index3)) should be ((x3 / z) +- 0.000000001) - result.get(List(c1Index5)) should be ((x5 / z) +- 0.000000001) - result.get(List(c1Index7)) should be ((x7 / z) +- 0.000000001) - result.get(List(c1Index9)) should be ((x9 / z) +- 0.000000001) + result.get(List(c1Index3)) should be ((x3 / z) +- tol) + result.get(List(c1Index5)) should be ((x5 / z) +- tol) + result.get(List(c1Index7)) should be ((x7 / z) +- tol) + result.get(List(c1Index9)) should be ((x9 / z) +- tol) } } @@ -313,7 +327,7 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -327,10 +341,10 @@ class BPSolverTest extends WordSpec with Matchers { val x7 = 0.25 * (0.7 * 0.5 + 0.3 * 0.2) val x9 = 0.25 * (0.9 * 0.5 + 0.1 * 0.2) val z = x3 + x5 + x7 + x9 - result.get(List(c1Index3)) should be (x3 / z +- 0.000000001) - result.get(List(c1Index5)) should be (x5 / z +- 0.000000001) - result.get(List(c1Index7)) should be (x7 / z +- 0.000000001) - result.get(List(c1Index9)) should be (x9 / z +- 0.000000001) + result.get(List(c1Index3)) should be (x3 / z +- tol) + result.get(List(c1Index5)) should be (x5 / z +- tol) + result.get(List(c1Index7)) should be (x7 / z +- tol) + result.get(List(c1Index9)) should be (x9 / z +- tol) } } @@ -358,7 +372,7 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -372,10 +386,10 @@ class BPSolverTest extends WordSpec with Matchers { val x7 = 0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1) val x9 = 0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1) val z = x3 + x5 + x7 + x9 - result.get(List(c1Index3)) should be (x3 / z +- 0.000000001) - result.get(List(c1Index5)) should be (x5 / z +- 0.000000001) - result.get(List(c1Index7)) should be (x7 / z +- 0.000000001) - result.get(List(c1Index9)) should be (x9 / z +- 0.000000001) + result.get(List(c1Index3)) should be (x3 / z +- tol) + result.get(List(c1Index5)) should be (x5 / z +- tol) + result.get(List(c1Index7)) should be (x7 / z +- tol) + result.get(List(c1Index9)) should be (x9 / z +- tol) } } @@ -403,7 +417,7 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -417,10 +431,10 @@ class BPSolverTest extends WordSpec with Matchers { val x7 = 0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1) val x9 = 0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1) val z = x3 + x5 + x7 + x9 - result.get(List(c1Index3)) should be (x3 / z +- 0.000000001) - result.get(List(c1Index5)) should be (x5 / z +- 0.000000001) - result.get(List(c1Index7)) should be (x7 / z +- 0.000000001) - result.get(List(c1Index9)) should be (x9 / z +- 0.000000001) + result.get(List(c1Index3)) should be (x3 / z +- tol) + result.get(List(c1Index5)) should be (x5 / z +- tol) + result.get(List(c1Index7)) should be (x7 / z +- tol) + result.get(List(c1Index9)) should be (x9 / z +- tol) } } @@ -449,7 +463,7 @@ class BPSolverTest extends WordSpec with Matchers { c11.makeConstraintFactors() c12.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) pr.globals should equal (Set(c2)) val result = multiplyAll(pr.solution) @@ -459,8 +473,8 @@ class BPSolverTest extends WordSpec with Matchers { val x1 = (0.8 * 0.6) val x2 = (0.2 * 0.3) val z = x1 + x2 - result.get(List(c2Index1)) should be ((x1 / z) +- 0.000000001) - result.get(List(c2Index2)) should be ((x2 / z) +- 0.000000001) + result.get(List(c2Index1)) should be ((x1 / z) +- tol) + result.get(List(c2Index2)) should be ((x2 / z) +- tol) } } @@ -480,12 +494,12 @@ class BPSolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c1.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c2IndexT = c2.variable.range.indexOf(Regular(true)) val c2IndexF = c2.variable.range.indexOf(Regular(false)) - result.get(List(c2IndexT)) should be (1.0 +- 0.000000001) - result.get(List(c2IndexF)) should be (0.0 +- 0.000000001) + result.get(List(c2IndexT)) should equal (1.0) + result.get(List(c2IndexF)) should equal (0.0) } "with a constraint on an element that is used multiple times, only factor in the constraint once" in { @@ -495,7 +509,8 @@ class BPSolverTest extends WordSpec with Matchers { val f2 = Flip(0.3) val e1 = Apply(f1, f1, (b1: Boolean, b2: Boolean) => b1 == b2) val e2 = Apply(f1, f2, (b1: Boolean, b2: Boolean) => b1 == b2) - val d = Dist(0.5 -> e1, 0.5 -> e2) + val f3 = Flip(0.5) + val d = If(f3, e1, e2) f1.setConstraint((b: Boolean) => if (b) 3.0; else 2.0) val pr = new Problem(cc, List(d)) @@ -503,27 +518,33 @@ class BPSolverTest extends WordSpec with Matchers { pr.add(f2) pr.add(e1) pr.add(e2) + pr.add(f3) val cf1 = cc(f1) val cf2 = cc(f2) val ce1 = cc(e1) val ce2 = cc(e2) + val cf3 = cc(f3) val cd = cc(d) cf1.generateRange() cf2.generateRange() ce1.generateRange() ce2.generateRange() + cf3.generateRange() + cd.expand() cd.generateRange() cf1.makeNonConstraintFactors() cf2.makeNonConstraintFactors() ce1.makeNonConstraintFactors() ce2.makeNonConstraintFactors() + cf3.makeNonConstraintFactors() cd.makeNonConstraintFactors() cf1.makeConstraintFactors() cf2.makeConstraintFactors() ce1.makeConstraintFactors() ce2.makeConstraintFactors() + cf3.makeConstraintFactors() cd.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) // Probability that f1 is true = 0.6 // Probability that e1 is true = 1.0 @@ -534,7 +555,7 @@ class BPSolverTest extends WordSpec with Matchers { val dIndexF = cd.variable.range.indexOf(Regular(false)) val pT = result.get(List(dIndexT)) val pF = result.get(List(dIndexF)) - (pT / (pT + pF)) should be (0.73 +- 0.000000001) + (pT / (pT + pF)) should be (0.73 +- tol) } "with elements that are not used by the query or evidence, produce the correct result" in { @@ -551,6 +572,7 @@ class BPSolverTest extends WordSpec with Matchers { val ca = cc(a) cu.generateRange() cf.generateRange() + ca.expand() ca.generateRange() cu.makeNonConstraintFactors() cf.makeNonConstraintFactors() @@ -558,13 +580,13 @@ class BPSolverTest extends WordSpec with Matchers { cu.makeConstraintFactors() cf.makeConstraintFactors() ca.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val fIndexT = cf.variable.range.indexOf(Regular(true)) val fIndexF = cf.variable.range.indexOf(Regular(false)) val pT = result.get(List(fIndexT)) val pF = result.get(List(fIndexF)) - (pT / (pT + pF)) should be (0.6 +- 0.01) + (pT / (pT + pF)) should be (0.6 +- tol) } "with a model using chain and no conditions or constraints, when the outcomes are at the top level, produce the correct answer" in { @@ -595,10 +617,10 @@ class BPSolverTest extends WordSpec with Matchers { c2.makeConstraintFactors() c3.makeConstraintFactors() c4.makeConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) - result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) + result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- tol) } "with a model using chain and no conditions or constraints, when the outcomes are nested, produce the correct answer" in { @@ -626,12 +648,12 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default)))) c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) - result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) + result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- tol) } "with a model using chain and a condition on the result, when the outcomes are at the top level, correctly condition the parent" in { @@ -664,14 +686,14 @@ class BPSolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) val c1IndexF = c1.variable.range.indexOf(Regular(false)) val pT = result.get(List(c1IndexT)) val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be ((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) + (pT / (pT + pF)) should be ((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- tol) } "with a model using chain and a condition on the result, when the outcomes are nested, correctly condition the parent" in { @@ -701,16 +723,16 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default)))) c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) val c1IndexF = c1.variable.range.indexOf(Regular(false)) val pT = result.get(List(c1IndexT)) val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be ((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) + (pT / (pT + pF)) should be ((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- tol) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are at the top level, correctly condition the result" in { @@ -742,7 +764,7 @@ class BPSolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -751,7 +773,7 @@ class BPSolverTest extends WordSpec with Matchers { val p1 = result.get(List(c4Index1)) val p2 = result.get(List(c4Index2)) val p3 = result.get(List(c4Index3)) - (p1 / (p1 + p2 + p3)) should be ((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) + (p1 / (p1 + p2 + p3)) should be ((0.3 * 1 + 0.7 * 0.7) +- tol) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are at the top level, " + @@ -784,14 +806,14 @@ class BPSolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) val c1IndexF = c1.variable.range.indexOf(Regular(false)) val pT = result.get(List(c1IndexT)) val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be (0.3 +- 0.01) + (pT / (pT + pF)) should be (0.3 +- tol) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are nested, correctly condition the result" in { @@ -821,9 +843,9 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default)))) c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -832,7 +854,7 @@ class BPSolverTest extends WordSpec with Matchers { val p1 = result.get(List(c4Index1)) val p2 = result.get(List(c4Index2)) val p3 = result.get(List(c4Index3)) - (p1 / (p1 + p2 + p3)) should be ((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) + (p1 / (p1 + p2 + p3)) should be ((0.3 * 1 + 0.7 * 0.7) +- tol) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are nested, " + @@ -863,16 +885,16 @@ class BPSolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default)))) c4.makeNonConstraintFactors() - pr.solve(beliefPropagation()) + pr.solve(new ConstantStrategy(marginalGibbs(10000, 0, 1, BlockSampler.default))) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) val c1IndexF = c1.variable.range.indexOf(Regular(false)) val pT = result.get(List(c1IndexT)) val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be (0.3 +- 0.01) + (pT / (pT + pF)) should be (0.3 +- tol) } } @@ -880,4 +902,6 @@ class BPSolverTest extends WordSpec with Matchers { def multiplyAll(factors: List[Factor[Double]]): Factor[Double] = factors.foldLeft(Factory.unit(SumProductSemiring()))(_.product(_)) class EC1 extends ElementCollection { } + + val tol = 0.025 } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/VEBPChooserTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/VEBPChooserTest.scala similarity index 93% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/VEBPChooserTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/VEBPChooserTest.scala index f951f747..aadae275 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/VEBPChooserTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/VEBPChooserTest.scala @@ -10,18 +10,18 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured.solver +package com.cra.figaro.test.algorithm.structured.solver import org.scalatest.{WordSpec, Matchers} import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured._ -import solver._ import com.cra.figaro.library.compound._ -import com.cra.figaro.library.atomic.discrete.Uniform import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.experimental.structured.factory.Factory +import com.cra.figaro.algorithm.factored.factors.factory.Factory import com.cra.figaro.algorithm.factored.factors.SumProductSemiring import com.cra.figaro.algorithm.lazyfactored.Regular +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.solver._ class VEBPChooserTest extends WordSpec with Matchers { "Choosing VE or BP with a high score threshold so VE is chosen" when { @@ -47,7 +47,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c2)) pr.solved should equal (true) @@ -80,7 +80,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c2, c3)) val result = multiplyAll(pr.solution) @@ -132,7 +132,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -171,7 +171,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -211,7 +211,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -251,7 +251,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -292,7 +292,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c11.makeConstraintFactors() c12.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) pr.globals should equal (Set(c2)) val result = multiplyAll(pr.solution) @@ -320,7 +320,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c1.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c2IndexT = c2.variable.range.indexOf(Regular(true)) @@ -364,7 +364,7 @@ class VEBPChooserTest extends WordSpec with Matchers { ce1.makeConstraintFactors() ce2.makeConstraintFactors() cd.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) // Probability that f1 is true = 0.6 // Probability that e1 is true = 1.0 @@ -399,7 +399,7 @@ class VEBPChooserTest extends WordSpec with Matchers { cu.makeConstraintFactors() cf.makeConstraintFactors() ca.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val fIndexT = cf.variable.range.indexOf(Regular(true)) @@ -437,7 +437,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeConstraintFactors() c3.makeConstraintFactors() c4.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -469,9 +469,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -508,7 +508,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -545,9 +545,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -586,7 +586,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -628,7 +628,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -649,6 +649,8 @@ class VEBPChooserTest extends WordSpec with Matchers { val cc = new ComponentCollection val pr = new Problem(cc, List(e4)) pr.add(e1) + pr.add(e2) + pr.add(e3) val c1 = cc(e1) val c4 = cc(e4) c1.generateRange() @@ -665,9 +667,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -690,6 +692,8 @@ class VEBPChooserTest extends WordSpec with Matchers { e2.observe(1) val cc = new ComponentCollection val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) pr.add(e4) val c1 = cc(e1) val c4 = cc(e4) @@ -707,9 +711,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) + pr.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -744,7 +748,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) pr.globals should equal (Set(c2)) pr.solved should equal (true) @@ -777,7 +781,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) pr.globals should equal (Set(c2, c3)) val result = multiplyAll(pr.solution) @@ -829,7 +833,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -873,7 +877,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -918,7 +922,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -963,7 +967,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) pr.globals should equal (Set(c1)) val result = multiplyAll(pr.solution) @@ -1009,7 +1013,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c11.makeConstraintFactors() c12.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) pr.globals should equal (Set(c2)) val result = multiplyAll(pr.solution) @@ -1040,7 +1044,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c1.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c2IndexT = c2.variable.range.indexOf(Regular(true)) val c2IndexF = c2.variable.range.indexOf(Regular(false)) @@ -1083,7 +1087,7 @@ class VEBPChooserTest extends WordSpec with Matchers { ce1.makeConstraintFactors() ce2.makeConstraintFactors() cd.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) // Probability that f1 is true = 0.6 // Probability that e1 is true = 1.0 @@ -1111,6 +1115,7 @@ class VEBPChooserTest extends WordSpec with Matchers { val ca = cc(a) cu.generateRange() cf.generateRange() + ca.expand() ca.generateRange() cu.makeNonConstraintFactors() cf.makeNonConstraintFactors() @@ -1118,7 +1123,7 @@ class VEBPChooserTest extends WordSpec with Matchers { cu.makeConstraintFactors() cf.makeConstraintFactors() ca.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val fIndexT = cf.variable.range.indexOf(Regular(true)) val fIndexF = cf.variable.range.indexOf(Regular(false)) @@ -1155,7 +1160,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeConstraintFactors() c3.makeConstraintFactors() c4.makeConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) @@ -1186,9 +1191,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) @@ -1224,7 +1229,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -1261,9 +1266,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -1302,7 +1307,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -1344,7 +1349,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -1381,9 +1386,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -1423,9 +1428,9 @@ class VEBPChooserTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(beliefPropagation())) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalBeliefPropagation()))) c4.makeNonConstraintFactors() - pr.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) + pr.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) @@ -1473,7 +1478,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c11.makeConstraintFactors() c12.makeConstraintFactors() c13.makeConstraintFactors() - pr1.solve(chooseVEOrBP(Double.PositiveInfinity, 100)) // this should choose VE + pr1.solve(new VEBPStrategy(Double.PositiveInfinity, 100)) // this should choose VE val result1 = multiplyAll(pr1.solution) Universe.createNew() @@ -1510,7 +1515,7 @@ class VEBPChooserTest extends WordSpec with Matchers { c21.makeConstraintFactors() c22.makeConstraintFactors() c23.makeConstraintFactors() - pr2.solve(chooseVEOrBP(Double.NegativeInfinity, 100)) // this should choose BP + pr2.solve(new VEBPStrategy(Double.NegativeInfinity, 100)) // this should choose BP val result2 = multiplyAll(pr2.solution) val c13Index2 = c13.variable.range.indexOf(Regular(2)) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/VESolverTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/VESolverTest.scala similarity index 59% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/VESolverTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/VESolverTest.scala index 9ecbc7e1..255ff8ef 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/solver/VESolverTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/solver/VESolverTest.scala @@ -10,18 +10,20 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured.solver +package com.cra.figaro.test.algorithm.structured.solver -import org.scalatest.{WordSpec, Matchers} +import org.scalatest.{ WordSpec, Matchers } import com.cra.figaro.language._ -import com.cra.figaro.experimental.structured._ -import solver._ import com.cra.figaro.library.compound._ -import com.cra.figaro.library.atomic.discrete.Uniform import com.cra.figaro.algorithm.factored.factors.Factor -import com.cra.figaro.experimental.structured.factory.Factory +import com.cra.figaro.algorithm.factored.factors.factory.Factory import com.cra.figaro.algorithm.factored.factors.SumProductSemiring import com.cra.figaro.algorithm.lazyfactored.Regular +import com.cra.figaro.algorithm.structured.strategy.solve._ +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredMPEVE +import com.cra.figaro.library.atomic.discrete.Uniform class VESolverTest extends WordSpec with Matchers { "Running VariableElimination without *" when { @@ -47,17 +49,17 @@ class VESolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c2)) - pr.solved should equal (true) + pr.globals should equal(Set(c2)) + pr.solved should equal(true) val result = multiplyAll(pr.solution) - result.variables should equal (List(c2.variable)) - result.size should equal (2) + result.variables should equal(List(c2.variable)) + result.size should equal(2) val c2IndexT = c2.variable.range.indexOf(Regular(true)) val c2IndexF = c2.variable.range.indexOf(Regular(false)) - result.get(List(c2IndexT)) should be (0.6 +- 0.00000001) - result.get(List(c2IndexF)) should be (0.4 +- 0.00000001) + result.get(List(c2IndexT)) should be(0.6 +- 0.00000001) + result.get(List(c2IndexF)) should be(0.4 +- 0.00000001) } "produce the correct result over multiple elements" in { @@ -80,31 +82,31 @@ class VESolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c2, c3)) + pr.globals should equal(Set(c2, c3)) val result = multiplyAll(pr.solution) - result.variables.size should equal (2) + result.variables.size should equal(2) val c2IndexT = c2.variable.range.indexOf(Regular(true)) val c2IndexF = c2.variable.range.indexOf(Regular(false)) val c3IndexT = c3.variable.range.indexOf(Regular(true)) val c3IndexF = c3.variable.range.indexOf(Regular(false)) - result.size should equal (4) + result.size should equal(4) val var0 = result.variables(0) val var1 = result.variables(1) if (var0 == c2.variable) { - var1 should equal (c3.variable) - result.get(List(c2IndexT, c3IndexT)) should equal (0.6) - result.get(List(c2IndexT, c3IndexF)) should equal (0.0) - result.get(List(c2IndexF, c3IndexT)) should equal (0.0) - result.get(List(c2IndexF, c3IndexF)) should equal (0.4) + var1 should equal(c3.variable) + result.get(List(c2IndexT, c3IndexT)) should equal(0.6) + result.get(List(c2IndexT, c3IndexF)) should equal(0.0) + result.get(List(c2IndexF, c3IndexT)) should equal(0.0) + result.get(List(c2IndexF, c3IndexF)) should equal(0.4) } else { - var0 should equal (c3.variable) - var1 should equal (c2.variable) - result.get(List(c3IndexT, c2IndexT)) should equal (0.6) - result.get(List(c3IndexT, c2IndexF)) should equal (0.0) - result.get(List(c3IndexF, c2IndexT)) should equal (0.0) - result.get(List(c3IndexF, c2IndexF)) should equal (0.4) + var0 should equal(c3.variable) + var1 should equal(c2.variable) + result.get(List(c3IndexT, c2IndexT)) should equal(0.6) + result.get(List(c3IndexT, c2IndexF)) should equal(0.0) + result.get(List(c3IndexF, c2IndexT)) should equal(0.0) + result.get(List(c3IndexF, c2IndexF)) should equal(0.4) } } } @@ -132,19 +134,19 @@ class VESolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c1)) + pr.globals should equal(Set(c1)) val result = multiplyAll(pr.solution) val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) - result.size should equal (4) - result.get(List(c1Index3)) should be ((0.25 * 0.3) +- 0.000000001) - result.get(List(c1Index5)) should be ((0.25 * 0.5) +- 0.000000001) - result.get(List(c1Index7)) should be ((0.25 * 0.7) +- 0.000000001) - result.get(List(c1Index9)) should be ((0.25 * 0.9) +- 0.000000001) + result.size should equal(4) + result.get(List(c1Index3)) should be((0.25 * 0.3) +- 0.000000001) + result.get(List(c1Index5)) should be((0.25 * 0.5) +- 0.000000001) + result.get(List(c1Index7)) should be((0.25 * 0.7) +- 0.000000001) + result.get(List(c1Index9)) should be((0.25 * 0.9) +- 0.000000001) } } @@ -171,19 +173,19 @@ class VESolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c1)) + pr.globals should equal(Set(c1)) val result = multiplyAll(pr.solution) val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) - result.size should equal (4) - result.get(List(c1Index3)) should be ((0.25 * (0.3 * 0.5 + 0.7 * 0.2)) +- 0.000000001) - result.get(List(c1Index5)) should be ((0.25 * (0.5 * 0.5 + 0.5 * 0.2)) +- 0.000000001) - result.get(List(c1Index7)) should be ((0.25 * (0.7 * 0.5 + 0.3 * 0.2)) +- 0.000000001) - result.get(List(c1Index9)) should be ((0.25 * (0.9 * 0.5 + 0.1 * 0.2)) +- 0.000000001) + result.size should equal(4) + result.get(List(c1Index3)) should be((0.25 * (0.3 * 0.5 + 0.7 * 0.2)) +- 0.000000001) + result.get(List(c1Index5)) should be((0.25 * (0.5 * 0.5 + 0.5 * 0.2)) +- 0.000000001) + result.get(List(c1Index7)) should be((0.25 * (0.7 * 0.5 + 0.3 * 0.2)) +- 0.000000001) + result.get(List(c1Index9)) should be((0.25 * (0.9 * 0.5 + 0.1 * 0.2)) +- 0.000000001) } } @@ -211,19 +213,19 @@ class VESolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c1)) + pr.globals should equal(Set(c1)) val result = multiplyAll(pr.solution) val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) - result.size should equal (4) - result.get(List(c1Index3)) should be ((0.25 * (0.3 * 0.5 * 0.4 + 0.7 * 0.2 * 0.1)) +- 0.000000001) - result.get(List(c1Index5)) should be ((0.25 * (0.5 * 0.5 * 0.4 + 0.5 * 0.2 * 0.1)) +- 0.000000001) - result.get(List(c1Index7)) should be ((0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1)) +- 0.000000001) - result.get(List(c1Index9)) should be ((0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1)) +- 0.000000001) + result.size should equal(4) + result.get(List(c1Index3)) should be((0.25 * (0.3 * 0.5 * 0.4 + 0.7 * 0.2 * 0.1)) +- 0.000000001) + result.get(List(c1Index5)) should be((0.25 * (0.5 * 0.5 * 0.4 + 0.5 * 0.2 * 0.1)) +- 0.000000001) + result.get(List(c1Index7)) should be((0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1)) +- 0.000000001) + result.get(List(c1Index9)) should be((0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1)) +- 0.000000001) } } @@ -251,19 +253,19 @@ class VESolverTest extends WordSpec with Matchers { c1.makeConstraintFactors() c2.makeConstraintFactors() c3.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c1)) + pr.globals should equal(Set(c1)) val result = multiplyAll(pr.solution) val c1Index3 = c1.variable.range.indexOf(Regular(0.3)) val c1Index5 = c1.variable.range.indexOf(Regular(0.5)) val c1Index7 = c1.variable.range.indexOf(Regular(0.7)) val c1Index9 = c1.variable.range.indexOf(Regular(0.9)) - result.size should equal (4) - result.get(List(c1Index3)) should be ((0.25 * (0.3 * 0.5 * 0.4 + 0.7 * 0.2 * 0.1)) +- 0.000000001) - result.get(List(c1Index5)) should be ((0.25 * (0.5 * 0.5 * 0.4 + 0.5 * 0.2 * 0.1)) +- 0.000000001) - result.get(List(c1Index7)) should be ((0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1)) +- 0.000000001) - result.get(List(c1Index9)) should be ((0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1)) +- 0.000000001) + result.size should equal(4) + result.get(List(c1Index3)) should be((0.25 * (0.3 * 0.5 * 0.4 + 0.7 * 0.2 * 0.1)) +- 0.000000001) + result.get(List(c1Index5)) should be((0.25 * (0.5 * 0.5 * 0.4 + 0.5 * 0.2 * 0.1)) +- 0.000000001) + result.get(List(c1Index7)) should be((0.25 * (0.7 * 0.5 * 0.4 + 0.3 * 0.2 * 0.1)) +- 0.000000001) + result.get(List(c1Index9)) should be((0.25 * (0.9 * 0.5 * 0.4 + 0.1 * 0.2 * 0.1)) +- 0.000000001) } } @@ -292,15 +294,15 @@ class VESolverTest extends WordSpec with Matchers { c11.makeConstraintFactors() c12.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) - pr.globals should equal (Set(c2)) + pr.globals should equal(Set(c2)) val result = multiplyAll(pr.solution) val c2Index1 = c2.variable.range.indexOf(Regular(ec1)) val c2Index2 = c2.variable.range.indexOf(Regular(ec2)) - result.size should equal (2) - result.get(List(c2Index1)) should be ((0.8 * 0.6) +- 0.000000001) - result.get(List(c2Index2)) should be ((0.2 * 0.3) +- 0.000000001) + result.size should equal(2) + result.get(List(c2Index1)) should be((0.8 * 0.6) +- 0.000000001) + result.get(List(c2Index2)) should be((0.2 * 0.3) +- 0.000000001) } } @@ -320,12 +322,12 @@ class VESolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c1.makeConstraintFactors() c2.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c2IndexT = c2.variable.range.indexOf(Regular(true)) val c2IndexF = c2.variable.range.indexOf(Regular(false)) - result.get(List(c2IndexT)) should be (1.0 +- 0.000000001) - result.get(List(c2IndexF)) should be (0.0 +- 0.000000001) + result.get(List(c2IndexT)) should be(1.0 +- 0.000000001) + result.get(List(c2IndexF)) should be(0.0 +- 0.000000001) } "with a constraint on an element that is used multiple times, only factor in the constraint once" in { @@ -363,7 +365,7 @@ class VESolverTest extends WordSpec with Matchers { ce1.makeConstraintFactors() ce2.makeConstraintFactors() cd.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) // Probability that f1 is true = 0.6 // Probability that e1 is true = 1.0 @@ -374,7 +376,7 @@ class VESolverTest extends WordSpec with Matchers { val dIndexF = cd.variable.range.indexOf(Regular(false)) val pT = result.get(List(dIndexT)) val pF = result.get(List(dIndexF)) - (pT / (pT + pF)) should be (0.73 +- 0.000000001) + (pT / (pT + pF)) should be(0.73 +- 0.000000001) } "with elements that are not used by the query or evidence, produce the correct result" in { @@ -391,6 +393,7 @@ class VESolverTest extends WordSpec with Matchers { val ca = cc(a) cu.generateRange() cf.generateRange() + ca.expand() ca.generateRange() cu.makeNonConstraintFactors() cf.makeNonConstraintFactors() @@ -398,13 +401,13 @@ class VESolverTest extends WordSpec with Matchers { cu.makeConstraintFactors() cf.makeConstraintFactors() ca.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val fIndexT = cf.variable.range.indexOf(Regular(true)) val fIndexF = cf.variable.range.indexOf(Regular(false)) val pT = result.get(List(fIndexT)) val pF = result.get(List(fIndexF)) - (pT / (pT + pF)) should be (0.6 +- 0.000000001) + (pT / (pT + pF)) should be(0.6 +- 0.000000001) } "with a model using chain and no conditions or constraints, when the outcomes are at the top level, produce the correct answer" in { @@ -435,10 +438,10 @@ class VESolverTest extends WordSpec with Matchers { c2.makeConstraintFactors() c3.makeConstraintFactors() c4.makeConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) - result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) + result.get(List(c4Index1)) should be((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) } "with a model using chain and no conditions or constraints, when the outcomes are nested, produce the correct answer" in { @@ -466,17 +469,17 @@ class VESolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) - result.get(List(c4Index1)) should be ((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) + result.get(List(c4Index1)) should be((0.3 * 0.1 + 0.7 * 0.7) +- 0.000000001) } "with a model using chain and a condition on the result, when the outcomes are at the top level, correctly condition the parent" in { Universe.createNew() - val e1= Flip(0.3) + val e1 = Flip(0.3) val e2 = Select(0.1 -> 1, 0.9 -> 2) val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) @@ -504,19 +507,19 @@ class VESolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) val c1IndexF = c1.variable.range.indexOf(Regular(false)) val pT = result.get(List(c1IndexT)) val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be ((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) + (pT / (pT + pF)) should be((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) } "with a model using chain and a condition on the result, when the outcomes are nested, correctly condition the parent" in { Universe.createNew() - val e1= Flip(0.3) + val e1 = Flip(0.3) val e2 = Select(0.1 -> 1, 0.9 -> 2) val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) @@ -541,16 +544,16 @@ class VESolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c1IndexT = c1.variable.range.indexOf(Regular(true)) val c1IndexF = c1.variable.range.indexOf(Regular(false)) val pT = result.get(List(c1IndexT)) val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be ((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) + (pT / (pT + pF)) should be((0.3 * 0.1 / (0.3 * 0.1 + 0.7 * 0.7)) +- 0.000000001) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are at the top level, correctly condition the result" in { @@ -582,7 +585,7 @@ class VESolverTest extends WordSpec with Matchers { c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() c4.makeNonConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -591,48 +594,48 @@ class VESolverTest extends WordSpec with Matchers { val p1 = result.get(List(c4Index1)) val p2 = result.get(List(c4Index2)) val p3 = result.get(List(c4Index3)) - (p1 / (p1 + p2 + p3)) should be ((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) + (p1 / (p1 + p2 + p3)) should be((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are at the top level, " + - "not change the belief about the parent" in { - Universe.createNew() - val e1 = Flip(0.3) - val e2 = Select(0.1 -> 1, 0.9 -> 2) - val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) - val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) - e2.observe(1) - val cc = new ComponentCollection - val pr = new Problem(cc, List(e1)) - pr.add(e2) - pr.add(e3) - pr.add(e4) - val c1 = cc(e1) - val c2 = cc(e2) - val c3 = cc(e3) - val c4 = cc(e4) - c1.generateRange() - c2.generateRange() - c3.generateRange() - c4.expand() - c4.generateRange() - c1.makeConstraintFactors() - c2.makeConstraintFactors() - c3.makeConstraintFactors() - c4.makeConstraintFactors() - c1.makeNonConstraintFactors() - c2.makeNonConstraintFactors() - c3.makeNonConstraintFactors() - c4.makeNonConstraintFactors() - pr.solve(variableElimination) + "not change the belief about the parent" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e2.observe(1) + val cc = new ComponentCollection + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.expand() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalVariableElimination)) - val result = multiplyAll(pr.solution) - val c1IndexT = c1.variable.range.indexOf(Regular(true)) - val c1IndexF = c1.variable.range.indexOf(Regular(false)) - val pT = result.get(List(c1IndexT)) - val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be (0.3 +- 0.000000001) - } + val result = multiplyAll(pr.solution) + val c1IndexT = c1.variable.range.indexOf(Regular(true)) + val c1IndexF = c1.variable.range.indexOf(Regular(false)) + val pT = result.get(List(c1IndexT)) + val pF = result.get(List(c1IndexF)) + (pT / (pT + pF)) should be(0.3 +- 0.000000001) + } "with a model using chain and a condition on one of the outcome elements, when the outcomes are nested, correctly condition the result" in { Universe.createNew() @@ -645,6 +648,8 @@ class VESolverTest extends WordSpec with Matchers { val cc = new ComponentCollection val pr = new Problem(cc, List(e4)) pr.add(e1) + pr.add(e2) + pr.add(e3) val c1 = cc(e1) val c4 = cc(e4) c1.generateRange() @@ -661,9 +666,9 @@ class VESolverTest extends WordSpec with Matchers { c1.makeNonConstraintFactors() c2.makeNonConstraintFactors() c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) c4.makeNonConstraintFactors() - pr.solve(variableElimination) + pr.solve(new ConstantStrategy(marginalVariableElimination)) val result = multiplyAll(pr.solution) val c4Index1 = c4.variable.range.indexOf(Regular(1)) @@ -672,52 +677,183 @@ class VESolverTest extends WordSpec with Matchers { val p1 = result.get(List(c4Index1)) val p2 = result.get(List(c4Index2)) val p3 = result.get(List(c4Index3)) - (p1 / (p1 + p2 + p3)) should be ((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) + (p1 / (p1 + p2 + p3)) should be((0.3 * 1 + 0.7 * 0.7) +- 0.000000001) } "with a model using chain and a condition on one of the outcome elements, when the outcomes are nested, " + - "not change the belief about the parent" in { - Universe.createNew() - val e1 = Flip(0.3) - val e2 = Select(0.1 -> 1, 0.9 -> 2) - val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) - val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) - e2.observe(1) + "not change the belief about the parent" in { + Universe.createNew() + val e1 = Flip(0.3) + val e2 = Select(0.1 -> 1, 0.9 -> 2) + val e3 = Select(0.7 -> 1, 0.2 -> 2, 0.1 -> 3) + val e4 = Chain(e1, (b: Boolean) => if (b) e2; else e3) + e2.observe(1) - val cc = new ComponentCollection - val pr = new Problem(cc, List(e1)) - pr.add(e4) - val c1 = cc(e1) - val c4 = cc(e4) - c1.generateRange() - c4.expand() - val c2 = cc(e2) - val c3 = cc(e3) - c2.generateRange() - c3.generateRange() - c4.generateRange() - c1.makeConstraintFactors() - c2.makeConstraintFactors() - c3.makeConstraintFactors() - c4.makeConstraintFactors() - c1.makeNonConstraintFactors() - c2.makeNonConstraintFactors() - c3.makeNonConstraintFactors() - c4.subproblems.values.foreach(_.solve(variableElimination)) - c4.makeNonConstraintFactors() - pr.solve(variableElimination) + val cc = new ComponentCollection + val pr = new Problem(cc, List(e1)) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c4 = cc(e4) + c1.generateRange() + c4.expand() + val c2 = cc(e2) + val c3 = cc(e3) + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.subproblems.values.foreach(_.solve(new ConstantStrategy(marginalVariableElimination))) + c4.makeNonConstraintFactors() + pr.solve(new ConstantStrategy(marginalVariableElimination)) - val result = multiplyAll(pr.solution) - val c1IndexT = c1.variable.range.indexOf(Regular(true)) - val c1IndexF = c1.variable.range.indexOf(Regular(false)) - val pT = result.get(List(c1IndexT)) - val pF = result.get(List(c1IndexF)) - (pT / (pT + pF)) should be (0.3 +- 0.000000001) + val result = multiplyAll(pr.solution) + val c1IndexT = c1.variable.range.indexOf(Regular(true)) + val c1IndexF = c1.variable.range.indexOf(Regular(false)) + val pT = result.get(List(c1IndexT)) + val pF = result.get(List(c1IndexF)) + (pT / (pT + pF)) should be(0.3 +- 0.000000001) + } + + } + + "Running MPE VariableElimination" when { + "given a target" should { + "produce the most likely factor over the target" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.75 -> 0.2, 0.25 -> 0.3) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + val pr = new Problem(cc, List(e1)) + pr.add(e1) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // p(e1=.2,e2=T,e3=F,e4=F) = 0.75 * 0.2 * 0.8 = .12 + // p(e1=.2,e2=F,e3=T,e4=F) = 0.75 * 0.8 * 0.2 = .12 + // p(e1=.3,e2=T,e3=F,e4=F) = 0.25 * 0.3 * 0.7 = .0525 + // p(e1=.3,e2=F,e3=T,e4=F) = 0.25 * 0.7 * 0.3 = .0525 + // MPE: e1=.2,e2=F,e3=F,e4=T + // If we leave e1 un-eliminated, we should end up with a factor that has e1=.2 at .48 and e1=.3 at .1225 + pr.solve(new ConstantStrategy(mpeVariableElimination)) + val f = pr.solution reduceLeft (_.product(_)) + f.numVars should equal(1) + f.get(List(0)) should be({ if (c1.variable.range(0).value == .2) 0.48 else 0.1225 } +- 0.000000001) + f.get(List(1)) should be({ if (c1.variable.range(1).value == .2) 0.48 else 0.1225 } +- 0.000000001) + } } + "given a flat model" should { + "produce the correct most likely values for all elements with no conditions or constraints" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.75 -> 0.2, 0.25 -> 0.3) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + val pr = new Problem(cc, List()) + pr.add(e1) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // p(e1=.2,e2=T,e3=F,e4=F) = 0.75 * 0.2 * 0.8 = .12 + // p(e1=.2,e2=F,e3=T,e4=F) = 0.75 * 0.8 * 0.2 = .12 + // p(e1=.3,e2=T,e3=F,e4=F) = 0.25 * 0.3 * 0.7 = .0525 + // p(e1=.3,e2=F,e3=T,e4=F) = 0.25 * 0.7 * 0.3 = .0525 + // MPE: e1=.2,e2=F,e3=F,e4=T + pr.solve(new ConstantStrategy(mpeVariableElimination)) + pr.recordingFactors(c1.variable).get(List()).asInstanceOf[Double] should be(0.2 +- .0000001) + pr.recordingFactors(c2.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c3.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c4.variable).get(List()).asInstanceOf[Boolean] should be(true) + } + + "produce the correct most likely values for all elements with conditions and constraints" in { + Universe.createNew() + val cc = new ComponentCollection + val e1 = Select(0.5 -> 0.2, 0.5 -> 0.3) + e1.addConstraint((d: Double) => if (d < 0.25) 3.0 else 1.0) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + e4.observe(true) + val pr = new Problem(cc, List()) + pr.add(e1) + pr.add(e2) + pr.add(e3) + pr.add(e4) + val c1 = cc(e1) + val c2 = cc(e2) + val c3 = cc(e3) + val c4 = cc(e4) + c1.generateRange() + c2.generateRange() + c3.generateRange() + c4.generateRange() + c1.makeConstraintFactors() + c2.makeConstraintFactors() + c3.makeConstraintFactors() + c4.makeConstraintFactors() + c1.makeNonConstraintFactors() + c2.makeNonConstraintFactors() + c3.makeNonConstraintFactors() + c4.makeNonConstraintFactors() + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // MPE: e1=.2,e2=F,e3=F,e4=T + pr.solve(new ConstantStrategy(mpeVariableElimination)) + pr.recordingFactors(c1.variable).get(List()).asInstanceOf[Double] should be(0.2 +- .0000001) + pr.recordingFactors(c2.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c3.variable).get(List()).asInstanceOf[Boolean] should be(false) + pr.recordingFactors(c4.variable).get(List()).asInstanceOf[Boolean] should be(true) + } + } } def multiplyAll(factors: List[Factor[Double]]): Factor[Double] = factors.foldLeft(Factory.unit(SumProductSemiring()))(_.product(_)) - class EC1 extends ElementCollection { } + class EC1 extends ElementCollection {} } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/FlatTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/FlatTest.scala new file mode 100644 index 00000000..c7b4b8c5 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/FlatTest.scala @@ -0,0 +1,199 @@ +/* + * FlatTest.scala + * Test of flat strategies. + * + * Created By: Brian Ruttenberg (bruttenberg@cra.com) + * Creation Date: July 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.test.algorithm.structured.strategy + +import org.scalatest.{WordSpec, Matchers} +import com.cra.figaro.language._ +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.algorithm.structured._ +import com.cra.figaro.algorithm.structured.strategy.decompose._ +import com.cra.figaro.algorithm.structured.algorithm.flat.FlatVE +import com.cra.figaro.algorithm.structured.strategy.solve.ConstantStrategy +import com.cra.figaro.algorithm.structured.strategy.decompose.DecompositionStrategy +import com.cra.figaro.algorithm.structured.solver._ +import com.cra.figaro.language.Element.toBooleanElement + + +class FlatTest extends WordSpec with Matchers { + "Executing a flat strategy" when { + + "expanding the model" should { + "produce the correct factors" in { + Universe.createNew() + val e1 = Flip(0.4) + val r1 = Chain(e1, (b: Boolean) => { + if (b) Chain(Flip(0.2), (b: Boolean) => if (b) Uniform(1,2) else Uniform(3,4)) + else Chain(Flip(0.1), (b: Boolean) => if (b) Uniform(5,6) else Uniform(7,8)) + }) + val cc = new ComponentCollection + val problem = new Problem(cc, List(r1)) + val fs = DecompositionStrategy.recursiveFlattenStrategy(problem, new ConstantStrategy(marginalVariableElimination), defaultRangeSizer, Lower, false) + fs.backwardChain(problem.components , Set()) + val factors =problem.components.flatMap(_.nonConstraintFactors) + factors.foreach(f => println(f.toReadableString)) + factors.size should be(16) + FlatVE.probability(r1, 1) should equal (0.5*0.2*.4 +- 0.000001) + } + } + + "given a flat model with a compound Flip without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + FlatVE.probability(e3, true) should equal (0.6) + } + } + + "given a flat model with evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + FlatVE.probability(e1, 0.3) should be (0.125 +- 0.000000001) + } + } + + "given a model with multiple targets and no evidence" should { + "produce the correct probability over both targets" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + val alg = FlatVE(e2, e3) + alg.start() + alg.probability(e2, true) should equal (0.6) + alg.probability(e3, true) should equal (0.6) + } + } + + "given a model with multiple targets with evidence" should { + "produce the correct probability over both targets" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + val alg = FlatVE(e2, e1) + alg.start() + alg.probability(e2, true) should equal (1.0) + alg.probability(e1, 0.3) should be (0.125 +- 0.000000001) + } + } + + "given a one-level nested model without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, Constant(true), Constant(false)) + val alg = FlatVE(e3) + alg.start() + alg.probability(e3, true) should equal (0.6) + } + } + + /* + "given a one-level nested model with nested evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, { val e = Flip(0.5); e.observe(true); e }, Constant(false)) + val alg = FlatVE(e3) + alg.start() + alg.probability(e3, true) should equal (0.6) + } + } + * + */ + + "given a two-level nested model" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, If(Flip(0.9), Constant(true), Constant(false)), Constant(false)) + val alg = FlatVE(e3) + alg.start() + alg.probability(e3, true) should be ((0.6 * 0.9) +- 0.000000001) + } + } + + "expanding an element with two different arguments" should { + "expand both the arguments" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = Flip(0.3) + val e3 = e1 && e2 + FlatVE.probability(e3, true) should be (0.12 +- 0.000000001) + } + } + + "expanding an argument that is used more than once" should { + "only expand the argument once" in { + var count = 0 + Universe.createNew() + val e1 = Apply(Constant(true), (b: Boolean) => { count += 1; 5 }) + val e2 = e1 === e1 + FlatVE.probability(e2, true) should equal (1.0) + count should equal (1) + // Note that this should now only expand once since Apply Maps have been added to Components + } + } + + "expanding an argument that needs another argument later expanded" should { + "create values for the ancestor argument first" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = If(e1, Constant(1), Constant(2)) + val e3 = Apply(e2, e1, (i: Int, b: Boolean) => if (b) i + 1 else i + 2) + // e3 is 2 iff e1 is true, because then e2 is 1 + FlatVE.probability(e3, 2) should be (0.4 +- 0.000000001) + } + } + + "solving a problem with a reused nested subproblem" should { + "only process the nested subproblem once" in { + var count = 0 + val f = (p: Boolean) => { + count += 1 + Constant(p) + } + val e1 = Chain(Flip(0.5), f) + val e2 = Chain(Flip(0.4), f) + val e3 = e1 && e2 + FlatVE.probability(e3, true) should be ((0.5 * 0.4) +- 0.000000001) + count should equal (2) // One each for p = true and p = false, but only expanded once + } + } + + "given a problem with unneeded elements in the universe" should { + "not create factors for the unneeded elements" in { + var count = 0 + val e1 = Apply(Constant(1), (i: Int) => { count += 1; 5 }) + val e2 = Flip(0.5) + val alg = FlatVE(e2) + alg.start + alg.probability(e2, true) should equal (0.5) + alg.problem.collection(e1).nonConstraintFactors.isEmpty should be (true) + //count should equal (0) + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredBPTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredBPTest.scala new file mode 100644 index 00000000..bcf67db4 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredBPTest.scala @@ -0,0 +1,257 @@ +/* + * StructuredBPTest.scala + * Test of structure belief propagation algorithm. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.test.algorithm.structured.strategy + +import org.scalatest.{ WordSpec, Matchers } +import com.cra.figaro.language._ +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredBP +import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.language.Element.toBooleanElement +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredMPEBP + +class StructuredBPTest extends WordSpec with Matchers { + "Executing a recursive structured BP solver strategy" when { + "given a flat model with an atomic flip without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e2 = Flip(0.6) + val e3 = Apply(e2, (b: Boolean) => b) + StructuredBP.probability(e3, true) should be(0.6 +- 0.000000001) + } + } + + "given a flat model with a compound Flip without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + StructuredBP.probability(e3, true) should be(0.6 +- 0.000000001) + } + } + + "given a flat model with evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + StructuredBP.probability(e1, 0.3) should be(0.125 +- 0.000000001) + } + } + + "given a model with multiple targets and no evidence" should { + "produce the correct probability over both targets" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + val alg = StructuredBP(100, e2, e3) + alg.start() + alg.probability(e2, true) should be(0.6 +- 0.000000001) + alg.probability(e3, true) should equal(0.6 +- 0.000000001) + } + } + + "given a model with multiple targets with evidence" should { + "produce the correct probability over both targets" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + val alg = StructuredBP(100, e2, e1) + alg.start() + alg.probability(e2, true) should equal(1.0) + alg.probability(e1, 0.3) should be(0.125 +- 0.000000001) + } + } + + "given a one-level nested model without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, Constant(true), Constant(false)) + val alg = StructuredBP(100, e3) + alg.start() + alg.probability(e3, true) should be(0.6 +- 0.000000001) + } + } + + /* + "given a one-level nested model with nested evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, { val e = Flip(0.5); e.observe(true); e }, Constant(false)) + val alg = StructuredBP(100, e3) + alg.start() + alg.probability(e3, true) should be(0.6 +- 0.000000001) + } + } + * + */ + + "given a two-level nested model" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, If(Flip(0.9), Constant(true), Constant(false)), Constant(false)) + val alg = StructuredBP(100, e3) + alg.start() + alg.probability(e3, true) should be((0.6 * 0.9) +- 0.000000001) + } + } + + "expanding an element with two different arguments" should { + "expand both the arguments" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = Flip(0.3) + val e3 = e1 && e2 + StructuredBP.probability(e3, true) should be(0.12 +- 0.000000001) + } + } + + "expanding an argument that is used more than once" should { + "only expand the argument once" in { + var count = 0 + Universe.createNew() + val e1 = Apply(Constant(true), (b: Boolean) => { count += 1; 5 }) + val e2 = e1 === e1 + StructuredBP.probability(e2, true) should equal(1.0) + count should equal(1) + // Note that this should now only expand once since Apply Maps have been added to Components + } + } + + // The below test is loopy so BP's answer can't be predicted easily + // "expanding an argument that needs another argument later expanded" should { + // "create values for the ancestor argument first" in { + // Universe.createNew() + // val e1 = Flip(0.4) + // val e2 = If(e1, Constant(1), Constant(2)) + // val e3 = Apply(e2, e1, (i: Int, b: Boolean) => if (b) i + 1 else i + 2) + // // e3 is 2 iff e1 is true, because then e2 is 1 + // StructuredBP.probability(e3, 2) should be (0.4 +- 0.000000001) + // } + // } + + "solving a problem with a reused nested subproblem" should { + "only process the nested subproblem once" in { + var count = 0 + val f = (p: Boolean) => { + count += 1 + Constant(p) + } + val e1 = Chain(Flip(0.5), f) + val e2 = Chain(Flip(0.4), f) + val e3 = e1 && e2 + StructuredBP.probability(e3, true) should be((0.5 * 0.4) +- 0.000000001) + count should equal(2) // One each for p = true and p = false, but only expanded once + } + } + + "given a problem with unneeded elements in the universe" should { + "not process the unneeded elements" in { + var count = 0 + val e1 = Apply(Constant(1), (i: Int) => { count += 1; 5 }) + val e2 = Flip(0.5) + StructuredBP.probability(e2, true) should equal(0.5) + count should equal(0) + } + } + } + + "MPE BP" when { + "given a flat model without evidence should produce the right answer" in { + Universe.createNew() + val e1 = Select(0.75 -> 0.2, 0.25 -> 0.3) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + val alg = StructuredMPEBP(20) + alg.start + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // p(e1=.2,e2=T,e3=F,e4=F) = 0.75 * 0.2 * 0.8 = .12 + // p(e1=.2,e2=F,e3=T,e4=F) = 0.75 * 0.8 * 0.2 = .12 + // p(e1=.3,e2=T,e3=F,e4=F) = 0.25 * 0.3 * 0.7 = .0525 + // p(e1=.3,e2=F,e3=T,e4=F) = 0.25 * 0.7 * 0.3 = .0525 + // MPE: e1=.2,e2=F,e3=F,e4=T + alg.mostLikelyValue(e1) should be(0.2 +- 0.0000001) + alg.mostLikelyValue(e2) should equal(false) + alg.mostLikelyValue(e3) should equal(false) + alg.mostLikelyValue(e4) should equal(true) + alg.kill + } + + "given a flat model with evidence should produce the right answer" in { + Universe.createNew() + val e1 = Select(0.5 -> 0.2, 0.5 -> 0.3) + e1.addConstraint((d: Double) => if (d < 0.25) 3.0 else 1.0) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + e4.observe(true) + val alg = StructuredMPEBP(20) + alg.start + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // MPE: e1=.2,e2=F,e3=F,e4=T + alg.mostLikelyValue(e1) should be(0.2 +- 0.0000001) + alg.mostLikelyValue(e2) should equal(false) + alg.mostLikelyValue(e3) should equal(false) + alg.mostLikelyValue(e4) should equal(true) + alg.kill + } + + "given a structured model with evidence should produce the right answer" in { + Universe.createNew() + val e1 = Flip(0.5) + e1.setConstraint((b: Boolean) => if (b) 3.0; else 1.0) + val e2 = Chain(e1, (b: Boolean) => { + if (b) Flip(0.4) || Flip(0.2) + else Flip(0.9) || Flip(0.2) + }) + val e3 = If(e1, Flip(0.52), Flip(0.4)) + val e4 = e2 === e3 + e4.observe(true) + // p(e1=T,e2=T,f1=T,f2=T,e3=T) = 0.75 * 0.4 * 0.2 * 0.52 = .0312 + // p(e1=T,e2=T,f1=T,f2=F,e3=T) = 0.75 * 0.4 * 0.8 * 0.52 = .1248 + // p(e1=T,e2=T,f1=F,f2=T,e3=T) = 0.75 * 0.6 * 0.2 * 0.52 = .0468 + // p(e1=T,e2=F,f1=F,f2=F,e3=F) = 0.75 * 0.6 * 0.8 * 0.48 = .1728 + // p(e1=F,e2=T,f1=T,f2=T,e3=T) = 0.25 * 0.9 * 0.2 * 0.4 = .018 + // p(e1=F,e2=T,f1=T,f2=F,e3=T) = 0.25 * 0.9 * 0.8 * 0.4 = .072 + // p(e1=F,e2=T,f1=F,f2=T,e3=T) = 0.25 * 0.1 * 0.2 * 0.4 = .002 + // p(e1=F,e2=F,f1=F,f2=F,e3=F) = 0.25 * 0.1 * 0.8 * 0.6 = .012 + // MPE: e1=T,e2=F,e3=F,e4=T + val alg = StructuredMPEBP(20) + alg.start + alg.mostLikelyValue(e1) should equal(true) + alg.mostLikelyValue(e2) should equal(false) + alg.mostLikelyValue(e3) should equal(false) + alg.mostLikelyValue(e4) should equal(true) + alg.kill + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredBPTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredGibbsTest.scala similarity index 64% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredBPTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredGibbsTest.scala index bbfaf395..04541517 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredBPTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredGibbsTest.scala @@ -1,32 +1,31 @@ /* - * StructuredBPTest.scala - * Test of structure belief propagation algorithm. + * StructuredGibbsTest.scala + * Test of structured Gibbs sampling algorithm. * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 25, 2015 * * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured.strategy +package com.cra.figaro.test.algorithm.structured.strategy import org.scalatest.{WordSpec, Matchers} import com.cra.figaro.language._ -import com.cra.figaro.library.compound.If -import com.cra.figaro.experimental.structured.algorithm.StructuredBP -import com.cra.figaro.algorithm.ValuesMaker +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredGibbs import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.language.Element.toBooleanElement -class StructuredBPTest extends WordSpec with Matchers { +class StructuredGibbsTest extends WordSpec with Matchers { "Executing a recursive structured BP solver strategy" when { "given a flat model with an atomic flip without evidence" should { "produce the correct answer" in { Universe.createNew() val e2 = Flip(0.6) val e3 = Apply(e2, (b: Boolean) => b) - StructuredBP.probability(e3, true) should be (0.6 +- 0.000000001) + StructuredGibbs.probability(e3, true) should be (0.6 +- tol) } } @@ -36,7 +35,7 @@ class StructuredBPTest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) - StructuredBP.probability(e3, true) should be (0.6 +- 0.000000001) + StructuredGibbs.probability(e3, true) should be (0.6 +- tol) } } @@ -47,7 +46,7 @@ class StructuredBPTest extends WordSpec with Matchers { val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) e3.observe(true) - StructuredBP.probability(e1, 0.3) should be (0.125 +- 0.000000001) + StructuredGibbs.probability(e1, 0.3) should be (0.125 +- tol) } } @@ -57,10 +56,10 @@ class StructuredBPTest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) - val alg = StructuredBP(100, e2, e3) + val alg = StructuredGibbs(10000, e2, e3) alg.start() - alg.probability(e2, true) should be (0.6 +- 0.000000001) - alg.probability(e3, true) should equal (0.6 +- 0.000000001) + alg.probability(e2, true) should be (0.6 +- tol) + alg.probability(e3, true) should equal (0.6 +- tol) } } @@ -71,22 +70,23 @@ class StructuredBPTest extends WordSpec with Matchers { val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) e3.observe(true) - val alg = StructuredBP(100, e2, e1) + val alg = StructuredGibbs(10000, e2, e1) alg.start() alg.probability(e2, true) should equal (1.0) - alg.probability(e1, 0.3) should be (0.125 +- 0.000000001) + alg.probability(e1, 0.3) should be (0.125 +- tol) } } - "given a one-level nested model without evidence" should { + // These tests don't work because of blocking problems. The Chain results have disjoint ranges. + /*"given a one-level nested model without evidence" should { "produce the correct answer" in { Universe.createNew() val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = If(e2, Constant(true), Constant(false)) - val alg = StructuredBP(100, e3) + val alg = StructuredGibbs(10000, e3) alg.start() - alg.probability(e3, true) should be (0.6 +- 0.000000001) + alg.probability(e3, true) should be (0.6 +- tol) } } @@ -96,9 +96,9 @@ class StructuredBPTest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = If(e2, { val e = Flip(0.5); e.observe(true); e }, Constant(false)) - val alg = StructuredBP(100, e3) + val alg = StructuredGibbs(10000, e3) alg.start() - alg.probability(e3, true) should be (0.6 +- 0.000000001) + alg.probability(e3, true) should be (0.6 +- tol) } } @@ -108,11 +108,11 @@ class StructuredBPTest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = If(e2, If(Flip(0.9), Constant(true), Constant(false)), Constant(false)) - val alg = StructuredBP(100, e3) + val alg = StructuredGibbs(10000, e3) alg.start() - alg.probability(e3, true) should be ((0.6 * 0.9) +- 0.000000001) + alg.probability(e3, true) should be ((0.6 * 0.9) +- tol) } - } + }*/ "expanding an element with two different arguments" should { "expand both the arguments" in { @@ -120,7 +120,7 @@ class StructuredBPTest extends WordSpec with Matchers { val e1 = Flip(0.4) val e2 = Flip(0.3) val e3 = e1 && e2 - StructuredBP.probability(e3, true) should be (0.12 +- 0.000000001) + StructuredGibbs.probability(e3, true) should be (0.12 +- tol) } } @@ -130,22 +130,23 @@ class StructuredBPTest extends WordSpec with Matchers { Universe.createNew() val e1 = Apply(Constant(true), (b: Boolean) => { count += 1; 5 }) val e2 = e1 === e1 - StructuredBP.probability(e2, true) should equal (1.0) - count should equal (2) // One for generating the range and one for creating the non-constraint factor. Both require applying the function. + StructuredGibbs.probability(e2, true) should equal (1.0) + count should equal (1) + // Note that this should now only expand once since Apply Maps have been added to Components } } - // The below test is loopy so BP's answer can't be predicted easily -// "expanding an argument that needs another argument later expanded" should { -// "create values for the ancestor argument first" in { -// Universe.createNew() -// val e1 = Flip(0.4) -// val e2 = If(e1, Constant(1), Constant(2)) -// val e3 = Apply(e2, e1, (i: Int, b: Boolean) => if (b) i + 1 else i + 2) -// // e3 is 2 iff e1 is true, because then e2 is 1 -// StructuredBP.probability(e3, 2) should be (0.4 +- 0.000000001) -// } -// } + // These ones too + /*"expanding an argument that needs another argument later expanded" should { + "create values for the ancestor argument first" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = If(e1, Constant(1), Constant(2)) + val e3 = Apply(e2, e1, (i: Int, b: Boolean) => if (b) i + 1 else i + 2) + // e3 is 2 iff e1 is true, because then e2 is 1 + StructuredGibbs.probability(e3, 2) should be (0.4 +- tol) + } + } "solving a problem with a reused nested subproblem" should { "only process the nested subproblem once" in { @@ -157,19 +158,21 @@ class StructuredBPTest extends WordSpec with Matchers { val e1 = Chain(Flip(0.5), f) val e2 = Chain(Flip(0.4), f) val e3 = e1 && e2 - StructuredBP.probability(e3, true) should be ((0.5 * 0.4) +- 0.000000001) + StructuredGibbs.probability(e3, true) should be ((0.5 * 0.4) +- tol) count should equal (2) // One each for p = true and p = false, but only expanded once } - } + }*/ "given a problem with unneeded elements in the universe" should { "not process the unneeded elements" in { var count = 0 val e1 = Apply(Constant(1), (i: Int) => { count += 1; 5 }) val e2 = Flip(0.5) - StructuredBP.probability(e2, true) should equal (0.5) + StructuredGibbs.probability(e2, true) should be (0.5 +- tol) count should equal (0) } } } + + val tol = 0.025 } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredVEBPChooserTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVEBPChooserTest.scala similarity index 94% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredVEBPChooserTest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVEBPChooserTest.scala index bf56016e..81dbfc6e 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredVEBPChooserTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVEBPChooserTest.scala @@ -10,14 +10,14 @@ * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured.strategy +package com.cra.figaro.test.algorithm.structured.strategy import org.scalatest.{WordSpec, Matchers} import com.cra.figaro.language._ import com.cra.figaro.library.compound.If -import com.cra.figaro.experimental.structured.algorithm.StructuredVEBPChooser -import com.cra.figaro.algorithm.ValuesMaker +import com.cra.figaro.algorithm.structured.algorithm.hybrid.StructuredVEBPChooser import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.language.Element.toBooleanElement class StructuredVEBPChooserTest extends WordSpec with Matchers { "Executing a recursive structured VE solver strategy" when { @@ -90,6 +90,7 @@ class StructuredVEBPChooserTest extends WordSpec with Matchers { } } + /* "given a one-level nested model with nested evidence" should { "produce the correct answer" in { Universe.createNew() @@ -101,6 +102,8 @@ class StructuredVEBPChooserTest extends WordSpec with Matchers { alg.probability(e3, true) should equal (0.6) } } + * + */ "given a two-level nested model" should { "produce the correct answer" in { @@ -131,7 +134,8 @@ class StructuredVEBPChooserTest extends WordSpec with Matchers { val e1 = Apply(Constant(true), (b: Boolean) => { count += 1; 5 }) val e2 = e1 === e1 StructuredVEBPChooser.probability(e2, true) should equal (1.0) - count should equal (2) // One for generating the range and one for creating the non-constraint factor. Both require applying the function. + count should equal (1) + // Note that this should now only expand once since Apply Maps have been added to Components } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredVETest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVEGibbsChooserTest.scala similarity index 70% rename from Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredVETest.scala rename to Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVEGibbsChooserTest.scala index bc6ef34c..6faf8454 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/structured/strategy/StructuredVETest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVEGibbsChooserTest.scala @@ -1,32 +1,32 @@ /* - * StructuredVETest.scala - * Test of structured variable elimination algorithm. + * StructuredVEGibbsChooserTest.scala + * Test of structured Gibbs sampling/VE chooser algorithm. * - * Created By: Avi Pfeffer (apfeffer@cra.com) - * Creation Date: March 1, 2015 + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 25, 2015 * * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ -package com.cra.figaro.test.experimental.structured.strategy +package com.cra.figaro.test.algorithm.structured.strategy import org.scalatest.{WordSpec, Matchers} import com.cra.figaro.language._ import com.cra.figaro.library.compound.If -import com.cra.figaro.experimental.structured.algorithm.StructuredVE -import com.cra.figaro.algorithm.ValuesMaker +import com.cra.figaro.algorithm.structured.algorithm.hybrid.StructuredVEGibbsChooser import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.language.Element.toBooleanElement -class StructuredVETest extends WordSpec with Matchers { - "Executing a recursive structured VE solver strategy" when { +class StructuredVEGibbsChooserTest extends WordSpec with Matchers { + "Executing a recursive structured BP solver strategy" when { "given a flat model with an atomic flip without evidence" should { "produce the correct answer" in { Universe.createNew() val e2 = Flip(0.6) val e3 = Apply(e2, (b: Boolean) => b) - StructuredVE.probability(e3, true) should equal (0.6) + StructuredVEGibbsChooser.probability(e3, true) should be (0.6 +- tol) } } @@ -36,7 +36,7 @@ class StructuredVETest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) - StructuredVE.probability(e3, true) should equal (0.6) + StructuredVEGibbsChooser.probability(e3, true) should be (0.6 +- tol) } } @@ -47,7 +47,7 @@ class StructuredVETest extends WordSpec with Matchers { val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) e3.observe(true) - StructuredVE.probability(e1, 0.3) should be (0.125 +- 0.000000001) + StructuredVEGibbsChooser.probability(e1, 0.3) should be (0.125 +- tol) } } @@ -57,10 +57,10 @@ class StructuredVETest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) - val alg = StructuredVE(e2, e3) + val alg = StructuredVEGibbsChooser(100, 10000, e2, e3) alg.start() - alg.probability(e2, true) should equal (0.6) - alg.probability(e3, true) should equal (0.6) + alg.probability(e2, true) should be (0.6 +- tol) + alg.probability(e3, true) should equal (0.6 +- tol) } } @@ -71,10 +71,10 @@ class StructuredVETest extends WordSpec with Matchers { val e2 = Flip(e1) val e3 = Apply(e2, (b: Boolean) => b) e3.observe(true) - val alg = StructuredVE(e2, e1) + val alg = StructuredVEGibbsChooser(100, 10000, e2, e1) alg.start() alg.probability(e2, true) should equal (1.0) - alg.probability(e1, 0.3) should be (0.125 +- 0.000000001) + alg.probability(e1, 0.3) should be (0.125 +- tol) } } @@ -84,23 +84,26 @@ class StructuredVETest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = If(e2, Constant(true), Constant(false)) - val alg = StructuredVE(e3) + val alg = StructuredVEGibbsChooser(100, 10000, e3) alg.start() - alg.probability(e3, true) should equal (0.6) + alg.probability(e3, true) should be (0.6 +- tol) } } + /* "given a one-level nested model with nested evidence" should { "produce the correct answer" in { Universe.createNew() val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = If(e2, { val e = Flip(0.5); e.observe(true); e }, Constant(false)) - val alg = StructuredVE(e3) + val alg = StructuredVEGibbsChooser(100, 10000, e3) alg.start() - alg.probability(e3, true) should equal (0.6) + alg.probability(e3, true) should be (0.6 +- tol) } } + * + */ "given a two-level nested model" should { "produce the correct answer" in { @@ -108,9 +111,9 @@ class StructuredVETest extends WordSpec with Matchers { val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) val e2 = Flip(e1) val e3 = If(e2, If(Flip(0.9), Constant(true), Constant(false)), Constant(false)) - val alg = StructuredVE(e3) + val alg = StructuredVEGibbsChooser(100, 10000, e3) alg.start() - alg.probability(e3, true) should be ((0.6 * 0.9) +- 0.000000001) + alg.probability(e3, true) should be ((0.6 * 0.9) +- tol) } } @@ -120,7 +123,7 @@ class StructuredVETest extends WordSpec with Matchers { val e1 = Flip(0.4) val e2 = Flip(0.3) val e3 = e1 && e2 - StructuredVE.probability(e3, true) should be (0.12 +- 0.000000001) + StructuredVEGibbsChooser.probability(e3, true) should be (0.12 +- tol) } } @@ -130,8 +133,9 @@ class StructuredVETest extends WordSpec with Matchers { Universe.createNew() val e1 = Apply(Constant(true), (b: Boolean) => { count += 1; 5 }) val e2 = e1 === e1 - StructuredVE.probability(e2, true) should equal (1.0) - count should equal (2) // One for generating the range and one for creating the non-constraint factor. Both require applying the function. + StructuredVEGibbsChooser.probability(e2, true) should equal (1.0) + count should equal (1) + // Note that this should now only expand once since Apply Maps have been added to Components } } @@ -142,7 +146,7 @@ class StructuredVETest extends WordSpec with Matchers { val e2 = If(e1, Constant(1), Constant(2)) val e3 = Apply(e2, e1, (i: Int, b: Boolean) => if (b) i + 1 else i + 2) // e3 is 2 iff e1 is true, because then e2 is 1 - StructuredVE.probability(e3, 2) should be (0.4 +- 0.000000001) + StructuredVEGibbsChooser.probability(e3, 2) should be (0.4 +- tol) } } @@ -156,7 +160,7 @@ class StructuredVETest extends WordSpec with Matchers { val e1 = Chain(Flip(0.5), f) val e2 = Chain(Flip(0.4), f) val e3 = e1 && e2 - StructuredVE.probability(e3, true) should be ((0.5 * 0.4) +- 0.000000001) + StructuredVEGibbsChooser.probability(e3, true) should be ((0.5 * 0.4) +- tol) count should equal (2) // One each for p = true and p = false, but only expanded once } } @@ -166,9 +170,11 @@ class StructuredVETest extends WordSpec with Matchers { var count = 0 val e1 = Apply(Constant(1), (i: Int) => { count += 1; 5 }) val e2 = Flip(0.5) - StructuredVE.probability(e2, true) should equal (0.5) + StructuredVEGibbsChooser.probability(e2, true) should be (0.5 +- tol) count should equal (0) } } } + + val tol = 0.025 } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVETest.scala b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVETest.scala new file mode 100644 index 00000000..8b4ff067 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/algorithm/structured/strategy/StructuredVETest.scala @@ -0,0 +1,270 @@ +/* + * StructuredVETest.scala + * Test of structured variable elimination algorithm. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: March 1, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.test.algorithm.structured.strategy + +import org.scalatest.{Matchers, WordSpec} +import com.cra.figaro.language._ +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredVE +import com.cra.figaro.algorithm.lazyfactored.ValueSet._ +import com.cra.figaro.language.Element.toBooleanElement +import com.cra.figaro.algorithm.structured.algorithm.structured.StructuredMPEVE + +class StructuredVETest extends WordSpec with Matchers { + "Executing a recursive structured VE solver strategy" when { + "given a flat model with an atomic flip without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e2 = Flip(0.6) + val e3 = Apply(e2, (b: Boolean) => b) + StructuredVE.probability(e3, true) should equal(0.6) + } + } + + "given a flat model with a compound Flip without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + StructuredVE.probability(e3, true) should equal(0.6) + } + } + + "given a flat model with evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + StructuredVE.probability(e1, 0.3) should be(0.125 +- 0.000000001) + } + } + + "given a model with multiple targets and no evidence" should { + "produce the correct probability over both targets" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + val alg = StructuredVE(e2, e3) + alg.start() + alg.probability(e2, true) should equal(0.6) + alg.probability(e3, true) should equal(0.6) + } + } + + "given a model with multiple targets with evidence" should { + "produce the correct probability over both targets" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = Apply(e2, (b: Boolean) => b) + e3.observe(true) + val alg = StructuredVE(e2, e1) + alg.start() + alg.probability(e2, true) should equal(1.0) + alg.probability(e1, 0.3) should be(0.125 +- 0.000000001) + } + } + + "given a one-level nested model without evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, Constant(true), Constant(false)) + val alg = StructuredVE(e3) + alg.start() + alg.probability(e3, true) should equal(0.6) + } + } + + /* + "given a one-level nested model with nested evidence" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, { val e = Flip(0.5); e.observe(true); e }, Constant(false)) + val alg = StructuredVE(e3) + alg.start() + alg.probability(e3, true) should equal(0.6) + } + } + * + */ + + "given a two-level nested model" should { + "produce the correct answer" in { + Universe.createNew() + val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) + val e2 = Flip(e1) + val e3 = If(e2, If(Flip(0.9), Constant(true), Constant(false)), Constant(false)) + val alg = StructuredVE(e3) + alg.start() + alg.probability(e3, true) should be((0.6 * 0.9) +- 0.000000001) + } + } + + "expanding an element with two different arguments" should { + "expand both the arguments" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = Flip(0.3) + val e3 = e1 && e2 + StructuredVE.probability(e3, true) should be(0.12 +- 0.000000001) + } + } + + "expanding an argument that is used more than once" should { + "only expand the argument once" in { + var count = 0 + Universe.createNew() + val e1 = Apply(Constant(true), (b: Boolean) => { count += 1; 5 }) + val e2 = e1 === e1 + StructuredVE.probability(e2, true) should equal(1.0) + count should equal(1) + // Note that this should now only expand once since Apply Maps have been added to Components + } + } + + "expanding an argument that needs another argument later expanded" should { + "create values for the ancestor argument first" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = If(e1, Constant(1), Constant(2)) + val e3 = Apply(e2, e1, (i: Int, b: Boolean) => if (b) i + 1 else i + 2) + // e3 is 2 iff e1 is true, because then e2 is 1 + StructuredVE.probability(e3, 2) should be(0.4 +- 0.000000001) + } + } + + "solving a problem with a reused nested subproblem" should { + "only process the nested subproblem once" in { + var count = 0 + val f = (p: Boolean) => { + count += 1 + Constant(p) + } + val e1 = Chain(Flip(0.5), f) + val e2 = Chain(Flip(0.4), f) + val e3 = e1 && e2 + StructuredVE.probability(e3, true) should be((0.5 * 0.4) +- 0.000000001) + count should equal(2) // One each for p = true and p = false, but only expanded once + } + } + + "given a problem with unneeded elements in the universe" should { + "not process the unneeded elements" in { + var count = 0 + val e1 = Apply(Constant(1), (i: Int) => { count += 1; 5 }) + val e2 = Flip(0.5) + StructuredVE.probability(e2, true) should equal(0.5) + count should equal(0) + } + } + } + + "MPE VE" when { + "given a disconnected model should produce the right answer" in { + Universe.createNew() + val e1 = Flip(0.4) + val e2 = Flip(0.6) + val alg = StructuredMPEVE() + alg.start + alg.mostLikelyValue(e1) should equal(false) + alg.mostLikelyValue(e2) should equal(true) + alg.kill + } + + "given a flat model without evidence should produce the right answer" in { + Universe.createNew() + val e1 = Select(0.75 -> 0.2, 0.25 -> 0.3) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + val alg = StructuredMPEVE() + alg.start + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // p(e1=.2,e2=T,e3=F,e4=F) = 0.75 * 0.2 * 0.8 = .12 + // p(e1=.2,e2=F,e3=T,e4=F) = 0.75 * 0.8 * 0.2 = .12 + // p(e1=.3,e2=T,e3=F,e4=F) = 0.25 * 0.3 * 0.7 = .0525 + // p(e1=.3,e2=F,e3=T,e4=F) = 0.25 * 0.7 * 0.3 = .0525 + // MPE: e1=.2,e2=F,e3=F,e4=T + alg.mostLikelyValue(e1) should be(0.2 +- 0.0000001) + alg.mostLikelyValue(e2) should equal(false) + alg.mostLikelyValue(e3) should equal(false) + alg.mostLikelyValue(e4) should equal(true) + alg.kill + } + + "given a flat model with evidence should produce the right answer" in { + Universe.createNew() + val e1 = Select(0.5 -> 0.2, 0.5 -> 0.3) + e1.addConstraint((d: Double) => if (d < 0.25) 3.0 else 1.0) + val e2 = Flip(e1) + val e3 = Flip(e1) + val e4 = e2 === e3 + e4.observe(true) + val alg = StructuredMPEVE() + alg.cc.useSingleChainFactor = true + alg.start + // p(e1=.2,e2=T,e3=T,e4=T) = 0.75 * 0.2 * 0.2 = .03 + // p(e1=.2,e2=F,e3=F,e4=T) = 0.75 * 0.8 * 0.8 = .48 + // p(e1=.3,e2=T,e3=T,e4=T) = 0.25 * 0.3 * 0.3 = .0225 + // p(e1=.3,e2=F,e3=F,e4=T) = 0.25 * 0.7 * 0.7 = .1225 + // MPE: e1=.2,e2=F,e3=F,e4=T + alg.mostLikelyValue(e1) should be(0.2 +- 0.0000001) + alg.mostLikelyValue(e2) should equal(false) + alg.mostLikelyValue(e3) should equal(false) + alg.mostLikelyValue(e4) should equal(true) + alg.kill + } + + "given a structured model with evidence should produce the right answer" in { + Universe.createNew() + val e1 = Flip(0.5) + e1.setConstraint((b: Boolean) => if (b) 3.0; else 1.0) + val e2 = Chain(e1, (b: Boolean) => { + if (b) Flip(0.4) || Flip(0.2) + else Flip(0.9) || Flip(0.2) + }) + val e3 = If(e1, Flip(0.52), Flip(0.4)) + val e4 = e2 === e3 + e4.observe(true) + // p(e1=T,e2=T,f1=T,f2=T,e3=T) = 0.75 * 0.4 * 0.2 * 0.52 = .0312 + // p(e1=T,e2=T,f1=T,f2=F,e3=T) = 0.75 * 0.4 * 0.8 * 0.52 = .1248 + // p(e1=T,e2=T,f1=F,f2=T,e3=T) = 0.75 * 0.6 * 0.2 * 0.52 = .0468 + // p(e1=T,e2=F,f1=F,f2=F,e3=F) = 0.75 * 0.6 * 0.8 * 0.48 = .1728 + // p(e1=F,e2=T,f1=T,f2=T,e3=T) = 0.25 * 0.9 * 0.2 * 0.4 = .018 + // p(e1=F,e2=T,f1=T,f2=F,e3=T) = 0.25 * 0.9 * 0.8 * 0.4 = .072 + // p(e1=F,e2=T,f1=F,f2=T,e3=T) = 0.25 * 0.1 * 0.2 * 0.4 = .002 + // p(e1=F,e2=F,f1=F,f2=F,e3=F) = 0.25 * 0.1 * 0.8 * 0.6 = .012 + // MPE: e1=T,e2=F,e3=F,e4=T + val alg = StructuredMPEVE() + alg.cc.useSingleChainFactor = true + alg.start + alg.mostLikelyValue(e1) should equal(true) + alg.mostLikelyValue(e2) should equal(false) + alg.mostLikelyValue(e3) should equal(false) + alg.mostLikelyValue(e4) should equal(true) + alg.kill + } + + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/appendixA/Test.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/appendixA/Test.scala new file mode 100644 index 00000000..0f194777 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/appendixA/Test.scala @@ -0,0 +1,41 @@ +/* + * Test.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.appendixA + +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.sampling._ + +object Test { + def main(args: Array[String]) { + val test = Constant("Test") + val algorithm = Importance(1000, test) + algorithm.start() + println(algorithm.probability(test, "Test")) + } +} + +class Test extends WordSpec with Matchers { + Universe.createNew() + "AppendixA Importance value" should { + "equal 1.0" taggedAs (BookExample) in { + val test = Constant("Test") + val algorithm = Importance(1000, test) + algorithm.start() + algorithm.probability(test, "Test") should be(1.0) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap01/HelloWorld.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap01/HelloWorld.scala new file mode 100644 index 00000000..6cea517e --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap01/HelloWorld.scala @@ -0,0 +1,80 @@ +/* + * HelloWorld.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap01 + +import com.cra.figaro.language.Universe +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.language.{Flip, Select} //#A +import com.cra.figaro.library.compound.If //#A +import com.cra.figaro.algorithm.factored.VariableElimination //#A + +object HelloWorldFigaro { + val sunnyToday = Flip(0.2) //#B + val greetingToday = If(sunnyToday, //#C + Select(0.6 -> "Hello, world!", 0.4 -> "Howdy, universe!"), //#C + Select(0.2 -> "Hello, world!", 0.8 -> "Oh no, not again")) //#C + val sunnyTomorrow = If(sunnyToday, Flip(0.8), Flip(0.05)) //#D + val greetingTomorrow = If(sunnyTomorrow, //#E + Select(0.5 -> "Hello, world!", 0.5 -> "Howdy, universe!"), //#E + Select(0.1 -> "Hello, world!", 0.9 -> "Oh no, not again")) //#E + + def predict() : Double = { + val result = VariableElimination.probability(greetingToday, "Hello, world!") // #F + println("Today's greeting is \"Hello, world!\" " + + "with probability " + result + ".") //#G + result + } + + def infer() : Double = { + greetingToday.observe("Hello, world!") //#H + val result = VariableElimination.probability(sunnyToday, true) //#F + println("If today's greeting is \"Hello, world!\", today's " + + "weather is sunny with probability " + result + ".") //#G + result + } + + def learnAndPredict() : Double = { + val result = + VariableElimination.probability(greetingTomorrow, "Hello, world!") //#F + println("If today's greeting is \"Hello, world!\", " + + "tomorrow's greeting will be \"Hello, world!\" " + + "with probability " + result + ".") //#G + result + } + + def main(args: Array[String]) { //#I + predict() //#I + infer() //#I + learnAndPredict() //#I + } +} + +class HelloWorldTest extends WordSpec with Matchers { + Universe.createNew() + "Chap01 Variable Elimination value" should { + "equal 0.27999999999999997 when predicted" taggedAs (BookExample) in { + HelloWorldFigaro.predict() should be(0.27999999999999997) //#I + } + + "equal 0.4285714285714285 when inferred" taggedAs (BookExample) in { + HelloWorldFigaro.infer() should be(0.4285714285714285) //#I + } + + "equal 0.24857142857142858 when learned and predicted" taggedAs (BookExample) in { + HelloWorldFigaro.learnAndPredict() should be(0.24857142857142858) //#I + } + } +} \ No newline at end of file diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Dictionary.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Dictionary.scala new file mode 100644 index 00000000..f38d7951 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Dictionary.scala @@ -0,0 +1,89 @@ +/* + * Dictionary.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap03 + +import com.cra.figaro.language.Universe +import scala.collection.mutable.Map +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +class Dictionary(initNumEmails: Int) { + val counts: Map[String, Int] = Map() + var numEmails = initNumEmails + + def addWord(word: String) { + counts += word -> (getCount(word) + 1) + } + + def addEmail(email: Email) { + numEmails += 1 + for { word <- email.allWords } { + addWord(word) + } + } + + object OrderByCount extends Ordering[String] { + def compare(a: String, b: String) = getCount(b) - getCount(a) + } + def words = counts.keySet.toList.sorted(OrderByCount) + + def nonStopWords = words.dropWhile(counts(_) >= numEmails * Dictionary.stopWordFraction) + def featureWords = nonStopWords.take(Dictionary.numFeatures) + + def getCount(word: String) = + counts.getOrElse(word, 0) + + def isUnusual(word: String, learning: Boolean) = + if (learning) getCount(word) <= 1 + else getCount(word) <= 0 +} + +object Dictionary { + def fromEmails(emails: Traversable[Email]) = { + val result = new Dictionary(0) + for { email <- emails } { result.addEmail(email) } + result + } + + val stopWordFraction = 0.15 + val numFeatures = 100 + + def main(args: Array[String]) = { + val emails = LearningComponent.readEmails("src/test/resources/BookData/Test") + val dict = Dictionary.fromEmails(emails.map(_._2)) + println("Total number of words: " + dict.words.length) + println("Number of feature words: " + dict.featureWords.length) + println("\nAll words and counts:\n") + println(dict.words.map(word => word + " " + dict.getCount(word)).mkString("\n")) + println("\nFeature words and counts:\n") + println("Feature words:\n") + println(dict.featureWords.map(word => word + " " + dict.getCount(word)).mkString("\n")) + } +} + +class DictionaryTest extends WordSpec with Matchers { + Universe.createNew() + val emails = LearningComponent.readEmails("src/test/resources/BookData/Test") + val dict = Dictionary.fromEmails(emails.map(_._2)) + + "Dictionary" should { + "have a total number of words equal to 12922" taggedAs (BookExample) in { + dict.words.length should be(12922) + } + "have a number of feature words equal to 100" taggedAs (BookExample) in { + dict.featureWords.length should be(100) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Email.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Email.scala new file mode 100644 index 00000000..6abbeb7c --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Email.scala @@ -0,0 +1,82 @@ +/* + * Email.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap03 + +import com.cra.figaro.language.Universe +import java.io.File +import scala.io.Source +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +class Email(file: File) { + def getAllWords() = { + def getWords(line: String): List[String] = { + for { + rawWord <- line.split(Array(' ', '\t', '\n')).toList + word = rawWord.filter((c: Char) => c.isLetterOrDigit) + if !word.isEmpty + } yield word.toLowerCase() + } + + val source = Source.fromFile(file)("ISO-8859-1") + val allLines = source.getLines().toList + + val allWordsWithRepeats = + for { + line <- allLines + word <- getWords(line) + } yield word + + allWordsWithRepeats.toSet + } + + val allWords: Set[String] = getAllWords() + + def observeEvidence(model: Model, label: Option[Boolean], learning: Boolean) { + label match { + case Some(b) => model.isSpam.observe(b) + case None => () + } + + for { + (word, element) <- model.hasWordElements + } { + element.observe(allWords.contains(word)) + } + + val obsNumUnusualWords = + allWords.filter((word: String) => model.dictionary.isUnusual(word, learning)).size + val unusualWordFraction = obsNumUnusualWords * Model.binomialNumTrials / allWords.size + model.numUnusualWords.observe(unusualWordFraction) + } +} + +object Email { + def main(args: Array[String]) { + val email = new Email(new File("src/test/resources/BookData/Test/TestEmail_9.txt")) + println(email.allWords) + } +} + +class EmailTest extends WordSpec with Matchers { + Universe.createNew() + val email = new Email(new File("src/test/resources/BookData/Test/TestEmail_9.txt")) + + "Email" should { + "have a total number of words equal to 358" taggedAs (BookExample) in { + email.allWords.size should be(358) + } + } +} \ No newline at end of file diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Evaluator.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Evaluator.scala new file mode 100644 index 00000000..23836adf --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Evaluator.scala @@ -0,0 +1,121 @@ +/* + * Evaluator.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap03 + +import com.cra.figaro.language.Universe +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object Evaluator { + def main(args: Array[String]) = { + val testDirectoryName = "src/test/resources/BookData/Test" + val labelFileName = "src/test/resources/BookData/Labels.txt" + val kbFileName = "src/test/resources/BookData/LearnedModel.txt" + val threshold = 0.5 + + val emails = LearningComponent.readEmails(testDirectoryName) + val labels = LearningComponent.readLabels(labelFileName) + val (dictionary, learningResults) = ReasoningComponent.loadResults(kbFileName) + + var truePositives = 0 + var falseNegatives = 0 + var falsePositives = 0 + var trueNegatives = 0 + + for { (fileName, email) <- emails } { + println(fileName) + Universe.createNew() + val isSpamProbability = ReasoningComponent.classify(dictionary, learningResults, testDirectoryName + "/" + fileName) + val prediction = isSpamProbability >= threshold + (labels.get(fileName), prediction) match { + case (Some(true), true) => truePositives += 1 + case (Some(true), false) => falseNegatives += 1 + case (Some(false), true) => falsePositives += 1 + case (Some(false), false) => trueNegatives += 1 + case _ => () + } + } + + val accuracy = (truePositives + trueNegatives).toDouble / (truePositives + falseNegatives + falsePositives + trueNegatives) + val precision = truePositives.toDouble / (truePositives + falsePositives) + val recall = truePositives.toDouble / (truePositives + falseNegatives) + + println("True positives: " + truePositives) + println("False negatives: " + falseNegatives) + println("False positives: " + falsePositives) + println("True negatives: " + trueNegatives) + println("Threshold: " + threshold) + println("Accuracy: " + accuracy) + println("Precision: " + precision) + println("Recall: " + recall) + } +} + +class EvaluatorTest extends WordSpec with Matchers { + Universe.createNew() + val testDirectoryName = "src/test/resources/BookData/Test" + val labelFileName = "src/test/resources/BookData/Labels.txt" + val kbFileName = "src/test/resources/BookData/LearnedModel.txt" + val threshold = 0.5 + + val emails = LearningComponent.readEmails(testDirectoryName) + val labels = LearningComponent.readLabels(labelFileName) + val (dictionary, learningResults) = ReasoningComponent.loadResults(kbFileName) + + var truePositives = 0 + var falseNegatives = 0 + var falsePositives = 0 + var trueNegatives = 0 + + for { (fileName, email) <- emails } { + Universe.createNew() + val isSpamProbability = ReasoningComponent.classify(dictionary, learningResults, testDirectoryName + "/" + fileName) + val prediction = isSpamProbability >= threshold + (labels.get(fileName), prediction) match { + case (Some(true), true) => truePositives += 1 + case (Some(true), false) => falseNegatives += 1 + case (Some(false), true) => falsePositives += 1 + case (Some(false), false) => trueNegatives += 1 + case _ => () + } + } + val accuracy = (truePositives + trueNegatives).toDouble / (truePositives + falseNegatives + falsePositives + trueNegatives) + val precision = truePositives.toDouble / (truePositives + falsePositives) + val recall = truePositives.toDouble / (truePositives + falseNegatives) + + "Evaluator" should { + "have true positives equal to 39" taggedAs (BookExample) in { + truePositives should be(39) + } + "have false negatives equal to 0" taggedAs (BookExample) in { + falseNegatives should be(0) + } + "have false positives equal to 0" taggedAs (BookExample) in { + falsePositives should be(0) + } + "have true negatives equal to 61" taggedAs (BookExample) in { + trueNegatives should be(61) + } + "have an accuracy equal to 1.0" taggedAs (BookExample) in { + accuracy should be(1.0) + } + "have a precision equal to 1.0" taggedAs (BookExample) in { + precision should be(1.0) + } + "have a recall equal to 1.0" taggedAs (BookExample) in { + recall should be(1.0) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/LearningComponent.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/LearningComponent.scala new file mode 100644 index 00000000..44ea341c --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/LearningComponent.scala @@ -0,0 +1,177 @@ +/* + * LearningComponent.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap03 + +import java.nio.file.{Files,Paths,Path} +import java.io._ +import scala.io.Source +import com.cra.figaro.language.{Universe, Constant} +import com.cra.figaro.library.atomic.continuous.{Beta, AtomicBeta} +import com.cra.figaro.algorithm.learning._ +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +// MXR (10-MAR-2016): +// Some println statements commented out to minimize testing output +// MXR (11-MAR-2016): +// Calls to saveResults() were commented out to preserve the integrity +// of the LearnedModels.txt file, which is used by other tests. +object LearningComponent { + + def readEmails(directoryName: String): Map[String, Email] = { + val directory = Paths.get(directoryName) + val directoryIterator = Files.newDirectoryStream(directory).iterator() + + var result: Map[String, Email] = Map() + while (directoryIterator.hasNext()) { + val nextFile = directoryIterator.next().toFile() + val fileName = nextFile.getName + // println("Reading " + fileName) + result += fileName -> new Email(nextFile) + } + result + } + + def readLabels(labelFileName: String): Map[String, Boolean] = { + val source = Source.fromFile(labelFileName) + var result: Map[String, Boolean] = Map() + for { + line <- source.getLines() + } { + val parts = line.split(' ') + val isSpam = parts(0) == "1" + val emailFileName = parts(1) + result += emailFileName -> isSpam + } + result + } + + def learnMAP(params: PriorParameters): LearnedParameters = { +println("Beginning training") +println("Number of elements: " + Universe.universe.activeElements.length) + val algorithm = EMWithBP(params.fullParameterList:_*) +val time0 = System.currentTimeMillis() + algorithm.start() +val time1 = System.currentTimeMillis() +println("Training time: " + ((time1 - time0) / 1000.0)) + val spamProbability = params.spamProbability.MAPValue + val hasUnusualWordsGivenSpamProbability = params.hasManyUnusualWordsGivenSpamProbability.MAPValue + val hasUnusualWordsGivenNormalProbability = params.hasManyUnusualWordsGivenNormalProbability.MAPValue + val unusualWordGivenHasUnusualProbability = params.unusualWordGivenManyProbability.MAPValue + val unusualWordGivenNotHasUnusualProbability = params.unusualWordGivenFewProbability.MAPValue + val wordGivenSpamProbabilities = + for { (word, param) <- params.wordGivenSpamProbabilities } + yield (word, param.MAPValue) + val wordGivenNormalProbabilities = + for { (word, param) <- params.wordGivenNormalProbabilities } + yield (word, param.MAPValue) + algorithm.kill() + new LearnedParameters( + spamProbability, + hasUnusualWordsGivenSpamProbability, + hasUnusualWordsGivenNormalProbability, + unusualWordGivenHasUnusualProbability, + unusualWordGivenNotHasUnusualProbability, + wordGivenSpamProbabilities.toMap, + wordGivenNormalProbabilities.toMap + ) + } + + def saveResults( + fileName: String, + dictionary: Dictionary, + learningResults: LearnedParameters + ) = { + val file = new File(fileName) + val output = new PrintWriter(new BufferedWriter(new FileWriter(file))) + + output.println(dictionary.numEmails) + output.println(learningResults.spamProbability) + output.println(learningResults.hasManyUnusualWordsGivenSpamProbability) + output.println(learningResults.hasManyUnusualWordsGivenNormalProbability) + output.println(learningResults.unusualWordGivenManyProbability) + output.println(learningResults.unusualWordGivenFewProbability) + output.println(dictionary.words.length) + output.println(dictionary.featureWords.length) + + for { + word <- dictionary.words + } { + output.println(word) + output.println(dictionary.getCount(word)) + } + + for { + word <- dictionary.featureWords + } { + output.println(word) + output.println(learningResults.wordGivenSpamProbabilities(word)) + output.println(learningResults.wordGivenNormalProbabilities(word)) + } + + output.close() + } + + def main(args: Array[String]) { + val trainingDirectoryName = "src/test/resources/BookData/Training" + val labelFileName = "src/test/resources/BookData/Labels.txt" + val learningFileName = "src/test/resources/BookData/LearnedModel.txt" + + val emails = readEmails(trainingDirectoryName) + val labels = readLabels(labelFileName) + val dictionary = Dictionary.fromEmails(emails.values) + + val params = new PriorParameters(dictionary) + val models = + for { (fileName, email) <- emails } + yield { + val model = new LearningModel(dictionary, params) + email.observeEvidence(model, labels.get(fileName), true) + model + } + + val results = learnMAP(params) + // saveResults(learningFileName, dictionary, results) + println("Done!") + } +} + +class LearningComponentTest extends WordSpec with Matchers { + Universe.createNew() + val trainingDirectoryName = "src/test/resources/BookData/Training" + val labelFileName = "src/test/resources/BookData/Labels.txt" + val learningFileName = "src/test/resources/BookData/LearnedModel.txt" + + "Learning Component" should { + "produce the correct results" taggedAs (BookExample) in { + val emails = LearningComponent.readEmails(trainingDirectoryName) + val labels = LearningComponent.readLabels(labelFileName) + val dictionary = Dictionary.fromEmails(emails.values) + + val params = new PriorParameters(dictionary) + val models = + for { (fileName, email) <- emails } + yield { + val model = new LearningModel(dictionary, params) + email.observeEvidence(model, labels.get(fileName), true) + model + } + + val results = LearningComponent.learnMAP(params) + results should not be(null) + // LearningComponent.saveResults(learningFileName, dictionary, results) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Model.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Model.scala new file mode 100644 index 00000000..3b987a88 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/Model.scala @@ -0,0 +1,109 @@ +/* + * Model.scala + * Book example unit test support file. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap03 + +import com.cra.figaro.language.{Element, Constant, Flip, Universe} +import com.cra.figaro.library.compound.If +import com.cra.figaro.library.atomic.continuous.{Beta, AtomicBeta} +import com.cra.figaro.library.atomic.discrete.Binomial +import com.cra.figaro.algorithm.ProbQueryAlgorithm +import scala.collection.Map + +class PriorParameters(dictionary: Dictionary) { + val spamProbability = Beta(2,3) + val wordGivenSpamProbabilities = dictionary.featureWords.map(word => (word, Beta(2,2))) + val wordGivenNormalProbabilities = dictionary.featureWords.map(word => (word, Beta(2,2))) + val hasManyUnusualWordsGivenSpamProbability = Beta(2,2) + val hasManyUnusualWordsGivenNormalProbability = Beta(2, 21) + val unusualWordGivenManyProbability = Beta(2,2) + val unusualWordGivenFewProbability = Beta(2,7) + val fullParameterList = + spamProbability :: + hasManyUnusualWordsGivenSpamProbability :: + hasManyUnusualWordsGivenNormalProbability :: + unusualWordGivenManyProbability :: + unusualWordGivenFewProbability :: + wordGivenSpamProbabilities.map(pair => pair._2) ::: + wordGivenNormalProbabilities.map(pair => pair._2) + } + +class LearnedParameters( + val spamProbability: Double, + val hasManyUnusualWordsGivenSpamProbability: Double, + val hasManyUnusualWordsGivenNormalProbability: Double, + val unusualWordGivenManyProbability: Double, + val unusualWordGivenFewProbability: Double, + val wordGivenSpamProbabilities: Map[String, Double], + val wordGivenNormalProbabilities: Map[String, Double] +) + +abstract class Model(val dictionary: Dictionary) { + val isSpam: Element[Boolean] + + val hasWordElements: List[(String, Element[Boolean])] + + val hasManyUnusualWords: Element[Boolean] + + val numUnusualWords: Element[Int] +} + +class LearningModel(dictionary: Dictionary, parameters: PriorParameters) extends Model(dictionary) { + val isSpam = Flip(parameters.spamProbability) + + val hasWordElements = { + val wordGivenSpamMap = Map(parameters.wordGivenSpamProbabilities:_*) + val wordGivenNormalMap = Map(parameters.wordGivenNormalProbabilities:_*) + for { word <- dictionary.featureWords } yield { + val givenSpamProbability = wordGivenSpamMap(word) + val givenNormalProbability = wordGivenNormalMap(word) + val hasWordIfSpam = Flip(givenSpamProbability) + val hasWordIfNormal = Flip(givenNormalProbability) + (word, If(isSpam, hasWordIfSpam, hasWordIfNormal)) + } + } + + val hasManyUnusualIfSpam = Flip(parameters.hasManyUnusualWordsGivenSpamProbability) + val hasManyUnusualIfNormal = Flip(parameters.hasManyUnusualWordsGivenNormalProbability) + val hasManyUnusualWords = If(isSpam, hasManyUnusualIfSpam, hasManyUnusualIfNormal) + + val numUnusualIfHasMany = Binomial(Model.binomialNumTrials, parameters.unusualWordGivenManyProbability) + val numUnusualIfHasFew = Binomial(Model.binomialNumTrials, parameters.unusualWordGivenFewProbability) + val numUnusualWords = If(hasManyUnusualWords, numUnusualIfHasMany, numUnusualIfHasFew) +} + +class ReasoningModel(dictionary: Dictionary, parameters: LearnedParameters) extends Model(dictionary) { + val isSpam = Flip(parameters.spamProbability) + + val hasWordElements = { + for { word <- dictionary.featureWords } yield { + val givenSpamProbability = parameters.wordGivenSpamProbabilities(word) + val givenNormalProbability = parameters.wordGivenNormalProbabilities(word) + val hasWordIfSpam = Flip(givenSpamProbability) + val hasWordIfNormal = Flip(givenNormalProbability) + (word, If(isSpam, hasWordIfSpam, hasWordIfNormal)) + } + } + + val hasManyUnusualIfSpam = Flip(parameters.hasManyUnusualWordsGivenSpamProbability) + val hasManyUnusualIfNormal = Flip(parameters.hasManyUnusualWordsGivenNormalProbability) + val hasManyUnusualWords = If(isSpam, hasManyUnusualIfSpam, hasManyUnusualIfNormal) + + val numUnusualIfHasMany = Binomial(Model.binomialNumTrials, parameters.unusualWordGivenManyProbability) + val numUnusualIfHasFew = Binomial(Model.binomialNumTrials, parameters.unusualWordGivenFewProbability) + val numUnusualWords = If(hasManyUnusualWords, numUnusualIfHasMany, numUnusualIfHasFew) +} + +object Model { + val binomialNumTrials = 20 +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/ReasoningComponent.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/ReasoningComponent.scala new file mode 100644 index 00000000..7a07cd3c --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap03/ReasoningComponent.scala @@ -0,0 +1,114 @@ +/* + * ReasoningComponent.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap03 + +import scala.io.Source +import java.io.File +import com.cra.figaro.language.{Constant, Element, Universe} +import com.cra.figaro.algorithm.sampling.Importance +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.algorithm.factored.beliefpropagation.BeliefPropagation +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +// MXR (10-MAR-2016): some println statements commented out to minimize testing output +object ReasoningComponent { + def loadResults(fileName: String) = { + val source = Source.fromFile(fileName) + val lines = source.getLines().toList + val (numEmailsLine :: spamLine :: hasManyUnusualWordsGivenSpamLine :: hasManyUnusualWordsGivenNormalLine :: + unusualWordGivenManyLine :: unusualWordGivenFewLine :: numWordsLine :: numFeatureWordsLine :: rest) = lines + val numEmails = numEmailsLine.toInt + val spamProbability = spamLine.toDouble + val hasUnusualWordsGivenSpamProbability = hasManyUnusualWordsGivenSpamLine.toDouble + val hasUnusualWordsGivenNormalProbability = hasManyUnusualWordsGivenNormalLine.toDouble + val unusualWordGivenHasUnusualProbability = unusualWordGivenManyLine.toDouble + val unusualWordGivenNotHasUnusualProbability = unusualWordGivenFewLine.toDouble + val numWords = numWordsLine.toInt + val numFeatureWords = numFeatureWordsLine.toInt + + var linesRemaining = rest + var wordsGivenSpamProbabilities = Map[String, Double]() + var wordsGivenNormalProbabilities = Map[String, Double]() + var wordsAndCounts = List[(String, Int)]() + + for { i <- 0 until numWords } { + val word :: countLine :: rest = linesRemaining + linesRemaining = rest + wordsAndCounts ::= (word, countLine.toInt) + } + + for { i <- 0 until numFeatureWords } { + val word :: givenSpamLine :: givenNormalLine :: rest = linesRemaining + linesRemaining = rest + wordsGivenSpamProbabilities += word -> givenSpamLine.toDouble + wordsGivenNormalProbabilities += word -> givenNormalLine.toDouble + } + + val dictionary = new Dictionary(numEmails) + for { + (word, count) <- wordsAndCounts + i <- 0 until count + } { + dictionary.addWord(word) + } + + val params = new LearnedParameters( + spamProbability, + hasUnusualWordsGivenSpamProbability, + hasUnusualWordsGivenNormalProbability, + unusualWordGivenHasUnusualProbability, + unusualWordGivenNotHasUnusualProbability, + wordsGivenSpamProbabilities, + wordsGivenNormalProbabilities + ) + (dictionary, params) + } + + def classify(dictionary: Dictionary, parameters: LearnedParameters, fileName: String) = { + val file = new File(fileName) + val email = new Email(file) + val model = new ReasoningModel(dictionary, parameters) + email.observeEvidence(model, None, false) + val algorithm = VariableElimination(model.isSpam) + algorithm.start() + val isSpamProbability = algorithm.probability(model.isSpam, true) + // println("Spam probability: " + isSpamProbability) + algorithm.kill() + isSpamProbability + } + + def main(args: Array[String]) { + val emailFileName = "src/test/resources/BookData/Test/TestEmail_9.txt" + val learningFileName = "src/test/resources/BookData/LearnedModel.txt" + val (dictionary, parameters) = loadResults(learningFileName) + classify(dictionary, parameters, emailFileName) + println("Done!") + } +} + +class ReasoningComponentTest extends WordSpec with Matchers { + Universe.createNew() + val emailFileName = "src/test/resources/BookData/Test/TestEmail_9.txt" + val learningFileName = "src/test/resources/BookData/LearnedModel.txt" + + "Reasoning Component" should { + "produce a spam probability of 0.999999999632511" taggedAs (BookExample) in { + val (dictionary, parameters) = ReasoningComponent.loadResults(learningFileName) + val isSpamProbability = ReasoningComponent.classify(dictionary, parameters, emailFileName) + isSpamProbability should be(0.999999999632511) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/ImageRecovery.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/ImageRecovery.scala new file mode 100644 index 00000000..14e3b1b9 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/ImageRecovery.scala @@ -0,0 +1,119 @@ +/* + * ImageRecovery.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap05 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.beliefpropagation._ +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object ImageRecovery { + val pixels = Array.fill(10, 10)(Flip(0.4)) + def setConstraint(i1: Int, j1: Int, i2: Int, j2: Int) { + val pixel1 = pixels(i1)(j1) + val pixel2 = pixels(i2)(j2) + val pair = ^^(pixel1, pixel2) + pair.addConstraint(bb => if (bb._1 == bb._2) 0.9; else 0.1) + } + for { + i <- 0 until 10 + j <- 0 until 10 + } { + if (i <= 8) setConstraint(i, j, i+1, j) + if (j <= 8) setConstraint(i, j, i, j+1) + } + + def setEvidence(data: String) = { + for { n <- 0 until data.length } { + val i = n / 10 + val j = n % 10 + data(n) match { + case '0' => pixels(i)(j).observe(false) + case '1' => pixels(i)(j).observe(true) + case _ => () + } + } + } + + def main(args: Array[String]) { + val data = + """00?000?000 + 0?010?0010 + 110?010011 + 11??000111 + 11011000?1 + 1?0?100?10 + 00001?0?00 + 0010??0100 + 01?01001?0 + 0??000110?""".filterNot(_.isWhitespace) + + setEvidence(data) + val algorithm = MPEBeliefPropagation(10) + algorithm.start() + for { + i <- 0 until 10 + } { + for { j <- 0 until 10 } { + val mlv = algorithm.mostLikelyValue(pixels(i)(j)) + if (mlv) print('1') else print('0') + } + println() + } + } +} + +class ImageRecoveryTest extends WordSpec with Matchers { + Universe.createNew() + "Image Recovery" should { + "produce the correct results" taggedAs (BookExample, NonDeterministic) in { + val pixels = Array.fill(10, 10)(Flip(0.4)) + for { + i <- 0 until 10 + j <- 0 until 10 + } { + if (i <= 8) ImageRecovery.setConstraint(i, j, i+1, j) + if (j <= 8) ImageRecovery.setConstraint(i, j, i, j+1) + } + + val data = + """00?000?000 + 0?010?0010 + 110?010011 + 11??000111 + 11011000?1 + 1?0?100?10 + 00001?0?00 + 0010??0100 + 01?01001?0 + 0??000110?""".filterNot(_.isWhitespace) + + ImageRecovery.setEvidence(data) + val algorithm = MPEBeliefPropagation(10) + algorithm.start() + for { + i <- 0 until 10 + } { + for { j <- 0 until 10 } { + val mlv = algorithm.mostLikelyValue(pixels(i)(j)) + if (mlv) print('1') else print('0') + } + println() + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/PrinterProblem.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/PrinterProblem.scala new file mode 100644 index 00000000..320b64b7 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/PrinterProblem.scala @@ -0,0 +1,209 @@ +/* + * PrinterProblem.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap05 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object PrinterProblem { + val printerPowerButtonOn = Flip(0.95) + val tonerLevel = Select(0.7 -> 'high, 0.2 -> 'low, 0.1 -> 'out) + val tonerLowIndicatorOn = + If(printerPowerButtonOn, + CPD(tonerLevel, + 'high -> Flip(0.2), + 'low -> Flip(0.6), + 'out -> Flip(0.99)), + Constant(false)) + val paperFlow = Select(0.6 -> 'smooth, 0.2 -> 'uneven, 0.2 -> 'jammed) + val paperJamIndicatorOn = + If(printerPowerButtonOn, + CPD(paperFlow, + 'smooth -> Flip(0.1), + 'uneven -> Flip(0.3), + 'jammed -> Flip(0.99)), + Constant(false)) + val printerState = + Apply(printerPowerButtonOn, tonerLevel, paperFlow, + (power: Boolean, toner: Symbol, paper: Symbol) => { + if (power) { + if (toner == 'high && paper == 'smooth) 'good + else if (toner == 'out || paper == 'out) 'out + else 'poor + } else 'out + }) + val softwareState = Select(0.8 -> 'correct, 0.15 -> 'glitchy, 0.05 -> 'crashed) + val networkState = Select(0.7 -> 'up, 0.2 -> 'intermittent, 0.1 -> 'down) + val userCommandCorrect = Flip(0.65) + val numPrintedPages = + RichCPD(userCommandCorrect, networkState, softwareState, printerState, + (*, *, *, OneOf('out)) -> Constant('zero), + (*, *, OneOf('crashed), *) -> Constant('zero), + (*, OneOf('down), *, *) -> Constant('zero), + (OneOf(false), *, *, *) -> Select(0.3 -> 'zero, 0.6 -> 'some, 0.1 -> 'all), + (OneOf(true), *, *, *) -> Select(0.01 -> 'zero, 0.01 -> 'some, 0.98 -> 'all)) + val printsQuickly = + Chain(networkState, softwareState, + (network: Symbol, software: Symbol) => + if (network == 'down || software == 'crashed) Constant(false) + else if (network == 'intermittent || software == 'glitchy) Flip(0.5) + else Flip(0.9)) + val goodPrintQuality = + CPD(printerState, + 'good -> Flip(0.95), + 'poor -> Flip(0.3), + 'out -> Constant(false)) + val printResultSummary = + Apply(numPrintedPages, printsQuickly, goodPrintQuality, + (pages: Symbol, quickly: Boolean, quality: Boolean) => + if (pages == 'zero) 'none + else if (pages == 'some || !quickly || !quality) 'poor + else 'excellent) + + def step1() : Double = { + val answerWithNoEvidence = VariableElimination.probability(printerPowerButtonOn, true) + println("Prior probability the printer power button is on = " + answerWithNoEvidence) + answerWithNoEvidence + } + + def step2() : Double = { + printResultSummary.observe('poor) + val answerIfPrintResultPoor = VariableElimination.probability(printerPowerButtonOn, true) + println("Probability the printer power button is on given a poor result = " + answerIfPrintResultPoor) + answerIfPrintResultPoor + } + + def step3() : Double = { + printResultSummary.observe('none) + val answerIfPrintResultNone = VariableElimination.probability(printerPowerButtonOn, true) + println("Probability the printer power button is on given empty result = " + answerIfPrintResultNone) + answerIfPrintResultNone + } + + def step4() : Double = { + printResultSummary.unobserve() + printerState.observe('out) + val answerIfPrinterStateOut = VariableElimination.probability(printerPowerButtonOn, true) + println("Probability the printer power button is on given " + "out printer state = " + answerIfPrinterStateOut) + answerIfPrinterStateOut + } + + def step4a() : Double = { + printResultSummary.observe('none) + val answerIfPrinterStateOutAndResultNone = VariableElimination.probability(printerPowerButtonOn, true) + println("Probability the printer power button is on given out printer state and empty result = " + answerIfPrinterStateOutAndResultNone) + answerIfPrinterStateOutAndResultNone + } + + def step5() : Double = { + printResultSummary.unobserve() + printerState.unobserve() + val printerStateGoodPrior = VariableElimination.probability(printerState, 'good) + println("Prior probability the printer state is good = " + printerStateGoodPrior) + printerStateGoodPrior + } + + def step5a() : Double = { + tonerLowIndicatorOn.observe(true) + val printerStateGoodGivenTonerLowIndicatorOn = VariableElimination.probability(printerState, 'good) + println("Probability printer state is good given low toner indicator = " + printerStateGoodGivenTonerLowIndicatorOn) + printerStateGoodGivenTonerLowIndicatorOn + } + + def step6() : Double = { + tonerLowIndicatorOn.unobserve() + val softwareStateCorrectPrior = VariableElimination.probability(softwareState, 'correct) + println("Prior probability the software state is correct = " + softwareStateCorrectPrior) + softwareStateCorrectPrior + } + + def step6a() : Double = { + networkState.observe('up) + val softwareStateCorrectGivenNetworkUp = VariableElimination.probability(softwareState, 'correct) + println("Probability software state is correct given network up = " + softwareStateCorrectGivenNetworkUp) + softwareStateCorrectGivenNetworkUp + } + + def step7() : Double = { + networkState.unobserve() + printsQuickly.observe(false) + val softwareStateCorrectGivenPrintsSlowly = VariableElimination.probability(softwareState, 'correct) + println("Probability software state is correct given prints slowly = " + softwareStateCorrectGivenPrintsSlowly) + softwareStateCorrectGivenPrintsSlowly + } + + def step7a() : Double = { + networkState.observe('up) + val softwareStateCorrectGivenPrintsSlowlyAndNetworkUp = VariableElimination.probability(softwareState, 'correct) + println("Probability software state is correct given prints slowly and network up = " + softwareStateCorrectGivenPrintsSlowlyAndNetworkUp) + softwareStateCorrectGivenPrintsSlowlyAndNetworkUp + } + + def main(args: Array[String]) { + step1() + step2() + step3() + step4() + step4a() + step5() + step5a() + step6() + step6a() + step7() + step7a() + } +} + +class PrinterProblemTest extends WordSpec with Matchers { + Universe.createNew() + "Printer Problem" should { + "answerWithNoEvidence equals 0.95" taggedAs (BookExample) in { + PrinterProblem.step1() should be(0.95) + } + "answerIfPrintResultPoor equals 1.0" taggedAs (BookExample) in { + PrinterProblem.step2() should be(1.0) + } + "answerIfPrintResultNone equals 0.8573402523786461" taggedAs (BookExample) in { + PrinterProblem.step3() should be(0.8573402523786461) + } + "answerIfPrinterStateOut equals 0.6551724137931032" taggedAs (BookExample) in { + PrinterProblem.step4() should be(0.6551724137931032) + } + "answerIfPrinterStateOutAndResultNone equals 0.6551724137931033" taggedAs (BookExample) in { + PrinterProblem.step4a() should be(0.6551724137931033) + } + "printerStateGoodPrior equals 0.39899999999999997" taggedAs (BookExample) in { + PrinterProblem.step5() should be(0.39899999999999997) + } + "printerStateGoodGivenTonerLowIndicatorOn approximately equals 0.2339832869" taggedAs (BookExample) in { + PrinterProblem.step5a() should be(0.2339832869 +- 0.00000000001) + } + "softwareStateCorrectPrior equals 0.8" taggedAs (BookExample) in { + PrinterProblem.step6() should be(0.8) + } + "softwareStateCorrectGivenNetworkUp equals 0.7999999999999999" taggedAs (BookExample) in { + PrinterProblem.step6a() should be(0.7999999999999999) + } + "softwareStateCorrectGivenPrintsSlowly equals 0.6197991391678623" taggedAs (BookExample) in { + PrinterProblem.step7() should be(0.6197991391678623) + } + "softwareStateCorrectGivenPrintsSlowlyAndNetworkUp equals 0.39024390243902435" taggedAs (BookExample) in { + PrinterProblem.step7a() should be(0.39024390243902435) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/ProductDistribution.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/ProductDistribution.scala new file mode 100644 index 00000000..271bef36 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap05/ProductDistribution.scala @@ -0,0 +1,92 @@ +/* + * ProductDistribution.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap05 + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.discrete.{Binomial, Poisson} +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.sampling.Importance +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object ProductDistribution { + class Network(popularity: Double) { + val numNodes = Poisson(popularity) + } + + class Model(targetPopularity: Double, productQuality: Double, affordability: Double) { + def generateLikes(numFriends: Int, productQuality: Double): Element[Int] = { + def helper(friendsVisited: Int, totalLikes: Int, unprocessedLikes: Int): Element[Int] = { + if (unprocessedLikes == 0) Constant(totalLikes) + else { + val unvisitedFraction = //#C + 1.0 - (friendsVisited.toDouble - 1)/ (numFriends - 1) //#C + val newlyVisited = Binomial(2, unvisitedFraction) + val newlyLikes = Binomial(newlyVisited, Constant(productQuality)) + Chain(newlyVisited, newlyLikes, + (visited: Int, likes: Int) => + helper(friendsVisited + visited, totalLikes + likes, unprocessedLikes + likes - 1)) + } + } + helper(1, 1, 1) + } + + val targetSocialNetwork = new Network(targetPopularity) + val targetLikes = Flip(productQuality) + val numberFriendsLike = + Chain(targetLikes, targetSocialNetwork.numNodes, + (l: Boolean, n: Int) => + if (l) generateLikes(n, productQuality); else Constant(0)) + val numberBuy = Binomial(numberFriendsLike, Constant(affordability)) + } + + def predict(targetPopularity: Double, productQuality: Double, affordability: Double): Double = { + val model = new Model(targetPopularity, productQuality, affordability) + val algorithm = Importance(1000, model.numberBuy) + algorithm.start() + val result = algorithm.expectation(model.numberBuy)(i=> i.toDouble) + algorithm.kill() + result + } + + def main(args: Array[String]) { + println("Popularity\tProduct quality\tAffordability\tPredicted number of buyers") + println("100 \t0.5 \t0.5 \t" + predict(100, 0.5, 0.5)) + println("100 \t0.5 \t0.9 \t" + predict(100, 0.5, 0.9)) + println("100 \t0.9 \t0.5 \t" + predict(100, 0.9, 0.5)) + println("100 \t0.9 \t0.9 \t" + predict(100, 0.9, 0.9)) + println("10 \t0.5 \t0.5 \t" + predict(10, 0.5, 0.5)) + println("10 \t0.5 \t0.9 \t" + predict(10, 0.5, 0.9)) + println("10 \t0.9 \t0.5 \t" + predict(10, 0.9, 0.5)) + println("10 \t0.9 \t0.9 \t" + predict(10, 0.9, 0.9)) + } +} + +class ProductDistributionTest extends WordSpec with Matchers { + Universe.createNew() + "Product Distribution" should { + "produce the correct results at each step" taggedAs (BookExample, NonDeterministic) in { + ProductDistribution.predict(100, 0.5, 0.5) should be(2.0259999999999976 +- 1.0) + ProductDistribution.predict(100, 0.5, 0.9) should be(3.562999999999997 +- 1.0) + ProductDistribution.predict(100, 0.9, 0.5) should be(29.016999999999967 +- 1.0) + ProductDistribution.predict(100, 0.9, 0.9) should be(52.58599999999993 +- 2.0) + ProductDistribution.predict(10, 0.5, 0.5) should be(0.8219999999999981 +- 1.0) + ProductDistribution.predict(10, 0.5, 0.9) should be(1.3489999999999978 +- 1.0) + ProductDistribution.predict(10, 0.9, 0.5) should be(3.385999999999989 +- 1.0) + ProductDistribution.predict(10, 0.9, 0.9) should be(6.143999999999982 +- 1.0) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/CoinTosses.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/CoinTosses.scala new file mode 100644 index 00000000..8d27c794 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/CoinTosses.scala @@ -0,0 +1,87 @@ +/* + * CoinTosses.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.language.Flip +import com.cra.figaro.algorithm.sampling.Importance +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object CoinTosses { + def main(args: Array[String]) { + val outcomes = "10" + val numTosses = outcomes.length + + val bias = Beta(2,5) + val tosses = Array.fill(numTosses)(Flip(bias)) + val nextToss = Flip(bias) + + for { + toss <- 0 until numTosses + } { + val outcome = outcomes(toss) == 'H' + tosses(toss).observe(outcome) + } + + val algorithm = Importance(nextToss, bias) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + + println("Average bias = " + algorithm.mean(bias)) + println("Probability of heads on next toss = " + algorithm.probability(nextToss, true)) + + algorithm.kill() + } +} + +class CoinTossesTest extends WordSpec with Matchers { + Universe.createNew() + "Coin Tosses" should { + val outcomes = "10" + val numTosses = outcomes.length + + val bias = Beta(2,5) + val tosses = Array.fill(numTosses)(Flip(bias)) + val nextToss = Flip(bias) + + for { + toss <- 0 until numTosses + } { + val outcome = outcomes(toss) == 'H' + tosses(toss).observe(outcome) + } + + val algorithm = Importance(nextToss, bias) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + + val avgBias = algorithm.mean(bias) + val pHeads = algorithm.probability(nextToss, true) + + algorithm.kill() + + "have an average bias equal to 0.222 +- 0.003" taggedAs (BookExample, NonDeterministic) in { + avgBias should be(0.222 +- 0.003) + } + "have a probability of heads on next toss equal to 0.227 +- 0.003" taggedAs (BookExample, NonDeterministic) in { + pHeads should be(0.227 +- 0.003) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/HealthProcess.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/HealthProcess.scala new file mode 100644 index 00000000..a5f733d1 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/HealthProcess.scala @@ -0,0 +1,108 @@ +/* + * HealthProcess.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.language._ +import com.cra.figaro.library.collection.Process +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.compound.{^^, If} +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.library.compound.If +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object HealthProcess extends Process[Double, Boolean] { + val healthyPrior = Uniform((0.05 to 0.95 by 0.1):_*) + val healthChangeRate = Uniform((0.001 to 0.1 by 0.002):_*) + + def generate(time: Double): Element[Boolean] = Flip(healthyPrior) + + def generate(times: List[Double]): Map[Double, Element[Boolean]] = { + val sortedTimes = times.sorted + val healthy = sortedTimes.map(time => (time, generate(time))).toMap + def makePairs(remaining: List[Double]) { + if (remaining.length >= 2) { + val time1 :: time2 :: rest = remaining + val probChange = Apply(healthChangeRate, (d: Double) => 1 - math.exp(- (time2 - time1) / d)) + val equalHealth = healthy(time1) === healthy(time2) + val healthStatusChecker = If(equalHealth, Constant(true), Flip(probChange)) + healthStatusChecker.observe(true) + makePairs(time2 :: rest) + } + } + makePairs(sortedTimes) + healthy + } + + def rangeCheck(time: Double) = time >= 0 + + def main(args: Array[String]) { + val data = Map(0.1 -> true, 0.25 -> true, 0.3 -> false, 0.31 -> false, 0.34 -> false, 0.36 -> false, 0.4 -> true, 0.5 -> true, 0.55 -> true) + val queries = List(0.35, 0.37, 0.45, 0.6) + val targets = queries ::: data.keys.toList + + val healthy = generate(targets) + for { (time, value) <- data } { + healthy(time).observe(value) + } + + val queryElements = queries.map(healthy(_)) + val queryTargets = healthyPrior :: healthChangeRate :: queryElements + val algorithm = VariableElimination(queryTargets:_*) + algorithm.start() + for { query <- queries } { + println("Probability the patient is healthy at time " + query + " = " + algorithm.probability(healthy(query), true)) + } + println("Expected prior probability of healthy = " + algorithm.mean(healthyPrior)) + println("Expected health change rate = " + algorithm.mean(healthChangeRate)) + algorithm.kill() + } +} + +class HealthProcessTest extends WordSpec with Matchers { + Universe.createNew() + "Health Process" should { + val healthyPrior = Uniform((0.05 to 0.95 by 0.1):_*) + val healthChangeRate = Uniform((0.001 to 0.1 by 0.002):_*) + + val data = Map(0.1 -> true, 0.25 -> true, 0.3 -> false, 0.31 -> false, 0.34 -> false, 0.36 -> false, 0.4 -> true, 0.5 -> true, 0.55 -> true) + val queries = List(0.35, 0.37, 0.45, 0.6) + val targets = queries ::: data.keys.toList + + val healthy = HealthProcess.generate(targets) + for { (time, value) <- data } { + healthy(time).observe(value) + } + + val queryElements = queries.map(healthy(_)) + val queryTargets = healthyPrior :: healthChangeRate :: queryElements + val algorithm = VariableElimination(queryTargets:_*) + algorithm.start() + for { query <- queries } { + println("Probability the patient is healthy at time " + query + " = " + algorithm.probability(healthy(query), true)) + } + val pHealthy = algorithm.mean(healthyPrior) + val expectedHcr = algorithm.mean(healthChangeRate) + algorithm.kill() + + "have an expected prior probability of healthy equal to 0.449 +- 0.003" taggedAs (BookExample, NonDeterministic) in { + pHealthy should be(0.449 +- 0.003) + } + "have an expected health change rate equal to 0.050 +- 0.003" taggedAs (BookExample, NonDeterministic) in { + expectedHcr should be(0.050 +- 0.003) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/Hierarchical.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/Hierarchical.scala new file mode 100644 index 00000000..c4942ada --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/Hierarchical.scala @@ -0,0 +1,116 @@ +/* + * Hierarchical.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.language.{Flip, Constant, Universe} //#A +import com.cra.figaro.library.atomic.continuous.{Uniform, Beta} //#A +import com.cra.figaro.library.compound.If //#A +import com.cra.figaro.algorithm.sampling.Importance //#A +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object Hierarchical { + def main(args: Array[String]) { + val initialTosses : Array[String] = Array("010101010101010101010101010101") + val numCoins = initialTosses.length + + val fairProbability = Uniform(0.0, 1.0) + + val isFair = + for { coin <- 0 until numCoins } + yield Flip(fairProbability) + val biases = + for { coin <- 0 until numCoins } + yield If(isFair(coin), Constant(0.5), Beta(2,5)) + + val tosses = + for { coin <- 0 until numCoins } + yield { + for { toss <- 0 until initialTosses(coin).length } + yield Flip(biases(coin)) + } + + for { + coin <- 0 until numCoins + toss <- 0 until initialTosses(coin).length + } { + val outcome = initialTosses(coin)(toss) == 'H' + tosses(coin)(toss).observe(outcome) + } + + val algorithm = Importance(fairProbability, biases(0)) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + + val averageFairProbability = algorithm.mean(fairProbability) + val firstCoinAverageBias = algorithm.mean(biases(0)) + println("Average fairness probability: " + averageFairProbability) + println("First coin average bias: " + firstCoinAverageBias) + + algorithm.kill() + } +} + +class HierarchicalTest extends WordSpec with Matchers { + Universe.createNew() + "Hierarchical" should { + val initialTosses : Array[String] = Array("010101010101010101010101010101") + val numCoins = initialTosses.length + + val fairProbability = Uniform(0.0, 1.0) + + val isFair = + for { coin <- 0 until numCoins } + yield Flip(fairProbability) + val biases = + for { coin <- 0 until numCoins } + yield If(isFair(coin), Constant(0.5), Beta(2,5)) + + val tosses = + for { coin <- 0 until numCoins } + yield { + for { toss <- 0 until initialTosses(coin).length } + yield Flip(biases(coin)) + } + + for { + coin <- 0 until numCoins + toss <- 0 until initialTosses(coin).length + } { + val outcome = initialTosses(coin)(toss) == 'H' + tosses(coin)(toss).observe(outcome) + } + + val algorithm = Importance(fairProbability, biases(0)) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + + val averageFairProbability = algorithm.mean(fairProbability) + val firstCoinAverageBias = algorithm.mean(biases(0)) + + algorithm.kill() + + "have an average fairness probability equal to 0.03 +- 0.03" taggedAs (BookExample, NonDeterministic) in { + averageFairProbability should be(0.34 +- 0.03) + } + "have a first coin average bias equal to 0.05 +- 0.03" taggedAs (BookExample, NonDeterministic) in { + firstCoinAverageBias should be(0.05 +- 0.03) + } + } +} + diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/HierarchicalContainers.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/HierarchicalContainers.scala new file mode 100644 index 00000000..55a014b3 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/HierarchicalContainers.scala @@ -0,0 +1,102 @@ +/* + * HierarchicalContainers.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.language.{Flip, Constant, Universe} +import com.cra.figaro.library.atomic.continuous.{Uniform, Beta} +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.sampling.Importance +import com.cra.figaro.library.collection.FixedSizeArray +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object HierarchicalContainers { + def main(args: Array[String]) { + val initialTosses : Array[String] = Array("0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1") + val numCoins = initialTosses.length + + val fairProbability = Uniform(0.0, 1.0) + + val isFair = new FixedSizeArray(numCoins + 1, i => Flip(fairProbability)) + val biases = isFair.chain(if (_) Constant(0.5) else Beta(2,5)) + + val tosses = + for { coin <- 0 until numCoins } yield new FixedSizeArray(initialTosses(coin).length, i => Flip(biases(coin))) + + val hasHeads = + for { coin <- 0 until numCoins } yield tosses(coin).exists(b => b) + + for { + coin <- 0 until numCoins + toss <- 0 until initialTosses(coin).length + } { + initialTosses(coin)(toss) match { + case 'H' => tosses(coin)(toss).observe(true) + case 'T' => tosses(coin)(toss).observe(false) + case _ => () + } + } + + val algorithm = Importance(fairProbability, hasHeads(2)) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + println("Probability at least one of the tosses of the third coin was heads = " + algorithm.probability(hasHeads(2), true)) + algorithm.kill() + } +} + +class HierarchicalContainersTest extends WordSpec with Matchers { + Universe.createNew() + "Hierarchical Containers" should { + val initialTosses : Array[String] = Array("0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1","0","1") + val numCoins = initialTosses.length + + val fairProbability = Uniform(0.0, 1.0) + + val isFair = new FixedSizeArray(numCoins + 1, i => Flip(fairProbability)) + val biases = isFair.chain(if (_) Constant(0.5) else Beta(2,5)) + + val tosses = + for { coin <- 0 until numCoins } yield new FixedSizeArray(initialTosses(coin).length, i => Flip(biases(coin))) + + val hasHeads = + for { coin <- 0 until numCoins } yield tosses(coin).exists(b => b) + + for { + coin <- 0 until numCoins + toss <- 0 until initialTosses(coin).length + } { + initialTosses(coin)(toss) match { + case 'H' => tosses(coin)(toss).observe(true) + case 'T' => tosses(coin)(toss).observe(false) + case _ => () + } + } + + val algorithm = Importance(fairProbability, hasHeads(2)) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + val prob = algorithm.probability(hasHeads(2), true) + algorithm.kill() + + "have a Probability at least one of the tosses of the third coin was heads equal to 0.36 +- 0.05" taggedAs (BookExample, NonDeterministic) in { + prob should be(0.36 +- 0.05) + } + } +} + diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/NewProducts.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/NewProducts.scala new file mode 100644 index 00000000..6bda6608 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/NewProducts.scala @@ -0,0 +1,64 @@ +/* + * NewProducts.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.library.atomic.discrete.Geometric +import com.cra.figaro.library.atomic.continuous.{Beta, Normal} +import com.cra.figaro.library.collection.VariableSizeArray +import com.cra.figaro.algorithm.sampling.Importance +import com.cra.figaro.language.Universe +import com.cra.figaro.language.Flip +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object NewProducts { + def runExperiment(rNDLevel: Double) { + Universe.createNew() + val numNewProducts = Geometric(rNDLevel) + val productQuality = VariableSizeArray(numNewProducts, i => Beta(1, i + 1)) + val productSalesRaw = productQuality.chain(Normal(_, 0.5)) + val productSales = productSalesRaw.map(_.max(0)) + val totalSales = productSales.foldLeft(0.0)(_ + _) + val algorithm = Importance(totalSales) + algorithm.start() + Thread.sleep(5000) + algorithm.stop() + println("With R&D at " + rNDLevel + ", expected sales will be " + algorithm.mean(totalSales)) + algorithm.kill() + } + + def main(args: Array[String]) { + var i : Double = 0.0 + for { i <- 0.05 to 1.0 by 0.1 } { + println(i) + runExperiment(i) + } + } +} + +class NewProductsTest extends WordSpec with Matchers { + def testFunc { + for { i <- 0.05 to 1.0 by 0.1 } { + NewProducts.runExperiment(i) + } + } + + "New Products" should { + "produce correct results at each step" taggedAs (BookExample, NonDeterministic) in { + testFunc + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/Sales.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/Sales.scala new file mode 100644 index 00000000..20224839 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/Sales.scala @@ -0,0 +1,120 @@ +/* + * Sales.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.language.Flip +import com.cra.figaro.algorithm.sampling.Importance +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object Sales { + + def main(args: Array[String]) { + val regions : Array[String] = Array("A","B","C","D","E") + val numProducts = regions.length + val numRegions = regions(0).length + + /* The model */ + val productQuality = Array.fill(numProducts)(Beta(2,2)) + val regionPenetration = Array.fill(numRegions)(Beta(2,2)) + def makeSales(i: Int, j: Int) = Flip(productQuality(i) * regionPenetration(j)) + val highSales = Array.tabulate(numProducts, numRegions)(makeSales _) + + /* Observe all the sales */ + for { + i <- 0 until numProducts + j <- 0 until numRegions + } { + val observation = regions(i)(j) == 'T' + highSales(i)(j).observe(observation) + } + + /* Run inference */ + val targets = productQuality ++ regionPenetration + val algorithm = Importance(targets:_*) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + + /* Report the results */ + for { i <- 0 until numProducts } { + println("Product " + i + " quality: " + algorithm.mean(productQuality(i))) + } + for { j <- 0 until numRegions } { + println("Region " + j + " penetration: " + algorithm.mean(regionPenetration(j))) + } + algorithm.kill() + } +} + +class SalesTest extends WordSpec with Matchers { + Universe.createNew() + "Sales" should { + val products : Array[String] = Array("A","B","C","D","E") + val numProducts = products.length + val numRegions = products(0).length + + /* The model */ + val productQuality = Array.fill(numProducts)(Beta(2,2)) + val regionPenetration = Array.fill(numRegions)(Beta(2,2)) + def makeSales(i: Int, j: Int) = Flip(productQuality(i) * regionPenetration(j)) + val highSales = Array.tabulate(numProducts, numRegions)(makeSales _) + + /* Observe all the sales */ + for { + i <- 0 until numProducts + j <- 0 until numRegions + } { + val observation = products(i)(j) == 'T' + highSales(i)(j).observe(observation) + } + + /* Run inference */ + val targets = productQuality ++ regionPenetration + val algorithm = Importance(targets:_*) + algorithm.start() + Thread.sleep(1000) + algorithm.stop() + val prod0 = algorithm.mean(productQuality(0)) + val prod1 = algorithm.mean(productQuality(1)) + val prod2 = algorithm.mean(productQuality(2)) + val prod3 = algorithm.mean(productQuality(3)) + val prod4 = algorithm.mean(productQuality(4)) + val reg0 = algorithm.mean(regionPenetration(0)) + algorithm.kill() + + "produce a Product 0 quality of 0.47 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + prod0 should be(0.47 +- 0.3) + } + "produce a Product 1 quality of 0.47 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + prod1 should be(0.47 +- 0.3) + } + "produce a Product 2 quality of 0.47 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + prod2 should be(0.47 +- 0.3) + } + "produce a Product 3 quality of 0.47 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + prod3 should be(0.47 +- 0.3) + } + "produce a Product 4 quality of 0.47 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + prod4 should be(0.47 +- 0.3) + } + "produce a Region 0 penetration of 0.35 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + reg0 should be(0.35 +- 0.3) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/SalesPrediction.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/SalesPrediction.scala new file mode 100644 index 00000000..00c28f20 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap06/SalesPrediction.scala @@ -0,0 +1,148 @@ +/* + * SalesPrediction.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap06 + +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.language.Flip +import com.cra.figaro.algorithm.sampling.Importance +import com.cra.figaro.library.collection.Container +import com.cra.figaro.library.atomic.discrete.Poisson +import com.cra.figaro.language.Chain +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object SalesPrediction { + + def main(args: Array[String]) { + val products : Array[String] = Array("A","B","C","D","E") + val numProducts = products.length + val numRegions = products(0).length + + /* The model */ + val productQuality = Array.fill(numProducts)(Beta(2,2)) + val regionPenetration = Array.fill(numRegions)(Beta(2,2)) + def makeSales(i: Int, j: Int) = Flip(productQuality(i) * regionPenetration(j)) + val highSalesLastYear = Array.tabulate(numProducts, numRegions)(makeSales _) + val highSalesNextYear = Array.tabulate(numProducts, numRegions)(makeSales _) + + def getSalesByProduct(i: Int) = + for { j <- 0 until numRegions } yield highSalesNextYear(i)(j) + val salesPredictionByProduct = + Array.tabulate(numProducts)(i => Container(getSalesByProduct(i):_*)) + + val numHighSales = + for { predictions <- salesPredictionByProduct } + yield predictions.count(b => b) + + val numHiresByProduct = +// for { i <- 0 until numProducts } +// yield Chain(numHighSales(i), (n: Int) => Poisson(n + 1)) + Container(numHighSales:_*).chain((n: Int) => Poisson(n + 1)) + + /* Observe all the sales */ + for { + i <- 0 until numProducts + j <- 0 until numRegions + } { + val observation = products(i)(j) == 'T' + highSalesLastYear(i)(j).observe(observation) + } + + /* Run inference */ + val targets = numHiresByProduct.elements + val algorithm = Importance(targets:_*) + algorithm.start() + Thread.sleep(10000) + algorithm.stop() + + /* Report the results */ + for { i <- 0 until numProducts } { + println("Number of hires for product " + i + ": " + algorithm.expectation(numHiresByProduct(i), (n: Int) => n.toDouble)) + } + algorithm.kill() + } +} + +class SalesPredictionTest extends WordSpec with Matchers { + Universe.createNew() + "Sales Prediction" should { + val products : Array[String] = Array("A","B","C","D","E") + val numProducts = products.length + val numRegions = products(0).length + + /* The model */ + val productQuality = Array.fill(numProducts)(Beta(2,2)) + val regionPenetration = Array.fill(numRegions)(Beta(2,2)) + def makeSales(i: Int, j: Int) = Flip(productQuality(i) * regionPenetration(j)) + val highSalesLastYear = Array.tabulate(numProducts, numRegions)(makeSales _) + val highSalesNextYear = Array.tabulate(numProducts, numRegions)(makeSales _) + + def getSalesByProduct(i: Int) = + for { j <- 0 until numRegions } yield highSalesNextYear(i)(j) + val salesPredictionByProduct = + Array.tabulate(numProducts)(i => Container(getSalesByProduct(i):_*)) + + val numHighSales = + for { predictions <- salesPredictionByProduct } + yield predictions.count(b => b) + + val numHiresByProduct = +// for { i <- 0 until numProducts } +// yield Chain(numHighSales(i), (n: Int) => Poisson(n + 1)) + Container(numHighSales:_*).chain((n: Int) => Poisson(n + 1)) + + /* Observe all the sales */ + for { + i <- 0 until numProducts + j <- 0 until numRegions + } { + val observation = products(i)(j) == 'T' + highSalesLastYear(i)(j).observe(observation) + } + + /* Run inference */ + val targets = numHiresByProduct.elements + val algorithm = Importance(targets:_*) + algorithm.start() + Thread.sleep(10000) + algorithm.stop() + + val hiresProd0 = algorithm.expectation(numHiresByProduct(0), (n: Int) => n.toDouble) + val hiresProd1 = algorithm.expectation(numHiresByProduct(1))(n => n.toDouble) + val hiresProd2 = algorithm.expectation(numHiresByProduct(2), (n: Int) => n.toDouble) + val hiresProd3 = algorithm.expectation(numHiresByProduct(3))(n => n.toDouble) + val hiresProd4 = algorithm.expectation(numHiresByProduct(4), (n: Int) => n.toDouble) + + algorithm.kill() + + "produce a number of hires for Product 0 of 1.15 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + hiresProd0 should be(1.15 +- 0.3) + } + "produce a number of hires for Product 1 of 1.15 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + hiresProd1 should be(1.15 +- 0.3) + } + "produce a number of hires for Product 2 of 1.15 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + hiresProd2 should be(1.15 +- 0.3) + } + "produce a number of hires for Product 3 of 1.15 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + hiresProd3 should be(1.15 +- 0.3) + } + "produce a number of hires for Product 4 of 1.15 +- 0.3" taggedAs (BookExample, NonDeterministic) in { + hiresProd4 should be(1.15 +- 0.3) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemInheritance.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemInheritance.scala new file mode 100644 index 00000000..3acfe27a --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemInheritance.scala @@ -0,0 +1,193 @@ +/* + * PrinterProblemInheritance.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap07 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object PrinterProblemInheritance { + abstract class Printer { + val powerButtonOn = Flip(0.95) + + val paperFlow = Select(0.6 -> 'smooth, 0.2 -> 'uneven, 0.2 -> 'jammed) + val paperJamIndicatorOn = + If(powerButtonOn, + CPD(paperFlow, + 'smooth -> Flip(0.1), + 'uneven -> Flip(0.3), + 'jammed -> Flip(0.99)), + Constant(false)) + + val state: Element[Symbol] + } + + class LaserPrinter extends Printer { + val tonerLevel = Select(0.7 -> 'high, 0.2 -> 'low, 0.1 -> 'out) + val tonerLowIndicatorOn = + If(powerButtonOn, + CPD(tonerLevel, + 'high -> Flip(0.2), + 'low -> Flip(0.6), + 'out -> Flip(0.99)), + Constant(false)) + val state = + Apply(powerButtonOn, tonerLevel, paperFlow, + (power: Boolean, toner: Symbol, paper: Symbol) => { + if (power) { + if (toner == 'high && paper == 'smooth) 'good + else if (toner == 'out || paper == 'out) 'out + else 'poor + } else 'out + } + ) + } + + class InkjetPrinter extends Printer { + val inkCartridgeEmpty = Flip(0.1) + val inkCartridgeEmptyIndicator = If(inkCartridgeEmpty, Flip(0.99), Flip(0.3)) + val cloggedNozzle = Flip(0.001) + val state = + Apply(powerButtonOn, inkCartridgeEmpty, cloggedNozzle, paperFlow, + (power: Boolean, ink: Boolean, nozzle: Boolean, paper: Symbol) => { + if (power && !ink && !nozzle) { + if (paper == 'smooth) 'good + else if (paper == 'uneven) 'poor + else 'out + } else 'out + } + ) + } + + class Software { + val state = Select(0.8 -> 'correct, 0.15 -> 'glitchy, 0.05 -> 'crashed) + } + + class Network { + val state = Select(0.7 -> 'up, 0.2 -> 'intermittent, 0.1 -> 'down) + } + + class User { + val commandCorrect = Flip(0.65) + } + + class PrintExperience(printer: Printer, software: Software, network: Network, user: User) { + val numPrintedPages = + RichCPD(user.commandCorrect, network.state, software.state, printer.state, + (*, *, *, OneOf('out)) -> Constant('zero), + (*, *, OneOf('crashed), *) -> Constant('zero), + (*, OneOf('down), *, *) -> Constant('zero), + (OneOf(false), *, *, *) -> Select(0.3 -> 'zero, 0.6 -> 'some, 0.1 -> 'all), + (OneOf(true), *, *, *) -> Select(0.01 -> 'zero, 0.01 -> 'some, 0.98 -> 'all)) + val printsQuickly = + Chain(network.state, software.state, + (network: Symbol, software: Symbol) => + if (network == 'down || software == 'crashed) Constant(false) + else if (network == 'intermittent || software == 'glitchy) Flip(0.5) + else Flip(0.9)) + val goodPrintQuality = + CPD(printer.state, + 'good -> Flip(0.95), + 'poor -> Flip(0.3), + 'out -> Constant(false)) + val summary = + Apply(numPrintedPages, printsQuickly, goodPrintQuality, + (pages: Symbol, quickly: Boolean, quality: Boolean) => + if (pages == 'zero) 'none + else if (pages == 'some || !quickly || !quality) 'poor + else 'excellent) + } + + val myLaserPrinter = new LaserPrinter + val myInkjetPrinter = new InkjetPrinter + val mySoftware = new Software + val myNetwork = new Network + val me = new User + val myExperience1 = new PrintExperience(myLaserPrinter, mySoftware, myNetwork, me) + val myExperience2 = new PrintExperience(myInkjetPrinter, mySoftware, myNetwork, me) + + def step1() { + myExperience1.summary.observe('none) + val alg = VariableElimination(myLaserPrinter.powerButtonOn, myNetwork.state) + alg.start() + println("After observing that printing with the laser printer produces no result:") + println("Probability laser printer power button is on = " + alg.probability(myLaserPrinter.powerButtonOn, true)) + println("Probability network is down = " + alg.probability(myNetwork.state, 'down)) + alg.kill() + } + + def step2() { + myExperience2.summary.observe('none) + val alg = VariableElimination(myLaserPrinter.powerButtonOn, myNetwork.state) + alg.start() + println("\nAfter observing that printing with the inkjet printer also produces no result:") + println("Probability laser printer power button is on = " + alg.probability(myLaserPrinter.powerButtonOn, true)) + println("Probability network is down = " + alg.probability(myNetwork.state, 'down)) + alg.kill() + } + + def main(args: Array[String]) { + step1() + step2() + } +} + +class PrinterProblemInheritanceTest extends WordSpec with Matchers { + Universe.createNew() + val myLaserPrinter = new PrinterProblemInheritance.LaserPrinter + val myInkjetPrinter = new PrinterProblemInheritance.InkjetPrinter + val mySoftware = new PrinterProblemInheritance.Software + val myNetwork = new PrinterProblemInheritance.Network + val me = new PrinterProblemInheritance.User + val myExperience1 = new PrinterProblemInheritance.PrintExperience(myLaserPrinter, mySoftware, myNetwork, me) + val myExperience2 = new PrinterProblemInheritance.PrintExperience(myInkjetPrinter, mySoftware, myNetwork, me) + + "Printer Problem Inheritance" should { + myExperience1.summary.observe('none) + val alg1 = VariableElimination(myLaserPrinter.powerButtonOn, myNetwork.state) + alg1.start() + val btnOn1 = alg1.probability(myLaserPrinter.powerButtonOn, true) + val netDown1 = alg1.probability(myNetwork.state, 'down) + alg1.kill() + + "after observing that printing with the laser printer produces no result:" should { + "produce a probability laser printer power button is on = 0.8573402523786461" taggedAs (BookExample) in { + btnOn1 should be(0.8573402523786461) + } + "produce a probability network is down = 0.2853194952427" taggedAs (BookExample) in { + netDown1 should be(0.2853194952427 +- 0.0000000000001) + } + } + + myExperience2.summary.observe('none) + val alg2 = VariableElimination(myLaserPrinter.powerButtonOn, myNetwork.state) + alg2.start() + val btnOn2 = alg2.probability(myLaserPrinter.powerButtonOn, true) + val netDown2 = alg2.probability(myNetwork.state, 'down) + alg2.kill() + + "after observing that printing with the inkjet printer also produces no result:" should { + "produce a probability laser printer power button is on = 0.8978039844824163" taggedAs (BookExample) in { + btnOn2 should be(0.8978039844824163) + } + "produce a probability network is down = 0.4250135950242" taggedAs (BookExample) in { + netDown2 should be(0.4250135950242 +- 0.0000000000001) + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemOO.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemOO.scala new file mode 100644 index 00000000..2dad8fcc --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemOO.scala @@ -0,0 +1,138 @@ +/* + * PrinterProblemOO.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap07 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object PrinterProblemOO { + class Printer { //#A + val powerButtonOn = Flip(0.95) //#A + val tonerLevel = Select(0.7 -> 'high, 0.2 -> 'low, 0.1 -> 'out) //#A + val tonerLowIndicatorOn = //#A + If(powerButtonOn, //#A + CPD(tonerLevel, //#A + 'high -> Flip(0.2), //#A + 'low -> Flip(0.6), //#A + 'out -> Flip(0.99)), //#A + Constant(false)) //#A + val paperFlow = Select(0.6 -> 'smooth, 0.2 -> 'uneven, 0.2 -> 'jammed) //#A + val paperJamIndicatorOn = //#A + If(powerButtonOn, //#A + CPD(paperFlow, //#A + 'smooth -> Flip(0.1), //#A + 'uneven -> Flip(0.3), //#A + 'jammed -> Flip(0.99)), //#A + Constant(false)) //#A + val state = //#A + Apply(powerButtonOn, tonerLevel, paperFlow, //#A + (power: Boolean, toner: Symbol, paper: Symbol) => { //#A + if (power) { //#A + if (toner == 'high && paper == 'smooth) 'good //#A + else if (toner == 'out || paper == 'out) 'out //#A + else 'poor //#A + } else 'out //#A + }) //#A + } //#A + + class Software { //#A + val state = Select(0.8 -> 'correct, 0.15 -> 'glitchy, 0.05 -> 'crashed) //#A + } //#A + + class Network { //#A + val state = Select(0.7 -> 'up, 0.2 -> 'intermittent, 0.1 -> 'down) //#A + } //#A + + class User { //#A + val commandCorrect = Flip(0.65) //#A + } //#A + + class PrintExperience(printer: Printer, software: Software, network: Network, user: User) { //#B + + val numPrintedPages = + RichCPD(user.commandCorrect, network.state, software.state, printer.state, //#C + (*, *, *, OneOf('out)) -> Constant('zero), + (*, *, OneOf('crashed), *) -> Constant('zero), + (*, OneOf('down), *, *) -> Constant('zero), + (OneOf(false), *, *, *) -> Select(0.3 -> 'zero, 0.6 -> 'some, 0.1 -> 'all), + (OneOf(true), *, *, *) -> Select(0.01 -> 'zero, 0.01 -> 'some, 0.98 -> 'all)) + val printsQuickly = + Chain(network.state, software.state, //#C + (network: Symbol, software: Symbol) => + if (network == 'down || software == 'crashed) Constant(false) + else if (network == 'intermittent || software == 'glitchy) Flip(0.5) + else Flip(0.9)) + val goodPrintQuality = + CPD(printer.state, //#C + 'good -> Flip(0.95), + 'poor -> Flip(0.3), + 'out -> Constant(false)) + val summary = + Apply(numPrintedPages, printsQuickly, goodPrintQuality, //#C + (pages: Symbol, quickly: Boolean, quality: Boolean) => + if (pages == 'zero) 'none + else if (pages == 'some || !quickly || !quality) 'poor + else 'excellent) + } + + val myPrinter = new Printer //#D + val mySoftware = new Software //#D + val myNetwork = new Network //#D + val me = new User //#D + val myExperience = new PrintExperience(myPrinter, mySoftware, myNetwork, me) //#D + + def step1() { + val answerWithNoEvidence = VariableElimination.probability(myPrinter.powerButtonOn, true) //#E + println("Prior probability the printer power button is on = " + answerWithNoEvidence) + } + + def step2() { + myExperience.summary.observe('poor) //#E + val answerIfPrintResultPoor = VariableElimination.probability(myPrinter.powerButtonOn, true) + println("Probability the printer power button is on given a poor result = " + answerIfPrintResultPoor) + } + + def main(args: Array[String]) { + step1() + step2() + } +} + +class PrinterProblemOOTest extends WordSpec with Matchers { + Universe.createNew() + val myPrinter = new PrinterProblemOO.Printer //#D + val mySoftware = new PrinterProblemOO.Software //#D + val myNetwork = new PrinterProblemOO.Network //#D + val me = new PrinterProblemOO.User //#D + val myExperience = new PrinterProblemOO.PrintExperience(myPrinter, mySoftware, myNetwork, me) //#D + + "Printer Problem OO" should { + val answerWithNoEvidence = VariableElimination.probability(myPrinter.powerButtonOn, true) //#E + + "produce a prior probability the printer power button is on = 0.95" taggedAs (BookExample) in { + answerWithNoEvidence should be(0.95) + } + + myExperience.summary.observe('poor) //#E + val answerIfPrintResultPoor = VariableElimination.probability(myPrinter.powerButtonOn, true) + + "produce a probability the printer power button is on given a poor result = 1.0" taggedAs (BookExample) in { + answerIfPrintResultPoor should be(1.0) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemTypeUncertainty.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemTypeUncertainty.scala new file mode 100644 index 00000000..6c9aac4b --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/PrinterProblemTypeUncertainty.scala @@ -0,0 +1,164 @@ +/* + * PrinterProblemTypeUncertainty.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap07 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object PrinterProblemTypeUncertainty extends ElementCollection { + abstract class Printer extends ElementCollection { + val powerButtonOn = Flip(0.95)("power button on", this) + + val paperFlow = Select(0.6 -> 'smooth, 0.2 -> 'uneven, 0.2 -> 'jammed) + val paperJamIndicatorOn = + If(powerButtonOn, + CPD(paperFlow, + 'smooth -> Flip(0.1), + 'uneven -> Flip(0.3), + 'jammed -> Flip(0.99)), + Constant(false)) + + val state: Element[Symbol] + } + + class LaserPrinter extends Printer { + val tonerLevel = Select(0.7 -> 'high, 0.2 -> 'low, 0.1 -> 'out) + val tonerLowIndicatorOn = + If(powerButtonOn, + CPD(tonerLevel, + 'high -> Flip(0.2), + 'low -> Flip(0.6), + 'out -> Flip(0.99)), + Constant(false)) + val state = + Apply(powerButtonOn, tonerLevel, paperFlow, + (power: Boolean, toner: Symbol, paper: Symbol) => { + if (power) { + if (toner == 'high && paper == 'smooth) 'good + else if (toner == 'out || paper == 'out) 'out + else 'poor + } else 'out + } + ) + } + + class InkjetPrinter extends Printer { + val inkCartridgeEmpty = Flip(0.1) + val inkCartridgeEmptyIndicator = If(inkCartridgeEmpty, Flip(0.99), Flip(0.3)) + val cloggedNozzle = Flip(0.001) + val state = + Apply(powerButtonOn, inkCartridgeEmpty, cloggedNozzle, paperFlow, + (power: Boolean, ink: Boolean, nozzle: Boolean, paper: Symbol) => { + if (power && !ink && !nozzle) { + if (paper == 'smooth) 'good + else if (paper == 'uneven) 'poor + else 'out + } else 'out + } + ) + } + + class Software { + val state = Select(0.8 -> 'correct, 0.15 -> 'glitchy, 0.05 -> 'crashed) + } + + class Network { + val state = Select(0.7 -> 'up, 0.2 -> 'intermittent, 0.1 -> 'down) + } + + class User { + val commandCorrect = Flip(0.65) + } + + class PrintExperience(printer: Printer, software: Software, network: Network, user: User) extends ElementCollection { + val numPrintedPages = + RichCPD(user.commandCorrect, network.state, software.state, printer.state, + (*, *, *, OneOf('out)) -> Constant('zero), + (*, *, OneOf('crashed), *) -> Constant('zero), + (*, OneOf('down), *, *) -> Constant('zero), + (OneOf(false), *, *, *) -> Select(0.3 -> 'zero, 0.6 -> 'some, 0.1 -> 'all), + (OneOf(true), *, *, *) -> Select(0.01 -> 'zero, 0.01 -> 'some, 0.98 -> 'all)) + val printsQuickly = + Chain(network.state, software.state, + (network: Symbol, software: Symbol) => + if (network == 'down || software == 'crashed) Constant(false) + else if (network == 'intermittent || software == 'glitchy) Flip(0.5) + else Flip(0.9)) + val goodPrintQuality = + CPD(printer.state, + 'good -> Flip(0.95), + 'poor -> Flip(0.3), + 'out -> Constant(false)) + val summary = + Apply(numPrintedPages, printsQuickly, goodPrintQuality, + (pages: Symbol, quickly: Boolean, quality: Boolean) => + if (pages == 'zero) 'none + else if (pages == 'some || !quickly || !quality) 'poor + else 'excellent)("summary", this) + } + + val myPrinter = Select(0.3 -> new LaserPrinter, 0.7 -> new InkjetPrinter)("my printer", this) + val mySoftware = new Software + val myNetwork = new Network + val me = new User + val myExperience = + Apply(myPrinter, (p: Printer) => new PrintExperience(p, mySoftware, myNetwork, me))("print experience", this) + + def step1() { + val summary = get[Symbol]("print experience.summary") + summary.observe('none) + + val powerButtonOn = get[Boolean]("my printer.power button on") + val isLaser = Apply(myPrinter, (p: Printer) => p.isInstanceOf[LaserPrinter]) + val alg = VariableElimination(powerButtonOn, isLaser) + alg.start() + println("After observing no print result:") + println("Probability printer power button is on = " + alg.probability(powerButtonOn, true)) + println("Probability printer is a laser printer = " + alg.probability(isLaser, true)) + alg.kill() + } + + def main(args: Array[String]) { + step1() + } +} + +class PrinterProblemTypeUncertaintyTest extends WordSpec with Matchers { + Universe.createNew() + val summary = PrinterProblemTypeUncertainty.get[Symbol]("print experience.summary") + summary.observe('none) + + val powerButtonOn = PrinterProblemTypeUncertainty.get[Boolean]("my printer.power button on") + val isLaser = Apply(PrinterProblemTypeUncertainty.myPrinter, (p: PrinterProblemTypeUncertainty.Printer) => p.isInstanceOf[PrinterProblemTypeUncertainty.LaserPrinter]) + val alg = VariableElimination(powerButtonOn, isLaser) + alg.start() + val btnOn = alg.probability(powerButtonOn, true) + val laserPrinter = alg.probability(isLaser, true) + alg.kill() + + "Printer Problem Type Uncertainty" should { + "after observing no print result:" should { + "produce a probability printer power button is on = 0.8868215502107176" taggedAs (BookExample) in { + btnOn should be(0.8868215502107176) + } + "produce a probability printer is a laser printer = 0.2380036100085 +- 0.00000000000001" taggedAs (BookExample) in { + laserPrinter should be(0.2380036100085 +- 0.00000000000001) + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaBasic.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaBasic.scala new file mode 100644 index 00000000..04e8db61 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaBasic.scala @@ -0,0 +1,128 @@ +/* + * SocialMediaBasic.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap07 + +import com.cra.figaro.language.{Flip, Universe} +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.compound.{If, ^^} +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.util.memo +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object SocialMediaBasic { + class Person() { + val interest = Uniform("sports", "politics") + } + + class Post(val poster: Person) { + val topic = If(Flip(0.9), poster.interest, Uniform("sports", "politics")) + } + + class Connection(person1: Person, person2: Person) { + val connectionType = Uniform("acquaintance", "close friend", "family") + } + def generateConnection(pair: (Person, Person)) = new Connection(pair._1, pair._2) + val connection = memo(generateConnection _) + + class Comment(val post: Post, val commenter: Person) { + val topicMatch = post.topic === commenter.interest + val pair = ^^(topicMatch, connection(post.poster, commenter).connectionType) + def constraint(pair: (Boolean, String)) = { + val (topicMatch, connectionType) = pair + if (topicMatch) 1.0 + else if (connectionType == "family") 0.8 + else if (connectionType == "close friend") 0.5 + else 0.1 + } + pair.addConstraint(constraint _) + } + + def main(args: Array[String]) { + val amy = new Person() + val brian = new Person() + val cheryl = new Person() + + val post1 = new Post(amy) + val post2 = new Post(brian) + val post3 = new Post(amy) + + val comment1 = new Comment(post1, brian) + val comment2 = new Comment(post1, cheryl) + val comment3 = new Comment(post2, amy) + val comment4 = new Comment(post3, cheryl) + + post1.topic.observe("politics") + post2.topic.observe("sports") + post3.topic.observe("politics") + + println("Probability Amy's interest is politics = " + VariableElimination.probability(amy.interest, "politics")) + println("Probability Brian's interest is politics = " + VariableElimination.probability(brian.interest, "politics")) + println("Probability Cheryl's interest is politics = " + VariableElimination.probability(cheryl.interest, "politics")) + println("Probability Brian is Amy's family = " + VariableElimination.probability(connection(amy, brian).connectionType, "family")) + println("Probability Cheryl is Amy's family = " + VariableElimination.probability(connection(amy, cheryl).connectionType, "family")) + println("Probability Cheryl is Brian's family = " + VariableElimination.probability(connection(brian, cheryl).connectionType, "family")) + } +} + +class SocialMediaBasicTest extends WordSpec with Matchers { + Universe.createNew() + val connection = memo(SocialMediaBasic.generateConnection _) + + val amy = new SocialMediaBasic.Person() + val brian = new SocialMediaBasic.Person() + val cheryl = new SocialMediaBasic.Person() + + val post1 = new SocialMediaBasic.Post(amy) + val post2 = new SocialMediaBasic.Post(brian) + val post3 = new SocialMediaBasic.Post(amy) + + val comment1 = new SocialMediaBasic.Comment(post1, brian) + val comment2 = new SocialMediaBasic.Comment(post1, cheryl) + val comment3 = new SocialMediaBasic.Comment(post2, amy) + val comment4 = new SocialMediaBasic.Comment(post3, cheryl) + + post1.topic.observe("politics") + post2.topic.observe("sports") + post3.topic.observe("politics") + + val amyPolitics = VariableElimination.probability(amy.interest, "politics") + val brianPolitics = VariableElimination.probability(brian.interest, "politics") + val cherylPolitics = VariableElimination.probability(cheryl.interest, "politics") + val brianAmyFamily = VariableElimination.probability(connection(amy, brian).connectionType, "family") + val cherylAmyFamily = VariableElimination.probability(connection(amy, cheryl).connectionType, "family") + val cherylBrianFamily = VariableElimination.probability(connection(brian, cheryl).connectionType, "family") + + "Social Media Basic" should { + "produce a probability Amy's interest is politics = 0.9940991345397325" taggedAs (BookExample) in { + amyPolitics should be(0.9940991345397325) + } + "produce a probability Brian's interest is politics = 0.10135135135135133" taggedAs (BookExample) in { + brianPolitics should be(0.10135135135135133) + } + "produce a probability Cheryl's interest is politics = 0.7692307692307692" taggedAs (BookExample) in { + cherylPolitics should be(0.7692307692307692) + } + "produce a probability Brian is Amy's family = 0.33333333333333337" taggedAs (BookExample) in { + brianAmyFamily should be(0.33333333333333337) + } + "produce a probability Cheryl is Amy's family = 0.3333333333333333" taggedAs (BookExample) in { + cherylAmyFamily should be(0.3333333333333333) + } + "produce a probability Cheryl is Brian's family = 0.3333333333333333" taggedAs (BookExample) in { + cherylBrianFamily should be(0.3333333333333333) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaMultipleInterests.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaMultipleInterests.scala new file mode 100644 index 00000000..64707434 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaMultipleInterests.scala @@ -0,0 +1,119 @@ +/* + * SocialMediaMultipleInterests.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap07 + +import com.cra.figaro.language.{Flip, Chain, Universe} +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.compound.{If, ^^} +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.util.memo +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object SocialMediaMultipleInterests { + val topics = List("sports", "politics", "movies") + + class Person() { + def generateInterest(topic: String) = Flip(0.4) + val interested = memo(generateInterest _) + } + + class Post(val poster: Person) { + val topic = Uniform(topics:_*) + val interestMatch = Chain(topic, (s: String) => poster.interested(s)) + interestMatch.addConstraint((b: Boolean) => if (b) 1.0 else 0.2) + } + + class Connection(person1: Person, person2: Person) { + val connectionType = Uniform("acquaintance", "close friend", "family") + } + def generateConnection(pair: (Person, Person)) = new Connection(pair._1, pair._2) + val connection = memo(generateConnection _) + + class Comment(val post: Post, val commenter: Person) { + val interestMatch = Chain(post.topic, (s: String) => commenter.interested(s)) + val pair = ^^(interestMatch, connection(post.poster, commenter).connectionType) + def constraint(pair: (Boolean, String)) = { + val (interestMatch, connectionType) = pair + if (interestMatch) 1.0 + else if (connectionType == "family") 0.8 + else if (connectionType == "close friend") 0.5 + else 0.1 + } + pair.addConstraint(constraint _) + } + + def main(args: Array[String]) { + val amy = new Person() + val brian = new Person() + val cheryl = new Person() + + val post1 = new Post(amy) + val post2 = new Post(brian) + val post3 = new Post(amy) + + val comment1 = new Comment(post1, brian) + val comment2 = new Comment(post1, cheryl) + val comment3 = new Comment(post2, amy) + val comment4 = new Comment(post3, cheryl) + + post1.topic.observe("politics") + post2.topic.observe("sports") + post3.topic.observe("politics") + + println("Probability Brian is interested in sports = " + VariableElimination.probability(brian.interested("sports"), true)) + println("Probability Cheryl is interested in sports = " + VariableElimination.probability(cheryl.interested("sports"), true)) + println("Probability Brian is Amy's family = " + VariableElimination.probability(connection(amy, brian).connectionType, "family")) + } +} + +class SocialMediaMultipleInterestsTest extends WordSpec with Matchers { + Universe.createNew() + val topics = List("sports", "politics", "movies") + val connection = memo(SocialMediaMultipleInterests.generateConnection _) + + val amy = new SocialMediaMultipleInterests.Person() + val brian = new SocialMediaMultipleInterests.Person() + val cheryl = new SocialMediaMultipleInterests.Person() + + val post1 = new SocialMediaMultipleInterests.Post(amy) + val post2 = new SocialMediaMultipleInterests.Post(brian) + val post3 = new SocialMediaMultipleInterests.Post(amy) + + val comment1 = new SocialMediaMultipleInterests.Comment(post1, brian) + val comment2 = new SocialMediaMultipleInterests.Comment(post1, cheryl) + val comment3 = new SocialMediaMultipleInterests.Comment(post2, amy) + val comment4 = new SocialMediaMultipleInterests.Comment(post3, cheryl) + + post1.topic.observe("politics") + post2.topic.observe("sports") + post3.topic.observe("politics") + + val brianSports = VariableElimination.probability(brian.interested("sports"), true) + val cherylSports = VariableElimination.probability(cheryl.interested("sports"), true) + val brianAmyFamily = VariableElimination.probability(connection(amy, brian).connectionType, "family") + + "Social Media Multiple Interests" should { + "produce a probability Brian's interest is sports = 0.7692307692307693" taggedAs (BookExample) in { + brianSports should be(0.7692307692307693) + } + "produce a probability Cheryl's interest is sports = 0.4" taggedAs (BookExample) in { + cherylSports should be(0.4) + } + "produce a probability Brian is Amy's family = 0.3333333333333333" taggedAs (BookExample) in { + brianAmyFamily should be(0.3333333333333333) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaTopicObjects.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaTopicObjects.scala new file mode 100644 index 00000000..ed7fec7e --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap07/SocialMediaTopicObjects.scala @@ -0,0 +1,180 @@ +/* + * SocialMediaTopicObjects.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap07 + +import com.cra.figaro.language.Universe +import com.cra.figaro.language.Flip +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.compound.{If, ^^} +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.language.ElementCollection +import com.cra.figaro.language.Apply +import com.cra.figaro.util.memo +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object SocialMediaTopicObjects { + class Topic() extends ElementCollection { + val hot = Flip(0.1)("hot", this) + } + + val sports = new Topic() + val politics = new Topic() + + class Person() { + val interest = Uniform(sports, politics) + } + + class Post(val poster: Person) extends ElementCollection { + val topic = If(Flip(0.9), poster.interest, Uniform(sports, politics))("topic", this) + } + + class Connection(person1: Person, person2: Person) { + val connectionType = Uniform("acquaintance", "close friend", "family") + } + def generateConnection(pair: (Person, Person)) = new Connection(pair._1, pair._2) + val connection = memo(generateConnection _) + + class Comment(val post: Post, val commenter: Person) { + val isHot = post.get[Boolean]("topic.hot") + val appropriateComment = + Apply(post.topic, commenter.interest, isHot, (t1: Topic, t2: Topic, b: Boolean) => (t1 == t2) || b) + val pair = ^^(appropriateComment, connection(post.poster, commenter).connectionType) + def constraint(pair: (Boolean, String)) = { + if (pair._1) 1.0 + else if (pair._2 == "family") 0.8 + else if (pair._2 == "close friend") 0.5 + else 0.1 + } + pair.addConstraint(constraint _) + } + + def main(args: Array[String]) { + val amy = new Person() + val brian = new Person() + val cheryl = new Person() + + val post1 = new Post(amy) + val post2 = new Post(brian) + val post3 = new Post(amy) + + val comment1 = new Comment(post1, brian) + val comment2 = new Comment(post1, cheryl) + val comment3 = new Comment(post2, amy) + val comment4 = new Comment(post3, cheryl) + + post1.topic.observe(politics) + + println("Prior probabilities") + println("Probability Amy's interest is politics = " + VariableElimination.probability(amy.interest, politics)) + println("Probability post 2's topic is sports = " + VariableElimination.probability(post2.topic, sports)) + println("Probability post 3's topic is sports = " + VariableElimination.probability(post3.topic, sports)) + println("Probability Amy is Brian's family = " + VariableElimination.probability(connection(brian, amy).connectionType, "family")) + + sports.hot.observe(true) + + println("\nAfter observing that sports is hot") + println("Probability Amy's interest is politics = " + VariableElimination.probability(amy.interest, politics)) + println("Probability post 2's topic is sports = " + VariableElimination.probability(post2.topic, sports)) + println("Probability post 3's topic is sports = " + VariableElimination.probability(post3.topic, sports)) + println("Probability Amy is Brian's family = " + VariableElimination.probability(connection(brian, amy).connectionType, "family")) + } +} + +class SocialMediaTopicObjectsTest extends WordSpec with Matchers { + Universe.createNew() + + class Topic() extends ElementCollection { + val hot = Flip(0.1)("hot", this) + } + + val sports = new Topic() + val politics = new Topic() + + class Person() { + val interest = Uniform(sports, politics) + } + + class Post(val poster: Person) extends ElementCollection { + val topic = If(Flip(0.9), poster.interest, Uniform(sports, politics))("topic", this) + } + + class Connection(person1: Person, person2: Person) { + val connectionType = Uniform("acquaintance", "close friend", "family") + } + def generateConnection(pair: (Person, Person)) = new Connection(pair._1, pair._2) + val connection = memo(generateConnection _) + + class Comment(val post: Post, val commenter: Person) { + val isHot = post.get[Boolean]("topic.hot") + val appropriateComment = + Apply(post.topic, commenter.interest, isHot, (t1: Topic, t2: Topic, b: Boolean) => (t1 == t2) || b) + val pair = ^^(appropriateComment, connection(post.poster, commenter).connectionType) + def constraint(pair: (Boolean, String)) = { + if (pair._1) 1.0 + else if (pair._2 == "family") 0.8 + else if (pair._2 == "close friend") 0.5 + else 0.1 + } + pair.addConstraint(constraint _) + } + + val amy = new Person() + val brian = new Person() + val cheryl = new Person() + + val post1 = new Post(amy) + val post2 = new Post(brian) + val post3 = new Post(amy) + + val comment1 = new Comment(post1, brian) + val comment2 = new Comment(post1, cheryl) + val comment3 = new Comment(post2, amy) + val comment4 = new Comment(post3, cheryl) + + post1.topic.observe(politics) + + "Social Media Topic Objects for prior probabilities" should { + "produce a probability Amy's interest is politics = 0.9656697011762803" taggedAs (BookExample) in { + VariableElimination.probability(amy.interest, politics) should be(0.9656697011762803) + } + "produce a probability post 2's topic is sports = 0.24190044937009827" taggedAs (BookExample) in { + VariableElimination.probability(post2.topic, sports) should be(0.24190044937009827) + } + "produce a probability post 3's topic is sports = 0.06958146174469142" taggedAs (BookExample) in { + VariableElimination.probability(post3.topic, sports) should be(0.06958146174469142) + } + "produce a probability Brian is Amy's family = 0.3791735849751334" taggedAs (BookExample) in { + VariableElimination.probability(connection(brian, amy).connectionType, "family") should be(0.3791735849751334) + } + } + + sports.hot.observe(true) + + "Social Media Topic Objects after observing that sports is hot" should { + "produce a probability Amy's interest is politics = 0.9609178093049061" taggedAs (BookExample) in { + VariableElimination.probability(amy.interest, politics) should be(0.9609178093049061) + } + "produce a probability post 2's topic is sports = 0.3729396845525878" taggedAs (BookExample) in { + VariableElimination.probability(post2.topic, sports) should be(0.3729396845525878) + } + "produce a probability post 3's topic is sports = 0.09005551240389947" taggedAs (BookExample) in { + VariableElimination.probability(post3.topic, sports) should be(0.09005551240389947) + } + "produce a probability Amy is Brian's family = 0.33670840928905443" taggedAs (BookExample) in { + VariableElimination.probability(connection(brian, amy).connectionType, "family") should be(0.33670840928905443) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/DynamicBayesianNetwork.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/DynamicBayesianNetwork.scala new file mode 100644 index 00000000..568e9b9d --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/DynamicBayesianNetwork.scala @@ -0,0 +1,84 @@ +/* + * DynamicBayesianNetwork.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap08 + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.library.compound.{If, CPD} +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object DynamicBayesianNetwork { + val length = 91 + val winning: Array[Element[String]] = Array.fill(length)(Constant("")) + val confident: Array[Element[Boolean]] = Array.fill(length)(Constant(false)) + val ourPossession: Array[Element[Boolean]] = Array.fill(length)(Constant(false)) + val goal: Array[Element[Boolean]] = Array.fill(length)(Constant(false)) + val scoreDifferential: Array[Element[Int]] = Array.fill(length)(Constant(0)) + confident(0) = Flip(0.4) + scoreDifferential(0) = Constant(0) + for { minute <- 1 until length } { + winning(minute) = + Apply(scoreDifferential(minute - 1), (i: Int) => if (i > 0) "us" else if (i < 0) "them" else "none") + confident(minute) = + CPD(confident(minute - 1), winning(minute), + (true, "us") -> Flip(0.9), + (true, "none") -> Flip(0.7), + (true, "them") -> Flip(0.5), + (false, "us") -> Flip(0.5), + (false, "none") -> Flip(0.3), + (false, "them") -> Flip(0.1)) + ourPossession(minute) = If(confident(minute), Flip(0.7), Flip(0.3)) + goal(minute) = + CPD(ourPossession(minute), confident(minute), + (true, true) -> Flip(0.04), + (true, false) -> Flip(0.01), + (false, true) -> Flip(0.045), + (false, false) -> Flip(0.02)) + scoreDifferential(minute) = + If(goal(minute), + Apply(ourPossession(minute), scoreDifferential(minute - 1), + (poss: Boolean, diff: Int) => if (poss) (diff + 1).min(5) else (diff - 1).max(-5)), + scoreDifferential(minute - 1)) + } + + def main(args: Array[String]) { + println("Probability we will win the game") + println("Prior probability: " + + VariableElimination.probability(scoreDifferential(length - 1), (i: Int) => i > 0)) + ourPossession(4).observe(true) + goal(4).observe(true) + println("After observing that we scored at time step 4: " + + VariableElimination.probability(scoreDifferential(length - 1), (i: Int) => i > 0)) + } +} + +class DynamicBayesianNetworkTest extends WordSpec with Matchers { + Universe.createNew() + val prior = VariableElimination.probability(DynamicBayesianNetwork.scoreDifferential(DynamicBayesianNetwork.length - 1), (i: Int) => i > 0) + DynamicBayesianNetwork.ourPossession(4).observe(true) + DynamicBayesianNetwork.goal(4).observe(true) + val afterScore = VariableElimination.probability(DynamicBayesianNetwork.scoreDifferential(DynamicBayesianNetwork.length - 1), (i: Int) => i > 0) + + "Dynamic Bayesian Network" should { + "produce a prior probability we will win the game = 0.400617362192 +- 0.000000000002" taggedAs (BookExample) in { + prior should be(0.400617362192 +- 0.000000000002) + } + "produce a probability after observing that we scored at time step 4 = 0.731889855275 +- 0.000000000002" taggedAs (BookExample) in { + afterScore should be(0.731889855275 +- 0.000000000002) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/HiddenMarkovModel.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/HiddenMarkovModel.scala new file mode 100644 index 00000000..ecc4436c --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/HiddenMarkovModel.scala @@ -0,0 +1,86 @@ +/* + * HiddenMarkovModel.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap08 + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object HiddenMarkovModel { + val length = 90 + val confident: Array[Element[Boolean]] = Array.fill(length)(Constant(false)) + val ourPossession: Array[Element[Boolean]] = Array.fill(length)(Constant(false)) + confident(0) = Flip(0.4) + for { minute <- 1 until length } { + confident(minute) = If(confident(minute - 1), Flip(0.6), Flip(0.3)) + } + for { minute <- 0 until length } { + ourPossession(minute) = If(confident(minute), Flip(0.7), Flip(0.3)) + } + + def main(args: Array[String]) { + println("Probability we are confident at time step 2") + println("Prior probability: " + VariableElimination.probability(confident(2), true)) + ourPossession(2).observe(true) + println("After observing current possession at time step 2: " + VariableElimination.probability(confident(2), true)) + ourPossession(1).observe(true) + println("After observing previous possession at time step 1: " + VariableElimination.probability(confident(2), true)) + ourPossession(0).observe(true) + println("After observing previous possession at time step 0: " + VariableElimination.probability(confident(2), true)) + ourPossession(3).observe(true) + println("After observing future possession at time step 3: " + VariableElimination.probability(confident(2), true)) + ourPossession(4).observe(true) + println("After observing future possession at time step 4: " + VariableElimination.probability(confident(2), true)) + } +} + +class HiddenMarkovModelTest extends WordSpec with Matchers { + Universe.createNew() + val prior = VariableElimination.probability(HiddenMarkovModel.confident(2), true) + HiddenMarkovModel.ourPossession(2).observe(true) + val posStep2 = VariableElimination.probability(HiddenMarkovModel.confident(2), true) + HiddenMarkovModel.ourPossession(1).observe(true) + val posStep1 = VariableElimination.probability(HiddenMarkovModel.confident(2), true) + HiddenMarkovModel.ourPossession(0).observe(true) + val posStep0 = VariableElimination.probability(HiddenMarkovModel.confident(2), true) + HiddenMarkovModel.ourPossession(3).observe(true) + val posStep3 = VariableElimination.probability(HiddenMarkovModel.confident(2), true) + HiddenMarkovModel.ourPossession(4).observe(true) + val posStep4 = VariableElimination.probability(HiddenMarkovModel.confident(2), true) + + "Hidden Markov Model" should { + "produce a prior probability we are confident at time step 2: 0.426 +- .001" taggedAs (BookExample) in { + prior should be(0.426 +- .001) + } + "after observing current possession at time step 2: 0.6339285714285714" taggedAs (BookExample) in { + posStep2 should be(0.6339285714285714) + } + "after observing previous possession at time step 1: 0.6902173913043478" taggedAs (BookExample) in { + posStep1 should be(0.6902173913043478) + } + "after observing previous possession at time step 0: 0.7046460176991151" taggedAs (BookExample) in { + posStep0 should be(0.7046460176991151) + } + "after observing future possession at time step 3: 0.7541436464088398" taggedAs (BookExample) in { + posStep3 should be(0.7541436464088398) + } + "after observing future possession at time step 4: 0.7663786503335885" taggedAs (BookExample) in { + posStep4 should be(0.7663786503335885) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/MarkovChain.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/MarkovChain.scala new file mode 100644 index 00000000..2e18bfa5 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/MarkovChain.scala @@ -0,0 +1,75 @@ +/* + * MarkovChain.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap08 + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.library.compound.If +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object MarkovChain { + val length = 90 + val ourPossession: Array[Element[Boolean]] = Array.fill(length)(Constant(false)) + ourPossession(0) = Flip(0.5) + for { minute <- 1 until length } { + ourPossession(minute) = If(ourPossession(minute - 1), Flip(0.6), Flip(0.3)) + } + + def main(args: Array[String]) { + println("Probability we have possession at time step 5") + println("Prior probability: " + VariableElimination.probability(ourPossession(5), true)) + ourPossession(4).observe(true) + println("After observing that we have possession at time step 4: " + VariableElimination.probability(ourPossession(5), true)) + ourPossession(3).observe(true) + println("After observing that we have possession at time step 3: " + VariableElimination.probability(ourPossession(5), true)) + ourPossession(6).observe(true) + println("After observing that we have possession at time step 6: " + VariableElimination.probability(ourPossession(5), true)) + ourPossession(7).observe(true) + println("After observing that we have possession at time step 7: " + VariableElimination.probability(ourPossession(5), true)) + } +} + +class MarkovChainTest extends WordSpec with Matchers { + Universe.createNew() + val prior = VariableElimination.probability(MarkovChain.ourPossession(5), true) + MarkovChain.ourPossession(4).observe(true) + val posStep2 = VariableElimination.probability(MarkovChain.ourPossession(5), true) + MarkovChain.ourPossession(3).observe(true) + val posStep1 = VariableElimination.probability(MarkovChain.ourPossession(5), true) + MarkovChain.ourPossession(6).observe(true) + val posStep0 = VariableElimination.probability(MarkovChain.ourPossession(5), true) + MarkovChain.ourPossession(7).observe(true) + val posStep3 = VariableElimination.probability(MarkovChain.ourPossession(5), true) + + "Markov Chain" should { + "produce a probability we have posession at time step 5: 0.42874500000000004" taggedAs (BookExample) in { + prior should be(0.42874500000000004) + } + "after observing that we have possession at time step 4: 0.6" taggedAs (BookExample) in { + posStep2 should be(0.6) + } + "after observing that we have possession at time step 3: 0.600 +- 0.001" taggedAs (BookExample) in { + posStep1 should be(0.600 +- 0.001) + } + "after observing that we have possession at time step 6: 0.75" taggedAs (BookExample) in { + posStep0 should be(0.75) + } + "after observing that we have possession at time step 7: 0.75" taggedAs (BookExample) in { + posStep3 should be(0.75) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/Restaurant.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/Restaurant.scala new file mode 100644 index 00000000..f76c0464 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/Restaurant.scala @@ -0,0 +1,81 @@ +/* + * Restaurant.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap08 + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Normal +import com.cra.figaro.library.atomic.discrete.{FromRange, Poisson} +import com.cra.figaro.library.compound.{If, ^^} +import com.cra.figaro.algorithm.sampling.Importance +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object Restaurant { + val numSteps = 12 + val capacity = 10 + + val seated: Array[Element[List[Int]]] = Array.fill(numSteps)(Constant(List())) + val waiting: Array[Element[Int]] = Array.fill(numSteps)(Constant(0)) + seated(0) = Constant(List(0, 5, 15, 15, 25, 30, 40, 60, 65, 75)) + waiting(0) = Constant(3) + + def transition(seated: List[Int], waiting: Int): (Element[(List[Int], Int)]) = { + val newTimes: List[Element[Int]] = + for { time <- seated } + yield Apply(Flip(time / 80.0), (b: Boolean) => if (b) -1 else time + 5) + val newTimesListElem: Element[List[Int]] = Inject(newTimes:_*) + val staying = Apply(newTimesListElem, (l: List[Int]) => l.filter(_ >= 0)) + + val arriving = Poisson(2) + val totalWaiting = Apply(arriving, (i: Int) => i + waiting) + val placesOccupied = Apply(staying, (l: List[Int]) => l.length.min(capacity)) + val placesAvailable = Apply(placesOccupied, (i: Int) => capacity - i) + val numNewlySeated = Apply(totalWaiting, placesAvailable, (tw: Int, pa: Int) => tw.min(pa)) + + val newlySeated = Apply(numNewlySeated, (i: Int) => List.fill(i)(0)) + val allSeated = Apply(newlySeated, staying, (l1: List[Int], l2: List[Int]) => l1 ::: l2) + val newWaiting = Apply(totalWaiting, numNewlySeated, (tw: Int, ns: Int) => tw - ns) + ^^(allSeated, newWaiting) + } + + for { step <- 1 until numSteps } { + val newState = + Chain(seated(step - 1), waiting(step - 1), (l: List[Int], i: Int) => transition(l, i)) + seated(step) = newState._1 + waiting(step) = newState._2 + } + + def main(args: Array[String]) { + val alg = Importance(10000, waiting(numSteps - 1)) + alg.start() + println(alg.probability(waiting(numSteps - 1))(_ > 4)) + alg.kill() + } +} + +class RestaurantTest extends WordSpec with Matchers { + Universe.createNew() + "Restaurant" should { + "produce a probability = 0.465 +- 0.01" taggedAs (BookExample, NonDeterministic) in { + val alg = Importance(10000, Restaurant.waiting(Restaurant.numSteps - 1)) + alg.start() + val prob = alg.probability(Restaurant.waiting(Restaurant.numSteps - 1))(_ > 4) + alg.kill() + + prob should be(0.465 +- 0.01) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/RestaurantMonitoring.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/RestaurantMonitoring.scala new file mode 100644 index 00000000..0968060b --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap08/RestaurantMonitoring.scala @@ -0,0 +1,103 @@ +/* + * RestaurantMonitoring.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap08 + +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Normal +import com.cra.figaro.library.atomic.discrete.{FromRange, Poisson} +import com.cra.figaro.library.compound.{If, ^^} +import com.cra.figaro.algorithm.filtering.ParticleFilter +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object RestaurantMonitoring { + val capacity = 10 + + val initial = Universe.createNew() + Constant(List(0, 5, 15, 15, 25, 30, 40, 60, 65, 75))("seated", initial) + Constant(3)("waiting", initial) + Constant(0)("arriving", initial) + + def transition(seated: List[Int], waiting: Int): (Element[(List[Int], Int, Int)]) = { + val newTimes: List[Element[Int]] = + for { time <- seated } + yield Apply(Flip(time / 80.0), (b: Boolean) => if (b) -1 else time + 5) + val newTimesListElem: Element[List[Int]] = Inject(newTimes:_*) + val staying = Apply(newTimesListElem, (l: List[Int]) => l.filter(_ >= 0)) + + val arriving = Poisson(2) + val totalWaiting = arriving.map(_ + waiting) + val placesOccupied = staying.map(_.length.min(capacity)) + val placesAvailable = placesOccupied.map(capacity - _) + val numNewlySeated = Apply(totalWaiting, placesAvailable, (tw: Int, pa: Int) => tw.min(pa)) + + val newlySeated = numNewlySeated.map(List.fill(_)(0)) + val allSeated = Apply(newlySeated, staying, (l1: List[Int], l2: List[Int]) => l1 ::: l2) + val newWaiting = Apply(totalWaiting, numNewlySeated, (tw: Int, ns: Int) => tw - ns) + ^^(allSeated, newWaiting, arriving) + } + + def nextUniverse(previous: Universe): Universe = { + val next = Universe.createNew() + val previousSeated = previous.get[List[Int]]("seated") + val previousWaiting = previous.get[Int]("waiting") + val newState = Chain(previousSeated, previousWaiting, transition _) + Apply(newState, (s: (List[Int], Int, Int)) => s._1)("seated", next) + Apply(newState, (s: (List[Int], Int, Int)) => s._2)("waiting", next) + Apply(newState, (s: (List[Int], Int, Int)) => s._3)("arriving", next) + next + } + + def main(args: Array[String]) { + val arrivingObservation = List(None, None, Some(1), None, None, Some(2), None, Some(0), Some(3), None, None, None, Some(1)) + val alg = ParticleFilter(initial, nextUniverse, 10000) + alg.start() + for { time <- 1 to 12 } { + val evidence = { + arrivingObservation(time) match { + case None => List() + case Some(n) => List(NamedEvidence("arriving", Observation(n))) + } + } + alg.advanceTime(evidence) + print("Time " + time + ": ") + print("expected customers = " + alg.currentExpectation("seated", (l: List[Int]) => l.length)) + println(", expected waiting = " + alg.currentExpectation("waiting", (i: Int) => i)) + } + } +} + +class RestaurantMonitoringTest extends WordSpec with Matchers { + "Restaurant Monitoring" should { + "produce the correct results" taggedAs (BookExample, NonDeterministic) in { + val arrivingObservation = List(None, None, Some(1), None, None, Some(2), None, Some(0), Some(3), None, None, None, Some(1)) + val alg = ParticleFilter(RestaurantMonitoring.initial, RestaurantMonitoring.nextUniverse, 10000) + alg.start() + for { time <- 1 to 12 } { + val evidence = { + arrivingObservation(time) match { + case None => List() + case Some(n) => List(NamedEvidence("arriving", Observation(n))) + } + } + alg.advanceTime(evidence) + print("Time " + time + ": ") + print("expected customers = " + alg.currentExpectation("seated", (l: List[Int]) => l.length)) + println(", expected waiting = " + alg.currentExpectation("waiting", (i: Int) => i)) + } + } + } +} \ No newline at end of file diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap11/ActorsAndMovies.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap11/ActorsAndMovies.scala new file mode 100644 index 00000000..52b63245 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap11/ActorsAndMovies.scala @@ -0,0 +1,101 @@ +/* + * ActorsAndMovies.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap11 + +import com.cra.figaro.language._ +import com.cra.figaro.algorithm.sampling._ +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object ActorsAndMovies { + val random = new scala.util.Random() + + class Actor { + val famous = Flip(0.1) + } + + class Movie { + val quality = Select(0.3 -> 'low, 0.5 -> 'medium, 0.2 -> 'high) + } + + class Appearance(val actor: Actor, val movie: Movie) { + def getProb(quality: Symbol, famous: Boolean) = + (quality, famous) match { + case ('low, false) => 0.001 + case ('low, true) => 0.01 + case ('medium, false) => 0.01 + case ('medium, true) => 0.05 + case ('high, false) => 0.05 + case ('high, true) => 0.2 + } + val probability = + Apply(movie.quality, actor.famous, (q: Symbol, f: Boolean) => getProb(q, f)) + val award: Element[Boolean] = + Flip(probability) + } + + val numActors = 200 + val numMovies = 100 + val numAppearances = 300 + + val actors = Array.fill(numActors)(new Actor) + val movies = Array.fill(numMovies)(new Movie) + val appearances = Array.fill(numAppearances)(new Appearance(actors(random.nextInt(numActors)), movies(random.nextInt(numMovies)))) + + // Ensure that exactly one appearance gets an award. + def uniqueAwardConstraint(awards: List[Boolean]) = { + val n = awards.count(b => b) + if (n == 0) 0.0 else 1.0 / (n * n) + } + val allAwards: Element[List[Boolean]] = Inject(appearances.map(_.award): _*) + allAwards.setConstraint(uniqueAwardConstraint) + + val scheme: ProposalScheme = { + DisjointScheme( + (0.5, () => ProposalScheme(appearances(random.nextInt(numAppearances)).award, appearances(random.nextInt(numAppearances)).award)), + (0.25, () => ProposalScheme(actors(random.nextInt(numActors)).famous)), + (0.25, () => ProposalScheme(movies(random.nextInt(numMovies)).quality))) + } + + def main(args: Array[String]): Unit = { + appearances(0).actor.famous.observe(true) + appearances(0).movie.quality.observe('high) + + val alg = MetropolisHastings(1000000, scheme, 100000, appearances(0).award) + alg.start() + println("Probability the first appearance gets an award, given that the actor is famous and the movie is high quality = " + + alg.probability(appearances(0).award, true)) + alg.kill() + } +} + +class ActorsAndMoviesTest extends WordSpec with Matchers { + Universe.createNew() + ActorsAndMovies.appearances(0).actor.famous.observe(true) + ActorsAndMovies.appearances(0).movie.quality.observe('high) + + val alg = MetropolisHastings(1000000, ActorsAndMovies.scheme, 100000, ActorsAndMovies.appearances(0).award) + alg.start() + val prob = alg.probability(ActorsAndMovies.appearances(0).award, true) + alg.kill() + + "Actors and Movies" should { + "produce a probability the first appearance gets an award, given that the actor is famous and the movie is high quality = 0.12 +- 0.02" taggedAs (BookExample, NonDeterministic) in { + prob should be(0.12 +- 0.02) + } + } +} + diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap11/TwoVariableMCMC.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap11/TwoVariableMCMC.scala new file mode 100644 index 00000000..2973cfde --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap11/TwoVariableMCMC.scala @@ -0,0 +1,50 @@ +/* + * TwoVariableMCMC.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap11 + +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Normal +import com.cra.figaro.library.compound.^^ +import com.cra.figaro.algorithm.sampling.MetropolisHastings +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object TwoVariableMCMC { + def main(args: Array[String]) { + val x = Normal(0.75, 0.2) + val y = Normal(0.4, 0.2) + x.setCondition((d: Double) => d > 0 && d < 1) + y.setCondition((d: Double) => d > 0 && d < 1) + val pair = ^^(x, y) + println(MetropolisHastings.probability(pair, (xy: (Double, Double)) => xy._1 > 0.5 && xy._2 > 0.5)) + } +} + +class TwoVariableMCMCTest extends WordSpec with Matchers { + Universe.createNew() + val x = Normal(0.75, 0.2) + val y = Normal(0.4, 0.2) + x.setCondition((d: Double) => d > 0 && d < 1) + y.setCondition((d: Double) => d > 0 && d < 1) + val pair = ^^(x, y) + val prob = MetropolisHastings.probability(pair, (xy: (Double, Double)) => xy._1 > 0.5 && xy._2 > 0.5) + + "Two Variable MCMC" should { + "produce a probability = 0.283 +- 0.002" taggedAs (BookExample, NonDeterministic) in { + prob should be(0.283 +- 0.002) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/JointDistribution.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/JointDistribution.scala new file mode 100644 index 00000000..78118343 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/JointDistribution.scala @@ -0,0 +1,92 @@ +/* + * JointDistribution.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap12 + +import com.cra.figaro.language.{Element, Flip, Universe} +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.compound.{If, ^^} +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.library.collection.Container +import com.cra.figaro.algorithm.sampling.Importance +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object JointDistribution { + val economicClimateGood = Flip(0.5) + def makeSales: Element[Int] = If(economicClimateGood, Uniform(80, 120, 160), Uniform(60, 90, 120)) + val sales = Array.fill(20)(makeSales) + val totalSales = Container(sales:_*).reduce(_ + _) + + def main(args: Array[String]) { + val salesPair = ^^(sales(0), sales(1)) + val ve = VariableElimination(sales(0), sales(1), salesPair) + ve.start() + val imp = Importance(10000, totalSales) + imp.start() + println("Probability first sales will be less than 100 = " + + ve.probability(sales(0))(_ < 100)) + println("Probability second sales will be less than 100 = " + + ve.probability(sales(1))(_ < 100)) + println("Probability both sales will be less than 100 = " + + ve.probability(salesPair)(pair => pair._1 < 100 && pair._2 < 100)) + println("Probability total sales are less than 2000 = " + + imp.probability(totalSales)(_ < 2000)) + println("Mean individual sales = " + + ve.expectation(sales(0), (i: Int) => i.toDouble)) + println("Mean total sales = " + + imp.expectation(totalSales, (i: Int) => i.toDouble)) + ve.kill() + imp.kill() + } +} + +class JointDistributionTest extends WordSpec with Matchers { + Universe.createNew() + val salesPair = ^^(JointDistribution.sales(0), JointDistribution.sales(1)) + val ve = VariableElimination(JointDistribution.sales(0), JointDistribution.sales(1), salesPair) + ve.start() + val imp = Importance(10000, JointDistribution.totalSales) + imp.start() + val firstSales = ve.probability(JointDistribution.sales(0), (i: Int) => i < 100) + val secondSales = ve.probability(JointDistribution.sales(1))(_ < 100) + val bothSales = ve.probability(salesPair, (pair: (Int, Int)) => pair._1 < 100 && pair._2 < 100) + val totalSales = imp.probability(JointDistribution.totalSales)(_ < 2000) + val meanIndSales = ve.expectation(JointDistribution.sales(0))(i => i.toDouble) + val meanTotSales = imp.expectation(JointDistribution.totalSales, (i: Int) => i.toDouble) + ve.kill() + imp.kill() + + "Joint Distribution" should { + "produce a probability first sales will be less than 100 = 0.5" taggedAs (BookExample) in { + firstSales should be(0.5) + } + "produce a probability second sales will be less than 100 = 0.5" taggedAs (BookExample) in { + secondSales should be(0.5) + } + "produce a probability both sales will be less than 100 = 0.2777777777777778" taggedAs (BookExample) in { + bothSales should be(0.2777777777777778) + } + "produce a probability total sales are less than 2000 = 0.48 +- 0.02" taggedAs (BookExample, NonDeterministic) in { + totalSales should be(0.48 +- 0.02) + } + "produce mean individual sales = 105.0" taggedAs (BookExample) in { + meanIndSales should be(105.0) + } + "produce mean total sales = 2090 +- 20.0" taggedAs (BookExample, NonDeterministic) in { + meanTotSales should be(2090.0 +- 20.0) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/MPE.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/MPE.scala new file mode 100644 index 00000000..129a5d1e --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/MPE.scala @@ -0,0 +1,88 @@ +/* + * MPE.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap12 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound.^^ +import com.cra.figaro.algorithm.OneTimeMPE +import com.cra.figaro.algorithm.factored.MPEVariableElimination +import com.cra.figaro.algorithm.factored.beliefpropagation.MPEBeliefPropagation +import com.cra.figaro.algorithm.sampling.MetropolisHastingsAnnealer +import com.cra.figaro.algorithm.sampling.ProposalScheme +import com.cra.figaro.algorithm.sampling.Schedule +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object MPE { + val pixels = Array.fill(4, 4)(Flip(0.4)) + + def makeConstraint(pixel1: Element[Boolean], pixel2: Element[Boolean]) { + val pairElem = ^^(pixel1, pixel2) + pairElem.setConstraint(pair => if (pair._1 == pair._2) 1.0 else 0.5) + } + + for { + i <- 0 until 4 + j <- 0 until 4 + } { + if (i > 0) makeConstraint(pixels(i-1)(j), pixels(i)(j)) + if (j > 0) makeConstraint(pixels(i)(j-1), pixels(i)(j)) + } + + pixels(0)(0).observe(true) + pixels(0)(2).observe(false) + pixels(1)(1).observe(true) + pixels(2)(0).observe(true) + pixels(2)(3).observe(false) + pixels(3)(1).observe(true) + + def run(algorithm: OneTimeMPE) { + algorithm.start() + for { i <- 0 until 4 } { + for { j <- 0 until 4 } { + print(algorithm.mostLikelyValue(pixels(i)(j))) + print("\t") + } + println() + } + println() + algorithm.kill() + } + + def main(args: Array[String]) { + println("MPE variable elimination") + run(MPEVariableElimination()) + println("MPE belief propagation") + run(MPEBeliefPropagation(10)) + println("Simulated annealing") + run(MetropolisHastingsAnnealer(100000, ProposalScheme.default, Schedule.default(1.0))) + } +} + +class MPETest extends WordSpec with Matchers { + Universe.createNew() + "MPE" should { + "produce the correct result with Variable Elimination" taggedAs (BookExample, NonDeterministic) in { + MPE.run(MPEVariableElimination()) + } + "produce the correct result with Belief Propagation" taggedAs (BookExample, NonDeterministic) in { + MPE.run(MPEBeliefPropagation(10)) + } + "produce the correct result with Simulated Annealing" taggedAs (BookExample, NonDeterministic) in { + MPE.run(MetropolisHastingsAnnealer(100000, ProposalScheme.default, Schedule.default(1.0))) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/ProbabilityOfEvidence.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/ProbabilityOfEvidence.scala new file mode 100644 index 00000000..0ec683a4 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap12/ProbabilityOfEvidence.scala @@ -0,0 +1,69 @@ +/* + * ProbabilityOfEvidence.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap12 + +import com.cra.figaro.language._ +import com.cra.figaro.library.compound.^^ +import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler +import com.cra.figaro.algorithm.factored.beliefpropagation.ProbEvidenceBeliefPropagation +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object ProbabilityOfEvidence { + val pixels = Array.tabulate(4, 4)((i: Int, j: Int) => Flip(0.4)("pixel(" + i + "," + j + ")", Universe.universe)) + + def makeConstraint(pixel1: Element[Boolean], pixel2: Element[Boolean]) { + val pairElem = ^^(pixel1, pixel2) + pairElem.setConstraint(pair => if (pair._1 == pair._2) 1.0 else 0.5) + } + + for { + i <- 0 until 4 + j <- 0 until 4 + } { + if (i > 0) makeConstraint(pixels(i-1)(j), pixels(i)(j)) + if (j > 0) makeConstraint(pixels(i)(j-1), pixels(i)(j)) + } + + def makeNamedEvidence(i: Int, j: Int, obs: Boolean) = + NamedEvidence("pixel(" + i + "," + j + ")", Observation(true)) + val evidence = + List(makeNamedEvidence(0, 0, true), + makeNamedEvidence(0, 2, false), + makeNamedEvidence(1, 1, true), + makeNamedEvidence(2, 0, true), + makeNamedEvidence(2, 3, false), + makeNamedEvidence(3, 1, true)) + + def main(args: Array[String]) { + println("Probability of evidence sampling") + println(ProbEvidenceSampler.computeProbEvidence(100000, evidence)) + println("Probability of evidence belief propagation") + println(ProbEvidenceBeliefPropagation.computeProbEvidence(20, evidence)) + } +} + +class ProbabilityOfEvidenceTest extends WordSpec with Matchers { + Universe.createNew() + "Probability of Evidence" should { + "produce a probability of evidence sampling = 0.002 +- 0.001" taggedAs (BookExample, NonDeterministic) in { + ProbEvidenceSampler.computeProbEvidence(100000, ProbabilityOfEvidence.evidence) should be (0.002 +- 0.001) + } + "produce a probability of evidence belief propagation = 4.676522804 +- 0.000000001" taggedAs (BookExample, NonDeterministic) in { + ProbEvidenceBeliefPropagation.computeProbEvidence(20, ProbabilityOfEvidence.evidence) should be(4.676522804 +- 0.000000001) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/BayesianLearning.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/BayesianLearning.scala new file mode 100644 index 00000000..e0e24cc4 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/BayesianLearning.scala @@ -0,0 +1,102 @@ +/* + * BayesianLearning.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap13 + +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.language.Flip +import com.cra.figaro.library.compound.If +import com.cra.figaro.language.Element +import com.cra.figaro.algorithm.sampling.MetropolisHastings +import com.cra.figaro.algorithm.sampling.ProposalScheme +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample +import com.cra.figaro.test.tags.NonDeterministic + +object BayesianLearning { + class Email(val text: List[String], val label: String) + + // In Chapter 3, the emails were drawn automatically from the training and test sets. + // They are hardcoded here to keep the example simple. + val trainingEmail1 = new Email(List("Hello, would you like to receive a free book?"), "spam") + val trainingEmail2 = new Email(List("Please reply to my question by Tuesday!"), "normal") + val trainingEmail3 = new Email(List("Hello, I have a question about your book."), "normal") + val trainingEmails = List(trainingEmail1, trainingEmail2, trainingEmail3) + + // Illustrative list of feature words. In Chapter 3, the feature words were derived from the training emails. + val featureWords = List("hello", "reply", "question", "free", "book") + val spamProbability = Beta(2,3) + + val wordGivenSpamProbabilities = featureWords.map(word => (word, Beta(2,2))).toMap + val wordGivenNormalProbabilities = featureWords.map(word => (word, Beta(2,2))).toMap + + class EmailModel { + val isSpam = Flip(spamProbability) + + val hasWordElements = { + for { word <- featureWords } yield { + val givenSpamProbability = wordGivenSpamProbabilities(word) + val givenNormalProbability = wordGivenNormalProbabilities(word) + val hasWord = + If(isSpam, Flip(givenSpamProbability), Flip(givenNormalProbability)) + (word, hasWord) + } + } + + val hasWord = hasWordElements.toMap + } + + for { email <- trainingEmails } { + val model = new EmailModel + for { word <- featureWords } { + model.hasWord(word).observe(email.text.contains(word)) + } + model.isSpam.observe(email.label == "spam") + } + + def main(args: Array[String]) { + val futureEmail = new Email(List("Feel free to reply if you have any ideas."), "unknown") + val futureModel = new EmailModel + for { word <- featureWords } { + futureModel.hasWord(word).observe(futureEmail.text.contains(word)) + } + + val alg = MetropolisHastings(1000000, ProposalScheme.default, 100000, futureModel.isSpam) + alg.start() + val result = alg.probability(futureModel.isSpam, true) + println("Probability new email is spam = " + result) + alg.kill() + } +} + +class BayesianLearningTest extends WordSpec with Matchers { + Universe.createNew() + val futureEmail = new BayesianLearning.Email(List("Feel free to reply if you have any ideas."), "unknown") + val futureModel = new BayesianLearning.EmailModel + for { word <- BayesianLearning.featureWords } { + futureModel.hasWord(word).observe(futureEmail.text.contains(word)) + } + + val alg = MetropolisHastings(1000000, ProposalScheme.default, 100000, futureModel.isSpam) + alg.start() + val result = alg.probability(futureModel.isSpam, true) + alg.kill() + + "Bayesian Learning" should { + "produce a probability new email is spam = 0.25 +- 0.03" taggedAs (BookExample, NonDeterministic) in { + result should be (0.25 +- 0.03) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/MAPLearning.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/MAPLearning.scala new file mode 100644 index 00000000..e069e362 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/MAPLearning.scala @@ -0,0 +1,119 @@ +/* + * MAPLearning.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap13 + +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Beta +import com.cra.figaro.language.Flip +import com.cra.figaro.library.compound.If +import com.cra.figaro.language.Element +import com.cra.figaro.algorithm.sampling.MetropolisHastings +import com.cra.figaro.algorithm.sampling.ProposalScheme +import com.cra.figaro.patterns.learning.ModelParameters +import com.cra.figaro.patterns.learning.ParameterCollection +import com.cra.figaro.algorithm.learning.EMWithBP +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.algorithm.learning.EMWithVE +import com.cra.figaro.algorithm.learning.ExpectationMaximization +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object MAPLearning { + class Email(val text: List[String], val label: String) + + // In Chapter 3, the emails were drawn automatically from the training and test sets. + // They are hardcoded here to keep the example simple. + val trainingEmail1 = new Email(List("Hello, would you like to receive a free book?"), "spam") + val trainingEmail2 = new Email(List("Please reply to my question by Tuesday!"), "normal") + val trainingEmail3 = new Email(List("Hello, I have a question about your book."), "normal") + val trainingEmails = List(trainingEmail1, trainingEmail2, trainingEmail3) + + // Illustrative list of feature words. In Chapter 3, the feature words were derived from the training emails. + val featureWords = List("hello", "reply", "question", "free", "book") + + val params = ModelParameters() + val spamProbability = Beta(2,3)("spam probability", params) + val wordGivenSpamProbabilities = + featureWords.map(word => (word, Beta(2,2)(word + " given spam", params))).toMap + val wordGivenNormalProbabilities = + featureWords.map(word => (word, Beta(2,2)(word + " given normal", params))).toMap + + class EmailModel(paramCollection: ParameterCollection) { + val isSpam = Flip(paramCollection.get("spam probability")) + + val hasWordElements = { + for { word <- featureWords } yield { + val givenSpamProbability = paramCollection.get(word + " given spam") + val givenNormalProbability = paramCollection.get(word + " given normal") + val hasWord = + If(isSpam, Flip(givenSpamProbability), Flip(givenNormalProbability)) + (word, hasWord) + } + } + + val hasWord = hasWordElements.toMap + } + + for { email <- trainingEmails } { + val model = new EmailModel(params.priorParameters) + for { word <- featureWords } { + model.hasWord(word).observe(email.text.contains(word)) + } + model.isSpam.observe(email.label == "spam") + } + + def main(args: Array[String]) { + val learningAlg = EMWithVE(10, params) + learningAlg.start() + + val futureEmail = new Email(List("Feel free to reply if you have any ideas."), "unknown") + val futureModel = new EmailModel(params.posteriorParameters) + for { word <- featureWords } { + futureModel.hasWord(word).observe(futureEmail.text.contains(word)) + } + + learningAlg.stop() + + val result = VariableElimination.probability(futureModel.isSpam, true) + println("Probability new email is spam = " + result) + + learningAlg.kill() + } +} + +class MAPLearningTest extends WordSpec with Matchers { + Universe.createNew() + val learningAlg = EMWithVE(10, MAPLearning.params) + learningAlg.start() + + val futureEmail = new MAPLearning.Email(List("Feel free to reply if you have any ideas."), "unknown") + val futureModel = new MAPLearning.EmailModel(MAPLearning.params.posteriorParameters) + for { word <- MAPLearning.featureWords } { + futureModel.hasWord(word).observe(futureEmail.text.contains(word)) + } + + learningAlg.stop() + + val result = VariableElimination.probability(futureModel.isSpam, true) + println("Probability new email is spam = " + result) + + learningAlg.kill() + + "MAP Learning" should { + "produce a probability new email is spam = 0.2171438039742 +- 0.0000000000001" taggedAs (BookExample) in { + result should be (0.2171438039742 +- 0.0000000000001) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/OnlineEM.scala b/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/OnlineEM.scala new file mode 100644 index 00000000..1f08cbab --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/book/chap13/OnlineEM.scala @@ -0,0 +1,106 @@ +/* + * OnlineEM.scala + * Book example unit test. + * + * Created By: Michael Reposa (mreposa@cra.com), Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 26, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.book.chap13 + +import com.cra.figaro.algorithm.learning.EMWithVE +import com.cra.figaro.patterns.learning.{ModelParameters, ParameterCollection} +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Dirichlet +import com.cra.figaro.algorithm.factored.VariableElimination +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.test.tags.BookExample + +object OnlineEM { + def main(args: Array[String]) { + val parameters = ModelParameters() + val d = Dirichlet(2.0,2.0,2.0)("d",parameters) + + class Model(parameters: ParameterCollection, modelUniverse: Universe) { + val s = Select(parameters.get("d"), 1, 2, 3)("s", modelUniverse) + } + + def f = () => { + val modelUniverse = new Universe + new Model(parameters.priorParameters, modelUniverse) + modelUniverse + } + + val em = EMWithVE.online(f, parameters) + em.start() + + for (i <- 1 to 100) { + val evidence = List(NamedEvidence("s", Observation(1))) + em.update(evidence) + } + + val futureUniverse1 = Universe.createNew() + val futureModel1 = new Model(parameters.posteriorParameters, futureUniverse1) + println(VariableElimination.probability(futureModel1.s, 1)) + + for (i <- 101 to 200) { + val evidence = List(NamedEvidence("s", Observation(2))) + em.update(evidence) + } + + val futureUniverse2 = Universe.createNew() + val futureModel2 = new Model(parameters.posteriorParameters, futureUniverse2) + println(VariableElimination.probability(futureModel2.s, 1)) + } +} + +class OnlineEMTest extends WordSpec with Matchers { + val parameters = ModelParameters() + val d = Dirichlet(2.0,2.0,2.0)("d",parameters) + + class Model(parameters: ParameterCollection, modelUniverse: Universe) { + val s = Select(parameters.get("d"), 1, 2, 3)("s", modelUniverse) + } + + def f = () => { + val modelUniverse = new Universe + new Model(parameters.priorParameters, modelUniverse) + modelUniverse + } + + val em = EMWithVE.online(f, parameters) + em.start() + + for (i <- 1 to 100) { + val evidence = List(NamedEvidence("s", Observation(1))) + em.update(evidence) + } + + val futureUniverse1 = Universe.createNew() + val futureModel1 = new Model(parameters.posteriorParameters, futureUniverse1) + val result1 = VariableElimination.probability(futureModel1.s, 1) + + for (i <- 101 to 200) { + val evidence = List(NamedEvidence("s", Observation(2))) + em.update(evidence) + } + + val futureUniverse2 = Universe.createNew() + val futureModel2 = new Model(parameters.posteriorParameters, futureUniverse2) + val result2 = VariableElimination.probability(futureModel2.s, 1) + + "Online EM" should { + "produce a probability for future model 1 = 0.9805825242718447" taggedAs (BookExample) in { + result1 should be (0.9805825242718447) + } + "produce a probability for future model 2 = 0.4975369458128079" taggedAs (BookExample) in { + result2 should be (0.4975369458128079) + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/BayesianNetworkTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/BayesianNetworkTest.scala index 7e47b79d..7cf4e7ee 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/BayesianNetworkTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/BayesianNetworkTest.scala @@ -1,13 +1,13 @@ /* - * BayesianNetworkTest.scala + * BayesianNetworkTest.scala * Bayesian network examples tests. - * + * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jan 1, 2009 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -24,6 +24,8 @@ import com.cra.figaro.algorithm.sampling._ import com.cra.figaro.test._ import com.cra.figaro.test.tags.Example import com.cra.figaro.test.tags.NonDeterministic +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.algorithm.factored.gibbs.BlockSampler class BayesianNetworkTest extends WordSpec with Matchers { "A simple Bayesian network" should { @@ -41,18 +43,30 @@ class BayesianNetworkTest extends WordSpec with Matchers { val model = new Model Importance.probability(model.burglary, true) should be (model.pBurglary +- 0.02) } - + "produce the correct probability under Metropolis-Hastings without burn-in or interval" taggedAs (Example, NonDeterministic) in { val model = new Model MetropolisHastings.probability(model.burglary, true) should be (model.pBurglary +- 0.02) } - + "produce the correct probability under Metropolis-Hastings with burn-in and interval" taggedAs (Example, NonDeterministic) in { val model = new Model val alg = MetropolisHastings(200000, ProposalScheme.default, 800, 10, model.burglary) alg.start() alg.probability(model.burglary, true) should be (model.pBurglary +- 0.02) } + + "produce the correct probability under Gibbs sampling without burn-in or interval" taggedAs (Example, NonDeterministic) in { + val model = new Model + Gibbs.probability(model.burglary, true) should be (model.pBurglary +- 0.02) + } + + "produce the correct probability under Gibbs sampling with burn-in and interval" taggedAs (Example, NonDeterministic) in { + val model = new Model + val alg = Gibbs(200000, 800, 10, BlockSampler.default, model.burglary) + alg.start() + alg.probability(model.burglary, true) should be (model.pBurglary +- 0.02) + } } class Model { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/FairDiceTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/FairDiceTest.scala index 4cf89f17..93e0dd06 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/FairDiceTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/FairDiceTest.scala @@ -15,17 +15,17 @@ package com.cra.figaro.test.example import org.scalatest.Matchers import org.scalatest.WordSpec -import com.cra.figaro.algorithm._ -import com.cra.figaro.algorithm.sampling._ -import com.cra.figaro.algorithm.factored._ -import com.cra.figaro.algorithm.learning._ -import com.cra.figaro.library.compound._ -import com.cra.figaro.library.atomic.continuous._ -import com.cra.figaro.language._ -import com.cra.figaro.test._ -import com.cra.figaro.test.tags.Example + +import com.cra.figaro.algorithm.factored.VariableElimination +import com.cra.figaro.algorithm.learning.EMWithBP +import com.cra.figaro.language.Apply +import com.cra.figaro.language.Name.stringToName +import com.cra.figaro.language.Select +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Dirichlet import com.cra.figaro.patterns.learning.ModelParameters import com.cra.figaro.patterns.learning.ParameterCollection +import com.cra.figaro.test.tags.Example class FairDiceTest extends WordSpec with Matchers { "A simple FairDiceTest" should { @@ -97,19 +97,22 @@ class FairDiceTest extends WordSpec with Matchers { val d2 = Select(params.posteriorParameters.get("fairness2"), outcomes: _*) val d3 = Select(params.posteriorParameters.get("fairness3"), outcomes: _*) - val probsAndSides1 = d1.probs zip outcomes - probsAndSides1.map(a => println("\t" + a._1 + " -> " + a._2)) + val ve = VariableElimination(d1,d2,d3) + ve.start() + ve.stop() + + println("\nThe probabilities of seeing each side of d_1 are: ") + outcomes.foreach { o => println("\t" + ve.probability(d1)(_ == o) + " -> " + o) } println("\nThe probabilities of seeing each side of d_2 are: ") - val probsAndSides2 = d2.probs zip outcomes - probsAndSides2.map(a => println("\t" + a._1 + " -> " + a._2)) + outcomes.foreach { o => println("\t" + ve.probability(d2)(_ == o) + " -> " + o) } println("\nThe probabilities of seeing each side of d_3 are: ") - val probsAndSides3 = d3.probs zip outcomes - probsAndSides3.map(a => println("\t" + a._1 + " -> " + a._2)) + outcomes.foreach { o => println("\t" + ve.probability(d3)(_ == o) + " -> " + o) } - (d1.probs(0) > .8) should be(true) - (d2.probs(1) > .8) should be(true) - (d3.probs(5) > .8) should be(true) + (ve.probability(d1)(_ == outcomes(0)) > .8) should be(true) + (ve.probability(d2)(_ == outcomes(1)) > .8) should be(true) + (ve.probability(d3)(_ == outcomes(5)) > .8) should be(true) + ve.kill() algorithm.kill } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/MultiDecisionTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/MultiDecisionTest.scala index 72fe3e06..61953081 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/MultiDecisionTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/MultiDecisionTest.scala @@ -58,13 +58,13 @@ class MultiDecisionTest extends WordSpec with Matchers { override def oneTest = { val result = doTest((e1: List[Element[Double]], e2: List[Decision[_, _]]) => - MultiDecisionMetropolisHastings(200000, propmaker, 10000, e1, e2: _*)) + MultiDecisionMetropolisHastings(300000, propmaker, 20000, e1, e2: _*)) update(result(0), NDTest.BOOLEAN, "MHMulti-DecisionFound(-1)", true, .90) update(result(1), NDTest.BOOLEAN, "MHMulti-DecisionFound(0)", false, .90) update(result(2), NDTest.BOOLEAN, "MHMulti-DecisionFound(1)", true, .90) update(result(3), NDTest.BOOLEAN, "MHMulti-DecisionFound(2)", true, .90) - update(result(4), NDTest.BOOLEAN, "MHMulti-DecisionTest(0)", true, .90) + update(result(4), NDTest.BOOLEAN, "MHMulti-DecisionTest(0)", true, .70) } } ndtest.run(10) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/MultiValuedTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/MultiValuedTest.scala index 34e3cf8b..144a7ece 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/MultiValuedTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/MultiValuedTest.scala @@ -1,13 +1,13 @@ /* - * MultiValuedTest.scala + * MultiValuedTest.scala * Multi-valued reference uncertainty example tests. - * + * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jan 1, 2009 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/MutableMovieTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/MutableMovieTest.scala index f781b137..4d330d43 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/MutableMovieTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/MutableMovieTest.scala @@ -35,8 +35,7 @@ class MutableMovieTest extends WordSpec with Matchers { } "produce the correct probability under Metropolis-Hastings" taggedAs (Example, NonDeterministic) in { - test((e: Element[Boolean]) => MetropolisHastings(200000, chooseScheme, 10000, e)) - println(com.cra.figaro.util.seed) + test((e: Element[Boolean]) => MetropolisHastings(300000, chooseScheme, 20000, e)) } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/OpenUniverseTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/OpenUniverseTest.scala index dd830831..f8eb1b86 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/OpenUniverseTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/OpenUniverseTest.scala @@ -58,7 +58,7 @@ class OpenUniverseTest extends WordSpec with Matchers { (0.25, () => ProposalScheme(numSources)), (0.25, () => ProposalScheme(sources.items(random.nextInt(numSources.value)))), (0.25, () => ProposalScheme(samples(random.nextInt(numSamples)).sourceNum)), - (0.25, () => ProposalScheme(samples(random.nextInt(numSamples)).position.resultElement))) + (0.25, () => ProposalScheme(samples(random.nextInt(numSamples)).position))) sample1.position.addCondition((y: Double) => y >= 0.5 && y < 0.8) sample2.position.addCondition((y: Double) => y >= 0.5 && y < 0.8) @@ -83,7 +83,7 @@ class OpenUniverseTest extends WordSpec with Matchers { val totalProbSame = (0.0 /: (1 to limitNumSources))(_ + probSame(_)) val totalProbDifferent = (0.0 /: (1 to limitNumSources))(_ + probDifferent(_)) val answer = totalProbSame / (totalProbSame + totalProbDifferent) - val alg = MetropolisHastings(2000000, chooseScheme, 50000, equal) + val alg = MetropolisHastings(200000, chooseScheme, 5000, equal) alg.start() alg.probability(equal, true) should be(answer +- 0.02) alg.kill diff --git a/Figaro/src/test/scala/com/cra/figaro/test/example/SmokersTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/example/SmokersTest.scala index 55b23934..c95da992 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/example/SmokersTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/example/SmokersTest.scala @@ -1,13 +1,13 @@ /* - * SmokersTest.scala + * SmokersTest.scala * Friends and smokes example tests. - * + * * Created By: Avi Pfeffer (apfeffer@cra.com) * Creation Date: Jan 1, 2009 - * + * * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. * See http://www.cra.com or email figaro@cra.com for information. - * + * * See http://www.github.com/p2t2/figaro for a copy of the software license. */ @@ -18,6 +18,7 @@ import org.scalatest.WordSpec import com.cra.figaro.algorithm._ import com.cra.figaro.algorithm.sampling._ import com.cra.figaro.algorithm.factored._ +import com.cra.figaro.algorithm.factored.gibbs.Gibbs import com.cra.figaro.language._ import com.cra.figaro.library.compound._ import com.cra.figaro.test._ @@ -37,6 +38,10 @@ class SmokersTest extends WordSpec with Matchers { "produce the correct answer under variable elimination" taggedAs (Example, NonDeterministic) in { test((e: Element[Boolean]) => VariableElimination(e)) } + + "produce the correct answer under Gibbs sampling" taggedAs (Example, NonDeterministic) in { + test((e: Element[Boolean]) => Gibbs(20000, e)) + } } private class Person { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/CollapsedGibbsTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/experimental/CollapsedGibbsTest.scala new file mode 100644 index 00000000..911d505a --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/experimental/CollapsedGibbsTest.scala @@ -0,0 +1,262 @@ +/* + * CollapsedGibbsTest.scala + * Collapsed Gibbs sampler tests. + * + * Created By: Cory Scott (scottcb@uci.edu) + * Creation Date: July 21, 2016 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.experimental + +import com.cra.figaro.algorithm.factored.factors._ +import com.cra.figaro.algorithm.lazyfactored.LazyValues +import com.cra.figaro.language._ +import com.cra.figaro.library.compound.If +import com.cra.figaro.library.compound.^^ +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.algorithm.factored.factors.factory.Factory +import com.cra.figaro.algorithm.factored.gibbs.Gibbs +import com.cra.figaro.experimental.collapsedgibbs._ + +class CollapsedGibbsTest extends WordSpec with Matchers { + "A Collapsed Gibbs sampler" should { + + "with an unconstrained model produce the correct result" in { + Universe.createNew + val f = Flip(0.3) + val s1 = Select(0.1 -> 1, 0.4 -> 2, 0.5 -> 3) + val s2 = Select(0.7 -> 2, 0.1 -> 3, 0.2 -> 4) + val c = If(f, s1, s2) + // 0.3 * 0.4 + 0.7 * 0.7 = 0.61 + test[Int](c, _ == 2, 0.61) + } + + "with a constrained model produce the correct result" in { + Universe.createNew + val f = Flip(0.3) + val s1 = Select(0.1 -> 1, 0.4 -> 2, 0.5 -> 3) + val s2 = Select(0.7 -> 2, 0.1 -> 3, 0.2 -> 4) + val c = If(f, s1, s2) + c.addConstraint(identity) + // 0.61 * 2 / (0.3*0.1*1 + 0.3*0.4*2 + 0.3*0.5*3 + 0.7*0.7*2 + 0.7*0.1*3 + 0.7*0.2*4) + test[Int](c, _ == 2, 0.4939) + } + + "with a constraint on a Chain produce the correct result for the parent" in { + Universe.createNew + val f = Flip(0.3) + val c = If(f, Flip(0.8), Constant(false)) + c.addConstraint(b => if (b) 2.0 else 1.0) + // (0.3 * 0.8 * 2) / (0.3 * 0.8 * 2 + 0.3 * 0.2 + 0.7) + test[Boolean](c, identity, 0.3871) + } + + "with a constraint on a Chain result correctly constrain the Chain but not the parent" in { + Universe.createNew + val f = Flip(0.3) + val r1 = Flip(0.8) + r1.addConstraint(b => if (b) 2.0 else 1.0) + val c = If(f, r1, Constant(false)) + test[Boolean](f, identity, 0.3) + // r1 true with probability (0.8 * 2) / (0.8 * 2 + 0.2) = 0.8889 + // 0.3 * 0.8889 + test[Boolean](c, identity, 0.2667) + } + + "with an element used multiple times use the same value each time" in { + Universe.createNew + val f = Flip(0.3) + val e = f === f + test[Boolean](e, identity, 1.0) + } + + "not underflow" in { + Universe.createNew() + val x = Flip(0.99) + for (i <- 0 until 10) { + x.addConstraint((b: Boolean) => if (b) 1e-100; else 1e-120) + } + test[Boolean](x, identity, 1.0, 2) + } + + "with the default collapser (default parameters) produce the correct result in a deterministic model" in { + testStrategyDeterministicModel[Boolean]("") + } + "with the DETERM collapser (default parameters) produce the correct result in a deterministic model" in { + testStrategyDeterministicModel[Boolean]("DETERM") + } + "with the RECURR collapser (default parameters) produce the correct result in a deterministic model" in { + testStrategyDeterministicModel[Boolean]("RECURR") + } + "with the FACTOR collapser (default parameters) produce the correct result in a deterministic model" in { + testStrategyDeterministicModel[Boolean]("FACTOR") + } + "with the SIMPLE collapser (default parameters) produce the correct result in a deterministic model" in { + testStrategyDeterministicModel[Boolean]("SIMPLE") + } + + "with the default collapser (high alpha and gamma) produce the correct result in a deterministic model" in { + testStrategyWithParamsDeterministicModel[Boolean]("", Seq(10, 2000)) + } + "with the DETERM collapser (high alpha and gamma) produce the correct result in a deterministic model" in { + testStrategyWithParamsDeterministicModel[Boolean]("DETERM", Seq(10, 2000)) + } + "with the RECURR collapser (high alpha, gamma, frequent sample saving) produce the correct result in a deterministic model" in { + testStrategyWithParamsDeterministicModel[Boolean]("RECURR", Seq(10, 2000, 2000, 1)) + } + "with the FACTOR collapser (high alpha, gamma, and factorThreshhold) produce the correct result in a deterministic model" in { + testStrategyWithParamsDeterministicModel[Boolean]("FACTOR", Seq(10, 2000, 10000)) + } + "with the SIMPLE collapser (high alpha and gamma) produce the correct result in a deterministic model" in { + testStrategyWithParamsDeterministicModel[Boolean]("SIMPLE", Seq(10, 2000)) + } + + "with the default collapser (default parameters) produce the correct result in an Ising model" in { + testStrategyIsingModel[Boolean]("") + } + "with the DETERM collapser (default parameters) produce the correct result in an Ising model" in { + testStrategyIsingModel[Boolean]("DETERM") + } + "with the RECURR collapser (default parameters) produce the correct result in an Ising model" in { + testStrategyIsingModel[Boolean]("RECURR") + } + "with the FACTOR collapser (default parameters) produce the correct result in an Ising model" in { + testStrategyIsingModel[Boolean]("FACTOR") + } + "with the SIMPLE collapser (default parameters) produce the correct result in an Ising model" in { + testStrategyIsingModel[Boolean]("SIMPLE") + } + + "with the default collapser (high alpha and gamma) produce the correct result in a Ising model" in { + testStrategyWithParamsIsingModel[Boolean]("", Seq(10, 2000)) + } + "with the DETERM collapser (high alpha and gamma) produce the correct result in a Ising model" in { + testStrategyWithParamsIsingModel[Boolean]("DETERM", Seq(10, 2000)) + } + "with the RECURR collapser (high alpha, gamma, frequent sample saving) produce the correct result in a Ising model" in { + testStrategyWithParamsIsingModel[Boolean]("RECURR", Seq(10, 2000, 2000, 1)) + } + "with the FACTOR collapser (high alpha, gamma, and factorThreshhold) produce the correct result in a Ising model" in { + testStrategyWithParamsIsingModel[Boolean]("FACTOR", Seq(10, 2000, 10000)) + } + "with the SIMPLE collapser (high alpha and gamma) produce the correct result in a Ising model" in { + testStrategyWithParamsIsingModel[Boolean]("SIMPLE", Seq(10, 2000)) + } + + "with the default collapser and default parameters collapse down to query variables" in { + Universe.createNew + val a1 = Flip(0.5) + val a2 = Flip(0.5) + val a3 = Flip(0.5) + val b1 = Apply(a1, a2, (b1: Boolean, b2: Boolean) => b1 || b2) + val b2 = Apply(a3, a2, (b1: Boolean, b2: Boolean) => b1 || b2) + val b3 = Apply(a1, a3, (b1: Boolean, b2: Boolean) => b1 || b2) + val c1 = Apply(b1, b2, (b1: Boolean, b2: Boolean) => b1 || b2) + val c2 = Apply(b3, b2, (b1: Boolean, b2: Boolean) => b1 || b2) + val c3 = Apply(b1, b3, (b1: Boolean, b2: Boolean) => b1 || b2) + val d3 = Apply(c1, c2, c3, (b1: Boolean, b2: Boolean, b3: Boolean) => b1 || b2 || b3) + val alg2 = CollapsedGibbs(10000, d3) + alg2.initialize() + alg2.variables.toList.length should be (1) + } + + } + + def makeFactors(): List[Factor[Double]] = { + LazyValues(Universe.universe).expandAll(Universe.universe.activeElements.toSet.map((elem: Element[_]) => ((elem, Integer.MAX_VALUE)))) + Universe.universe.activeElements.foreach(Variable(_)) + Universe.universe.activeElements flatMap (Factory.makeFactorsForElement(_)) + } + + def test[T](target: Element[T], predicate: T => Boolean, prob: Double, tol: Double = 0.025) { + val algorithm = CollapsedGibbs(100000, target) + algorithm.start() + algorithm.stop() + algorithm.probability(target, predicate) should be(prob +- tol) + algorithm.kill() + } + def testStrategyDeterministicModel[T](strategy:String, tol: Double = 0.025) { + Universe.universe.finalize() + Universe.createNew + val a1 = Flip(0.5) + val a2 = Flip(0.5) + val a3 = Flip(0.5) + val b1 = Apply(a1, a2, (b1: Boolean, b2: Boolean) => b1 || b2) + val b2 = Apply(a3, a2, (b1: Boolean, b2: Boolean) => b1 || b2) + val b3 = Apply(a1, a3, (b1: Boolean, b2: Boolean) => b1 || b2) + val c1 = Apply(b1, b2, (b1: Boolean, b2: Boolean) => b1 || b2) + val c2 = Apply(b3, b2, (b1: Boolean, b2: Boolean) => b1 || b2) + val c3 = Apply(b1, b3, (b1: Boolean, b2: Boolean) => b1 || b2) + val d3 = Apply(c1, c2, c3, (b1: Boolean, b2: Boolean, b3: Boolean) => b1 || b2 || b3) + val algorithm = CollapsedGibbs(strategy:String, 100000, d3) + algorithm.start() + algorithm.stop() + algorithm.probability(d3, true) should be(.875 +- tol) + algorithm.kill() + } + def testStrategyWithParamsDeterministicModel[T](strategy:String, params:Seq[Int], tol: Double = 0.025) { + Universe.universe.finalize() + Universe.createNew + val a1 = Flip(0.5) + val a2 = Flip(0.5) + val a3 = Flip(0.5) + val b1 = Apply(a1, a2, (b1: Boolean, b2: Boolean) => b1 || b2) + val b2 = Apply(a3, a2, (b1: Boolean, b2: Boolean) => b1 || b2) + val b3 = Apply(a1, a3, (b1: Boolean, b2: Boolean) => b1 || b2) + val c1 = Apply(b1, b2, (b1: Boolean, b2: Boolean) => b1 || b2) + val c2 = Apply(b3, b2, (b1: Boolean, b2: Boolean) => b1 || b2) + val c3 = Apply(b1, b3, (b1: Boolean, b2: Boolean) => b1 || b2) + val d3 = Apply(c1, c2, c3, (b1: Boolean, b2: Boolean, b3: Boolean) => b1 || b2 || b3) + val algorithm = CollapsedGibbs(strategy:String, params, 100000, d3) + algorithm.start() + algorithm.stop() + algorithm.probability(d3, true) should be(.875 +- tol) + algorithm.kill() + } + def testStrategyIsingModel[T](strategy:String, tol: Double = 0.025) { + Universe.universe.finalize() + Universe.createNew() + def IsingConstraint(pair: (Boolean, Boolean)) = if (pair._1 == pair._2) 1.1; else 1.0 + val IsingSize = 4 + var allFlips = for {i <- 0 until IsingSize} yield (for {j <- 0 until IsingSize} yield Flip(0.5)).toList + for {i <- 0 until IsingSize} { + for {j <- 0 until IsingSize} { + ^^(allFlips(i)(j), allFlips((i+1) % IsingSize)(j)).setConstraint(IsingConstraint) + ^^(allFlips(i)(j), allFlips(i)((j + 1) % IsingSize)).setConstraint(IsingConstraint) + } + } + allFlips(IsingSize/2)(IsingSize/2).observe(false) + val toTest = allFlips(0)(0) + val algorithm = CollapsedGibbs(strategy:String, 100000, toTest) + algorithm.start() + algorithm.stop() + algorithm.probability(toTest, true) should be(Gibbs.probability(toTest, true) +- tol) + algorithm.kill() + } + def testStrategyWithParamsIsingModel[T](strategy:String, params:Seq[Int], tol: Double = 0.025) { + Universe.universe.finalize() + Universe.createNew() + def IsingConstraint(pair: (Boolean, Boolean)) = if (pair._1 == pair._2) 1.1; else 1.0 + val IsingSize = 4 + var allFlips = for {i <- 0 until IsingSize} yield (for {j <- 0 until IsingSize} yield Flip(0.5)).toList + for {i <- 0 until IsingSize} { + for {j <- 0 until IsingSize} { + ^^(allFlips(i)(j), allFlips((i+1) % IsingSize)(j)).setConstraint(IsingConstraint) + ^^(allFlips(i)(j), allFlips(i)((j + 1) % IsingSize)).setConstraint(IsingConstraint) + } + } + allFlips(IsingSize/2)(IsingSize/2).observe(false) + val toTest = allFlips(0)(0) + val algorithm = CollapsedGibbs(strategy:String, params, 100000, toTest) + algorithm.start() + algorithm.stop() + algorithm.probability(toTest, true) should be(Gibbs.probability(toTest, true) +- tol) + algorithm.kill() + } +} \ No newline at end of file diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/MarginalMAPBPTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/MarginalMAPBPTest.scala new file mode 100644 index 00000000..3e9c9d79 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/MarginalMAPBPTest.scala @@ -0,0 +1,241 @@ +/* + * MarginalMAPBPTest.scala + * Marginal MAP Belief Propagation tests. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 11, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.experimental.marginalmap + +import com.cra.figaro.experimental.marginalmap.MarginalMAPBeliefPropagation +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.collection.Container +import com.cra.figaro.library.compound.If +import org.scalatest.{Matchers, WordSpec} + +class MarginalMAPBPTest extends WordSpec with Matchers { + "Marginal MAP BP" when { + "given a model with MAP queries on all elements" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.8) + val b11 = Flip(0.7) + val b12 = Flip(0.6) + val b1 = b11 && b12 + val b2 = Constant(false) + val b = If(a, b1, b2) + // Even though b is a Chain, the result elements of b are MAP elements + // This should produce the same result as an MPE query + + // p(a=T,b11=T,b12=T) = 0.8 * 0.7 * 0.6 = 0.336 + // p(a=T,b11=T,b12=F) = 0.8 * 0.7 * 0.4 = 0.224 + // p(a=T,b11=F,b12=T) = 0.8 * 0.3 * 0.6 = 0.144 + // p(a=T,b11=F,b12=F) = 0.8 * 0.3 * 0.4 = 0.096 + // p(a=F,b11=T,b12=T) = 0.2 * 0.7 * 0.6 = 0.084 + // p(a=F,b11=T,b12=F) = 0.2 * 0.7 * 0.4 = 0.054 + // p(a=F,b11=F,b12=T) = 0.2 * 0.3 * 0.6 = 0.036 + // p(a=F,b11=F,b12=F) = 0.2 * 0.3 * 0.4 = 0.024 + // MAP: a=T,b11=T,b12=T which implies b1=T,b2=F,b=T + val alg = MarginalMAPBeliefPropagation(20, a, b11, b12, b1, b2, b) + alg.start + alg.mostLikelyValue(a) should equal(true) + alg.mostLikelyValue(b11) should equal(true) + alg.mostLikelyValue(b12) should equal(true) + alg.mostLikelyValue(b1) should equal(true) + alg.mostLikelyValue(b2) should equal(false) + alg.mostLikelyValue(b) should equal(true) + alg.kill + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.8) + val b11 = Flip(0.7) + val b12 = Flip(0.6) + val b1 = b11 && b12 + val b2 = Constant(false) + val b = If(a, b1, b2) + b.observe(false) + // Even though b is a Chain, the result elements of b are MAP elements + // This should produce the same result as an MPE query + + // These weights are not normalized + // p(a=T,b11=T,b12=T) = 0 + // p(a=T,b11=T,b12=F) = 0.8 * 0.7 * 0.4 = 0.224 + // p(a=T,b11=F,b12=T) = 0.8 * 0.3 * 0.6 = 0.144 + // p(a=T,b11=F,b12=F) = 0.8 * 0.3 * 0.4 = 0.096 + // p(a=F,b11=T,b12=T) = 0.2 * 0.7 * 0.6 = 0.084 + // p(a=F,b11=T,b12=F) = 0.2 * 0.7 * 0.4 = 0.054 + // p(a=F,b11=F,b12=T) = 0.2 * 0.3 * 0.6 = 0.036 + // p(a=F,b11=F,b12=F) = 0.2 * 0.3 * 0.4 = 0.024 + // MAP: a=T,b11=T,b12=F which implies b1=F,b2=F,b=F + val alg = MarginalMAPBeliefPropagation(20, a, b11, b12, b1, b2, b) + alg.start + alg.mostLikelyValue(a) should equal(true) + alg.mostLikelyValue(b11) should equal(true) + alg.mostLikelyValue(b12) should equal(false) + alg.mostLikelyValue(b1) should equal(false) + alg.mostLikelyValue(b2) should equal(false) + alg.mostLikelyValue(b) should equal(false) + alg.kill + } + } + + "given a model with MAP queries on all permanent elements" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.3) && Flip(0.5), Flip(0.4)) + // The result elements of b are marginalized + // Then, the first result element of b is effectively a Flip(0.15) + + // p(a=T,b=T) = 0.6 * 0.15 = 0.09 + // p(a=T,b=F) = 0.6 * 0.85 = 0.51 + // p(a=F,b=T) = 0.4 * 0.4 = 0.16 + // p(a=F,b=F) = 0.4 * 0.6 = 0.24 + // MAP: a=T,b=F + val alg = MarginalMAPBeliefPropagation(20, a, b) + alg.start + alg.mostLikelyValue(a) should equal(true) + alg.mostLikelyValue(b) should equal(false) + alg.kill + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.3) && Flip(0.5), Flip(0.4)) + b.observe(true) + // The result elements of b are marginalized + // Then, the first result element of b is effectively a Flip(0.15) + + // These weights are not normalized + // p(a=T,b=T) = 0.6 * 0.15 = 0.09 + // p(a=T,b=F) = 0 + // p(a=F,b=T) = 0.4 * 0.4 = 0.16 + // p(a=F,b=F) = 0 + // MAP: a=F,b=T + val alg = MarginalMAPBeliefPropagation(20, a, b) + alg.start + alg.mostLikelyValue(a) should equal(false) + alg.mostLikelyValue(b) should equal(true) + alg.kill + } + } + + "given a model with MAP queries on a single element" should { + "produce the right answer without evidence" in { + Universe.createNew() + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + // num4 is effectively a binomial distribution with n=10, p=0.25 + // The mode is floor(p*(n+1))=2 + val alg = MarginalMAPBeliefPropagation(20, num4) + alg.start + alg.mostLikelyValue(num4) should equal(2) + alg.kill + } + + "produce the right answer with evidence" in { + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + num4.addCondition(_ >= 5) + + // Since the pmf of a binomial distribution is strictly decreasing past the mode, + // the most likely value should be the least possible value given the evidence + val alg = MarginalMAPBeliefPropagation(20, num4) + alg.start + alg.mostLikelyValue(num4) should equal(5) + alg.kill + } + } + + "given a model with MAP queries on more than one element" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0.1512+0.0288+0.1296+0.0864=0.396 -> MAP + + val alg = MarginalMAPBeliefPropagation(10, c, d) + alg.start() + alg.mostLikelyValue(c) should equal(false) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // These weights are not normalized + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 -> MAP + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0 + + (c || d).observe(true) + + val alg = MarginalMAPBeliefPropagation(10, c, d) + alg.start() + alg.mostLikelyValue(c) should equal(true) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/ProbEvidenceMarginalMAPTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/ProbEvidenceMarginalMAPTest.scala new file mode 100644 index 00000000..9a195e54 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/ProbEvidenceMarginalMAPTest.scala @@ -0,0 +1,362 @@ +/* + * ProbEvidenceMarginalMAPTest.scala + * Tests for probability of evidence-based marginal MAP. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 1, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.experimental.marginalmap + +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.experimental.marginalmap.ProbEvidenceMarginalMAP +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.Normal +import com.cra.figaro.library.atomic.discrete.{Uniform, Util} +import com.cra.figaro.library.collection.Container +import com.cra.figaro.library.compound.If +import org.scalatest.{Matchers, WordSpec} + +class ProbEvidenceMarginalMAPTest extends WordSpec with Matchers { + // For testing, it's more efficient to use a linear schedule on simple models where we just want quick convergence, + // rather than exploration of a large state space + val linearSchedule = Schedule((temp, iter) => iter) + + def anytime(elems: Element[_]*) = + ProbEvidenceMarginalMAP(0.05, 100, 100, ProposalScheme.default, linearSchedule, elems:_*) + + def oneTime(elems: Element[_]*) = + ProbEvidenceMarginalMAP(2000, 0.05, 100, 100, ProposalScheme.default, linearSchedule, elems:_*) + + "Marginal MAP using probability of evidence" should { + "increase temperature with additional iterations" in { + Universe.createNew() + val elem = Flip(0.6) + + val alg = ProbEvidenceMarginalMAP(elem) + alg.start() + val temp1 = alg.getTemperature + Thread.sleep(500) + val temp2 = alg.getTemperature + alg.kill() + + temp2 should be > temp1 + } + + "increase temperature faster with lower k" in { + Universe.createNew() + val elem = Flip(0.6) + + // k = 2.0 + val alg1 = ProbEvidenceMarginalMAP(100, 0.05, 2, 100, ProposalScheme(elem), Schedule.default(2.0), elem) + alg1.start() + val temp1 = alg1.getTemperature + alg1.kill() + + // k = 4.0 + val alg2 = ProbEvidenceMarginalMAP(100, 0.05, 2, 100, ProposalScheme(elem), Schedule.default(4.0), elem) + alg2.start() + val temp2 = alg2.getTemperature + alg2.kill() + + temp2 should be < temp1 + } + } + + "Running anytime marginal MAP using probability of evidence" when { + "given a model with a MAP query on a top-level parameter" should { + "correctly estimate the parameter with evidence" in { + Universe.createNew() + val parameterMean = 5.0 + val parameterVariance = 1.0 + val variance = 1.0 + + // We're using paramater as a prior to estimate a normal with known variance + val parameter = Normal(parameterMean, parameterVariance) + + val observations = List(6.1, 7.3, 5.8, 5.3, 6.4) + for(obs <- observations) { + Normal(parameter, variance).observe(obs) + } + + // The MAP value of parameter is just the the posterior mean + val meanEstimate = (parameterMean / parameterVariance + observations.sum / variance) / + (1.0 / parameterVariance + observations.length / variance) + + val alg = ProbEvidenceMarginalMAP(0.05, 2, 1, ProposalScheme(parameter), linearSchedule, parameter) + alg.start() + Thread.sleep(2500) + alg.stop() + alg.mostLikelyValue(parameter) should equal(meanEstimate +- 0.05) + alg.kill() + } + + } + + "given a model with MAP queries on more than one element" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0.1512+0.0288+0.1296+0.0864=0.396 -> MAP + + val alg = anytime(c, d) + alg.start() + Thread.sleep(2500) + alg.stop() + alg.mostLikelyValue(c) should equal(false) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // These weights are not normalized + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 -> MAP + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0 + + (c || d).observe(true) + + val alg = anytime(c, d) + alg.start() + Thread.sleep(2500) + alg.stop() + alg.mostLikelyValue(c) should equal(true) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + } + + "given evidence on MAP elements" should { + "produce the right answer with a condition" in { + Universe.createNew() + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + num4.addCondition(_ >= 5) + + // Since the pmf of a binomial distribution is strictly decreasing past the mode, + // the most likely value should be the least possible value given the evidence + val alg = anytime(num4) + alg.start() + Thread.sleep(10000) + alg.stop() + alg.mostLikelyValue(num4) should equal(5) + alg.kill() + } + + "produce the right answer with a constraint" in { + Universe.createNew() + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + val constraint = (x: Int) => math.exp(- (x - 6) * (x - 6)) + num4.addConstraint(constraint) + val max = (0 to 10).maxBy(x => Util.binomialDensity(10, 0.25, x) * constraint(x)) + max should equal(5) + + val alg = anytime(num4) + alg.start() + Thread.sleep(10000) + alg.stop() + alg.mostLikelyValue(num4) should equal(max) + alg.kill() + } + } + } + + "Running one time marginal MAP using probability of evidence" when { + "given a model with a MAP query on a top-level parameter" should { + "correctly estimate the parameter with evidence" in { + Universe.createNew() + val parameterMean = 5.0 + val parameterVariance = 1.0 + val variance = 1.0 + + // We're using paramater as a prior to estimate a normal with known variance + val parameter = Normal(parameterMean, parameterVariance) + + val observations = List(6.1, 7.3, 5.8, 5.3, 6.4) + for(obs <- observations) { + Normal(parameter, variance).observe(obs) + } + + // The MAP value of parameter is just the the posterior mean + val meanEstimate = (parameterMean / parameterVariance + observations.sum / variance) / + (1.0 / parameterVariance + observations.length / variance) + + val alg = ProbEvidenceMarginalMAP(1000, 0.05, 2, 1, ProposalScheme(parameter), linearSchedule, parameter) + alg.start() + alg.mostLikelyValue(parameter) should equal(meanEstimate +- 0.05) + alg.kill() + } + + } + + "given a model with MAP queries on more than one element" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0.1512+0.0288+0.1296+0.0864=0.396 -> MAP + + val alg = oneTime(c, d) + alg.start() + alg.mostLikelyValue(c) should equal(false) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // These weights are not normalized + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 -> MAP + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0 + + (c || d).observe(true) + + val alg = oneTime(c, d) + alg.start() + alg.mostLikelyValue(c) should equal(true) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + } + + "given evidence on MAP elements" should { + "produce the right answer with a condition" in { + Universe.createNew() + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + num4.addCondition(_ >= 5) + + // Since the pmf of a binomial distribution is strictly decreasing past the mode, + // the most likely value should be the least possible value given the evidence + val alg = oneTime(num4) + alg.start() + alg.mostLikelyValue(num4) should equal(5) + alg.kill() + } + + "produce the right answer with a constraint" in { + Universe.createNew() + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + val constraint = (x: Int) => math.exp(- (x - 6) * (x - 6)) + num4.addConstraint(constraint) + val max = (0 to 10).maxBy(x => Util.binomialDensity(10, 0.25, x) * constraint(x)) + max should equal(5) + + val alg = oneTime(num4) + alg.start() + alg.mostLikelyValue(num4) should equal(max) + alg.kill() + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/StructuredMMAPVETest.scala b/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/StructuredMMAPVETest.scala new file mode 100644 index 00000000..b4f0b45d --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/experimental/marginalmap/StructuredMMAPVETest.scala @@ -0,0 +1,241 @@ +/* + * StructuredMMAPVETest.scala + * Structured marginal MAP VE tests. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 11, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.experimental.marginalmap + +import com.cra.figaro.experimental.marginalmap.StructuredMarginalMAPVE +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.discrete.Uniform +import com.cra.figaro.library.collection.Container +import com.cra.figaro.library.compound.If +import org.scalatest.{Matchers, WordSpec} + +class StructuredMMAPVETest extends WordSpec with Matchers { + "Marginal MAP VE" when { + "given a model with MAP queries on all elements" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.8) + val b11 = Flip(0.7) + val b12 = Flip(0.6) + val b1 = b11 && b12 + val b2 = Constant(false) + val b = If(a, b1, b2) + // Even though b is a Chain, the result elements of b are MAP elements + // This should produce the same result as an MPE query + + // p(a=T,b11=T,b12=T) = 0.8 * 0.7 * 0.6 = 0.336 + // p(a=T,b11=T,b12=F) = 0.8 * 0.7 * 0.4 = 0.224 + // p(a=T,b11=F,b12=T) = 0.8 * 0.3 * 0.6 = 0.144 + // p(a=T,b11=F,b12=F) = 0.8 * 0.3 * 0.4 = 0.096 + // p(a=F,b11=T,b12=T) = 0.2 * 0.7 * 0.6 = 0.084 + // p(a=F,b11=T,b12=F) = 0.2 * 0.7 * 0.4 = 0.054 + // p(a=F,b11=F,b12=T) = 0.2 * 0.3 * 0.6 = 0.036 + // p(a=F,b11=F,b12=F) = 0.2 * 0.3 * 0.4 = 0.024 + // MAP: a=T,b11=T,b12=T which implies b1=T,b2=F,b=T + val alg = StructuredMarginalMAPVE(a, b11, b12, b1, b2, b) + alg.start + alg.mostLikelyValue(a) should equal(true) + alg.mostLikelyValue(b11) should equal(true) + alg.mostLikelyValue(b12) should equal(true) + alg.mostLikelyValue(b1) should equal(true) + alg.mostLikelyValue(b2) should equal(false) + alg.mostLikelyValue(b) should equal(true) + alg.kill + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.8) + val b11 = Flip(0.7) + val b12 = Flip(0.6) + val b1 = b11 && b12 + val b2 = Constant(false) + val b = If(a, b1, b2) + b.observe(false) + // Even though b is a Chain, the result elements of b are MAP elements + // This should produce the same result as an MPE query + + // These weights are not normalized + // p(a=T,b11=T,b12=T) = 0 + // p(a=T,b11=T,b12=F) = 0.8 * 0.7 * 0.4 = 0.224 + // p(a=T,b11=F,b12=T) = 0.8 * 0.3 * 0.6 = 0.144 + // p(a=T,b11=F,b12=F) = 0.8 * 0.3 * 0.4 = 0.096 + // p(a=F,b11=T,b12=T) = 0.2 * 0.7 * 0.6 = 0.084 + // p(a=F,b11=T,b12=F) = 0.2 * 0.7 * 0.4 = 0.054 + // p(a=F,b11=F,b12=T) = 0.2 * 0.3 * 0.6 = 0.036 + // p(a=F,b11=F,b12=F) = 0.2 * 0.3 * 0.4 = 0.024 + // MAP: a=T,b11=T,b12=F which implies b1=F,b2=F,b=F + val alg = StructuredMarginalMAPVE(a, b11, b12, b1, b2, b) + alg.start + alg.mostLikelyValue(a) should equal(true) + alg.mostLikelyValue(b11) should equal(true) + alg.mostLikelyValue(b12) should equal(false) + alg.mostLikelyValue(b1) should equal(false) + alg.mostLikelyValue(b2) should equal(false) + alg.mostLikelyValue(b) should equal(false) + alg.kill + } + } + + "given a model with MAP queries on all permanent elements" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.3) && Flip(0.5), Flip(0.4)) + // The result elements of b are marginalized + // Then, the first result element of b is effectively a Flip(0.15) + + // p(a=T,b=T) = 0.6 * 0.15 = 0.09 + // p(a=T,b=F) = 0.6 * 0.85 = 0.51 + // p(a=F,b=T) = 0.4 * 0.4 = 0.16 + // p(a=F,b=F) = 0.4 * 0.6 = 0.24 + // MAP: a=T,b=F + val alg = StructuredMarginalMAPVE(a, b) + alg.start + alg.mostLikelyValue(a) should equal(true) + alg.mostLikelyValue(b) should equal(false) + alg.kill + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.3) && Flip(0.5), Flip(0.4)) + b.observe(true) + // The result elements of b are marginalized + // Then, the first result element of b is effectively a Flip(0.15) + + // These weights are not normalized + // p(a=T,b=T) = 0.6 * 0.15 = 0.09 + // p(a=T,b=F) = 0 + // p(a=F,b=T) = 0.4 * 0.4 = 0.16 + // p(a=F,b=F) = 0 + // MAP: a=F,b=T + val alg = StructuredMarginalMAPVE(a, b) + alg.start + alg.mostLikelyValue(a) should equal(false) + alg.mostLikelyValue(b) should equal(true) + alg.kill + } + } + + "given a model with MAP queries on a single element" should { + "produce the right answer without evidence" in { + Universe.createNew() + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + // num4 is effectively a binomial distribution with n=10, p=0.25 + // The mode is floor(p*(n+1))=2 + val alg = StructuredMarginalMAPVE(num4) + alg.start + alg.mostLikelyValue(num4) should equal(2) + alg.kill + } + + "produce the right answer with evidence" in { + val rolls = for { i <- 1 to 10 } yield Uniform(1,2,3,4) + val c = Container(rolls: _*) + val num4 = c.count(_ == 4) + + num4.addCondition(_ >= 5) + + // Since the pmf of a binomial distribution is strictly decreasing past the mode, + // the most likely value should be the least possible value given the evidence + val alg = StructuredMarginalMAPVE(num4) + alg.start + alg.mostLikelyValue(num4) should equal(5) + alg.kill + } + } + + "given a model with MAP queries on more than one element" should { + "produce the right answer without evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0.1512+0.0288+0.1296+0.0864=0.396 -> MAP + + val alg = StructuredMarginalMAPVE(c, d) + alg.start() + alg.mostLikelyValue(c) should equal(false) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + + "produce the right answer with evidence" in { + Universe.createNew() + val a = Flip(0.6) + val b = If(a, Flip(0.7), Flip(0.4)) + val c = If(a, Flip(0.6), Flip(0.1)) + val d = If(b, Flip(0.1), Flip(0.6)) + + // p(a=T,b=T,c=T,d=T)=0.6*0.7*0.6*0.1=0.0252 + // p(a=T,b=T,c=T,d=F)=0.6*0.7*0.6*0.9=0.2268 + // p(a=T,b=T,c=F,d=T)=0.6*0.7*0.4*0.1=0.0168 + // p(a=T,b=T,c=F,d=F)=0.6*0.7*0.4*0.9=0.1512 + // p(a=T,b=F,c=T,d=T)=0.6*0.3*0.6*0.6=0.0648 + // p(a=T,b=F,c=T,d=F)=0.6*0.3*0.6*0.4=0.0432 + // p(a=T,b=F,c=F,d=T)=0.6*0.3*0.4*0.6=0.0432 + // p(a=T,b=F,c=F,d=F)=0.6*0.3*0.4*0.4=0.0288 + // p(a=F,b=T,c=T,d=T)=0.4*0.4*0.1*0.1=0.0016 + // p(a=F,b=T,c=T,d=F)=0.4*0.4*0.1*0.9=0.0144 + // p(a=F,b=T,c=F,d=T)=0.4*0.4*0.9*0.1=0.0144 + // p(a=F,b=T,c=F,d=F)=0.4*0.4*0.9*0.9=0.1296 + // p(a=F,b=F,c=T,d=T)=0.4*0.6*0.1*0.6=0.0144 + // p(a=F,b=F,c=T,d=F)=0.4*0.6*0.1*0.4=0.0096 + // p(a=F,b=F,c=F,d=T)=0.4*0.6*0.9*0.6=0.1296 + // p(a=F,b=F,c=F,d=F)=0.4*0.6*0.9*0.4=0.0864 + + // These weights are not normalized + // p(c=T,d=T)=0.0252+0.0648+0.0016+0.0144=0.106 + // p(c=T,d=F)=0.2268+0.0432+0.0144+0.0096=0.294 -> MAP + // p(c=F,d=T)=0.0168+0.0432+0.0144+0.1296=0.204 + // p(c=F,d=F)=0 + + (c || d).observe(true) + + val alg = StructuredMarginalMAPVE(c, d) + alg.start() + alg.mostLikelyValue(c) should equal(true) + alg.mostLikelyValue(d) should equal(false) + alg.kill() + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/normalproposals/NormalProposerTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/experimental/normalproposals/NormalProposerTest.scala new file mode 100644 index 00000000..fd917e9d --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/experimental/normalproposals/NormalProposerTest.scala @@ -0,0 +1,291 @@ +/* + * NormalProposerTest.scala + * Atomic continuous element tests. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 25, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.experimental.normalproposals + +import org.scalatest.Matchers +import org.scalatest.WordSpec +import scala.math.exp +import com.cra.figaro.algorithm.sampling._ +import com.cra.figaro.experimental.normalproposals._ +import com.cra.figaro.language._ +import JSci.maths.statistics._ +import com.cra.figaro.test.tags.NonDeterministic +import com.cra.figaro.ndtest._ + +// Largely copied from ContinuousTest, but applied only to NormalProposer elements using MH. The idea is that if we +// integrate the NormalProposer elements into the main library, then we can reuse the previous tests. Some additional +// tests were added for multiple Beta parameters, since Beta only conditionally uses Normal proposals. +class NormalProposerTest extends WordSpec with Matchers { + + val alpha: Double = 0.05 + + def varStatistic(value: Double, target: Double, n: Int) = { + val df = n - 1 + + // df * sample value / target value is distributed as chisq with df degrees of freedom + val chisq = df * value / target + + // (chisq - df) / sqrt(2 * df) is approximately N(0, 1) for high df + val stat = (chisq - df) / math.sqrt(2 * df) + + stat + } + + "An AtomicUniform" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + val target = 0.25 + Universe.createNew() + val elem = Uniform(0.0, 2.0) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicUniformTestResults", target, alpha) + } + } + + ndtest.run(10) + } + } + + "An AtomicNormal" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val elem = Normal(1.0, 2.0) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new NormalDistribution(1.0, 2.0) + val target = dist.cumulative(1.2) - dist.cumulative(0.7) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result - target, NDTest.TTEST, "AtomicNormalTestResults", 0.0, alpha) + } + } + + ndtest.run(10) + } + } + + "An AtomicExponential" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val elem = Exponential(2.0) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new ExponentialDistribution(2.0) + val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicExponentialTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "An AtomicGamma" when { + "k > 1.0, theta = 1.0" should { + "compute the correct value under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val k = 2.5 + val elem = Gamma(k) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new GammaDistribution(k) + val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "k = 1.0, theta is not 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val theta = 2.0 + val elem = Gamma(1.0, theta) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + // Using the fact that for Gamma(1,theta), the CDF is given by F(x) = 1 - exp(-x/theta) + def cdf(x: Double) = 1 - exp(-x / theta) + val targetProb = cdf(1.2) - cdf(0.7) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "k = 1.0, theta = 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val k = 1.0 + val elem = Gamma(k) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new GammaDistribution(k) + val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "k < 1.0, theta = 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val k = 0.6 + val elem = Gamma(k) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new GammaDistribution(k) + val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + } + + "An AtomicBeta" when { + "a >= 1.0 and b >= 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val a = 1.3 + val b = 2.7 + val elem = Beta(a, b) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new BetaDistribution(a, b) + val targetProb = dist.cumulative(0.3) - dist.cumulative(0.2) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicBetaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "a >= 1.0 and b < 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val a = 1.2 + val b = 0.5 + val elem = Beta(a, b) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new BetaDistribution(a, b) + val targetProb = dist.cumulative(0.3) - dist.cumulative(0.2) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicBetaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "a < 1.0 and b >= 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val a = 0.3 + val b = 1.0 + val elem = Beta(a, b) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new BetaDistribution(a, b) + val targetProb = dist.cumulative(0.3) - dist.cumulative(0.2) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicBetaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + + "a < 1.0 and b < 1.0" should { + "compute the correct probability under Metropolis-Hastings" taggedAs NonDeterministic in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + val a = 0.3 + val b = 0.6 + val elem = Beta(a, b) + val alg = MetropolisHastings(20000, ProposalScheme.default, elem) + alg.start() + val dist = new BetaDistribution(a, b) + val targetProb = dist.cumulative(0.3) - dist.cumulative(0.2) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) + alg.stop() + alg.kill() + update(result, NDTest.TTEST, "AtomicBetaTestResults", targetProb, alpha) + } + } + + ndtest.run(10) + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/experimental/particlebp/PBPTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/experimental/particlebp/PBPTest.scala index 1a867242..1444f82f 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/experimental/particlebp/PBPTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/experimental/particlebp/PBPTest.scala @@ -37,6 +37,8 @@ import com.cra.figaro.experimental.particlebp.AutomaticDensityEstimator import com.cra.figaro.algorithm.sampling.ProbEvidenceSampler import com.cra.figaro.ndtest._ import org.apache.commons.math3.distribution.MultivariateNormalDistribution +import akka.util.Timeout +import java.util.concurrent.TimeUnit class PBPTest extends WordSpec with Matchers { @@ -76,12 +78,12 @@ class PBPTest extends WordSpec with Matchers { Inject(f: _*) }) val pbpSampler = ParticleGenerator(Universe.universe) - pbpSampler.update(n, pbpSampler.numArgSamples, List[(Double, _)]((1.0, 2.0))) + pbpSampler.update(n, pbpSampler.numSamplesFromAtomics, List[(Double, _)]((1.0, 2.0))) val bpb = ParticleBeliefPropagation(1, 1, items) bpb.runOuterLoop val fg_2 = bpb.bp.factorGraph.getNodes.filter(p => p.isInstanceOf[VariableNode]).toSet - pbpSampler.update(n, pbpSampler.numArgSamples, List[(Double, _)]((1.0, 3.0))) + pbpSampler.update(n, pbpSampler.numSamplesFromAtomics, List[(Double, _)]((1.0, 3.0))) val dependentElems = Set[Element[_]](n, number, items) bpb.runInnerLoop(dependentElems, Set()) // Currently have to subtract 3 since the old factors for n = 2 also get created since they exist in the chain cache @@ -114,9 +116,9 @@ class PBPTest extends WordSpec with Matchers { val b = bp.distribution(ep).toList bp.probability(e2, (i: Int) => i == 0) should be(e2_0 +- tol) - bp.probability(e2, (i: Int) => i == 1) should be(e2_1 +- tol) + bp.probability(e2)(_ == 1) should be(e2_1 +- tol) bp.probability(e2, (i: Int) => i == 2) should be(e2_2 +- tol) - bp.probability(e2, (i: Int) => i == 3) should be(e2_3 +- tol) + bp.probability(e2)(_ == 3) should be(e2_3 +- tol) } "correctly retrieve the last messages to recompute sample densities" in { @@ -209,7 +211,7 @@ class PBPTest extends WordSpec with Matchers { // U(false) = \int_{0.2}^{1.0) (1-p) val u1 = 0.35 * 0.96 val u2 = 0.32 - val result = test(f, 5000, 250, 100, 100, (b: Boolean) => b, u1 / (u1 + u2), globalTol, false) + val result = test(f, 5000, 500, 100, 100, (b: Boolean) => b, u1 / (u1 + u2), globalTol, false) update(result, NDTest.TTEST, "ConditionOnDependentElement", u1 / (u1 + u2), alpha) } } @@ -228,7 +230,7 @@ class PBPTest extends WordSpec with Matchers { // U(false) = \int_{0.2}^(1.0) (2 * (1-p)) = 0.64 val u1 = 0.85 * 0.96 val u2 = 0.64 - val result = test(f, 5000, 250, 100, 100, (b: Boolean) => b, u1 / (u1 + u2), globalTol, false) + val result = test(f, 5000, 500, 100, 100, (b: Boolean) => b, u1 / (u1 + u2), globalTol, false) update(result, NDTest.TTEST, "ConstraintOnDependentElement", u1 / (u1 + u2), alpha) } } @@ -268,7 +270,7 @@ class PBPTest extends WordSpec with Matchers { Universe.createNew() val algorithm = ParticleBeliefPropagation(1, 20, 100, 100, f)(u1) algorithm.start() - val result = algorithm.probability(f, (b: Boolean) => b) + val result = algorithm.probability(f)(b => b) algorithm.kill() update(result, NDTest.TTEST, "DifferentUniverse", 0.6, alpha) } @@ -329,7 +331,7 @@ class PBPTest extends WordSpec with Matchers { }) val algorithm = ParticleBeliefPropagation(10, 4, 15, 15, loc, locX, locY) algorithm.start() - val locE = algorithm.expectation(loc, (d: (Double, Double)) => d._1 * d._2) + val locE = algorithm.expectation(loc)(d => d._1 * d._2) val cov = locE - algorithm.mean(locX) * algorithm.mean(locY) ParticleGenerator.clear @@ -412,7 +414,9 @@ class PBPTest extends WordSpec with Matchers { val algorithm = if (oneTime) { ParticleBeliefPropagation(outer, inner, argSamples, totalSamples, target) } else { - ParticleBeliefPropagation(inner.toLong, argSamples, totalSamples, target) + val alg = ParticleBeliefPropagation(inner.toLong, argSamples, totalSamples, target) + alg.messageTimeout = Timeout(30000, TimeUnit.MILLISECONDS) + alg } algorithm.start() if (!oneTime) Thread.sleep(outer.toLong) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/language/ElementsTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/language/ElementsTest.scala index aff4d68d..2f028e5f 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/language/ElementsTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/language/ElementsTest.scala @@ -91,6 +91,8 @@ class ElementsTest extends WordSpec with Matchers { NonCachingChain(f1, fn).toString should equal("Chain(" + f1 + ", " + fn + ")") } + // No more caching, removed + /* "return a cached result" in { var sum = 0 def fn(b: Boolean) = { @@ -103,7 +105,11 @@ class ElementsTest extends WordSpec with Matchers { c.get(true) sum should equal(2) } + * + */ + // No more caching, removed + /* "call the CPD when the cache is full" in { Universe.createNew() var sum = 0 @@ -122,9 +128,29 @@ class ElementsTest extends WordSpec with Matchers { // see implementation of NonCachingChain and use of oldParentValue sum should be > (3) } + * + */ + "call the CPD for each Chain access" in { + Universe.createNew() + var sum = 0 + def fn(b: Int) = { + sum += 1 + Constant(b) + } + val f1 = Uniform(0, 1, 2) + val c = NonCachingChain(f1, fn _) + sum = 0 + c.get(0) + c.get(1) + c.get(2) + c.get(0) + sum should equal(4) + } } - "managing the context" should { + // No more local context of chains + /* + "managing the context" should { "store new elements in the correct subContext" in { Universe.createNew() val c = Chain(Flip(0.5), (b: Boolean) => if (b) Constant(0) else Constant(1)) @@ -135,7 +161,7 @@ class ElementsTest extends WordSpec with Matchers { c.myMappedContextContents(false).size should equal(1) c.elemInContext(c.myMappedContextContents(false).head) should equal(false) } - + "remove deactivated elements from context when resizing the cache" in { Universe.createNew() val c = NonCachingChain(Uniform(0, 1, 2), (b: Int) => Constant(b)) @@ -145,7 +171,7 @@ class ElementsTest extends WordSpec with Matchers { c.directContextContents.size should equal(2) c.elemInContext.size should equal(2) } - + "remove deactivated elements from context when removing temporaries" in { Universe.createNew() val c = CachingChain(com.cra.figaro.library.atomic.discrete.Uniform(0, 10), (b: Int) => Constant(b)) @@ -156,7 +182,7 @@ class ElementsTest extends WordSpec with Matchers { c.directContextContents.size should equal(1) c.elemInContext.size should equal(1) } - + "only remove elements defined in subContext" in { Universe.createNew() def fcn(b: Int) = { @@ -176,6 +202,8 @@ class ElementsTest extends WordSpec with Matchers { Universe.universe.contextContents(c) forall (_.active) should equal(true) } } + * + */ } "A chain with two parents" when { @@ -213,22 +241,18 @@ class ElementsTest extends WordSpec with Matchers { "evaluate the CPD each time get is called" in { Universe.createNew() var sum = 0 - def fn(b: (Boolean, Boolean)) = { + def fn(b1: Boolean, b2: Boolean): Element[Boolean] = { sum += 1 - Constant(b._1 && b._2) + Constant(b1 && b2) } val f1 = Flip(0.5) val f2 = Flip(0.5) f1.set(true) f2.set(false) - val c = new Chain("", ^^(f1, f2), fn, 1, Universe.universe) - //val c = NonCachingChain(f1, f2, fn _) - sum = 0 + val c = Chain(f1, f2, fn) c.get(true, true) c.get(false, false) c.get(true, true) - // either 3 or 4 depending on whether the value on initialization is true or false - // see implementation of NonCachingChain and use of oldParentValue sum should equal(3) } } @@ -265,22 +289,7 @@ class ElementsTest extends WordSpec with Matchers { CachingChain(f1, f2, fn).toString should equal("Chain(Apply(" + f1 + ", " + f2 + ", " + fn + "), )") } //((((((((((80 * .50) * 1.08) + 82.4 * .50) * 1.08) + 84.8 * .50) * 1.08) + 87.3 * .50) * 1.08) + 89.9 * .50) * 1.08) - "evaluate the CPD only once for each input" in { - var sum = 0 - def fn(b1: Boolean, b2: Boolean) = { - sum += 1 - Constant(b1 && b2) - } - val f1 = Flip(0.5) - val f2 = Flip(0.5) - f1.set(true) - f2.set(true) - val c = CachingChain(f1, f2, fn) - c.get(true, true) - c.get(false, false) - c.get(true, true) - sum should equal(2) - } + } } @@ -302,6 +311,25 @@ class ElementsTest extends WordSpec with Matchers { } } + "An ApplyC with one argument" should { + "have value equal to its function applied to its argument" in { + Universe.createNew() + val u = Uniform(0.0, 2.0) + val a = ApplyC(u)(_ + 1.0) + u.value = 1.3 + a.generate() + a.value should equal(2.3) + } + + "convert to the correct string" in { + Universe.createNew() + val u = Uniform(0.0, 2.0) + val f = (d: Double) => d + 1.0 + ApplyC(u)(f).toString should equal("Apply(" + u + ", " + f + ")") + } + + } + "An Apply with two arguments" should { "have value equal to its function applied to its arguments" in { Universe.createNew() @@ -322,6 +350,27 @@ class ElementsTest extends WordSpec with Matchers { } } + "An ApplyC with two arguments" should { + "have value equal to its function applied to its arguments" in { + Universe.createNew() + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val a = ApplyC(u, v)( _ + _ + 1.0) + u.value = 1.3 + v.value = 1.0 + a.generate() + a.value should equal(3.3) + } + + "convert to the correct string" in { + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val f = (d1: Double, d2: Double) => d1 + d2 + 1.0 + ApplyC(u, v)(f).toString should equal("Apply(" + u + ", " + v + ", " + f + ")") + } + } + + "An Apply with three arguments" should { "have value equal to its function applied to its arguments" in { Universe.createNew() @@ -346,6 +395,31 @@ class ElementsTest extends WordSpec with Matchers { } } + "An ApplyC with three arguments" should { + "have value equal to its function applied to its arguments" in { + Universe.createNew() + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val w = Select(0.5 -> 0.0, 0.5 -> 5.0) + val a = ApplyC(u, v, w)(_ + _ + _ + 1.0) + u.value = 1.3 + v.value = 1.0 + w.value = 5.0 + a.generate() + a.value should equal(8.3) + } + + "convert to the correct string" in { + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val w = Select(0.5 -> 0.0, 0.5 -> 5.0) + val f = (d1: Double, d2: Double, d3: Double) => d1 + d2 + d3 + 1.0 + ApplyC(u, v, w)(f).toString should equal( + "Apply(" + u + ", " + v + ", " + w + ", " + f + ")") + } + } + + "An Apply with four arguments" should { "have value equal to its function applied to its arguments" in { Universe.createNew() @@ -373,6 +447,34 @@ class ElementsTest extends WordSpec with Matchers { } } + "An ApplyC with four arguments" should { + "have value equal to its function applied to its arguments" in { + Universe.createNew() + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val w = Select(0.5 -> 0.0, 0.5 -> 5.0) + val x = Constant(-2.0) + val a = ApplyC(u, v, w, x)(_ + _ + _ + _ + 1.0) + u.value = 1.3 + v.value = 1.0 + w.value = 5.0 + x.value = -2.0 + a.generate() + a.value should equal(6.3) + } + + "convert to the correct string" in { + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val w = Select(0.5 -> 0.0, 0.5 -> 5.0) + val x = Constant(-2.0) + val f = (d1: Double, d2: Double, d3: Double, d4: Double) => d1 + d2 + d3 + d4 + 1.0 + ApplyC(u, v, w, x)(f).toString should equal( + "Apply(" + u + ", " + v + ", " + w + ", " + x + ", " + f + ")") + } + } + + "An Apply with five arguments" should { "have value equal to its function applied to its arguments" in { Universe.createNew() @@ -406,6 +508,40 @@ class ElementsTest extends WordSpec with Matchers { } } + "An ApplyC with five arguments" should { + "have value equal to its function applied to its arguments" in { + Universe.createNew() + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val w = Select(0.5 -> 0.0, 0.5 -> 5.0) + val x = Constant(-2.0) + val y = Constant(0.5) + val a = + ApplyC(u, v, w, x, y)(_ + _ + _ + _ + _ + 1.0) + u.value = 1.3 + v.value = 1.0 + w.value = 5.0 + x.value = -2.0 + y.value = 0.5 + a.generate() + a.value should equal(6.8) + } + + "convert to the correct string" in { + val u = Uniform(0.0, 2.0) + val v = Constant(1.0) + val w = Select(0.5 -> 0.0, 0.5 -> 5.0) + val x = Constant(-2.0) + val y = Constant(0.5) + val f = + (d1: Double, d2: Double, d3: Double, d4: Double, d5: Double) => d1 + d2 + d3 + d4 + d5 + 1.0 + ApplyC(u, v, w, x, y)(f).toString should equal( + "Apply(" + u + ", " + v + ", " + w + ", " + x + ", " + y + ", " + f + ")") + } + } + + + "An Inject" should { "have value equal to the sequence of values of its arguments" in { Universe.createNew() diff --git a/Figaro/src/test/scala/com/cra/figaro/test/language/ReferenceTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/language/ReferenceTest.scala index ac533afa..1b2571c7 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/language/ReferenceTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/language/ReferenceTest.scala @@ -24,6 +24,7 @@ import com.cra.figaro.library.atomic.continuous._ import com.cra.figaro.library.atomic.discrete import com.cra.figaro.util._ import scala.math.log +import com.cra.figaro.algorithm.factored.beliefpropagation.BeliefPropagation class ReferenceTest extends WordSpec with PrivateMethodTester with Matchers { "A Reference" when { @@ -592,7 +593,7 @@ class ReferenceTest extends WordSpec with PrivateMethodTester with Matchers { } val w = Select(0.1 -> List(new C4), 0.9 -> List(new C4, new C4))("w", u) val a = u.getAggregate((s: MultiSet[Int]) => (0 /: s)(_ + _))("w.y.x.s") - val alg = VariableElimination(a) + val alg = BeliefPropagation(100, a) //VariableElimination.debugged(a) alg.start() // C3(0) could be (1) (prob 0.2 * 0.2), (2) (prob 0.2 * 0.3), (3) (prob 0.2 * 0.5), () (prob 0.8) // C3(1) could be (1,1) (prob 0.2 * 0.2 * 0.2), (1,2) or (2,1) (prob 0.2 * 0.2 * 0.3 * 2), (1,3) or (3,1) (prob 0.2 * 0.2 * 0.5 * 2), diff --git a/Figaro/src/test/scala/com/cra/figaro/test/language/UniverseStateTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/language/UniverseStateTest.scala new file mode 100644 index 00000000..1c880f61 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/language/UniverseStateTest.scala @@ -0,0 +1,412 @@ +/* + * UniverseStateTest.scala + * Tests for saving and restoring universe state. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 3, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.language + +import com.cra.figaro.algorithm.OneTime +import com.cra.figaro.language.Element.ElemVal +import com.cra.figaro.language._ +import com.cra.figaro.library.atomic.continuous.{Normal, Uniform} +import org.scalatest.{Matchers, WordSpec} + +import scala.collection.mutable + +class UniverseStateTest extends WordSpec with Matchers { + "A universe state" when { + "restoring elements in the universe" should { + "restore value and randomness" in { + val universe = Universe.createNew() + val e1 = Uniform(0.0, 1.0) + e1.generate() + val e1Randomness = e1.randomness + val e1Value = e1.value + val state = new UniverseState(universe) + + e1.generate() + + state.restore() + e1.randomness should equal(e1Randomness) + e1.value should equal(e1Value) + } + + "restore set and unset elements" in { + val universe = Universe.createNew() + val e1 = Normal(0.0, 1.0) + val e2 = Uniform(0.0, 1.0) + e1.set(100.0) + e2.generate() + val state = new UniverseState(universe) + + e1.unset() + e1.generate() + e2.set(0.5) + + state.restore() + e1.generate() // Won't produce anything other than 100.0 unless e1 is unset + e2.generate() // Produces something other than 0.5 with probability 1 + e1.value should equal(100.0) + e2.value should not equal 0.5 + } + + "restore previous evidence with and without contingencies" when { + "conditions are changed" in { + val universe = Universe.createNew() + val e1 = Uniform(0.0, 1.0) + val e2 = Flip(0.2) + e1.addCondition(_ > 0.3) + e1.addCondition(_ < 0.8, List(ElemVal(e2, true))) + val e1Conditions = e1.allConditions + val state = new UniverseState(universe) + + e1.removeConditions() // Only removes the first (non-contingent) condition + e1.addCondition(_ > 0.5, List(ElemVal(e2, false))) + e2.addCondition((b: Boolean) => b) + + state.restore() + e1.allConditions should contain theSameElementsAs e1Conditions + e2.allConditions should be(empty) + universe.conditionedElements should contain theSameElementsAs List(e1) + } + + "constraints are changed" in { + val universe = Universe.createNew() + val e1 = Uniform(0.0, 1.0) + val e2 = Flip(0.2) + e1.addConstraint(_ * 2.0) + e1.addConstraint(_ + 1.0, List(ElemVal(e2, true))) + val e1Constraints = e1.allConstraints + val state = new UniverseState(universe) + + e1.removeConstraints() // Only removes the first (non-contingent) constraint + e1.addConstraint(_ * 3.0, List(ElemVal(e2, false))) + e2.addConstraint((b: Boolean) => if(b) 2.0 else 1.0) + + state.restore() + e1.allConstraints should contain theSameElementsAs e1Constraints + e2.allConstraints should be(empty) + universe.constrainedElements should contain theSameElementsAs List(e1) + } + + "observations are changed" in { + val universe = Universe.createNew() + val e1 = Uniform(0.0, 1.0) + val e2 = Flip(0.2) + e1.observe(0.3) + val state = new UniverseState(universe) + + e1.unobserve() + e2.observe(true) + + state.restore() + e1.observation should equal(Some(0.3)) + e2.observation should equal(None) + universe.conditionedElements should contain theSameElementsAs List(e1) + } + } + + "activate the correct elements" when { + "elements are deactivated manually" in { + val universe = Universe.createNew() + val e1 = Constant(8) + val e2 = Constant(7) + e2.deactivate() + val state = new UniverseState(universe) + + e1.deactivate() + e2.activate() + + state.restore() + e1.active should be(true) + e2.active should be(false) + universe.activeElements should contain theSameElementsAs List(e1) + } + + "temporary elements are cleared" in { + val universe = Universe.createNew() + val e1 = Constant(8) + universe.pushContext(e1) + val e2 = Constant(7) // This makes e2 temporary + val state = new UniverseState(universe) + + universe.clearTemporaries() + + state.restore() + universe.activeElements should contain theSameElementsAs List(e1, e2) + } + + "the universe is cleared" in { + val universe = Universe.createNew() + val e1 = Constant(8) + val state = new UniverseState(universe) + + universe.clear() + + state.restore() + e1.active should be(true) + universe.activeElements should contain theSameElementsAs List(e1) + } + } + + "restore context and direct context contents" in { + val universe = Universe.createNew() + val e1 = Constant(8) + universe.pushContext(e1) + val e2 = Constant(7) + val state = new UniverseState(universe) + + e2.deactivate() + + state.restore() + e1.directContextContents should contain theSameElementsAs List(e2) + e2.context should contain theSameElementsAs List(e1) + } + } + + "restoring information about the universe structure" should { + "copy the previous context stack" when { + "elements in the stack are deactivated" in { + val universe = Universe.createNew() + val e1 = Constant(8) + universe.pushContext(e1) + val e2 = Constant(7) + universe.pushContext(e2) + val e3 = Constant(6) + universe.pushContext(e3) + val state = new UniverseState(universe) + + e2.deactivate() + + state.restore() + universe.contextStack should equal(List(e3, e2, e1)) + } + + "the context stack is modified manually" in { + val universe = Universe.createNew() + val e1 = Constant(8) + universe.pushContext(e1) + val e2 = Constant(7) + universe.pushContext(e2) + val e3 = Constant(6) + universe.pushContext(e3) + val state = new UniverseState(universe) + + universe.popContext(e2) + + state.restore() + universe.contextStack should equal(List(e3, e2, e1)) + } + + "temporary elements are cleared" in { + val universe = Universe.createNew() + val e1 = Constant(8) + universe.pushContext(e1) + val e2 = Constant(7) + universe.pushContext(e2) + val e3 = Constant(6) + universe.pushContext(e3) + val state = new UniverseState(universe) + + universe.clearTemporaries() + + state.restore() + universe.contextStack should equal(List(e3, e2, e1)) + } + + "the universe is cleared" in { + val universe = Universe.createNew() + val e1 = Constant(8) + universe.pushContext(e1) + val state = new UniverseState(universe) + + universe.clear() + + state.restore() + universe.contextStack should equal(List(e1)) + } + } + + "repopulate the same stochastic elements" in { + val universe = Universe.createNew() + val e1 = Normal(0.0, 1.0) + val e2 = Uniform(0.0, 1.0) + e2.deactivate() + val state = new UniverseState(universe) + + e1.deactivate() + e2.activate() + + state.restore() + universe.stochasticElements should contain theSameElementsAs List(e1) + } + + "repopulate uses and usedBy" when { + "registered uses are changed manually" in { + val universe = Universe.createNew() + val e1 = Constant(8) + val e2 = Constant(7) + val e3 = Constant(6) + val e4 = Constant(5) + // e1 -> e2, e2 -> e3, e1 -> e4, e2 -> e4 + universe.registerUses(e2, e1) + universe.registerUses(e3, e2) + universe.registerUses(e4, e1) + universe.registerUses(e4, e2) + val state = new UniverseState(universe) + + universe.deregisterUses(e2, e1) + universe.deregisterUses(e3, e2) + universe.registerUses(e3, e1) + + state.restore() + universe.uses(e1) should be(empty) + universe.usedBy(e1) should equal(Set(e2, e3, e4)) + universe.directlyUsedBy(e1) should equal(Set(e2, e4)) + universe.uses(e2) should equal(Set(e1)) + universe.usedBy(e2) should equal(Set(e3, e4)) + universe.directlyUsedBy(e2) should equal(Set(e3, e4)) + universe.uses(e3) should equal(Set(e1, e2)) + universe.usedBy(e3) should be(empty) + universe.directlyUsedBy(e3) should be(empty) + universe.uses(e4) should equal(Set(e1, e2)) + universe.usedBy(e4) should be(empty) + universe.directlyUsedBy(e4) should be(empty) + } + + "elements are added to the model" in { + val universe = Universe.createNew() + val e1 = Constant(8) + val e2 = -e1 + val state = new UniverseState(universe) + + val e3 = e1 ++ e2 + val e4 = -e3 + + state.restore() + universe.uses(e1) should be(empty) + universe.usedBy(e1) should equal(Set(e2)) + universe.directlyUsedBy(e1) should equal(Set(e2)) + universe.uses(e2) should equal(Set(e1)) + universe.usedBy(e2) should be(empty) + universe.directlyUsedBy(e2) should be(empty) + universe.uses(e3) should be(empty) + universe.usedBy(e3) should be(empty) + universe.directlyUsedBy(e3) should be(empty) + universe.uses(e4) should be(empty) + universe.usedBy(e4) should be(empty) + universe.directlyUsedBy(e4) should be(empty) + } + + "elements are activated or deactivated" in { + val universe = Universe.createNew() + val e1 = Constant(8) + val e2 = -e1 + val e3 = -e2 + e3.deactivate() + val state = new UniverseState(universe) + val a = state.myUses(e2) + + e3.activate() + e2.deactivate() + + val b = state.myUses(e2) + + state.restore() + universe.uses(e1) should be(empty) + universe.usedBy(e1) should equal(Set(e2)) + universe.directlyUsedBy(e1) should equal(Set(e2)) + universe.uses(e2) should equal(Set(e1)) + universe.usedBy(e2) should be(empty) + universe.directlyUsedBy(e2) should be(empty) + universe.uses(e3) should be(empty) + universe.usedBy(e3) should be(empty) + universe.directlyUsedBy(e3) should be(empty) + } + + "contingent evidence changes" in { + val universe = Universe.createNew() + val e1 = Flip(0.2) + val e2 = Flip(0.3) + e1.addCondition((b: Boolean) => b, List(ElemVal(e2, false))) + val state = new UniverseState(universe) + + e1.removeConditions() + e2.addConstraint((b: Boolean) => if(b) 2.0 else 1.0, List(ElemVal(e1, true))) + + state.restore() + universe.uses(e1) should equal(Set(e2)) + universe.usedBy(e1) should be(empty) + universe.directlyUsedBy(e1) should be(empty) + universe.uses(e2) should be(empty) + universe.usedBy(e2) should equal(Set(e1)) + universe.directlyUsedBy(e2) should equal(Set(e1)) + } + } + + "not modify registered maps" when { + "registered algorithms change" in { + val universe = Universe.createNew() + var alg1Killed = false + var alg2Killed = false + val alg1 = new OneTime { override def run() = {} ; override def kill() = { alg1Killed = true } } + val alg2 = new OneTime { override def run() = {} ; override def kill() = { alg2Killed = true } } + alg1.start() + alg2.start() + universe.registerAlgorithm(alg1) + universe.deregisterAlgorithm(alg2) + val state = new UniverseState(universe) + + universe.deregisterAlgorithm(alg1) + universe.registerAlgorithm(alg2) + + state.restore() + universe.clear() // Kills registered algorithms (i.e. alg2) + alg1Killed should be(false) + alg2Killed should be(true) + } + + "registered element maps change" in { + val universe = Universe.createNew() + val e1 = Flip(0.2) + val set1: mutable.Set[Element[_]] = mutable.Set(e1) + val set2: mutable.Set[Element[_]] = mutable.Set(e1) + universe.register(set1) + val state = new UniverseState(universe) + + universe.deregister(set1) + universe.register(set2) + + state.restore() + universe.clear() // Clears registered element maps (i.e. set2) + set1 should equal(Set(e1)) + set2 should be(empty) + } + + "registered universe maps change" in { + val universe = Universe.createNew() + val map1 = mutable.Map(universe -> 1) + val map2 = mutable.Map(universe -> 2) + universe.registerUniverse(map1) + val state = new UniverseState(universe) + + universe.deregisterUniverse(map1) + universe.registerUniverse(map2) + + state.restore() + universe.clear() // Removes universe from registered maps (i.e. map2) + map1 should equal(Map(universe -> 1)) + map2 should be(empty) + } + } + } + } +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/continuous/ContinuousTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/continuous/ContinuousTest.scala index 664393c8..bac54451 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/continuous/ContinuousTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/continuous/ContinuousTest.scala @@ -24,6 +24,7 @@ import JSci.maths.SpecialMath.gamma import org.apache.commons.math3.distribution.MultivariateNormalDistribution import com.cra.figaro.test.tags.NonDeterministic import com.cra.figaro.ndtest._ +import scala.collection.mutable.ArrayBuffer class ContinuousTest extends WordSpec with Matchers { @@ -50,7 +51,7 @@ class ContinuousTest extends WordSpec with Matchers { val elem = Uniform(0.0, 2.0) val alg = Importance(20000, elem) alg.start() - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicUniformTestResults", target, alpha) @@ -68,7 +69,7 @@ class ContinuousTest extends WordSpec with Matchers { val elem = Uniform(0.0, 2.0) val alg = MetropolisHastings(20000, ProposalScheme.default, elem) alg.start() - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicUniformTestResults", target, alpha) @@ -109,7 +110,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() // p(1.25 < x < 1.5 | lower = l) = 0.25 / (2-l) // Expectation of l = \int_{0}^{1} 1 / (2-l) dl = 0.25(-ln(2-1) + ln(2-0)) = 0.1733 - val result = alg.probability(uniformComplex, (d: Double) => 1.25 <= d && d < 1.5) + val result = alg.probability(uniformComplex)(d => 1.25 <= d && d < 1.5) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundUniformTestResults", target, alpha) @@ -137,7 +138,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new NormalDistribution(1.0, 2.0) val target = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result - target, NDTest.TTEST, "AtomicNormalTestResults", 0.0, alpha) @@ -156,7 +157,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new NormalDistribution(1.0, 2.0) val target = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result - target, NDTest.TTEST, "AtomicNormalTestResults", 0.0, alpha) @@ -194,7 +195,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist2 = new NormalDistribution(1.0, 2.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(1.2) - dist.cumulative(0.7) val target = 0.5 * getProb(dist1) + 0.5 * getProb(dist2) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result - target, NDTest.TTEST, "CompoundNormalMeanResultsDiff", 0.0, alpha) @@ -225,7 +226,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist2 = new NormalDistribution(2.0, 3.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(1.2) - dist.cumulative(0.7) val targetProb = 0.5 * getProb(dist1) + 0.5 * getProb(dist2) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundNormalMeanTestResults", targetProb, alpha) @@ -257,7 +258,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist4 = new NormalDistribution(1.0, 3.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(1.2) - dist.cumulative(0.7) val target = 0.25 * getProb(dist1) + 0.25 * getProb(dist2) + 0.25 * getProb(dist3) + 0.25 * getProb(dist4) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result - target, NDTest.TTEST, "CompoundNormalTestResults", 0.0, alpha) @@ -316,12 +317,9 @@ class ContinuousTest extends WordSpec with Matchers { override def oneTest = { val sampleUniverse = Universe.createNew() val nSamples = Normal(2.5, 2.0)("", sampleUniverse) - val samples = for (i <- 1 to 25) + val samples = for (i <- 1 to 200) yield nSamples.generateValue(nSamples.generateRandomness()) -// val samplesMean = samples.sum / samples.size -// val samplesVariance = samples.map(s => (s - samplesMean) * (s - samplesMean)).sum / (samples.size - 1) - val universe = Universe.createNew() val mean = Uniform(-5, 5)("mean", universe) val variance = Uniform(0, 5)("variance", universe) @@ -330,7 +328,7 @@ class ContinuousTest extends WordSpec with Matchers { normal.observe(sample) } - val alg = Importance(20000, mean, variance) + val alg = Importance(2000, mean, variance) alg.start() val result1 = alg.mean(mean) val result2 = alg.mean(variance) @@ -431,6 +429,58 @@ class ContinuousTest extends WordSpec with Matchers { } } + "A KernelDensity" should { + "have the correct density" in { + Universe.createNew() + + val bandwidth = 2.0 + val data = Seq(1.0, 2.0, 3.0) + val dataDistrs = data.map(d => new NormalDistribution(d, bandwidth)) + + val elem = KernelDensity(data, bandwidth) + val result = elem.density(2.5) + val target = dataDistrs.map(dist => dist.probability(2.5)).sum / 3.0 + result should be(target +- 1e-6) + } + + "produce samples centered at an input point" taggedAs (NonDeterministic) in { + val ndtest = new NDTest { + override def oneTest = { + Universe.createNew() + // this gives get three disjoint normals, simplifying testing + val data = Seq(1.0, 100, 200) + // choosing a weird bandwidth to make sure it's actually getting used + val elem = KernelDensity(data, 2.3) + + val deviations = ArrayBuffer[Double]() + val selectedIndices = ArrayBuffer[Int]() + for (i <- 0 to 1000) { + val rand = elem.generateRandomness() + val value = elem.generateValue(rand) + val selectedInfo = data.zipWithIndex.map(v => (Math.pow(v._1 - value, 2), v._2)).sortBy(v => v._1).head + deviations += selectedInfo._1 + selectedIndices += selectedInfo._2 + } + val sampleStdDev = deviations.sum / (deviations.length - 1) + // selected indices should be uniform + val indDistr = selectedIndices.groupBy(d=>d).mapValues(s => 1.0 * s.length / selectedIndices.length) + + update(sampleStdDev, NDTest.TTEST, "KernelDensityTestResultsStdDev", 2.3, alpha) + update(indDistr(0), NDTest.TTEST, "KernelDensityTestResultsSampleDistr1", 0.33, alpha) + update(indDistr(1), NDTest.TTEST, "KernelDensityTestResultsSampleDistr2", 0.33, alpha) + update(indDistr(2), NDTest.TTEST, "KernelDensityTestResultsSampleDistr3", 0.33, alpha) + } + } + + ndtest.run(10) + } + + "convert to the correct string" in { + Universe.createNew() + KernelDensity(Seq(1, 2, 3), 2.0).toString should equal("KernelDensity(bandwidth=" + 2.0 + ")") + } + } + "An AtomicMultivariateNormal" should { val means = List(1.0, 2.0) val covariances = List(List(.25, .15), List(.15, .25)) @@ -526,7 +576,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new ExponentialDistribution(2.0) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicExponentialTestResults", targetProb, alpha) @@ -545,7 +595,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new ExponentialDistribution(2.0) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicExponentialTestResults", targetProb, alpha) @@ -583,7 +633,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist2 = new ExponentialDistribution(2.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(1.2) - dist.cumulative(0.7) val targetProb = 0.5 * getProb(dist1) + 0.5 * getProb(dist2) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundExponentialTestResults", targetProb, alpha) @@ -634,7 +684,7 @@ class ContinuousTest extends WordSpec with Matchers { override def oneTest = { val sampleUniverse = Universe.createNew() val nSamples = Exponential(2)("", sampleUniverse) - val samples = for (i <- 1 to 25) + val samples = for (i <- 1 to 200) yield nSamples.generateValue(nSamples.generateRandomness()) val universe = Universe.createNew() @@ -643,7 +693,7 @@ class ContinuousTest extends WordSpec with Matchers { val exponential = Exponential(lambda) exponential.observe(sample) } - val alg = Importance(20000, lambda) + val alg = Importance(2000, lambda) alg.start() val target = 2.0 val result = alg.mean(lambda) @@ -669,7 +719,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new GammaDistribution(k) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -689,7 +739,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new GammaDistribution(k) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -725,7 +775,7 @@ class ContinuousTest extends WordSpec with Matchers { // Using the fact that for Gamma(1,theta), the CDF is given by F(x) = 1 - exp(-x/theta) def cdf(x: Double) = 1 - exp(-x / theta) val targetProb = cdf(1.2) - cdf(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -746,7 +796,7 @@ class ContinuousTest extends WordSpec with Matchers { // Using the fact that for Gamma(1,theta), the CDF is given by F(x) = 1 - exp(-x/theta) def cdf(x: Double) = 1 - exp(-x / theta) val targetProb = cdf(1.2) - cdf(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -783,7 +833,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new GammaDistribution(k) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -803,7 +853,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new GammaDistribution(k) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -825,7 +875,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new GammaDistribution(k) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -845,7 +895,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new GammaDistribution(k) val targetProb = dist.cumulative(1.2) - dist.cumulative(0.7) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicGammaTestResults", targetProb, alpha) @@ -870,7 +920,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist2 = new GammaDistribution(3.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(1.2) - dist.cumulative(0.7) val targetProb = 0.5 * getProb(dist1) + 0.5 * getProb(dist2) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundGammaKTestResults", targetProb, alpha) @@ -900,7 +950,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist2 = new GammaDistribution(1.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(1.2) - dist.cumulative(0.7) val targetProb = 0.5 * getProb(dist1) + 0.5 * getProb(dist2) - val result = alg.probability(elem, (d: Double) => 0.7 <= d && d < 1.2) + val result = alg.probability(elem)(d => 0.7 <= d && d < 1.2) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundGammaTestResults", targetProb, alpha) @@ -955,7 +1005,7 @@ class ContinuousTest extends WordSpec with Matchers { val sampleUniverse = Universe.createNew() val nSamples = Gamma(2, 2)("", sampleUniverse) - val samples = for (i <- 1 to 25) + val samples = for (i <- 1 to 200) yield nSamples.generateValue(nSamples.generateRandomness()) val universe = Universe.createNew() @@ -966,7 +1016,7 @@ class ContinuousTest extends WordSpec with Matchers { gamma.observe(sample) } - val alg = Importance(20000, k, theta) + val alg = Importance(2000, k, theta) alg.start() val resultK = alg.mean(k) val resultTheta = alg.mean(theta) @@ -993,7 +1043,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new BetaDistribution(a, b) val targetProb = dist.cumulative(0.3) - dist.cumulative(0.2) - val result = alg.probability(elem, (d: Double) => 0.2 <= d && d < 0.3) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicBetaTestResults", targetProb, alpha) @@ -1014,7 +1064,7 @@ class ContinuousTest extends WordSpec with Matchers { alg.start() val dist = new BetaDistribution(a, b) val targetProb = dist.cumulative(0.3) - dist.cumulative(0.2) - val result = alg.probability(elem, (d: Double) => 0.2 <= d && d < 0.3) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicBetaTestResults", targetProb, alpha) @@ -1055,7 +1105,7 @@ class ContinuousTest extends WordSpec with Matchers { val dist4 = new BetaDistribution(1.0, 3.0) def getProb(dist: ProbabilityDistribution) = dist.cumulative(0.3) - dist.cumulative(0.2) val targetProb = 0.25 * getProb(dist1) + 0.25 * getProb(dist2) + 0.25 * getProb(dist3) + 0.25 * getProb(dist4) - val result = alg.probability(elem, (d: Double) => 0.2 <= d && d < 0.3) + val result = alg.probability(elem)(d => 0.2 <= d && d < 0.3) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundBetaTestResults", targetProb, alpha) @@ -1109,7 +1159,7 @@ class ContinuousTest extends WordSpec with Matchers { override def oneTest = { val sampleUniverse = Universe.createNew() val nSamples = Beta(2, 5)("", sampleUniverse) - val samples = for (i <- 1 to 25) + val samples = for (i <- 1 to 200) yield nSamples.generateValue(nSamples.generateRandomness()) val universe = Universe.createNew() @@ -1119,7 +1169,7 @@ class ContinuousTest extends WordSpec with Matchers { val beta = Beta(a, b) beta.observe(sample) } - val alg = Importance(40000, a, b) + val alg = Importance(2000, a, b) alg.start() val resultA = alg.mean(a) val resultB = alg.mean(b) @@ -1151,7 +1201,7 @@ class ContinuousTest extends WordSpec with Matchers { val r = ds(0) / (ds(0) + ds(1)) 0.2 <= r && r < 0.3 } - val result = alg.probability(elem, (ds: Array[Double]) => check(ds)) + val result = alg.probability(elem)(ds => check(ds)) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicDirichletTestResults", targetProb, alpha) @@ -1176,7 +1226,7 @@ class ContinuousTest extends WordSpec with Matchers { val r = ds(0) / (ds(0) + ds(1)) 0.2 <= r && r < 0.3 } - val result = alg.probability(elem, (ds: Array[Double]) => check(ds)) + val result = alg.probability(elem)(ds => check(ds)) alg.stop() alg.kill update(result, NDTest.TTEST, "AtomicDirichletTestResults", targetProb, alpha) @@ -1226,7 +1276,7 @@ class ContinuousTest extends WordSpec with Matchers { val r = ds(0) / (ds(0) + ds(1)) 0.2 <= r && r < 0.3 } - val result = alg.probability(elem, (ds: Array[Double]) => check(ds)) + val result = alg.probability(elem)(ds => check(ds)) alg.stop() alg.kill update(result, NDTest.TTEST, "CompoundDirichletTestResults", targetProb, alpha) @@ -1293,7 +1343,7 @@ class ContinuousTest extends WordSpec with Matchers { override def oneTest = { val sampleUniverse = Universe.createNew() val nSamples = Dirichlet(1, 2, 3)("", sampleUniverse) - val samples = for (i <- 1 to 25) + val samples = for (i <- 1 to 200) yield nSamples.generateValue(nSamples.generateRandomness()) val universe = Universe.createNew() @@ -1304,7 +1354,7 @@ class ContinuousTest extends WordSpec with Matchers { val dirichlet = Dirichlet(alpha1, alpha2, alpha3) dirichlet.observe(sample) } - val alg = Importance(30000, alpha1, alpha2, alpha3) + val alg = Importance(2000, alpha1, alpha2, alpha3) alg.start() val resultA = alg.mean(alpha1) val resultB = alg.mean(alpha2) diff --git a/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/discrete/DiscreteTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/discrete/DiscreteTest.scala index 002bb0f5..07380772 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/discrete/DiscreteTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/library/atomic/discrete/DiscreteTest.scala @@ -30,7 +30,7 @@ class DiscreteTest extends WordSpec with Matchers { val elem = Uniform(1, 2, 3, 4) val alg = Importance(20000, elem) alg.start() - alg.probability(elem, (i: Int) => 1 <= i && i < 3) should be(0.5 +- 0.01) + alg.probability(elem)(i => 1 <= i && i < 3) should be(0.5 +- 0.01) } "for an input within the options have density equal to 1 divided by the number of options" in { @@ -76,7 +76,7 @@ class DiscreteTest extends WordSpec with Matchers { val alg = Importance(20000, elem) alg.start() val targetProb = 0.5 * 0.5 + 0.5 * 0.3 - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "convert to the correct string" in { @@ -95,7 +95,7 @@ class DiscreteTest extends WordSpec with Matchers { val alg = Importance(20000, elem) alg.start() val targetProb = 0.9 * 0.9 * 0.1 - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "produce the correct result under Metropolis-Hastings" in { @@ -104,7 +104,7 @@ class DiscreteTest extends WordSpec with Matchers { val alg = MetropolisHastings(50000, ProposalScheme.default, elem) alg.start() val targetProb = 0.9 * 0.9 * 0.1 - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "have the correct density" in { @@ -127,7 +127,7 @@ class DiscreteTest extends WordSpec with Matchers { val alg = Importance(20000, elem) alg.start() val targetProb = 0.5 * 0.9 * 0.9 * 0.1 + 0.5 * 0.7 * 0.7 * 0.3 - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "convert to the correct string" in { @@ -145,7 +145,7 @@ class DiscreteTest extends WordSpec with Matchers { alg.start() val dist = new PoissonDistribution(0.9) val targetProb = dist.probability(3) - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "produce the correct result under Metropolis-Hastings" in { @@ -155,7 +155,7 @@ class DiscreteTest extends WordSpec with Matchers { alg.start() val dist = new PoissonDistribution(0.9) val targetProb = dist.probability(3) - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "have the correct density" in { @@ -191,7 +191,7 @@ class DiscreteTest extends WordSpec with Matchers { val dist1 = new PoissonDistribution(0.7) val dist2 = new PoissonDistribution(0.9) val targetProb = 0.5 * dist1.probability(3) + 0.5 * dist2.probability(3) - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "convert to the correct string" in { @@ -209,7 +209,7 @@ class DiscreteTest extends WordSpec with Matchers { alg.start() val dist = new BinomialDistribution(5, 0.9) val targetProb = dist.probability(3) - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "produce the correct result under Metropolis-Hastings" in { @@ -219,7 +219,7 @@ class DiscreteTest extends WordSpec with Matchers { alg.start() val dist = new BinomialDistribution(5, 0.9) val targetProb = dist.probability(3) - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "have the correct density" in { @@ -283,7 +283,7 @@ class DiscreteTest extends WordSpec with Matchers { val dist1 = new BinomialDistribution(5, 0.7) val dist2 = new BinomialDistribution(5, 0.9) val targetProb = 0.5 * dist1.probability(3) + 0.5 * dist2.probability(3) - alg.probability(elem, (i: Int) => i == 3) should be(targetProb +- 0.01) + alg.probability(elem)(_ == 3) should be(targetProb +- 0.01) } "convert to the correct string" in { @@ -300,7 +300,7 @@ class DiscreteTest extends WordSpec with Matchers { val elem = SwitchingFlip(0.8) val alg = Importance(40000, elem) alg.start() - alg.probability(elem, (b: Boolean) => b) should be(0.8 +- 0.01) + alg.probability(elem)(b => b) should be(0.8 +- 0.01) } "always produce the opposite value for the next randomness" in { @@ -324,7 +324,7 @@ class DiscreteTest extends WordSpec with Matchers { val elem = SwitchingFlip(0.7) val alg = MetropolisHastings(20000, ProposalScheme.default, elem) alg.start() - alg.probability(elem, (b: Boolean) => b) should be(0.7 +- 0.01) + alg.probability(elem)(b => b) should be(0.7 +- 0.01) } "produce the right probability when conditioned under Metropolis-Hastings" in { @@ -336,7 +336,7 @@ class DiscreteTest extends WordSpec with Matchers { alg.start() val p1 = 0.2 * 0.7 val p2 = 0.8 * 0.4 - alg.probability(elem1, (b: Boolean) => b) should be(p1 / (p1 + p2) +- 0.01) + alg.probability(elem1)(b => b) should be(p1 / (p1 + p2) +- 0.01) } } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/library/collection/ContainerElementTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/library/collection/ContainerElementTest.scala index 80907c46..e2175a7d 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/library/collection/ContainerElementTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/library/collection/ContainerElementTest.scala @@ -61,7 +61,7 @@ class ContainerElementTest extends WordSpec with Matchers { "map a function through all possible values correctly" in { Universe.createNew() - val contElem = create() + val contElem = create() VariableElimination.probability(contElem.map(!_)(0), false) should be ((0.5 * 0.1 + 0.5 * 0.3) +- 0.0000000001) } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/library/compound/CompoundTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/library/compound/CompoundTest.scala index 56fc4c53..1f21f098 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/library/compound/CompoundTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/library/compound/CompoundTest.scala @@ -69,14 +69,17 @@ class CompoundTest extends WordSpec with Matchers { "not generate a consequent that's not needed when sampling" in { Universe.createNew() var count = 0 - def makeElem() = { + def makeElem1() = { + Constant(1) + } + def makeElem2() = { count += 1 Constant(1) } - val e = If(Constant(true), makeElem(), makeElem()) + val e = If(Constant(true), makeElem1(), makeElem2()) val alg = Importance(100, e) alg.start() - count should equal (1) + count should equal (0) alg.kill() } } @@ -526,6 +529,21 @@ class CompoundTest extends WordSpec with Matchers { +- 0.00000000001) } } + + "A Rich CPD using an implicit universe" should { + "should add elements automatically to implicit universe instead of default" in { + Universe.createNew() + val otherUniverse = new Universe + implicit val implicitUniverse: Universe = otherUniverse + val x = Constant(true) + val y = Constant(true) + val z: Element[Boolean] = RichCPD(x, y, + (OneOf(true), *) -> Constant(true), + (OneOf(false), *) -> Constant(false)) + Universe.universe.activeElements.size should be(0) + otherUniverse.activeElements.size should be(6) + } + } "A MakeList" should { "have the number of items be distributed according to the first argument" in { diff --git a/Figaro/src/test/scala/com/cra/figaro/test/library/decision/DecisionTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/library/decision/DecisionTest.scala index 92daa5e2..e58abebf 100644 --- a/Figaro/src/test/scala/com/cra/figaro/test/library/decision/DecisionTest.scala +++ b/Figaro/src/test/scala/com/cra/figaro/test/library/decision/DecisionTest.scala @@ -45,20 +45,6 @@ class DecisionTest extends WordSpec with Matchers { } - "A NonCachingDecision" should { - - "remove previous elements when parent value changes" in { - val U = Universe.createNew() - val u1 = Uniform((0 until 5): _*) - val d1 = NonCachingDecision(u1, 5 until 10) - d1.setPolicy((i: Int) => Constant(i)) - val prev = d1.get(u1.value) - val curr = d1.get((u1.value + 1) % 5) - U.uses(d1) should not contain (prev) - prev.active should be(false) - } - - } } diff --git a/Figaro/src/test/scala/com/cra/figaro/test/tags/BookExample.scala b/Figaro/src/test/scala/com/cra/figaro/test/tags/BookExample.scala new file mode 100644 index 00000000..205dcd70 --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/tags/BookExample.scala @@ -0,0 +1,18 @@ +/* + * BookExample.scala + * Tag for book example tests. + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Feb 23, 2016 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.tags + +import org.scalatest.Tag + +object BookExample extends Tag("com.cra.figaro.test.BookExample") diff --git a/Figaro/src/test/scala/com/cra/figaro/test/util/CacheTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/util/CacheTest.scala new file mode 100644 index 00000000..669c266b --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/util/CacheTest.scala @@ -0,0 +1,136 @@ +/* + * CacheTest.scala + * Needs description + * + * Created By: Avi Pfeffer (apfeffer@cra.com) + * Creation Date: Jan 1, 2009 + * + * Copyright 2013 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.util + +import org.scalatest.Matchers +import org.scalatest.WordSpec +import com.cra.figaro.language.CachingChain +import com.cra.figaro.language.Chain +import com.cra.figaro.language.Constant +import com.cra.figaro.language.Flip +import com.cra.figaro.language.Universe +import com.cra.figaro.library.atomic.continuous.Uniform +import com.cra.figaro.library.cache.MHCache +import com.cra.figaro.library.compound.If +import com.cra.figaro.library.cache.PermanentCache + +class CacheTest extends WordSpec with Matchers { + "A MH cache" should { + "correctly retrieve cache elements for caching chains" in { + val u = Universe.createNew() + val cc = new MHCache(u) + var sum = 0 + def fn(b: Boolean) = { + sum += 1 + Constant(b) + } + val f = Flip(0.5) + val c = CachingChain(f, fn) + f.value = true; cc(c) + f.value = false; cc(c) + f.value = true; cc(c) + sum should equal(2) + } + + "keep the stack at maximum of two for non-caching chains" in { + val u = Universe.createNew() + val cc = new MHCache(u) + val f = Uniform(0.0, 1.0) + val c = Chain(f, (d: Double) => Constant(d)) + for { _ <- 0 until 10 } { + f.generate + cc(c) + } + cc.nccCache(c).size should equal(2) + } + + "remove deactivated elements from the cache" in { + val u = Universe.createNew() + val cc = new MHCache(u) + val a1 = Flip(0.1) + val a2 = Flip(0.2) + val s = Flip(0.5) + val c = If(s, a1, a2) + s.value = true; cc(c) + s.value = false; cc(c) + cc.ccCache(c).size should equal(2) + a1.deactivate + cc.ccCache(c).size should equal(1) + } + + "correctly clear the context of elements removed from the stack" in { + val u = Universe.createNew() + val cc = new MHCache(u) + def fn(d: Double) = { + Flip(d); Flip(d); Flip(d) + } + val f = Uniform(0.0, 1.0) + val c = Chain(f, fn) + + f.generate; cc(c) + c.directContextContents.size should equal(3) + f.generate; cc(c) + c.directContextContents.size should equal(6) + f.generate; cc(c) + c.directContextContents.size should equal(6) + u.activeElements.size should equal(8) + } + + "correctly clear the caches when clearing temporaries" in { + val u = Universe.createNew() + val cc = new MHCache(u) + def fn(d: Double) = { + Flip(d); Flip(d); Flip(d) + } + val f = Uniform(0.0, 1.0) + val perm = Constant(false) + val fl = Flip(0.5) + val c2 = CachingChain(fl, (b: Boolean) => { + if (b) perm + else { + Flip(f) + } + }) + + for { _ <- 0 until 100 } { + f.generate + fl.generate + cc(c2) + } + u.clearTemporaries + u.activeElements.size should equal(4) + cc.ccCache(c2).size should equal(1) + cc.ccInvertedCache(perm).size should equal(1) + } + } + + "A Permanent cache" should { + "not throw an exception if something is already removed" in { + val f1 = Flip(0.1) + val f2 = Flip(0.3) + val s = Flip(0.5) + val c = Chain(s, (b: Boolean) => if (b) f1 else f2) + + s.generate + f1.generate + f2.generate + val cache = new PermanentCache(Universe.universe) + val r = cache(c) + cache -= c + cache -= f1 + cache -= f2 + } + } + +} diff --git a/Figaro/src/test/scala/com/cra/figaro/test/util/LogStatisticsTest.scala b/Figaro/src/test/scala/com/cra/figaro/test/util/LogStatisticsTest.scala new file mode 100644 index 00000000..6280c35f --- /dev/null +++ b/Figaro/src/test/scala/com/cra/figaro/test/util/LogStatisticsTest.scala @@ -0,0 +1,203 @@ +/* + * LogStatistics.scala + * Tests for statistics in log space. + * + * Created By: William Kretschmer (kretsch@mit.edu) + * Creation Date: Aug 22, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.test.util + +import com.cra.figaro.util._ +import com.cra.figaro.util.LogStatistics.oneSidedTTest +import org.scalatest.{Matchers, WordSpec} + +class LogStatisticsTest extends WordSpec with Matchers { + // Used to test underflow/overflow + val logOffset = 100000 + + "Running a one-sided t-test" should { + "throw an exception" when { + "one of the counts is less than 2" in { + val v1 = LogStatistics(0.0, 1.0, 1) + val v2 = LogStatistics(1.0, 1.0, 30) + an [IllegalArgumentException] shouldBe thrownBy(oneSidedTTest(v1, v2)) + an [IllegalArgumentException] shouldBe thrownBy(oneSidedTTest(v2, v1)) + } + } + + "handle 0 variance" when { + "one variance is 0" in { + val v1 = LogStatistics(0.0, Double.NegativeInfinity, 20) + val v2 = LogStatistics(Double.NegativeInfinity, 0.0, 20) + val v3 = LogStatistics(0.2, 0.0, 20) + val v4 = LogStatistics(0.0, 0.0, 20) + + oneSidedTTest(v1, v2) should be (1.3060E-4 +- 1E-8) + oneSidedTTest(v1, v3) should be (0.16727 +- 1E-5) + oneSidedTTest(v1, v4) should be (0.5 +- 1E-5) + } + + "both variances are 0" in { + val v1 = LogStatistics(0.0, Double.NegativeInfinity, 20) + val v2 = LogStatistics(Double.NegativeInfinity, Double.NegativeInfinity, 20) + + oneSidedTTest(v1, v2) should be (0.0 +- 1E-5) + oneSidedTTest(v1, v1) should be (0.5 +- 1E-5) + oneSidedTTest(v2, v2) should be (0.5 +- 1E-5) + } + } + + "handle 0 mean" when { + "one mean is 0" in { + val v1 = LogStatistics(Double.NegativeInfinity, 0.0, 20) + val v2 = LogStatistics(0.0, 1.0, 20) + + oneSidedTTest(v1, v2) should be (0.013536 +- 1E-6) + } + + "both means are 0" in { + val v1 = LogStatistics(Double.NegativeInfinity, 0.0, 20) + val v2 = LogStatistics(Double.NegativeInfinity, 1.0, 20) + + oneSidedTTest(v1, v2) should be (0.5 +- 1E-5) + } + } + + "compute degrees of freedom" when { + "the counts are the same" in { + val v1 = LogStatistics(0.0, 0.0, 5) + val v2 = LogStatistics(0.1, 0.1, 5) + + oneSidedTTest(v1, v2) should be (0.43763 +- 1E-5) + } + + "the counts are different" in { + val v1 = LogStatistics(0.0, 0.0, 3) + val v2 = LogStatistics(0.1, 0.1, 7) + + oneSidedTTest(v1, v2) should be (0.44396 +- 1E-5) + } + } + + "compare the lesser mean to the greater mean" when { + "the means are the same" in { + val v1 = LogStatistics(0.0, 0.0, 20) + val v2 = LogStatistics(0.0, 0.1, 30) + + oneSidedTTest(v1, v2) should be (0.5 +- 1E-5) + oneSidedTTest(v2, v1) should be (0.5 +- 1E-5) + } + + "the means are different" in { + val v1 = LogStatistics(0.0, 0.0, 20) + val v2 = LogStatistics(0.1, 0.1, 30) + + oneSidedTTest(v1, v2) should be (0.36147 +- 1E-5) + oneSidedTTest(v2, v1) should be (0.36147 +- 1E-5) + } + } + + "remain stable in log space" when { + "exponentiation could underflow" in { + val v1 = LogStatistics(0.0, 0.0, 20).multiplyByConstant(-logOffset) + val v2 = LogStatistics(0.1, 0.1, 30).multiplyByConstant(-logOffset) + + oneSidedTTest(v1, v2) should be (0.36147 +- 1E-5) + } + + "exponentiation could overflow" in { + val v1 = LogStatistics(0.0, 0.0, 20).multiplyByConstant(logOffset) + val v2 = LogStatistics(0.1, 0.1, 30).multiplyByConstant(logOffset) + + oneSidedTTest(v1, v2) should be (0.36147 +- 1E-5) + } + } + } + + "Running online log statistics" should { + "handle 0 observations" when { + "all of the observations are 0" in { + val ols = new OnlineLogStatistics {} + val numObservations = 100 + for(_ <- 1 to numObservations) ols.record(Double.NegativeInfinity) + + val logStats = ols.totalLogStatistics + logStats.logMean should be (Double.NegativeInfinity) + logStats.logVariance should be (Double.NegativeInfinity) + logStats.count should be (numObservations) + } + + "some of the observations are 0" in { + val ols = new OnlineLogStatistics {} + val observations = List(Double.NegativeInfinity, 0.0, -0.5, Double.NegativeInfinity, -2.2, 0.1) + observations.foreach(ols.record) + + val logStats = ols.totalLogStatistics + logStats.logMean should be (-0.75413 +- 1E-5) + logStats.logVariance should be (-1.3674 +- 1E-4) + } + } + + "compute the correct result" when { + "given no observations" in { + val ols = new OnlineLogStatistics {} + + val logStats = ols.totalLogStatistics + logStats.logMean should be (Double.NegativeInfinity) + logStats.logVariance.isNaN should be (true) + logStats.count should be (0) + } + + "given one observation" in { + val ols = new OnlineLogStatistics {} + ols.record(1.5) + + val logStats = ols.totalLogStatistics + logStats.logMean should be (1.5 +- 1E-4) + logStats.logVariance.isNaN should be (true) + logStats.count should be (1) + } + + "given several observations" in { + val ols = new OnlineLogStatistics {} + val observations = List(-3.7, 2.1, 0.8, 1.2, 0.8, -0.5) + observations.foreach(ols.record) + + val logStats = ols.totalLogStatistics + logStats.logMean should be (1.0158 +- 1E-4) + logStats.logVariance should be (2.1337 +- 1E-4) + logStats.count should be (observations.length) + } + } + + "remain stable in log space" when { + "exponentiation could underflow" in { + val ols = new OnlineLogStatistics {} + val observations = List(-3.7, 2.1, 0.8, 1.2, 0.8, -0.5).map(_ - logOffset) + observations.foreach(ols.record) + + val logStats = ols.totalLogStatistics.multiplyByConstant(logOffset) + logStats.logMean should be (1.0158 +- 1E-4) + logStats.logVariance should be (2.1337 +- 1E-4) + logStats.count should be (observations.length) + } + + "exponentiation could overflow" in { + val ols = new OnlineLogStatistics {} + val observations = List(-3.7, 2.1, 0.8, 1.2, 0.8, -0.5).map(_ + logOffset) + observations.foreach(ols.record) + + val logStats = ols.totalLogStatistics.multiplyByConstant(-logOffset) + logStats.logMean should be (1.0158 +- 1E-4) + logStats.logVariance should be (2.1337 +- 1E-4) + logStats.count should be (observations.length) + } + } + } +} diff --git a/FigaroExamples/.gitignore b/FigaroExamples/.gitignore index ae3c1726..9bfdb952 100644 --- a/FigaroExamples/.gitignore +++ b/FigaroExamples/.gitignore @@ -1 +1,2 @@ /bin/ +/.cache-main diff --git a/FigaroExamples/META-INF/MANIFEST.MF b/FigaroExamples/META-INF/MANIFEST.MF index 45a4a81e..9a6ff07d 100644 --- a/FigaroExamples/META-INF/MANIFEST.MF +++ b/FigaroExamples/META-INF/MANIFEST.MF @@ -2,8 +2,8 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: FigaroExamples Bundle-SymbolicName: com.cra.figaro.examples -Bundle-Version: 3.2.0 -Require-Bundle: com.cra.figaro;bundle-version="3.2.0", +Bundle-Version: 4.1.0 +Require-Bundle: com.cra.figaro;bundle-version="4.1.0", org.scala-lang.scala-library Bundle-Vendor: Charles River Analytics Bundle-RequiredExecutionEnvironment: JavaSE-1.6 diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/CarAndEngine.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/CarAndEngine.scala index 561e29fc..7156e70e 100644 --- a/FigaroExamples/src/main/scala/com/cra/figaro/example/CarAndEngine.scala +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/CarAndEngine.scala @@ -55,7 +55,7 @@ object CarAndEngine { val alg = VariableElimination(car.speed) alg.start() alg.stop() - println(alg.expectation(car.speed, (d: Double) => d)) + println(alg.expectation(car.speed)(d => d)) alg.kill() } } diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/FairDice.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/FairDice.scala index aa3c3e29..c555e4d4 100644 --- a/FigaroExamples/src/main/scala/com/cra/figaro/example/FairDice.scala +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/FairDice.scala @@ -20,6 +20,7 @@ import com.cra.figaro.library.atomic.continuous._ import com.cra.figaro.patterns.learning.ParameterCollection import com.cra.figaro.patterns.learning.ModelParameters import com.cra.figaro.language.Select +import com.cra.figaro.algorithm.factored.VariableElimination /** * A model for learning fairness of three dice. In each trial, choose two die randomly and their sum is observed. * This model will learn the fairness of each die. @@ -89,15 +90,19 @@ object FairDice { val d2 = Select(params.posteriorParameters.get("fairness2"), outcomes:_*) val d3 = Select(params.posteriorParameters.get("fairness3"), outcomes:_*) - val probsAndSides1 = d1.probs zip outcomes - probsAndSides1.map(a => println("\t" + a._1 + " -> " + a._2)) + val ve = VariableElimination(d1,d2,d3) + ve.start() + ve.stop() + + println("\nThe probabilities of seeing each side of d_1 are: ") + outcomes.foreach { o => println("\t" + ve.probability(d1)(_ == o) + " -> " + o) } println("\nThe probabilities of seeing each side of d_2 are: ") - val probsAndSides2 = d2.probs zip outcomes - probsAndSides2.map(a => println("\t" + a._1 + " -> " + a._2)) + outcomes.foreach { o => println("\t" + ve.probability(d2)( _ == o) + " -> " + o) } println("\nThe probabilities of seeing each side of d_3 are: ") - val probsAndSides3 = d3.probs zip outcomes - probsAndSides3.map(a => println("\t" + a._1 + " -> " + a._2)) + outcomes.foreach { o => println("\t" + ve.probability(d3)( _ == o) + " -> " + o) } + + ve.kill() algorithm.kill } diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/GaussianProcessTraining.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/GaussianProcessTraining.scala new file mode 100644 index 00000000..63876981 --- /dev/null +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/GaussianProcessTraining.scala @@ -0,0 +1,131 @@ +/* + * GaussianProcessTraining.scala + * Demonstrates use of externally trained models in Figaro. + * + * Created By: Dan Garant (dgarant@cra.com) + * Creation Date: May 20, 2016 + * + * Copyright 2016 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ + +package com.cra.figaro.example + +import org.apache.commons.math3.linear._ +import com.cra.figaro.language._ +import com.cra.figaro.util.random +import com.cra.figaro.library.atomic._ +import com.cra.figaro.library.atomic.continuous.Normal +import com.cra.figaro.algorithm.sampling.Importance + +/** + * Trains a GaussianProcess model and uses it as part of a chain + */ +object GaussianProcessTraining { + + def main(args:Array[String]) = { + + // set up the model + // y = x^2 + eps, eps ~ N(0, 1) + val x = Range.Double(1, 10, 1) + val y = x.map(xi => math.pow(xi, 2) + random.nextGaussian()) + + // wire together dependence structure + val gp = new GaussianProcess(new GaussianCovarianceFunction(1 / 2.0)) + gp.train(x zip y toList) + val xElement = continuous.Uniform(0, 11) + val yElement = Chain(xElement, gp.model) + + // estimate conditional expectation + xElement.observe(7.5) + var importance = Importance(1000, yElement) + importance.start() + val expectedYVal = importance.computeExpectation(yElement, (v: Double) => v) + importance.kill() + println("E[Y|X=7.5] = " + expectedYVal) + + // now adding an effect of y + val zElement = Chain(yElement, (v:Double) => Normal(v + 3, 1)) + + importance = Importance(1000, zElement) + importance.start() + val expectedZVal = importance.computeExpectation(zElement, (v: Double) => v) + importance.kill() + println("E[Z|X=7.5] = " + expectedZVal) + } +} + +/** + * General form of a covariance function, + * taking two items of type T and producing a measure + */ +trait CovarianceFunction[T] { + def apply(v1: T, v2: T): Double +} + +/** The Gaussian, or radial basis function kernel / covariance function between univariate observations */ +class GaussianCovarianceFunction(var gamma: Double) extends CovarianceFunction[Double] { + + /** Computes covariance using the L2 norm */ + override def apply(v1: Double, v2: Double): Double = { + Math.exp(-gamma * Math.pow(v1 - v2, 2)) + } + + override def toString = "GaussianCovarianceFunction(gamma=" + gamma + ")" +} + +/** + * Estimates and performs posterior prediction from a Gaussian process. + * @param covarianceFunction Defines the inner product between feature vectors + * @param noiseVariance Prior variance of noise at design points + */ +class GaussianProcess[Input](var covarianceFunction: CovarianceFunction[Input], noiseVariance: Double = 0.001) { + + var priorMean: Double = 0 + var covarianceInverse: RealMatrix = null + var inputs: Seq[Input] = null + var alpha: RealMatrix = null + var responses: RealVector = null + var numDimensions: Integer = null + + /** + * Estimate the conditional density of a new point. + * @param newInputs The point to evaluate at + * @returns Posterior normal distribution + */ + def model(newInput: Input): Element[Double] = { + if (covarianceInverse == null) { + throw new IllegalArgumentException("The Gaussian process must be fit before 'model' can be called") + } + + val newCovariance = new ArrayRealVector((0 until inputs.length).map(i => covarianceFunction(inputs(i), newInput)).toArray) + var variance = 1 - covarianceInverse.preMultiply(newCovariance).dotProduct(newCovariance) + val mean = alpha.preMultiply(newCovariance).getEntry(0) + + Normal(priorMean + mean, Math.sqrt(variance)) + } + + /** + * Estimates the parameters of the Gaussian process (the inverse of the covariance matrix) from data + * @param data A sequence of input, output pairs to use when fitting the model + */ + def train(data: List[(Input, Double)]) = { + inputs = data map { _._1 } + priorMean = (data map { _._2 } sum) / data.length + responses = new ArrayRealVector(data map { _._2 - priorMean } toArray) + + // construct covariance matrix + val rows = (0 until data.length).map(i => { + (0 until data.length).map(j => { + covarianceFunction(data(i)._1, data(j)._1) + }).toArray + }).toArray + + val covariance = new Array2DRowRealMatrix(rows, false) + covarianceInverse = MatrixUtils.inverse(covariance.add(MatrixUtils.createRealIdentityMatrix(data.length).scalarMultiply(noiseVariance))) + + alpha = covarianceInverse.multiply(new Array2DRowRealMatrix(responses toArray)) + } +} diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/Sources.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/Sources.scala index 01794a39..b52aaf37 100644 --- a/FigaroExamples/src/main/scala/com/cra/figaro/example/Sources.scala +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/Sources.scala @@ -77,7 +77,7 @@ object Sources { def peAlg(universe: Universe, evidence: List[NamedEvidence[_]]) = () => ProbEvidenceSampler.computeProbEvidence(100000, evidence)(universe) val alg = VariableElimination(List(ue1, ue2, ue3, ue4), peAlg _, sample1.fromSource) alg.start() - val result = alg.probability(sample1.fromSource, (s: Source) => s == source1) + val result = alg.probability(sample1.fromSource)(_ == source1) println("Probability of Source 1: " + result) alg.kill() diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/Burglary.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/Burglary.scala new file mode 100644 index 00000000..e1605d19 --- /dev/null +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/Burglary.scala @@ -0,0 +1,53 @@ +/* + * Burglary.scala + * A Bayesian network example with visualization + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jun 15, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.example.visualization + +import com.cra.figaro.algorithm.factored._ +import com.cra.figaro.language._ +import com.cra.figaro.library.compound._ +import com.cra.figaro.util.visualization.{ResultsGUI} +import com.cra.figaro.util.visualization.results.{DiscreteData} +/** + * A Bayesian network example + */ +object Burglary { + Universe.createNew() + + private val burglary = Flip(0.01) + + private val earthquake = Flip(0.0001) + + private val alarm = CPD(burglary, earthquake, + (false, false) -> Flip(0.001), + (false, true) -> Flip(0.1), + (true, false) -> Flip(0.9), + (true, true) -> Flip(0.99)) + + private val johnCalls = CPD(alarm, + false -> Flip(0.01), + true -> Flip(0.7)) + + def main(args: Array[String]) { + val gui = ResultsGUI + gui.startup(args) + + johnCalls.observe(true) + val alg = VariableElimination(burglary, earthquake) + alg.start() + println("Probability of burglary: " + alg.probability(burglary, true)) + + gui.addResult("burglary", alg.distribution(burglary).toList) + + alg.kill + } +} diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/ContinuousExample.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/ContinuousExample.scala new file mode 100644 index 00000000..ac248a42 --- /dev/null +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/ContinuousExample.scala @@ -0,0 +1,30 @@ +/* + * ContinuousExample.scala + * An example using continuous elements. + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jul 3, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.example.visualization + +import com.cra.figaro.library.atomic.continuous.{Exponential, Normal} +import com.cra.figaro.util.visualization._ +import com.cra.figaro.util.visualization.results.{ContinuousData} + +/** + * @author gtakata + */ +object ContinuousExample { + + def main(args: Array[String]) { + val gui = ResultsGUI + gui.startup(args) + gui.addResult("normal", Normal(20, 25)) + gui.addResult("exponential", Exponential(.2)) + } +} \ No newline at end of file diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/DiscreteExample.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/DiscreteExample.scala new file mode 100644 index 00000000..5306f87c --- /dev/null +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/DiscreteExample.scala @@ -0,0 +1,30 @@ +/* + * DiscreteExample.scala + * An example using discrete distributions that demonstrates user interaction. + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jun 16, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.example.visualization + +import com.cra.figaro.util.visualization._ +import com.cra.figaro.util.visualization.results.{DiscreteData} + +/** + * @author gtakata + */ +object DiscreteExample { + + def main(args: Array[String]) { + val gui = ResultsGUI + gui.startup(args) + gui.addResult("test", List((.20, true), (.80, false))) + gui.addResult("test2", List((1.0, true), (.00, false))) + + } +} \ No newline at end of file diff --git a/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/Regression.scala b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/Regression.scala new file mode 100644 index 00000000..de018365 --- /dev/null +++ b/FigaroExamples/src/main/scala/com/cra/figaro/example/visualization/Regression.scala @@ -0,0 +1,48 @@ +/* + * Regression.scala + * A Bayesian network example with visualization + * + * Created By: Glenn Takata (gtakata@cra.com) + * Creation Date: Jul 7, 2015 + * + * Copyright 2015 Avrom J. Pfeffer and Charles River Analytics, Inc. + * See http://www.cra.com or email figaro@cra.com for information. + * + * See http://www.github.com/p2t2/figaro for a copy of the software license. + */ +package com.cra.figaro.example.visualization + +import com.cra.figaro.language.{Universe} +import com.cra.figaro.algorithm.sampling.{Importance} +import com.cra.figaro.library.atomic.continuous.{Normal, Uniform} +import com.cra.figaro.util.visualization.ResultsGUI + +/** + * @author Glenn Takata (gtakata@cra.com) + */ +object Regression { + Universe.createNew() + + private val mean = Uniform(0, 1) + + for (_ <- 0 until 100) { + val n = Normal(mean, 1.0) + n.addConstraint((m: Double) => m + 5) + } + + + def main(args: Array[String]) { + val gui = ResultsGUI + gui.startup(args) + + val alg = Importance(1000, mean) + alg.start() + + val dist = alg.distribution(mean) + println("Probability of mean: " + dist.toList) + + gui.addResult("mean", alg.distribution(mean).toList) + + alg.kill + } +} \ No newline at end of file diff --git a/FigaroLaTeX/Tutorial/FigaroTutorial.pdf b/FigaroLaTeX/Tutorial/FigaroTutorial.pdf index d504e63c..a86d6ccd 100644 Binary files a/FigaroLaTeX/Tutorial/FigaroTutorial.pdf and b/FigaroLaTeX/Tutorial/FigaroTutorial.pdf differ diff --git a/FigaroLaTeX/Tutorial/FigaroTutorial.tex b/FigaroLaTeX/Tutorial/FigaroTutorial.tex index ad0b1449..355b46f8 100644 --- a/FigaroLaTeX/Tutorial/FigaroTutorial.tex +++ b/FigaroLaTeX/Tutorial/FigaroTutorial.tex @@ -83,7 +83,8 @@ \include{Sections/9HierReasoning} \include{Sections/10Elements} \include{Sections/11CreateNew} -\include{Sections/12Conclusion} +\include{Sections/12Experimental} +\include{Sections/13Conclusion} %---------------------------------------------------------------------------------------- diff --git a/FigaroLaTeX/Tutorial/Sections/10Elements.tex b/FigaroLaTeX/Tutorial/Sections/10Elements.tex index 1c8495cb..543891ab 100644 --- a/FigaroLaTeX/Tutorial/Sections/10Elements.tex +++ b/FigaroLaTeX/Tutorial/Sections/10Elements.tex @@ -4,7 +4,7 @@ \chapter{Creating a new element class} % Chapter title \label{Creating a new element class} % For referencing the chapter elsewhere, use -For many applications, Figaro's built-in element classes will suffice. However, if you do need a new element class, it is usually not hard to create one. The easiest way to create a new class is to inherit from an existing class. We describe how to do this for atomic and compound classes. Then we describe how to create an atomic or compound class without inheritance. After that, we describe how to make a class usable by range computation and variable elimination. Next, an explanation is given on how to create parameters and parameterized elements suitable for use with expectation maximization or other learning algorithms. Finally, we show how to create a class with special behavior under Metropolis-Hastings. +For many applications, Figaro's built-in element classes will suffice. However, if you do need a new element class, it is usually not hard to create one. The easiest way to create a new class is to inherit from an existing class. We describe how to do this for atomic and compound classes. Then we describe how to create an atomic or compound class without inheritance. After that, we describe how to make a class usable by range computation and factored algorithms. Next, an explanation is given on how to create parameters and parameterized elements suitable for use with expectation maximization or other learning algorithms. Finally, we show how to create a class with special behavior under Metropolis-Hastings. More examples of element classes can be found under \texttt{com.cra.fig\-aro.library}. If you do create a new element class and think it might be generally useful, we would appreciate if you would consider sharing it, either as a library or possibly as part of a future Figaro release. @@ -84,7 +84,7 @@ \subsection{Inheriting from Chain} } \end{flushleft} -First, note that it inherits from \texttt{CachingChain}. When you inherit from \texttt{Chain}, you have two options. You can specify either a caching or non-caching version of the chain (which has preset cache capacities), or you can directly instantiate \texttt{Chain} with a specified cache capacity. Note that chains themselves do not extend the \texttt{Cacheable} trait, since the support of a can be infinite. Also, when you inherit from a class, you have to explicitly pass along the name and collection arguments. +First, note that it inherits from \texttt{CachingChain}. When you inherit from \texttt{Chain}, you have two options. You can specify either a caching or non-caching version of the chain. Chains themselves perform no caching, but the cacheable or non--cacheable status of a chain tells an algorithm whether it is allowed to cache chain results. Note that chains themselves do not extend the \texttt{Cacheable} trait, since the support of a can be infinite. Also, when you inherit from a class, you have to explicitly pass along the name and collection arguments. The operation of the chain can be thought of as follows: first, produce specific values for each of the options. Then, given such a specific set of values, create an atomic uniform element over those values. Finally, generate a specific value from the atomic uniform element, i.e., a uniformly chosen value from those values. @@ -173,23 +173,22 @@ \section{Creating a compound class without inheritance} For non-deterministic classes, you need to define the additional elements \texttt{Randomness}, \texttt{generateRandomness}, and \texttt{density}, as before. -\section{Making a class usable by variable elimination} +\section{Making a class usable by factored algorithms} Certain algorithms rely on element classes being able to support specific functionality. For example, computing ranges requires that it be possible to enumerate the values of every element in the universe. One way to make a new element class support value enumeration would be to modify the code that enumerates values in \texttt{Values.scala}, This approach would not be modular; it is undesirable for a user to have to modify library code. Figaro provides a different solution. There is a trait called \texttt{ValuesMa\-ker} that characterizes element classes for which values can be enumerated. If you want your element class to support range computation, make it extend \texttt{ValuesMaker} and have it implement the \texttt{makeValues} method, which produces an enumeration of the possible values of the element. For example, we might want to enumerate the possible values of an atomic binomial element. If n is the number of trials of the binomial, we can define the function: \begin{flushleft} -\texttt{def makeValues: Set[Int] = (for \{ i <- 0 to n \} yield i).toSet} +\texttt{def makeValues(depth: Int): ValueSet[Int] = ValueSet.withoutStar((for \{ i <- 0 to n \} yield i).toSet}) \end{flushleft} -The \texttt{makeValues} method returns a set of values. For a binomial, this is simply all the integers from -0 to the number of trials. This set is computed through a for comprehension whose result is turned into a set. We also make \texttt{AtomicBinomial} extend \texttt{ValuesMaker}. +The \texttt{makeValues} method returns a set of values in a set called a ValueSet. A ValueSet is a set of over a type but also includes a special value call * (star) that is used for lazy inference. In addition, makeValues function requires a depth value that is also needed for lazy inference. For a binomial, the makeValues function is simply all the integers from 0 to the number of trials. This set is computed through a for comprehension whose result is turned into a set. We also make \texttt{AtomicBinomial} extend \texttt{ValuesMaker}. -Similarly, variable elimination requires both that it be possible to enumerate the values of an element and that it be possible to turn into a set of factors. To specify that it has the latter capability, you make it -extend \texttt{ProbFactorMaker} and implement the \texttt{makeFactors} method. Factors are parameterized by the type of values they contain; in this case, since we are creating a factor representing probabilities, we make a \texttt{Factor[Double]}. +Generally, factored algorithms like variable elimination requires both that it be possible to enumerate the values of an element and that it be possible to turn into a set of factors. However, factored algorithms will operate on any Figaro model by sampling values from an element if no ValuesMaker is defined. It is still better though to explicitly specify how to make factors for a new element. To do this, you make it +extend \texttt{FactorMaker} and implement the \texttt{makeFactors} method. Factors are parameterized by the type of values they contain; in this case, since we are creating a factor representing probabilities, we make a \texttt{Factor[Double]}. -For example, the \texttt{AtomicBinomial} class extends \texttt{ProbFactorMaker} and includes the following code: +For example, the one could extend the \texttt{AtomicBinomial} class to extend \texttt{FactorMaker} and include the following code to generate factors: \begin{flushleft} \texttt{def makeFactors(binomial: AtomicBinomial): List[Factor[Double]] = \{ @@ -202,6 +201,7 @@ \section{Making a class usable by variable elimination} \newline \} } \end{flushleft} +(Note that the \texttt{AtomicBinomial} class is included in Figaro and thus its factor creation is defined in the language, not in the element itself.) The \texttt{makeFactors} method returns a list of factors. A factor is a table defined over a set of \emph{variables}. To create a variable out of an element, use \texttt{Variable}. For example, the \texttt{Variable(binomial)} line above creates a variable out of this atomic binomial element. Creating variables is memoized, so you can be sure that every time you call \texttt{Variable} on an element you get the same variable. This is important if an element participates in multiple factors. To create a factor, you pass it an array of its variables. @@ -216,7 +216,7 @@ \section{Making a class usable by variable elimination} The first line uses a for comprehension to get at pairs of values of the binomial variable together with their index into the range. The standard Scala library method \texttt{zipWithIndex} takes a list and associates each element of the list with its index in the list. For example, \texttt{List("a", "b").zipWithIndex} is \texttt{List(("a", 0), ("b", 1))}. The first argument to \texttt{factor.set} is an list of indices into the ranges of the variables, in the same order as the array used to create the factor. The second argument is the value to associate with those indices. -At the end, \texttt{makeFactors} returns a list consisting of this single factor. This is the basic principle behind creating factors. You can find a variety of more complex examples, including some with multiple variables, in \texttt{ProbFactor.scala} For atomic elements, the process should usually be similarly simple to that for binomials. +At the end, \texttt{makeFactors} returns a list consisting of this single factor. This is the basic principle behind creating factors. You can find a variety of more complex examples, including some with multiple variables, in \texttt{Factory.scala}. For atomic elements, the process should usually be similarly simple to that for binomials. \section{Making parameters and parameterized elements} diff --git a/FigaroLaTeX/Tutorial/Sections/11CreateNew.tex b/FigaroLaTeX/Tutorial/Sections/11CreateNew.tex index edbe57ff..cd04c042 100644 --- a/FigaroLaTeX/Tutorial/Sections/11CreateNew.tex +++ b/FigaroLaTeX/Tutorial/Sections/11CreateNew.tex @@ -49,13 +49,13 @@ \subsection{Expansion and factors} A useful operation is to expand all chains in a universe to obtain the complete set of elements in the universe. This is achieved using the syntax: \begin{flushleft} -\texttt{Expand(universe).expand()} +\texttt{LazyValues(universe).expandAll(universe.activeElements)} \end{flushleft} As usual, the \texttt{universe} argument can be omitted, using the current default universe. Support is provided for algorithms that are based on factors. Variable elimination is one example, but there are many other such algorithms. To create all the factors for an element, use: \begin{flushleft} -\texttt{ProbFactor.make(element)} +\texttt{Factory.makeFactorsForElement(element)} \end{flushleft} The standard procedure to turn a universe into a list of factors is to: diff --git a/FigaroLaTeX/Tutorial/Sections/12Conclusion.tex b/FigaroLaTeX/Tutorial/Sections/13Conclusion.tex similarity index 100% rename from FigaroLaTeX/Tutorial/Sections/12Conclusion.tex rename to FigaroLaTeX/Tutorial/Sections/13Conclusion.tex diff --git a/FigaroLaTeX/Tutorial/Sections/1Introduction.tex b/FigaroLaTeX/Tutorial/Sections/1Introduction.tex index d6c12143..542dd10c 100644 --- a/FigaroLaTeX/Tutorial/Sections/1Introduction.tex +++ b/FigaroLaTeX/Tutorial/Sections/1Introduction.tex @@ -40,6 +40,7 @@ \section{What is Figaro?} \item Probability of evidence using importance sampling, belief propagation, variable elimination, and particle filtering \item Particle filtering \item Factored frontier +\item Gibbs sampling \item Parameter learning using expectation maximization \end{itemize} diff --git a/FigaroLaTeX/Tutorial/Sections/5Reasoning.tex b/FigaroLaTeX/Tutorial/Sections/5Reasoning.tex index 3e78ab14..f18573ad 100644 --- a/FigaroLaTeX/Tutorial/Sections/5Reasoning.tex +++ b/FigaroLaTeX/Tutorial/Sections/5Reasoning.tex @@ -149,7 +149,7 @@ \section{Lazy factored inference} \section{Importance sampling} -Figaro's importance sampling algorithm is actually a combination of importance and rejection sampling. It uses a simple forward sampling approach. When it encounters a condition, it checks to see if the condition is satisfied and rejects if it is not. When it encounters a constraint, it multiplies the weight of the sample by the value of the constraint. +Figaro's importance sampling algorithm is actually a combination of likelihood weighting and rejection sampling. When Figaro's Importance sampling encounters a condition, it attempts to push the condition through any Chains in the model and weight the sample by the probability of the condition. However, it is not always possible to do this (especially when it encounters an Apply), so in that case it will reject a sample if it does not match the constraint. When it encounters a constraint, it multiplies the weight of the sample by the value of the constraint. Unlike variable elimination, this algorithm can be applied to models whose expansion produces an infinite number of elements, provided any particular possible world only requires a finite number of elements to be generated. Also, this algorithm works for atomic continuous models. In addition, as an approximate algorithm, it can produce reasonably accurate answers much more quickly than the exact variable elimination. @@ -184,7 +184,9 @@ \section{Importance sampling} Importance sampling also provides a one-line query shortcut. -\section{Markov chain Monte Carlo} +There is also a parallel version of Importance sampling that uses Scala's built in parallel collections. The interface to use parallel Importance sampling is similar to the original algorithm with a few exceptions. First, this version sampling uses a model generator, which is a function that produces a universe (Importance sampling is run in parallel on separate but identical universes). Second, the user must indicate the number of threads to use. Finally, instead of taking a set of elements to query, the algorithm takes in a set of references, where each reference refers to the same element on each of the parallel universes. + +\section{Metropolis-Hastings Markov chain Monte Carlo} Figaro provides a Metropolis-Hastings Markov chain Monte Carlo algorithm. Metropolis-Hastings uses a proposal distribution to propose a new state at each step of the algorithm, and either accepts or rejects the proposal. In Figaro, a proposal involves proposing new randomnesses for any number of elements. After proposing these new randomnesses, any element that depends on those randomnesses must have its value updated. Recall that the value of an element is a deterministic function of its randomness and the values of its arguments, so this update process is a deterministic result of the randomness proposal. @@ -268,17 +270,18 @@ \subsection{Defining a proposal scheme} In some cases, it might be useful to have the decision as to which later elements to propose depend on the proposed values of earlier elements. \texttt{TypedScheme} is provided for this purpose. It has a type parameter \texttt{T} which is the value type of the first element to be proposed. The first argument to \texttt{TypedScheme} is a function of zero arguments that returns an \texttt{Element[T]}. The second argument is a function from a value of type \texttt{T} to an \texttt{Option[ProposalScheme]}. An \texttt{Option[Proposal\-Scheme]}, as its name implies, is an optional proposal scheme. It can take the value \texttt{None}, meaning that there is no proposal scheme, or the value \texttt{Some(ps)}, where \texttt{ps} is a proposal scheme. This allows the proposed value of the first element to determine, first of all, whether there will be any more proposals, and if there will be more proposals, what the subsequent proposal scheme will be. -\subsection{Chains and Metropolis-Hastings} - -In designing a Metropolis-Hastings algorithm using chains, there are design considerations of the model that can affect the run-time and memory performance of the algorithm. Chains contain an internal cache of previously generated elements from different combinations of its argument values. When a chain's function is invoked on an argument to produce a result element, the cache is first checked to determine if there exists an entry for the argument value. If an entry does exist, the cached element is retrieved and used to determine the value of the chain. If no entry exists, then the chain's function is invoked, an element is returned from the function and placed in the cache. The cache also contains a maximum capacity; once the capacity of the cache is reached, a random element is selected in the cache and discarded. The capacity of the cache can significantly impact the performance of the Metropolis-Hastings algorithm. - -The standard advantage of a large cache capacity is that it can save significant time if the function is executed repeatedly on a finite set of argument values. In Metropolis-Hastings, there is an important additional advantage. After an element is created, it may go through a sequence of proposals and eventually reach a region of high probability. Large capacity caches increases the chance that this work is saved and reused every time the parent of the chain returns to the same value. With a small capacity -cache, elements can be evicted if there are many different parent values. If at a later stage the parent returns to the same original value, it may have been evicted from the cache and we'll need to begin the -search process from scratch. - -However, the standard disadvantage of large caches is that they use more memory. In particular, a different element is stored for every value of the parent that has been seen, and may never be released if the cache is large enough. If the parent can have a large or infinite number of possible values, this can lead to exhausting the memory of the machine. - -Fortunately, most of the cache management is automatically handled internally by Figaro. There are two types of chains defined in Figaro: \texttt{CachingChain} and \texttt{NonCachingChain}. \texttt{CachingChain} by default instantiates a chain with a cache capacity of 1000, whereas a \texttt{NonCachi\-ngChain} instantiates a chain with a capacity of 1. In general, a \texttt{Caching\-Chain} is usually better for elements with discrete parents with relatively few values, and a \texttt{NonCachingChain} is better for elements with continuous parents. When a user creates a new \texttt{Chain} class, Figaro attempts to determine the best chain to use given the parents of the chain. In most cases, the cache capacity selected by Figaro will be adequate to use the model efficiently in a Metropolis-Hastings algorithm. However, should you need to ensure the efficiency of the model in a Metropolis-Hastings algorithm, the user can still explicitly instantiate a \texttt{Chain} class with specific cache capacity. +% Removed 3/4/16 by Brian - this is no longer relevant since Caching is handled inside algorithms and has no impact on the user. +%\subsection{Chains and Metropolis-Hastings} +% +%In designing a Metropolis-Hastings algorithm using chains, there are design considerations of the model that can affect the run-time and memory performance of the algorithm. Chains contain an internal cache of previously generated elements from different combinations of its argument values. When a chain's function is invoked on an argument to produce a result element, the cache is first checked to determine if there exists an entry for the argument value. If an entry does exist, the cached element is retrieved and used to determine the value of the chain. If no entry exists, then the chain's function is invoked, an element is returned from the function and placed in the cache. The cache also contains a maximum capacity; once the capacity of the cache is reached, a random element is selected in the cache and discarded. The capacity of the cache can significantly impact the performance of the Metropolis-Hastings algorithm. +% +%The standard advantage of a large cache capacity is that it can save significant time if the function is executed repeatedly on a finite set of argument values. In Metropolis-Hastings, there is an important additional advantage. After an element is created, it may go through a sequence of proposals and eventually reach a region of high probability. Large capacity caches increases the chance that this work is saved and reused every time the parent of the chain returns to the same value. With a small capacity +%cache, elements can be evicted if there are many different parent values. If at a later stage the parent returns to the same original value, it may have been evicted from the cache and we'll need to begin the +%search process from scratch. +% +%However, the standard disadvantage of large caches is that they use more memory. In particular, a different element is stored for every value of the parent that has been seen, and may never be released if the cache is large enough. If the parent can have a large or infinite number of possible values, this can lead to exhausting the memory of the machine. +% +%Fortunately, most of the cache management is automatically handled internally by Figaro. There are two types of chains defined in Figaro: \texttt{CachingChain} and \texttt{NonCachingChain}. \texttt{CachingChain} by default instantiates a chain with a cache capacity of 1000, whereas a \texttt{NonCachi\-ngChain} instantiates a chain with a capacity of 1. In general, a \texttt{Caching\-Chain} is usually better for elements with discrete parents with relatively few values, and a \texttt{NonCachingChain} is better for elements with continuous parents. When a user creates a new \texttt{Chain} class, Figaro attempts to determine the best chain to use given the parents of the chain. In most cases, the cache capacity selected by Figaro will be adequate to use the model efficiently in a Metropolis-Hastings algorithm. However, should you need to ensure the efficiency of the model in a Metropolis-Hastings algorithm, the user can still explicitly instantiate a \texttt{Chain} class with specific cache capacity. \subsection{Debugging Metropolis-Hastings} @@ -288,6 +291,63 @@ \subsection{Debugging Metropolis-Hastings} In addition, if you have a \texttt{Metropolis-Hastings} object \texttt{mh}, you can define an initial state by setting the values of elements. Then call \texttt{mh.test} and provide it a number of samples to generate. It will repeatedly propose a new state from the initial state and either accept or reject it, restoring to the original state each time. You can provide a sequence of predicates, and it will report how often each predicate was satisfied after one step of Metropolis-Hastings from the initial state. You can also provide a sequence of elements to track, and it will report how often each element is proposed. For example, in the movies example, you could set the initial state to be one in which exactly one appearance is awarded and test the fraction of times this condition holds after one step. +\section{Gibbs Sampling Markov chain Monte Carlo} + +Figaro also provides another Markov chain Monte Carlo algorithm known as Gibbs sampling. Gibbs sampling is an algorithm that traditionally will iterate through all the variables in the model, and sample each variable conditioned on the rest of the model (or the Markov blanket, as that is all that is needed). By successively sampling variables in the model in this manner, the Markov chain eventually reaches convergence, and subsequent samples from the model are from the joint distribution defined by the model. + +Figaro's Gibbs sampling is similar to traditional implementations of Gibbs sampling except for two key differences: It is implemented on a factor graph, and blocks of variables are sampled at each iteration instead of a single variable (this is required because of chains). This means that when Gibbs sampling is run, factors are generated based on the model; continuous variables are sampled into factors. Hence, Gibbs samples in Figaro is really a mix between a factored algorithm and a sampling algorithm. + +To create an anytime Gibbs sampling algorithm, use: + +\begin{flushleft} +\texttt{import com.cra.figaro.language.\_ +\newline import com.cra.figaro.algorithm.factored.gibbs.\_ +\newline +\newline val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) +\newline val e2 = Flip(e1) +\newline val e3 = If(e2, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) +\newline e3.setCondition((i: Int) => i == 2) +\newline +\newline val gs = Gibbs(e2) +} +\end{flushleft} + +Gibbs sampling takes three additional optional arguments. The first represents the burn-in, which is the number of steps the algorithm goes through before collecting samples, the second is the number of steps between samples, and the final is the method of creating blocks (which most people do not need to change). The default burn-in is 0, while the default interval is 1. These arguments appear before the query elements. + +To use a one-time (i.e., non-anytime) Gibbs sampling algorithm, simply provide the number of samples as the first argument. Gibbs sampling also provides a one-line query shortcut. + +\section{Structured Factored Inference} + +Figaro also contains a set of algorithms known as structured factored inference (SFI) algorithms. These are not new algorithms per se, but rather a method of decomposing a model into a set of smaller sub--models that are solved recursively (i.e., structured) using one of Figaro's three existing factored algorithms (VE, BP, and Gibbs sampling). There is a rich and extensible library of the different strategies that can be applied to model that guide how a model is decomposed, and which algorithms are applied to the generated sub--models. However, we provide a base set of structured algorithms. + +The basic structured algorithm will decompose a model based on Chains. That is, every Chain defined in the model creates at most \textit{n} sub--models, one for each value of the parent of the Chain. Each of these sub--models is recursively decomposed. When a sub--model can no longer be decomposed, a solving strategy is applied to the sub--model. Figaro comes with a set of strategies that either applies the same algorithm to each model or chooses to apply either VE, BP, or Gibbs sampling based on some heuristics. + +To create an SFI algorithm that uses the same algorithm for each sub--model, use: + +\begin{flushleft} +\texttt{import com.cra.figaro.language.\_ +\newline import com.cra.figaro.algorithm.structured.algorithm.structured.\_ +\newline +\newline val e1 = Select(0.25 -> 0.3, 0.25 -> 0.5, 0.25 -> 0.7, 0.25 -> 0.9) +\newline val e2 = Flip(e1) +\newline val e3 = If(e2, Select(0.3 -> 1, 0.7 -> 2), Constant(2)) +\newline e3.setCondition((i: Int) => i == 2) +\newline +\newline val sve = StructuredVE(e2) +} +\end{flushleft} + +The algorithm can them be used like a normal Figaro algorithm. There are also structured BP and Gibbs algorithms. To use a strategy that automatically chooses between VE, BP, and Gibbs sampling, use: + +\begin{flushleft} +\texttt{ +\newline import com.cra.figaro.algorithm.structured.algorithm.hybrid.\_ +\newline val structAlg = StructuredVEBPGibbsChooser(0.0, 0.5, 100, 1000, e2) +} +\end{flushleft} + +Where the first argument is the threshold that determines when VE is run (e.g., the cost of running VE is less than the threshold). The second argument controls when to choose between BP and Gibbs sampling. If the fraction of the model that is deterministic is greater than the second argument, then BP is run, otherwise Gibbs is run. The following arguments are the number of BP and Gibbs iterations to perform, and the final argument is the query target. + \section{Probability of evidence algorithms} The previous three algorithms all computed the conditional probability of query variables given evidence. Sometimes we just want to compute the probability of evidence. Since there is the potential for ambiguity here, Figaro is careful to define what constitutes evidence for computing probability of evidence. Conditions and constraints often constitute evidence. Sometimes, however, they can be considered to be part of the model specification. Consider, for example, the constraint on pairs of friends that they share the same smoking habits (this is part of the model definition, not evidence). @@ -647,3 +707,4 @@ \section{Reproducing inference results} \end{flushleft} To reproduce the results of an inference algorithm, you must set the seed in the random number generator. Repeated runs of the same algorithm with the same seed will then be identical, making debugging much easier since errors can be tracked between runs. To set the seed, you import the util package, and simply call \texttt{setSeed(s: Long)}. To retrieve the current random number generator seed, one calls \texttt{getSeed()}. + diff --git a/FigaroLaTeX/Tutorial/Sections/6DynamicModels.tex b/FigaroLaTeX/Tutorial/Sections/6DynamicModels.tex index be68f094..9b600b42 100644 --- a/FigaroLaTeX/Tutorial/Sections/6DynamicModels.tex +++ b/FigaroLaTeX/Tutorial/Sections/6DynamicModels.tex @@ -64,6 +64,8 @@ \subsection{Particle filtering} There are a couple of implementation notes about particle filtering that a user should be aware of. First, since particle filtering estimates probabilities of elements using many particles, it can get expensive (memory wise) to store the state estimates for every element in the model. Therefore, the states of only \emph{named} elements are tracked through time. This means that queries to filter for an element probability or expectation must be on named elements. In addition, because filtering tracks estimates through time, we want to free up memory from old universes that are no longer used. To accomplish this, when \texttt{advanceTime} is called, all named elements from the previous universe are copied as constants to a new, temporary universe. The temporary universe is then used in the transition function, allowing the real previous universe to be freed while still letting the new universe use the correct values from the old universe. +There is also a parallel version of particle filtering that uses Scala's built in parallel collections. The interface to use parallel particle filtering is similar to the original algorithm with a few exceptions. First, this version uses a model generator, which is a function that produces a universe (particle filtering is run in parallel on separate but identical universes). Second, the user must indicate the number of threads to use. + \subsection{Factored frontier} Factored frontier is a dynamic reasoning algorithm that uses factors instead of sampling. To create an instance of factored frontier, use: diff --git a/README.md b/README.md index 383cab37..13abb06a 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,4 @@ Figaro is a probabilistic programming language that supports development of very Figaro makes it possible to express probabilistic models using the power of programming languages, giving the modeler the expressive tools to create a wide variety of models. Figaro comes with a number of built-in reasoning algorithms that can be applied automatically to new models. In addition, Figaro models are data structures in the Scala programming language, which is interoperable with Java, and can be constructed, manipulated, and used directly within any Scala or Java program. -Figaro is free and is released under an [open-source license](https://github.com/p2t2/figaro/blob/master/LICENSE). The current, stable binary release of Figaro can be found [here](https://www.cra.com/figaro). For more information please see the [Figaro Release Notes](https://www.cra.com/pdf/Figaro-Release-Notes.pdf) and [Figaro Tutorial](https://www.cra.com/pdf/Figaro-tutorial.pdf). Documentation of the Figaro library interface can be found [here](https://www.cra.com/files/Figaro_Scaladoc/index.html). +Figaro is free and is released under an [open-source license](https://github.com/p2t2/figaro/blob/master/LICENSE). The current, stable binary release of Figaro can be found [here](https://www.cra.com/work/case-studies/figaro). For more information please see the [Figaro Release Notes](https://www.cra.com/sites/default/files/pdf/Figaro_Release_Notes.pdf) and [Figaro Tutorial](https://www.cra.com/sites/default/files/pdf/Figaro_Tutorial.pdf). Documentation of the Figaro library interface can be found [here](https://www.cra.com/Figaro_Scaladoc/index.html#package). diff --git a/doc/Figaro Quick Start Guide.pdf b/doc/Figaro Quick Start Guide.pdf new file mode 100644 index 00000000..0ad2718f Binary files /dev/null and b/doc/Figaro Quick Start Guide.pdf differ diff --git a/doc/Figaro Release Notes.pdf b/doc/Figaro Release Notes.pdf index d7394855..39c09810 100644 Binary files a/doc/Figaro Release Notes.pdf and b/doc/Figaro Release Notes.pdf differ diff --git a/doc/Figaro Tutorial.pdf b/doc/Figaro Tutorial.pdf index d504e63c..a6d7c64a 100644 Binary files a/doc/Figaro Tutorial.pdf and b/doc/Figaro Tutorial.pdf differ diff --git a/project/Build.scala b/project/Build.scala index 14605a73..59e8eb0c 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -21,24 +21,11 @@ import com.typesafe.sbteclipse.plugin.EclipsePlugin.EclipseKeys object FigaroBuild extends Build { - // Copy dependency JARs to /target//lib - // Courtesy of - // http://stackoverflow.com/questions/7351280/collecting-dependencies-under-sbt-0-10-putting-all-dependency-jars-to-target-sc - lazy val copyDependencies = TaskKey[Unit]("copy-deps") - - def copyDepTask = copyDependencies <<= (update, crossTarget, scalaVersion) map { - (updateReport, out, scalaVer) => - updateReport.allFiles foreach { srcPath => - val destPath = out / "lib" / srcPath.getName - IO.copyFile(srcPath, destPath, preserveLastModified=true) - } - } - override val settings = super.settings ++ Seq( organization := "com.cra.figaro", description := "Figaro: a language for probablistic programming", - version := "3.2.0.0", - scalaVersion := "2.11.6", + version := "4.1.0.0", + scalaVersion := "2.11.7", crossPaths := true, publishMavenStyle := true, pomExtra := @@ -95,18 +82,23 @@ object FigaroBuild extends Build { "asm" % "asm" % "3.3.1", "org.apache.commons" % "commons-math3" % "3.3", "net.sf.jsci" % "jsci" % "1.2", - "com.typesafe.akka" %% "akka-actor" % "2.3.8", + "com.typesafe.akka" %% "akka-actor" % "2.3.14", "org.scalanlp" %% "breeze" % "0.10", "io.argonaut" %% "argonaut" % "6.0.4", - "com.storm-enroute" %% "scalameter" % "0.6" % "provided", + "org.prefuse" % "prefuse" % "beta-20071021", + "org.scala-lang.modules" %% "scala-swing" % "1.0.1", + "com.storm-enroute" %% "scalameter" % "0.7" % "provided", "org.scalatest" %% "scalatest" % "2.2.4" % "provided, test" )) + // Copy all managed dependencies to \lib_managed directory + .settings(retrieveManaged := true) // Enable forking .settings(fork := true) // Increase max memory for JVM for both testing and runtime - .settings(javaOptions in (Test,run) += "-Xmx8G") + .settings(javaOptions in (Test,run) += "-Xmx6G") // test settings .settings(parallelExecution in Test := false) + .settings(testOptions in Test += Tests.Argument("-oD")) .configs(detTest) .settings(inConfig(detTest)(Defaults.testTasks): _*) .settings(testOptions in detTest := Seq(Tests.Argument("-l", "com.cra.figaro.test.nonDeterministic"))) @@ -122,8 +114,6 @@ object FigaroBuild extends Build { val cp = (fullClasspath in assembly).value cp filter {_.data.getName == "arpack_combined_all-0.1-javadoc.jar"} }) - // Copy dependency JARs - .settings(copyDepTask) // ScalaMeter settings .settings(testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework")) .settings(logBuffered := false) @@ -133,10 +123,10 @@ object FigaroBuild extends Build { lazy val examples = Project("FigaroExamples", file("FigaroExamples")) .dependsOn(figaro) .settings(packageOptions := Seq(Package.JarManifest(examplesManifest))) - // Copy dependency JARs - .settings(copyDepTask) // SBTEclipse settings .settings(EclipseKeys.eclipseOutput := Some("target/scala-2.11/classes")) + // Copy all managed dependencies to \lib_managed directory + .settings(retrieveManaged := true) lazy val detTest = config("det") extend(Test) lazy val nonDetTest = config("nonDet") extend(Test) diff --git a/project/build.properties b/project/build.properties index 9ad7e847..176a863a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.8 \ No newline at end of file +sbt.version=0.13.9 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 287138a7..472d8904 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,5 @@ -addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "3.0.0") +addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.3") resolvers += Classpaths.sbtPluginReleases