diff --git a/README.md b/README.md index ebc7eec..7f50f42 100644 --- a/README.md +++ b/README.md @@ -121,12 +121,12 @@ Then [re]start Solr, e.g.: Now start the Stanford Named Entity Recognition server, which is used to pull names, places etc. out of the source data: - $ cd stanford-ner-2011-09-14 + $ cd stanford-ner-2014-06-16 $ ./server.sh & or on Windows: - C:\ cd stanford-ner-2011-09-14 + C:\ cd stanford-ner-2014-06-16 C:\ server.bat Finally, add the example documents (which are provided as plaintext files): diff --git a/go2.bat b/go2.bat index 6751533..91c6351 100644 --- a/go2.bat +++ b/go2.bat @@ -4,5 +4,5 @@ rem Each must run in its own command line shell call paths.bat -cd stanford-ner-2011-09-14 +cd stanford-ner-2014-06-16 server.bat diff --git a/stanford-ner-2011-09-14/Makefile b/stanford-ner-2011-09-14/Makefile deleted file mode 100644 index 2f8d594..0000000 --- a/stanford-ner-2011-09-14/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# This is a rudimentary Makefile for rebuilding the CRF NER distribution. -# We actually use ant (q.v.) or a Java IDE. - -JAVAC = javac -JAVAFLAGS = -O -d classes -encoding utf-8 - -ner: - mkdir -p classes - $(JAVAC) $(JAVAFLAGS) src/edu/stanford/nlp/*/*.java src/edu/stanford/nlp/*/*/*.java - cd classes ; jar -cfm ../stanford-ner-`date +%Y-%m-%d`.jar ../src/edu/stanford/nlp/ie/crf/ner-manifest.txt edu ; cd .. - cp stanford-ner-`date +%Y-%m-%d`.jar stanford-ner.jar - diff --git a/stanford-ner-2011-09-14/NERDemo.java b/stanford-ner-2011-09-14/NERDemo.java deleted file mode 100644 index bc7bc50..0000000 --- a/stanford-ner-2011-09-14/NERDemo.java +++ /dev/null @@ -1,77 +0,0 @@ -import edu.stanford.nlp.ie.crf.*; -import edu.stanford.nlp.ie.AbstractSequenceClassifier; -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.ling.CoreLabel; -import edu.stanford.nlp.ling.CoreAnnotations.AnswerAnnotation; - -import java.util.List; -import java.io.IOException; - - - -/** This is a demo of calling CRFClassifier programmatically. - *

- * Usage: java -cp "stanford-ner.jar:." NERDemo [serializedClassifier [fileName]] - *

- * If arguments aren't specified, they default to - * ner-eng-ie.crf-3-all2006.ser.gz and some hardcoded sample text. - *

- * To use CRFClassifier from the command line: - * java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier - * [classifier] -textFile [file] - * Or if the file is already tokenized and one word per line, perhaps in - * a tab-separated value format with extra columns for part-of-speech tag, - * etc., use the version below (note the 's' instead of the 'x'): - * java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier - * [classifier] -testFile [file] - * - * @author Jenny Finkel - * @author Christopher Manning - */ - -public class NERDemo { - - public static void main(String[] args) throws IOException { - - String serializedClassifier = "classifiers/ner-eng-ie.crf-3-all2008.ser.gz"; - - if (args.length > 0) { - serializedClassifier = args[0]; - } - - AbstractSequenceClassifier classifier = CRFClassifier.getClassifierNoExceptions(serializedClassifier); - - /* For either a file to annotate or for the hardcoded text example, - this demo file shows two ways to process the output, for teaching - purposes. For the file, it shows both how to run NER on a String - and how to run it on a whole file. For the hard-coded String, - it shows how to run it on a single sentence, and how to do this - and produce an inline XML output format. - */ - if (args.length > 1) { - String fileContents = IOUtils.slurpFile(args[1]); - List> out = classifier.classify(fileContents); - for (List sentence : out) { - for (CoreLabel word : sentence) { - System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' '); - } - System.out.println(); - } - out = classifier.classifyFile(args[1]); - for (List sentence : out) { - for (CoreLabel word : sentence) { - System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' '); - } - System.out.println(); - } - - } else { - String s1 = "Good afternoon Rajat Raina, how are you today?"; - String s2 = "I go to school at Stanford University, which is located in California."; - System.out.println(classifier.classifyToString(s1)); - System.out.println(classifier.classifyWithInlineXML(s2)); - System.out.println(classifier.classifyToString(s2, "xml", true)); - } - } - -} diff --git a/stanford-ner-2011-09-14/ner-gui.sh b/stanford-ner-2011-09-14/ner-gui.sh deleted file mode 100755 index ea2cb5e..0000000 --- a/stanford-ner-2011-09-14/ner-gui.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -java -mx500m -cp "stanford-ner.jar:" edu.stanford.nlp.ie.crf.NERGUI diff --git a/stanford-ner-2011-09-14/ner.bat b/stanford-ner-2011-09-14/ner.bat deleted file mode 100755 index e87b3c8..0000000 --- a/stanford-ner-2011-09-14/ner.bat +++ /dev/null @@ -1 +0,0 @@ -java -mx1500m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers\all.3class.distsim.crf.ser.gz -textFile %1 diff --git a/stanford-ner-2011-09-14/ner.sh b/stanford-ner-2011-09-14/ner.sh deleted file mode 100755 index 56827f1..0000000 --- a/stanford-ner-2011-09-14/ner.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -java -mx700m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers/all.3class.distsim.crf.ser.gz -textFile $1 diff --git a/stanford-ner-2011-09-14/server.bat b/stanford-ner-2011-09-14/server.bat deleted file mode 100644 index 81f810a..0000000 --- a/stanford-ner-2011-09-14/server.bat +++ /dev/null @@ -1 +0,0 @@ -java -mx1000m -cp stanford-ner.jar edu.stanford.nlp.ie.NERServer -loadClassifier classifiers/all.3class.distsim.crf.ser.gz -port 9000 \ No newline at end of file diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AbstractLinearClassifierFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AbstractLinearClassifierFactory.java deleted file mode 100644 index a671b16..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AbstractLinearClassifierFactory.java +++ /dev/null @@ -1,91 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.HashIndex; - -import java.lang.ref.Reference; -import java.util.Collection; -import java.util.List; - -/** - * Shared methods for training a {@link LinearClassifier}. - * Inheriting classes need to implement the - * trainWeights method. - * - * @author Dan Klein - * - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * - * @param The type of the labels in the Dataset and Datum - * @param The type of the features in the Dataset and Datum - */ - -public abstract class AbstractLinearClassifierFactory implements ClassifierFactory> { - - private static final long serialVersionUID = 1L; - - Index labelIndex = new HashIndex(); - Index featureIndex = new HashIndex(); - - public AbstractLinearClassifierFactory() { - } - - int numFeatures() { - return featureIndex.size(); - } - - int numClasses() { - return labelIndex.size(); - } - - public Classifier trainClassifier(List> examples) { - Dataset dataset = new Dataset(); - dataset.addAll(examples); - return trainClassifier(dataset); - } - - protected abstract double[][] trainWeights(GeneralDataset dataset) ; - - /** - * Takes a {@link Collection} of {@link Datum} objects and gives you back a - * {@link Classifier} trained on it. - * - * @param examples {@link Collection} of {@link Datum} objects to train the - * classifier on - * @return A {@link Classifier} trained on it. - */ - public LinearClassifier trainClassifier(Collection> examples) { - Dataset dataset = new Dataset(); - dataset.addAll(examples); - return trainClassifier(dataset); - } - - /** - * Takes a {@link Reference} to a {@link Collection} of {@link Datum} - * objects and gives you back a {@link Classifier} trained on them - * - * @param ref {@link Reference} to a {@link Collection} of {@link - * Datum} objects to train the classifier on - * @return A Classifier trained on a collection of Datum - */ - public LinearClassifier trainClassifier(Reference>> ref) { - Collection> examples = ref.get(); - return trainClassifier(examples); - } - - - /** - * Trains a {@link Classifier} on a {@link Dataset}. - * - * @return A {@link Classifier} trained on the data. - */ - public LinearClassifier trainClassifier(GeneralDataset data) { - labelIndex = data.labelIndex(); - featureIndex = data.featureIndex(); - double[][] weights = trainWeights(data); - return new LinearClassifier(weights, featureIndex, labelIndex); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AdaptedGaussianPriorObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AdaptedGaussianPriorObjectiveFunction.java deleted file mode 100644 index 16521e1..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AdaptedGaussianPriorObjectiveFunction.java +++ /dev/null @@ -1,118 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.math.ArrayMath; -import java.util.Arrays; - - -/** - * Adapt the mean of the Gaussian Prior by shifting the mean to the previously trained weights - * @author Pi-Chuan Chang - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * - * @param The type of the labels in the Dataset (one can be passed in to the constructor) - * @param The type of the features in the Dataset - */ - -public class AdaptedGaussianPriorObjectiveFunction extends LogConditionalObjectiveFunction { - - double[] weights; - - /** - * Calculate the conditional likelihood. - */ - @Override - protected void calculate(double[] x) { - if (useSummedConditionalLikelihood) { - calculateSCL(x); - } else { - calculateCL(x); - } - } - - - /** - */ - private void calculateSCL(double[] x) { - throw new UnsupportedOperationException(); - } - - /** - */ - private void calculateCL(double[] x) { - value = 0.0; - if (derivativeNumerator == null) { - derivativeNumerator = new double[x.length]; - for (int d = 0; d < data.length; d++) { - int[] features = data[d]; - for (int f = 0; f < features.length; f++) { - int i = indexOf(features[f], labels[d]); - if (dataweights == null) { - derivativeNumerator[i] -= 1; - } else { - derivativeNumerator[i] -= dataweights[d]; - } - } - } - } - copy(derivative, derivativeNumerator); - - double[] sums = new double[numClasses]; - double[] probs = new double[numClasses]; - - for (int d = 0; d < data.length; d++) { - int[] features = data[d]; - // activation - Arrays.fill(sums, 0.0); - - for (int c = 0; c < numClasses; c++) { - for (int f = 0; f < features.length; f++) { - int i = indexOf(features[f], c); - sums[c] += x[i]; - } - } - double total = ArrayMath.logSum(sums); - for (int c = 0; c < numClasses; c++) { - probs[c] = Math.exp(sums[c] - total); - if (dataweights != null) { - probs[c] *= dataweights[d]; - } - for (int f = 0; f < features.length; f++) { - int i = indexOf(features[f], c); - derivative[i] += probs[c]; - } - } - - double dV = sums[labels[d]] - total; - if (dataweights != null) { - dV *= dataweights[d]; - } - value -= dV; - } - //System.err.println("x length="+x.length); - //System.err.println("weights length="+weights.length); - double[] newX = ArrayMath.pairwiseSubtract(x, weights); - value += prior.compute(newX, derivative); - } - - /** - */ - @Override - protected void rvfcalculate(double[] x) { - throw new UnsupportedOperationException(); - } - - public AdaptedGaussianPriorObjectiveFunction(GeneralDataset dataset, LogPrior prior, double weights[][]) { - super(dataset, prior); - this.weights = to1D(weights); - } - - public double[] to1D(double[][] x2) { - double[] x = new double[numFeatures*numClasses]; - for (int i = 0; i < numFeatures; i++) { - for (int j = 0; j < numClasses; j++) { - x[indexOf(i, j)] = x2[i][j]; - } - } - return x; - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/BiasedLogConditionalObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/BiasedLogConditionalObjectiveFunction.java deleted file mode 100644 index 5378d14..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/BiasedLogConditionalObjectiveFunction.java +++ /dev/null @@ -1,137 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.optimization.AbstractCachingDiffFunction; - -import java.util.Arrays; - - -/** - * Maximizes the conditional likelihood with a given prior. - * - * @author Jenny Finkel - */ - -public class BiasedLogConditionalObjectiveFunction extends AbstractCachingDiffFunction { - - public void setPrior(LogPrior prior) { - this.prior = prior; - } - - protected LogPrior prior; - - protected int numFeatures = 0; - protected int numClasses = 0; - - protected int[][] data = null; - protected int[] labels = null; - - private double[][] confusionMatrix; - - @Override - public int domainDimension() { - return numFeatures * numClasses; - } - - int classOf(int index) { - return index % numClasses; - } - - int featureOf(int index) { - return index / numClasses; - } - - protected int indexOf(int f, int c) { - return f * numClasses + c; - } - - public double[][] to2D(double[] x) { - double[][] x2 = new double[numFeatures][numClasses]; - for (int i = 0; i < numFeatures; i++) { - for (int j = 0; j < numClasses; j++) { - x2[i][j] = x[indexOf(i, j)]; - } - } - return x2; - } - - @Override - protected void calculate(double[] x) { - - if (derivative == null) { - derivative = new double[x.length]; - } else { - Arrays.fill(derivative, 0.0); - } - - value = 0.0; - - double[] sums = new double[numClasses]; - double[] probs = new double[numClasses]; - double[] weightedProbs = new double[numClasses]; - - for (int d = 0; d < data.length; d++) { - int[] features = data[d]; - int observedLabel = labels[d]; - // activation - Arrays.fill(sums, 0.0); - - for (int c = 0; c < numClasses; c++) { - for (int f = 0; f < features.length; f++) { - int i = indexOf(features[f], c); - sums[c] += x[i]; - } - } - - double total = ArrayMath.logSum(sums); - - double[] weightedSums = new double[numClasses]; - for (int trueLabel = 0; trueLabel < numClasses; trueLabel++) { - weightedSums[trueLabel] = Math.log(confusionMatrix[observedLabel][trueLabel]) + sums[trueLabel]; - } - - double weightedTotal = ArrayMath.logSum(weightedSums); - - for (int c = 0; c < numClasses; c++) { - probs[c] = Math.exp(sums[c] - total); - weightedProbs[c] = Math.exp(weightedSums[c] - weightedTotal); - for (int f = 0; f < features.length; f++) { - int i = indexOf(features[f], c); - derivative[i] += probs[c] - weightedProbs[c]; - } - } - - double tmpValue = 0.0; - for (int c = 0; c < numClasses; c++) { - tmpValue += confusionMatrix[observedLabel][c] * Math.exp(sums[c] - total); - } - value -= Math.log(tmpValue); - } - - value += prior.compute(x, derivative); - - } - - - - public BiasedLogConditionalObjectiveFunction(GeneralDataset dataset, double[][] confusionMatrix) { - this(dataset, confusionMatrix, new LogPrior(LogPrior.LogPriorType.QUADRATIC)); - } - - public BiasedLogConditionalObjectiveFunction(GeneralDataset dataset, double[][] confusionMatrix, LogPrior prior) { - this(dataset.numFeatures(), dataset.numClasses(), dataset.getDataArray(), dataset.getLabelsArray(), confusionMatrix, prior); - } - - public BiasedLogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, double[][] confusionMatrix) { - this(numFeatures, numClasses, data, labels, confusionMatrix, new LogPrior(LogPrior.LogPriorType.QUADRATIC)); - } - - public BiasedLogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, double[][] confusionMatrix, LogPrior prior) { - this.numFeatures = numFeatures; - this.numClasses = numClasses; - this.data = data; - this.labels = labels; - this.prior = prior; - this.confusionMatrix = confusionMatrix; - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Classifier.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Classifier.java deleted file mode 100644 index 4998323..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Classifier.java +++ /dev/null @@ -1,28 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.stats.Counter; - -import java.io.Serializable; -import java.util.Collection; - -/** - * A simple interface for classifying and scoring data points, implemented - * by most of the classifiers in this package. A basic Classifier - * works over a List of categorical features. For classifiers over - * real-valued features, see {@link RVFClassifier}. - * - * @author Dan Klein - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * - * @param The type of the label(s) in each Datum - * @param The type of the features in each Datum - */ - -public interface Classifier extends Serializable { - public L classOf(Datum example); - - public Counter scoresOf(Datum example); - - public Collection labels(); -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierCreator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierCreator.java deleted file mode 100644 index d58905b..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierCreator.java +++ /dev/null @@ -1,10 +0,0 @@ -package edu.stanford.nlp.classify; - -/** - * Creates a classifier with given weights - * - * @author Angel Chang - */ -public interface ClassifierCreator { - public Classifier createClassifier(double[] weights); -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierFactory.java deleted file mode 100644 index 4d4cb38..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierFactory.java +++ /dev/null @@ -1,24 +0,0 @@ -package edu.stanford.nlp.classify; - -import java.io.Serializable; -import java.util.List; - -import edu.stanford.nlp.ling.RVFDatum; - -/** - * A simple interface for training a Classifier from a Dataset of training - * examples. - * - * @author Dan Klein - * - * Templatized by Sarah Spikes (sdspikes@cs.stanford.edu) - */ - -public interface ClassifierFactory> extends Serializable { - - @Deprecated //ClassifierFactory should implement trainClassifier(GeneralDataset) instead. - public C trainClassifier(List> examples); - - public C trainClassifier(GeneralDataset dataset); - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/CrossValidator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/CrossValidator.java deleted file mode 100644 index aba5462..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/CrossValidator.java +++ /dev/null @@ -1,93 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.Triple; -import edu.stanford.nlp.util.Function; - -import java.util.Iterator; - -/** - * This class is meant to simplify performing cross validation on - * classifiers for hyper-parameters. It has the ability to save - * state for each fold (for instance, the weights for a MaxEnt - * classifier, and the alphas for an SVM). - * - * @author Aria Haghighi - * @author Jenny Finkel - * @author Sarah Spikes (Templatization) - */ - -public class CrossValidator { - private GeneralDataset originalTrainData; - private int kfold; - private SavedState[] savedStates; - - public CrossValidator(GeneralDataset trainData) { - this (trainData,5); - } - - public CrossValidator(GeneralDataset trainData, int kfold) { - originalTrainData = trainData; - this.kfold = kfold; - savedStates = new SavedState[kfold]; - for (int i = 0; i < savedStates.length; i++) { - savedStates[i] = new SavedState(); - } - } - - /** - * Returns and Iterator over train/test/saved states - */ - private Iterator,GeneralDataset,SavedState>> iterator() { return new CrossValidationIterator(); } - - /** - * This computes the average over all folds of the function we're trying to optimize. - * The input triple contains, in order, the train set, the test set, and the saved state. - * You don't have to use the saved state if you don't want to. - */ - public double computeAverage (Function,GeneralDataset,SavedState>,Double> function) - { - double sum = 0; - Iterator,GeneralDataset,SavedState>> foldIt = iterator(); - while (foldIt.hasNext()) { - sum += function.apply(foldIt.next()); - } - return sum / kfold; - } - - class CrossValidationIterator implements Iterator,GeneralDataset,SavedState>> - { - int iter = 0; - public boolean hasNext() { return iter < kfold; } - - public void remove() - { - throw new RuntimeException("CrossValidationIterator doesn't support remove()"); - } - - public Triple,GeneralDataset,SavedState> next() - { - if (iter == kfold) return null; - int start = originalTrainData.size() * iter / kfold; - int end = originalTrainData.size() * (iter + 1) / kfold; - //System.err.println("##train data size: " + originalTrainData.size() + " start " + start + " end " + end); - Pair, GeneralDataset> split = originalTrainData.split(start, end); - - return new Triple,GeneralDataset,SavedState>(split.first(),split.second(),savedStates[iter++]); - } - } - - public static class SavedState { - public Object state; - } - - public static void main(String[] args) { - Dataset d = Dataset.readSVMLightFormat(args[0]); - Iterator,GeneralDataset,SavedState>> it = (new CrossValidator(d)).iterator(); - while (it.hasNext()) - { - it.next(); - break; - } - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Dataset.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Dataset.java deleted file mode 100644 index 36d7e52..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Dataset.java +++ /dev/null @@ -1,728 +0,0 @@ -package edu.stanford.nlp.classify; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import edu.stanford.nlp.ling.BasicDatum; -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.stats.TwoDimensionalCounter; -import edu.stanford.nlp.objectbank.ObjectBank; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.HashIndex; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.ScoredComparator; -import edu.stanford.nlp.util.ScoredObject; - - -/** - * An interfacing class for {@link ClassifierFactory} that incrementally - * builds a more memory-efficient representation of a {@link List} of - * {@link Datum} objects for the purposes of training a {@link Classifier} - * with a {@link ClassifierFactory}. - * - * @author Roger Levy (rog@stanford.edu) - * @author Anna Rafferty (various refactoring with GeneralDataset/RVFDataset) - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (templatization) - * @author nmramesh@cs.stanford.edu {@link #getL1NormalizedTFIDFDatum(Datum, Counter) and #getL1NormalizedTFIDFDataset()} - * - * @param Label type - * @param Feature type - */ -public class Dataset extends GeneralDataset { - - private static final long serialVersionUID = -3883164942879961091L; - - public Dataset() { - this(10); - } - - - public Dataset(int numDatums, Index featureIndex, Index labelIndex) { - initialize(numDatums); - this.labelIndex = labelIndex; - this.featureIndex = featureIndex; - } - - public Dataset(Index featureIndex, Index labelIndex) { - this(10); - this.labelIndex = labelIndex; - this.featureIndex = featureIndex; - } - - public Dataset(int numDatums) { - initialize(numDatums); - } - - /** - * Constructor that fully specifies a Dataset. Needed this for MulticlassDataset. - */ - public Dataset(Index labelIndex, int[] labels, Index featureIndex, int[][] data) { - this (labelIndex, labels, featureIndex, data, data.length); - } - - /** - * Constructor that fully specifies a Dataset. Needed this for MulticlassDataset. - */ - public Dataset(Index labelIndex, int[] labels, Index featureIndex, int[][] data, int size) { - this.labelIndex = labelIndex; - this.labels = labels; - this.featureIndex = featureIndex; - this.data = data; - this.size = size; - } - - @Override - public Pair, GeneralDataset> split(double percentDev) { - int devSize = (int)(percentDev * size()); - int trainSize = size() - devSize; - - int[][] devData = new int[devSize][]; - int[] devLabels = new int[devSize]; - - int[][] trainData = new int[trainSize][]; - int[] trainLabels = new int[trainSize]; - - System.arraycopy(data, 0, devData, 0, devSize); - System.arraycopy(labels, 0, devLabels, 0, devSize); - - System.arraycopy(data, devSize, trainData, 0, trainSize); - System.arraycopy(labels, devSize, trainLabels, 0, trainSize); - - if (this instanceof WeightedDataset) { - float[] trainWeights = new float[trainSize]; - float[] devWeights = new float[devSize]; - - WeightedDataset w = (WeightedDataset)this; - - System.arraycopy(w.weights, 0, devWeights, 0, devSize); - System.arraycopy(w.weights, devSize, trainWeights, 0, trainSize); - - WeightedDataset dev = new WeightedDataset(labelIndex, devLabels, featureIndex, devData, devSize, devWeights); - WeightedDataset train = new WeightedDataset(labelIndex, trainLabels, featureIndex, trainData, trainSize, trainWeights); - - return new Pair,GeneralDataset>(train, dev); - } - Dataset dev = new Dataset(labelIndex, devLabels, featureIndex, devData, devSize); - Dataset train = new Dataset(labelIndex, trainLabels, featureIndex, trainData, trainSize); - - return new Pair,GeneralDataset>(train, dev); - } - - @Override - public Pair,GeneralDataset> split(int start, int end) { - int devSize = end - start; - int trainSize = size() - devSize; - - int[][] devData = new int[devSize][]; - int[] devLabels = new int[devSize]; - - int[][] trainData = new int[trainSize][]; - int[] trainLabels = new int[trainSize]; - - System.arraycopy(data, start, devData, 0, devSize); - System.arraycopy(labels, start, devLabels, 0, devSize); - - System.arraycopy(data, 0, trainData, 0, start); - System.arraycopy(data, end, trainData, start, size()-end); - System.arraycopy(labels, 0, trainLabels, 0, start); - System.arraycopy(labels, end, trainLabels, start, size()-end); - - if (this instanceof WeightedDataset) { - float[] trainWeights = new float[trainSize]; - float[] devWeights = new float[devSize]; - - WeightedDataset w = (WeightedDataset)this; - - System.arraycopy(w.weights, start, devWeights, 0, devSize); - System.arraycopy(w.weights, 0, trainWeights, 0, start); - System.arraycopy(w.weights, end, trainWeights, start, size()-end); - - WeightedDataset dev = new WeightedDataset(labelIndex, devLabels, featureIndex, devData, devSize, devWeights); - WeightedDataset train = new WeightedDataset(labelIndex, trainLabels, featureIndex, trainData, trainSize, trainWeights); - - return new Pair,GeneralDataset>(train, dev); - } - Dataset dev = new Dataset(labelIndex, devLabels, featureIndex, devData, devSize); - Dataset train = new Dataset(labelIndex, trainLabels, featureIndex, trainData, trainSize); - - return new Pair,GeneralDataset>(train, dev); - } - - - public Dataset getRandomSubDataset(double p, int seed) { - int newSize = (int)(p * size()); - Set indicesToKeep = new HashSet(); - Random r = new Random(seed); - int s = size(); - while (indicesToKeep.size() < newSize) { - indicesToKeep.add(r.nextInt(s)); - } - - int[][] newData = new int[newSize][]; - int[] newLabels = new int[newSize]; - - int i = 0; - for (int j : indicesToKeep) { - newData[i] = data[j]; - newLabels[i] = labels[j]; - i++; - } - - return new Dataset(labelIndex, newLabels, featureIndex, newData); - } - - @Override - public double[][] getValuesArray() { - return null; - } - - /** - * Constructs a Dataset by reading in a file in SVM light format. - */ - public static Dataset readSVMLightFormat(String filename) { - return readSVMLightFormat(filename, new HashIndex(), new HashIndex()); - } - - /** - * Constructs a Dataset by reading in a file in SVM light format. - * The lines parameter is filled with the lines of the file for further processing - * (if lines is null, it is assumed no line information is desired) - */ - public static Dataset readSVMLightFormat(String filename, List lines) { - return readSVMLightFormat(filename, new HashIndex(), new HashIndex(), lines); - } - - /** - * Constructs a Dataset by reading in a file in SVM light format. - * the created dataset has the same feature and label index as given - */ - public static Dataset readSVMLightFormat(String filename, Index featureIndex, Index labelIndex) { - return readSVMLightFormat(filename, featureIndex, labelIndex, null); - } - /** - * Constructs a Dataset by reading in a file in SVM light format. - * the created dataset has the same feature and label index as given - */ - public static Dataset readSVMLightFormat(String filename, Index featureIndex, Index labelIndex, List lines) { - Dataset dataset; - try { - dataset = new Dataset(10, featureIndex, labelIndex); - for (String line : ObjectBank.getLineIterator(new File(filename))) { - if(lines != null) - lines.add(line); - dataset.add(svmLightLineToDatum(line)); - } - - } catch (Exception e) { - throw new RuntimeException(e); - } - return dataset; - } - - private static int line1 = 0; - - public static Datum svmLightLineToDatum(String l) { - line1++; - l = l.replaceAll("#.*", ""); // remove any trailing comments - String[] line = l.split("\\s+"); - Collection features = new ArrayList(); - for (int i = 1; i < line.length; i++) { - String[] f = line[i].split(":"); - if (f.length != 2) { - System.err.println("Dataset error: line " + line1); - } - int val = (int) Double.parseDouble(f[1]); - for (int j = 0; j < val; j++) { - features.add(f[0]); - } - } - features.add(String.valueOf(Integer.MAX_VALUE)); // a constant feature for a class - Datum d = new BasicDatum(features, line[0]); - return d; - } - - /** - * Get Number of datums a given feature appears in. - */ - public Counter getFeatureCounter() - { - Counter featureCounts = new ClassicCounter(); - for (int i=0; i < this.size(); i++) - { - BasicDatum datum = (BasicDatum) getDatum(i); - Set featureSet = new HashSet(datum.asFeatures()); - for (F key : featureSet) { - featureCounts.incrementCount(key, 1.0); - } - } - return featureCounts; - } - - /** - * Method to convert features from counts to L1-normalized TFIDF based features - * @param datum with a collection of features. - * @param featureDocCounts a counter of doc-count for each feature. - * @return RVFDatum with l1-normalized tf-idf features. - */ - public RVFDatum getL1NormalizedTFIDFDatum(Datum datum,Counter featureDocCounts){ - Counter tfidfFeatures = new ClassicCounter(); - for(F feature : datum.asFeatures()){ - if(featureDocCounts.containsKey(feature)) - tfidfFeatures.incrementCount(feature,1.0); - } - double l1norm = 0; - for(F feature: tfidfFeatures.keySet()){ - double idf = Math.log(((double)(this.size()+1))/(featureDocCounts.getCount(feature)+0.5)); - double tf = tfidfFeatures.getCount(feature); - tfidfFeatures.setCount(feature, tf*idf); - l1norm += tf*idf; - } - for(F feature: tfidfFeatures.keySet()){ - double tfidf = tfidfFeatures.getCount(feature); - tfidfFeatures.setCount(feature, tfidf/l1norm); - } - RVFDatum rvfDatum = new RVFDatum(tfidfFeatures,datum.label()); - return rvfDatum; - } - - /** - * Method to convert this dataset to RVFDataset using L1-normalized TF-IDF features - * @return RVFDataset - */ - public RVFDataset getL1NormalizedTFIDFDataset(){ - RVFDataset rvfDataset = new RVFDataset(this.size(),this.featureIndex,this.labelIndex); - Counter featureDocCounts = getFeatureCounter(); - for(int i = 0; i < this.size(); i++){ - Datum datum = this.getDatum(i); - RVFDatum rvfDatum = getL1NormalizedTFIDFDatum(datum,featureDocCounts); - rvfDataset.add(rvfDatum); - } - return rvfDataset; - } - - @Override - public void add(Datum d) { - add(d.asFeatures(), d.label()); - } - - public void add(Collection features, L label) { - ensureSize(); - addLabel(label); - addFeatures(features); - size++; - } - - protected void ensureSize() { - if (labels.length == size) { - int[] newLabels = new int[size * 2]; - System.arraycopy(labels, 0, newLabels, 0, size); - labels = newLabels; - int[][] newData = new int[size * 2][]; - System.arraycopy(data, 0, newData, 0, size); - data = newData; - } - } - - protected void addLabel(L label) { - labelIndex.add(label); - labels[size] = labelIndex.indexOf(label); - } - - protected void addFeatures(Collection features) { - int[] intFeatures = new int[features.size()]; - int j = 0; - for (F feature : features) { - featureIndex.add(feature); - int index = featureIndex.indexOf(feature); - if (index >= 0) { - intFeatures[j] = featureIndex.indexOf(feature); - j++; - } - } - data[size] = new int[j]; - System.arraycopy(intFeatures, 0, data[size], 0, j); - } - - - - @Override - protected final void initialize(int numDatums) { - labelIndex = new HashIndex(); - featureIndex = new HashIndex(); - labels = new int[numDatums]; - data = new int[numDatums][]; - size = 0; - } - - /** - * @return the index-ed datum - */ - @Override - public Datum getDatum(int index) { - return new BasicDatum(featureIndex.objects(data[index]), labelIndex.get(labels[index])); - } - - /** - * @return the index-ed datum - */ - @Override - public RVFDatum getRVFDatum(int index) { - ClassicCounter c = new ClassicCounter(); - for (F key : featureIndex.objects(data[index])) { - c.incrementCount(key); - } - return new RVFDatum(c, labelIndex.get(labels[index])); - } - - /** - * Prints some summary statistics to stderr for the Dataset. - */ - @Override - public void summaryStatistics() { - System.err.println(toSummaryStatistics()); - } - - public String toSummaryStatistics() { - StringBuilder sb = new StringBuilder(); - sb.append("numDatums: ").append(size).append('\n'); - sb.append("numLabels: ").append(labelIndex.size()).append(" ["); - Iterator iter = labelIndex.iterator(); - while (iter.hasNext()) { - sb.append(iter.next()); - if (iter.hasNext()) { - sb.append(", "); - } - } - sb.append("]\n"); - sb.append("numFeatures (Phi(X) types): ").append(featureIndex.size()).append('\n'); - // List l = new ArrayList(featureIndex); -// Collections.sort(l); -// sb.append(l); - return sb.toString(); - } - - - /** - * Applies feature count thresholds to the Dataset. - * Only features that match pattern_i and occur at - * least threshold_i times (for some i) are kept. - * - * @param thresholds a list of pattern, threshold pairs - */ - public void applyFeatureCountThreshold(List> thresholds) { - - // get feature counts - float[] counts = getFeatureCounts(); - - // build a new featureIndex - Index newFeatureIndex = new HashIndex(); - LOOP: for (F f : featureIndex) { - for (Pair threshold : thresholds) { - Pattern p = threshold.first(); - Matcher m = p.matcher(f.toString()); - if (m.matches()) { - if (counts[featureIndex.indexOf(f)] >= threshold.second) { - newFeatureIndex.add(f); - } - continue LOOP; - } - } - // we only get here if it didn't match anything on the list - newFeatureIndex.add(f); - } - - counts = null; - - int[] featMap = new int[featureIndex.size()]; - for (int i = 0; i < featMap.length; i++) { - featMap[i] = newFeatureIndex.indexOf(featureIndex.get(i)); - } - - featureIndex = null; - - for (int i = 0; i < size; i++) { - List featList = new ArrayList(data[i].length); - for (int j = 0; j < data[i].length; j++) { - if (featMap[data[i][j]] >= 0) { - featList.add(featMap[data[i][j]]); - } - } - data[i] = new int[featList.size()]; - for (int j = 0; j < data[i].length; j++) { - data[i][j] = featList.get(j); - } - } - - //TODO: continuation of hack from above... hope those features are really strings - //if you call this function. ~Sarah - featureIndex = newFeatureIndex; - } - - - /** - * prints the full feature matrix in tab-delimited form. These can be BIG - * matrices, so be careful! - */ - public void printFullFeatureMatrix(PrintWriter pw) { - String sep = "\t"; - for (int i = 0; i < featureIndex.size(); i++) { - pw.print(sep + featureIndex.get(i)); - } - pw.println(); - for (int i = 0; i < labels.length; i++) { - pw.print(labelIndex.get(i)); - Set feats = new HashSet(); - for (int j = 0; j < data[i].length; j++) { - int feature = data[i][j]; - feats.add(Integer.valueOf(feature)); - } - for (int j = 0; j < featureIndex.size(); j++) { - if (feats.contains(Integer.valueOf(j))) { - pw.print(sep + '1'); - } else { - pw.print(sep + '0'); - } - } - } - } - - /** - * prints the sparse feature matrix using {@link #printSparseFeatureMatrix()} - * to {@link System#out System.out}. - */ - public void printSparseFeatureMatrix() { - printSparseFeatureMatrix(new PrintWriter(System.out, true)); - } - - /** - * prints a sparse feature matrix representation of the Dataset. Prints the actual - * {@link Object#toString()} representations of features. - */ - public void printSparseFeatureMatrix(PrintWriter pw) { - String sep = "\t"; - for (int i = 0; i < size; i++) { - pw.print(labelIndex.get(labels[i])); - int[] datum = data[i]; - for (int j : datum) { - pw.print(sep + featureIndex.get(j)); - } - pw.println(); - } - } - - - public void changeLabelIndex(Index newLabelIndex) { - - labels = trimToSize(labels); - - for (int i = 0; i < labels.length; i++) { - labels[i] = newLabelIndex.indexOf(labelIndex.get(labels[i])); - } - labelIndex = newLabelIndex; - } - - public void changeFeatureIndex(Index newFeatureIndex) { - - data = trimToSize(data); - labels = trimToSize(labels); - - int[][] newData = new int[data.length][]; - for (int i = 0; i < data.length; i++) { - int[] newD = new int[data[i].length]; - int k = 0; - for (int j = 0; j < data[i].length; j++) { - int newIndex = newFeatureIndex.indexOf(featureIndex.get(data[i][j])); - if (newIndex >= 0) { - newD[k++] = newIndex; - } - } - newData[i] = new int[k]; - System.arraycopy(newD, 0, newData[i], 0, k); - } - data = newData; - featureIndex = newFeatureIndex; - } - - public void selectFeaturesBinaryInformationGain(int numFeatures) { - double[] scores = getInformationGains(); - selectFeatures(numFeatures,scores); - } - - /** - * Generic method to select features based on the feature scores vector provided as an argument. - * @param numFeatures number of features to be selected. - * @param scores a vector of size total number of features in the data. - */ - public void selectFeatures(int numFeatures, double[] scores) { - - List> scoredFeatures = new ArrayList>(); - - for (int i = 0; i < scores.length; i++) { - scoredFeatures.add(new ScoredObject(featureIndex.get(i), scores[i])); - } - - Collections.sort(scoredFeatures, ScoredComparator.DESCENDING_COMPARATOR); - Index newFeatureIndex = new HashIndex(); - for (int i = 0; i < scoredFeatures.size() && i < numFeatures; i++) { - newFeatureIndex.add(scoredFeatures.get(i).object()); - //System.err.println(scoredFeatures.get(i)); - } - - for (int i = 0; i < size; i++) { - int[] newData = new int[data[i].length]; - int curIndex = 0; - for (int j = 0; j < data[i].length; j++) { - int index; - if ((index = newFeatureIndex.indexOf(featureIndex.get(data[i][j]))) != -1) { - newData[curIndex++] = index; - } - } - int[] newDataTrimmed = new int[curIndex]; - System.arraycopy(newData, 0, newDataTrimmed, 0, curIndex); - data[i] = newDataTrimmed; - } - featureIndex = newFeatureIndex; - } - - - public double[] getInformationGains() { - - data = trimToSize(data); - labels = trimToSize(labels); - - // counts the number of times word X is present - ClassicCounter featureCounter = new ClassicCounter(); - - // counts the number of time a document has label Y - ClassicCounter labelCounter = new ClassicCounter(); - - // counts the number of times the document has label Y given word X is present - TwoDimensionalCounter condCounter = new TwoDimensionalCounter(); - - for (int i = 0; i < labels.length; i++) { - labelCounter.incrementCount(labelIndex.get(labels[i])); - - // convert the document to binary feature representation - boolean[] doc = new boolean[featureIndex.size()]; - //System.err.println(i); - for (int j = 0; j < data[i].length; j++) { - doc[data[i][j]] = true; - } - - for (int j = 0; j < doc.length; j++) { - if (doc[j]) { - featureCounter.incrementCount(featureIndex.get(j)); - condCounter.incrementCount(featureIndex.get(j), labelIndex.get(labels[i]), 1.0); - } - } - } - - double entropy = 0.0; - for (int i = 0; i < labelIndex.size(); i++) { - double labelCount = labelCounter.getCount(labelIndex.get(i)); - double p = labelCount / size(); - entropy -= p * (Math.log(p) / Math.log(2)); - } - - double[] ig = new double[featureIndex.size()]; - Arrays.fill(ig, entropy); - - for (int i = 0; i < featureIndex.size(); i++) { - F feature = featureIndex.get(i); - - double featureCount = featureCounter.getCount(feature); - double notFeatureCount = size() - featureCount; - - double pFeature = featureCount / size(); - double pNotFeature = (1.0 - pFeature); - - if (featureCount == 0) { ig[i] = 0; continue; } - if (notFeatureCount == 0) { ig[i] = 0; continue; } - - double sumFeature = 0.0; - double sumNotFeature = 0.0; - - for (int j = 0; j < labelIndex.size(); j++) { - L label = labelIndex.get(j); - - double featureLabelCount = condCounter.getCount(feature, label); - double notFeatureLabelCount = size() - featureLabelCount; - - // yes, these dont sum to 1. that is correct. - // one is the prob of the label, given that the - // feature is present, and the other is the prob - // of the label given that the feature is absent - double p = featureLabelCount / featureCount; - double pNot = notFeatureLabelCount / notFeatureCount; - - if (featureLabelCount != 0) { - sumFeature += p * (Math.log(p) / Math.log(2)); - } - - if (notFeatureLabelCount != 0) { - sumNotFeature += pNot * (Math.log(pNot) / Math.log(2)); - } - //System.out.println(pNot+" "+(Math.log(pNot)/Math.log(2))); - - } - - //System.err.println(pFeature+" * "+sumFeature+" = +"+); - //System.err.println("^ "+pNotFeature+" "+sumNotFeature); - - ig[i] += pFeature*sumFeature + pNotFeature*sumNotFeature; - /* earlier the line above used to be: ig[i] = pFeature*sumFeature + pNotFeature*sumNotFeature; - * This completely ignored the entropy term computed above. So added the "+=" to take that into account. - * -Ramesh (nmramesh@cs.stanford.edu) - */ - } - return ig; - } - - - - @Override - public String toString() { - return "Dataset of size " + size; - } - - public String toSummaryString() { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - pw.println("Number of data points: " + size()); - pw.println("Number of active feature tokens: " + numFeatureTokens()); - pw.println("Number of active feature types:" + numFeatureTypes()); - return pw.toString(); - } - - /** - * Need to sort the counter by feature keys and dump it - * - */ - public static void printSVMLightFormat(PrintWriter pw, ClassicCounter c, int classNo) { - Integer[] features = c.keySet().toArray(new Integer[c.keySet().size()]); - Arrays.sort(features); - StringBuilder sb = new StringBuilder(); - sb.append(classNo); - sb.append(' '); - for (int f: features) { - sb.append(f + 1).append(':').append(c.getCount(f)).append(' '); - } - pw.println(sb.toString()); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralDataset.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralDataset.java deleted file mode 100644 index e25adb4..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralDataset.java +++ /dev/null @@ -1,488 +0,0 @@ -package edu.stanford.nlp.classify; - -import java.io.PrintWriter; -import java.io.Serializable; -import java.util.*; - -import edu.stanford.nlp.ling.BasicDatum; -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.util.HashIndex; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.Pair; - -/** - * The purpose of this interface is to unify {@link Dataset} and {@link RVFDataset}. - * @author Kristina Toutanova (kristina@cs.stanford.edu) - * @author Anna Rafferty (various refactoring with subclasses) - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * - * @param The type of the labels in the Dataset - * @param The type of the features in the Dataset - * - * @author Ramesh Nallapati (nmramesh@cs.stanford.edu) - * Added an abstract method getDatum. Juily 17th, 2008. - */ -public abstract class GeneralDataset implements Serializable, Iterable> { - - private static final long serialVersionUID = 19157757130054829L; - - public Index labelIndex; - public Index featureIndex; - - protected int[] labels; - protected int[][] data; - - protected int size; - - public GeneralDataset() { - } - - public Index labelIndex() { return labelIndex; } - - public Index featureIndex() { return featureIndex; } - - public int numFeatures() { return featureIndex.size(); } - - public int numClasses() { return labelIndex.size(); } - - public int[] getLabelsArray() { - labels = trimToSize(labels); - return labels; - } - - public int[][] getDataArray() { - data = trimToSize(data); - return data; - } - - public abstract double[][] getValuesArray(); - - /** - * Resets the Dataset so that it is empty and ready to collect data. - */ - public void clear() { - clear(10); - } - - /** - * Resets the Dataset so that it is empty and ready to collect data. - * @param numDatums initial capacity of dataset - */ - public void clear(int numDatums) { - initialize(numDatums); - } - - /** - * This method takes care of resetting values of the dataset - * such that it is empty with an initial capacity of numDatums - * - * Should be accessed only by appropriate methods within the class, - * such as clear(), which take care of other parts of the emptying of data - * - * @param numDatums initial capacity of dataset - */ - protected abstract void initialize(int numDatums); - - - public abstract RVFDatum getRVFDatum(int index); - - public abstract Datum getDatum(int index); - - - public abstract void add(Datum d); - - /** - * Get the total count (over all data instances) of each feature - * - * @return an array containing the counts (indexed by index) - */ - public float[] getFeatureCounts() { - float[] counts = new float[featureIndex.size()]; - for (int i = 0, m = size; i < m; i++) { - for (int j = 0, n = data[i].length; j < n; j++) { - counts[data[i][j]] += 1.0; - } - } - return counts; - } - - /** - * Applies a feature count threshold to the Dataset. All features that - * occur fewer than k times are expunged. - */ - public void applyFeatureCountThreshold(int k) { - float[] counts = getFeatureCounts(); - Index newFeatureIndex = new HashIndex(); - - int[] featMap = new int[featureIndex.size()]; - for (int i = 0; i < featMap.length; i++) { - F feat = featureIndex.get(i); - if (counts[i] >= k) { - int newIndex = newFeatureIndex.size(); - newFeatureIndex.add(feat); - featMap[i] = newIndex; - } else { - featMap[i] = -1; - } - // featureIndex.remove(feat); - } - - featureIndex = newFeatureIndex; - // counts = null; // This is unnecessary; JVM can clean it up - - for (int i = 0; i < size; i++) { - List featList = new ArrayList(data[i].length); - for (int j = 0; j < data[i].length; j++) { - if (featMap[data[i][j]] >= 0) { - featList.add(featMap[data[i][j]]); - } - } - data[i] = new int[featList.size()]; - for (int j = 0; j < data[i].length; j++) { - data[i][j] = featList.get(j); - } - } - } - - - /** - * Applies a max feature count threshold to the Dataset. All features that - * occur greater than k times are expunged. - */ - public void applyFeatureMaxCountThreshold(int k) { - float[] counts = getFeatureCounts(); - HashIndex newFeatureIndex = new HashIndex(); - - int[] featMap = new int[featureIndex.size()]; - for (int i = 0; i < featMap.length; i++) { - F feat = featureIndex.get(i); - if (counts[i] <= k) { - int newIndex = newFeatureIndex.size(); - newFeatureIndex.add(feat); - featMap[i] = newIndex; - } else { - featMap[i] = -1; - } - // featureIndex.remove(feat); - } - - featureIndex = newFeatureIndex; - // counts = null; // This is unnecessary; JVM can clean it up - - for (int i = 0; i < size; i++) { - List featList = new ArrayList(data[i].length); - for (int j = 0; j < data[i].length; j++) { - if (featMap[data[i][j]] >= 0) { - featList.add(featMap[data[i][j]]); - } - } - data[i] = new int[featList.size()]; - for (int j = 0; j < data[i].length; j++) { - data[i][j] = featList.get(j); - } - } - } - - - /** - * returns the number of feature tokens in the Dataset. - */ - public int numFeatureTokens() { - int x = 0; - for (int i = 0, m = size; i < m; i++) { - x += data[i].length; - } - return x; - } - - /** - * returns the number of distinct feature types in the Dataset. - */ - public int numFeatureTypes() { - return featureIndex.size(); - } - - - - /** - * Adds all Datums in the given collection of data to this dataset - * @param data collection of datums you would like to add to the dataset - */ - public void addAll(Iterable> data) { - for (Datum d : data) { - add(d); - } - } - - public abstract Pair, GeneralDataset> split (int start, int end) ; - public abstract Pair, GeneralDataset> split (double p) ; - - /** - * Returns the number of examples ({@link Datum}s) in the Dataset. - */ - public int size() { return size; } - - protected void trimData() { - data = trimToSize(data); - } - - protected void trimLabels() { - labels = trimToSize(labels); - } - - protected int[] trimToSize(int[] i) { - int[] newI = new int[size]; - System.arraycopy(i, 0, newI, 0, size); - return newI; - } - - protected int[][] trimToSize(int[][] i) { - int[][] newI = new int[size][]; - System.arraycopy(i, 0, newI, 0, size); - return newI; - } - - protected double[][] trimToSize(double[][] i) { - double[][] newI = new double[size][]; - System.arraycopy(i, 0, newI, 0, size); - return newI; - } - - /** - * Randomizes the data array in place - * Note: this cannot change the values array or the datum weights, so redefine this for RVFDataset and WeightedDataset! - * @param randomSeed - */ - public void randomize(int randomSeed) { - Random rand = new Random(randomSeed); - for(int j = size - 1; j > 0; j --){ - int randIndex = rand.nextInt(j); - - int [] tmp = data[randIndex]; - data[randIndex] = data[j]; - data[j] = tmp; - - int tmpl = labels[randIndex]; - labels[randIndex] = labels[j]; - labels[j] = tmpl; - } - } - - public GeneralDataset sampleDataset(int randomSeed, double sampleFrac, boolean sampleWithReplacement) { - int sampleSize = (int)(this.size()*sampleFrac); - Random rand = new Random(randomSeed); - GeneralDataset subset; - if(this instanceof RVFDataset) - subset = new RVFDataset(); - else if (this instanceof Dataset) { - subset = new Dataset(); - } - else { - throw new RuntimeException("Can't handle this type of GeneralDataset."); - } - if (sampleWithReplacement) { - for(int i = 0; i < sampleSize; i++){ - int datumNum = rand.nextInt(this.size()); - subset.add(this.getDatum(datumNum)); - } - } else { - Set indicedSampled = new HashSet(); - while (subset.size() < sampleSize) { - int datumNum = rand.nextInt(this.size()); - if (!indicedSampled.contains(datumNum)) { - subset.add(this.getDatum(datumNum)); - indicedSampled.add(datumNum); - } - } - } - return subset; - } - - /** - * Print some statistics summarizing the dataset - * - */ - public abstract void summaryStatistics(); - - /** - * Returns an iterator over the class labels of the Dataset - * - * @return An iterator over the class labels of the Dataset - */ - public Iterator labelIterator() { - return labelIndex.iterator(); - } - - - /** - * - * @param dataset - * @return a new GeneralDataset whose features and ids map exactly to those of this GeneralDataset. - * Useful when two Datasets are created independently and one wants to train a model on one dataset and test on the other. -Ramesh. - */ - public GeneralDataset mapDataset(GeneralDataset dataset){ - GeneralDataset newDataset; - if(dataset instanceof RVFDataset) - newDataset = new RVFDataset(this.featureIndex,this.labelIndex); - else newDataset = new Dataset(this.featureIndex,this.labelIndex); - this.featureIndex.lock(); - this.labelIndex.lock(); - //System.out.println("inside mapDataset: dataset size:"+dataset.size()); - for(int i = 0; i < dataset.size(); i++) - //System.out.println("inside mapDataset: adding datum number"+i); - newDataset.add(dataset.getDatum(i)); - - //System.out.println("old Dataset stats: numData:"+dataset.size()+" numfeatures:"+dataset.featureIndex().size()+" numlabels:"+dataset.labelIndex.size()); - //System.out.println("new Dataset stats: numData:"+newDataset.size()+" numfeatures:"+newDataset.featureIndex().size()+" numlabels:"+newDataset.labelIndex.size()); - //System.out.println("this dataset stats: numData:"+size()+" numfeatures:"+featureIndex().size()+" numlabels:"+labelIndex.size()); - - this.featureIndex.unlock(); - this.labelIndex.unlock(); - return newDataset; - } - - public static Datum mapDatum(Datum d, Map labelMapping, L2 defaultLabel) - { - Datum d2 = null; - // TODO: How to copy datum? - L2 newLabel = labelMapping.get(d.label()); - if (newLabel == null) { - newLabel = defaultLabel; - } - if (d instanceof RVFDatum) { - d2 = new RVFDatum( ((RVFDatum) d).asFeaturesCounter(), newLabel ); - } else { - d2 = new BasicDatum( d.asFeatures(), newLabel ); - } - return d2; - } - - - /** - * - * @param dataset - * @return a new GeneralDataset whose features and ids map exactly to those of this GeneralDataset. But labels are converted to be another set of labels - */ - public GeneralDataset mapDataset(GeneralDataset dataset, Index newLabelIndex, Map labelMapping, L2 defaultLabel) - { - GeneralDataset newDataset; - if(dataset instanceof RVFDataset) - newDataset = new RVFDataset(this.featureIndex, newLabelIndex); - else newDataset = new Dataset(this.featureIndex, newLabelIndex); - this.featureIndex.lock(); - this.labelIndex.lock(); - //System.out.println("inside mapDataset: dataset size:"+dataset.size()); - for(int i = 0; i < dataset.size(); i++) { - //System.out.println("inside mapDataset: adding datum number"+i); - Datum d = dataset.getDatum(i); - Datum d2 = mapDatum(d, labelMapping, defaultLabel); - newDataset.add(d2); - } - //System.out.println("old Dataset stats: numData:"+dataset.size()+" numfeatures:"+dataset.featureIndex().size()+" numlabels:"+dataset.labelIndex.size()); - //System.out.println("new Dataset stats: numData:"+newDataset.size()+" numfeatures:"+newDataset.featureIndex().size()+" numlabels:"+newDataset.labelIndex.size()); - //System.out.println("this dataset stats: numData:"+size()+" numfeatures:"+featureIndex().size()+" numlabels:"+labelIndex.size()); - - this.featureIndex.unlock(); - this.labelIndex.unlock(); - return newDataset; - } - - /** - * Dumps the Dataset as a training/test file for SVMLight.
- * class [fno:val]+ - * The features must occur in consecutive order. - */ - public void printSVMLightFormat() { - printSVMLightFormat(new PrintWriter(System.out)); - } - - /** - * Maps our labels to labels that are compatible with svm_light - * @return array of strings - */ - public String[] makeSvmLabelMap() { - String[] labelMap = new String[numClasses()]; - if (numClasses() > 2) { - for (int i = 0; i < labelMap.length; i++) { - labelMap[i] = String.valueOf((i + 1)); - } - } else { - labelMap = new String[]{"+1", "-1"}; - } - return labelMap; - } - - /** - * Print SVM Light Format file. - * - * The following comments are no longer applicable because I am - * now printing out the exact labelID for each example. -Ramesh (nmramesh@cs.stanford.edu) 12/17/2009. - * - * If the Dataset has more than 2 classes, then it - * prints using the label index (+1) (for svm_struct). If it is 2 classes, then the labelIndex.get(0) - * is mapped to +1 and labelIndex.get(1) is mapped to -1 (for svm_light). - */ - - public void printSVMLightFormat(PrintWriter pw) { - //assumes each data item has a few features on, and sorts the feature keys while collecting the values in a counter - - // old comment: - // the following code commented out by Ramesh (nmramesh@cs.stanford.edu) 12/17/2009. - // why not simply print the exact id of the label instead of mapping to some values?? - // new comment: - // mihai: we NEED this, because svm_light has special conventions not supported by default by our labels, - // e.g., in a multiclass setting it assumes that labels start at 1 whereas our labels start at 0 (08/31/2010) - String[] labelMap = makeSvmLabelMap(); - - for (int i = 0; i < size; i++) { - RVFDatum d = getRVFDatum(i); - Counter c = d.asFeaturesCounter(); - ClassicCounter printC = new ClassicCounter(); - for (F f : c.keySet()) { - printC.setCount(featureIndex.indexOf(f), c.getCount(f)); - } - Integer[] features = printC.keySet().toArray(new Integer[printC.keySet().size()]); - Arrays.sort(features); - StringBuilder sb = new StringBuilder(); - sb.append(labelMap[labels[i]]).append(' '); - // sb.append(labels[i]).append(' '); // commented out by mihai: labels[i] breaks svm_light conventions! - - /* Old code: assumes that F is Integer.... - * - for (int f: features) { - sb.append((f + 1)).append(":").append(c.getCount(f)).append(" "); - } - */ - //I think this is what was meant (using printC rather than c), but not sure - // ~Sarah Spikes (sdspikes@cs.stanford.edu) - for (int f: features) { - sb.append((f + 1)).append(':').append(printC.getCount(f)).append(' '); - } - pw.println(sb.toString()); - } - } - - - public Iterator> iterator() { - return new Iterator>() { - int id = 0; - - public boolean hasNext() { - return id < size(); - } - - public RVFDatum next() { - return getRVFDatum(id++); - } - - public void remove() { - throw new UnsupportedOperationException(); - } - - }; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralizedExpectationObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralizedExpectationObjectiveFunction.java deleted file mode 100644 index 8c1ce5b..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralizedExpectationObjectiveFunction.java +++ /dev/null @@ -1,217 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.optimization.AbstractCachingDiffFunction; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.util.Triple; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - - -/** - * Implementation of Generalized Expectation Objective function for an I.I.D. log-linear model. See Mann and McCallum, ACL 2008. - * IMPORTANT : the current implementation is only correct as long as the labeled features passed to GE are binary. - * However, other features are allowed to be real valued. - * The original paper also discusses GE only for binary features. - * @author Ramesh Nallapati (nmramesh@cs.stanford.edu) - */ - -public class GeneralizedExpectationObjectiveFunction extends AbstractCachingDiffFunction { - - - private GeneralDataset labeledDataset; - private List> unlabeledDataList; - private List geFeatures; - private LinearClassifier classifier; - private double[][] geFeature2EmpiricalDist; //empirical label distributions of each feature. - private List> geFeature2DatumList; //an inverted list of active unlabeled documents for each feature. - - protected int numFeatures = 0; - protected int numClasses = 0; - - @Override - public int domainDimension() { - return numFeatures * numClasses; - } - - int classOf(int index) { - return index % numClasses; - } - - int featureOf(int index) { - return index / numClasses; - } - - protected int indexOf(int f, int c) { - return f * numClasses + c; - } - - public double[][] to2D(double[] x) { - double[][] x2 = new double[numFeatures][numClasses]; - for (int i = 0; i < numFeatures; i++) { - for (int j = 0; j < numClasses; j++) { - x2[i][j] = x[indexOf(i, j)]; - } - } - return x2; - } - - @Override - protected void calculate(double[] x) { - classifier.setWeights(to2D(x)); - if (derivative == null) { - derivative = new double[x.length]; - } else { - Arrays.fill(derivative, 0.0); - } - Counter> feature2classPairDerivatives = new ClassicCounter>(); - - value = 0.0; - for(int n = 0; n < geFeatures.size(); n++){ - //F feature = geFeatures.get(n); - double[] modelDist = new double[numClasses]; - Arrays.fill(modelDist,0); - - //go over the unlabeled active data to compute expectations - List activeData = geFeature2DatumList.get(n); - for(int i = 0; i < activeData.size(); i++){ - Datum datum = unlabeledDataList.get(activeData.get(i)); - double[] probs = getModelProbs(datum); - for(int c = 0; c < numClasses; c++) - modelDist[c]+= probs[c]; - updateDerivative(datum,probs,feature2classPairDerivatives); //computes p(y_d)*(1-p(y_d))*f_d for all active features. - } - - //now compute the value (KL-divergence) and the final value of the derivative. - if(activeData.size()>0){ - for(int c = 0; c < numClasses; c++) - modelDist[c]/= activeData.size(); - smoothDistribution(modelDist); - - for(int c = 0; c < numClasses; c++) - value += -geFeature2EmpiricalDist[n][c]*Math.log(modelDist[c]); - - for(int f = 0; f < labeledDataset.featureIndex().size(); f++){ - for(int c = 0; c < numClasses; c++){ - int wtIndex = indexOf(f,c); - for(int cPrime = 0; cPrime < numClasses; cPrime++){ - derivative[wtIndex] += feature2classPairDerivatives.getCount(new Triple(f,c,cPrime))*geFeature2EmpiricalDist[n][cPrime]/modelDist[cPrime]; - } - derivative[wtIndex] /= activeData.size(); - } - } // loop over each feature for derivative computation - } //end of if condition - } //loop over each GE feature - } - - - private void updateDerivative(Datum datum, double[] probs,Counter> feature2classPairDerivatives){ - for(F feature : datum.asFeatures()){ - int fID = labeledDataset.featureIndex.indexOf(feature); - if(fID >= 0){ - for(int c = 0; c < numClasses; c++){ - for(int cPrime = 0; cPrime < numClasses; cPrime++){ - if(cPrime == c) - feature2classPairDerivatives.incrementCount(new Triple(fID,c,cPrime), - probs[c]*(1-probs[c])*valueOfFeature(feature,datum)); - else - feature2classPairDerivatives.incrementCount(new Triple(fID,c,cPrime), probs[c]*probs[cPrime]*valueOfFeature(feature,datum)); - } - } - } - } - } - - /* - * This method assumes the feature already exists in the datum. - */ - private double valueOfFeature(F feature, Datum datum){ - if(datum instanceof RVFDatum) - return ((RVFDatum)datum).asFeaturesCounter().getCount(feature); - else return 1.0; - } - - private void computeEmpiricalStatistics(List geFeatures){ - //allocate memory to the containers and initialize them - geFeature2EmpiricalDist = new double[geFeatures.size()][labeledDataset.labelIndex.size()]; - geFeature2DatumList = new ArrayList>(geFeatures.size()); - Map geFeatureMap = new HashMap(); - Set activeUnlabeledExamples = new HashSet(); - for(int n = 0; n < geFeatures.size(); n++){ - F geFeature = geFeatures.get(n); - geFeature2DatumList.add(new ArrayList()); - Arrays.fill(geFeature2EmpiricalDist[n], 0); - geFeatureMap.put(geFeature,n); - } - - //compute the empirical label distribution for each GE feature - for(int i = 0; i < labeledDataset.size(); i++){ - Datum datum = labeledDataset.getDatum(i); - int labelID = labeledDataset.labelIndex.indexOf(datum.label()); - for(F feature : datum.asFeatures()){ - if(geFeatureMap.containsKey(feature)){ - int geFnum = geFeatureMap.get(feature); - geFeature2EmpiricalDist[geFnum][labelID]++; - } - } - } - //now normalize and smooth the label distribution for each feature. - for(int n = 0; n < geFeatures.size(); n++){ - ArrayMath.normalize(geFeature2EmpiricalDist[n]); - smoothDistribution(geFeature2EmpiricalDist[n]); - } - - //now build the inverted index from each GE feature to unlabeled datums that contain it. - for(int i = 0; i < unlabeledDataList.size(); i++){ - Datum datum = unlabeledDataList.get(i); - for(F feature : datum.asFeatures()){ - if(geFeatureMap.containsKey(feature)){ - int geFnum = geFeatureMap.get(feature); - geFeature2DatumList.get(geFnum).add(i); - activeUnlabeledExamples.add(i); - } - } - } - System.out.println("Number of active unlabeled examples:"+activeUnlabeledExamples.size()); - } - - private void smoothDistribution(double [] dist){ - //perform Laplace smoothing - double epsilon = 1e-6; - for(int i = 0; i < dist.length; i++) - dist[i] += epsilon; - ArrayMath.normalize(dist); - } - - private double[] getModelProbs(Datum datum){ - double[] condDist = new double[labeledDataset.numClasses()]; - Counter probCounter = classifier.probabilityOf(datum); - for(L label : probCounter.keySet()){ - int labelID = labeledDataset.labelIndex.indexOf(label); - condDist[labelID] = probCounter.getCount(label); - } - return condDist; - } - - public GeneralizedExpectationObjectiveFunction(GeneralDataset labeledDataset, List> unlabeledDataList,List geFeatures) { - System.out.println("Number of labeled examples:"+labeledDataset.size+"\nNumber of unlabeled examples:"+unlabeledDataList.size()); - System.out.println("Number of GE features:"+geFeatures.size()); - this.numFeatures = labeledDataset.numFeatures(); - this.numClasses = labeledDataset.numClasses(); - this.labeledDataset = labeledDataset; - this.unlabeledDataList = unlabeledDataList; - this.geFeatures = geFeatures; - this.classifier = new LinearClassifier(null,labeledDataset.featureIndex,labeledDataset.labelIndex); - computeEmpiricalStatistics(geFeatures); - //empirical distributions don't change with iterations, so compute them only once. - //model distributions will have to be recomputed every iteration though. - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LinearClassifier.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LinearClassifier.java deleted file mode 100644 index 5655256..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LinearClassifier.java +++ /dev/null @@ -1,1402 +0,0 @@ -// Stanford Classifier - a multiclass maxent classifier -// LinearClassifier -// Copyright (c) 2003-2007 The Board of Trustees of -// The Leland Stanford Junior University. All Rights Reserved. -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -// -// For more information, bug reports, fixes, contact: -// Christopher Manning -// Dept of Computer Science, Gates 1A -// Stanford CA 94305-9010 -// USA -// Support/Questions: java-nlp-user@lists.stanford.edu -// Licensing: java-nlp-support@lists.stanford.edu -// http://www-nlp.stanford.edu/software/classifier.shtml - -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.ling.BasicDatum; -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.util.*; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.stats.Distribution; -import edu.stanford.nlp.stats.Counters; - -import java.io.*; -import java.text.DecimalFormat; -import java.text.NumberFormat; -import java.util.*; - - -/** - * Implements a multiclass linear classifier. At classification time this - * can be any generalized linear model classifier (such as a perceptron, - * naive logistic regression, SVM). - * - * @author Dan Klein - * @author Jenny Finkel - * @author Galen Andrew (converted to arrays and indices) - * @author Christopher Manning (most of the printing options) - * @author Eric Yeh (save to text file, new constructor w/thresholds) - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * @author (nmramesh@cs.stanford.edu) {@link #weightsAsMapOfCounters()} - * @author Angel Chang (Add functions to get top features, and number of features with weights above a certain threshold) - * - * @param The type of the labels in the Classifier - * @param The type of the features in the Classifier - */ -public class LinearClassifier implements ProbabilisticClassifier, RVFClassifier { - - /** Classifier weights. First index is the featureIndex value and second - * index is the labelIndex value. - */ - private double[][] weights; - private Index labelIndex; - private Index featureIndex; - public boolean intern = false; // variable should be deleted when breaking serialization anyway.... - private double[] thresholds = null; - - private static final long serialVersionUID = 8499574525453275255L; - - private static final int MAX_FEATURE_ALIGN_WIDTH = 50; - - public static final String TEXT_SERIALIZATION_DELIMITER = "\t"; - - public Collection labels() { - return labelIndex.objectsList(); - } - - public Collection features() { - return featureIndex.objectsList(); - } - - public Index labelIndex() { - return labelIndex; - } - - public Index featureIndex() { - return featureIndex; - } - - private double weight(int iFeature, int iLabel) { - if (iFeature < 0) { - //System.err.println("feature not seen "); - return 0.0; - } - return weights[iFeature][iLabel]; - } - - private double weight(F feature, int iLabel) { - int f = featureIndex.indexOf(feature); - return weight(f, iLabel); - } - - public double weight(F feature, L label) { - int f = featureIndex.indexOf(feature); - int iLabel = labelIndex.indexOf(label); - return weight(f, iLabel); - } - - /* --- obsolete method from before this class was rewritten using arrays - public Counter scoresOf(Datum example) { - Counter scores = new Counter(); - for (Object l : labels()) { - scores.setCount(l, scoreOf(example, l)); - } - return scores; - } - --- */ - - /** Construct a counter with keys the labels of the classifier and - * values the score (unnormalized log probability) of each class. - */ - public Counter scoresOf(Datum example) { - if(example instanceof RVFDatum)return scoresOfRVFDatum((RVFDatum)example); - Collection feats = example.asFeatures(); - int[] features = new int[feats.size()]; - int i = 0; - for (F f : feats) { - int index = featureIndex.indexOf(f); - if (index >= 0) { - features[i++] = index; - } else { - //System.err.println("FEATURE LESS THAN ZERO: " + f); - } - } - int[] activeFeatures = new int[i]; - System.arraycopy(features, 0, activeFeatures, 0, i); - Counter scores = new ClassicCounter(); - for (L lab : labels()) { - scores.setCount(lab, scoreOf(activeFeatures, lab)); - } - return scores; - } - - /** Returns of the score of the Datum for the specified label. - * Ignores the true label of the Datum. - */ - public double scoreOf(Datum example, L label) { - if(example instanceof RVFDatum)return scoreOfRVFDatum((RVFDatum)example, label); - int iLabel = labelIndex.indexOf(label); - double score = 0.0; - for (F f : example.asFeatures()) { - score += weight(f, iLabel); - } - return score + thresholds[iLabel]; - } - - /** Construct a counter with keys the labels of the classifier and - * values the score (unnormalized log probability) of each class - * for an RVFDatum. - */ - @Deprecated - public Counter scoresOf(RVFDatum example) { - Counter scores = new ClassicCounter(); - for (L l : labels()) { - scores.setCount(l, scoreOf(example, l)); - } - //System.out.println("Scores are: " + scores + " (gold: " + example.label() + ")"); - return scores; - } - - /** Construct a counter with keys the labels of the classifier and - * values the score (unnormalized log probability) of each class - * for an RVFDatum. - */ - private Counter scoresOfRVFDatum(RVFDatum example) { - Counter scores = new ClassicCounter(); - for (L l : labels()) { - scores.setCount(l, scoreOfRVFDatum(example, l)); - } - //System.out.println("Scores are: " + scores + " (gold: " + example.label() + ")"); - return scores; - } - - /** Returns the score of the RVFDatum for the specified label. - * Ignores the true label of the RVFDatum. - */ - @Deprecated - public double scoreOf(RVFDatum example, L label) { - int iLabel = labelIndex.indexOf(label); - double score = 0.0; - Counter features = example.asFeaturesCounter(); - for (F f : features.keySet()) { - score += weight(f, iLabel) * features.getCount(f); - } - return score + thresholds[iLabel]; - } - - /** Returns the score of the RVFDatum for the specified label. - * Ignores the true label of the RVFDatum. - */ - private double scoreOfRVFDatum(RVFDatum example, L label) { - int iLabel = labelIndex.indexOf(label); - double score = 0.0; - Counter features = example.asFeaturesCounter(); - for (F f : features.keySet()) { - score += weight(f, iLabel) * features.getCount(f); - } - return score + thresholds[iLabel]; - } - - - /** Returns of the score of the Datum as internalized features for the - * specified label. Ignores the true label of the Datum. - * Doesn't consider a value for each feature. - */ - private double scoreOf(int[] feats, L label) { - int iLabel = labelIndex.indexOf(label); - double score = 0.0; - for (int feat : feats) { - score += weight(feat, iLabel); - } - return score + thresholds[iLabel]; - } - - - /** - * Returns a counter mapping from each class name to the probability of - * that class for a certain example. - * Looking at the the sum of each count v, should be 1.0. - */ - public Counter probabilityOf(Datum example) { - if(example instanceof RVFDatum)return probabilityOfRVFDatum((RVFDatum)example); - Counter scores = logProbabilityOf(example); - for (L label : scores.keySet()) { - scores.setCount(label, Math.exp(scores.getCount(label))); - } - return scores; - } - - /** - * Returns a counter mapping from each class name to the probability of - * that class for a certain example. - * Looking at the the sum of each count v, should be 1.0. - */ - private Counter probabilityOfRVFDatum(RVFDatum example) { - // NB: this duplicate method is needed so it calls the scoresOf method - // with a RVFDatum signature - Counter scores = logProbabilityOfRVFDatum(example); - for (L label : scores.keySet()) { - scores.setCount(label, Math.exp(scores.getCount(label))); - } - return scores; - } - - /** - * Returns a counter mapping from each class name to the probability of - * that class for a certain example. - * Looking at the the sum of each count v, should be 1.0. - */ - @Deprecated - public Counter probabilityOf(RVFDatum example) { - // NB: this duplicate method is needed so it calls the scoresOf method - // with a RVFDatum signature - Counter scores = logProbabilityOf(example); - for (L label : scores.keySet()) { - scores.setCount(label, Math.exp(scores.getCount(label))); - } - return scores; - } - - /** - * Returns a counter mapping from each class name to the log probability of - * that class for a certain example. - * Looking at the the sum of e^v for each count v, should be 1.0. - */ - public Counter logProbabilityOf(Datum example) { - if(example instanceof RVFDatum)return logProbabilityOfRVFDatum((RVFDatum)example); - Counter scores = scoresOf(example); - Counters.logNormalizeInPlace(scores); - return scores; - } - - /** - * Returns a counter for the log probability of each of the classes - * looking at the the sum of e^v for each count v, should be 1 - */ - private Counter logProbabilityOfRVFDatum(RVFDatum example) { - // NB: this duplicate method is needed so it calls the scoresOf method - // with an RVFDatum signature!! Don't remove it! - // JLS: type resolution of method parameters is static - Counter scores = scoresOfRVFDatum(example); - Counters.logNormalizeInPlace(scores); - return scores; - } - - /** - * Returns a counter for the log probability of each of the classes - * looking at the the sum of e^v for each count v, should be 1 - */ - @Deprecated - public Counter logProbabilityOf(RVFDatum example) { - // NB: this duplicate method is needed so it calls the scoresOf method - // with an RVFDatum signature!! Don't remove it! - // JLS: type resolution of method parameters is static - Counter scores = scoresOf(example); - Counters.logNormalizeInPlace(scores); - return scores; - } - - /** - * Returns indices of labels - * @param labels - Set of labels to get indicies - * @return Set of indicies - */ - protected Set getLabelIndices(Set labels) { - Set iLabels = new HashSet(); - for (L label:labels) { - int iLabel = labelIndex.indexOf(label); - iLabels.add(iLabel); - if (iLabel < 0) throw new IllegalArgumentException("Unknown label " + label); - } - return iLabels; - } - - /** - * Returns number of features with weight above a certain threshold - * (across all labels) - * @param threshold Threshold above which we will count the feature - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @return number of features satisfying the specified conditions - */ - public int getFeatureCount(double threshold, boolean useMagnitude) - { - int n = 0; - for (int feat = 0; feat < weights.length; feat++) { - for (int lab = 0; lab < weights[feat].length; lab++) { - double thisWeight = (useMagnitude)? Math.abs(weights[feat][lab]):weights[feat][lab]; - if (thisWeight > threshold) { - n++; - } - } - } - return n; - } - - /** - * Returns number of features with weight above a certain threshold - * @param labels Set of labels we care about when counting features - * Use null to get counts across all labels - * @param threshold Threshold above which we will count the feature - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @return number of features satisfying the specified conditions - */ - public int getFeatureCount(Set labels, double threshold, boolean useMagnitude) - { - if (labels != null) { - Set iLabels = getLabelIndices(labels); - return getFeatureCountLabelIndices(iLabels, threshold, useMagnitude); - } else { - return getFeatureCount(threshold, useMagnitude); - } - } - - /** - * Returns number of features with weight above a certain threshold - * @param iLabels Set of label indices we care about when counting features - * Use null to get counts across all labels - * @param threshold Threshold above which we will count the feature - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @return number of features satisfying the specified conditions - */ - protected int getFeatureCountLabelIndices(Set iLabels, double threshold, boolean useMagnitude) - { - int n = 0; - for (int feat = 0; feat < weights.length; feat++) { - for (int labIndex:iLabels) { - double thisWeight = (useMagnitude)? Math.abs(weights[feat][labIndex]):weights[feat][labIndex]; - if (thisWeight > threshold) { - n++; - } - } - } - return n; - } - - /** - * Returns list of top features with weight above a certain threshold - * (list is descending and across all labels) - * @param threshold Threshold above which we will count the feature - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @param numFeatures How many top features to return (-1 for unlimited) - * @return List of triples indicating feature, label, weight - */ - public List> getTopFeatures(double threshold, boolean useMagnitude, int numFeatures) - { - return getTopFeatures(null, threshold, useMagnitude, numFeatures, true); - } - - /** - * Returns list of top features with weight above a certain threshold - * @param labels Set of labels we care about when getting features - * Use null to get features across all labels - * @param threshold Threshold above which we will count the feature - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @param numFeatures How many top features to return (-1 for unlimited) - * @param descending Return weights in descending order - * @return List of triples indicating feature, label, weight - */ - public List> getTopFeatures(Set labels, - double threshold, boolean useMagnitude, int numFeatures, - boolean descending) - { - if (labels != null) { - Set iLabels = getLabelIndices(labels); - return getTopFeaturesLabelIndices(iLabels, threshold, useMagnitude, numFeatures, descending); - } else { - return getTopFeaturesLabelIndices(null, threshold, useMagnitude, numFeatures, descending); - } - } - - /** - * Returns list of top features with weight above a certain threshold - * @param iLabels Set of label indices we care about when getting features - * Use null to get features across all labels - * @param threshold Threshold above which we will count the feature - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @param numFeatures How many top features to return (-1 for unlimited) - * @param descending Return weights in descending order - * @return List of triples indicating feature, label, weight - */ - protected List> getTopFeaturesLabelIndices(Set iLabels, - double threshold, boolean useMagnitude, int numFeatures, - boolean descending) - { - edu.stanford.nlp.util.PriorityQueue> biggestKeys = - new FixedPrioritiesPriorityQueue>(); - - // locate biggest keys - for (int feat = 0; feat < weights.length; feat++) { - for (int lab = 0; lab < weights[feat].length; lab++) { - if (iLabels != null && !iLabels.contains(lab)) { - continue; - } - double thisWeight; - if (useMagnitude) { - thisWeight = Math.abs(weights[feat][lab]); - } else { - thisWeight = weights[feat][lab]; - } - - if (thisWeight > threshold) { - // reverse the weight, so get smallest first - thisWeight = -thisWeight; - if (biggestKeys.size() == numFeatures) { - // have enough features, add only if bigger - double lowest = biggestKeys.getPriority(); - if (thisWeight < lowest) { - // remove smallest - biggestKeys.removeFirst(); - biggestKeys.add(new Pair(feat, lab), thisWeight); - } - } else { - // always add it if don't have enough features yet - biggestKeys.add(new Pair(feat, lab), thisWeight); - } - } - } - } - - List> topFeatures = new ArrayList>(biggestKeys.size()); - while (!biggestKeys.isEmpty()) { - Pair p = biggestKeys.removeFirst(); - double weight = weights[p.first()][p.second()]; - F feat = featureIndex.get(p.first()); - L label = labelIndex.get(p.second()); - topFeatures.add(new Triple(feat, label, weight)); - } - if (descending) { - Collections.reverse(topFeatures); - } - return topFeatures; - } - - /** - * Returns string representation of a list of top features - * @param topFeatures List of triples indicating feature, label, weight - * @return String representation of the list of features - */ - public String topFeaturesToString(List> topFeatures) - { - // find longest key length (for pretty printing) with a limit - int maxLeng = 0; - for (Triple t : topFeatures) { - String key = "(" + t.first + "," + t.second + ")"; - int leng = key.length(); - if (leng > maxLeng) { - maxLeng = leng; - } - } - maxLeng = Math.min(64, maxLeng); - - // set up pretty printing of weights - NumberFormat nf = NumberFormat.getNumberInstance(); - nf.setMinimumFractionDigits(4); - nf.setMaximumFractionDigits(4); - if (nf instanceof DecimalFormat) { - ((DecimalFormat) nf).setPositivePrefix(" "); - } - - //print high weight features to a String - StringBuilder sb = new StringBuilder(); - for (Triple t : topFeatures) { - String key = "(" + t.first + "," + t.second + ")"; - sb.append(StringUtils.pad(key, maxLeng)); - sb.append(" "); - double cnt = t.third(); - if (Double.isInfinite(cnt)) { - sb.append(cnt); - } else { - sb.append(nf.format(cnt)); - } - sb.append("\n"); - } - return sb.toString(); - } - - /** Return a String that prints features with large weights. - * - * @param useMagnitude Whether the notion of "large" should ignore - * the sign of the feature weight. - * @param numFeatures How many top features to print - * @param printDescending Print weights in descending order - * @return The String representation of features with large weights - */ - public String toBiggestWeightFeaturesString(boolean useMagnitude, - int numFeatures, - boolean printDescending) { - // this used to try to use a treeset, but that was WRONG.... - edu.stanford.nlp.util.PriorityQueue> biggestKeys = - new FixedPrioritiesPriorityQueue>(); - - // locate biggest keys - for (int feat = 0; feat < weights.length; feat++) { - for (int lab = 0; lab < weights[feat].length; lab++) { - double thisWeight; - // reverse the weight, so get smallest first - if (useMagnitude) { - thisWeight = -Math.abs(weights[feat][lab]); - } else { - thisWeight = -weights[feat][lab]; - } - if (biggestKeys.size() == numFeatures) { - // have enough features, add only if bigger - double lowest = biggestKeys.getPriority(); - if (thisWeight < lowest) { - // remove smallest - biggestKeys.removeFirst(); - biggestKeys.add(new Pair(feat, lab), thisWeight); - } - } else { - // always add it if don't have enough features yet - biggestKeys.add(new Pair(feat, lab), thisWeight); - } - } - } - - // Put in List either reversed or not - // (Note: can't repeatedly iterate over PriorityQueue.) - int actualSize = biggestKeys.size(); - Pair[] bigArray = ErasureUtils.>mkTArray(Pair.class,actualSize); - // System.err.println("biggestKeys is " + biggestKeys); - if (printDescending) { - for (int j = actualSize - 1; j >= 0; j--) { - bigArray[j] = biggestKeys.removeFirst(); - } - } else { - for (int j = 0; j < actualSize; j--) { - bigArray[j] = biggestKeys.removeFirst(); - } - } - List> bigColl = Arrays.asList(bigArray); - // System.err.println("bigColl is " + bigColl); - - // find longest key length (for pretty printing) with a limit - int maxLeng = 0; - for (Pair p : bigColl) { - String key = "(" + featureIndex.get(p.first) + "," + labelIndex.get(p.second) + ")"; - int leng = key.length(); - if (leng > maxLeng) { - maxLeng = leng; - } - } - maxLeng = Math.min(64, maxLeng); - - // set up pretty printing of weights - NumberFormat nf = NumberFormat.getNumberInstance(); - nf.setMinimumFractionDigits(4); - nf.setMaximumFractionDigits(4); - if (nf instanceof DecimalFormat) { - ((DecimalFormat) nf).setPositivePrefix(" "); - } - - //print high weight features to a String - StringBuilder sb = new StringBuilder("LinearClassifier [printing top " + numFeatures + " features]\n"); - for (Pair p : bigColl) { - String key = "(" + featureIndex.get(p.first) + "," + labelIndex.get(p.second) + ")"; - sb.append(StringUtils.pad(key, maxLeng)); - sb.append(" "); - double cnt = weights[p.first][p.second]; - if (Double.isInfinite(cnt)) { - sb.append(cnt); - } else { - sb.append(nf.format(cnt)); - } - sb.append("\n"); - } - return sb.toString(); - } - - /** - * Similar to histogram but exact values of the weights - * to see whether there are many equal weights. - * - * @return A human readable string about the classifier distribution. - */ - public String toDistributionString(int treshold) { - Counter weightCounts = new ClassicCounter(); - StringBuilder s = new StringBuilder(); - s.append("Total number of weights: ").append(totalSize()); - for (int f = 0; f < weights.length; f++) { - for (int l = 0; l < weights[f].length; l++) { - weightCounts.incrementCount(weights[f][l]); - } - } - - s.append("Counts of weights\n"); - Set keys = Counters.keysAbove(weightCounts, treshold); - s.append(keys.size()).append(" keys occur more than ").append(treshold).append(" times "); - return s.toString(); - } - - public int totalSize() { - return labelIndex.size() * featureIndex.size(); - } - - public String toHistogramString() { - // big classifiers - double[][] hist = new double[3][202]; - Object[][] histEg = new Object[3][202]; - int num = 0; - int pos = 0; - int neg = 0; - int zero = 0; - double total = 0.0; - double x2total = 0.0; - double max = 0.0, min = 0.0; - for (int f = 0; f < weights.length; f++) { - for (int l = 0; l < weights[f].length; l++) { - Pair feat = new Pair(featureIndex.get(f), labelIndex.get(l)); - num++; - double wt = weights[f][l]; - total += wt; - x2total += wt * wt; - if (wt > max) { - max = wt; - } - if (wt < min) { - min = wt; - } - if (wt < 0.0) { - neg++; - } else if (wt > 0.0) { - pos++; - } else { - zero++; - } - int index; - index = bucketizeValue(wt); - hist[0][index]++; - if (histEg[0][index] == null) { - histEg[0][index] = feat; - } - if (wt < 0.1 && wt >= -0.1) { - index = bucketizeValue(wt * 100.0); - hist[1][index]++; - if (histEg[1][index] == null) { - histEg[1][index] = feat; - } - if (wt < 0.001 && wt >= -0.001) { - index = bucketizeValue(wt * 10000.0); - hist[2][index]++; - if (histEg[2][index] == null) { - histEg[2][index] = feat; - } - } - } - } - } - double ave = total / num; - double stddev = (x2total / num) - ave * ave; - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - pw.println("Linear classifier with " + num + " f(x,y) features"); - pw.println("Average weight: " + ave + "; std dev: " + stddev); - pw.println("Max weight: " + max + " min weight: " + min); - pw.println("Weights: " + neg + " negative; " + pos + " positive; " + zero + " zero."); - - printHistCounts(0, "Counts of lambda parameters between [-10, 10)", pw, hist, histEg); - printHistCounts(1, "Closeup view of [-0.1, 0.1) depicted * 10^2", pw, hist, histEg); - printHistCounts(2, "Closeup view of [-0.001, 0.001) depicted * 10^4", pw, hist, histEg); - pw.close(); - return sw.toString(); - } - - /** Print out a partial representation of a linear classifier. - * This just calls toString("WeightHistogram", 0) - */ - @Override - public String toString() { - return toString("WeightHistogram", 0); - } - - - /** - * Print out a partial representation of a linear classifier in one of - * several ways. - * - * @param style Options are: - * HighWeight: print out the param parameters with largest weights; - * HighMagnitude: print out the param parameters for which the absolute - * value of their weight is largest; - * AllWeights: print out the weights of all features; - * WeightHistogram: print out a particular hard-coded textual histogram - * representation of a classifier; - * WeightDistribution; - * - * @param param Determines the number of things printed in certain styles - * @throws IllegalArgumentException if the style name is unrecognized - */ - public String toString(String style, int param) { - if (style == null || "".equals(style)) { - return "LinearClassifier with " + featureIndex.size() + " features, " + - labelIndex.size() + " classes, and " + - labelIndex.size() * featureIndex.size() + " parameters.\n"; - } else if (style.equalsIgnoreCase("HighWeight")) { - return toBiggestWeightFeaturesString(false, param, true); - } else if (style.equalsIgnoreCase("HighMagnitude")) { - return toBiggestWeightFeaturesString(true, param, true); - } else if (style.equalsIgnoreCase("AllWeights")) { - return toAllWeightsString(); - } else if (style.equalsIgnoreCase("WeightHistogram")) { - return toHistogramString(); - } else if (style.equalsIgnoreCase("WeightDistribution")) { - return toDistributionString(param); - } else { - throw new IllegalArgumentException("Unknown style: " + style); - } - } - - - /** - * Convert parameter value into number between 0 and 201 - */ - private static int bucketizeValue(double wt) { - int index; - if (wt >= 0.0) { - index = ((int) (wt * 10.0)) + 100; - } else { - index = ((int) (Math.floor(wt * 10.0))) + 100; - } - if (index < 0) { - index = 201; - } else if (index > 200) { - index = 200; - } - return index; - } - - /** - * Print histogram counts from hist and examples over a certain range - */ - private static void printHistCounts(int ind, String title, PrintWriter pw, double[][] hist, Object[][] histEg) { - pw.println(title); - for (int i = 0; i < 200; i++) { - int intpart, fracpart; - if (i < 100) { - intpart = 10 - ((i + 9) / 10); - fracpart = (10 - (i % 10)) % 10; - } else { - intpart = (i / 10) - 10; - fracpart = i % 10; - } - pw.print("[" + ((i < 100) ? "-" : "") + intpart + "." + fracpart + ", " + ((i < 100) ? "-" : "") + intpart + "." + fracpart + "+0.1): " + hist[ind][i]); - if (histEg[ind][i] != null) { - pw.print(" [" + histEg[ind][i] + ((hist[ind][i] > 1) ? ", ..." : "") + "]"); - } - pw.println(); - } - } - - - //TODO: Sort of assumes that Labels are Strings... - public String toAllWeightsString() { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - pw.println("Linear classifier with the following weights"); - Datum allFeatures = new BasicDatum(features(), (L)null); - justificationOf(allFeatures, pw); - return sw.toString(); - } - - - /** - * Print all features in the classifier and the weight that they assign - * to each class. - */ - public void dump() { - Datum allFeatures = new BasicDatum(features(), (L)null); - justificationOf(allFeatures); - } - - public void dump(PrintWriter pw) { - Datum allFeatures = new BasicDatum(features(), (L)null); - justificationOf(allFeatures, pw); - } - - - - @Deprecated - public void justificationOf(RVFDatum example) { - PrintWriter pw = new PrintWriter(System.err, true); - justificationOf(example, pw); - } - - /** - * Print all features active for a particular datum and the weight that - * the classifier assigns to each class for those features. - */ - private void justificationOfRVFDatum(RVFDatum example, PrintWriter pw) { - int featureLength = 0; - int labelLength = 6; - NumberFormat nf = NumberFormat.getNumberInstance(); - nf.setMinimumFractionDigits(2); - nf.setMaximumFractionDigits(2); - if (nf instanceof DecimalFormat) { - ((DecimalFormat) nf).setPositivePrefix(" "); - } - Counter features = example.asFeaturesCounter(); - for (F f : features.keySet()) { - featureLength = Math.max(featureLength, f.toString().length() + 2 + - nf.format(features.getCount(f)).length()); - } - // make as wide as total printout - featureLength = Math.max(featureLength, "Total:".length()); - // don't make it ridiculously wide - featureLength = Math.min(featureLength, MAX_FEATURE_ALIGN_WIDTH); - - for (Object l : labels()) { - labelLength = Math.max(labelLength, l.toString().length()); - } - - StringBuilder header = new StringBuilder(""); - for (int s = 0; s < featureLength; s++) { - header.append(' '); - } - for (L l : labels()) { - header.append(' '); - header.append(StringUtils.pad(l, labelLength)); - } - pw.println(header); - for (F f : features.keySet()) { - String fStr = f.toString(); - StringBuilder line = new StringBuilder(fStr); - line.append("[").append(nf.format(features.getCount(f))).append("]"); - fStr = line.toString(); - for (int s = fStr.length(); s < featureLength; s++) { - line.append(' '); - } - for (L l : labels()) { - String lStr = nf.format(weight(f, l)); - line.append(' '); - line.append(lStr); - for (int s = lStr.length(); s < labelLength; s++) { - line.append(' '); - } - } - pw.println(line); - } - Counter scores = scoresOfRVFDatum(example); - StringBuilder footer = new StringBuilder("Total:"); - for (int s = footer.length(); s < featureLength; s++) { - footer.append(' '); - } - for (L l : labels()) { - footer.append(' '); - String str = nf.format(scores.getCount(l)); - footer.append(str); - for (int s = str.length(); s < labelLength; s++) { - footer.append(' '); - } - } - pw.println(footer); - Distribution distr = Distribution.distributionFromLogisticCounter(scores); - footer = new StringBuilder("Prob:"); - for (int s = footer.length(); s < featureLength; s++) { - footer.append(' '); - } - for (L l : labels()) { - footer.append(' '); - String str = nf.format(distr.getCount(l)); - footer.append(str); - for (int s = str.length(); s < labelLength; s++) { - footer.append(' '); - } - } - pw.println(footer); - } - - - /** - * Print all features active for a particular datum and the weight that - * the classifier assigns to each class for those features. - */ - @Deprecated - public void justificationOf(RVFDatum example, PrintWriter pw) { - int featureLength = 0; - int labelLength = 6; - NumberFormat nf = NumberFormat.getNumberInstance(); - nf.setMinimumFractionDigits(2); - nf.setMaximumFractionDigits(2); - if (nf instanceof DecimalFormat) { - ((DecimalFormat) nf).setPositivePrefix(" "); - } - Counter features = example.asFeaturesCounter(); - for (F f : features.keySet()) { - featureLength = Math.max(featureLength, f.toString().length() + 2 + - nf.format(features.getCount(f)).length()); - } - // make as wide as total printout - featureLength = Math.max(featureLength, "Total:".length()); - // don't make it ridiculously wide - featureLength = Math.min(featureLength, MAX_FEATURE_ALIGN_WIDTH); - - for (Object l : labels()) { - labelLength = Math.max(labelLength, l.toString().length()); - } - - StringBuilder header = new StringBuilder(""); - for (int s = 0; s < featureLength; s++) { - header.append(' '); - } - for (L l : labels()) { - header.append(' '); - header.append(StringUtils.pad(l, labelLength)); - } - pw.println(header); - for (F f : features.keySet()) { - String fStr = f.toString(); - StringBuilder line = new StringBuilder(fStr); - line.append("[").append(nf.format(features.getCount(f))).append("]"); - fStr = line.toString(); - for (int s = fStr.length(); s < featureLength; s++) { - line.append(' '); - } - for (L l : labels()) { - String lStr = nf.format(weight(f, l)); - line.append(' '); - line.append(lStr); - for (int s = lStr.length(); s < labelLength; s++) { - line.append(' '); - } - } - pw.println(line); - } - Counter scores = scoresOf(example); - StringBuilder footer = new StringBuilder("Total:"); - for (int s = footer.length(); s < featureLength; s++) { - footer.append(' '); - } - for (L l : labels()) { - footer.append(' '); - String str = nf.format(scores.getCount(l)); - footer.append(str); - for (int s = str.length(); s < labelLength; s++) { - footer.append(' '); - } - } - pw.println(footer); - Distribution distr = Distribution.distributionFromLogisticCounter(scores); - footer = new StringBuilder("Prob:"); - for (int s = footer.length(); s < featureLength; s++) { - footer.append(' '); - } - for (L l : labels()) { - footer.append(' '); - String str = nf.format(distr.getCount(l)); - footer.append(str); - for (int s = str.length(); s < labelLength; s++) { - footer.append(' '); - } - } - pw.println(footer); - } - - - public void justificationOf(Datum example) { - PrintWriter pw = new PrintWriter(System.err, true); - justificationOf(example, pw); - } - - public void justificationOf(Datum example, PrintWriter pw, Function printer) { - justificationOf(example, pw, printer, false); - } - - /** Print all features active for a particular datum and the weight that - * the classifier assigns to each class for those features. - * - * @param example The datum for which features are to be printed - * @param pw Where to print it to - * @param printer If this is non-null, then it is applied to each - * feature to convert it to a more readable form - * @param sortedByFeature Whether to sort by feature names - */ - public void justificationOf(Datum example, PrintWriter pw, - Function printer, boolean sortedByFeature) { - - if(example instanceof RVFDatum) { - justificationOfRVFDatum((RVFDatum)example,pw); - return; - } - NumberFormat nf = NumberFormat.getNumberInstance(); - nf.setMinimumFractionDigits(2); - nf.setMaximumFractionDigits(2); - if (nf instanceof DecimalFormat) { - ((DecimalFormat) nf).setPositivePrefix(" "); - } - - // determine width for features, making it at least total's width - int featureLength = 0; - //TODO: not really sure what this Printer is supposed to spit out... - for (F f : example.asFeatures()) { - int length = f.toString().length(); - if (printer != null) { - length = printer.apply(f).toString().length(); - } - featureLength = Math.max(featureLength, length); - } - // make as wide as total printout - featureLength = Math.max(featureLength, "Total:".length()); - // don't make it ridiculously wide - featureLength = Math.min(featureLength, MAX_FEATURE_ALIGN_WIDTH); - - // determine width for labels - int labelLength = 6; - for (L l : labels()) { - labelLength = Math.max(labelLength, l.toString().length()); - } - - // print header row of output listing classes - StringBuilder header = new StringBuilder(""); - for (int s = 0; s < featureLength; s++) { - header.append(' '); - } - for (L l : labels()) { - header.append(' '); - header.append(StringUtils.pad(l, labelLength)); - } - pw.println(header); - - // print active features and weights per class - Collection featColl = example.asFeatures(); - if (sortedByFeature){ - featColl = ErasureUtils.sortedIfPossible(featColl); - } - for (F f : featColl) { - String fStr; - if (printer != null) { - fStr = printer.apply(f).toString(); - } else { - fStr = f.toString(); - } - StringBuilder line = new StringBuilder(fStr); - for (int s = fStr.length(); s < featureLength; s++) { - line.append(' '); - } - for (L l : labels()) { - String lStr = nf.format(weight(f, l)); - line.append(' '); - line.append(lStr); - for (int s = lStr.length(); s < labelLength; s++) { - line.append(' '); - } - } - pw.println(line); - } - - // Print totals, probs, etc. - Counter scores = scoresOf(example); - StringBuilder footer = new StringBuilder("Total:"); - for (int s = footer.length(); s < featureLength; s++) { - footer.append(' '); - } - for (L l : labels()) { - footer.append(' '); - String str = nf.format(scores.getCount(l)); - footer.append(str); - for (int s = str.length(); s < labelLength; s++) { - footer.append(' '); - } - } - pw.println(footer); - Distribution distr = Distribution.distributionFromLogisticCounter(scores); - footer = new StringBuilder("Prob:"); - for (int s = footer.length(); s < featureLength; s++) { - footer.append(' '); - } - for (L l : labels()) { - footer.append(' '); - String str = nf.format(distr.getCount(l)); - footer.append(str); - for (int s = str.length(); s < labelLength; s++) { - footer.append(' '); - } - } - pw.println(footer); - } - -/** - * This method returns a map from each label to a counter of feature weights for that label. - * Useful for feature analysis. - * @return a map of counters - */ - - public Map> weightsAsMapOfCounters() { - Map> mapOfCounters = new HashMap>(); - for(L label : labelIndex){ - int labelID = labelIndex.indexOf(label); - Counter c = new ClassicCounter(); - mapOfCounters.put(label, c); - for (F f : featureIndex) { - c.incrementCount(f, weights[featureIndex.indexOf(f)][labelID]); - } - } - return mapOfCounters; - } - - /** - * Print all features active for a particular datum and the weight that - * the classifier assigns to each class for those features. - */ - public void justificationOf(Datum example, PrintWriter pw) { - justificationOf(example, pw, null); - } - - - /** - * Print all features in the classifier and the weight that they assign - * to each class. The feature names are printed in sorted order. - */ - public void dumpSorted() { - Datum allFeatures = new BasicDatum(features(), (L)null); - justificationOf(allFeatures, new PrintWriter(System.err, true), true); - } - - /** - * Print all features active for a particular datum and the weight that - * the classifier assigns to each class for those features. Sorts by feature - * name if 'sorted' is true. - */ - public void justificationOf(Datum example, PrintWriter pw, boolean sorted) { - if(example instanceof RVFDatum) - justificationOf(example, pw, null, sorted); - } - - - public Counter scoresOf(Datum example, Collection possibleLabels) { - Counter scores = new ClassicCounter(); - for (L l : possibleLabels) { - if (labelIndex.indexOf(l) == -1) { - continue; - } - double score = scoreOf(example, l); - scores.setCount(l, score); - } - return scores; - } - - - public L experimentalClassOf(Datum example) { - if(example instanceof RVFDatum) { - throw new UnsupportedOperationException(); - } - - int labelCount = weights[0].length; - //System.out.printf("labelCount: %d\n", labelCount); - Collection features = example.asFeatures(); - - int[] featureInts = new int[features.size()]; - int fI = 0; - for (F feature : features) { - featureInts[fI++] = featureIndex.indexOf(feature); - } - //System.out.println("Features: "+features); - double bestScore = Double.NEGATIVE_INFINITY; - int bestI = 0; - for (int i = 0; i < labelCount; i++) { - double score = 0; - for (int j = 0; j < featureInts.length; j++) { - if (featureInts[j] < 0) continue; - score += weights[featureInts[j]][i]; - } - if (score > bestScore) { - bestI = i; - bestScore = score; - } - //System.out.printf("Score: %s(%d): %e\n", labelIndex.get(i), i, score); - } - //System.out.printf("label(%d): %s\n", bestI, labelIndex.get(bestI));; - return labelIndex.get(bestI); - } - - public L classOf(Datum example) { - if(example instanceof RVFDatum)return classOfRVFDatum((RVFDatum)example); - Counter scores = scoresOf(example); - return Counters.argmax(scores); - } - - - private L classOfRVFDatum(RVFDatum example) { - Counter scores = scoresOfRVFDatum(example); - return Counters.argmax(scores); - } - - @Deprecated - public L classOf(RVFDatum example) { - Counter scores = scoresOf(example); - return Counters.argmax(scores); - } - - public LinearClassifier(double[][] weights, Index featureIndex, Index labelIndex) { - this.featureIndex = featureIndex; - this.labelIndex = labelIndex; - this.weights = weights; - thresholds = new double[labelIndex.size()]; - Arrays.fill(thresholds, 0.0); - } - - public LinearClassifier(double[][] weights, Index featureIndex, Index labelIndex, - double[] thresholds) throws Exception { - this.featureIndex = featureIndex; - this.labelIndex = labelIndex; - this.weights = weights; - if (thresholds.length != labelIndex.size()) - throw new Exception("Number of thresholds and number of labels do not match."); - thresholds = new double[thresholds.length]; - int curr = 0; - for (double tval : thresholds) { - thresholds[curr++] = tval; - } - Arrays.fill(thresholds, 0.0); - } - - public LinearClassifier(double[] weights, Index> weightIndex) { - Counter> weightCounter = new ClassicCounter>(); - for (int i = 0; i < weightIndex.size(); i++) { - if (weights[i] == 0) { - continue; // no need to save 0 weights - } - weightCounter.setCount(weightIndex.get(i), weights[i]); - } - init(weightCounter, new ClassicCounter()); - } - - public LinearClassifier(Counter> weightCounter) { - this(weightCounter, new ClassicCounter()); - } - - public LinearClassifier(Counter> weightCounter, Counter thresholdsC) { - init(weightCounter,thresholdsC); - } - - private void init(Counter> weightCounter, Counter thresholdsC) { - Collection> keys = weightCounter.keySet(); - featureIndex = new HashIndex(); - labelIndex = new HashIndex(); - for (Pair p : keys) { - featureIndex.add(p.first()); - labelIndex.add(p.second()); - } - thresholds = new double[labelIndex.size()]; - for (L label : labelIndex) { - thresholds[labelIndex.indexOf(label)] = thresholdsC.getCount(label); - } - weights = new double[featureIndex.size()][labelIndex.size()]; - Pair tempPair = new Pair(); - for (int f = 0; f < weights.length; f++) { - for (int l = 0; l < weights[f].length; l++) { - tempPair.first = featureIndex.get(f); - tempPair.second = labelIndex.get(l); - weights[f][l] = weightCounter.getCount(tempPair); - } - } - } - - - public void adaptWeights(Dataset adapt,LinearClassifierFactory lcf) { - System.err.println("before adapting, weights size="+weights.length); - weights = lcf.adaptWeights(weights,adapt); - System.err.println("after adapting, weights size="+weights.length); - } - - public double[][] weights() { - return weights; - } - - public void setWeights(double[][] newWeights) { - weights = newWeights; - } - - /** - * Loads a classifier from a file. - * Simple convenience wrapper for IOUtils.readFromString. - */ - public static LinearClassifier readClassifier(String loadPath) { - System.err.print("Deserializing classifier from " + loadPath + "..."); - - try { - ObjectInputStream ois = IOUtils.readStreamFromString(loadPath); - LinearClassifier classifier = ErasureUtils.>uncheckedCast(ois.readObject()); - ois.close(); - return classifier; - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException("Deserialization failed: "+e.getMessage()); - } - } - - /** - * Convenience wrapper for IOUtils.writeObjectToFile - */ - public static void writeClassifier(LinearClassifier classifier, String writePath) { - try { - IOUtils.writeObjectToFile(classifier, writePath); - } catch (Exception e) { - throw new RuntimeException("Serialization failed: "+e.getMessage(), e); - } - } - - /** - * Saves this out to a standard text file, instead of as a serialized Java object. - * NOTE: this currently assumes feature and weights are represented as Strings. - * @param file String filepath to write out to. - */ - public void saveToFilename(String file) { - try { - File tgtFile = new File(file); - BufferedWriter out = new BufferedWriter(new FileWriter(tgtFile)); - // output index first, blank delimiter, outline feature index, then weights - labelIndex.saveToWriter(out); - featureIndex.saveToWriter(out); - int numLabels = labelIndex.size(); - int numFeatures = featureIndex.size(); - for (int featIndex=0; featIndex objective = new AdaptedGaussianPriorObjectiveFunction(adaptDataset, logPrior,newWeights); - - double[] initial = objective.initial(); - - double[] weights = minimizer.minimize(objective, TOL, initial); - return objective.to2D(weights); - - //Question: maybe the adaptWeights can be done just in LinearClassifier ?? (pichuan) - } - - @Override - public double[][] trainWeights(GeneralDataset dataset) { - return trainWeights(dataset, null); - } - - public double[][] trainWeights(GeneralDataset dataset, double[] initial) { - return trainWeights(dataset, initial, false); - } - - public double[][] trainWeights(GeneralDataset dataset, double[] initial, boolean bypassTuneSigma) { - if(dataset instanceof RVFDataset) - ((RVFDataset)dataset).ensureRealValues(); - double[] interimWeights = null; - if(! bypassTuneSigma) { - if (tuneSigmaHeldOut) { - interimWeights = heldOutSetSigma(dataset); // the optimum interim weights from held-out training data have already been found. - } else if (tuneSigmaCV) { - crossValidateSetSigma(dataset,folds); // TODO: assign optimum interim weights as part of this process. - } - } - LogConditionalObjectiveFunction objective = new LogConditionalObjectiveFunction(dataset, logPrior); - if(initial == null && interimWeights != null && ! retrainFromScratchAfterSigmaTuning) { - //System.err.println("## taking advantage of interim weights as starting point."); - initial = interimWeights; - } - if (initial == null) { - initial = objective.initial(); - } - - double[] weights = minimizer.minimize(objective, TOL, initial); - return objective.to2D(weights); - } - - /** - * IMPORTANT: dataset and biasedDataset must have same featureIndex, labelIndex - */ - public Classifier trainClassifierSemiSup(GeneralDataset data, GeneralDataset biasedData, double[][] confusionMatrix, double[] initial) { - double[][] weights = trainWeightsSemiSup(data, biasedData, confusionMatrix, initial); - LinearClassifier classifier = new LinearClassifier(weights, data.featureIndex(), data.labelIndex()); - return classifier; - } - - public double[][] trainWeightsSemiSup(GeneralDataset data, GeneralDataset biasedData, double[][] confusionMatrix, double[] initial) { - LogConditionalObjectiveFunction objective = new LogConditionalObjectiveFunction(data, new LogPrior(LogPrior.LogPriorType.NULL)); - BiasedLogConditionalObjectiveFunction biasedObjective = new BiasedLogConditionalObjectiveFunction(biasedData, confusionMatrix, new LogPrior(LogPrior.LogPriorType.NULL)); - SemiSupervisedLogConditionalObjectiveFunction semiSupObjective = new SemiSupervisedLogConditionalObjectiveFunction(objective, biasedObjective, logPrior); - if (initial == null) { - initial = objective.initial(); - } - double[] weights = minimizer.minimize(semiSupObjective, TOL, initial); - return objective.to2D(weights); - } - - /** - * Trains the linear classifier using Generalized Expectation criteria as described in - * Generalized Expectation Criteria for Semi Supervised Learning of Conditional Random Fields, Mann and McCallum, ACL 2008. - * The original algorithm is proposed for CRFs but has been adopted to LinearClassifier (which is a simpler special case of a CRF). - * IMPORTANT: the labeled features that are passed as an argument are assumed to be binary valued, although - * other features are allowed to be real valued. - */ - public LinearClassifier trainSemiSupGE(GeneralDataset labeledDataset, List> unlabeledDataList, List GEFeatures, double convexComboCoeff) { - LogConditionalObjectiveFunction objective = new LogConditionalObjectiveFunction(labeledDataset, new LogPrior(LogPrior.LogPriorType.NULL)); - GeneralizedExpectationObjectiveFunction geObjective = new GeneralizedExpectationObjectiveFunction(labeledDataset, unlabeledDataList, GEFeatures); - SemiSupervisedLogConditionalObjectiveFunction semiSupObjective = new SemiSupervisedLogConditionalObjectiveFunction(objective, geObjective, null,convexComboCoeff); - double[] initial = objective.initial(); - double[] weights = minimizer.minimize(semiSupObjective, TOL, initial); - return new LinearClassifier(objective.to2D(weights), labeledDataset.featureIndex(), labeledDataset.labelIndex()); - } - - - /** - * Trains the linear classifier using Generalized Expectation criteria as described in - * Generalized Expectation Criteria for Semi Supervised Learning of Conditional Random Fields, Mann and McCallum, ACL 2008. - * The original algorithm is proposed for CRFs but has been adopted to LinearClassifier (which is a simpler, special case of a CRF). - * Automatically discovers high precision, high frequency labeled features to be used as GE constraints. - * IMPORTANT: the current feature selector assumes the features are binary. The GE constraints assume the constraining features are binary anyway, although - * it doesn't make such assumptions about other features. - */ - public LinearClassifier trainSemiSupGE(GeneralDataset labeledDataset, List> unlabeledDataList) { - List GEFeatures = getHighPrecisionFeatures(labeledDataset,0.9,10); - return trainSemiSupGE(labeledDataset, unlabeledDataList, GEFeatures,0.5); - } - - public LinearClassifier trainSemiSupGE(GeneralDataset labeledDataset, List> unlabeledDataList, double convexComboCoeff) { - List GEFeatures = getHighPrecisionFeatures(labeledDataset,0.9,10); - return trainSemiSupGE(labeledDataset, unlabeledDataList, GEFeatures,convexComboCoeff); - } - - - /** - * Returns a list of featured thresholded by minPrecision and sorted by their frequency of occurrence. - * precision in this case, is defined as the frequency of majority label over total frequency for that feature. - * @return list of high precision features. - */ - private List getHighPrecisionFeatures(GeneralDataset dataset, double minPrecision, int maxNumFeatures){ - int[][] feature2label = new int[dataset.numFeatures()][dataset.numClasses()]; - for(int f = 0; f < dataset.numFeatures(); f++) - Arrays.fill(feature2label[f],0); - - int[][] data = dataset.data; - int[] labels = dataset.labels; - for(int d = 0; d < data.length; d++){ - int label = labels[d]; - //System.out.println("datum id:"+d+" label id: "+label); - if(data[d] != null){ - //System.out.println(" number of features:"+data[d].length); - for(int n = 0; n < data[d].length; n++){ - feature2label[data[d][n]][label]++; - } - } - } - Counter feature2freq = new ClassicCounter(); - for(int f = 0; f < dataset.numFeatures(); f++){ - int maxF = ArrayMath.max(feature2label[f]); - int total = ArrayMath.sum(feature2label[f]); - double precision = ((double)maxF)/total; - F feature = dataset.featureIndex.get(f); - if(precision >= minPrecision){ - feature2freq.incrementCount(feature, total); - } - } - if(feature2freq.size() > maxNumFeatures){ - Counters.retainTop(feature2freq, maxNumFeatures); - } - //for(F feature : feature2freq.keySet()) - //System.out.println(feature+" "+feature2freq.getCount(feature)); - //System.exit(0); - return Counters.toSortedList(feature2freq); - } - - /** - * Train a classifier with a sigma tuned on a validation set. - * - * @return The constructed classifier - */ - public LinearClassifier trainClassifierV(GeneralDataset train, GeneralDataset validation, double min, double max, boolean accuracy) { - labelIndex = train.labelIndex(); - featureIndex = train.featureIndex(); - this.min = min; - this.max = max; - heldOutSetSigma(train, validation); - double[][] weights = trainWeights(train); - return new LinearClassifier(weights, train.featureIndex(), train.labelIndex()); - } - - /** - * Train a classifier with a sigma tuned on a validation set. - * In this case we are fitting on the last 30% of the training data. - * - * @param train The data to train (and validate) on. - * @return The constructed classifier - */ - public LinearClassifier trainClassifierV(GeneralDataset train, double min, double max, boolean accuracy) { - labelIndex = train.labelIndex(); - featureIndex = train.featureIndex(); - tuneSigmaHeldOut = true; - this.min = min; - this.max = max; - heldOutSetSigma(train); - double[][] weights = trainWeights(train); - return new LinearClassifier(weights, train.featureIndex(), train.labelIndex()); - } - - - public LinearClassifierFactory() { - this(new QNMinimizer(15)); - } - - public LinearClassifierFactory(Minimizer min) { - this(min, false); - } - - public LinearClassifierFactory(boolean useSum) { - this(new QNMinimizer(15), useSum); - } - - public LinearClassifierFactory(double tol) { - this(new QNMinimizer(15), tol, false); - } - public LinearClassifierFactory(Minimizer min, boolean useSum) { - this(min, 1e-4, useSum); - } - public LinearClassifierFactory(Minimizer min, double tol, boolean useSum) { - this(min, tol, useSum, 1.0); - } - public LinearClassifierFactory(double tol, boolean useSum, double sigma) { - this(new QNMinimizer(15), tol, useSum, sigma); - } - public LinearClassifierFactory(Minimizer min, double tol, boolean useSum, double sigma) { - this(min, tol, useSum, LogPrior.LogPriorType.QUADRATIC.ordinal(), sigma); - } - public LinearClassifierFactory(Minimizer min, double tol, boolean useSum, int prior, double sigma) { - this(min, tol, useSum, prior, sigma, 0.0); - } - public LinearClassifierFactory(double tol, boolean useSum, int prior, double sigma, double epsilon) { - this(new QNMinimizer(15), tol, useSum, new LogPrior(prior, sigma, epsilon)); - } - - public LinearClassifierFactory(double tol, boolean useSum, int prior, double sigma, double epsilon, int mem) { - this(new QNMinimizer(mem), tol, useSum, new LogPrior(prior, sigma, epsilon)); - }; - - /** - * Create a factory that builds linear classifiers from training data. - * - * @param min The method to be used for optimization (minimization) (default: {@link QNMinimizer}) - * @param tol The convergence threshold for the minimization (default: 1e-4) - * @param useSum Asks to the optimizer to minimize the sum of the - * likelihoods of individual data items rather than their product (default: false) - * NOTE: this is currently ignored!!! - * @param prior What kind of prior to use, as an enum constant from class - * LogPrior - * @param sigma The strength of the prior (smaller is stronger for most - * standard priors) (default: 1.0) - * @param epsilon A second parameter to the prior (currently only used - * by the Huber prior) - */ - public LinearClassifierFactory(Minimizer min, double tol, boolean useSum, int prior, double sigma, double epsilon) { - this(min, tol, useSum, new LogPrior(prior, sigma, epsilon)); - } - - public LinearClassifierFactory(Minimizer min, double tol, boolean useSum, LogPrior logPrior) { - this.minimizer = min; - this.TOL = tol; - //this.useSum = useSum; - this.logPrior = logPrior; - } - - /** - * Set the tolerance. 1e-4 is the default. - */ - public void setTol(double tol) { - this.TOL = tol; - } - - /** - * Set the prior. - * - * @param logPrior One of the priors defined in - * LogConditionalObjectiveFunction. - * LogPrior.QUADRATIC is the default. - */ - public void setPrior(LogPrior logPrior) { - this.logPrior = logPrior; - } - - /** - * Set the verbose flag for {@link CGMinimizer}. - * Only used with conjugate-gradient minimization. - * false is the default. - */ - - public void setVerbose(boolean verbose) { - this.verbose = verbose; - } - - /** - * Sets the minimizer. {@link QNMinimizer} is the default. - */ - public void setMinimizer(Minimizer min) { - this.minimizer = min; - } - - /** - * Sets the epsilon value for {@link LogConditionalObjectiveFunction}. - */ - public void setEpsilon(double eps) { - logPrior.setEpsilon(eps); - } - - public void setSigma(double sigma) { - logPrior.setSigma(sigma); - } - - public double getSigma() { - return logPrior.getSigma(); - } - - /** - * Sets the minimizer to QuasiNewton. {@link QNMinimizer} is the default. - */ - public void useQuasiNewton() { - this.minimizer = new QNMinimizer(mem); - } - - public void useQuasiNewton(boolean useRobust) { - this.minimizer = new QNMinimizer(mem,useRobust); - } - - public void useStochasticQN(double initialSMDGain, int stochasticBatchSize){ - this.minimizer = new SQNMinimizer(mem,initialSMDGain,stochasticBatchSize,false); - } - - public void useStochasticMetaDescent(){ - useStochasticMetaDescent(0.1,15,StochasticCalculateMethods.ExternalFiniteDifference,20); - } - - public void useStochasticMetaDescent(double initialSMDGain, int stochasticBatchSize,StochasticCalculateMethods stochasticMethod,int passes) { - this.minimizer = new SMDMinimizer(initialSMDGain, stochasticBatchSize,stochasticMethod,passes); - } - - public void useStochasticGradientDescent(){ - useStochasticGradientDescent(0.1,15); - } - - public void useStochasticGradientDescent(double gainSGD, int stochasticBatchSize){ - this.minimizer = new SGDMinimizer(gainSGD,stochasticBatchSize); - } - - public void useInPlaceStochasticGradientDescent(){ - useInPlaceStochasticGradientDescent(-1, -1, 1.0); - } - - public void useInPlaceStochasticGradientDescent(int SGDPasses, int tuneSampleSize, double sigma) { - this.minimizer = new StochasticInPlaceMinimizer(sigma, SGDPasses, tuneSampleSize); - } - - public void useHybridMinimizerWithInPlaceSGD(int SGDPasses, int tuneSampleSize, double sigma) { - Minimizer firstMinimizer = new StochasticInPlaceMinimizer(sigma, SGDPasses, tuneSampleSize); - Minimizer secondMinimizer = new QNMinimizer(mem); - this.minimizer = new HybridMinimizer(firstMinimizer, secondMinimizer, SGDPasses); - } - - public void useStochasticGradientDescentToQuasiNewton(SeqClassifierFlags p){ - this.minimizer = new SGDToQNMinimizer(p); - } - - public void useHybridMinimizer(){ - useHybridMinimizer(0.1,15,StochasticCalculateMethods.ExternalFiniteDifference , 0); - } - - public void useHybridMinimizer(double initialSMDGain, int stochasticBatchSize,StochasticCalculateMethods stochasticMethod,int cutoffIteration){ - Minimizer firstMinimizer = new SMDMinimizer(initialSMDGain, stochasticBatchSize,stochasticMethod,cutoffIteration); - Minimizer secondMinimizer = new QNMinimizer(mem); - this.minimizer = new HybridMinimizer(firstMinimizer,secondMinimizer,cutoffIteration); - } - - /** - * Set the mem value for {@link QNMinimizer}. - * Only used with quasi-newton minimization. 15 is the default. - * - * @param mem Number of previous function/derivative evaluations to store - * to estimate second derivative. Storing more previous evaluations - * improves training convergence speed. This number can be very - * small, if memory conservation is the priority. For large - * optimization systems (of 100,000-1,000,000 dimensions), setting this - * to 15 produces quite good results, but setting it to 50 can - * decrease the iteration count by about 20% over a value of 15. - */ - public void setMem(int mem) { - this.mem = mem; - } - - /** - * Sets the minimizer to {@link CGMinimizer}, with the passed verbose flag. - */ - public void useConjugateGradientAscent(boolean verbose) { - this.verbose = verbose; - useConjugateGradientAscent(); - } - - /** - * Sets the minimizer to {@link CGMinimizer}. - */ - public void useConjugateGradientAscent() { - this.minimizer = new CGMinimizer(!this.verbose); - } - - /** - * NOTE: nothing is actually done with this value! - * - * SetUseSum sets the useSum flag: when turned on, - * the Summed Conditional Objective Function is used. Otherwise, the - * LogConditionalObjectiveFunction is used. The default is false. - */ - public void setUseSum(boolean useSum) { - //this.useSum = useSum; - } - - /** - * setTuneSigmaHeldOut sets the tuneSigmaHeldOut flag: when turned on, - * the sigma is tuned by means of held-out (70%-30%). Otherwise no tuning on sigma is done. - * The default is false. - */ - public void setTuneSigmaHeldOut() { - tuneSigmaHeldOut = true; - tuneSigmaCV = false; - } - - /** - * setTuneSigmaCV sets the tuneSigmaCV flag: when turned on, - * the sigma is tuned by cross-validation. The number of folds is the parameter. - * If there is less data than the number of folds, leave-one-out is used. - * The default is false. - */ - public void setTuneSigmaCV(int folds) { - tuneSigmaCV = true; - tuneSigmaHeldOut = false; - this.folds = folds; - } - - /** - * NOTE: Nothing is actually done with this value. - * - * resetWeight sets the restWeight flag. This flag makes sense only if sigma is tuned: - * when turned on, the weights outputed by the tuneSigma method will be reset to zero when training the - * classifier. - * The default is false. - */ - public void resetWeight() { - //resetWeight = true; - } - - static protected double[] sigmasToTry = {0.5,1.0,2.0,4.0,10.0, 20.0, 100.0}; - - /** - * Calls the method {@link #crossValidateSetSigma(GeneralDataset, int)} with 5-fold cross-validation. - * @param dataset the data set to optimize sigma on. - */ - public void crossValidateSetSigma(GeneralDataset dataset) { - crossValidateSetSigma(dataset, 5); - } - - /** - * callls the method {@link #crossValidateSetSigma(GeneralDataset, int, Scorer, LineSearcher)} with - * multi-class log-likelihood scoring (see {@link MultiClassAccuracyStats}) and golden-section line search - * (see {@link GoldenSectionLineSearch}). - * @param dataset the data set to optimize sigma on. - */ - public void crossValidateSetSigma(GeneralDataset dataset,int kfold) { - System.err.println("##you are here."); - crossValidateSetSigma(dataset, kfold, new MultiClassAccuracyStats(MultiClassAccuracyStats.USE_LOGLIKELIHOOD), new GoldenSectionLineSearch(true, 1e-2, min, max)); - } - - public void crossValidateSetSigma(GeneralDataset dataset,int kfold, final Scorer scorer) { - crossValidateSetSigma(dataset, kfold, scorer, new GoldenSectionLineSearch(true, 1e-2, min, max)); - } - public void crossValidateSetSigma(GeneralDataset dataset,int kfold, LineSearcher minimizer) { - crossValidateSetSigma(dataset, kfold, new MultiClassAccuracyStats(MultiClassAccuracyStats.USE_LOGLIKELIHOOD), minimizer); - } - /** - * Sets the sigma parameter to a value that optimizes the cross-validation score given by scorer. Search for an optimal value - * is carried out by minimizer - * @param dataset the data set to optimize sigma on. - */ - public void crossValidateSetSigma(GeneralDataset dataset,int kfold, final Scorer scorer, LineSearcher minimizer) { - System.err.println("##in Cross Validate, folds = " + kfold); - System.err.println("##Scorer is " + scorer); - - featureIndex = dataset.featureIndex; - labelIndex = dataset.labelIndex; - - final CrossValidator crossValidator = new CrossValidator(dataset,kfold); - final Function,GeneralDataset,CrossValidator.SavedState>,Double> score = - new Function,GeneralDataset,CrossValidator.SavedState>,Double> () - { - public Double apply (Triple,GeneralDataset,CrossValidator.SavedState> fold) { - GeneralDataset trainSet = fold.first(); - GeneralDataset devSet = fold.second(); - - double[] weights = (double[])fold.third().state; - double[][] weights2D; - - weights2D = trainWeights(trainSet, weights,true); // must of course bypass sigma tuning here. - - fold.third().state = ArrayUtils.flatten(weights2D); - - LinearClassifier classifier = new LinearClassifier(weights2D, trainSet.featureIndex, trainSet.labelIndex); - - double score = scorer.score(classifier, devSet); - //System.out.println("score: "+score); - System.out.print("."); - return score; - } - }; - - Function negativeScorer = - new Function () - { - public Double apply(Double sigmaToTry) { - //sigma = sigmaToTry; - setSigma(sigmaToTry); - Double averageScore = crossValidator.computeAverage(score); - System.err.print("##sigma = "+getSigma()+" "); - System.err.println("-> average Score: "+averageScore); - return -averageScore; - } - }; - - double bestSigma = minimizer.minimize(negativeScorer); - System.err.println("##best sigma: " + bestSigma); - setSigma(bestSigma); - } - - /** - * Set the {@link LineSearcher} to be used in {@link #heldOutSetSigma(GeneralDataset, GeneralDataset)}. - */ - public void setHeldOutSearcher(LineSearcher heldOutSearcher) { - this.heldOutSearcher = heldOutSearcher; - } - - private LineSearcher heldOutSearcher = null; - public double[] heldOutSetSigma(GeneralDataset train) { - Pair, GeneralDataset> data = train.split(0.3); - return heldOutSetSigma(data.first(), data.second()); - } - - public double[] heldOutSetSigma(GeneralDataset train, Scorer scorer) { - Pair, GeneralDataset> data = train.split(0.3); - return heldOutSetSigma(data.first(), data.second(), scorer); - } - - public double[] heldOutSetSigma(GeneralDataset train, GeneralDataset dev) { - return heldOutSetSigma(train, dev, new MultiClassAccuracyStats(MultiClassAccuracyStats.USE_LOGLIKELIHOOD), heldOutSearcher == null ? new GoldenSectionLineSearch(true, 1e-2, min, max) : heldOutSearcher); - } - - public double[] heldOutSetSigma(GeneralDataset train, GeneralDataset dev, final Scorer scorer) { - return heldOutSetSigma(train, dev, scorer, new GoldenSectionLineSearch(true, 1e-2, min, max)); - } - public double[] heldOutSetSigma(GeneralDataset train, GeneralDataset dev, LineSearcher minimizer) { - return heldOutSetSigma(train, dev, new MultiClassAccuracyStats(MultiClassAccuracyStats.USE_LOGLIKELIHOOD), minimizer); - } - - /** - * Sets the sigma parameter to a value that optimizes the held-out score given by scorer. Search for an optimal value - * is carried out by minimizer - * dataset the data set to optimize sigma on. - * kfold - * @return an interim set of optimal weights: the weights - */ - public double[] heldOutSetSigma(final GeneralDataset trainSet, final GeneralDataset devSet, final Scorer scorer, LineSearcher minimizer) { - - featureIndex = trainSet.featureIndex; - labelIndex = trainSet.labelIndex; - //double[] resultWeights = null; - Timing timer = new Timing(); - - NegativeScorer negativeScorer = new NegativeScorer(trainSet,devSet,scorer,timer); - - timer.start(); - double bestSigma = minimizer.minimize(negativeScorer); - System.err.println("##best sigma: " + bestSigma); - setSigma(bestSigma); - - return ArrayUtils.flatten(trainWeights(trainSet,negativeScorer.weights,true)); // make sure it's actually the interim weights from best sigma - } - - class NegativeScorer implements Function { - public double[] weights = null; - GeneralDataset trainSet; - GeneralDataset devSet; - Scorer scorer; - Timing timer; - - public NegativeScorer(GeneralDataset trainSet, GeneralDataset devSet, Scorer scorer,Timing timer) { - super(); - this.trainSet = trainSet; - this.devSet = devSet; - this.scorer = scorer; - this.timer = timer; - } - - public Double apply(Double sigmaToTry) { - double[][] weights2D; - setSigma(sigmaToTry); - - weights2D = trainWeights(trainSet, weights,true); //bypass. - - weights = ArrayUtils.flatten(weights2D); - - LinearClassifier classifier = new LinearClassifier(weights2D, trainSet.featureIndex, trainSet.labelIndex); - - double score = scorer.score(classifier, devSet); - //System.out.println("score: "+score); - //System.out.print("."); - System.err.print("##sigma = "+getSigma()+" "); - System.err.println("-> average Score: "+ score); - System.err.println("##time elapsed: " + timer.stop() + " milliseconds."); - timer.restart(); - return -score; - } - } - - /** If set to true, then when training a classifier, after an optimal sigma is chosen a model is relearned from - * scratch. If set to false (the default), then the model is updated from wherever it wound up in the sigma-tuning process. - * The latter is likely to be faster, but it's not clear which model will wind up better. */ - public void setRetrainFromScratchAfterSigmaTuning( boolean retrainFromScratchAfterSigmaTuning) { - this.retrainFromScratchAfterSigmaTuning = retrainFromScratchAfterSigmaTuning; - } - - - public Classifier trainClassifier(Iterable> dataIterable) { - Index featureIndex = Generics.newIndex(); - Index labelIndex = Generics.newIndex(); - for(Datum d : dataIterable) { - labelIndex.add(d.label()); - featureIndex.addAll(d.asFeatures());//If there are duplicates, it doesn't add them again. - } - System.err.println(String.format("Training linear classifier with %d features and %d labels", featureIndex.size(), labelIndex.size())); - - LogConditionalObjectiveFunction objective = new LogConditionalObjectiveFunction(dataIterable, logPrior, featureIndex, labelIndex); - objective.setPrior(new LogPrior(LogPrior.LogPriorType.QUADRATIC)); - - double[] initial = objective.initial(); - double[] weights = minimizer.minimize(objective, TOL, initial); - - LinearClassifier classifier = new LinearClassifier(objective.to2D(weights), featureIndex, labelIndex); - return - classifier; - - } - - public Classifier trainClassifier(GeneralDataset dataset, float[] dataWeights, LogPrior prior) { - if(dataset instanceof RVFDataset) - ((RVFDataset)dataset).ensureRealValues(); - LogConditionalObjectiveFunction objective = new LogConditionalObjectiveFunction(dataset, dataWeights, logPrior); - - double[] initial = objective.initial(); - double[] weights = minimizer.minimize(objective, TOL, initial); - - LinearClassifier classifier = new LinearClassifier(objective.to2D(weights), dataset.featureIndex(), dataset.labelIndex()); - return classifier; - } - - - @Override - public LinearClassifier trainClassifier(GeneralDataset dataset) { - return trainClassifier(dataset, null); - } - public LinearClassifier trainClassifier(GeneralDataset dataset, double[] initial) { - if(dataset instanceof RVFDataset) - ((RVFDataset)dataset).ensureRealValues(); - double[][] weights = trainWeights(dataset, initial, false); - LinearClassifier classifier = new LinearClassifier(weights, dataset.featureIndex(), dataset.labelIndex()); - return classifier; - } - - /** - * Given the path to a file representing the text based serialization of a - * Linear Classifier, reconstitutes and returns that LinearClassifier. - * - * TODO: Leverage Index - */ - public Classifier loadFromFilename(String file) { - try { - File tgtFile = new File(file); - BufferedReader in = new BufferedReader(new FileReader(tgtFile)); - - // Format: read indicies first, weights, then thresholds - Index labelIndex = HashIndex.loadFromReader(in); - Index featureIndex = HashIndex.loadFromReader(in); - double[][] weights = new double[featureIndex.size()][labelIndex.size()]; - String line = in.readLine(); - int currLine = 1; - while (line != null && line.length()>0) { - String[] tuples = line.split(LinearClassifier.TEXT_SERIALIZATION_DELIMITER); - if (tuples.length != 3) { - throw new Exception("Error: incorrect number of tokens in weight specifier, line=" - +currLine+" in file "+tgtFile.getAbsolutePath()); - } - currLine++; - int feature = Integer.valueOf(tuples[0]); - int label = Integer.valueOf(tuples[1]); - double value = Double.valueOf(tuples[2]); - weights[feature][label] = value; - line = in.readLine(); - } - - // First line in thresholds is the number of thresholds - int numThresholds = Integer.valueOf(in.readLine()); - double[] thresholds = new double[numThresholds]; - int curr = 0; - while ((line = in.readLine()) != null) { - double tval = Double.valueOf(line.trim()); - thresholds[curr++] = tval; - } - in.close(); - LinearClassifier classifier = new LinearClassifier(weights, featureIndex, labelIndex); - return classifier; - } catch (Exception e) { - System.err.println("Error in LinearClassifierFactory, loading from file="+file); - e.printStackTrace(); - return null; - } - } - - @Deprecated - public LinearClassifier trainClassifier(List> examples) { - // TODO Auto-generated method stub - return null; - } - - public boolean setEvaluators(int iters, Evaluator[] evaluators) - { - if (minimizer instanceof HasEvaluators) { - ((HasEvaluators) minimizer).setEvaluators(iters, evaluators); - return true; - } else { - return false; - } - } - - public LinearClassifierCreator getClassifierCreator(GeneralDataset dataset) { -// LogConditionalObjectiveFunction objective = new LogConditionalObjectiveFunction(dataset, logPrior); - return new LinearClassifierCreator(dataset.featureIndex, dataset.labelIndex); - } - - public static class LinearClassifierCreator implements ClassifierCreator, ProbabilisticClassifierCreator - { - LogConditionalObjectiveFunction objective; - Index featureIndex; - Index labelIndex; - - public LinearClassifierCreator(LogConditionalObjectiveFunction objective, Index featureIndex, Index labelIndex) - { - this.objective = objective; - this.featureIndex = featureIndex; - this.labelIndex = labelIndex; - } - - public LinearClassifierCreator(Index featureIndex, Index labelIndex) - { - this.featureIndex = featureIndex; - this.labelIndex = labelIndex; - } - - public LinearClassifier createLinearClassifier(double[] weights) { - double[][] weights2D; - if (objective != null) { - weights2D = objective.to2D(weights); - } else { - weights2D = ArrayUtils.to2D(weights, featureIndex.size(), labelIndex.size()); - } - return new LinearClassifier(weights2D, featureIndex, labelIndex); - } - - public Classifier createClassifier(double[] weights) { - return createLinearClassifier(weights); - } - - public ProbabilisticClassifier createProbabilisticClassifier(double[] weights) { - return createLinearClassifier(weights); - } - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LogConditionalObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LogConditionalObjectiveFunction.java deleted file mode 100644 index e6a6c62..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LogConditionalObjectiveFunction.java +++ /dev/null @@ -1,899 +0,0 @@ -package edu.stanford.nlp.classify; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; - -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.math.ADMath; -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.math.DoubleAD; -import edu.stanford.nlp.optimization.AbstractStochasticCachingDiffUpdateFunction; -import edu.stanford.nlp.optimization.StochasticCalculateMethods; -import edu.stanford.nlp.util.Index; - - -/** - * Maximizes the conditional likelihood with a given prior. - * - * @author Dan Klein - * @author Galen Andrew - * @author Chris Cox (merged w/ SumConditionalObjectiveFunction, 2/16/05) - * @author Sarah Spikes (Templatization, allowing an Iterable> to be passed in instead of a GeneralDataset) - * @author Angel Chang (support in place SGD - extend AbstractStochasticCachingDiffUpdateFunction) - */ - -public class LogConditionalObjectiveFunction extends AbstractStochasticCachingDiffUpdateFunction { - - public void setPrior(LogPrior prior) { - this.prior = prior; - clearCache(); - } - - protected LogPrior prior; - - protected int numFeatures = 0; - protected int numClasses = 0; - - protected int[][] data = null; - protected Iterable> dataIterable = null; - protected double[][] values = null; - protected int[] labels = null; - protected float[] dataweights = null; - protected double[] derivativeNumerator = null; - - protected DoubleAD[] xAD = null; - protected double [] priorDerivative = null; //The only reason this is around is because the Prior Functions don't handle stochastic calculations yet. - protected DoubleAD[] derivativeAD = null; - protected DoubleAD[] sums = null; - protected DoubleAD[] probs = null; - - protected Index labelIndex = null; - protected Index featureIndex = null; - protected boolean useIterable = false; - - protected boolean useSummedConditionalLikelihood = false; //whether to use sumConditional or logConditional - - @Override - public int domainDimension() { - return numFeatures * numClasses; - } - - @Override - public int dataDimension(){ - return data.length; - } - - int classOf(int index) { - return index % numClasses; - } - - int featureOf(int index) { - return index / numClasses; - } - - protected int indexOf(int f, int c) { - return f * numClasses + c; - } - - public double[][] to2D(double[] x) { - double[][] x2 = new double[numFeatures][numClasses]; - for (int i = 0; i < numFeatures; i++) { - for (int j = 0; j < numClasses; j++) { - x2[i][j] = x[indexOf(i, j)]; - } - } - return x2; - } - - /** - * Calculate the conditional likelihood. - * If useSummedConditionalLikelihood is false (the default), - * this calculates standard(product) CL, otherwise this calculates summed CL. - * What's the difference? See Klein and Manning's 2002 EMNLP paper. - */ - @Override - protected void calculate(double[] x) { - //If the batchSize is 0 then use the regular calculate methods - if (useSummedConditionalLikelihood) { - calculateSCL(x); - } else { - calculateCL(x); - } - - } - - - - /* - * This function is used to comme up with an estimate of the value / gradient based on only a small - * portion of the data (refered to as the batchSize for lack of a better term. In this case batch does - * not mean All!! It should be thought of in the sense of "a small batch of the data". - */ - - - @Override - public void calculateStochastic(double[] x, double[] v, int[] batch){ - - if(method.calculatesHessianVectorProduct() && v != null){ - // This is used for Stochastic Methods that involve second order information (SMD for example) - if(method.equals(StochasticCalculateMethods.AlgorithmicDifferentiation)){ - calculateStochasticAlgorithmicDifferentiation(x,v,batch); - }else if(method.equals(StochasticCalculateMethods.IncorporatedFiniteDifference)){ - calculateStochasticFiniteDifference(x,v,finiteDifferenceStepSize,batch); - } - } else{ - //This is used for Stochastic Methods that don't need anything but the gradient (SGD) - calculateStochasticGradientOnly(x,batch); - } - - } - - - - - /** - * Calculate the summed conditional likelihood of this data by summing - * conditional estimates. - * - */ - private void calculateSCL(double[] x) { - //System.out.println("Checking at: "+x[0]+" "+x[1]+" "+x[2]); - value = 0.0; - Arrays.fill(derivative, 0.0); - double[] sums = new double[numClasses]; - double[] probs = new double[numClasses]; - double[] counts = new double[numClasses]; - Arrays.fill(counts, 0.0); - for (int d = 0; d < data.length; d++) { - // if (d == testMin) { - // d = testMax - 1; - // continue; - // } - int[] features = data[d]; - // activation - Arrays.fill(sums, 0.0); - for (int c = 0; c < numClasses; c++) { - for (int f = 0; f < features.length; f++) { - int i = indexOf(features[f], c); - sums[c] += x[i]; - } - } - // expectation (slower routine replaced by fast way) - // double total = Double.NEGATIVE_INFINITY; - // for (int c=0; c datum : dataIterable) { - // if (d == testMin) { - // d = testMax - 1; - // continue; - // } - Collection features = datum.asFeatures(); - for (F feature : features) { - int i = indexOf(featureIndex.indexOf(feature), labelIndex.indexOf(datum.label())); - if (dataweights == null) { - derivativeNumerator[i] -= 1; - } /*else { - derivativeNumerator[i] -= dataweights[index]; - }*/ - } - } - } - else { - System.err.println("Both were null! Couldn't calculate."); - System.exit(-1); - } - } - copy(derivative, derivativeNumerator); - // Arrays.fill(derivative, 0.0); - double[] sums = new double[numClasses]; - double[] probs = new double[numClasses]; - // double[] counts = new double[numClasses]; - // Arrays.fill(counts, 0.0); - - Iterator> iter = null; - int d = -1; - if(useIterable) - iter = dataIterable.iterator(); - Datum datum = null; - while(true){ - if(useIterable) { - if(!iter.hasNext()) break; - datum = iter.next(); - } else { - d++; - if(d >= data.length) break; - } - - // if (d == testMin) { - // d = testMax - 1; - // continue; - // } - - // activation - Arrays.fill(sums, 0.0); - double total = 0; - if(!useIterable) { - int[] featuresArr = data[d]; - - for (int c = 0; c < numClasses; c++) { - for (int f = 0; f < featuresArr.length; f++) { - int i = indexOf(featuresArr[f], c); - sums[c] += x[i]; - } - } - // expectation (slower routine replaced by fast way) - // double total = Double.NEGATIVE_INFINITY; - // for (int c=0; c features = datum.asFeatures(); - for (int c = 0; c < numClasses; c++) { - for (F feature : features) { - int i = indexOf(featureIndex.indexOf(feature), c); - sums[c] += x[i]; - } - } - // expectation (slower routine replaced by fast way) - // double total = Double.NEGATIVE_INFINITY; - // for (int c=0; c dataset) { - this(dataset, new LogPrior(LogPrior.LogPriorType.QUADRATIC)); - } - - public LogConditionalObjectiveFunction(GeneralDataset dataset, LogPrior prior) { - this(dataset, prior, false); - } - - public LogConditionalObjectiveFunction(GeneralDataset dataset, float[] dataWeights, LogPrior prior) { - this(dataset, prior, false); - this.dataweights = dataWeights; - System.err.println("correct constructor"); - } - - public LogConditionalObjectiveFunction(GeneralDataset dataset, LogPrior prior, boolean useSumCondObjFun) { - setPrior(prior); - setUseSumCondObjFun(useSumCondObjFun); - this.numFeatures = dataset.numFeatures(); - this.numClasses = dataset.numClasses(); - this.data = dataset.getDataArray(); - this.labels = dataset.getLabelsArray(); - this.values = dataset.getValuesArray(); - if (dataset instanceof WeightedDataset) { - this.dataweights = ((WeightedDataset)dataset).getWeights(); - } - } - - //TODO: test this - public LogConditionalObjectiveFunction(Iterable> dataIterable, LogPrior logPrior, Index featureIndex, Index labelIndex) { - setPrior(prior); - setUseSumCondObjFun(false); - this.useIterable = true; - this.numFeatures = featureIndex.size(); - this.numClasses = labelIndex.size(); - this.data = null; - this.dataIterable = dataIterable; - - this.labelIndex = labelIndex; - this.featureIndex = featureIndex; - this.labels = null;//dataset.getLabelsArray(); - this.values = null;//dataset.getValuesArray(); - //this.dataweights //leave it null? - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, boolean useSumCondObjFun) { - this(numFeatures, numClasses, data, labels); - this.useSummedConditionalLikelihood = useSumCondObjFun; - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels) { - this(numFeatures, numClasses, data, labels, new LogPrior(LogPrior.LogPriorType.QUADRATIC)); - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, LogPrior prior) { - this(numFeatures, numClasses, data, labels, null, prior); - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, float[] dataweights) { - this(numFeatures, numClasses, data, labels, dataweights, new LogPrior(LogPrior.LogPriorType.QUADRATIC)); - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, float[] dataweights, LogPrior prior) { - this.numFeatures = numFeatures; - this.numClasses = numClasses; - this.data = data; - this.labels = labels; - this.prior = prior; - this.dataweights = dataweights; - // this.testMin = data.length; - // this.testMax = data.length; - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, int intPrior, double sigma, double epsilon) { - this(numFeatures, numClasses, data, null, labels, intPrior, sigma, epsilon); - } - - public LogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, double[][] values, int[] labels, int intPrior, double sigma, double epsilon) { - this.numFeatures = numFeatures; - this.numClasses = numClasses; - this.data = data; - this.values = values; - this.labels = labels; - this.prior = new LogPrior(intPrior, sigma, epsilon); - // this.testMin = data.length; - // this.testMax = data.length; - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LogPrior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LogPrior.java deleted file mode 100644 index 19257c1..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/LogPrior.java +++ /dev/null @@ -1,334 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.util.ArrayUtils; - -import java.io.Serializable; - - -/** - * A Prior for functions. Immutable. - * - * @author Galen Andrew - */ -public class LogPrior implements Serializable { - - /** - * - */ - private static final long serialVersionUID = 7826853908892790965L; - - public enum LogPriorType { NULL, QUADRATIC, HUBER, QUARTIC, COSH, ADAPT, MULTIPLE_QUADRATIC } - - public static LogPriorType getType(String name) { - if (name.equalsIgnoreCase("null")) { return LogPriorType.NULL; } - else if (name.equalsIgnoreCase("quadratic")) { return LogPriorType.QUADRATIC; } - else if (name.equalsIgnoreCase("huber")) { return LogPriorType.HUBER; } - else if (name.equalsIgnoreCase("quartic")) { return LogPriorType.QUARTIC; } - else if (name.equalsIgnoreCase("cosh")) { return LogPriorType.COSH; } -// else if (name.equalsIgnoreCase("multiple")) { return LogPriorType.MULTIPLE; } - else { throw new RuntimeException("Unknown LogPriorType: "+name); } - } - - // these fields are just for the ADAPT prior - - // is there a better way to do this? - private double[] means = null; - private LogPrior otherPrior = null; - - public static LogPrior getAdaptationPrior(double[] means, LogPrior otherPrior) { - LogPrior lp = new LogPrior(LogPriorType.ADAPT); - lp.means = means; - lp.otherPrior = otherPrior; - return lp; - } - - public LogPriorType getType() { - return type; - } - - public final LogPriorType type; - - public LogPrior() { - this(LogPriorType.QUADRATIC); - } - - public LogPrior(int intPrior) { - this(intPrior, 1.0, 0.1); - } - - public LogPrior(LogPriorType type) { - this(type, 1.0, 0.1); - } - - // why isn't this functionality in enum? - private static LogPriorType intToType(int intPrior) { - LogPriorType[] values = LogPriorType.values(); - for (LogPriorType val : values) { - if (val.ordinal() == intPrior) { - return val; - } - } - throw new IllegalArgumentException(intPrior + " is not a legal LogPrior."); - } - - public LogPrior(int intPrior, double sigma, double epsilon) { - this(intToType(intPrior), sigma, epsilon); - } - - public LogPrior(LogPriorType type, double sigma, double epsilon) { - this.type = type; - if (type != LogPriorType.ADAPT) { - setSigma(sigma); - setEpsilon(epsilon); - } - } - - - // this is the C variable in CSFoo's MM paper C = 1/\sigma^2 -// private double[] regularizationHyperparameters = null; - - private double[] sigmaSqM = null; - private double[] sigmaQuM = null; - - -// public double[] getRegularizationHyperparameters() { -// return regularizationHyperparameters; -// } -// -// public void setRegularizationHyperparameters( -// double[] regularizationHyperparameters) { -// this.regularizationHyperparameters = regularizationHyperparameters; -// } - - /** - * IMPORTANT NOTE: This constructor allows non-uniform regularization, but it - * transforms the inputs C (like the machine learning people like) to sigma - * (like we NLP folks like). C = 1/\sigma^2 - */ - public LogPrior(double[] C) { - this.type = LogPriorType.MULTIPLE_QUADRATIC; - double[] sigmaSqM = new double[C.length]; - for (int i=0;i 30.0) { - val = norm - Math.log(2); - d = 1.0 / sigmaSq; - } else { - val = Math.log(Math.cosh(norm)); - d = (2 * (1 / (Math.exp(-2.0 * norm) + 1)) - 1.0) / sigmaSq; - } - for (int i=0; i < x.length; i++) { - grad[i] += Math.signum(x[i]) * d; - } - return val; - case MULTIPLE_QUADRATIC: -// for (int i = 0; i < x.length; i++) { -// val += x[i] * x[i]* 1/2 * regularizationHyperparameters[i]; -// grad[i] += x[i] * regularizationHyperparameters[i]; -// } - - for (int i = 0; i < x.length; i++) { - val += x[i] * x[i] / 2.0 / sigmaSqM[i]; - grad[i] += x[i] / sigmaSqM[i]; - } - - - return val; - default: - throw new RuntimeException("LogPrior.valueAt is undefined for prior of type " + this); - } - } - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/PRCurve.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/PRCurve.java deleted file mode 100644 index 16c1295..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/PRCurve.java +++ /dev/null @@ -1,367 +0,0 @@ -package edu.stanford.nlp.classify; - -import java.util.ArrayList; -import java.util.List; -import java.io.File; - -import edu.stanford.nlp.util.BinaryHeapPriorityQueue; -import edu.stanford.nlp.objectbank.ObjectBank; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.PriorityQueue; -import edu.stanford.nlp.util.StringUtils; -import edu.stanford.nlp.util.Triple; - -/** - * @author Kristina Toutanova - * May 23, 2005 - * A class to create recall-precision curves given scores - * used to fit the best monotonic function for logistic regression and svms - */ -public class PRCurve { - double[] scores; //sorted scores - int[] classes; // the class of example i - int[] guesses; // the guess of example i according to the argmax - int[] numpositive; // number positive in the i-th highest scores - int[] numnegative; // number negative in the i-th lowest scores - - /** - * reads scores with classes from a file, sorts by score and creates the arrays - * - */ - public PRCurve(String filename) { - try { - ArrayList> dataScores = new ArrayList>(); - for(String line : ObjectBank.getLineIterator(new File(filename))) { - List elems = StringUtils.split(line); - Pair p = new Pair(new Double(elems.get(0).toString()), Integer.valueOf(elems.get(1).toString())); - dataScores.add(p); - } - init(dataScores); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - - /** - * reads scores with classes from a file, sorts by score and creates the arrays - * - */ - public PRCurve(String filename, boolean svm) { - try { - - ArrayList> dataScores = new ArrayList>(); - for(String line : ObjectBank.getLineIterator(new File(filename))) { - List elems = StringUtils.split(line); - int cls = (new Double(elems.get(0).toString())).intValue(); - if (cls == -1) { - cls = 0; - } - double score = Double.parseDouble(elems.get(1).toString()) + 0.5; - Pair p = new Pair(new Double(score), Integer.valueOf(cls)); - dataScores.add(p); - } - init(dataScores); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - public double optimalAccuracy() { - return precision(numSamples()) / (double) numSamples(); - } - - public double accuracy() { - return logPrecision(numSamples()) / (double) numSamples(); - } - - - public PRCurve(List> dataScores) { - init(dataScores); - } - - public void init(List> dataScores) { - PriorityQueue>> q = new BinaryHeapPriorityQueue>>(); - for (int i = 0; i < dataScores.size(); i++) { - q.add(new Pair>(Integer.valueOf(i), dataScores.get(i)), -dataScores.get(i).first().doubleValue()); - } - List>> sorted = q.toSortedList(); - scores = new double[sorted.size()]; - classes = new int[sorted.size()]; - System.err.println("incoming size " + dataScores.size() + " resulting " + sorted.size()); - - for (int i = 0; i < sorted.size(); i++) { - Pair next = sorted.get(i).second(); - scores[i] = next.first().doubleValue(); - classes[i] = next.second().intValue(); - } - init(); - } - - - public void initMC(ArrayList> dataScores) { - PriorityQueue>> q = new BinaryHeapPriorityQueue>>(); - for (int i = 0; i < dataScores.size(); i++) { - q.add(new Pair>(Integer.valueOf(i), dataScores.get(i)), -dataScores.get(i).first().doubleValue()); - } - List>> sorted = q.toSortedList(); - scores = new double[sorted.size()]; - classes = new int[sorted.size()]; - guesses = new int[sorted.size()]; - System.err.println("incoming size " + dataScores.size() + " resulting " + sorted.size()); - - for (int i = 0; i < sorted.size(); i++) { - Triple next = sorted.get(i).second(); - scores[i] = next.first().doubleValue(); - classes[i] = next.second().intValue(); - guesses[i] = next.third().intValue(); - } - init(); - } - - - /** - * initialize the numpositive and the numnegative arrays - */ - void init() { - numnegative = new int[numSamples() + 1]; - numpositive = new int[numSamples() + 1]; - numnegative[0] = 0; - numpositive[0] = 0; - int num = numSamples(); - for (int i = 1; i <= num; i++) { - numnegative[i] = numnegative[i - 1] + (classes[i - 1] == 0 ? 1 : 0); - } - for (int i = 1; i <= num; i++) { - numpositive[i] = numpositive[i - 1] + (classes[num - i] == 0 ? 0 : 1); - } - System.err.println("total positive " + numpositive[num] + " total negative " + numnegative[num] + " total " + num); - for (int i = 1; i < numpositive.length; i++) { - //System.out.println(i + " positive " + numpositive[i] + " negative " + numnegative[i] + " classes " + classes[i - 1] + " " + classes[num - i]); - } - } - - int numSamples() { - return scores.length; - } - - /** - * what is the best precision at the given recall - * - */ - public int precision(int recall) { - int optimum = 0; - for (int right = 0; right <= recall; right++) { - int candidate = numpositive[right] + numnegative[recall - right]; - if (candidate > optimum) { - optimum = candidate; - } - } - return optimum; - } - - public static double f1(int tp, int fp, int fn) { - double prec = 1; - double recall = 1; - if (tp + fp > 0) { - prec = tp / (double) (tp + fp); - } - if (tp + fn > 0) { - recall = tp / (double) (tp + fn); - } - return 2 * prec * recall / (prec + recall); - } - - /** - * the f-measure if we just guess as negativ the first numleft and guess as poitive the last numright - * - */ - public double fmeasure(int numleft, int numright) { - int tp = 0, fp = 0, fn = 0; - tp = numpositive[numright]; - fp = numright - tp; - fn = numleft - numnegative[numleft]; - return f1(tp, fp, fn); - } - - - /** - * what is the precision at this recall if we look at the score as the probability of class 1 given x - * as if coming from logistic regression - * - */ - public int logPrecision(int recall) { - int totaltaken = 0; - int rightIndex = numSamples() - 1; //next right candidate - int leftIndex = 0; //next left candidate - int totalcorrect = 0; - - while (totaltaken < recall) { - double confr = Math.abs(scores[rightIndex] - .5); - double confl = Math.abs(scores[leftIndex] - .5); - int chosen = leftIndex; - if (confr > confl) { - chosen = rightIndex; - rightIndex--; - } else { - leftIndex++; - } - //System.err.println("chose "+chosen+" score "+scores[chosen]+" class "+classes[chosen]+" correct "+correct(scores[chosen],classes[chosen])); - if ((scores[chosen] >= .5) && (classes[chosen] == 1)) { - totalcorrect++; - } - if ((scores[chosen] < .5) && (classes[chosen] == 0)) { - totalcorrect++; - } - totaltaken++; - } - - return totalcorrect; - } - - /** - * what is the optimal f-measure we can achieve given recall guesses - * using the optimal monotonic function - * - */ - public double optFmeasure(int recall) { - double max = 0; - for (int i = 0; i < (recall + 1); i++) { - double f = fmeasure(i, recall - i); - if (f > max) { - max = f; - } - } - return max; - } - - public double opFmeasure() { - return optFmeasure(numSamples()); - } - - /** - * what is the f-measure at this recall if we look at the score as the probability of class 1 given x - * as if coming from logistic regression same as logPrecision but calculating f-measure - * - * @param recall make this many guesses for which we are most confident - */ - public double fmeasure(int recall) { - int totaltaken = 0; - int rightIndex = numSamples() - 1; //next right candidate - int leftIndex = 0; //next left candidate - int tp = 0, fp = 0, fn = 0; - while (totaltaken < recall) { - double confr = Math.abs(scores[rightIndex] - .5); - double confl = Math.abs(scores[leftIndex] - .5); - int chosen = leftIndex; - if (confr > confl) { - chosen = rightIndex; - rightIndex--; - } else { - leftIndex++; - } - //System.err.println("chose "+chosen+" score "+scores[chosen]+" class "+classes[chosen]+" correct "+correct(scores[chosen],classes[chosen])); - if ((scores[chosen] >= .5)) { - if (classes[chosen] == 1) { - tp++; - } else { - fp++; - } - } - if ((scores[chosen] < .5)) { - if (classes[chosen] == 1) { - fn++; - } - } - totaltaken++; - } - - return f1(tp, fp, fn); - - } - - - /** - * assuming the scores are probability of 1 given x - * - */ - public double logLikelihood() { - double loglik = 0; - for (int i = 0; i < scores.length; i++) { - loglik += Math.log(classes[i] == 0 ? 1 - scores[i] : scores[i]); - } - return loglik; - } - - /** - * confidence weighted accuracy assuming the scores are probabilities and using .5 as treshold - * - */ - public double cwa() { - double acc = 0; - for (int recall = 1; recall <= numSamples(); recall++) { - acc += logPrecision(recall) / (double) recall; - } - return acc / numSamples(); - } - - /** - * confidence weighted accuracy assuming the scores are probabilities and using .5 as treshold - * - */ - public int[] cwaArray() { - int[] arr = new int[numSamples()]; - for (int recall = 1; recall <= numSamples(); recall++) { - arr[recall - 1] = logPrecision(recall); - } - return arr; - } - - /** - * confidence weighted accuracy assuming the scores are probabilities and using .5 as treshold - * - */ - public int[] optimalCwaArray() { - int[] arr = new int[numSamples()]; - for (int recall = 1; recall <= numSamples(); recall++) { - arr[recall - 1] = precision(recall); - } - return arr; - } - - /** - * optimal confidence weighted accuracy assuming for each recall we can fit an optimal monotonic function - * - */ - public double optimalCwa() { - double acc = 0; - for (int recall = 1; recall <= numSamples(); recall++) { - acc += precision(recall) / (double) recall; - } - return acc / numSamples(); - } - - - public static boolean correct(double score, int cls) { - return ((score >= .5) && (cls == 1)) || ((score < .5) && (cls == 0)); - } - - public static void main(String[] args) { - - PriorityQueue q = new BinaryHeapPriorityQueue(); - q.add("bla", 2); - q.add("bla3", 2); - System.err.println("size of q " + q.size()); - - PRCurve pr = new PRCurve("c:/data0204/precsvm", true); - System.err.println("acc " + pr.accuracy() + " opt " + pr.optimalAccuracy() + " cwa " + pr.cwa() + " optcwa " + pr.optimalCwa()); - for (int r = 1; r <= pr.numSamples(); r++) { - System.err.println("optimal precision at recall " + r + " " + pr.precision(r)); - System.err.println("model precision at recall " + r + " " + pr.logPrecision(r)); - } - } - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ProbabilisticClassifier.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ProbabilisticClassifier.java deleted file mode 100644 index 7cae490..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ProbabilisticClassifier.java +++ /dev/null @@ -1,10 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.stats.Counter; - -public interface ProbabilisticClassifier extends Classifier -{ - public Counter probabilityOf(Datum example); - public Counter logProbabilityOf(Datum example); -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ProbabilisticClassifierCreator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ProbabilisticClassifierCreator.java deleted file mode 100644 index ee06806..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ProbabilisticClassifierCreator.java +++ /dev/null @@ -1,10 +0,0 @@ -package edu.stanford.nlp.classify; - -/** - * Creates a probablic classifier with given weights - * - * @author Angel Chang - */ -public interface ProbabilisticClassifierCreator { - public ProbabilisticClassifier createProbabilisticClassifier(double[] weights); -} \ No newline at end of file diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/RVFClassifier.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/RVFClassifier.java deleted file mode 100644 index 1a7d438..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/RVFClassifier.java +++ /dev/null @@ -1,20 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.stats.Counter; - -import java.io.Serializable; - -/** - * A simple interface for classifying and scoring data points with - * real-valued features. Implemented by the linear classifier. - * - * @author Jenny Finkel - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - */ - -public interface RVFClassifier extends Serializable { - public L classOf(RVFDatum example); - - public Counter scoresOf(RVFDatum example); -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/RVFDataset.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/RVFDataset.java deleted file mode 100644 index 3df87be..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/RVFDataset.java +++ /dev/null @@ -1,961 +0,0 @@ -package edu.stanford.nlp.classify; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.ling.RVFDatum; -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.stats.Counters; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.HashIndex; - - -/** - * An interfacing class for {@link ClassifierFactory} that incrementally - * builds a more memory-efficient representation of a {@link List} of - * {@link RVFDatum} objects for the purposes of training a {@link Classifier} - * with a {@link ClassifierFactory}. - * - * @author Jenny Finkel (jrfinkel@stanford.edu) - * @author Rajat Raina (added methods to record data sources and ids) - * @author Anna Rafferty (various refactoring with GeneralDataset/Dataset) - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * - * @param The type of the labels in the Dataset - * @param The type of the features in the Dataset - */ -public class RVFDataset extends GeneralDataset implements Iterable> { // Serializable - - private static final long serialVersionUID = -3841757837680266182L; - - private double[][] values; - private double[] minValues; // = null; //stores the minValues of all features for normalization. - private double[] maxValues; // = null; //stores the maxValues of all features for normalization. - double[] means; - double[] stdevs; //means and stdevs of features, used for - - - /* - * Store source and id of each datum; optional, and not fully supported. - */ - private ArrayList> sourcesAndIds; - - public RVFDataset() { - this(10); - } - - public RVFDataset(int numDatums, Index featureIndex, Index labelIndex) { - this(numDatums); - this.labelIndex = labelIndex; - this.featureIndex = featureIndex; - } - - public RVFDataset(Index featureIndex, Index labelIndex) { - this(10); - this.labelIndex = labelIndex; - this.featureIndex = featureIndex; - } - - public RVFDataset(int numDatums) { - initialize(numDatums); - } - - /** - * Constructor that fully specifies a Dataset. Needed this for MulticlassDataset. - */ - public RVFDataset(Index labelIndex, int[] labels, Index featureIndex, int[][] data, double[][] values) { - this.labelIndex = labelIndex; - this.labels = labels; - this.featureIndex = featureIndex; - this.data = data; - this.values = values; - this.size = labels.length; - } - - @Override - public Pair, GeneralDataset> split(double percentDev) { - int devSize = (int)(percentDev * size()); - int trainSize = size() - devSize; - - int[][] devData = new int[devSize][]; - double[][] devValues = new double[devSize][]; - int[] devLabels = new int[devSize]; - - int[][] trainData = new int[trainSize][]; - double[][] trainValues = new double[trainSize][]; - int[] trainLabels = new int[trainSize]; - - System.arraycopy(data, 0, devData, 0, devSize); - System.arraycopy(values, 0, devValues, 0, devSize); - System.arraycopy(labels, 0, devLabels, 0, devSize); - - System.arraycopy(data, devSize, trainData, 0, trainSize); - System.arraycopy(values, devSize, trainValues, 0, trainSize); - System.arraycopy(labels, devSize, trainLabels, 0, trainSize); - - - RVFDataset dev = new RVFDataset(labelIndex, devLabels, featureIndex, devData, devValues); - RVFDataset train = new RVFDataset(labelIndex, trainLabels, featureIndex, trainData, trainValues); - - return new Pair,GeneralDataset>(train, dev); - - } - - public void scaleFeaturesGaussian(){ - means = new double[this.numFeatures()]; - Arrays.fill(means, 0); - - for(int i = 0; i < this.size(); i++){ - for(int j = 0; j < data[i].length; j++) - means[data[i][j]]+=values[i][j]; - } - ArrayMath.multiplyInPlace(means, 1.0/this.size()); - - stdevs = new double[this.numFeatures()]; - Arrays.fill(stdevs, 0); - double[] deltaX = new double[this.numFeatures()]; - - for(int i = 0; i < this.size(); i++){ - for(int f = 0; f < this.numFeatures(); f++) - deltaX[f] = -means[f]; - for(int j = 0; j < data[i].length; j++) - deltaX[data[i][j]] += values[i][j]; - for(int f = 0; f < this.numFeatures(); f++){ - stdevs[f] += deltaX[f]*deltaX[f]; - } - } - for(int f = 0; f < this.numFeatures(); f++){ - stdevs[f] /= (this.size()-1); - stdevs[f] = Math.sqrt(stdevs[f]); - } - for(int i = 0; i < this.size(); i++){ - for(int j = 0; j < data[i].length; j++){ - int fID = data[i][j]; - if(stdevs[fID] != 0) - values[i][j] = (values[i][j] - means[fID])/stdevs[fID]; - } - } - - } - /** - * scales feature values linearly such that each feature value lies between 0 and 1. - * - */ - public void scaleFeatures(){ - // TODO: should also implement a method that scales the features using the mean and std. - minValues = new double[featureIndex.size()]; - maxValues = new double[featureIndex.size()]; - Arrays.fill(minValues, Double.POSITIVE_INFINITY); - Arrays.fill(maxValues, Double.NEGATIVE_INFINITY); - - //first identify the max and min values for each feature. - //System.out.printf("number of datums: %d dataset size: %d\n",data.length,size()); - for(int i = 0; i < size(); i++){ - //System.out.printf("datum %d length %d\n", i,data[i].length); - for(int j = 0; j < data[i].length; j++){ - int f = data[i][j]; - if(values[i][j] < minValues[f])minValues[f] = values[i][j]; - if(values[i][j] > maxValues[f])maxValues[f] = values[i][j]; - } - } - - for(int f = 0; f < featureIndex.size(); f++){ - if(minValues[f] == Double.POSITIVE_INFINITY) - throw new RuntimeException("minValue for feature "+f+" not assigned. "); - if(maxValues[f] == Double.NEGATIVE_INFINITY) - throw new RuntimeException("maxValue for feature "+f+" not assigned."); - } - - //now scale each value such that it's between 0 and 1. - for(int i = 0; i < size(); i++){ - for(int j = 0; j < data[i].length; j++){ - int f = data[i][j]; - if(minValues[f] != maxValues[f])//the equality can happen for binary features which always take the value of 1.0 - values[i][j] = (values[i][j] - minValues[f])/(maxValues[f] - minValues[f]); - } - } - - /* - for(int f = 0; f < featureIndex.size(); f++){ - if(minValues[f] == maxValues[f]) - throw new RuntimeException("minValue for feature "+f+" is equal to maxValue:"+minValues[f]); - } - */ - } - - /** - * checks if the dataset has any unbounded values. - * Always good to use this before training a model on the dataset. - * This way, one can avoid seeing the infamous 4's that get printed by the QuasiNewton Method when NaNs exist in the data! - * -Ramesh - */ - public void ensureRealValues(){ - double[][] values = getValuesArray(); - int[][] data = getDataArray(); - for(int i = 0; i < size(); i++){ - for(int j = 0; j < values[i].length; j++){ - if(Double.isNaN(values[i][j])){ - int fID = data[i][j]; - F feature = featureIndex.get(fID); - throw new RuntimeException("datum "+i+" has a NaN value for feature:"+feature); - } - if(Double.isInfinite(values[i][j])){ - int fID = data[i][j]; - F feature = featureIndex.get(fID); - throw new RuntimeException("datum "+i+" has infinite value for feature:"+feature); - } - } - } - } - - /** - * Scales the values of each feature in each linearly using the min and max values found in the training set. - * NOTE1: Not guaranteed to be between 0 and 1 for a test datum. - * NOTE2: Also filters out features from each datum that are not seen at training time. - * @param dataset - * @return a new dataset - */ - public RVFDataset scaleDataset(RVFDataset dataset){ - RVFDataset newDataset = new RVFDataset(this.featureIndex,this.labelIndex); - for(int i = 0; i < dataset.size(); i++){ - RVFDatum datum = dataset.getDatum(i); - newDataset.add(scaleDatum(datum)); - } - return newDataset; - } - - /** - * Scales the values of each feature linearly using the min and max values found in the training set. - * NOTE1: Not guaranteed to be between 0 and 1 for a test datum. - * NOTE2: Also filters out features from the datum that are not seen at training time. - * @param datum - * @return a new datum - */ - public RVFDatum scaleDatum(RVFDatum datum){ - //scale this dataset before scaling the datum - if(minValues == null || maxValues == null) - scaleFeatures(); - Counter scaledFeatures = new ClassicCounter(); - for(F feature : datum.asFeatures()){ - int fID = this.featureIndex.indexOf(feature); - if(fID >= 0){ - double oldVal = datum.asFeaturesCounter().getCount(feature); - double newVal; - if(minValues[fID] != maxValues[fID]) - newVal = (oldVal - minValues[fID])/(maxValues[fID] - minValues[fID]); - else newVal = oldVal; - scaledFeatures.incrementCount(feature, newVal); - } - } - return new RVFDatum(scaledFeatures, datum.label()); - } - - public RVFDataset scaleDatasetGaussian(RVFDataset dataset){ - RVFDataset newDataset = new RVFDataset(this.featureIndex,this.labelIndex); - for(int i = 0; i < dataset.size(); i++){ - RVFDatum datum = dataset.getDatum(i); - newDataset.add(scaleDatumGaussian(datum)); - } - return newDataset; - } - - public RVFDatum scaleDatumGaussian(RVFDatum datum){ - //scale this dataset before scaling the datum - if(means == null || stdevs == null) - scaleFeaturesGaussian(); - Counter scaledFeatures = new ClassicCounter(); - for(F feature : datum.asFeatures()){ - int fID = this.featureIndex.indexOf(feature); - if(fID >= 0){ - double oldVal = datum.asFeaturesCounter().getCount(feature); - double newVal; - if(stdevs[fID] != 0) - newVal = (oldVal - means[fID])/stdevs[fID]; - else newVal = oldVal; - scaledFeatures.incrementCount(feature, newVal); - } - } - return new RVFDatum(scaledFeatures, datum.label()); - } - - @Override - public Pair,GeneralDataset> split(int start, int end) { - int devSize = end - start; - int trainSize = size() - devSize; - - int[][] devData = new int[devSize][]; - double[][] devValues = new double[devSize][]; - int[] devLabels = new int[devSize]; - - int[][] trainData = new int[trainSize][]; - double[][] trainValues = new double[trainSize][]; - int[] trainLabels = new int[trainSize]; - - System.arraycopy(data, start, devData, 0, devSize); - System.arraycopy(values, start, devValues, 0, devSize); - System.arraycopy(labels, start, devLabels, 0, devSize); - - System.arraycopy(data, 0, trainData, 0, start); - System.arraycopy(data, end, trainData, start, size()-end); - System.arraycopy(values, 0, trainValues, 0, start); - System.arraycopy(values, end, trainValues, start, size()-end); - System.arraycopy(labels, 0, trainLabels, 0, start); - System.arraycopy(labels, end, trainLabels, start, size()-end); - - GeneralDataset dev = new RVFDataset(labelIndex, devLabels, featureIndex, devData, devValues); - GeneralDataset train = new RVFDataset(labelIndex, trainLabels, featureIndex, trainData, trainValues); - - return new Pair,GeneralDataset>(train, dev); - - } - - - - - //TODO: Check that this does what we want for Datum other than RVFDatum - @Override - public void add(Datum d) { - if (d instanceof RVFDatum) { - addLabel(d.label()); - addFeatures(((RVFDatum)d).asFeaturesCounter()); - size++; - } else { - addLabel(d.label()); - addFeatures(Counters.asCounter(d.asFeatures())); - size++; - } - } - - - - public void add(Datum d, String src, String id) { - if (d instanceof RVFDatum) { - addLabel(d.label()); - addFeatures(((RVFDatum)d).asFeaturesCounter()); - addSourceAndId(src, id); - size++; - } else { - addLabel(d.label()); - addFeatures(Counters.asCounter(d.asFeatures())); - addSourceAndId(src, id); - size++; - } - } - - - // TODO shouldn't have both this and getRVFDatum - @Override - public RVFDatum getDatum(int index) { - return getRVFDatum(index); - } - - /** - * @return the index-ed datum - * - * Note, this returns a new RVFDatum object, not the original RVFDatum that was added to the dataset. - */ - @Override - public RVFDatum getRVFDatum(int index) { - ClassicCounter c = new ClassicCounter(); - for (int i = 0; i < data[index].length; i++) { - c.incrementCount(featureIndex.get(data[index][i]), values[index][i]); - } - return new RVFDatum(c, labelIndex.get(labels[index])); - } - - public String getRVFDatumSource(int index) { - return sourcesAndIds.get(index).first(); - } - public String getRVFDatumId(int index) { - return sourcesAndIds.get(index).second(); - } - private void addSourceAndId(String src, String id) { - sourcesAndIds.add(new Pair(src, id)); - } - - private void addLabel(L label) { - if (labels.length == size) { - int[] newLabels = new int[size * 2]; - System.arraycopy(labels, 0, newLabels, 0, size); - labels = newLabels; - } - labelIndex.add(label); - labels[size] = labelIndex.indexOf(label); - } - - - private void addFeatures(Counter features) { - if (data.length == size) { - int[][] newData = new int[size * 2][]; - double[][] newValues = new double[size * 2][]; - System.arraycopy(data, 0, newData, 0, size); - System.arraycopy(values, 0, newValues, 0, size); - data = newData; - values = newValues; - } - int[] intFeatures = new int[features.size()]; - double[] featureValues = new double[features.size()]; - - int j = 0; - for (F feature : features.keySet()) { - featureIndex.add(feature); - int fID = featureIndex.indexOf(feature); - if(fID >= 0){ - intFeatures[j] = fID; - featureValues[j] = features.getCount(feature); - j++; - } - } - data[size] = intFeatures; - values[size] = featureValues; - } - - /** - * Resets the Dataset so that it is empty and ready to collect data. - */ - @Override - public void clear() { - clear(10); - } - - /** - * Resets the Dataset so that it is empty and ready to collect data. - */ - @Override - public void clear(int numDatums) { - initialize(numDatums); - } - - @Override - protected void initialize(int numDatums) { - labelIndex = new HashIndex(); - featureIndex = new HashIndex(); - labels = new int[numDatums]; - data = new int[numDatums][]; - values = new double[numDatums][]; - sourcesAndIds = new ArrayList>(numDatums); - size = 0; - } - - - /** - * Prints some summary statistics to stderr for the Dataset. - */ - @Override - public void summaryStatistics() { - System.err.println("numDatums: " + size); - System.err.print("numLabels: " + labelIndex.size() + " ["); - Iterator iter = labelIndex.iterator(); - while (iter.hasNext()) { - System.err.print(iter.next()); - if (iter.hasNext()) { - System.err.print(", "); - } - } - System.err.println("]"); - System.err.println("numFeatures (Phi(X) types): " + featureIndex.size()); - /*for(int i = 0; i < data.length; i++) { - for(int j = 0; j < data[i].length; j++) { - System.out.println(data[i][j]); - } - }*/ - } - -// private int[] trimToSize(int[] i, int size) { -// int[] newI = new int[size]; -// System.arraycopy(i, 0, newI, 0, size); -// return newI; -// } -// -// private int[][] trimToSize(int[][] i, int size) { -// int[][] newI = new int[size][]; -// System.arraycopy(i, 0, newI, 0, size); -// return newI; -// } - - private static double[][] trimToSize(double[][] i, int size) { - double[][] newI = new double[size][]; - System.arraycopy(i, 0, newI, 0, size); - return newI; - } - - - - /** - * prints the full feature matrix in tab-delimited form. These can be BIG - * matrices, so be careful! [Can also use printFullFeatureMatrixWithValues] - */ - public void printFullFeatureMatrix(PrintWriter pw) { - String sep = "\t"; - for (int i = 0; i < featureIndex.size(); i++) { - pw.print(sep + featureIndex.get(i)); - } - pw.println(); - for (int i = 0; i < labels.length; i++) { - pw.print(labelIndex.get(i)); - Set feats = new HashSet(); - for (int j = 0; j < data[i].length; j++) { - int feature = data[i][j]; - feats.add(Integer.valueOf(feature)); - } - for (int j = 0; j < featureIndex.size(); j++) { - if (feats.contains(Integer.valueOf(j))) { - pw.print(sep + "1"); - } else { - pw.print(sep + "0"); - } - } - pw.println(); - } - } - - /** - * Modification of printFullFeatureMatrix to correct bugs & print values (Rajat). - * Prints the full feature matrix in tab-delimited form. These can be BIG - * matrices, so be careful! - */ - public void printFullFeatureMatrixWithValues(PrintWriter pw) { - String sep = "\t"; - for (int i = 0; i < featureIndex.size(); i++) { - pw.print(sep + featureIndex.get(i)); - } - pw.println(); - for (int i = 0; i < size; i++) { // changed labels.length to size - pw.print(labelIndex.get(labels[i])); // changed i to labels[i] - HashMap feats = new HashMap(); - for (int j = 0; j < data[i].length; j++) { - int feature = data[i][j]; - double val = values[i][j]; - feats.put(Integer.valueOf(feature), new Double(val)); - } - for (int j = 0; j < featureIndex.size(); j++) { - if (feats.containsKey(Integer.valueOf(j))) { - pw.print(sep + feats.get(Integer.valueOf(j))); - } else { - pw.print(sep + " "); - } - } - pw.println(); - } - pw.flush(); - } - - /** - * Constructs a Dataset by reading in a file in SVM light format. - * - */ - public static RVFDataset readSVMLightFormat(String filename) { - return readSVMLightFormat(filename, new HashIndex(), new HashIndex()); - } - - /** - * Constructs a Dataset by reading in a file in SVM light format. - * The lines parameter is filled with the lines of the file for further processing - * (if lines is null, it is assumed no line information is desired) - */ - public static RVFDataset readSVMLightFormat(String filename, List lines) { - return readSVMLightFormat(filename, new HashIndex(), new HashIndex(), lines); - } - - /** - * Constructs a Dataset by reading in a file in SVM light format. - * the created dataset has the same feature and label index as given - */ - public static RVFDataset readSVMLightFormat(String filename, Index featureIndex, Index labelIndex) { - return readSVMLightFormat(filename, featureIndex, labelIndex, null); - } - - - /** - * Removes all features from the dataset that are not in featureSet. - * @param featureSet - */ - public void selectFeaturesFromSet(Set featureSet){ - HashIndex newFeatureIndex = new HashIndex(); - int[] featMap = new int[featureIndex.size()]; - Arrays.fill(featMap, -1); - for(F feature : featureSet){ - int oldID = featureIndex.indexOf(feature); - if(oldID >= 0){ //it's a valid feature in the index - int newID = newFeatureIndex.indexOf(feature, true); - featMap[oldID] = newID; - } - } - featureIndex = newFeatureIndex; - for (int i = 0; i < size; i++) { - List featList = new ArrayList(data[i].length); - List valueList = new ArrayList(values[i].length); - for (int j = 0; j < data[i].length; j++) { - if (featMap[data[i][j]] >= 0) { - featList.add(featMap[data[i][j]]); - valueList.add(values[i][j]); - } - } - data[i] = new int[featList.size()]; - values[i] = new double[valueList.size()]; - for (int j = 0; j < data[i].length; j++) { - data[i][j] = featList.get(j); - values[i][j] = valueList.get(j); - } - } - } - - - - /** - * Applies a feature count threshold to the RVFDataset. All features that - * occur fewer than k times are expunged. - */ - public void applyFeatureCountThreshold(int k) { - float[] counts = getFeatureCounts(); - HashIndex newFeatureIndex = new HashIndex(); - - int[] featMap = new int[featureIndex.size()]; - for (int i = 0; i < featMap.length; i++) { - F feat = featureIndex.get(i); - if (counts[i] >= k) { - int newIndex = newFeatureIndex.size(); - newFeatureIndex.add(feat); - featMap[i] = newIndex; - } else { - featMap[i] = -1; - } - // featureIndex.remove(feat); - } - - featureIndex = newFeatureIndex; - // counts = null; // This is unnecessary; JVM can clean it up - - for (int i = 0; i < size; i++) { - List featList = new ArrayList(data[i].length); - List valueList = new ArrayList(values[i].length); - for (int j = 0; j < data[i].length; j++) { - if (featMap[data[i][j]] >= 0) { - featList.add(featMap[data[i][j]]); - valueList.add(values[i][j]); - } - } - data[i] = new int[featList.size()]; - values[i] = new double[valueList.size()]; - for (int j = 0; j < data[i].length; j++) { - data[i][j] = featList.get(j); - values[i][j] = valueList.get(j); - } - } - } - - - /** - * Applies a feature max count threshold to the RVFDataset. All features that - * occur greater than k times are expunged. - */ - public void applyFeatureMaxCountThreshold(int k) { - float[] counts = getFeatureCounts(); - HashIndex newFeatureIndex = new HashIndex(); - - int[] featMap = new int[featureIndex.size()]; - for (int i = 0; i < featMap.length; i++) { - F feat = featureIndex.get(i); - if (counts[i] <= k) { - int newIndex = newFeatureIndex.size(); - newFeatureIndex.add(feat); - featMap[i] = newIndex; - } else { - featMap[i] = -1; - } - // featureIndex.remove(feat); - } - - featureIndex = newFeatureIndex; - // counts = null; // This is unnecessary; JVM can clean it up - - for (int i = 0; i < size; i++) { - List featList = new ArrayList(data[i].length); - List valueList = new ArrayList(values[i].length); - for (int j = 0; j < data[i].length; j++) { - if (featMap[data[i][j]] >= 0) { - featList.add(featMap[data[i][j]]); - valueList.add(values[i][j]); - } - } - data[i] = new int[featList.size()]; - values[i] = new double[valueList.size()]; - for (int j = 0; j < data[i].length; j++) { - data[i][j] = featList.get(j); - values[i][j] = valueList.get(j); - } - } - } - - private static RVFDataset readSVMLightFormat(String filename, Index featureIndex, Index labelIndex, List lines) { - BufferedReader in = null; - RVFDataset dataset; - try { - dataset = new RVFDataset(10, featureIndex, labelIndex); - in = new BufferedReader(new FileReader(filename)); - - while (in.ready()) { - String line = in.readLine(); - if(lines != null) - lines.add(line); - dataset.add(svmLightLineToRVFDatum(line)); - } - - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - IOUtils.closeIgnoringExceptions(in); - } - return dataset; - } - - public static RVFDatum svmLightLineToRVFDatum(String l) { - l = l.replaceAll("#.*", ""); // remove any trailing comments - String[] line = l.split("\\s+"); - ClassicCounter features = new ClassicCounter(); - for (int i = 1; i < line.length; i++) { - String[] f = line[i].split(":"); - double val = Double.parseDouble(f[1]); - features.incrementCount(f[0], val); - } - return new RVFDatum(features, line[0]); - } - - /** - * Read SVM-light formatted data into this dataset. - * - * A strict SVM-light format is expected, where labels and features are both - * encoded as integers. These integers are converted into the dataset label - * and feature types using the indexes stored in this dataset. - * - * @param file The file from which the data should be read. - */ - public void readSVMLightFormat(File file) { - for (String line: IOUtils.readLines(file)) { - line = line.replaceAll("#.*", ""); // remove any trailing comments - String[] items = line.split("\\s+"); - Integer label = Integer.parseInt(items[0]); - Counter features = new ClassicCounter(); - for (int i = 1; i < items.length; i++) { - String[] featureItems = items[i].split(":"); - int feature = Integer.parseInt(featureItems[0]); - double value = Double.parseDouble(featureItems[1]); - features.incrementCount(this.featureIndex.get(feature), value); - } - this.add(new RVFDatum(features, this.labelIndex.get(label))); - } - } - - /** - * Write the dataset in SVM-light format to the file. - * - * A strict SVM-light format will be written, where labels and features are - * both encoded as integers, using the label and feature indexes of this - * datset. Datasets written by this method can be read by - * {@link #readSVMLightFormat(File)}. - * - * @param file The location where the dataset should be written. - */ - public void writeSVMLightFormat(File file) throws FileNotFoundException { - PrintWriter writer = new PrintWriter(file); - try { - for (RVFDatum datum: this) { - writer.print(this.labelIndex.indexOf(datum.label())); - Counter features = datum.asFeaturesCounter(); - for (F feature: features.keySet()) { - double count = features.getCount(feature); - writer.format(" %s:%f", this.featureIndex.indexOf(feature), count); - } - writer.println(); - } - } finally { - writer.close(); - } - } - - /** - * Prints the sparse feature matrix using {@link #printSparseFeatureMatrix(PrintWriter)} - * to {@link System#out System.out}. - */ - public void printSparseFeatureMatrix() { - printSparseFeatureMatrix(new PrintWriter(System.out, true)); - } - - /** - * Prints a sparse feature matrix representation of the Dataset. Prints the actual - * {@link Object#toString()} representations of features. - */ - public void printSparseFeatureMatrix(PrintWriter pw) { - String sep = "\t"; - for (int i = 0; i < size; i++) { - pw.print(labelIndex.get(labels[i])); - int[] datum = data[i]; - for (int feat : datum) { - pw.print(sep); - pw.print(featureIndex.get(feat)); - } - pw.println(); - } - } - - /** - * Prints a sparse feature-value output of the Dataset. Prints the actual - * {@link Object#toString()} representations of features. - * This is probably what you want for RVFDataset since the above two - * methods seem useless and unused. - */ - public void printSparseFeatureValues(PrintWriter pw) { - for (int i = 0; i < size; i++) { - printSparseFeatureValues(i, pw); - } - } - - /** - * Prints a sparse feature-value output of the Dataset. Prints the actual - * {@link Object#toString()} representations of features. - * This is probably what you want for RVFDataset since the above two - * methods seem useless and unused. - */ - public void printSparseFeatureValues(int datumNo, PrintWriter pw) { - pw.print(labelIndex.get(labels[datumNo])); - pw.print('\t'); - pw.println("LABEL"); - int[] datum = data[datumNo]; - double[] vals = values[datumNo]; - assert datum.length == vals.length; - for (int i = 0; i < datum.length; i++) { - pw.print(featureIndex.get(datum[i])); - pw.print('\t'); - pw.println(vals[i]); - } - pw.println(); - } - - public static void main(String[] args) { - RVFDataset data = new RVFDataset(); - ClassicCounter c1 = new ClassicCounter(); - c1.incrementCount("fever", 3.5); - c1.incrementCount("cough", 1.1); - c1.incrementCount("congestion", 4.2); - - ClassicCounter c2 = new ClassicCounter(); - c2.incrementCount("fever", 1.5); - c2.incrementCount("cough", 2.1); - c2.incrementCount("nausea", 3.2); - - ClassicCounter c3 = new ClassicCounter(); - c3.incrementCount("cough", 2.5); - c3.incrementCount("congestion", 3.2); - - data.add(new RVFDatum(c1, "cold")); - data.add(new RVFDatum(c2, "flu")); - data.add(new RVFDatum(c3, "cold")); - data.summaryStatistics(); - - LinearClassifierFactory factory = new LinearClassifierFactory(); - factory.useQuasiNewton(); - - LinearClassifier c = factory.trainClassifier(data); - - ClassicCounter c4 = new ClassicCounter(); - c4.incrementCount("cough", 2.3); - c4.incrementCount("fever", 1.3); - - RVFDatum datum = new RVFDatum(c4); - - c.justificationOf((Datum)datum); - } - - @Override - public double[][] getValuesArray() { - values = trimToSize(values, size); - return values; - } - - - @Override - public String toString() { - return "Dataset of size " + size; - } - - public String toSummaryString() { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - pw.println("Number of data points: " + size()); - - pw.print("Number of labels: " + labelIndex.size() + " ["); - Iterator iter = labelIndex.iterator(); - while (iter.hasNext()) { - pw.print(iter.next()); - if (iter.hasNext()) { - pw.print(", "); - } - } - pw.println("]"); - pw.println("Number of features (Phi(X) types): " + featureIndex.size()); - pw.println("Number of active feature types: " + numFeatureTypes()); - pw.println("Number of active feature tokens: " + numFeatureTokens()); - - return sw.toString(); - } - - /** {@inheritDoc} - */ - public Iterator> iterator() { - return new Iterator>() { - private int index; // = 0; - public boolean hasNext() { - return this.index < size; - } - public RVFDatum next() { - RVFDatum next = getRVFDatum(this.index); - ++this.index; - return next; - } - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - /** - * Randomizes the data array in place - * Needs to be redefined here because we need to randomize the values as well - */ - @Override - public void randomize(int randomSeed) { - Random rand = new Random(randomSeed); - for(int j = size - 1; j > 0; j --){ - int randIndex = rand.nextInt(j); - - int [] tmp = data[randIndex]; - data[randIndex] = data[j]; - data[j] = tmp; - - int tmpl = labels[randIndex]; - labels[randIndex] = labels[j]; - labels[j] = tmpl; - - double [] tmpv = values[randIndex]; - values[randIndex] = values[j]; - values[j] = tmpv; - } - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/SemiSupervisedLogConditionalObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/SemiSupervisedLogConditionalObjectiveFunction.java deleted file mode 100644 index 9112f42..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/SemiSupervisedLogConditionalObjectiveFunction.java +++ /dev/null @@ -1,67 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.optimization.AbstractCachingDiffFunction; - - -/** - * Maximizes the conditional likelihood with a given prior. - * - * @author Jenny Finkel - * @author Sarah Spikes (Templatization) - * @author Ramesh Nallapati (Made the function more general to support other AbstractCachingDiffFunctions involving the summation of two objective functions) - */ - -public class SemiSupervisedLogConditionalObjectiveFunction extends AbstractCachingDiffFunction { - - AbstractCachingDiffFunction objFunc; - //BiasedLogConditionalObjectiveFunction biasedObjFunc; - AbstractCachingDiffFunction biasedObjFunc; - double convexComboFrac = 0.5; - - LogPrior prior; - - public void setPrior(LogPrior prior) { - this.prior = prior; - } - - @Override - public int domainDimension() { - return objFunc.domainDimension(); - } - - @Override - protected void calculate(double[] x) { - if (derivative == null) { - derivative = new double[domainDimension()]; - } - - value = convexComboFrac*objFunc.valueAt(x) + (1.0-convexComboFrac)*biasedObjFunc.valueAt(x); - //value = objFunc.valueAt(x) + biasedObjFunc.valueAt(x); - double[] d1 = objFunc.derivativeAt(x); - double[] d2 = biasedObjFunc.derivativeAt(x); - - for (int i = 0; i < domainDimension(); i++) { - derivative[i] = convexComboFrac*d1[i] + (1.0-convexComboFrac)*d2[i]; - //derivative[i] = d1[i] + d2[i]; - } - if(prior != null) - value += prior.compute(x, derivative); - } - - public SemiSupervisedLogConditionalObjectiveFunction(AbstractCachingDiffFunction objFunc, AbstractCachingDiffFunction biasedObjFunc, LogPrior prior, double convexComboFrac) { - this.objFunc = objFunc; - this.biasedObjFunc = biasedObjFunc; - this.prior = prior; - this.convexComboFrac = convexComboFrac; - if(convexComboFrac < 0 || convexComboFrac > 1.0) - throw new RuntimeException ("convexComboFrac has to lie between 0 and 1 (both inclusive)."); - } - - public SemiSupervisedLogConditionalObjectiveFunction(AbstractCachingDiffFunction objFunc, AbstractCachingDiffFunction biasedObjFunc, LogPrior prior) { - //this.objFunc = objFunc; - //this.biasedObjFunc = biasedObjFunc; - //this.prior = prior; - this(objFunc,biasedObjFunc,prior,0.5); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/WeightedDataset.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/WeightedDataset.java deleted file mode 100644 index f5c4cd9..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/WeightedDataset.java +++ /dev/null @@ -1,110 +0,0 @@ -package edu.stanford.nlp.classify; - -import edu.stanford.nlp.ling.Datum; -import edu.stanford.nlp.util.Index; - -import java.util.Collection; -import java.util.Random; - -/** - * @author Galen Andrew - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - */ -public class WeightedDataset extends Dataset { - /** - * - */ - private static final long serialVersionUID = -5435125789127705430L; - protected float[] weights; - - public WeightedDataset(Index labelIndex, int[] labels, Index featureIndex, int[][] data, int size, float[] weights) { - super(labelIndex, labels, featureIndex, data, data.length); - this.weights = weights; - } - - public WeightedDataset() { - this(10); - } - - public WeightedDataset(int initSize) { - super(initSize); - weights = new float[initSize]; - } - - private float[] trimToSize(float[] i) { - float[] newI = new float[size]; - System.arraycopy(i, 0, newI, 0, size); - return newI; - } - - public float[] getWeights() { - weights = trimToSize(weights); - return weights; - } - - @Override - public float[] getFeatureCounts() { - float[] counts = new float[featureIndex.size()]; - for (int i = 0, m = size; i < m; i++) { - for (int j = 0, n = data[i].length; j < n; j++) { - counts[data[i][j]] += weights[i]; - } - } - return counts; - } - - @Override - public void add(Datum d) { - add(d, 1.0f); - } - - @Override - public void add(Collection features, L label) { - add(features, label, 1.0f); - } - - public void add(Datum d, float weight) { - add(d.asFeatures(), d.label(), weight); - } - - @Override - protected void ensureSize() { - super.ensureSize(); - if (weights.length == size) { - float[] newWeights = new float[size * 2]; - System.arraycopy(weights, 0, newWeights, 0, size); - weights = newWeights; - } - } - - public void add(Collection features, L label, float weight) { - ensureSize(); - addLabel(label); - addFeatures(features); - weights[size++] = weight; - } - - /** - * Randomizes the data array in place - * Needs to be redefined here because we need to randomize the weights as well - */ - @Override - public void randomize(int randomSeed) { - Random rand = new Random(randomSeed); - for(int j = size - 1; j > 0; j --){ - int randIndex = rand.nextInt(j); - - int [] tmp = data[randIndex]; - data[randIndex] = data[j]; - data[j] = tmp; - - int tmpl = labels[randIndex]; - labels[randIndex] = labels[j]; - labels[j] = tmpl; - - float tmpw = weights[randIndex]; - weights[randIndex] = weights[j]; - weights[j] = tmpw; - } - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSA.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSA.java deleted file mode 100644 index cc22b3f..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSA.java +++ /dev/null @@ -1,151 +0,0 @@ -package edu.stanford.nlp.fsm; - -import edu.stanford.nlp.util.Scored; - -import java.io.IOException; -import java.io.Writer; -import java.util.*; - -/** - * DFSA: A class for representing a deterministic finite state automaton - * without epsilon transitions. - * - * @author Dan Klein - * @author Michel Galley (AT&T FSM library format printing) - * @author Sarah Spikes (sdspikes@cs.stanford.edu) - cleanup and filling in types - */ -public final class DFSA implements Scored { - - Object dfsaID; - DFSAState initialState; - - public DFSA(DFSAState initialState, double score) { - this.initialState = initialState; - this.score = score; - } - - public DFSA(DFSAState initialState) { - this.initialState = initialState; - this.score = Double.NaN; - } - - public double score; - - public double score() { - return score; - } - - public DFSAState initialState() { - return initialState; - } - - public void setInitialState(DFSAState initialState) { - this.initialState = initialState; - } - - public Set> states() { - Set> visited = new HashSet>(); - List> toVisit = new ArrayList>(); - toVisit.add(initialState()); - exploreStates(toVisit, visited); - return visited; - } - - private static void exploreStates(List> toVisit, Set> visited) { - while (!toVisit.isEmpty()) { - DFSAState state = toVisit.get(toVisit.size() - 1); - toVisit.remove(toVisit.size() - 1); - if (!visited.contains(state)) { - toVisit.addAll(state.successorStates()); - visited.add(state); - } - } - } - - public DFSA(Object dfsaID) { - this.dfsaID = dfsaID; - this.score = 0; - } - - - private static void printTrieDFSAHelper(DFSAState state, int level) { - if (state.isAccepting()) { - return; - } - Set inputs = state.continuingInputs(); - for (T input : inputs) { - DFSATransition transition = state.transition(input); - System.out.print(level); - System.out.print(input); - for (int i = 0; i < level; i++) { - System.out.print(" "); - } - System.out.print(transition.score()); - System.out.print(" "); - System.out.println(input); - printTrieDFSAHelper(transition.target(), level + 1); - } - } - - public static void printTrieDFSA(DFSA dfsa) { - System.err.println("DFSA: " + dfsa.dfsaID); - printTrieDFSAHelper(dfsa.initialState(), 2); - } - - public void printAttFsmFormat(Writer w) throws IOException { - Queue> q = new LinkedList>(); - Set> visited = new HashSet>(); - q.offer(initialState); - while(q.peek() != null) { - DFSAState state = q.poll(); - if(state == null || visited.contains(state)) - continue; - visited.add(state); - if (state.isAccepting()) { - w.write(state.toString()+"\t"+state.score()+"\n"); - continue; - } - TreeSet inputs = new TreeSet(state.continuingInputs()); - for (T input : inputs) { - DFSATransition transition = state.transition(input); - DFSAState target = transition.target(); - if(!visited.contains(target)) - q.add(target); - w.write(state.toString()+"\t"+target.toString()+"\t"+transition.getInput()+"\t"+transition.score()+"\n"); - } - } - } - - private static void printTrieAsRulesHelper(DFSAState state, String prefix, Writer w) throws IOException { - if (state.isAccepting()) { - return; - } - Set inputs = state.continuingInputs(); - for (T input : inputs) { - DFSATransition transition = state.transition(input); - DFSAState target = transition.target(); - Set inputs2 = target.continuingInputs(); - boolean allTerminate = true; - for (T input2 : inputs2) { - DFSATransition transition2 = target.transition(input2); - DFSAState target2 = transition2.target(); - if (target2.isAccepting()) { - // it's a binary end rule. Print it. - w.write(prefix + " --> " + input + " " + input2 + "\n"); - } else { - allTerminate = false; - } - } - if (!allTerminate) { - // there are some longer continuations. Print continuation rule - String newPrefix = prefix + "_" + input; - w.write(prefix + " --> " + input + " " + newPrefix + "\n"); - printTrieAsRulesHelper(transition.target(), newPrefix, w); - } - } - } - - public static void printTrieAsRules(DFSA dfsa, Writer w) throws IOException { - printTrieAsRulesHelper(dfsa.initialState(), dfsa.dfsaID.toString(), w); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSAState.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSAState.java deleted file mode 100644 index 464604a..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSAState.java +++ /dev/null @@ -1,151 +0,0 @@ -package edu.stanford.nlp.fsm; - -import edu.stanford.nlp.util.Scored; - -import java.util.*; - -/** - * DFSAState - *

- * Class for representing the state of a deterministic finite state - * automaton without epsilon transitions. - * - * @author Dan Klein - * @version 12/14/2000 - * @author Sarah Spikes (sdspikes@cs.stanford.edu) - cleanup and filling in types - * @param stateID type - * @param transition type - */ -public final class DFSAState implements Scored { - - private S stateID; - private Map> inputToTransition; - public boolean accepting; - private DFSA dfsa; - - public double score; - - public double score() { - return score; - } - - public void setScore(double score) { - this.score = score; - } - - - public DFSA dfsa() { - return dfsa; - } - - public void setStateID(S stateID) { - this.stateID = stateID; - } - - public S stateID() { - return stateID; - } - - public void addTransition(DFSATransition transition) { - inputToTransition.put(transition.input(), transition); - } - - public DFSATransition transition(T input) { - return inputToTransition.get(input); - } - - public Collection> transitions() { - return inputToTransition.values(); - } - - public Set continuingInputs() { - return inputToTransition.keySet(); - } - - public Set> successorStates() { - Set> successors = new HashSet>(); - Collection> transitions = inputToTransition.values(); - for (DFSATransition transition : transitions) { - successors.add(transition.getTarget()); - } - return successors; - } - - public void setAccepting(boolean accepting) { - this.accepting = accepting; - } - - public boolean isAccepting() { - return accepting; - } - - public boolean isContinuable() { - return !inputToTransition.isEmpty(); - } - - @Override - public String toString() { - return stateID.toString(); - } - - private int hashCodeCache; // = 0; - - @Override - public int hashCode() { - if (hashCodeCache == 0) { - hashCodeCache = stateID.hashCode() ^ dfsa.hashCode(); - } - return hashCodeCache; - } - - // equals - @SuppressWarnings("unchecked") - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof DFSAState)) { - return false; - } - DFSAState s = (DFSAState) o; - // historically also checked: accepting == s.accepting && - //inputToTransition.equals(s.inputToTransition)) - return dfsa.equals(s.dfsa) && stateID.equals(s.stateID); - } - - public Set> statesReachable() { - Set> visited = new HashSet>(); - List> toVisit = new ArrayList>(); - toVisit.add(this); - exploreStates(toVisit, visited); - return visited; - } - - private void exploreStates(List> toVisit, Set> visited) { - while (!toVisit.isEmpty()) { - DFSAState state = toVisit.get(toVisit.size() - 1); - toVisit.remove(toVisit.size() - 1); - if (!visited.contains(state)) { - toVisit.addAll(state.successorStates()); - visited.add(state); - } - } - } - - public DFSAState(S id, DFSA dfsa) { - this.dfsa = dfsa; - this.stateID = id; - this.accepting = false; - this.inputToTransition = new HashMap>(); - this.score = Double.NEGATIVE_INFINITY; - } - - public DFSAState(S id, DFSA dfsa, double score) { - this(id,dfsa); - setScore(score); - } - - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSATransition.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSATransition.java deleted file mode 100644 index 9f72092..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/fsm/DFSATransition.java +++ /dev/null @@ -1,80 +0,0 @@ -package edu.stanford.nlp.fsm; - -import edu.stanford.nlp.util.Scored; - -/** - * (D)FSA Transition - *

- * Class for representing a transition in a weighted finite state - * transducer. For now, just null out fields that may not apply. - * This should really be FSATransition as there's nothing - * deterministic-specific. If FSA is ever made, this should be - * abstracted. The ID is a descriptor, not a unique ID. - * - * @author Dan Klein - * @version 12/14/00 - */ -public final class DFSATransition implements Scored { - - private Object transitionID; - private DFSAState source; - protected DFSAState target; // used directly in DFSAMinimizer (only) - private double score; - private T input; - private Object output; - - public DFSATransition(Object transitionID, DFSAState source, DFSAState target, T input, Object output, double score) { - this.transitionID = transitionID; - this.source = source; - this.target = target; - this.input = input; - this.output = output; - this.score = score; - } - - public DFSAState getSource() { - return source; - } - - public DFSAState source() { - return source; - } - - public DFSAState getTarget() { - return target; - } - - public DFSAState target() { - return target; - } - - public Object getID() { - return transitionID; - } - - public double score() { - return score; - } - - public T getInput() { - return input; - } - - public T input() { - return input; - } - - public Object getOutput() { - return output; - } - - public Object output() { - return output; - } - - @Override - public String toString() { - return "[" + transitionID + "]" + source + " -" + input + ":" + output + "-> " + target; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/AbstractSequenceClassifier.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/AbstractSequenceClassifier.java deleted file mode 100644 index 2cb0c86..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/AbstractSequenceClassifier.java +++ /dev/null @@ -1,1494 +0,0 @@ -// AbstractSequenceClassifier -- a framework for probabilistic sequence models. -// Copyright (c) 2002-2008 The Board of Trustees of -// The Leland Stanford Junior University. All Rights Reserved. -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -// -// For more information, bug reports, fixes, contact: -// Christopher Manning -// Dept of Computer Science, Gates 1A -// Stanford CA 94305-9010 -// USA -// Support/Questions: java-nlp-user@lists.stanford.edu -// Licensing: java-nlp-support@lists.stanford.edu -// http://nlp.stanford.edu/downloads/crf-classifier.shtml - -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.fsm.DFSA; -import edu.stanford.nlp.io.RegExFileFilter; -import edu.stanford.nlp.ling.CoreAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.*; -import edu.stanford.nlp.ling.CoreLabel; -import edu.stanford.nlp.ling.HasWord; -import edu.stanford.nlp.ling.CoreAnnotations; -import edu.stanford.nlp.objectbank.ObjectBank; -import edu.stanford.nlp.objectbank.ResettableReaderIteratorFactory; -import edu.stanford.nlp.process.CoreLabelTokenFactory; -import edu.stanford.nlp.process.CoreTokenFactory; -import edu.stanford.nlp.sequences.*; -import edu.stanford.nlp.sequences.FeatureFactory; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.stats.Counters; -import edu.stanford.nlp.stats.Sampler; -import edu.stanford.nlp.util.*; - -import java.io.*; -import java.text.DecimalFormat; -import java.text.NumberFormat; -import java.util.*; -import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; - -/** - * This class provides common functionality for (probabilistic) sequence models. - * It is a superclass of our CMM and CRF sequence classifiers, and is even used - * in the (deterministic) NumberSequenceClassifier. See implementing classes for - * more information. - *

- * A full implementation should implement these 5 abstract methods: - * List<CoreLabel> classify(List<CoreLabel> document); void - * train(Collection<List<CoreLabel>> docs); - * printProbsDocument(List<CoreLabel> document); void - * serializeClassifier(String serializePath); void - * loadClassifier(ObjectInputStream in, Properties props) throws IOException, - * ClassCastException, ClassNotFoundException; but a runtime (or rule-based) - * implementation can usefully implement just the first. - * - * @author Jenny Finkel - * @author Dan Klein - * @author Christopher Manning - * @author Dan Cer - * @author sonalg (made the class generic) - */ -public abstract class AbstractSequenceClassifier implements Function { - - public SeqClassifierFlags flags; - public Index classIndex; // = null; - public FeatureFactory featureFactory; - protected IN pad; - private CoreTokenFactory tokenFactory; - protected int windowSize; - // different threads can add or query knownLCWords at the same time, - // so we need a concurrent data structure - protected Set knownLCWords = new ConcurrentHashSet(); - - /** - * Construct a SeqClassifierFlags object based on the passed in properties, - * and then call the other constructor. - * - * @param props - * See SeqClassifierFlags for known properties. - */ - public AbstractSequenceClassifier(Properties props) { - this(new SeqClassifierFlags(props)); - } - - /** - * Initialize the featureFactory and other variables based on the passed in - * flags. - * - * @param flags - * A specification of the AbstractSequenceClassifier to construct. - */ - public AbstractSequenceClassifier(SeqClassifierFlags flags) { - this.flags = flags; - - try { - this.featureFactory = (FeatureFactory) Class.forName(flags.featureFactory).newInstance(); - if (flags.tokenFactory == null) - tokenFactory = (CoreTokenFactory) new CoreLabelTokenFactory(); - else - this.tokenFactory = (CoreTokenFactory) Class.forName(flags.tokenFactory).newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - pad = tokenFactory.makeToken(); - windowSize = flags.maxLeft + 1; - reinit(); - } - - /** - * This method should be called after there have been changes to the flags - * (SeqClassifierFlags) variable, such as after deserializing a classifier. It - * is called inside the loadClassifier methods. It assumes that the flags - * variable and the pad variable exist, but reinitializes things like the pad - * variable, featureFactory and readerAndWriter based on the flags. - *

- * Implementation note: At the moment this variable doesn't set - * windowSize or featureFactory, since they are being serialized separately in - * the file, but we should probably stop serializing them and just - * reinitialize them from the flags? - */ - protected final void reinit() { - pad.set(AnswerAnnotation.class, flags.backgroundSymbol); - pad.set(GoldAnswerAnnotation.class, flags.backgroundSymbol); - - featureFactory.init(flags); - - } - - public DocumentReaderAndWriter - makeReaderAndWriter() { - DocumentReaderAndWriter readerAndWriter; - try { - readerAndWriter = ((DocumentReaderAndWriter) - Class.forName(flags.readerAndWriter).newInstance()); - } catch (Exception e) { - throw new RuntimeException(e); - } - readerAndWriter.init(flags); - return readerAndWriter; - } - - /** - * Returns the background class for the classifier. - * - * @return The background class name - */ - public String backgroundSymbol() { - return flags.backgroundSymbol; - } - - public Set labels() { - return new HashSet(classIndex.objectsList()); - } - - /** - * Classify a List of IN. This method returns a new list of tokens, not - * the list of tokens passed in, and runs the new tokens through - * ObjectBankWrapper. (Both these behaviors are different from that of the - * classify(List) method. - * - * @param sentence The List of IN to be classified. - * @return The classified List of IN, where the classifier output for - * each token is stored in its {@link AnswerAnnotation} field. - */ - public List classifySentence(List sentence) { - List document = new ArrayList(); - int i = 0; - for (HasWord word : sentence) { - IN wi = null; - if (word instanceof CoreMap) { - // copy all annotations! some are required later in - // AbstractSequenceClassifier.classifyWithInlineXML - // wi = (IN) new ArrayCoreMap((ArrayCoreMap) word); - wi = tokenFactory.makeToken((IN) word); - } else { - wi = tokenFactory.makeToken(); - wi.set(CoreAnnotations.TextAnnotation.class, word.word()); - // wi.setWord(word.word()); - } - wi.set(PositionAnnotation.class, Integer.toString(i)); - wi.set(AnswerAnnotation.class, backgroundSymbol()); - document.add(wi); - i++; - } - - // TODO get rid of objectbankwrapper - ObjectBankWrapper wrapper = new ObjectBankWrapper(flags, null, knownLCWords); - wrapper.processDocument(document); - - classify(document); - - return document; - } - - /** - * Classify a List of IN using whatever additional information is passed in globalInfo. - * Used by SUTime (NumberSequenceClassifier), which requires the doc date to resolve relative dates - * - * @param tokenSequence - * The List of IN to be classified. - * @return The classified List of IN, where the classifier output for - * each token is stored in its "answer" field. - */ - public List classifySentenceWithGlobalInformation(List tokenSequence, final CoreMap doc, final CoreMap sentence) { - List document = new ArrayList(); - int i = 0; - for (HasWord word : tokenSequence) { - IN wi = null; - if (word instanceof CoreMap) { - // copy all annotations! some are required later in - // AbstractSequenceClassifier.classifyWithInlineXML - // wi = (IN) new ArrayCoreMap((ArrayCoreMap) word); - wi = tokenFactory.makeToken((IN) word); - } else { - wi = tokenFactory.makeToken(); - wi.set(CoreAnnotations.TextAnnotation.class, word.word()); - // wi.setWord(word.word()); - } - wi.set(PositionAnnotation.class, Integer.toString(i)); - wi.set(AnswerAnnotation.class, backgroundSymbol()); - document.add(wi); - i++; - } - - // TODO get rid of objectbankwrapper - ObjectBankWrapper wrapper = new ObjectBankWrapper(flags, null, knownLCWords); - wrapper.processDocument(document); - - classifyWithGlobalInformation(document, doc, sentence); - - return document; - } - - public SequenceModel getSequenceModel(List doc) { - throw new UnsupportedOperationException(); - } - - public Sampler> getSampler(final List input) { - return new Sampler>() { - SequenceModel model = getSequenceModel(input); - SequenceSampler sampler = new SequenceSampler(); - - public List drawSample() { - int[] sampleArray = sampler.bestSequence(model); - List sample = new ArrayList(); - int i = 0; - for (IN word : input) { - - IN newWord = tokenFactory.makeToken(word); - newWord.set(AnswerAnnotation.class, classIndex.get(sampleArray[i++])); - sample.add(newWord); - } - return sample; - } - }; - } - - public Counter> classifyKBest(List doc, Class> answerField, int k) { - - if (doc.isEmpty()) { - return new ClassicCounter>(); - } - - // TODO get rid of ObjectBankWrapper - // i'm sorry that this is so hideous - JRF - ObjectBankWrapper obw = new ObjectBankWrapper(flags, null, knownLCWords); - doc = obw.processDocument(doc); - - SequenceModel model = getSequenceModel(doc); - - KBestSequenceFinder tagInference = new KBestSequenceFinder(); - Counter bestSequences = tagInference.kBestSequences(model, k); - - Counter> kBest = new ClassicCounter>(); - - for (int[] seq : bestSequences.keySet()) { - List kth = new ArrayList(); - int pos = model.leftWindow(); - for (IN fi : doc) { - IN newFL = tokenFactory.makeToken(fi); - String guess = classIndex.get(seq[pos]); - fi.remove(AnswerAnnotation.class); // because fake answers will get - // added during testing - newFL.set(answerField, guess); - pos++; - kth.add(newFL); - } - kBest.setCount(kth, bestSequences.getCount(seq)); - } - - return kBest; - } - - public DFSA getViterbiSearchGraph(List doc, Class> answerField) { - if (doc.isEmpty()) { - return new DFSA(null); - } - // TODO get rid of objectbankwrapper - ObjectBankWrapper obw = new ObjectBankWrapper(flags, null, knownLCWords); - doc = obw.processDocument(doc); - SequenceModel model = getSequenceModel(doc); - return ViterbiSearchGraphBuilder.getGraph(model, classIndex); - } - - /** - * Classify the tokens in a String. Each sentence becomes a separate document. - * - * @param str - * A String with tokens in one or more sentences of text to be - * classified. - * @return {@link List} of classified sentences (each a List of something that - * extends {@link CoreMap}). - */ - public List> classify(String str) { - DocumentReaderAndWriter readerAndWriter = - new PlainTextDocumentReaderAndWriter(); - readerAndWriter.init(flags); - ObjectBank> documents = - makeObjectBankFromString(str, readerAndWriter); - List> result = new ArrayList>(); - - for (List document : documents) { - classify(document); - - List sentence = new ArrayList(); - for (IN wi : document) { - // TaggedWord word = new TaggedWord(wi.word(), wi.answer()); - // sentence.add(word); - sentence.add(wi); - } - result.add(sentence); - } - return result; - } - - /** - * Classify the tokens in a String. Each sentence becomes a separate document. - * Doesn't override default readerAndWriter. - * - * @param str - * A String with tokens in one or more sentences of text to be - * classified. - * @return {@link List} of classified sentences (each a List of something that - * extends {@link CoreMap}). - */ - public List> classifyRaw(String str, - DocumentReaderAndWriter readerAndWriter) { - ObjectBank> documents = - makeObjectBankFromString(str, readerAndWriter); - List> result = new ArrayList>(); - - for (List document : documents) { - classify(document); - - List sentence = new ArrayList(); - for (IN wi : document) { - // TaggedWord word = new TaggedWord(wi.word(), wi.answer()); - // sentence.add(word); - sentence.add(wi); - } - result.add(sentence); - } - return result; - } - - /** - * Classify the contents of a file. - * - * @param filename - * Contains the sentence(s) to be classified. - * @return {@link List} of classified List of IN. - */ - public List> classifyFile(String filename) { - DocumentReaderAndWriter readerAndWriter = - new PlainTextDocumentReaderAndWriter(); - readerAndWriter.init(flags); - ObjectBank> documents = - makeObjectBankFromFile(filename, readerAndWriter); - List> result = new ArrayList>(); - - for (List document : documents) { - // System.err.println(document); - classify(document); - - List sentence = new ArrayList(); - for (IN wi : document) { - sentence.add(wi); - // System.err.println(wi); - } - result.add(sentence); - } - return result; - } - - /** - * Maps a String input to an XML-formatted rendition of applying NER to the - * String. Implements the Function interface. Calls - * classifyWithInlineXML(String) [q.v.]. - */ - public String apply(String in) { - return classifyWithInlineXML(in); - } - - /** - * Classify the contents of a {@link String} to one of several String - * representations that shows the classes. Plain text or XML input is expected - * and the {@link PlainTextDocumentReaderAndWriter} is used. The classifier - * will tokenize the text and treat each sentence as a separate document. The - * output can be specified to be in a choice of three formats: slashTags - * (e.g., Bill/PERSON Smith/PERSON died/O ./O), inlineXML (e.g., - * <PERSON>Bill Smith</PERSON> went to - * <LOCATION>Paris</LOCATION> .), or xml, for stand-off XML (e.g., - * <wi num="0" entity="PERSON">Sue</wi> <wi num="1" - * entity="O">shouted</wi> ). There is also a binary choice as to - * whether the spacing between tokens of the original is preserved or whether - * the (tagged) tokens are printed with a single space (for inlineXML or - * slashTags) or a single newline (for xml) between each one. - *

- * Fine points: The slashTags and xml formats show tokens as - * transformed by any normalization processes inside the tokenizer, while - * inlineXML shows the tokens exactly as they appeared in the source text. - * When a period counts as both part of an abbreviation and as an end of - * sentence marker, it is included twice in the output String for slashTags or - * xml, but only once for inlineXML, where it is not counted as part of the - * abbreviation (or any named entity it is part of). For slashTags with - * preserveSpacing=true, there will be two successive periods such as "Jr.." - * The tokenized (preserveSpacing=false) output will have a space or a newline - * after the last token. - * - * @param sentences - * The String to be classified. It will be tokenized and - * divided into documents according to (heuristically - * determined) sentence boundaries. - * @param outputFormat - * The format to put the output in: one of "slashTags", "xml", or - * "inlineXML" - * @param preserveSpacing - * Whether to preserve the input spacing between tokens, which may - * sometimes be none (true) or whether to tokenize the text and print - * it with one space between each token (false) - * @return A {@link String} with annotated with classification information. - */ - public String classifyToString(String sentences, String outputFormat, boolean preserveSpacing) { - PlainTextDocumentReaderAndWriter.OutputStyle outFormat = - PlainTextDocumentReaderAndWriter.OutputStyle.fromShortName(outputFormat); - - PlainTextDocumentReaderAndWriter readerAndWriter = - new PlainTextDocumentReaderAndWriter(); - readerAndWriter.init(flags); - - ObjectBank> documents = - makeObjectBankFromString(sentences, readerAndWriter); - - StringBuilder sb = new StringBuilder(); - for (List doc : documents) { - List docOutput = classify(doc); - sb.append(readerAndWriter.getAnswers(docOutput, outFormat, - preserveSpacing)); - } - return sb.toString(); - } - - /** - * Classify the contents of a {@link String}. Plain text or XML is expected - * and the {@link PlainTextDocumentReaderAndWriter} is used. The classifier - * will treat each sentence as a separate document. The output can be - * specified to be in a choice of formats: Output is in inline XML format - * (e.g. <PERSON>Bill Smith</PERSON> went to - * <LOCATION>Paris</LOCATION> .) - * - * @param sentences - * The string to be classified - * @return A {@link String} with annotated with classification information. - */ - public String classifyWithInlineXML(String sentences) { - return classifyToString(sentences, "inlineXML", true); - } - - /** - * Classify the contents of a String to a tagged word/class String. Plain text - * or XML input is expected and the {@link PlainTextDocumentReaderAndWriter} - * is used. Output looks like: My/O name/O is/O Bill/PERSON Smith/PERSON ./O - * - * @param sentences - * The String to be classified - * @return A String annotated with classification information. - */ - public String classifyToString(String sentences) { - return classifyToString(sentences, "slashTags", true); - } - - /** - * Classify the contents of a {@link String} to classified character offset - * spans. Plain text or XML input text is expected and the - * {@link PlainTextDocumentReaderAndWriter} is used. Output is a (possibly - * empty, but not null) List of Triples. Each Triple is an entity - * name, followed by beginning and ending character offsets in the original - * String. Character offsets can be thought of as fenceposts between the - * characters, or, like certain methods in the Java String class, as character - * positions, numbered starting from 0, with the end index pointing to the - * position AFTER the entity ends. That is, end - start is the length of the - * entity in characters. - *

- * Fine points: Token offsets are true wrt the source text, even though - * the tokenizer may internally normalize certain tokens to String - * representations of different lengths (e.g., " becoming `` or ''). When a - * period counts as both part of an abbreviation and as an end of sentence - * marker, and that abbreviation is part of a named entity, the reported - * entity string excludes the period. - * - * @param sentences - * The string to be classified - * @return A {@link List} of {@link Triple}s, each of which gives an entity - * type and the beginning and ending character offsets. - */ - public List> classifyToCharacterOffsets(String sentences) { - DocumentReaderAndWriter readerAndWriter = - new PlainTextDocumentReaderAndWriter(); - readerAndWriter.init(flags); - ObjectBank> documents = - makeObjectBankFromString(sentences, readerAndWriter); - - List> entities = - new ArrayList>(); - for (List doc : documents) { - String prevEntityType = flags.backgroundSymbol; - Triple prevEntity = null; - - classify(doc); - - for (IN fl : doc) { - String guessedAnswer = fl.get(AnswerAnnotation.class); - if (guessedAnswer.equals(flags.backgroundSymbol)) { - if (prevEntity != null) { - entities.add(prevEntity); - prevEntity = null; - } - } else { - if (!guessedAnswer.equals(prevEntityType)) { - if (prevEntity != null) { - entities.add(prevEntity); - } - prevEntity = new Triple(guessedAnswer, fl - .get(CharacterOffsetBeginAnnotation.class), fl.get(CharacterOffsetEndAnnotation.class)); - } else { - assert prevEntity != null; // if you read the code carefully, this - // should always be true! - prevEntity.setThird(fl.get(CharacterOffsetEndAnnotation.class)); - } - } - prevEntityType = guessedAnswer; - } - - // include any entity at end of doc - if (prevEntity != null) { - entities.add(prevEntity); - } - - } - return entities; - } - - /** - * ONLY USE IF LOADED A CHINESE WORD SEGMENTER!!!!! - * - * @param sentence - * The string to be classified - * @return List of words - */ - public List segmentString(String sentence) { - // TODO: creating this object is annoying, is there any good way - // around it? Why can't we just use a - // PlainTextDocumentReaderAndWriter? - return segmentString(sentence, makeReaderAndWriter()); - } - - public List segmentString(String sentence, - DocumentReaderAndWriter readerAndWriter) { - ObjectBank> docs = makeObjectBankFromString(sentence, - readerAndWriter); - - StringWriter stringWriter = new StringWriter(); - PrintWriter stringPrintWriter = new PrintWriter(stringWriter); - for (List doc : docs) { - classify(doc); - readerAndWriter.printAnswers(doc, stringPrintWriter); - stringPrintWriter.println(); - } - stringPrintWriter.close(); - String segmented = stringWriter.toString(); - - return Arrays.asList(segmented.split("\\s")); - } - - /** - * Classify the contents of {@link SeqClassifierFlags scf.testFile}. The file - * should be in the format expected based on {@link SeqClassifierFlags - * scf.documentReader}. - * - * @return A {@link List} of {@link List}s of classified something that - * extends {@link CoreMap} where each {@link List} refers to a - * document/sentence. - */ - // public ObjectBank> test() { - // return test(flags.testFile); - // } - - /** - * Classify the contents of a file. The file should be in the format expected - * based on {@link SeqClassifierFlags scf.documentReader} if the file is - * specified in {@link SeqClassifierFlags scf.testFile}. If the file being - * read is from {@link SeqClassifierFlags scf.textFile} then the - * {@link PlainTextDocumentReaderAndWriter} is used. - * - * @param filename - * The path to the specified file - * @return A {@link List} of {@link List}s of classified {@link IN}s where - * each {@link List} refers to a document/sentence. - */ - // public ObjectBank> test(String filename) { - // // only for the OCR data does this matter - // flags.ocrTrain = false; - - // ObjectBank> docs = makeObjectBank(filename); - // return testDocuments(docs); - // } - - /** - * Classify a {@link List} of something that extends{@link CoreMap}. - * The classifications are added in place to the items of the document, - * which is also returned by this method - * - * @param document A {@link List} of something that extends {@link CoreMap}. - * @return The same {@link List}, but with the elements annotated with their - * answers (stored under the {@link AnswerAnnotation} key). - */ - public abstract List classify(List document); - - /** - * Classify a {@link List} of something that extends{@link CoreMap} using as additional information whatever is stored in globalInfo. - * This needed for SUTime (NumberSequenceClassifier), which requires the document date to resolve relative dates - * @param document - * @param globalInfo - * @return - */ - public abstract List classifyWithGlobalInformation(List tokenSequence, final CoreMap document, final CoreMap sentence); - - /** - * Train the classifier based on values in flags. It will use the first of - * these variables that is defined: trainFiles (and baseTrainDir), - * trainFileList, trainFile. - */ - public void train() { - if (flags.trainFiles != null) { - train(flags.baseTrainDir, flags.trainFiles, makeReaderAndWriter()); - } else if (flags.trainFileList != null) { - String[] files = flags.trainFileList.split(","); - train(files, makeReaderAndWriter()); - } else { - train(flags.trainFile, makeReaderAndWriter()); - } - } - - public void train(String filename) { - train(filename, makeReaderAndWriter()); - } - - public void train(String filename, - DocumentReaderAndWriter readerAndWriter) { - // only for the OCR data does this matter - flags.ocrTrain = true; - train(makeObjectBankFromFile(filename, readerAndWriter), readerAndWriter); - } - - public void train(String baseTrainDir, String trainFiles, - DocumentReaderAndWriter readerAndWriter) { - // only for the OCR data does this matter - flags.ocrTrain = true; - train(makeObjectBankFromFiles(baseTrainDir, trainFiles, readerAndWriter), - readerAndWriter); - } - - public void train(String[] trainFileList, - DocumentReaderAndWriter readerAndWriter) { - // only for the OCR data does this matter - flags.ocrTrain = true; - train(makeObjectBankFromFiles(trainFileList, readerAndWriter), - readerAndWriter); - } - - /** - * Trains a classifier from a Collection of sequences. - * Note that the Collection can be (and usually is) an ObjectBank. - * - * @param docs - * An Objectbank or a collection of sequences of IN - */ - public void train(Collection> docs) { - train(docs, makeReaderAndWriter()); - } - - /** - * Trains a classifier from a Collection of sequences. - * Note that the Collection can be (and usually is) an ObjectBank. - * - * @param docs - * An Objectbank or a collection of sequences of IN - * @param readerAndWriter - * A DocumentReaderAndWriter to use when loading test files - */ - public abstract void train(Collection> docs, - DocumentReaderAndWriter readerAndWriter); - - /** - * Reads a String into an ObjectBank object. NOTE: that the current - * implementation of ReaderIteratorFactory will first try to interpret each - * string as a filename, so this method will yield unwanted results if it - * applies to a string that is at the same time a filename. It prints out a - * warning, at least. - * - * @param string - * The String which will be the content of the ObjectBank - * @return The ObjectBank - */ - public ObjectBank> - makeObjectBankFromString(String string, - DocumentReaderAndWriter readerAndWriter) - { - if (flags.announceObjectBankEntries) { - System.err.print("Reading data using "); - System.err.println(flags.readerAndWriter); - - if (flags.inputEncoding == null) { - System.err.println("Getting data from " + string + " (default encoding)"); - } else { - System.err.println("Getting data from " + string + " (" + flags.inputEncoding + " encoding)"); - } - } - // return new ObjectBank>(new - // ResettableReaderIteratorFactory(string), readerAndWriter); - // TODO - return new ObjectBankWrapper(flags, new ObjectBank>(new ResettableReaderIteratorFactory(string), - readerAndWriter), knownLCWords); - } - - public ObjectBank> makeObjectBankFromFile(String filename, - DocumentReaderAndWriter readerAndWriter) { - String[] fileAsArray = { filename }; - return makeObjectBankFromFiles(fileAsArray, readerAndWriter); - } - - public ObjectBank> makeObjectBankFromFiles(String[] trainFileList, - DocumentReaderAndWriter readerAndWriter) { - // try{ - Collection files = new ArrayList(); - for (String trainFile : trainFileList) { - File f = new File(trainFile); - files.add(f); - } - // System.err.printf("trainFileList contains %d file%s.\n", files.size(), - // files.size() == 1 ? "": "s"); - // TODO get rid of objectbankwrapper - // return new ObjectBank>(new - // ResettableReaderIteratorFactory(files), readerAndWriter); - return new ObjectBankWrapper(flags, new ObjectBank>(new ResettableReaderIteratorFactory(files, flags.inputEncoding), - readerAndWriter), knownLCWords); - // } catch (IOException e) { - // throw new RuntimeException(e); - // } - } - - public ObjectBank> makeObjectBankFromFiles(String baseDir, String filePattern, DocumentReaderAndWriter readerAndWriter) { - - File path = new File(baseDir); - FileFilter filter = new RegExFileFilter(Pattern.compile(filePattern)); - File[] origFiles = path.listFiles(filter); - Collection files = new ArrayList(); - for (File file : origFiles) { - if (file.isFile()) { - if (flags.announceObjectBankEntries) { - System.err.println("Getting data from " + file + " (" + flags.inputEncoding + " encoding)"); - } - files.add(file); - } - } - - if (files.isEmpty()) { - throw new RuntimeException("No matching files: " + baseDir + '\t' + filePattern); - } - // return new ObjectBank>(new - // ResettableReaderIteratorFactory(files, flags.inputEncoding), - // readerAndWriter); - // TODO get rid of objectbankwrapper - return new ObjectBankWrapper(flags, new ObjectBank>(new ResettableReaderIteratorFactory(files, - flags.inputEncoding), readerAndWriter), knownLCWords); - } - - public ObjectBank> makeObjectBankFromFiles(Collection files, - DocumentReaderAndWriter readerAndWriter) { - if (files.isEmpty()) { - throw new RuntimeException("Attempt to make ObjectBank with empty file list"); - } - // return new ObjectBank>(new - // ResettableReaderIteratorFactory(files, flags.inputEncoding), - // readerAndWriter); - // TODO get rid of objectbankwrapper - return new ObjectBankWrapper(flags, new ObjectBank>(new ResettableReaderIteratorFactory(files, - flags.inputEncoding), readerAndWriter), knownLCWords); - } - - /** - * Set up an ObjectBank that will allow one to iterate over a collection of - * documents obtained from the passed in Reader. Each document will be - * represented as a list of IN. If the ObjectBank iterator() is called until - * hasNext() returns false, then the Reader will be read till end of file, but - * no reading is done at the time of this call. Reading is done using the - * reading method specified in flags.documentReader, and for some - * reader choices, the column mapping given in flags.map. - * - * @param in - * Input data addNEWLCWords do we add new lowercase words from this - * data to the word shape classifier - * @return The list of documents - */ - public ObjectBank> makeObjectBankFromReader(BufferedReader in, - DocumentReaderAndWriter readerAndWriter) { - if (flags.announceObjectBankEntries) { - System.err.println("Reading data using " + readerAndWriter.getClass()); - } - // TODO get rid of objectbankwrapper - // return new ObjectBank>(new ResettableReaderIteratorFactory(in), - // readerAndWriter); - return new ObjectBankWrapper(flags, new ObjectBank>(new ResettableReaderIteratorFactory(in), - readerAndWriter), knownLCWords); - } - - /** - * Takes the file, reads it in, and prints out the likelihood of each possible - * label at each point. - * - * @param filename - * The path to the specified file - */ - public void printProbs(String filename, - DocumentReaderAndWriter readerAndWriter) { - // only for the OCR data does this matter - flags.ocrTrain = false; - - ObjectBank> docs = - makeObjectBankFromFile(filename, readerAndWriter); - printProbsDocuments(docs); - } - - /** - * Takes a {@link List} of documents and prints the likelihood of each - * possible label at each point. - * - * @param documents - * A {@link List} of {@link List} of something that extends - * {@link CoreMap}. - */ - public void printProbsDocuments(ObjectBank> documents) { - for (List doc : documents) { - printProbsDocument(doc); - System.out.println(); - } - } - - public abstract void printProbsDocument(List document); - - /** - * Load a test file, run the classifier on it, and then print the answers to - * stdout (with timing to stderr). This uses the value of flags.documentReader - * to determine testFile format. - * - * @param testFile - * The file to test on. - */ - public void classifyAndWriteAnswers(String testFile, - DocumentReaderAndWriter readerWriter) - throws IOException - { - ObjectBank> documents = - makeObjectBankFromFile(testFile, readerWriter); - classifyAndWriteAnswers(documents, readerWriter); - } - - public void classifyAndWriteAnswers(String testFile, OutputStream outStream, - DocumentReaderAndWriter readerWriter) - throws IOException - { - ObjectBank> documents = - makeObjectBankFromFile(testFile, readerWriter); - classifyAndWriteAnswers(documents, outStream, readerWriter); - } - - public void classifyAndWriteAnswers(String baseDir, String filePattern, - DocumentReaderAndWriter readerWriter) - throws IOException - { - ObjectBank> documents = - makeObjectBankFromFiles(baseDir, filePattern, readerWriter); - classifyAndWriteAnswers(documents, readerWriter); - } - - public void classifyAndWriteAnswers(Collection testFiles, - DocumentReaderAndWriter readerWriter) - throws IOException - { - ObjectBank> documents = - makeObjectBankFromFiles(testFiles, readerWriter); - classifyAndWriteAnswers(documents, readerWriter); - } - - private void classifyAndWriteAnswers(ObjectBank> documents, - DocumentReaderAndWriter readerWriter) - throws IOException - { - classifyAndWriteAnswers(documents, System.out, readerWriter); - } - - public void classifyAndWriteAnswers(ObjectBank> documents, - OutputStream outStream, - DocumentReaderAndWriter readerWriter) - throws IOException - { - Timing timer = new Timing(); - - Counter entityTP = new ClassicCounter(); - Counter entityFP = new ClassicCounter(); - Counter entityFN = new ClassicCounter(); - boolean resultsCounted = true; - - int numWords = 0; - int numDocs = 0; - for (List doc : documents) { - classify(doc); - numWords += doc.size(); - writeAnswers(doc, outStream, readerWriter); - resultsCounted = (resultsCounted && - countResults(doc, entityTP, entityFP, entityFN)); - numDocs++; - } - long millis = timer.stop(); - double wordspersec = numWords / (((double) millis) / 1000); - NumberFormat nf = new DecimalFormat("0.00"); // easier way! - System.err.println(StringUtils.getShortClassName(this) + - " tagged " + numWords + " words in " + numDocs + - " documents at " + nf.format(wordspersec) + - " words per second."); - if (resultsCounted) { - printResults(entityTP, entityFP, entityFN); - } - } - - /** - * Load a test file, run the classifier on it, and then print the answers to - * stdout (with timing to stderr). This uses the value of flags.documentReader - * to determine testFile format. - * - * @param testFile - * The file to test on. - */ - public void classifyAndWriteAnswersKBest(String testFile, int k, DocumentReaderAndWriter readerAndWriter) throws Exception { - Timing timer = new Timing(); - ObjectBank> documents = - makeObjectBankFromFile(testFile, readerAndWriter); - int numWords = 0; - int numSentences = 0; - - for (List doc : documents) { - Counter> kBest = classifyKBest(doc, AnswerAnnotation.class, k); - numWords += doc.size(); - List> sorted = Counters.toSortedList(kBest); - int n = 1; - for (List l : sorted) { - System.out.println(""); - n++; - } - numSentences++; - } - - long millis = timer.stop(); - double wordspersec = numWords / (((double) millis) / 1000); - NumberFormat nf = new DecimalFormat("0.00"); // easier way! - System.err.println(this.getClass().getName() + " tagged " + numWords + " words in " + numSentences - + " documents at " + nf.format(wordspersec) + " words per second."); - } - - /** - * Load a test file, run the classifier on it, and then write a Viterbi search - * graph for each sequence. - * - * @param testFile - * The file to test on. - */ - public void classifyAndWriteViterbiSearchGraph(String testFile, String searchGraphPrefix, DocumentReaderAndWriter readerAndWriter) throws Exception { - Timing timer = new Timing(); - ObjectBank> documents = - makeObjectBankFromFile(testFile, readerAndWriter); - int numWords = 0; - int numSentences = 0; - - for (List doc : documents) { - DFSA tagLattice = getViterbiSearchGraph(doc, AnswerAnnotation.class); - numWords += doc.size(); - PrintWriter latticeWriter = new PrintWriter(new FileOutputStream(searchGraphPrefix + '.' + numSentences - + ".wlattice")); - PrintWriter vsgWriter = new PrintWriter(new FileOutputStream(searchGraphPrefix + '.' + numSentences + ".lattice")); - if (readerAndWriter instanceof LatticeWriter) - ((LatticeWriter) readerAndWriter).printLattice(tagLattice, doc, latticeWriter); - tagLattice.printAttFsmFormat(vsgWriter); - latticeWriter.close(); - vsgWriter.close(); - numSentences++; - } - - long millis = timer.stop(); - double wordspersec = numWords / (((double) millis) / 1000); - NumberFormat nf = new DecimalFormat("0.00"); // easier way! - System.err.println(this.getClass().getName() + " tagged " + numWords + " words in " + numSentences - + " documents at " + nf.format(wordspersec) + " words per second."); - } - - /** - * Write the classifications of the Sequence classifier out to stdout in a - * format determined by the DocumentReaderAndWriter used. If the flag - * outputEncoding is defined, the output is written in that - * character encoding, otherwise in the system default character encoding. - * - * @param doc - * Documents to write out - * @param outStream - * outstream to use - * @throws IOException - * If an IO problem - */ - public void writeAnswers(List doc, OutputStream outStream, - DocumentReaderAndWriter readerAndWriter) - throws IOException - { - if (flags.lowerNewgeneThreshold) { - return; - } - if (flags.numRuns <= 1) { - PrintWriter out; - if (flags.outputEncoding == null) { - out = new PrintWriter(outStream, true); - } else { - out = new PrintWriter(new OutputStreamWriter(outStream, flags.outputEncoding), true); - } - readerAndWriter.printAnswers(doc, out); - // out.println(); - out.flush(); - } - } - - /** - * Count the successes and failures of the model on the given document. - * Fills numbers in to counters for true positives, false positives, - * and false negatives, and also keeps track of the entities seen. - *
- * Returns false if we ever encounter null for gold or guess. - */ - public boolean countResults(List doc, Counter entityTP, - Counter entityFP, - Counter entityFN) { - int index = 0; - int goldIndex = 0, guessIndex = 0; - String lastGold = "O", lastGuess = "O"; - - // As we go through the document, there are two events we might be - // interested in. One is when a gold entity ends, and the other - // is when a guessed entity ends. If the gold and guessed - // entities end at the same time, started at the same time, and - // match entity type, we have a true positive. Otherwise we - // either have a false positive or a false negative. - for (IN line : doc) { - String gold = line.get(GoldAnswerAnnotation.class); - String guess = line.get(AnswerAnnotation.class); - - if (gold == null || guess == null) - return false; - - if (!lastGold.equals(gold) && !lastGold.equals("O")) { - if (lastGuess.equals(lastGold) && !lastGuess.equals(guess) && - goldIndex == guessIndex) { - entityTP.incrementCount(lastGold, 1.0); - } else { - entityFN.incrementCount(lastGold, 1.0); - } - } - - if (!lastGuess.equals(guess) && !lastGuess.equals("O")) { - if (lastGuess.equals(lastGold) && !lastGuess.equals(guess) && - goldIndex == guessIndex && !lastGold.equals(gold)) { - // correct guesses already tallied - // only need to tally false positives - } else { - entityFP.incrementCount(lastGuess, 1.0); - } - } - - if (!lastGold.equals(gold)) { - lastGold = gold; - goldIndex = index; - } - - if (!lastGuess.equals(guess)) { - lastGuess = guess; - guessIndex = index; - } - ++index; - } - - // We also have to account for entities at the very end of the - // document, since the above logic only occurs when we see - // something that tells us an entity has ended - if (!lastGold.equals("O")) { - if (lastGold.equals(lastGuess) && goldIndex == guessIndex) { - entityTP.incrementCount(lastGold, 1.0); - } else { - entityFN.incrementCount(lastGold, 1.0); - } - } - if (!lastGuess.equals("O")) { - if (lastGold.equals(lastGuess) && goldIndex == guessIndex) { - // correct guesses already tallied - } else { - entityFP.incrementCount(lastGuess, 1.0); - } - } - return true; - } - - /** - * Given counters of true positives, false positives, and false - * negatives, prints out precision, recall, and f1 for each key. - */ - public void printResults(Counter entityTP, Counter entityFP, - Counter entityFN) { - Set entities = new TreeSet(); - entities.addAll(entityTP.keySet()); - entities.addAll(entityFP.keySet()); - entities.addAll(entityFN.keySet()); - boolean printedHeader = false; - for (String entity : entities) { - double tp = entityTP.getCount(entity); - double fp = entityFP.getCount(entity); - double fn = entityFN.getCount(entity); - if (tp == 0.0 && (fp == 0.0 || fn == 0.0)) - continue; - double precision = tp / (tp + fp); - double recall = tp / (tp + fn); - double f1 = ((precision == 0.0 || recall == 0.0) ? - 0.0 : 2.0 / (1.0 / precision + 1.0 / recall)); - if (!printedHeader) { - System.err.println(" Entity\tP\tR\tF1\tTP\tFP\tFN"); - printedHeader = true; - } - System.err.format("%15s\t%.4f\t%.4f\t%.4f\t%.0f\t%.0f\t%.0f\n", - entity, precision, recall, f1, - tp, fp, fn); - } - } - - /** - * Serialize a sequence classifier to a file on the given path. - * - * @param serializePath - * The path/filename to write the classifier to. - */ - public abstract void serializeClassifier(String serializePath); - - /** - * Loads a classifier from the given input stream. The JVM shuts down - * (System.exit(1)) if there is an exception. This does not close the - * InputStream. - * - * @param in - * The InputStream to read from - */ - public void loadClassifierNoExceptions(InputStream in, Properties props) { - // load the classifier - try { - loadClassifier(in, props); - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - - /** - * Load a classifier from the specified InputStream. No extra properties are - * supplied. This does not close the InputStream. - * - * @param in - * The InputStream to load the serialized classifier from - * - * @throws IOException - * If there are problems accessing the input stream - * @throws ClassCastException - * If there are problems interpreting the serialized data - * @throws ClassNotFoundException - * If there are problems interpreting the serialized data - */ - public void loadClassifier(InputStream in) throws IOException, ClassCastException, ClassNotFoundException { - loadClassifier(in, null); - } - - /** - * Load a classifier from the specified InputStream. The classifier is - * reinitialized from the flags serialized in the classifier. This does not - * close the InputStream. - * - * @param in - * The InputStream to load the serialized classifier from - * @param props - * This Properties object will be used to update the - * SeqClassifierFlags which are read from the serialized classifier - * - * @throws IOException - * If there are problems accessing the input stream - * @throws ClassCastException - * If there are problems interpreting the serialized data - * @throws ClassNotFoundException - * If there are problems interpreting the serialized data - */ - public void loadClassifier(InputStream in, Properties props) throws IOException, ClassCastException, - ClassNotFoundException { - loadClassifier(new ObjectInputStream(in), props); - } - - /** - * Load a classifier from the specified input stream. The classifier is - * reinitialized from the flags serialized in the classifier. - * - * @param in - * The InputStream to load the serialized classifier from - * @param props - * This Properties object will be used to update the - * SeqClassifierFlags which are read from the serialized classifier - * - * @throws IOException - * If there are problems accessing the input stream - * @throws ClassCastException - * If there are problems interpreting the serialized data - * @throws ClassNotFoundException - * If there are problems interpreting the serialized data - */ - public abstract void loadClassifier(ObjectInputStream in, Properties props) throws IOException, ClassCastException, - ClassNotFoundException; - - private InputStream loadStreamFromClasspath(String path) { - InputStream is = getClass().getClassLoader().getResourceAsStream(path); - if (is == null) - return null; - try { - if (path.endsWith(".gz")) - is = new GZIPInputStream(new BufferedInputStream(is)); - else - is = new BufferedInputStream(is); - } catch (IOException e) { - System.err.println("CLASSPATH resource " + path + " is not a GZIP stream!"); - } - return is; - } - - /** - * Loads a classifier from the file specified by loadPath. If loadPath ends in - * .gz, uses a GZIPInputStream, else uses a regular FileInputStream. - */ - public void loadClassifier(String loadPath) throws ClassCastException, IOException, ClassNotFoundException { - loadClassifier(loadPath, null); - } - - /** - * Loads a classifier from the file specified by loadPath. If loadPath ends in - * .gz, uses a GZIPInputStream, else uses a regular FileInputStream. - */ - public void loadClassifier(String loadPath, Properties props) throws ClassCastException, IOException, ClassNotFoundException { - InputStream is; - // ms, 10-04-2010: check first is this path exists in our CLASSPATH. This - // takes priority over the file system. - if ((is = loadStreamFromClasspath(loadPath)) != null) { - Timing.startDoing("Loading classifier from " + loadPath); - loadClassifier(is); - is.close(); - Timing.endDoing(); - } else { - loadClassifier(new File(loadPath), props); - } - } - - public void loadClassifierNoExceptions(String loadPath) { - loadClassifierNoExceptions(loadPath, null); - } - - public void loadClassifierNoExceptions(String loadPath, Properties props) { - InputStream is; - // ms, 10-04-2010: check first is this path exists in our CLASSPATH. This - // takes priority over the file system. - if ((is = loadStreamFromClasspath(loadPath)) != null) { - Timing.startDoing("Loading classifier from " + loadPath); - loadClassifierNoExceptions(is, props); - try { - is.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - Timing.endDoing(); - } else { - loadClassifierNoExceptions(new File(loadPath), props); - } - } - - public void loadClassifier(File file) throws ClassCastException, IOException, ClassNotFoundException { - loadClassifier(file, null); - } - - /** - * Loads a classifier from the file specified. If the file's name ends in .gz, - * uses a GZIPInputStream, else uses a regular FileInputStream. This method - * closes the File when done. - * - * @param file - * Loads a classifier from this file. - * @param props - * Properties in this object will be used to overwrite those - * specified in the serialized classifier - * - * @throws IOException - * If there are problems accessing the input stream - * @throws ClassCastException - * If there are problems interpreting the serialized data - * @throws ClassNotFoundException - * If there are problems interpreting the serialized data - */ - public void loadClassifier(File file, Properties props) throws ClassCastException, IOException, - ClassNotFoundException { - Timing.startDoing("Loading classifier from " + file.getAbsolutePath()); - BufferedInputStream bis; - if (file.getName().endsWith(".gz")) { - bis = new BufferedInputStream(new GZIPInputStream(new FileInputStream(file))); - } else { - bis = new BufferedInputStream(new FileInputStream(file)); - } - loadClassifier(bis, props); - bis.close(); - Timing.endDoing(); - } - - public void loadClassifierNoExceptions(File file) { - loadClassifierNoExceptions(file, null); - } - - public void loadClassifierNoExceptions(File file, Properties props) { - try { - loadClassifier(file, props); - } catch (Exception e) { - System.err.println("Error deserializing " + file.getAbsolutePath()); - throw new RuntimeException(e); - } - } - - /** - * This function will load a classifier that is stored inside a jar file (if - * it is so stored). The classifier should be specified as its full filename, - * but the path in the jar file (/classifiers/) is coded in this - * class. If the classifier is not stored in the jar file or this is not run - * from inside a jar file, then this function will throw a RuntimeException. - * - * @param modelName - * The name of the model file. Iff it ends in .gz, then it is assumed - * to be gzip compressed. - * @param props - * A Properties object which can override certain properties in the - * serialized file, such as the DocumentReaderAndWriter. You can pass - * in null to override nothing. - */ - public void loadJarClassifier(String modelName, Properties props) { - Timing.startDoing("Loading JAR-internal classifier " + modelName); - try { - InputStream is = getClass().getResourceAsStream(modelName); - if (modelName.endsWith(".gz")) { - is = new GZIPInputStream(is); - } - is = new BufferedInputStream(is); - loadClassifier(is, props); - is.close(); - Timing.endDoing(); - } catch (Exception e) { - String msg = "Error loading classifier from jar file (most likely you are not running this code from a jar file or the named classifier is not stored in the jar file)"; - throw new RuntimeException(msg, e); - } - } - - private transient PrintWriter cliqueWriter; - private transient int writtenNum; // = 0; - - /** Print the String features generated from a IN */ - protected void printFeatures(IN wi, Collection features) { - if (flags.printFeatures == null || writtenNum > flags.printFeaturesUpto) { - return; - } - try { - if (cliqueWriter == null) { - cliqueWriter = new PrintWriter(new FileOutputStream("feats" + flags.printFeatures + ".txt"), true); - writtenNum = 0; - } - } catch (Exception ioe) { - throw new RuntimeException(ioe); - } - if (writtenNum >= flags.printFeaturesUpto) { - return; - } - if (wi instanceof CoreLabel) { - cliqueWriter.print(wi.get(TextAnnotation.class) + ' ' + wi.get(PartOfSpeechAnnotation.class) + ' ' - + wi.get(CoreAnnotations.GoldAnswerAnnotation.class) + '\t'); - } else { - cliqueWriter.print(wi.get(CoreAnnotations.TextAnnotation.class) - + wi.get(CoreAnnotations.GoldAnswerAnnotation.class) + '\t'); - } - boolean first = true; - for (Object feat : features) { - if (first) { - first = false; - } else { - cliqueWriter.print(" "); - } - cliqueWriter.print(feat); - } - cliqueWriter.println(); - writtenNum++; - } - - /** Print the String features generated from a token */ - protected void printFeatureLists(IN wi, Collection> features) { - if (flags.printFeatures == null || writtenNum > flags.printFeaturesUpto) { - return; - } - try { - if (cliqueWriter == null) { - cliqueWriter = new PrintWriter(new FileOutputStream("feats" + flags.printFeatures + ".txt"), true); - writtenNum = 0; - } - } catch (Exception ioe) { - throw new RuntimeException(ioe); - } - if (writtenNum >= flags.printFeaturesUpto) { - return; - } - if (wi instanceof CoreLabel) { - cliqueWriter.print(wi.get(TextAnnotation.class) + ' ' + wi.get(PartOfSpeechAnnotation.class) + ' ' - + wi.get(CoreAnnotations.GoldAnswerAnnotation.class) + '\t'); - } else { - cliqueWriter.print(wi.get(CoreAnnotations.TextAnnotation.class) - + wi.get(CoreAnnotations.GoldAnswerAnnotation.class) + '\t'); - } - boolean first = true; - for (List featList : features) { - for (String feat : featList) { - if (first) { - first = false; - } else { - cliqueWriter.print(" "); - } - cliqueWriter.print(feat); - } - cliqueWriter.print(" "); - } - cliqueWriter.println(); - writtenNum++; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/AcquisitionsPrior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/AcquisitionsPrior.java deleted file mode 100644 index f5c1080..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/AcquisitionsPrior.java +++ /dev/null @@ -1,264 +0,0 @@ -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.StringUtils; - -import java.util.Set; -import java.util.HashSet; -import java.util.List; -import java.util.ArrayList; - -/** - * @author Jenny Finkel - */ - -public class AcquisitionsPrior extends EntityCachingAbstractSequencePrior { - - double penalty = 4.0; - double penalty1 = 3.0; - double penalty2 = 4.0; - - public AcquisitionsPrior(String backgroundSymbol, Index classIndex, List doc) { - super(backgroundSymbol, classIndex, doc); - } - - public double scoreOf(int[] sequence) { - - Set purchasers = new HashSet(); - Set purchabrs = new HashSet(); - Set sellers = new HashSet(); - Set sellerabrs = new HashSet(); - Set acquireds = new HashSet(); - Set acqabrs = new HashSet(); - - List purchasersL = new ArrayList(); - List purchabrsL = new ArrayList(); - List sellersL = new ArrayList(); - List sellerabrsL = new ArrayList(); - List acquiredsL = new ArrayList(); - List acqabrsL = new ArrayList(); - - double p = 0.0; - for (int i = 0; i < entities.length; i++) { - Entity entity = entities[i]; - if ((i == 0 || entities[i-1] != entity) && entity != null) { - - String type = classIndex.get(entity.type); - String phrase = StringUtils.join(entity.words, " ").toLowerCase(); - if (type.equals("purchaser")) { - purchasers.add(phrase); - purchasersL.add(entity); - } else if (type.equals("purchabr")) { - purchabrs.add(phrase); - purchabrsL.add(entity); - } else if (type.equals("seller")) { - sellers.add(phrase); - sellersL.add(entity); - } else if (type.equals("sellerabr")) { - sellerabrs.add(phrase); - sellerabrsL.add(entity); - } else if (type.equals("acquired")) { - acquireds.add(phrase); - acquiredsL.add(entity); - } else if (type.equals("acqabr")) { - acqabrs.add(phrase); - acqabrsL.add(entity); - } else { - System.err.println("unknown entity type: "+type); - System.exit(0); - } - } - } - - for (Entity purchaser : purchasersL) { - if (purchasers.size() > 1) { - p -= purchaser.words.size() * penalty; - } - String s = StringUtils.join(purchaser.words, "").toLowerCase(); - boolean match = false; - for (Entity purchabr : purchabrsL) { - String s1 = StringUtils.join(purchabr.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s1.length() - 2) { - if (s.indexOf(s1) >= 0) { - match = true; - break; - } - } - if (!match && purchabrs.size() > 0) { - p -= purchaser.words.size() * penalty; - } - } - - for (Entity seller : sellersL) { - if (sellers.size() > 1) { - p -= seller.words.size() * penalty; - } - String s = StringUtils.join(seller.words, "").toLowerCase(); - boolean match = false; - for (Entity sellerabr : sellerabrsL) { - String s1 = StringUtils.join(sellerabr.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s1.length() - 2) { - if (s.indexOf(s1) >= 0) { - match = true; - break; - } - } - if (!match && sellerabrs.size() > 0) { - p -= seller.words.size() * penalty; - } - } - - for (Entity acquired : acquiredsL) { - if (acquireds.size() > 1) { - p -= acquired.words.size() * penalty; - } - String s = StringUtils.join(acquired.words, "").toLowerCase(); - boolean match = false; - for (Entity acqabr : acqabrsL) { - String s1 = StringUtils.join(acqabr.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s1.length() - 2) { - if (s.indexOf(s1) >= 0) { - match = true; - break; - } - } - if (!match && acqabrs.size() > 0) { - p -= acquired.words.size() * penalty; - } - } - - - for (Entity purchabr : purchabrsL) { - //p -= purchabr.words.size() * penalty; - String s = StringUtils.join(purchabr.words, "").toLowerCase(); - boolean match = false; - for (Entity purchaser : purchasersL) { - String s1 = StringUtils.join(purchaser.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s1.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - if (!match) { - p -= purchabr.words.size() * penalty2; - } - - match = false; - for (Entity acquired : acquiredsL) { - String s1 = StringUtils.join(acquired.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - for (Entity seller : sellersL) { - String s1 = StringUtils.join(seller.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - if (match) { - p -= purchabr.words.size() * penalty1; - } - } - - for (Entity sellerabr : sellerabrsL) { - //p -= sellerabr.words.size() * penalty; - String s = StringUtils.join(sellerabr.words, "").toLowerCase(); - boolean match = false; - for (Entity seller : sellersL) { - String s1 = StringUtils.join(seller.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s1.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - if (!match) { - p -= sellerabr.words.size() * penalty2; - } - - - match = false; - for (Entity acquired : acquiredsL) { - String s1 = StringUtils.join(acquired.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - for (Entity purchaser : purchasersL) { - String s1 = StringUtils.join(purchaser.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - if (match) { - p -= sellerabr.words.size() * penalty1; - } - } - - - for (Entity acqabr : acqabrsL) { - //p -= acqabr.words.size() * penalty; - String s = StringUtils.join(acqabr.words, "").toLowerCase(); - boolean match = false; - for (Entity acquired : acquiredsL) { - String s1 = StringUtils.join(acquired.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s1.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - if (!match) { - p -= acqabr.words.size() * penalty2; - } - - match = false; - for (Entity seller : sellersL) { - String s1 = StringUtils.join(seller.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s.length() - 2) { - if (s1.indexOf(s) >= 0) { - //System.err.println(acqabr.toString(classIndex)+"\n"+seller.toString(classIndex)+"\n"); - match = true; - break; - } - } - for (Entity purchaser : purchasersL) { - String s1 = StringUtils.join(purchaser.words, "").toLowerCase(); - //int dist = StringUtils.longestCommonSubstring(s, s1); - //if (dist > s.length() - 2) { - if (s1.indexOf(s) >= 0) { - match = true; - break; - } - } - if (match) { - p -= acqabr.words.size() * penalty1; - } - } - - return p; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/EmpiricalNERPrior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/EmpiricalNERPrior.java deleted file mode 100644 index 9736e36..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/EmpiricalNERPrior.java +++ /dev/null @@ -1,284 +0,0 @@ -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.ling.CoreLabel; - -import java.util.List; - - -/** - * @author Jenny Finkel - */ - -public class EmpiricalNERPrior extends EntityCachingAbstractSequencePrior { - - static final String ORG = "ORGANIZATION"; - static final String PER = "PERSON"; - static final String LOC = "LOCATION"; - static final String MISC = "MISC"; - - public EmpiricalNERPrior(String backgroundSymbol, Index classIndex, List doc) { - super(backgroundSymbol, classIndex, doc); - } - - double p1 = -Math.log(0.01); - - double dem1 = 6631.0; - double p2 = -Math.log(6436.0 / dem1)/2.0; - double p3 = -Math.log(188 / dem1)/2.0; - double p4 = -Math.log(4 / dem1)/2.0; - double p5 = -Math.log(3 / dem1)/2.0; - - double dem2 = 3169.0; - double p6 = -Math.log(188.0 / dem2)/2.0; - double p7 = -Math.log(2975 / dem2)/2.0; - double p8 = -Math.log(5 / dem2)/2.0; - double p9 = -Math.log(1 / dem2)/2.0; - - double dem3 = 3151.0; - double p10 = -Math.log(4.0 / dem3)/2.0; - double p11 = -Math.log(5 / dem3)/2.0; - double p12 = -Math.log(3141 / dem3)/2.0; - double p13 = -Math.log(1 / dem3)/2.0; - - double dem4 = 2035.0; - double p14 = -Math.log(3.0 / dem4)/2.0; - double p15 = -Math.log(1 / dem4)/2.0; - double p16 = -Math.log(1 / dem4)/2.0; - double p17 = -Math.log(2030 / dem4)/2.0; - - double dem5 = 724.0; - double p18 = -Math.log(167.0 / dem5); - double p19 = -Math.log(328.0 / dem5); - double p20 = -Math.log(5.0 / dem5); - double p21 = -Math.log(224.0 / dem5); - - double dem6 = 834.0; - double p22 = -Math.log(6.0 / dem6); - double p23 = -Math.log(819.0 / dem6); - double p24 = -Math.log(2.0 / dem6); - double p25 = -Math.log(7.0 / dem6); - - double dem7 = 1978.0; - double p26 = -Math.log(1.0 / dem7); - double p27 = -Math.log(22.0 / dem7); - double p28 = -Math.log(1941.0 / dem7); - double p29 = -Math.log(14.0 / dem7); - - double dem8 = 622.0; - double p30 = -Math.log(63.0 / dem8); - double p31 = -Math.log(191.0 / dem8); - double p32 = -Math.log(3.0 / dem8); - double p33 = -Math.log(365.0 / dem8); - - public double scoreOf(int[] sequence) { - double p = 0.0; - for (int i = 0; i < entities.length; i++) { - Entity entity = entities[i]; - //System.err.println(entity); - if ((i == 0 || entities[i-1] != entity) && entity != null) { - //System.err.println(1); - int length = entity.words.size(); - String tag1 = classIndex.get(entity.type); - - if (tag1.equals(LOC)) { tag1 = LOC; } - else if (tag1.equals(ORG)) { tag1 = ORG; } - else if (tag1.equals(PER)) { tag1 = PER; } - else if (tag1.equals(MISC)) { tag1 = MISC; } - - int[] other = entities[i].otherOccurrences; - for (int j = 0; j < other.length; j++) { - - Entity otherEntity = null; - for (int k = other[j]; k < other[j]+length && k < entities.length; k++) { - otherEntity = entities[k]; - if (otherEntity != null) { -// if (k > other[j]) { -// System.err.println(entity.words+" "+otherEntity.words); -// } - break; - } - } - // singleton + other instance null? - if (otherEntity == null) { - //p -= length * Math.log(0.1); - //if (entity.words.size() == 1) { - //p -= length * p1; - //} - continue; - } - - int oLength = otherEntity.words.size(); - String tag2 = classIndex.get(otherEntity.type); - - if (tag2.equals(LOC)) { tag2 = LOC; } - else if (tag2.equals(ORG)) { tag2 = ORG; } - else if (tag2.equals(PER)) { tag2 = PER; } - else if (tag2.equals(MISC)) { tag2 = MISC; } - - // exact match?? - boolean exact = false; - int[] oOther = otherEntity.otherOccurrences; - for (int k = 0; k < oOther.length; k++) { - if (oOther[k] >= i && oOther[k] <= i+length-1) { - exact = true; - break; - } - } - - if (exact) { - // entity not complete - if (length != oLength) { - if (tag1 == (tag2)) {// || ((tag1 == LOC && tag2 == ORG) || (tag1 == ORG && tag2 == LOC))) { // || - //p -= Math.abs(oLength - length) * Math.log(0.1); - p -= Math.abs(oLength - length) * p1; - } else if (!(tag1.equals(ORG) && tag2.equals(LOC)) && - !(tag2.equals(LOC) && tag1.equals(ORG))) { - // shorter - p -= (oLength + length) * p1; - } - } - if (tag1 == (LOC)) { - if (tag2 == (LOC)) { - //p -= length * Math.log(6436.0 / dem); - //p -= length * p2; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(188 / dem); - p -= length * p3; - } else if (tag2 == (PER)) { - //p -= length * Math.log(4 / dem); - p -= length * p4; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(3 / dem); - p -= length * p5; - } - } else if (tag1 == (ORG)) { - //double dem = 3169.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(188.0 / dem); - p -= length * p6; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(2975 / dem); - //p -= length * p7; - } else if (tag2 == (PER)) { - //p -= length * Math.log(5 / dem); - p -= length * p8; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(1 / dem); - p -= length * p9; - } - } else if (tag1 == (PER)) { - //double dem = 3151.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(4.0 / dem); - p -= length * p10; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(5 / dem); - p -= length * p11; - } else if (tag2 == (PER)) { - //p -= length * Math.log(3141 / dem); - //p -= length * p12; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(1 / dem); - p -= length * p13; - } - } else if (tag1 == (MISC)) { - //double dem = 2035.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(3.0 / dem); - p -= length * p14; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(1 / dem); - p -= length * p15; - } else if (tag2 == (PER)) { - //p -= length * Math.log(1 / dem); - p -= length * p16; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(2030 / dem); - //p -= length * p17; - } - } - } else { - if (tag1 == (LOC)) { - //double dem = 724.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(167.0 / dem); - //p -= length * p18; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(328.0 / dem); - //p -= length * p19; - } else if (tag2 == (PER)) { - //p -= length * Math.log(5.0 / dem); - p -= length * p20; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(224.0 / dem); - p -= length * p21; - } - } else if (tag1 == (ORG)) { - //double dem = 834.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(6.0 / dem); - p -= length * p22; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(819.0 / dem); - //p -= length * p23; - } else if (tag2 == (PER)) { - //p -= length * Math.log(2.0 / dem); - p -= length * p24; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(7.0 / dem); - p -= length * p25; - } - } else if (tag1 == (PER)) { - //double dem = 1978.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(1.0 / dem); - p -= length * p26; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(22.0 / dem); - p -= length * p27; - } else if (tag2 == (PER)) { - //p -= length * Math.log(1941.0 / dem); - //p -= length * p28; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(14.0 / dem); - p -= length * p29; - } - } else if (tag1 == (MISC)) { - //double dem = 622.0; - if (tag2 == (LOC)) { - //p -= length * Math.log(63.0 / dem); - p -= length * p30; - } else if (tag2 == (ORG)) { - //p -= length * Math.log(191.0 / dem); - p -= length * p31; - } else if (tag2 == (PER)) { - //p -= length * Math.log(3.0 / dem); - p -= length * p32; - } else if (tag2 == (MISC)) { - //p -= length * Math.log(365.0 / dem); - p -= length * p33; - } - } - } - -// if (tag1 == PER) { -// int personIndex = classIndex.indexOf(PER); -// String lastName = entity.words.get(entity.words.size()-1); -// for (int k = 0; k < doc.size(); k++) { -// String w = doc.get(k).word(); -// if (w.equalsIgnoreCase(lastName)) { -// if (sequence[k] != personIndex) { -// p -= p1; -// } -// } -// } -// } - } - } - } - return p; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/EntityCachingAbstractSequencePrior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/EntityCachingAbstractSequencePrior.java deleted file mode 100644 index 18563cf..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/EntityCachingAbstractSequencePrior.java +++ /dev/null @@ -1,500 +0,0 @@ -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.StringUtils; -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.sequences.SequenceModel; -import edu.stanford.nlp.sequences.SequenceListener; -import edu.stanford.nlp.ling.CoreAnnotations; - -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; - -/** - * This class keeps track of all labeled entities and updates the - * its list whenever the label at a point gets changed. This allows - * you to not have to regereate the list everytime, which can be quite - * inefficient. - * - * @author Jenny Finkel - **/ -public abstract class EntityCachingAbstractSequencePrior implements SequenceModel, SequenceListener { - - protected int[] sequence; - protected int backgroundSymbol; - protected int numClasses; - protected int[] possibleValues; - protected Index classIndex; - protected List doc; - - public EntityCachingAbstractSequencePrior(String backgroundSymbol, Index classIndex, List doc) { - this.classIndex = classIndex; - this.backgroundSymbol = classIndex.indexOf(backgroundSymbol); - this.numClasses = classIndex.size(); - this.possibleValues = new int[numClasses]; - for (int i=0; inot index+1) - **/ - public Entity extractEntity(int[] sequence, int position) { - Entity entity = new Entity(); - entity.type = sequence[position]; - entity.startPosition = position; - entity.words = new ArrayList(); - for ( ; position < sequence.length; position++) { - if (sequence[position] == entity.type) { - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - entity.words.add(word); - if (position == sequence.length - 1) { - entity.otherOccurrences = otherOccurrences(entity); - } - } else { - entity.otherOccurrences = otherOccurrences(entity); - break; - } - } - return entity; - } - - /** - * finds other locations in the sequence where the sequence of - * words in this entity occurs. - */ - public int[] otherOccurrences(Entity entity){ - List other = new ArrayList(); - for (int i = 0; i < doc.size(); i++) { - if (i == entity.startPosition) { continue; } - if (matches(entity, i)) { - other.add(Integer.valueOf(i)); - } - } - return toArray(other); - } - - public static int[] toArray(List list) { - int[] arr = new int[list.size()]; - for (int i = 0; i < arr.length; i++) { - arr[i] = list.get(i); - } - return arr; - } - - public boolean matches(Entity entity, int position) { - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - if (word.equalsIgnoreCase(entity.words.get(0))) { - //boolean matches = true; - for (int j = 1; j < entity.words.size(); j++) { - if (position + j >= doc.size()) { - return false; - } - String nextWord = doc.get(position+j).get(CoreAnnotations.TextAnnotation.class); - if (!nextWord.equalsIgnoreCase(entity.words.get(j))) { - return false; - } - } - return true; - } - return false; - } - - - public boolean joiningTwoEntities(int[] sequence, int position) { - if (sequence[position] == backgroundSymbol) { return false; } - if (position > 0 && position < sequence.length - 1) { - return (sequence[position] == sequence[position - 1] && - sequence[position] == sequence[position + 1]); - } - return false; - } - - public boolean splittingTwoEntities(int[] sequence, int position) { - if (position > 0 && position < sequence.length - 1) { - return (entities[position - 1] == entities[position + 1] && - entities[position - 1] != null); - } - return false; - } - - public boolean appendingEntity(int[] sequence, int position) { - if (position > 0) { - if (entities[position - 1] == null) { return false; } - Entity prev = entities[position - 1]; - return (sequence[position] == sequence[position - 1] && - prev.startPosition + prev.words.size() == position); - } - return false; - } - - public boolean prependingEntity(int[] sequence, int position) { - if (position < sequence.length - 1) { - if (entities[position + 1] == null) { return false; } - return (sequence[position] == sequence[position + 1]); - } - return false; - } - - public boolean addingSingletonEntity(int[] sequence, int position) { - if (sequence[position] == backgroundSymbol) { return false; } - if (position > 0) { - if (sequence[position - 1] == sequence[position]) { return false; } - } - if (position < sequence.length - 1) { - if (sequence[position + 1] == sequence[position]) { return false; } - } - return true; - } - - public boolean removingEndOfEntity(int[] sequence, int position) { - if (position > 0) { - if (sequence[position - 1] == backgroundSymbol) { return false; } - Entity prev = entities[position - 1]; - if (prev != null) { - return (prev.startPosition + prev.words.size() > position); - } - } - return false; - } - - public boolean removingBeginningOfEntity(int[] sequence, int position) { - if (position < sequence.length - 1) { - if (sequence[position + 1] == backgroundSymbol) { return false; } - Entity next = entities[position + 1]; - if (next != null) { - return (next.startPosition <= position); - } - } - return false; - } - - public boolean noChange(int[] sequence, int position) { - if (position > 0) { - if (sequence[position - 1] == sequence[position]) { - return entities[position - 1] == entities[position]; - } - } - if (position < sequence.length - 1) { - if (sequence[position + 1] == sequence[position]) { - return entities[position] == entities[position + 1]; - } - } - // actually, can't tell. either no change, or singleton - // changed type - return false; - } - - public void updateSequenceElement(int[] sequence, int position, int oldVal) { - if (VERBOSE) System.out.println("changing position "+position+" from " +classIndex.get(oldVal)+" to "+classIndex.get(sequence[position])); - - this.sequence = sequence; - - // no change? - if (noChange(sequence, position)) { - if (VERBOSE) System.out.println("no change"); - if (VERBOSE) System.out.println(this); - return; - } - // are we joining 2 entities? - else if (joiningTwoEntities(sequence, position)) { - if (VERBOSE) System.out.println("joining 2 entities"); - Entity newEntity = new Entity(); - Entity prev = entities[position - 1]; - Entity next = entities[position + 1]; - newEntity.startPosition = prev.startPosition; - newEntity.words = new ArrayList(); - newEntity.words.addAll(prev.words); - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - newEntity.words.add(word); - newEntity.words.addAll(next.words); - newEntity.type = sequence[position]; - List other = new ArrayList(); - for (int i = 0; i < prev.otherOccurrences.length; i++) { - int pos = prev.otherOccurrences[i]; - if (matches(newEntity, pos)) { - other.add(Integer.valueOf(pos)); - } - } - newEntity.otherOccurrences = toArray(other); - addEntityToEntitiesArray(newEntity); - if (VERBOSE) System.out.println(this); - return; - } - // are we splitting up an entity? - else if (splittingTwoEntities(sequence, position)) { - if (VERBOSE) System.out.println("splitting into 2 entities"); - Entity entity = entities[position]; - Entity prev = new Entity(); - prev.type = entity.type; - prev.startPosition = entity.startPosition; - prev.words = new ArrayList(entity.words.subList(0, position - entity.startPosition)); - prev.otherOccurrences = otherOccurrences(prev); - addEntityToEntitiesArray(prev); - Entity next = new Entity(); - next.type = entity.type; - next.startPosition = position + 1; - next.words = new ArrayList(entity.words.subList(position - entity.startPosition + 1, entity.words.size())); - next.otherOccurrences = otherOccurrences(next); - addEntityToEntitiesArray(next); - if (sequence[position] == backgroundSymbol) { - entities[position] = null; - } else { - Entity newEntity = new Entity(); - newEntity.startPosition = position; - newEntity.type = sequence[position]; - newEntity.words = new ArrayList(); - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - newEntity.words.add(word); - newEntity.otherOccurrences = otherOccurrences(newEntity); - entities[position] = newEntity; - } - if (VERBOSE) System.out.println(this); - return; - } - // are we prepending to an entity ? - else if (prependingEntity(sequence, position)) { - if (VERBOSE) System.out.println("prepending entity"); - Entity newEntity = new Entity(); - Entity next = entities[position + 1]; - newEntity.startPosition = position; - newEntity.words = new ArrayList(); - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - newEntity.words.add(word); - newEntity.words.addAll(next.words); - newEntity.type = sequence[position]; - //List other = new ArrayList(); - newEntity.otherOccurrences = otherOccurrences(newEntity); - addEntityToEntitiesArray(newEntity); - - if (removingEndOfEntity(sequence, position)) { - if (VERBOSE) System.out.println(" ... and removing end of previous entity."); - Entity prev = entities[position - 1]; - prev.words.remove(prev.words.size()-1); - prev.otherOccurrences = otherOccurrences(prev); - } - if (VERBOSE) System.out.println(this); - return; - } - // are we appending to an entity ? - else if (appendingEntity(sequence, position)) { - if (VERBOSE) System.out.println("appending entity"); - Entity newEntity = new Entity(); - Entity prev = entities[position - 1]; - newEntity.startPosition = prev.startPosition; - newEntity.words = new ArrayList(); - newEntity.words.addAll(prev.words); - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - newEntity.words.add(word); - newEntity.type = sequence[position]; - List other = new ArrayList(); - for (int i = 0; i < prev.otherOccurrences.length; i++) { - int pos = prev.otherOccurrences[i]; - if (matches(newEntity, pos)) { - other.add(Integer.valueOf(pos)); - } - } - newEntity.otherOccurrences = toArray(other); - addEntityToEntitiesArray(newEntity); - - if (removingBeginningOfEntity(sequence, position)) { - if (VERBOSE) System.out.println(" ... and removing beginning of next entity."); - entities[position + 1].words.remove(0); - entities[position + 1].startPosition++; - } - if (VERBOSE) System.out.println(this); - return; - } - // adding new singleton entity - else if (addingSingletonEntity(sequence, position)) { - Entity newEntity = new Entity(); - if (VERBOSE) System.out.println("adding singleton entity"); - newEntity.startPosition = position; - newEntity.words = new ArrayList(); - String word = doc.get(position).get(CoreAnnotations.TextAnnotation.class); - newEntity.words.add(word); - newEntity.type = sequence[position]; - newEntity.otherOccurrences = otherOccurrences(newEntity); - addEntityToEntitiesArray(newEntity); - - if (removingEndOfEntity(sequence, position)) { - if (VERBOSE) System.out.println(" ... and removing end of previous entity."); - Entity prev = entities[position - 1]; - prev.words.remove(prev.words.size()-1); - prev.otherOccurrences = otherOccurrences(prev); - } - - if (removingBeginningOfEntity(sequence, position)) { - if (VERBOSE) System.out.println(" ... and removing beginning of next entity."); - entities[position + 1].words.remove(0); - entities[position + 1].startPosition++; - } - - if (VERBOSE) System.out.println(this); - return; - } - // are splitting off the prev entity? - else if (removingEndOfEntity(sequence, position)) { - if (VERBOSE) System.out.println("splitting off prev entity"); - Entity prev = entities[position - 1]; - prev.words.remove(prev.words.size() - 1); - prev.otherOccurrences = otherOccurrences(prev); - entities[position] = null; - } - // are we splitting off the next entity? - else if (removingBeginningOfEntity(sequence, position)) { - if (VERBOSE) System.out.println("splitting off next entity"); - Entity next = entities[position + 1]; - next.words.remove(0); - next.startPosition++; - next.otherOccurrences = otherOccurrences(next); - entities[position] = null; - } else { - entities[position] = null; - } - if (VERBOSE) System.out.println(this); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < entities.length; i++) { - sb.append(i); - sb.append("\t"); - String word = doc.get(i).get(CoreAnnotations.TextAnnotation.class); - sb.append(word); - sb.append("\t"); - sb.append(classIndex.get(sequence[i])); - if (entities[i] != null) { - sb.append("\t"); - sb.append(entities[i].toString(classIndex)); - } - sb.append("\n"); - } - return sb.toString(); - } - - public String toString(int pos) { - StringBuffer sb = new StringBuffer(); - for (int i = Math.max(0, pos - 10); i < Math.min(entities.length, pos + 10); i++) { - sb.append(i); - sb.append("\t"); - String word = doc.get(i).get(CoreAnnotations.TextAnnotation.class); - sb.append(word); - sb.append("\t"); - sb.append(classIndex.get(sequence[i])); - if (entities[i] != null) { - sb.append("\t"); - sb.append(entities[i].toString(classIndex)); - } - sb.append("\n"); - } - return sb.toString(); - } - -} - -class Entity { - public int startPosition; - public List words; - public int type; - - /** - * the begining index of other locations where this sequence of - * words appears. - */ - public int[] otherOccurrences; - - public String toString(Index classIndex) { - StringBuffer sb = new StringBuffer(); - sb.append("\""); - sb.append(StringUtils.join(words, " ")); - sb.append("\" start: "); - sb.append(startPosition); - sb.append(" type: "); - sb.append(classIndex.get(type)); - sb.append(" other_occurrences: "); - sb.append(Arrays.toString(otherOccurrences)); - return sb.toString(); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERDemo.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERDemo.java deleted file mode 100644 index 71e980c..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERDemo.java +++ /dev/null @@ -1,79 +0,0 @@ -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.ie.crf.*; -import edu.stanford.nlp.ie.AbstractSequenceClassifier; -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.ling.CoreLabel; -import edu.stanford.nlp.ling.CoreAnnotations.AnswerAnnotation; - -import java.util.List; -import java.io.IOException; - - - -/** This is a demo of calling CRFClassifier programmatically. - *

- * Usage: java -cp "stanford-ner.jar:." NERDemo [serializedClassifier [fileName]] - *

- * If arguments aren't specified, they default to - * ner-eng-ie.crf-3-all2006.ser.gz and some hardcoded sample text. - *

- * To use CRFClassifier from the command line: - * java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier - * [classifier] -textFile [file] - * Or if the file is already tokenized and one word per line, perhaps in - * a tab-separated value format with extra columns for part-of-speech tag, - * etc., use the version below (note the 's' instead of the 'x'): - * java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier - * [classifier] -testFile [file] - * - * @author Jenny Finkel - * @author Christopher Manning - */ - -public class NERDemo { - - public static void main(String[] args) throws IOException { - - String serializedClassifier = "classifiers/ner-eng-ie.crf-3-all2008.ser.gz"; - - if (args.length > 0) { - serializedClassifier = args[0]; - } - - AbstractSequenceClassifier classifier = CRFClassifier.getClassifierNoExceptions(serializedClassifier); - - /* For either a file to annotate or for the hardcoded text example, - this demo file shows two ways to process the output, for teaching - purposes. For the file, it shows both how to run NER on a String - and how to run it on a whole file. For the hard-coded String, - it shows how to run it on a single sentence, and how to do this - and produce an inline XML output format. - */ - if (args.length > 1) { - String fileContents = IOUtils.slurpFile(args[1]); - List> out = classifier.classify(fileContents); - for (List sentence : out) { - for (CoreLabel word : sentence) { - System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' '); - } - System.out.println(); - } - out = classifier.classifyFile(args[1]); - for (List sentence : out) { - for (CoreLabel word : sentence) { - System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' '); - } - System.out.println(); - } - - } else { - String s1 = "Good afternoon Rajat Raina, how are you today?"; - String s2 = "I go to school at Stanford University, which is located in California."; - System.out.println(classifier.classifyToString(s1)); - System.out.println(classifier.classifyWithInlineXML(s2)); - System.out.println(classifier.classifyToString(s2, "xml", true)); - } - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERFeatureFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERFeatureFactory.java deleted file mode 100644 index 7ca05dc..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERFeatureFactory.java +++ /dev/null @@ -1,2143 +0,0 @@ -// NERFeatureFactory -- features for a probabilistic Named Entity Recognizer -// Copyright (c) 2002-2008 Leland Stanford Junior University -// Additional features (c) 2003 The University of Edinburgh -// -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -// -// For more information, bug reports, fixes, contact: -// Christopher Manning -// Dept of Computer Science, Gates 1A -// Stanford CA 94305-9010 -// USA -// Support/Questions: java-nlp-user@lists.stanford.edu -// Licensing: java-nlp-support@lists.stanford.edu -// http://nlp.stanford.edu/downloads/crf-classifier.shtml - -package edu.stanford.nlp.ie; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import edu.stanford.nlp.ling.CoreAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations; -import edu.stanford.nlp.ling.CoreLabel; -import edu.stanford.nlp.ling.CoreAnnotations.AbbrAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.AbgeneAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.AbstrAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ChunkAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.DictAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.DistSimAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.DomainAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.EntityRuleAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.EntityTypeAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.FreqAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.GazAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.GeniaAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.GovernorAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.IsDateRangeAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.IsURLAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.LemmaAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ParaPositionAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.PositionAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ProtoAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.SectionAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ShapeAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.StackedNamedEntityTagAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.TopicAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.UnknownAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.WebAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.WordPositionAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.WordnetSynAnnotation; -import edu.stanford.nlp.ling.CoreLabel.GenericAnnotation; -import edu.stanford.nlp.objectbank.ObjectBank; -import edu.stanford.nlp.process.WordShapeClassifier; -import edu.stanford.nlp.sequences.Clique; -import edu.stanford.nlp.sequences.CoNLLDocumentReaderAndWriter; -import edu.stanford.nlp.sequences.FeatureFactory; -import edu.stanford.nlp.sequences.SeqClassifierFlags; -import edu.stanford.nlp.trees.TreeCoreAnnotations; -import edu.stanford.nlp.util.PaddedList; -import edu.stanford.nlp.util.StringUtils; -import edu.stanford.nlp.util.Timing; - - -/** - * Features for Named Entity Recognition. The code here creates the features - * by processing Lists of CoreLabels. - * Look at {@link SeqClassifierFlags} to see where the flags are set for - * what options to use for what flags. - *

- * To add a new feature extractor, you should do the following: - *

    - *
  1. Add a variable (boolean, int, String, etc. as appropriate) to - * SeqClassifierFlags to mark if the new extractor is turned on or - * its value, etc. Add it at the bottom of the list of variables - * currently in the class (this avoids problems with older serialized - * files breaking). Make the default value of the variable false/null/0 - * (this is again for backwards compatibility).
  2. - *
  3. Add a clause to the big if/then/else of setProperties(Properties) in - * SeqClassifierFlags. Unless it is a macro option, make the option name - * the same as the variable name used in step 1.
  4. - *
  5. Add code to NERFeatureFactory for this feature. First decide which - * classes (hidden states) are involved in the feature. If only the - * current class, you add the feature extractor to the - * featuresC code, if both the current and previous class, - * then featuresCpC, etc.
  6. - *
- *

Parameters can be defined using a Properties file - * (specified on the command-line with -prop propFile), - * or directly on the command line. The following properties are recognized: - *

- * - * - * - * - * - * - * - *

- *

- * - * - * - * - * - * - * - * - * - *

- *

- * - * - * If true, a gazette feature fires when all tokens of a gazette entry match - *

- *

- * - * - * - * - * - * - * - * - * - * - *

- *

- * - * - * - *

- *

- * - *

- *

- * - * - * - * - * - *

- *

- * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Property NameTypeDefault ValueDescription
loadClassifier Stringn/aPath to serialized classifier to load
loadAuxClassifier Stringn/aPath to auxiliary classifier to load.
serializeToStringn/aPath to serialize classifier to
trainFileStringn/aPath of file to use as training data
testFileStringn/aPath of file to use as training data
useWordbooleantrueGives you feature for w
useBinnedLengthStringnullIf non-null, treat as a sequence of comma separated integer bounds, where items above the previous bound up to the next bound are binned Len-range
useNGramsbooleanfalseMake features from letter n-grams
lowercaseNGramsbooleanfalseMake features from letter n-grams only lowercase
dehyphenateNGramsbooleanfalseRemove hyphens before making features from letter n-grams
conjoinShapeNGramsbooleanfalseConjoin word shape and n-gram features
usePrevbooleanfalseGives you feature for (pw,c), and together with other options enables other previous features, such as (pt,c) [with useTags)
useNextbooleanfalseGives you feature for (nw,c), and together with other options enables other next features, such as (nt,c) [with useTags)
useTagsbooleanfalseGives you features for (t,c), (pt,c) [if usePrev], (nt,c) [if useNext]
useWordPairsbooleanfalseGives you - * features for (pw, w, c) and (w, nw, c)
useGazettesbooleanfalseIf true, use gazette features (defined by other flags)
gazetteStringnullThe value can be one or more filenames (names separated by a comma, semicolon or space). - * If provided gazettes are loaded from these files. Each line should be an entity class name, followed by whitespace followed by an entity (which might be a phrase of several tokens with a single space between words). - * Giving this property turns on useGazettes, so you normally don't need to specify it (but can use it to turn off gazettes specified in a properties file).
sloppyGazettebooleanfalseIf true, a gazette feature fires when any token of a gazette entry matches
cleanGazettebooleanfalse
wordShapeStringnoneEither "none" for no wordShape use, or the name of a word shape function recognized by {@link WordShapeClassifier#lookupShaper(String)}
useSequencesbooleantrue
usePrevSequencesbooleanfalse
useNextSequencesbooleanfalse
useLongSequencesbooleanfalseUse plain higher-order state sequences out to minimum of length or maxLeft
useBoundarySequencesbooleanfalseUse extra second order class sequence features when previous is CoNLL boundary, so entity knows it can span boundary.
useTaggySequencesbooleanfalseUse first, second, and third order class and tag sequence interaction features
useExtraTaggySequencesbooleanfalseAdd in sequences of tags with just current class features
useTaggySequencesShapeInteractionbooleanfalseAdd in terms that join sequences of 2 or 3 tags with the current shape
strictlyFirstOrderbooleanfalseAs an override to whatever other options are in effect, deletes all features other than C and CpC clique features when building the classifier
entitySubclassificationString"IO"If - * set, convert the labeling of classes (but not the background) into - * one of several alternate encodings (IO, IOB1, IOB2, IOE1, IOE2, SBIEO, with - * a S(ingle), B(eginning), - * E(nding), I(nside) 4-way classification for each class. By default, we - * either do no re-encoding, or the CoNLLDocumentIteratorFactory does a - * lossy encoding as IO. Note that this is all CoNLL-specific, and depends on - * their way of prefix encoding classes, and is only implemented by - * the CoNLLDocumentIteratorFactory.
useSumbooleanfalse
tolerancedouble1e-4Convergence tolerance in optimization
printFeaturesStringnullprint out the features of the classifier to a file based on this name (starting with feat-, suffixed "-1" and "-2")
printFeaturesUptoint-1Print out features for only the first this many datums, if the value is positive.
useSymTagsbooleanfalseGives you - * features (pt, t, nt, c), (t, nt, c), (pt, t, c)
useSymWordPairsbooleanfalseGives you - * features (pw, nw, c)
printClassifierStringnullStyle in which to print the classifier. One of: HighWeight, HighMagnitude, Collection, AllWeights, WeightHistogram
printClassifierParamint100A parameter - * to the printing style, which may give, for example the number of parameters - * to print
internbooleanfalseIf true, - * (String) intern read in data and classes and feature (pre-)names such - * as substring features
intern2booleanfalseIf true, intern all (final) feature names (if only current word and ngram features are used, these will already have been interned by intern, and this is an unnecessary no-op)
cacheNGramsbooleanfalseIf true, - * record the NGram features that correspond to a String (under the current - * option settings) and reuse rather than recalculating if the String is seen - * again.
selfTestbooleanfalse
noMidNGramsbooleanfalseDo not include character n-gram features for n-grams that contain neither the beginning or end of the word
maxNGramLengint-1If this number is - * positive, n-grams above this size will not be used in the model
useReversebooleanfalse
retainEntitySubclassificationbooleanfalseIf true, rather than undoing a recoding of entity tag subtypes (such as BIO variants), just leave them in the output.
useLemmasbooleanfalseInclude the lemma of a word as a feature.
usePrevNextLemmasbooleanfalseInclude the previous/next lemma of a word as a feature.
useLemmaAsWordbooleanfalseInclude the lemma of a word as a feature.
normalizeTermsbooleanfalseIf this is true, some words are normalized: day and month names are lowercased (as for normalizeTimex) and some British spellings are mapped to American English spellings (e.g., -our/-or, etc.).
normalizeTimexbooleanfalseIf this is true, capitalization of day and month names is normalized to lowercase
useNBbooleanfalse
useTypeSeqsbooleanfalseUse basic zeroeth order word shape features.
useTypeSeqs2booleanfalseAdd additional first and second order word shape features
useTypeSeqs3booleanfalseAdds one more first order shape sequence
useDisjunctivebooleanfalseInclude in features giving disjunctions of words anywhere in the left or right disjunctionWidth words (preserving direction but not position)
disjunctionWidthint4The number of words on each side of the current word that are included in the disjunction features
useDisjunctiveShapeInteractionbooleanfalseInclude in features giving disjunctions of words anywhere in the left or right disjunctionWidth words (preserving direction but not position) interacting with the word shape of the current word
useWideDisjunctivebooleanfalseInclude in features giving disjunctions of words anywhere in the left or right wideDisjunctionWidth words (preserving direction but not position)
wideDisjunctionWidthint4The number of words on each side of the current word that are included in the disjunction features
usePositionbooleanfalseUse combination of position in sentence and class as a feature
useBeginSentbooleanfalseUse combination of initial position in sentence and class (and word shape) as a feature. (Doesn't seem to help.)
useDisjShapebooleanfalseInclude features giving disjunctions of word shapes anywhere in the left or right disjunctionWidth words (preserving direction but not position)
useClassFeaturebooleanfalseInclude a feature for the class (as a class marginal)
useShapeConjunctionsbooleanfalseConjoin shape with tag or position
useWordTagbooleanfalseInclude word and tag pair features
useLastRealWordbooleanfalseIff the prev word is of length 3 or less, add an extra feature that combines the word two back and the current word's shape. Weird!
useNextRealWordbooleanfalseIff the next word is of length 3 or less, add an extra feature that combines the word after next and the current word's shape. Weird!
useTitlebooleanfalseMatch a word against a list of name titles (Mr, Mrs, etc.)
useOccurrencePatternsbooleanfalseThis is a very engineered feature designed to capture multiple references to names. If the current word isn't capitalized, followed by a non-capitalized word, and preceded by a word with alphabetic characters, it returns NO-OCCURRENCE-PATTERN. Otherwise, if the previous word is a capitalized NNP, then if in the next 150 words you find this PW-W sequence, you get XY-NEXT-OCCURRENCE-XY, else if you find W you get XY-NEXT-OCCURRENCE-Y. Similarly for backwards and XY-PREV-OCCURRENCE-XY and XY-PREV-OCCURRENCE-Y. Else (if the previous word isn't a capitalized NNP), under analogous rules you get one or more of X-NEXT-OCCURRENCE-YX, X-NEXT-OCCURRENCE-XY, X-NEXT-OCCURRENCE-X, X-PREV-OCCURRENCE-YX, X-PREV-OCCURRENCE-XY, X-PREV-OCCURRENCE-X.
useTypeySequencesbooleanfalseSome first order word shape patterns.
useGenericFeaturesbooleanfalseIf true, any features you include in the map will be incorporated into the model with values equal to those given in the file; values are treated as strings unless you use the "realValued" option (described below)
justifybooleanfalsePrint out all - * feature/class pairs and their weight, and then for each input data - * point, print justification (weights) for active features
normalizebooleanfalseFor the CMMClassifier (only) if this is true then the Scorer normalizes scores as probabilities.
useHuberbooleanfalseUse a Huber loss prior rather than the default quadratic loss.
useQuarticbooleanfalseUse a Quartic prior rather than the default quadratic loss.
sigmadouble1.0
epsilondouble0.01Used only as a parameter in the Huber loss: this is the distance from 0 at which the loss changes from quadratic to linear
beamSizeint30
maxLeftint2The number of things to the left that have to be cached to run the Viterbi algorithm: the maximum context of class features used.
dontExtendTaggybooleanfalseDon't extend the range of useTaggySequences when maxLeft is increased.
numFolds int1The number of folds to use for cross-validation.
startFold int1The starting fold to run.
numFoldsToRun int1The number of folds to run.
mergeTags booleanfalseWhether to merge B- and I- tags.
splitDocumentsbooleantrueWhether or not to split the data into separate documents for training/testing
maxDocSizeint10000If this number is greater than 0, attempt to split documents bigger than this value into multiple documents at sentence boundaries during testing; otherwise do nothing.
- *

- * Note: flags/properties overwrite left to right. That is, the parameter - * setting specified last is the one used. - *

- *

- * DOCUMENTATION ON FEATURE TEMPLATES
- * 

- * w = word - * t = tag - * p = position (word index in sentence) - * c = class - * p = paren - * g = gazette - * a = abbrev - * s = shape - * r = regent (dependency governor) - * h = head word of phrase - * n(w) = ngrams from w - * g(w) = gazette entries containing w - * l(w) = length of w - * o(...) = occurrence patterns of words - *

- * useReverse reverses meaning of prev, next everywhere below (on in macro) - *

- * "Prolog" booleans: , = AND and ; = OR - *

- * Mac: Y = turned on in -macro, - * + = additional positive things relative to -macro for CoNLL NERFeatureFactory - * (perhaps none...) - * - = Known negative for CoNLL NERFeatureFactory relative to -macro - *

p - * Bio: + = additional things that are positive for BioCreative - * - = things negative relative to -macro - *

- * HighMagnitude: There are no (0) to a few (+) to many (+++) high weight - * features of this template. (? = not used in goodCoNLL, but usually = 0) - *

- * Feature Mac Bio CRFFlags HighMagnitude - * --------------------------------------------------------------------- - * w,c Y useWord 0 (useWord is almost useless with unlimited ngram features, but helps a fraction in goodCoNLL, if only because of prior fiddling - * p,c usePosition ? - * p=0,c useBeginSent ? - * p=0,s,c useBeginSent ? - * t,c Y useTags ++ - * pw,c Y usePrev + - * pt,c Y usePrev,useTags 0 - * nw,c Y useNext ++ - * nt,c Y useNext,useTags 0 - * pw,w,c Y useWordPairs + - * w,nw,c Y useWordPairs + - * pt,t,nt,c useSymTags ? - * t,nt,c useSymTags ? - * pt,t,c useSymTags ? - * pw,nw,c useSymWordPairs ? - *

- * pc,c Y usePrev,useSequences,usePrevSequences +++ - * pc,w,c Y usePrev,useSequences,usePrevSequences 0 - * nc,c useNext,useSequences,useNextSequences ? - * w,nc,c useNext,useSequences,useNextSequences ? - * pc,nc,c useNext,usePrev,useSequences,usePrevSequences,useNextSequences ? - * w,pc,nc,c useNext,usePrev,useSequences,usePrevSequences,useNextSequences ? - *

- * (pw;p2w;p3w;p4w),c + useDisjunctive (out to disjunctionWidth now) +++ - * (nw;n2w;n3w;n4w),c + useDisjunctive (out to disjunctionWidth now) ++++ - * (pw;p2w;p3w;p4w),s,c + useDisjunctiveShapeInteraction ? - * (nw;n2w;n3w;n4w),s,c + useDisjunctiveShapeInteraction ? - * (pw;p2w;p3w;p4w),c + useWideDisjunctive (to wideDisjunctionWidth) ? - * (nw;n2w;n3w;n4w),c + useWideDisjunctive (to wideDisjunctionWidth) ? - * (ps;p2s;p3s;p4s),c useDisjShape (out to disjunctionWidth now) ? - * (ns;n2s;n3s;n4s),c useDisjShape (out to disjunctionWidth now) ? - *

- * pt,pc,t,c Y useTaggySequences + - * p2t,p2c,pt,pc,t,c Y useTaggySequences,maxLeft>=2 + - * p3t,p3c,p2t,p2c,pt,pc,t,c Y useTaggySequences,maxLeft>=3,!dontExtendTaggy ? - * p2c,pc,c Y useLongSequences ++ - * p3c,p2c,pc,c Y useLongSequences,maxLeft>=3 ? - * p4c,p3c,p2c,pc,c Y useLongSequences,maxLeft>=4 ? - * p2c,pc,c,pw=BOUNDARY useBoundarySequences 0 (OK, but!) - *

- * p2t,pt,t,c - useExtraTaggySequences ? - * p3t,p2t,pt,t,c - useExtraTaggySequences ? - *

- * p2t,pt,t,s,p2c,pc,c - useTaggySequencesShapeInteraction ? - * p3t,p2t,pt,t,s,p3c,p2c,pc,c useTaggySequencesShapeInteraction ? - *

- * s,pc,c Y useTypeySequences ++ - * ns,pc,c Y useTypeySequences // error for ps? not? 0 - * ps,pc,s,c Y useTypeySequences 0 - * // p2s,p2c,ps,pc,s,c Y useTypeySequences,maxLeft>=2 // duplicated a useTypeSeqs2 feature - *

- * n(w),c Y useNGrams (noMidNGrams, MaxNGramLeng, lowercaseNGrams, dehyphenateNGrams) +++ - * n(w),s,c useNGrams,conjoinShapeNGrams ? - *

- * g,c + useGazFeatures // test refining this? ? - * pg,pc,c + useGazFeatures ? - * ng,c + useGazFeatures ? - * // pg,g,c useGazFeatures ? - * // pg,g,ng,c useGazFeatures ? - * // p2g,p2c,pg,pc,g,c useGazFeatures ? - * g,w,c useMoreGazFeatures ? - * pg,pc,g,c useMoreGazFeatures ? - * g,ng,c useMoreGazFeatures ? - *

- * g(w),c useGazette,sloppyGazette (contains same word) ? - * g(w),[pw,nw,...],c useGazette,cleanGazette (entire entry matches) ? - *

- * s,c Y wordShape >= 0 +++ - * ps,c Y wordShape >= 0,useTypeSeqs + - * ns,c Y wordShape >= 0,useTypeSeqs + - * pw,s,c Y wordShape >= 0,useTypeSeqs + - * s,nw,c Y wordShape >= 0,useTypeSeqs + - * ps,s,c Y wordShape >= 0,useTypeSeqs 0 - * s,ns,c Y wordShape >= 0,useTypeSeqs ++ - * ps,s,ns,c Y wordShape >= 0,useTypeSeqs ++ - * pc,ps,s,c Y wordShape >= 0,useTypeSeqs,useTypeSeqs2 0 - * p2c,p2s,pc,ps,s,c Y wordShape >= 0,useTypeSeqs,useTypeSeqs2,maxLeft>=2 +++ - * pc,ps,s,ns,c wordShape >= 0,useTypeSeqs,useTypeSeqs3 ? - *

- * p2w,s,c if l(pw) <= 3 Y useLastRealWord // weird features, but work 0 - * n2w,s,c if l(nw) <= 3 Y useNextRealWord ++ - * o(pw,w,nw),c Y useOccurrencePatterns // don't fully grok but has to do with capitalized name patterns ++ - *

- * a,c useAbbr;useMinimalAbbr - * pa,a,c useAbbr - * a,na,c useAbbr - * pa,a,na,c useAbbr - * pa,pc,a,c useAbbr;useMinimalAbbr - * p2a,p2c,pa,pc,a useAbbr - * w,a,c useMinimalAbbr - * p2a,p2c,a,c useMinimalAbbr - *

- * RESTR. w,(pw,pc;p2w,p2c;p3w,p3c;p4w,p4c) + useParenMatching,maxLeft>=n - *

- * c - useClassFeature - *

- * p,s,c - useShapeConjunctions - * t,s,c - useShapeConjunctions - *

- * w,t,c + useWordTag ? - * w,pt,c + useWordTag ? - * w,nt,c + useWordTag ? - *

- * r,c useNPGovernor (only for baseNP words) - * r,t,c useNPGovernor (only for baseNP words) - * h,c useNPHead (only for baseNP words) - * h,t,c useNPHead (only for baseNP words) - *

- *

- * - * @author Dan Klein - * @author Jenny Finkel - * @author Christopher Manning - * @author Shipra Dingare - * @author Huy Nguyen - */ -public class NERFeatureFactory extends FeatureFactory { - - private static final long serialVersionUID = -2329726064739185544L; - - public NERFeatureFactory() { - super(); - } - - public void init(SeqClassifierFlags flags) { - super.init(flags); - initGazette(); - if (flags.useDistSim) { - initLexicon(flags); - } - } - - - /** - * Extracts all the features from the input data at a certain index. - * - * @param cInfo The complete data set as a List of WordInfo - * @param loc The index at which to extract features. - */ - @Override - public Collection getCliqueFeatures(PaddedList cInfo, int loc, Clique clique) { - Collection features = new HashSet(); - - boolean doFE = cInfo.get(0).containsKey(DomainAnnotation.class); - String domain = (doFE ? cInfo.get(0).get(DomainAnnotation.class) : null); - -// System.err.println(doFE+"\t"+domain); - - if (clique == cliqueC) { - //200710: tried making this clique null; didn't improve performance (rafferty) - Collection c = featuresC(cInfo, loc); - addAllInterningAndSuffixing(features, c, "C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-C"); - } - } else if (clique == cliqueCpC) { - Collection c = featuresCpC(cInfo, loc); - addAllInterningAndSuffixing(features, c, "CpC"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-CpC"); - } - - c = featuresCnC(cInfo, loc-1); - addAllInterningAndSuffixing(features, c, "CnC"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-CnC"); - } - } else if (clique == cliqueCp2C) { - Collection c = featuresCp2C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "Cp2C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-Cp2C"); - } - } else if (clique == cliqueCp3C) { - Collection c = featuresCp3C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "Cp3C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-Cp3C"); - } - } else if (clique == cliqueCp4C) { - Collection c = featuresCp4C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "Cp4C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-Cp4C"); - } - } else if (clique == cliqueCp5C) { - Collection c = featuresCp5C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "Cp5C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-Cp5C"); - } - } else if (clique == cliqueCpCp2C) { - Collection c = featuresCpCp2C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "CpCp2C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-CpCp2C"); - } - - c = featuresCpCnC(cInfo, loc-1); - addAllInterningAndSuffixing(features, c, "CpCnC"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-CpCnC"); - } - } else if (clique == cliqueCpCp2Cp3C) { - Collection c = featuresCpCp2Cp3C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "CpCp2Cp3C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-CpCp2Cp3C"); - } - } else if (clique == cliqueCpCp2Cp3Cp4C) { - Collection c = featuresCpCp2Cp3Cp4C(cInfo, loc); - addAllInterningAndSuffixing(features, c, "CpCp2Cp3Cp4C"); - if (doFE) { - addAllInterningAndSuffixing(features, c, domain+"-CpCp2Cp3Cp4C"); - } - } - - // System.err.println(StringUtils.join(features,"\n")+"\n"); - return features; - } - - - // TODO: when breaking serialization, it seems like it would be better to - // move the lexicon into (Abstract)SequenceClassifier and to do this - // annotation as part of the ObjectBankWrapper. But note that it is - // serialized in this object currently and it would then need to be - // serialized elsewhere or loaded each time - private Map lexicon; - - private void initLexicon(SeqClassifierFlags flags) { - if (flags.distSimLexicon == null) { - return; - } - if (lexicon != null) { - return; - } - Timing.startDoing("Loading distsim lexicon from " + flags.distSimLexicon); - lexicon = new HashMap(); - boolean terryKoo = "terryKoo".equals(flags.distSimFileFormat); - for (String line : ObjectBank.getLineIterator(flags.distSimLexicon, - flags.inputEncoding)) { - String word; - String wordClass; - if (terryKoo) { - String[] bits = line.split("\\t"); - word = bits[1]; - wordClass = bits[0]; - if (flags.distSimMaxBits > 0 && wordClass.length() > flags.distSimMaxBits) { - wordClass = wordClass.substring(0, flags.distSimMaxBits); - } - } else { - // "alexClark" - String[] bits = line.split("\\s+"); - word = bits[0]; - wordClass = bits[1]; - } - if ( ! flags.casedDistSim) { - word = word.toLowerCase(); - } - if (flags.numberEquivalenceDistSim) { - word = WordShapeClassifier.wordShape(word, WordShapeClassifier.WORDSHAPEDIGITS); - } - lexicon.put(word, wordClass); - } - Timing.endDoing(); - } - - - private void distSimAnnotate(PaddedList info) { - for (CoreLabel fl : info) { - if (fl.has(DistSimAnnotation.class)) { return; } - String word = fl.getString(TextAnnotation.class); - if ( ! flags.casedDistSim) { - word = word.toLowerCase(); - } - if (flags.numberEquivalenceDistSim) { - word = WordShapeClassifier.wordShape(word, WordShapeClassifier.WORDSHAPEDIGITS); - } - String distSim = lexicon.get(word); - if (distSim == null) { - distSim = flags.unknownWordDistSimClass; - } - fl.set(DistSimAnnotation.class, distSim); - } - } - - - private Map> wordToSubstrings = new HashMap>(); - - public void clearMemory() { - wordToSubstrings = new HashMap>(); - lexicon = null; - } - - private static String dehyphenate(String str) { - // don't take out leading or ending ones, just internal - // and remember padded with < > characters - String retStr = str; - int leng = str.length(); - int hyphen = 2; - do { - hyphen = retStr.indexOf('-', hyphen); - if (hyphen >= 0 && hyphen < leng - 2) { - retStr = retStr.substring(0, hyphen) + retStr.substring(hyphen + 1); - } else { - hyphen = -1; - } - } while (hyphen >= 0); - return retStr; - } - - private static String greekify(String str) { - // don't take out leading or ending ones, just internal - // and remember padded with < > characters - - String pattern = "(alpha)|(beta)|(gamma)|(delta)|(epsilon)|(zeta)|(kappa)|(lambda)|(rho)|(sigma)|(tau)|(upsilon)|(omega)"; - - Pattern p = Pattern.compile(pattern); - Matcher m = p.matcher(str); - return m.replaceAll("~"); - } - - /* end methods that do transformations */ - - /* - * static booleans that check strings for certain qualities * - */ - - // cdm: this could be improved to handle more name types, such as - // O'Reilly, DeGuzman, etc. (need a little classifier?!?) - private static boolean isNameCase(String str) { - if (str.length() < 2) { - return false; - } - if (!(Character.isUpperCase(str.charAt(0)) || Character.isTitleCase(str.charAt(0)))) { - return false; - } - for (int i = 1; i < str.length(); i++) { - if (Character.isUpperCase(str.charAt(i))) { - return false; - } - } - return true; - } - - private static boolean noUpperCase(String str) { - if (str.length() < 1) { - return false; - } - for (int i = 0; i < str.length(); i++) { - if (Character.isUpperCase(str.charAt(i))) { - return false; - } - } - return true; - } - - private static boolean hasLetter(String str) { - if (str.length() < 1) { - return false; - } - for (int i = 0; i < str.length(); i++) { - if (Character.isLetter(str.charAt(i))) { - return true; - } - } - return false; - } - - private static final Pattern ordinalPattern = Pattern.compile("(?:(?:first|second|third|fourth|fifth|"+ - "sixth|seventh|eighth|ninth|tenth|"+ - "eleventh|twelfth|thirteenth|"+ - "fourteenth|fifteenth|sixteenth|"+ - "seventeenth|eighteenth|nineteenth|"+ - "twenty|twentieth|thirty|thirtieth|"+ - "forty|fortieth|fifty|fiftieth|"+ - "sixty|sixtieth|seventy|seventieth|"+ - "eighty|eightieth|ninety|ninetieth|"+ - "one|two|three|four|five|six|seven|"+ - "eight|nine|hundred|hundredth)-?)+|[0-9]+(?:st|nd|rd|th)", Pattern.CASE_INSENSITIVE); - - - private static final Pattern numberPattern = Pattern.compile("[0-9]+"); - private static final Pattern ordinalEndPattern = Pattern.compile("(?:st|nd|rd|th)", Pattern.CASE_INSENSITIVE); - - private static boolean isOrdinal(List wordInfos, int pos) { - CoreLabel c = wordInfos.get(pos); - Matcher m = ordinalPattern.matcher(c.getString(TextAnnotation.class)); - if (m.matches()) { return true; } - m = numberPattern.matcher(c.getString(TextAnnotation.class)); - if (m.matches()) { - if (pos+1 < wordInfos.size()) { - CoreLabel n = wordInfos.get(pos+1); - m = ordinalEndPattern.matcher(n.getString(TextAnnotation.class)); - if (m.matches()) { return true; } - } - return false; - } - - m = ordinalEndPattern.matcher(c.getString(TextAnnotation.class)); - if (m.matches()) { - if (pos > 0) { - CoreLabel p = wordInfos.get(pos-1); - m = numberPattern.matcher(p.getString(TextAnnotation.class)); - if (m.matches()) { return true; } - } - } - if (c.getString(TextAnnotation.class).equals("-")) { - if (pos+1 < wordInfos.size() && pos > 0) { - CoreLabel p = wordInfos.get(pos-1); - CoreLabel n = wordInfos.get(pos+1); - m = ordinalPattern.matcher(p.getString(TextAnnotation.class)); - if (m.matches()) { - m = ordinalPattern.matcher(n.getString(TextAnnotation.class)); - if (m.matches()) { - return true; - } - } - } - } - return false; - } - - /* end static booleans that check strings for certain qualities */ - - /** - * Gazette Stuff. - */ - - private static class GazetteInfo implements Serializable { - String feature = ""; - int loc = 0; - String[] words = StringUtils.EMPTY_STRING_ARRAY; - private static final long serialVersionUID = -5903728481621584810L; - } // end class GazetteInfo - - private Map> wordToGazetteEntries = new HashMap>(); - private Map> wordToGazetteInfos = new HashMap>(); - - /** Reads a gazette file. Each line of it consists of a class name - * (a String not containing whitespace characters), followed by whitespace - * characters followed by a phrase, which is one or more tokens separated - * by a single space. - * - * @param in Where to read the gazette from - * @throws IOException If IO errors - */ - private void readGazette(BufferedReader in) throws IOException { - Pattern p = Pattern.compile("^(\\S+)\\s+(.+)$"); - for (String line; (line = in.readLine()) != null; ) { - Matcher m = p.matcher(line); - if (m.matches()) { - String type = intern(m.group(1)); - String phrase = m.group(2); - String[] words = phrase.split(" "); - for (int i = 0; i < words.length; i++) { - String word = intern(words[i]); - if (flags.sloppyGazette) { - Collection entries = wordToGazetteEntries.get(word); - if (entries == null) { - entries = new HashSet(); - wordToGazetteEntries.put(word, entries); - } - String feature = intern(type + "-GAZ" + words.length); - entries.add(feature); - } - if (flags.cleanGazette) { - Collection infos = wordToGazetteInfos.get(word); - if (infos == null) { - infos = new HashSet(); - wordToGazetteInfos.put(word, infos); - } - GazetteInfo info = new GazetteInfo(); - info.loc = i; - info.words = words; - info.feature = intern(type + "-GAZ" + words.length); - infos.add(info); - } - } - } - } - } - - private HashSet>> genericAnnotationKeys; // = null; //cache which keys are generic annotations so we don't have to do too many instanceof checks - - @SuppressWarnings({"unchecked", "SuspiciousMethodCalls"}) - private void makeGenericKeyCache(CoreLabel c) { - genericAnnotationKeys = new HashSet>>(); - for (Class key : c.keySet()) { - if (CoreLabel.genericValues.containsKey(key)) { - Class> genKey = (Class>) key; - genericAnnotationKeys.add(genKey); - } - } - } - - private HashSet lastNames; // = null; - private HashSet maleNames; // = null; - private HashSet femaleNames; // = null; - - private final Pattern titlePattern = Pattern.compile("(Mr|Ms|Mrs|Dr|Miss|Sen|Judge|Sir)\\.?"); // todo: should make static final and add more titles - - - protected Collection featuresC(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel n = cInfo.get(loc + 1); - CoreLabel n2 = cInfo.get(loc + 2); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - CoreLabel p3 = cInfo.get(loc - 3); - - String cWord = c.getString(TextAnnotation.class); - String pWord = p.getString(TextAnnotation.class); - String nWord = n.getString(TextAnnotation.class); - String cShape = c.getString(ShapeAnnotation.class); - - Collection featuresC = new ArrayList(); - - if (flags.useDistSim) { - distSimAnnotate(cInfo); - } - - if (flags.useDistSim && flags.useMoreTags) { - featuresC.add(p.get(DistSimAnnotation.class) + '-' + cWord + "-PDISTSIM-CWORD"); - } - - - if (flags.useDistSim) { - featuresC.add(c.get(DistSimAnnotation.class) + "-DISTSIM"); - } - - - if (flags.useTitle) { - Matcher m = titlePattern.matcher(cWord); - if (m.matches()) { - featuresC.add("IS_TITLE"); - } - } - - - if (flags.useInternal && flags.useExternal ) { - - if (flags.useWord) { - featuresC.add(cWord + "-WORD"); - } - - if (flags.use2W) { - featuresC.add(p2.getString(TextAnnotation.class) + "-P2W"); - featuresC.add(n2.getString(TextAnnotation.class) + "-N2W"); - } - - if (flags.useLC) { - featuresC.add(cWord.toLowerCase() + "-CL"); - featuresC.add(pWord.toLowerCase() + "-PL"); - featuresC.add(nWord.toLowerCase() + "-NL"); - } - - if (flags.useUnknown) { // for true casing - featuresC.add(c.get(UnknownAnnotation.class)+"-UNKNOWN"); - featuresC.add(p.get(UnknownAnnotation.class)+"-PUNKNOWN"); - featuresC.add(n.get(UnknownAnnotation.class)+"-NUNKNOWN"); - } - - if (flags.useLemmas) { - String lem = c.getString(LemmaAnnotation.class); - if (! "".equals(lem)) { - featuresC.add(lem + "-LEM"); - } - } - if (flags.usePrevNextLemmas) { - String plem = p.getString(LemmaAnnotation.class); - String nlem = n.getString(LemmaAnnotation.class); - if (! "".equals(plem)) { - featuresC.add(plem + "-PLEM"); - } - if (! "".equals(nlem)) { - featuresC.add(nlem + "-NLEM"); - } - } - - if (flags.checkNameList) { - try { - if (lastNames == null) { - lastNames = new HashSet(); - - for (String line : ObjectBank.getLineIterator(flags.lastNameList)) { - String[] cols = line.split("\\s+"); - lastNames.add(cols[0]); - } - } - if (maleNames == null) { - maleNames = new HashSet(); - for (String line : ObjectBank.getLineIterator(flags.maleNameList)) { - String[] cols = line.split("\\s+"); - maleNames.add(cols[0]); - } - } - if (femaleNames == null) { - femaleNames = new HashSet(); - for (String line : ObjectBank.getLineIterator(flags.femaleNameList)) { - String[] cols = line.split("\\s+"); - femaleNames.add(cols[0]); - } - } - - String name = cWord.toUpperCase(); - if (lastNames.contains(name)) { - featuresC.add("LAST_NAME"); - } - - if (maleNames.contains(name)) { - featuresC.add("MALE_NAME"); - } - - if (femaleNames.contains(name)) { - featuresC.add("FEMALE_NAME"); - } - - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - if (flags.binnedLengths != null) { - int len = cWord.length(); - String featureName = null; - for (int i = 0; i <= flags.binnedLengths.length; i++) { - if (i == flags.binnedLengths.length) { - featureName = "Len-" + flags.binnedLengths[flags.binnedLengths.length - 1] + "-Inf"; - } else if (len <= flags.binnedLengths[i]) { - featureName = "Len-" + ((i == 0) ? 1 : flags.binnedLengths[i - 1]) + '-' + flags.binnedLengths[i]; - break; - } - } - featuresC.add(featureName); - } - - if (flags.useABGENE) { - featuresC.add(c.get(AbgeneAnnotation.class) + "-ABGENE"); - featuresC.add(p.get(AbgeneAnnotation.class) + "-PABGENE"); - featuresC.add(n.get(AbgeneAnnotation.class) + "-NABGENE"); - } - - if (flags.useABSTRFreqDict) { - featuresC.add(c.get(AbstrAnnotation.class) + "-ABSTRACT" + c.get(FreqAnnotation.class) + "-FREQ" + c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - featuresC.add(c.get(AbstrAnnotation.class) + "-ABSTRACT" + c.get(DictAnnotation.class) + "-DICT" + c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - featuresC.add(c.get(AbstrAnnotation.class) + "-ABSTRACT" + c.get(DictAnnotation.class) + "-DICT" + c.get(FreqAnnotation.class) + "-FREQ" + c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - } - - if (flags.useABSTR) { - featuresC.add(c.get(AbstrAnnotation.class) + "-ABSTRACT"); - featuresC.add(p.get(AbstrAnnotation.class) + "-PABSTRACT"); - featuresC.add(n.get(AbstrAnnotation.class) + "-NABSTRACT"); - } - - if (flags.useGENIA) { - featuresC.add(c.get(GeniaAnnotation.class) + "-GENIA"); - featuresC.add(p.get(GeniaAnnotation.class) + "-PGENIA"); - featuresC.add(n.get(GeniaAnnotation.class) + "-NGENIA"); - } - if (flags.useWEBFreqDict) { - featuresC.add(c.get(WebAnnotation.class) + "-WEB" + c.get(FreqAnnotation.class) + "-FREQ" + c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - featuresC.add(c.get(WebAnnotation.class) + "-WEB" + c.get(DictAnnotation.class) + "-DICT" + c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - featuresC.add(c.get(WebAnnotation.class) + "-WEB" + c.get(DictAnnotation.class) + "-DICT" + c.get(FreqAnnotation.class) + "-FREQ" + c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - } - - if (flags.useWEB) { - featuresC.add(c.get(WebAnnotation.class) + "-WEB"); - featuresC.add(p.get(WebAnnotation.class) + "-PWEB"); - featuresC.add(n.get(WebAnnotation.class) + "-NWEB"); - } - - if (flags.useIsURL) { - featuresC.add(c.get(IsURLAnnotation.class) + "-ISURL"); - } - if (flags.useEntityRule) { - featuresC.add(c.get(EntityRuleAnnotation.class)+"-ENTITYRULE"); - } - if (flags.useEntityTypes) { - featuresC.add(c.get(EntityTypeAnnotation.class) + "-ENTITYTYPE"); - } - if (flags.useIsDateRange) { - featuresC.add(c.get(IsDateRangeAnnotation.class) + "-ISDATERANGE"); - } - - if (flags.useABSTRFreq) { - featuresC.add(c.get(AbstrAnnotation.class) + "-ABSTRACT" + c.get(FreqAnnotation.class) + "-FREQ"); - } - - if (flags.useFREQ) { - featuresC.add(c.get(FreqAnnotation.class) + "-FREQ"); - } - - if (flags.useMoreTags) { - featuresC.add(p.getString(PartOfSpeechAnnotation.class) + '-' + cWord + "-PTAG-CWORD"); - } - - if (flags.usePosition) { - featuresC.add(c.get(PositionAnnotation.class) + "-POSITION"); - } - if (flags.useBeginSent) { - if ("0".equals(c.get(PositionAnnotation.class))) { - featuresC.add("BEGIN-SENT"); - featuresC.add(cShape + "-BEGIN-SENT"); - } else { - featuresC.add("IN-SENT"); - featuresC.add(cShape + "-IN-SENT"); - } - } - if (flags.useTags) { - featuresC.add(c.getString(PartOfSpeechAnnotation.class) + "-TAG"); - } - - if (flags.useOrdinal) { - if (isOrdinal(cInfo, loc)) { - featuresC.add("C_ORDINAL"); - if (isOrdinal(cInfo, loc-1)) { - //System.err.print(p.getString(TextAnnotation.class)+" "); - featuresC.add("PC_ORDINAL"); - } - //System.err.println(c.getString(TextAnnotation.class)); - } - if (isOrdinal(cInfo, loc-1)) { - featuresC.add("P_ORDINAL"); - } - } - - if (flags.usePrev) { - featuresC.add(p.getString(TextAnnotation.class) + "-PW"); - if (flags.useTags) { - featuresC.add(p.getString(PartOfSpeechAnnotation.class) + "-PTAG"); - } - if (flags.useDistSim) { - featuresC.add(p.get(DistSimAnnotation.class) + "-PDISTSIM"); - } - if (flags.useIsURL) { - featuresC.add(p.get(IsURLAnnotation.class) + "-PISURL"); - } - if (flags.useEntityTypes) { - featuresC.add(p.get(EntityTypeAnnotation.class) + "-PENTITYTYPE"); - } - } - - if (flags.useNext) { - featuresC.add(n.getString(TextAnnotation.class) + "-NW"); - if (flags.useTags) { - featuresC.add(n.getString(PartOfSpeechAnnotation.class) + "-NTAG"); - } - if (flags.useDistSim) { - featuresC.add(n.get(DistSimAnnotation.class) + "-NDISTSIM"); - } - if (flags.useIsURL) { - featuresC.add(n.get(IsURLAnnotation.class) + "-NISURL"); - } - if (flags.useEntityTypes) { - featuresC.add(n.get(EntityTypeAnnotation.class) + "-NENTITYTYPE"); - } - } - /*here, entityTypes refers to the type in the PASCAL IE challenge: - * i.e. certain words are tagged "Date" or "Location" */ - - - if (flags.useEitherSideWord) { - featuresC.add(pWord + "-EW"); - featuresC.add(nWord + "-EW"); - } - - if (flags.useWordPairs) { - featuresC.add(cWord + '-' + pWord + "-W-PW"); - featuresC.add(cWord + '-' + nWord + "-W-NW"); - } - - if (flags.useSymTags) { - if (flags.useTags) { - featuresC.add(p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + '-' + n.getString(PartOfSpeechAnnotation.class) + "-PCNTAGS"); - featuresC.add(c.getString(PartOfSpeechAnnotation.class) + '-' + n.getString(PartOfSpeechAnnotation.class) + "-CNTAGS"); - featuresC.add(p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-PCTAGS"); - } - if (flags.useDistSim) { - featuresC.add(p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + '-' + n.get(DistSimAnnotation.class) + "-PCNDISTSIM"); - featuresC.add(c.get(DistSimAnnotation.class) + '-' + n.get(DistSimAnnotation.class) + "-CNDISTSIM"); - featuresC.add(p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-PCDISTSIM"); - } - - } - - - if (flags.useSymWordPairs) { - featuresC.add(pWord + '-' + nWord + "-SWORDS"); - } - - if (flags.useGazFeatures) { - if (!c.get(GazAnnotation.class).equals(flags.dropGaz)) { - featuresC.add(c.get(GazAnnotation.class) + "-GAZ"); - } - if (!n.get(GazAnnotation.class).equals(flags.dropGaz)) { - featuresC.add(n.get(GazAnnotation.class) + "-NGAZ"); - } - if (!p.get(GazAnnotation.class).equals(flags.dropGaz)) { - featuresC.add(p.get(GazAnnotation.class) + "-PGAZ"); - } - } - - if (flags.useMoreGazFeatures) { - if (!c.get(GazAnnotation.class).equals(flags.dropGaz)) { - featuresC.add(c.get(GazAnnotation.class) + '-' + cWord + "-CG-CW-GAZ"); - if (!n.get(GazAnnotation.class).equals(flags.dropGaz)) { - featuresC.add(c.get(GazAnnotation.class) + '-' + n.get(GazAnnotation.class) + "-CNGAZ"); - } - if (!p.get(GazAnnotation.class).equals(flags.dropGaz)) { - featuresC.add(p.get(GazAnnotation.class) + '-' + c.get(GazAnnotation.class) + "-PCGAZ"); - } - } - } - - if (flags.useAbbr || flags.useMinimalAbbr) { - featuresC.add(c.get(AbbrAnnotation.class) + "-ABBR"); - } - - if (flags.useAbbr1 || flags.useMinimalAbbr1) { - if (!c.get(AbbrAnnotation.class).equals("XX")) { - featuresC.add(c.get(AbbrAnnotation.class) + "-ABBR"); - } - } - - if (flags.useAbbr) { - featuresC.add(p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-PCABBR"); - featuresC.add(c.get(AbbrAnnotation.class) + '-' + n.get(AbbrAnnotation.class) + "-CNABBR"); - featuresC.add(p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + '-' + n.get(AbbrAnnotation.class) + "-PCNABBR"); - } - - if (flags.useAbbr1) { - if (!c.get(AbbrAnnotation.class).equals("XX")) { - featuresC.add(p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-PCABBR"); - featuresC.add(c.get(AbbrAnnotation.class) + '-' + n.get(AbbrAnnotation.class) + "-CNABBR"); - featuresC.add(p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + '-' + n.get(AbbrAnnotation.class) + "-PCNABBR"); - } - } - - if (flags.useChunks) { - featuresC.add(p.get(ChunkAnnotation.class) + '-' + c.get(ChunkAnnotation.class) + "-PCCHUNK"); - featuresC.add(c.get(ChunkAnnotation.class) + '-' + n.get(ChunkAnnotation.class) + "-CNCHUNK"); - featuresC.add(p.get(ChunkAnnotation.class) + '-' + c.get(ChunkAnnotation.class) + '-' + n.get(ChunkAnnotation.class) + "-PCNCHUNK"); - } - - if (flags.useMinimalAbbr) { - featuresC.add(cWord + '-' + c.get(AbbrAnnotation.class) + "-CWABB"); - } - - if (flags.useMinimalAbbr1) { - if (!c.get(AbbrAnnotation.class).equals("XX")) { - featuresC.add(cWord + '-' + c.get(AbbrAnnotation.class) + "-CWABB"); - } - } - - String prevVB = "", nextVB = ""; - if (flags.usePrevVB) { - for (int j = loc - 1; ; j--) { - CoreLabel wi = cInfo.get(j); - if (wi == cInfo.getPad()) { - prevVB = "X"; - featuresC.add("X-PVB"); - break; - } else if (wi.getString(PartOfSpeechAnnotation.class).startsWith("VB")) { - featuresC.add(wi.getString(TextAnnotation.class) + "-PVB"); - prevVB = wi.getString(TextAnnotation.class); - break; - } - } - } - - if (flags.useNextVB) { - for (int j = loc + 1; ; j++) { - CoreLabel wi = cInfo.get(j); - if (wi == cInfo.getPad()) { - featuresC.add("X-NVB"); - nextVB = "X"; - break; - } else if (wi.getString(PartOfSpeechAnnotation.class).startsWith("VB")) { - featuresC.add(wi.getString(TextAnnotation.class) + "-NVB"); - nextVB = wi.getString(TextAnnotation.class); - break; - } - } - } - - if (flags.useVB) { - featuresC.add(prevVB + '-' + nextVB + "-PNVB"); - } - - if (flags.useShapeConjunctions) { - featuresC.add(c.get(PositionAnnotation.class) + cShape + "-POS-SH"); - if (flags.useTags) { - featuresC.add(c.tag() + cShape + "-TAG-SH"); - } - if (flags.useDistSim) { - featuresC.add(c.get(DistSimAnnotation.class) + cShape + "-DISTSIM-SH"); - } - - } - - if (flags.useWordTag) { - featuresC.add(cWord + '-' + c.getString(PartOfSpeechAnnotation.class) + "-W-T"); - featuresC.add(cWord + '-' + p.getString(PartOfSpeechAnnotation.class) + "-W-PT"); - featuresC.add(cWord + '-' + n.getString(PartOfSpeechAnnotation.class) + "-W-NT"); - } - - if (flags.useNPHead) { - featuresC.add(c.get(TreeCoreAnnotations.HeadWordAnnotation.class) + "-HW"); - if (flags.useTags) { - featuresC.add(c.get(TreeCoreAnnotations.HeadWordAnnotation.class) + "-" + c.getString(PartOfSpeechAnnotation.class) + "-HW-T"); - } - if (flags.useDistSim) { - featuresC.add(c.get(TreeCoreAnnotations.HeadWordAnnotation.class) + "-" + c.get(DistSimAnnotation.class) + "-HW-DISTSIM"); - } - } - - if (flags.useNPGovernor) { - featuresC.add(c.get(GovernorAnnotation.class) + "-GW"); - if (flags.useTags) { - featuresC.add(c.get(GovernorAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-GW-T"); - } - if (flags.useDistSim) { - featuresC.add(c.get(GovernorAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-DISTSIM-T1"); - } - } - - if (flags.useHeadGov) { - featuresC.add(c.get(TreeCoreAnnotations.HeadWordAnnotation.class) + "-" + c.get(GovernorAnnotation.class) + "-HW_GW"); - } - - if (flags.useClassFeature) { - featuresC.add("###"); - } - - if (flags.useFirstWord) { - String firstWord = cInfo.get(0).getString(TextAnnotation.class); - featuresC.add(firstWord); - } - - if (flags.useNGrams) { - Collection subs = wordToSubstrings.get(cWord); - if (subs == null) { - subs = new ArrayList(); - String word = '<' + cWord + '>'; - if (flags.lowercaseNGrams) { - word = word.toLowerCase(); - } - if (flags.dehyphenateNGrams) { - word = dehyphenate(word); - } - if (flags.greekifyNGrams) { - word = greekify(word); - } - for (int i = 0; i < word.length(); i++) { - for (int j = i + 2; j <= word.length(); j++) { - if (flags.noMidNGrams && i != 0 && j != word.length()) { - continue; - } - if (flags.maxNGramLeng >= 0 && j - i > flags.maxNGramLeng) { - continue; - } - subs.add(intern('#' + word.substring(i, j) + '#')); - } - } - if (flags.cacheNGrams) { - wordToSubstrings.put(cWord, subs); - } - } - featuresC.addAll(subs); - if (flags.conjoinShapeNGrams) { - for (String str : subs) { - String feat = str + '-' + cShape + "-CNGram-CS"; - featuresC.add(feat); - } - } - } - - if (flags.useGazettes) { - if (flags.sloppyGazette) { - Collection entries = wordToGazetteEntries.get(cWord); - if (entries != null) { - featuresC.addAll(entries); - } - } - if (flags.cleanGazette) { - Collection infos = wordToGazetteInfos.get(cWord); - if (infos != null) { - for (GazetteInfo gInfo : infos) { - boolean ok = true; - for (int gLoc = 0; gLoc < gInfo.words.length; gLoc++) { - ok &= gInfo.words[gLoc].equals(cInfo.get(loc + gLoc - gInfo.loc).getString(TextAnnotation.class)); - } - if (ok) { - featuresC.add(gInfo.feature); - } - } - } - } - } - - if ((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || (flags.useShapeStrings)) { - featuresC.add(cShape + "-TYPE"); - if (flags.useTypeSeqs) { - String pShape = p.get(ShapeAnnotation.class); - String nShape = n.get(ShapeAnnotation.class); - featuresC.add(pShape + "-PTYPE"); - featuresC.add(nShape + "-NTYPE"); - featuresC.add(pWord + "..." + cShape + "-PW_CTYPE"); - featuresC.add(cShape + "..." + nWord + "-NW_CTYPE"); - featuresC.add(pShape + "..." + cShape + "-PCTYPE"); - featuresC.add(cShape + "..." + nShape + "-CNTYPE"); - featuresC.add(pShape + "..." + cShape + "..." + nShape + "-PCNTYPE"); - } - } - - if (flags.useLastRealWord) { - if (pWord.length() <= 3) { - // extending this to check for 2 short words doesn't seem to help.... - featuresC.add(p2.getString(TextAnnotation.class) + "..." + cShape + "-PPW_CTYPE"); - } - } - - if (flags.useNextRealWord) { - if (nWord.length() <= 3) { - // extending this to check for 2 short words doesn't seem to help.... - featuresC.add(n2.getString(TextAnnotation.class) + "..." + cShape + "-NNW_CTYPE"); - } - } - - if (flags.useOccurrencePatterns) { - featuresC.addAll(occurrencePatterns(cInfo, loc)); - } - - if (flags.useDisjunctive) { - for (int i = 1; i <= flags.disjunctionWidth; i++) { - CoreLabel dn = cInfo.get(loc + i); - CoreLabel dp = cInfo.get(loc - i); - featuresC.add(dn.getString(TextAnnotation.class) + "-DISJN"); - if (flags.useDisjunctiveShapeInteraction) { - featuresC.add(dn.getString(TextAnnotation.class) + '-' + cShape + "-DISJN-CS"); - } - featuresC.add(dp.getString(TextAnnotation.class) + "-DISJP"); - if (flags.useDisjunctiveShapeInteraction) { - featuresC.add(dp.getString(TextAnnotation.class) + '-' + cShape + "-DISJP-CS"); - } - } - } - - if (flags.useWideDisjunctive) { - for (int i = 1; i <= flags.wideDisjunctionWidth; i++) { - featuresC.add(cInfo.get(loc + i).getString(TextAnnotation.class) + "-DISJWN"); - featuresC.add(cInfo.get(loc - i).getString(TextAnnotation.class) + "-DISJWP"); - } - } - - if (flags.useEitherSideDisjunctive) { - for (int i = 1; i <= flags.disjunctionWidth; i++) { - featuresC.add(cInfo.get(loc + i).getString(TextAnnotation.class) + "-DISJWE"); - featuresC.add(cInfo.get(loc - i).getString(TextAnnotation.class) + "-DISJWE"); - } - } - - if (flags.useDisjShape) { - for (int i = 1; i <= flags.disjunctionWidth; i++) { - featuresC.add(cInfo.get(loc + i).get(ShapeAnnotation.class) + "-NDISJSHAPE"); - // featuresC.add(cInfo.get(loc - i).get(ShapeAnnotation.class) + "-PDISJSHAPE"); - featuresC.add(cShape + '-' + cInfo.get(loc + i).get(ShapeAnnotation.class) + "-CNDISJSHAPE"); - // featuresC.add(c.get(ShapeAnnotation.class) + "-" + cInfo.get(loc - i).get(ShapeAnnotation.class) + "-CPDISJSHAPE"); - } - } - - if (flags.useExtraTaggySequences) { - if (flags.useTags) { - featuresC.add(p2.getString(PartOfSpeechAnnotation.class) + '-' + p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-TTS"); - featuresC.add(p3.getString(PartOfSpeechAnnotation.class) + '-' + p2.getString(PartOfSpeechAnnotation.class) + '-' + p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-TTTS"); - } - if (flags.useDistSim) { - featuresC.add(p2.get(DistSimAnnotation.class) + '-' + p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-DISTSIM_TTS1"); - featuresC.add(p3.get(DistSimAnnotation.class) + '-' + p2.get(DistSimAnnotation.class) + '-' + p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-DISTSIM_TTTS1"); - } - } - - if (flags.useMUCFeatures) { - featuresC.add(c.get(SectionAnnotation.class)+"-SECTION"); - featuresC.add(c.get(WordPositionAnnotation.class)+"-WORD_POSITION"); - featuresC.add(c.get(CoreAnnotations.SentencePositionAnnotation.class)+"-SENT_POSITION"); - featuresC.add(c.get(ParaPositionAnnotation.class)+"-PARA_POSITION"); - featuresC.add(c.get(WordPositionAnnotation.class)+ '-' +c.get(ShapeAnnotation.class)+"-WORD_POSITION_SHAPE"); - } - } else if (flags.useInternal) { - - if (flags.useWord) { - featuresC.add(cWord + "-WORD"); - } - - if (flags.useNGrams) { - Collection subs = wordToSubstrings.get(cWord); - if (subs == null) { - subs = new ArrayList(); - String word = '<' + cWord + '>'; - if (flags.lowercaseNGrams) { - word = word.toLowerCase(); - } - if (flags.dehyphenateNGrams) { - word = dehyphenate(word); - } - if (flags.greekifyNGrams) { - word = greekify(word); - } - for (int i = 0; i < word.length(); i++) { - for (int j = i + 2; j <= word.length(); j++) { - if (flags.noMidNGrams && i != 0 && j != word.length()) { - continue; - } - if (flags.maxNGramLeng >= 0 && j - i > flags.maxNGramLeng) { - continue; - } - //subs.add(intern("#" + word.substring(i, j) + "#")); - subs.add(intern('#' + word.substring(i, j) + '#')); - } - } - if (flags.cacheNGrams) { - wordToSubstrings.put(cWord, subs); - } - } - featuresC.addAll(subs); - if (flags.conjoinShapeNGrams) { - String shape = c.get(ShapeAnnotation.class); - for (String str : subs) { - String feat = str + '-' + shape + "-CNGram-CS"; - featuresC.add(feat); - } - } - } - - if ((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || (flags.useShapeStrings)) { - featuresC.add(cShape + "-TYPE"); - } - - if (flags.useOccurrencePatterns) { - featuresC.addAll(occurrencePatterns(cInfo, loc)); - } - - } else if (flags.useExternal) { - - if (flags.usePrev) { - featuresC.add(pWord + "-PW"); - } - - if (flags.useNext) { - featuresC.add(nWord + "-NW"); - } - - if (flags.useWordPairs) { - featuresC.add(cWord + '-' + pWord + "-W-PW"); - featuresC.add(cWord + '-' + nWord + "-W-NW"); - } - - if (flags.useSymWordPairs) { - featuresC.add(pWord + '-' + nWord + "-SWORDS"); - } - - if ((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || (flags.useShapeStrings)) { - if (flags.useTypeSeqs) { - String pShape = p.get(ShapeAnnotation.class); - String nShape = n.get(ShapeAnnotation.class); - featuresC.add(pShape + "-PTYPE"); - featuresC.add(nShape + "-NTYPE"); - featuresC.add(pWord + "..." + cShape + "-PW_CTYPE"); - featuresC.add(cShape + "..." + nWord + "-NW_CTYPE"); - if (flags.maxLeft > 0) featuresC.add(pShape + "..." + cShape + "-PCTYPE"); // this one just isn't useful, at least given c,pc,s,ps. Might be useful 0th-order - featuresC.add(cShape + "..." + nShape + "-CNTYPE"); - featuresC.add(pShape + "..." + cShape + "..." + nShape + "-PCNTYPE"); - } - } - - if (flags.useLastRealWord) { - if (pWord.length() <= 3) { - featuresC.add(p2.getString(TextAnnotation.class) + "..." + cShape + "-PPW_CTYPE"); - } - } - - if (flags.useNextRealWord) { - if (nWord.length() <= 3) { - featuresC.add(n2.getString(TextAnnotation.class) + "..." + cShape + "-NNW_CTYPE"); - } - } - - if (flags.useDisjunctive) { - for (int i = 1; i <= flags.disjunctionWidth; i++) { - CoreLabel dn = cInfo.get(loc + i); - CoreLabel dp = cInfo.get(loc - i); - featuresC.add(dn.getString(TextAnnotation.class) + "-DISJN"); - if (flags.useDisjunctiveShapeInteraction) { - featuresC.add(dn.getString(TextAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-DISJN-CS"); - } - featuresC.add(dp.getString(TextAnnotation.class) + "-DISJP"); - if (flags.useDisjunctiveShapeInteraction) { - featuresC.add(dp.getString(TextAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-DISJP-CS"); - } - } - } - - if (flags.useWideDisjunctive) { - for (int i = 1; i <= flags.wideDisjunctionWidth; i++) { - featuresC.add(cInfo.get(loc + i).getString(TextAnnotation.class) + "-DISJWN"); - featuresC.add(cInfo.get(loc - i).getString(TextAnnotation.class) + "-DISJWP"); - } - } - - if (flags.useDisjShape) { - for (int i = 1; i <= flags.disjunctionWidth; i++) { - featuresC.add(cInfo.get(loc + i).get(ShapeAnnotation.class) + "-NDISJSHAPE"); - // featuresC.add(cInfo.get(loc - i).get(ShapeAnnotation.class) + "-PDISJSHAPE"); - featuresC.add(c.get(ShapeAnnotation.class) + '-' + cInfo.get(loc + i).get(ShapeAnnotation.class) + "-CNDISJSHAPE"); - // featuresC.add(c.get(ShapeAnnotation.class) + "-" + cInfo.get(loc - i).get(ShapeAnnotation.class) + "-CPDISJSHAPE"); - } - } - - } - - // Stuff to add binary features from the additional columns - if (flags.twoStage) { - featuresC.add(c.get(Bin1Annotation.class) + "-BIN1"); - featuresC.add(c.get(Bin2Annotation.class) + "-BIN2"); - featuresC.add(c.get(Bin3Annotation.class) + "-BIN3"); - featuresC.add(c.get(Bin4Annotation.class) + "-BIN4"); - featuresC.add(c.get(Bin5Annotation.class) + "-BIN5"); - featuresC.add(c.get(Bin6Annotation.class) + "-BIN6"); - } - - if(flags.useIfInteger){ - try { - int val = Integer.parseInt(cWord); - if(val > 0) featuresC.add("POSITIVE_INTEGER"); - else if(val < 0) featuresC.add("NEGATIVE_INTEGER"); - // System.err.println("FOUND INTEGER"); - } catch(NumberFormatException e){ - // not an integer value, nothing to do - } - } - - //Stuff to add arbitrary features - if (flags.useGenericFeatures) { - //see if we need to cach the keys - if (genericAnnotationKeys == null) { - makeGenericKeyCache(c); - } - //now look through the cached keys - for (Class key : genericAnnotationKeys) { - System.err.println("Adding feature: " + CoreLabel.genericValues.get(key) + " with value " + c.get(key)); - featuresC.add(c.get(key) + "-" + CoreLabel.genericValues.get(key)); - } - } - - if(flags.useTopics){ - //featuresC.add(p.get(TopicAnnotation.class) + '-' + cWord + "--CWORD"); - featuresC.add(c.get(TopicAnnotation.class)+ "-TopicID"); - featuresC.add(p.get(TopicAnnotation.class) + "-PTopicID"); - featuresC.add(n.get(TopicAnnotation.class) + "-NTopicID"); - //featuresC.add(p.get(TopicAnnotation.class) + '-' + c.get(TopicAnnotation.class) + '-' + n.get(TopicAnnotation.class) + "-PCNTopicID"); - //featuresC.add(c.get(TopicAnnotation.class) + '-' + n.get(TopicAnnotation.class) + "-CNTopicID"); - //featuresC.add(p.get(TopicAnnotation.class) + '-' + c.get(TopicAnnotation.class) + "-PCTopicID"); - //featuresC.add(c.get(TopicAnnotation.class) + cShape + "-TopicID-SH"); - //asdasd - } - - // NER tag annotations from a previous NER system - if (c.get(StackedNamedEntityTagAnnotation.class) != null) { - featuresC.add(c.get(StackedNamedEntityTagAnnotation.class)+ "-CStackedNERTag"); - featuresC.add(cWord + "-" + c.get(StackedNamedEntityTagAnnotation.class)+ "-WCStackedNERTag"); - - if (flags.useNext) { - featuresC.add(c.get(StackedNamedEntityTagAnnotation.class) + '-' + n.get(StackedNamedEntityTagAnnotation.class) + "-CNStackedNERTag"); - featuresC.add(cWord + "-" + c.get(StackedNamedEntityTagAnnotation.class) + '-' + n.get(StackedNamedEntityTagAnnotation.class) + "-WCNStackedNERTag"); - - if (flags.usePrev) { - featuresC.add(p.get(StackedNamedEntityTagAnnotation.class) + '-' + c.get(StackedNamedEntityTagAnnotation.class) + '-' + n.get(StackedNamedEntityTagAnnotation.class) + "-PCNStackedNERTag"); - featuresC.add(p.get(StackedNamedEntityTagAnnotation.class) + '-' + cWord + " -" + c.get(StackedNamedEntityTagAnnotation.class) - + '-' + n.get(StackedNamedEntityTagAnnotation.class) + "-PWCNStackedNERTag"); - } - } - if (flags.usePrev) { - featuresC.add(p.get(StackedNamedEntityTagAnnotation.class) + '-' + c.get(StackedNamedEntityTagAnnotation.class) + "-PCStackedNERTag"); - } - } - if(flags.useWordnetFeatures) - featuresC.add(c.get(WordnetSynAnnotation.class)+"-WordnetSyn"); - if(flags.useProtoFeatures) - featuresC.add(c.get(ProtoAnnotation.class)+"-Proto"); - - return featuresC; - } - - /** - * Binary feature annotations - */ - private static class Bin1Annotation implements CoreAnnotation { - public Class getType() { return String.class; } } - - private static class Bin2Annotation implements CoreAnnotation { - public Class getType() { return String.class; } } - - private static class Bin3Annotation implements CoreAnnotation { - public Class getType() { return String.class; } } - - private static class Bin4Annotation implements CoreAnnotation { - public Class getType() { return String.class; } } - - private static class Bin5Annotation implements CoreAnnotation { - public Class getType() { return String.class; } } - - private static class Bin6Annotation implements CoreAnnotation { - public Class getType() { return String.class; } } - - - - protected Collection featuresCpC(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel n = cInfo.get(loc + 1); - CoreLabel p = cInfo.get(loc - 1); - - String cWord = c.getString(TextAnnotation.class); - String pWord = p.getString(TextAnnotation.class); - String cDS = c.getString(DistSimAnnotation.class); - String pDS = p.getString(DistSimAnnotation.class); - String cShape = c.getString(ShapeAnnotation.class); - String pShape = p.getString(ShapeAnnotation.class); - Collection featuresCpC = new ArrayList(); - - if (flags.useInternal && flags.useExternal ) { - - if (flags.useOrdinal) { - if (isOrdinal(cInfo, loc)) { - featuresCpC.add("C_ORDINAL"); - if (isOrdinal(cInfo, loc-1)) { - featuresCpC.add("PC_ORDINAL"); - } - } - if (isOrdinal(cInfo, loc-1)) { - featuresCpC.add("P_ORDINAL"); - } - } - - if (flags.useAbbr || flags.useMinimalAbbr) { - featuresCpC.add(p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-PABBRANS"); - } - - if (flags.useAbbr1 || flags.useMinimalAbbr1) { - if (!c.get(AbbrAnnotation.class).equals("XX")) { - featuresCpC.add(p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-PABBRANS"); - } - } - - if (flags.useChunkySequences) { - featuresCpC.add(p.get(ChunkAnnotation.class) + '-' + c.get(ChunkAnnotation.class) + '-' + n.get(ChunkAnnotation.class) + "-PCNCHUNK"); - } - - if (flags.usePrev) { - if (flags.useSequences && flags.usePrevSequences) { - featuresCpC.add("PSEQ"); - featuresCpC.add(cWord + "-PSEQW"); - featuresCpC.add(pWord+ '-' +cWord + "-PSEQW2"); - - featuresCpC.add(pWord + "-PSEQpW"); - - featuresCpC.add(pDS + "-PSEQpDS"); - featuresCpC.add(cDS + "-PSEQcDS"); - featuresCpC.add(pDS+ '-' +cDS + "-PSEQpcDS"); - - if (((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || flags.useShapeStrings)) { - featuresCpC.add(pShape + "-PSEQpS"); - featuresCpC.add(cShape + "-PSEQcS"); - featuresCpC.add(pShape+ '-' +cShape + "-PSEQpcS"); - } - } - } - - if (((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || - flags.useShapeStrings) - && flags.useTypeSeqs && (flags.useTypeSeqs2 || flags.useTypeSeqs3)) { -// String pShape = p.get(ShapeAnnotation.class); -// String cShape = c.get(ShapeAnnotation.class); - if (flags.useTypeSeqs3) { - featuresCpC.add(pShape + '-' + cShape + '-' + n.get(ShapeAnnotation.class) + "-PCNSHAPES"); - } - if (flags.useTypeSeqs2) { - featuresCpC.add(pShape + '-' + cShape + "-TYPES"); - } - - if (flags.useYetMoreCpCShapes) { - String p2Shape = cInfo.get(loc - 2).getString(ShapeAnnotation.class); - featuresCpC.add(p2Shape + '-' + pShape + '-' + cShape + "-YMS"); - featuresCpC.add(pShape + '-' + cShape + "-" + n.getString(ShapeAnnotation.class) + "-YMSPCN"); - - } - } - - if (flags.useTypeySequences) { - featuresCpC.add(c.get(ShapeAnnotation.class) + "-TPS2"); - featuresCpC.add(n.get(ShapeAnnotation.class) + "-TNS1"); - // featuresCpC.add(p.get(ShapeAnnotation.class) + "-" + c.get(ShapeAnnotation.class) + "-TPS"); // duplicates -TYPES, so now omitted; you may need to slighly increase sigma to duplicate previous results, however. - } - - if (flags.useTaggySequences) { - if (flags.useTags) { - featuresCpC.add(p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-TS"); - } - if (flags.useDistSim) { - featuresCpC.add(p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-DISTSIM_TS1"); - } - } - - if (flags.useParenMatching) { - if (flags.useReverse) { - if (cWord.equals("(") || cWord.equals("[") || cWord.equals("-LRB-")) { - if (p.getString(TextAnnotation.class).equals(")") || p.getString(TextAnnotation.class).equals("]") || p.getString(TextAnnotation.class).equals("-RRB-")) { - featuresCpC.add("PAREN-MATCH"); - } - } - } else { - if (cWord.equals(")") || cWord.equals("]") || cWord.equals("-RRB-")) { - if (p.getString(TextAnnotation.class).equals("(") || p.getString(TextAnnotation.class).equals("[") || p.getString(TextAnnotation.class).equals("-LRB-")) { - featuresCpC.add("PAREN-MATCH"); - } - } - } - } - if (flags.useEntityTypeSequences) { - featuresCpC.add(p.get(EntityTypeAnnotation.class) + '-' + c.get(EntityTypeAnnotation.class) + "-ETSEQ"); - } - if (flags.useURLSequences) { - featuresCpC.add(p.get(IsURLAnnotation.class) + '-' + c.get(IsURLAnnotation.class) + "-URLSEQ"); - } - } else if (flags.useInternal) { - - if (flags.useSequences && flags.usePrevSequences) { - featuresCpC.add("PSEQ"); - featuresCpC.add(cWord + "-PSEQW"); - } - - if (flags.useTypeySequences) { - featuresCpC.add(c.get(ShapeAnnotation.class) + "-TPS2"); - } - - } else if (flags.useExternal) { - - if( ((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || - flags.useShapeStrings) - && flags.useTypeSeqs && (flags.useTypeSeqs2 || flags.useTypeSeqs3)) { -// String pShape = p.get(ShapeAnnotation.class); -// String cShape = c.get(ShapeAnnotation.class); - if (flags.useTypeSeqs3) { - featuresCpC.add(pShape + '-' + cShape + '-' + n.get(ShapeAnnotation.class) + "-PCNSHAPES"); - } - if (flags.useTypeSeqs2) { - featuresCpC.add(pShape + '-' + cShape + "-TYPES"); - } - } - - if (flags.useTypeySequences) { - featuresCpC.add(n.get(ShapeAnnotation.class) + "-TNS1"); - featuresCpC.add(p.get(ShapeAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-TPS"); - } - } - - return featuresCpC; - } - - protected Collection featuresCp2C(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - - String cWord = c.getString(TextAnnotation.class); - Collection featuresCp2C = new ArrayList(); - - if (flags.useMoreAbbr) { - featuresCp2C.add(p2.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-P2ABBRANS"); - } - - if (flags.useMinimalAbbr) { - featuresCp2C.add(p2.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-P2AP2CABB"); - } - - if (flags.useMinimalAbbr1) { - if (!c.get(AbbrAnnotation.class).equals("XX")) { - featuresCp2C.add(p2.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-P2AP2CABB"); - } - } - - if (flags.useParenMatching) { - if (flags.useReverse) { - if (cWord.equals("(") || cWord.equals("[") || cWord.equals("-LRB-")) { - if ((p2.getString(TextAnnotation.class).equals(")") || p2.getString(TextAnnotation.class).equals("]") || p2.getString(TextAnnotation.class).equals("-RRB-")) && ! (p.getString(TextAnnotation.class).equals(")") || p.getString(TextAnnotation.class).equals("]") || p.getString(TextAnnotation.class).equals("-RRB-"))) { - featuresCp2C.add("PAREN-MATCH"); - } - } - } else { - if (cWord.equals(")") || cWord.equals("]") || cWord.equals("-RRB-")) { - if ((p2.getString(TextAnnotation.class).equals("(") || p2.getString(TextAnnotation.class).equals("[") || p2.getString(TextAnnotation.class).equals("-LRB-")) && ! (p.getString(TextAnnotation.class).equals("(") || p.getString(TextAnnotation.class).equals("[") || p.getString(TextAnnotation.class).equals("-LRB-"))) { - featuresCp2C.add("PAREN-MATCH"); - } - } - } - } - - return featuresCp2C; - } - - protected Collection featuresCp3C(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - CoreLabel p3 = cInfo.get(loc - 3); - - String cWord = c.getString(TextAnnotation.class); - Collection featuresCp3C = new ArrayList(); - - if (flags.useParenMatching) { - if (flags.useReverse) { - if (cWord.equals("(") || cWord.equals("[")) { - if ((flags.maxLeft >= 3) && (p3.getString(TextAnnotation.class).equals(")") || p3.getString(TextAnnotation.class).equals("]")) && !(p2.getString(TextAnnotation.class).equals(")") || p2.getString(TextAnnotation.class).equals("]") || p.getString(TextAnnotation.class).equals(")") || p.getString(TextAnnotation.class).equals("]"))) { - featuresCp3C.add("PAREN-MATCH"); - } - } - } else { - if (cWord.equals(")") || cWord.equals("]")) { - if ((flags.maxLeft >= 3) && (p3.getString(TextAnnotation.class).equals("(") || p3.getString(TextAnnotation.class).equals("[")) && !(p2.getString(TextAnnotation.class).equals("(") || p2.getString(TextAnnotation.class).equals("[") || p.getString(TextAnnotation.class).equals("(") || p.getString(TextAnnotation.class).equals("["))) { - featuresCp3C.add("PAREN-MATCH"); - } - } - } - } - - return featuresCp3C; - } - - protected Collection featuresCp4C(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - CoreLabel p3 = cInfo.get(loc - 3); - CoreLabel p4 = cInfo.get(loc - 4); - - String cWord = c.getString(TextAnnotation.class); - Collection featuresCp4C = new ArrayList(); - - if (flags.useParenMatching) { - if (flags.useReverse) { - if (cWord.equals("(") || cWord.equals("[")) { - if ((flags.maxLeft >= 4) && (p4.getString(TextAnnotation.class).equals(")") || p4.getString(TextAnnotation.class).equals("]")) && !(p3.getString(TextAnnotation.class).equals(")") || p3.getString(TextAnnotation.class).equals("]") || p2.getString(TextAnnotation.class).equals(")") || p2.getString(TextAnnotation.class).equals("]") || p.getString(TextAnnotation.class).equals(")") || p.getString(TextAnnotation.class).equals("]"))) { - featuresCp4C.add("PAREN-MATCH"); - } - } - } else { - if (cWord.equals(")") || cWord.equals("]")) { - if ((flags.maxLeft >= 4) && (p4.getString(TextAnnotation.class).equals("(") || p4.getString(TextAnnotation.class).equals("[")) && !(p3.getString(TextAnnotation.class).equals("(") || p3.getString(TextAnnotation.class).equals("[") || p2.getString(TextAnnotation.class).equals("(") || p2.getString(TextAnnotation.class).equals("[") || p.getString(TextAnnotation.class).equals("(") || p.getString(TextAnnotation.class).equals("["))) { - featuresCp4C.add("PAREN-MATCH"); - } - } - } - } - - return featuresCp4C; - } - - protected Collection featuresCp5C(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - CoreLabel p3 = cInfo.get(loc - 3); - CoreLabel p4 = cInfo.get(loc - 4); - CoreLabel p5 = cInfo.get(loc - 5); - - String cWord = c.getString(TextAnnotation.class); - Collection featuresCp5C = new ArrayList(); - - if (flags.useParenMatching) { - if (flags.useReverse) { - if (cWord.equals("(") || cWord.equals("[")) { - if ((flags.maxLeft >= 5) && (p5.getString(TextAnnotation.class).equals(")") || p5.getString(TextAnnotation.class).equals("]")) && !(p4.getString(TextAnnotation.class).equals(")") || p4.getString(TextAnnotation.class).equals("]") || p3.getString(TextAnnotation.class).equals(")") || p3.getString(TextAnnotation.class).equals("]") || p2.getString(TextAnnotation.class).equals(")") || p2.getString(TextAnnotation.class).equals("]") || p.getString(TextAnnotation.class).equals(")") || p.getString(TextAnnotation.class).equals("]"))) { - featuresCp5C.add("PAREN-MATCH"); - } - } - } else { - if (cWord.equals(")") || cWord.equals("]")) { - if ((flags.maxLeft >= 5) && (p5.getString(TextAnnotation.class).equals("(") || p5.getString(TextAnnotation.class).equals("[")) && !(p4.getString(TextAnnotation.class).equals("(") || p4.getString(TextAnnotation.class).equals("[") || p3.getString(TextAnnotation.class).equals("(") || p3.getString(TextAnnotation.class).equals("[") || p2.getString(TextAnnotation.class).equals("(") || p2.getString(TextAnnotation.class).equals("[") || p.getString(TextAnnotation.class).equals("(") || p.getString(TextAnnotation.class).equals("["))) { - featuresCp5C.add("PAREN-MATCH"); - } - } - } - } - return featuresCp5C; - } - - - protected Collection featuresCpCp2C(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - - Collection featuresCpCp2C = new ArrayList(); - - if (flags.useInternal && flags.useExternal) { - - if (false && flags.useTypeySequences && flags.maxLeft >= 2) { // this feature duplicates -TYPETYPES one below, so don't include it (hurts to duplicate)!!! - featuresCpCp2C.add(p2.get(ShapeAnnotation.class) + '-' + p.get(ShapeAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-TTPS"); - } - - if (flags.useAbbr) { - featuresCpCp2C.add(p2.get(AbbrAnnotation.class) + '-' + p.get(AbbrAnnotation.class) + '-' + c.get(AbbrAnnotation.class) + "-2PABBRANS"); - } - - if (flags.useChunks) { - featuresCpCp2C.add(p2.get(ChunkAnnotation.class) + '-' + p.get(ChunkAnnotation.class) + '-' + c.get(ChunkAnnotation.class) + "-2PCHUNKS"); - } - - if (flags.useLongSequences) { - featuresCpCp2C.add("PPSEQ"); - } - if (flags.useBoundarySequences && p.getString(TextAnnotation.class).equals(CoNLLDocumentReaderAndWriter.BOUNDARY)) { - featuresCpCp2C.add("BNDRY-SPAN-PPSEQ"); - } - // This more complex consistency checker didn't help! - // if (flags.useBoundarySequences) { - // String pw = p.getString(TextAnnotation.class); - // // try enforce consistency over "and" and "," as well as boundary now - // if (pw.equals(CoNLLDocumentIteratorFactory.BOUNDARY) || - // pw.equalsIgnoreCase("and") || pw.equalsIgnoreCase("or") || - // pw.equals(",")) { - // } - // } - - if (flags.useTaggySequences) { - if (flags.useTags) { - featuresCpCp2C.add(p2.getString(PartOfSpeechAnnotation.class) + '-' + p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-TTS"); - if (flags.useTaggySequencesShapeInteraction) { - featuresCpCp2C.add(p2.getString(PartOfSpeechAnnotation.class) + '-' + p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-TTS-CS"); - } - } - if (flags.useDistSim) { - featuresCpCp2C.add(p2.get(DistSimAnnotation.class) + '-' + p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-DISTSIM_TTS1"); - if (flags.useTaggySequencesShapeInteraction) { - featuresCpCp2C.add(p2.get(DistSimAnnotation.class) + '-' + p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-DISTSIM_TTS1-CS"); - } - } - } - - if (((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || - flags.useShapeStrings) - && flags.useTypeSeqs && flags.useTypeSeqs2 && flags.maxLeft >= 2) { - String cShape = c.get(ShapeAnnotation.class); - String pShape = p.get(ShapeAnnotation.class); - String p2Shape = p2.get(ShapeAnnotation.class); - featuresCpCp2C.add(p2Shape + '-' + pShape + '-' + cShape + "-TYPETYPES"); - } - } else if (flags.useInternal) { - - if (flags.useLongSequences) { - featuresCpCp2C.add("PPSEQ"); - } - } else if (flags.useExternal) { - - if (flags.useLongSequences) { - featuresCpCp2C.add("PPSEQ"); - } - - if (((flags.wordShape > WordShapeClassifier.NOWORDSHAPE) || - flags.useShapeStrings) - && flags.useTypeSeqs && flags.useTypeSeqs2 && flags.maxLeft >= 2) { - String cShape = c.get(ShapeAnnotation.class); - String pShape = p.get(ShapeAnnotation.class); - String p2Shape = p2.get(ShapeAnnotation.class); - featuresCpCp2C.add(p2Shape + '-' + pShape + '-' + cShape + "-TYPETYPES"); - } - } - - return featuresCpCp2C; - } - - - protected Collection featuresCpCp2Cp3C(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - CoreLabel p = cInfo.get(loc - 1); - CoreLabel p2 = cInfo.get(loc - 2); - CoreLabel p3 = cInfo.get(loc - 3); - - Collection featuresCpCp2Cp3C = new ArrayList(); - - if (flags.useTaggySequences) { - if (flags.useTags) { - if (flags.maxLeft >= 3 && !flags.dontExtendTaggy) { - featuresCpCp2Cp3C.add(p3.getString(PartOfSpeechAnnotation.class) + '-' + p2.getString(PartOfSpeechAnnotation.class) + '-' + p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + "-TTTS"); - if (flags.useTaggySequencesShapeInteraction) { - featuresCpCp2Cp3C.add(p3.getString(PartOfSpeechAnnotation.class) + '-' + p2.getString(PartOfSpeechAnnotation.class) + '-' + p.getString(PartOfSpeechAnnotation.class) + '-' + c.getString(PartOfSpeechAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-TTTS-CS"); - } - } - } - if (flags.useDistSim) { - if (flags.maxLeft >= 3 && !flags.dontExtendTaggy) { - featuresCpCp2Cp3C.add(p3.get(DistSimAnnotation.class) + '-' + p2.get(DistSimAnnotation.class) + '-' + p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + "-DISTSIM_TTTS1"); - if (flags.useTaggySequencesShapeInteraction) { - featuresCpCp2Cp3C.add(p3.get(DistSimAnnotation.class) + '-' + p2.get(DistSimAnnotation.class) + '-' + p.get(DistSimAnnotation.class) + '-' + c.get(DistSimAnnotation.class) + '-' + c.get(ShapeAnnotation.class) + "-DISTSIM_TTTS1-CS"); - } - } - } - } - - if (flags.maxLeft >= 3) { - if (flags.useLongSequences) { - featuresCpCp2Cp3C.add("PPPSEQ"); - } - if (flags.useBoundarySequences && p.getString(TextAnnotation.class).equals(CoNLLDocumentReaderAndWriter.BOUNDARY)) { - featuresCpCp2Cp3C.add("BNDRY-SPAN-PPPSEQ"); - } - } - - return featuresCpCp2Cp3C; - } - - protected Collection featuresCpCp2Cp3Cp4C(PaddedList cInfo, int loc) { - Collection featuresCpCp2Cp3Cp4C = new ArrayList(); - - CoreLabel p = cInfo.get(loc - 1); - - if (flags.maxLeft >= 4) { - if (flags.useLongSequences) { - featuresCpCp2Cp3Cp4C.add("PPPPSEQ"); - } - if (flags.useBoundarySequences && p.getString(TextAnnotation.class).equals(CoNLLDocumentReaderAndWriter.BOUNDARY)) { - featuresCpCp2Cp3Cp4C.add("BNDRY-SPAN-PPPPSEQ"); - } - } - - return featuresCpCp2Cp3Cp4C; - } - - - protected Collection featuresCnC(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - - Collection featuresCnC = new ArrayList(); - - if (flags.useNext) { - if (flags.useSequences && flags.useNextSequences) { - featuresCnC.add("NSEQ"); - featuresCnC.add(c.getString(TextAnnotation.class) + "-NSEQW"); - } - } - - return featuresCnC; - } - - - protected Collection featuresCpCnC(PaddedList cInfo, int loc) { - CoreLabel c = cInfo.get(loc); - - Collection featuresCpCnC = new ArrayList(); - - if (flags.useNext && flags.usePrev) { - if (flags.useSequences && flags.usePrevSequences && flags.useNextSequences) { - featuresCpCnC.add("PNSEQ"); - featuresCpCnC.add(c.getString(TextAnnotation.class) + "-PNSEQW"); - } - } - - return featuresCpCnC; - } - - - int reverse(int i) { - return (flags.useReverse ? -1 * i : i); - } - - private Collection occurrencePatterns(PaddedList cInfo, int loc) { - // features on last Cap - String word = cInfo.get(loc).getString(TextAnnotation.class); - String nWord = cInfo.get(loc + reverse(1)).getString(TextAnnotation.class); - CoreLabel p = cInfo.get(loc - reverse(1)); - String pWord = p.getString(TextAnnotation.class); - // System.err.println(word+" "+nWord); - if (!(isNameCase(word) && noUpperCase(nWord) && hasLetter(nWord) && hasLetter(pWord) && p != cInfo.getPad())) { - return Collections.singletonList("NO-OCCURRENCE-PATTERN"); - } - // System.err.println("LOOKING"); - Set l = new HashSet(); - if (cInfo.get(loc - reverse(1)).getString(PartOfSpeechAnnotation.class) != null && isNameCase(pWord) && cInfo.get(loc - reverse(1)).getString(PartOfSpeechAnnotation.class).equals("NNP")) { - for (int jump = 3; jump < 150; jump++) { - if (cInfo.get(loc + reverse(jump)).getString(TextAnnotation.class).equals(word)) { - if (cInfo.get(loc + reverse(jump - 1)).getString(TextAnnotation.class).equals(pWord)) { - l.add("XY-NEXT-OCCURRENCE-XY"); - } else { - l.add("XY-NEXT-OCCURRENCE-Y"); - } - } - } - for (int jump = -3; jump > -150; jump--) { - if (cInfo.get(loc + reverse(jump)).getString(TextAnnotation.class).equals(word)) { - if (cInfo.get(loc + reverse(jump - 1)).getString(TextAnnotation.class).equals(pWord)) { - l.add("XY-PREV-OCCURRENCE-XY"); - } else { - l.add("XY-PREV-OCCURRENCE-Y"); - } - } - } - } else { - for (int jump = 3; jump < 150; jump++) { - if (cInfo.get(loc + reverse(jump)).getString(TextAnnotation.class).equals(word)) { - if (isNameCase(cInfo.get(loc + reverse(jump - 1)).getString(TextAnnotation.class)) && (cInfo.get(loc + reverse(jump - 1))).getString(PartOfSpeechAnnotation.class).equals("NNP")) { - l.add("X-NEXT-OCCURRENCE-YX"); - // System.err.println(cInfo.get(loc+reverse(jump-1)).getString(TextAnnotation.class)); - } else if (isNameCase((cInfo.get(loc + reverse(jump + 1))).getString(TextAnnotation.class)) && (cInfo.get(loc + reverse(jump + 1))).getString(PartOfSpeechAnnotation.class).equals("NNP")) { - // System.err.println(cInfo.get(loc+reverse(jump+1)).getString(TextAnnotation.class)); - l.add("X-NEXT-OCCURRENCE-XY"); - } else { - l.add("X-NEXT-OCCURRENCE-X"); - } - } - } - for (int jump = -3; jump > -150; jump--) { - if (cInfo.get(loc + jump).getString(TextAnnotation.class) != null && cInfo.get(loc + jump).getString(TextAnnotation.class).equals(word)) { - if (isNameCase(cInfo.get(loc + reverse(jump + 1)).getString(TextAnnotation.class)) && (cInfo.get(loc + reverse(jump + 1))).getString(PartOfSpeechAnnotation.class).equals("NNP")) { - l.add("X-PREV-OCCURRENCE-YX"); - // System.err.println(cInfo.get(loc+reverse(jump+1)).getString(TextAnnotation.class)); - } else if (isNameCase(cInfo.get(loc + reverse(jump - 1)).getString(TextAnnotation.class)) && cInfo.get(loc + reverse(jump - 1)).getString(PartOfSpeechAnnotation.class).equals("NNP")) { - l.add("X-PREV-OCCURRENCE-XY"); - // System.err.println(cInfo.get(loc+reverse(jump-1)).getString(TextAnnotation.class)); - } else { - l.add("X-PREV-OCCURRENCE-X"); - } - } - } - } - /* - if (!l.isEmpty()) { - System.err.println(pWord+" "+word+" "+nWord+" "+l); - } - */ - return l; - } - - String intern(String s) { - if (flags.intern) { - return s.intern(); - } else { - return s; - } - } - - public void initGazette() { - try { - // read in gazettes - if (flags.gazettes == null) { flags.gazettes = new ArrayList(); } - List gazettes = flags.gazettes; - for (String gazetteFile : gazettes) { - BufferedReader r = new BufferedReader(new FileReader(gazetteFile)); - readGazette(r); - r.close(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} // end class NERFeatureFactory diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERServer.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERServer.java deleted file mode 100644 index 05e33ab..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/NERServer.java +++ /dev/null @@ -1,335 +0,0 @@ -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.ie.crf.CRFClassifier; -import edu.stanford.nlp.util.StringUtils; -import edu.stanford.nlp.io.EncodingPrintWriter; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.io.OutputStreamWriter; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.UnknownHostException; -import java.util.Properties; - - -/***************************************************************************** - * A named-entity recognizer server for Stanford's NER. - * Runs on a socket and waits for text to annotate and returns the - * annotated text. (Internally, it uses the classifyString() - * method on a classifier, which can be either the default CRFClassifier - * which is serialized inside the jar file from which it is called, or another - * classifier which is passed as an argument to the main method. - * - * @version $Id: NERServer.java 36863 2011-07-12 22:58:19Z horatio $ - * @author - * Bjorn Aldag
- * Copyright © 2000 - 2004 Cycorp, Inc. All rights reserved. - * Permission granted for Stanford to distribute with their NER code - * by Bjorn Aldag - * @author Christopher Manning 2006 (considerably rewritten) - * -*****************************************************************************/ - -public class NERServer { - - //// Variables - - /** - * Debugging toggle. - */ - private boolean DEBUG = false; - - private final String charset; - - /** - * The listener socket of this server. - */ - private final ServerSocket listener; - - /** - * The classifier that does the actual tagging. - */ - private final AbstractSequenceClassifier ner; - - - //// Constructors - - /** - * Creates a new named entity recognizer server on the specified port. - * - * @param port the port this NERServer listens on. - * @param asc The classifier which will do the tagging - * @param charset The character set for encoding Strings over the socket stream, e.g., "utf-8" - * @throws IOException If there is a problem creating a ServerSocket - */ - public NERServer(int port, AbstractSequenceClassifier asc, String charset) throws IOException { - ner = asc; - listener = new ServerSocket(port); - this.charset = charset; - } - - //// Public Methods - - /** - * Runs this named entity recognizer server. - */ - @SuppressWarnings({"InfiniteLoopStatement", "ConstantConditions", "null"}) - public void run() { - Socket client = null; - while (true) { - try { - client = listener.accept(); - if (DEBUG) { - System.err.print("Accepted request from "); - System.err.println(client.getInetAddress().getHostName()); - } - new Session(client); - } catch (Exception e1) { - System.err.println("NERServer: couldn't accept"); - e1.printStackTrace(System.err); - try { - client.close(); - } catch (Exception e2) { - System.err.println("NERServer: couldn't close client"); - e2.printStackTrace(System.err); - } - } - } - } - - - //// Inner Classes - - /** - * A single user session, accepting one request, processing it, and - * sending back the results. - */ - private class Session extends Thread { - - //// Instance Fields - - /** - * The socket to the client. - */ - private final Socket client; - - /** - * The input stream from the client. - */ - private final BufferedReader in; - - /** - * The output stream to the client. - */ - private PrintWriter out; - - - //// Constructors - - private Session(Socket socket) throws IOException { - client = socket; - in = new BufferedReader(new InputStreamReader(client.getInputStream(), charset)); - out = new PrintWriter(new OutputStreamWriter(client.getOutputStream(), charset)); - start(); - } - - - //// Public Methods - - /** - * Runs this session by reading a string, tagging it, and writing - * back the result. The input should be a single line (no embedded - * newlines), which represents a whole sentence or document. - */ - @Override - public void run() { - if (DEBUG) {System.err.println("Created new session");} - String input = null; - try { - // TODO: why not allow for multiple lines of input? - input = in.readLine(); - if (DEBUG) { - EncodingPrintWriter.err.println("Receiving: \"" + input + '\"', charset); - } - } catch (IOException e) { - System.err.println("NERServer:Session: couldn't read input"); - e.printStackTrace(System.err); - } catch (NullPointerException npe) { - System.err.println("NERServer:Session: connection closed by peer"); - npe.printStackTrace(System.err); - } - try { - if (! (input == null)) { - String output = - ner.classifyToString(input, ner.flags.outputFormat, - !"slashTags".equals(ner.flags.outputFormat)); - - if (DEBUG) { - EncodingPrintWriter.err.println("Sending: \"" + output + '\"', charset); - } - out.print(output); - out.flush(); - } - } catch (RuntimeException e) { - // ah well, guess they won't be hearing back from us after all - } - close(); - } - - /** - * Terminates this session gracefully. - */ - private void close() { - try { - in.close(); - out.close(); - client.close(); - } catch (Exception e) { - System.err.println("NERServer:Session: can't close session"); - e.printStackTrace(System.err); - } - } - - } // end class Session - - /** This example sends material to the NER server one line at a time. - * Each line should be at least a whole sentence, or can be a whole - * document. - */ - public static class NERClient { - - private NERClient() {} - - public static void communicateWithNERServer(String host, int port, - String charset) - throws IOException - { - System.out.println("Input some text and press RETURN to NER tag it, " + - " or just RETURN to finish."); - - BufferedReader stdIn = - new BufferedReader(new InputStreamReader(System.in, charset)); - communicateWithNERServer(host, port, charset, stdIn, null, true); - stdIn.close(); - } - - public static void communicateWithNERServer(String host, int port, - String charset, - BufferedReader input, - BufferedWriter output, - boolean closeOnBlank) - throws IOException - { - if (host == null) { - host = "localhost"; - } - - for (String userInput; (userInput = input.readLine()) != null; ) { - if (userInput.matches("\\n?")) { - if (closeOnBlank) { - break; - } else { - continue; - } - } - try { - // TODO: why not keep the same socket for multiple lines? - Socket socket = new Socket(host, port); - PrintWriter out = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), charset), true); - BufferedReader in = new BufferedReader(new InputStreamReader( - socket.getInputStream(), charset)); - // send material to NER to socket - out.println(userInput); - // Print the results of NER - String result; - while ((result = in.readLine()) != null) { - if (output == null) { - EncodingPrintWriter.out.println(result, charset); - } else { - output.write(result); - output.newLine(); - } - } - in.close(); - socket.close(); - } catch (UnknownHostException e) { - System.err.print("Cannot find host: "); - System.err.println(host); - return; - } catch (IOException e) { - System.err.print("I/O error in the connection to: "); - System.err.println(host); - return; - } - } - } - } // end static class NERClient - - - private static final String USAGE = "Usage: NERServer [-loadClassifier file|-loadJarClassifier resource|-client] -port portNumber"; - - /** - * Starts this server on the specified port. The classifier used can be - * either a default one stored in the jar file from which this code is - * invoked or you can specify it as a filename or as another classifier - * resource name, which must correspond to the name of a resource in the - * /classifiers/ directory of the jar file. - *

- * Usage: java edu.stanford.nlp.ie.NERServer [-loadClassifier file|-loadJarClassifier resource|-client] -port portNumber - * - * @param args Command-line arguments (described above) - * @throws Exception If file or Java class problems with serialized classifier - */ - @SuppressWarnings({"StringEqualsEmptyString"}) - public static void main (String[] args) throws Exception { - Properties props = StringUtils.argsToProperties(args); - String loadFile = props.getProperty("loadClassifier"); - String loadJarFile = props.getProperty("loadJarClassifier"); - String client = props.getProperty("client"); - String portStr = props.getProperty("port"); - props.remove("port"); // so later code doesn't complain - if (portStr == null || portStr.equals("")) { - System.err.println(USAGE); - return; - } - String charset = "utf-8"; - String encoding = props.getProperty("encoding"); - if (encoding != null && ! "".equals(encoding)) { - charset = encoding; - } - int port; - try { - port = Integer.parseInt(portStr); - } catch (NumberFormatException e) { - System.err.println("Non-numerical port"); - System.err.println(USAGE); - return; - } - // default output format for if no output format is specified - if (props.getProperty("outputFormat") == null) { - props.setProperty("outputFormat", "slashTags"); - } - - if (client != null && ! client.equals("")) { - // run a test client for illustration/testing - String host = props.getProperty("host"); - NERClient.communicateWithNERServer(host, port, charset); - } else { - AbstractSequenceClassifier asc; - if (loadFile != null && ! loadFile.equals("")) { - asc = CRFClassifier.getClassifier(loadFile, props); - } else if (loadJarFile != null && ! loadJarFile.equals("")) { - asc = CRFClassifier.getJarClassifier(loadJarFile, props); - } else { - asc = CRFClassifier.getDefaultClassifier(props); - } - - new NERServer(port, asc, charset).run(); - } - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/SeminarsPrior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/SeminarsPrior.java deleted file mode 100644 index 4969d6e..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/SeminarsPrior.java +++ /dev/null @@ -1,186 +0,0 @@ -package edu.stanford.nlp.ie; - -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.StringUtils; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.ie.pascal.AcronymModel; -import edu.stanford.nlp.ling.CoreAnnotations; - - -import java.util.*; - -/** - * @author Jenny Finkel - */ - -public class SeminarsPrior extends EntityCachingAbstractSequencePrior { - - //double penalty = 4.0; - double penalty = 2.3; - //double penalty1 = 3.0; - //double penalty2 = 4.0; - - public SeminarsPrior(String backgroundSymbol, Index classIndex, List doc) { - super(backgroundSymbol, classIndex, doc); - init(doc); - } - - private void init(List doc) { - - interned = new String[doc.size()]; - int i = 0; - for (IN wi : doc) { - interned[i++] = wi.get(CoreAnnotations.TextAnnotation.class).toLowerCase().intern(); - } - - } - - private String[] interned; - - public double scoreOf(int[] sequence) { - - Set speakers = new HashSet(); - Set locations = new HashSet(); - Set stimes = new HashSet(); - Set etimes = new HashSet(); - - List speakersL = new ArrayList(); - List locationsL = new ArrayList(); - List stimesL = new ArrayList(); - List etimesL = new ArrayList(); - - double p = 0.0; - for (int i = 0; i < entities.length; i++) { - Entity entity = entities[i]; - if ((i == 0 || entities[i-1] != entity) && entity != null) { - - String type = classIndex.get(entity.type); - String phrase = StringUtils.join(entity.words, " ").toLowerCase(); - if (type.equalsIgnoreCase("SPEAKER")) { - speakers.add(phrase); - speakersL.add(entity); - } else if (type.equalsIgnoreCase("LOCATION")) { - locations.add(phrase); - locationsL.add(entity); - } else if (type.equals("STIME")) { - stimes.add(phrase); - stimesL.add(entity); - } else if (type.equals("ETIME")) { - etimes.add(phrase); - etimesL.add(entity); - } else { - System.err.println("unknown entity type: "+type); - System.exit(0); - } - } - } - - for (Entity stimeE : stimesL) { - if (stimes.size() == 1) { break; } - String stime = StringUtils.join(stimeE.words, " "); - String time = ""; - for (char c : stime.toCharArray()) { - if (c >= '0' && c <= '9') { - time += c; - } - } - if (time.length() == 1 || time.length() == 2) { time = time+"00"; } - boolean match = false; - for (String stime1 : stimes) { - String time1 = ""; - for (char c : stime1.toCharArray()) { - if (c >= '0' && c <= '9') { - time1 += c; - } - } - if (time1.length() == 1 || time1.length() == 2) { time1 = time1+"00"; } - if (!time.equals(time1)) { - p -= stimeE.words.size() * penalty; - //System.err.println(time+" ("+s+") "+time1+" ("+s1+") "+stimes); - } - } - } - - - for (Entity etimeE : etimesL) { - if (etimes.size() == 1) { break; } - String etime = StringUtils.join(etimeE.words, " "); - String time = ""; - for (char c : etime.toCharArray()) { - if (c >= '0' && c <= '9') { - time += c; - } - } - if (time.length() == 1 || time.length() == 2) { time = time+"00"; } - boolean match = false; - for (String etime1 : etimes) { - String time1 = ""; - for (char c : etime1.toCharArray()) { - if (c >= '0' && c <= '9') { - time1 += c; - } - } - if (time1.length() == 1 || time1.length() == 2) { time1 = time1+"00"; } - if (!time.equals(time1)) { - p -= etimeE.words.size() * penalty; - //System.err.println(time+" ("+s+") "+time1+" ("+s1+") "+etimes); - } - } - } - -// for (Entity locationE : locationsL) { -// String location = StringUtils.join(locationE.words, " "); -// for (String location1 : locations) { -// String s1 = location; -// String s2 = location1; -// if (s2.length() > s1.length()) { -// String tmp = s2; -// s2 = s1; -// s1 = tmp; -// } -// Pair pair = new Pair(s1, s2); -// Boolean b = aliasLocCache.get(pair); -// if (b == null) { -// double d = acronymModel.HearstSimilarity(s1, s2); -// b = (d >= 0.7); -// aliasLocCache.put(pair, b); -// } -// if (!b) { -// p -= locationE.words.size() * penalty; -// } -// } -// } - - int speakerIndex = classIndex.indexOf("SPEAKER"); - - for (Entity speakerE : speakersL) { - //String lastName = speakerE.words.get(speakerE.words.size()-1); - String lastName = interned[speakerE.startPosition+speakerE.words.size()-1]; - - for (int i = 0; i < interned.length; i++) { - String w = interned[i]; - if (w == lastName) { - if (sequence[i] != speakerIndex) { - p -= penalty; - } - } - } - } - - return p; - } - - private static Map, Boolean> aliasLocCache = new HashMap, Boolean>(); - - private static AcronymModel acronymModel; - - static { - try { - acronymModel = new AcronymModel(); - } catch (Exception e) { - throw new RuntimeException(e.getMessage()); - } - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/UniformPrior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/UniformPrior.java deleted file mode 100644 index 5ae102b..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/UniformPrior.java +++ /dev/null @@ -1,71 +0,0 @@ -package edu.stanford.nlp.ie; - -import java.util.List; - -import edu.stanford.nlp.sequences.SequenceListener; -import edu.stanford.nlp.sequences.SequenceModel; -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.Index; - -/** - * Uniform prior to be used for generic Gibbs inference in the ie.crf.CRFClassifier - * @author Mihai - * - */ -public class UniformPrior implements SequenceModel, SequenceListener { - - protected int[] sequence; - protected int backgroundSymbol; - protected int numClasses; - protected int[] possibleValues; - protected Index classIndex; - protected List doc; - - public UniformPrior(String backgroundSymbol, Index classIndex, List doc) { - this.classIndex = classIndex; - this.backgroundSymbol = classIndex.indexOf(backgroundSymbol); - this.numClasses = classIndex.size(); - this.possibleValues = new int[numClasses]; - for (int i=0; iColumnDocumentReaderAndWriter training data is 3 column input, - * with the columns containing a word, its POS, and its gold class, but - * this can be specified via the map property. - *

- * When run on a file with -textFile, - * the file is assumed to be plain English text (or perhaps simple HTML/XML), - * and a reasonable attempt is made at English tokenization by - * {@link PlainTextDocumentReaderAndWriter}. - *

- * Typical command-line usage - *

For running a trained model with a provided serialized classifier on a - * text file:

- * - * java -mx500m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier - * conll.ner.gz -textFile samplesentences.txt - *

- * When specifying all parameters in a properties file (train, test, or - * runtime):

- * - * java -mx1g edu.stanford.nlp.ie.crf.CRFClassifier -prop propFile - *

- * To train and test a simple NER model from the command line:
- * java -mx1000m edu.stanford.nlp.ie.crf.CRFClassifier - * -trainFile trainFile -testFile testFile -macro > output - *

- *

- * To train with multiple files: - *
- * java -mx1000m edu.stanford.nlp.ie.crf.CRFClassifier - * -trainFileList file1,file2,... -testFile testFile -macro > output - *

- * Features are defined by a {@link edu.stanford.nlp.sequences.FeatureFactory}. - * {@link NERFeatureFactory} is used by default, and - * you should look there for feature templates and properties or flags that - * will cause certain features to be used when training an NER classifier. - * There is also - * a {@link edu.stanford.nlp.wordseg.SighanFeatureFactory}, and various - * successors such as - * {@link edu.stanford.nlp.wordseg.ChineseSegmenterFeatureFactory}, - * which are used for Chinese word segmentation. - * Features are specified either by a Properties file (which is the - * recommended method) or by flags on the command line. The flags are read - * into a {@link SeqClassifierFlags} object, which the - * user need not be concerned with, unless wishing to add new features. - *

- * CRFClassifier may also be used programmatically. When creating a new - * instance, you must specify a Properties object. You may then - * call train methods to train a classifier, or load a classifier. - * The other way to get a CRFClassifier is to deserialize one via - * the static {@link CRFClassifier#getClassifier(String)} methods, which - * return a deserialized - * classifier. You may then tag (classify the items of) documents - * using either the assorted - * classify() or the assorted classify methods in - * {@link AbstractSequenceClassifier}. - * Probabilities assigned by the CRF can be interrogated using either the - * printProbsDocument() or - * getCliqueTrees() methods. - * - * @author Jenny Finkel - * @author Sonal Gupta (made the class generic) - */ -public class CRFClassifier extends AbstractSequenceClassifier { - - Index[] labelIndices; - /** Parameter weights of the classifier. */ - double[][] weights; - Index featureIndex; - int[] map; // caches the featureIndex - - /** Name of default serialized classifier resource to look for in a jar file. - */ - public static final String DEFAULT_CLASSIFIER = "/classifiers/ner-eng-ie.crf-3-all2008.ser.gz"; - private static final boolean VERBOSE = false; - - // List selftraindatums = new ArrayList(); - - - protected CRFClassifier() { - super(new SeqClassifierFlags()); - } - - public CRFClassifier(Properties props) { - super(props); - } - - public CRFClassifier(SeqClassifierFlags flags) { - super(flags); - } - - /** - * Makes a copy of the crf classifier - */ - public CRFClassifier(CRFClassifier crf) - { - super(crf.flags); - this.windowSize = crf.windowSize; - this.featureFactory = crf.featureFactory; - this.pad = crf.pad; - this.knownLCWords = (crf.knownLCWords != null)? new HashSet(crf.knownLCWords): null; - this.featureIndex = (crf.featureIndex != null)? - new HashIndex(crf.featureIndex.objectsList()):null; - this.classIndex = (crf.classIndex != null)? - new HashIndex(crf.classIndex.objectsList()):null; - if (crf.labelIndices != null) { - this.labelIndices = new HashIndex[crf.labelIndices.length]; - for (int i = 0; i < crf.labelIndices.length; i++) { - this.labelIndices[i] = (crf.labelIndices[i] != null)? - new HashIndex(crf.labelIndices[i].objectsList()): null; - } - } else { - this.labelIndices = null; - } - int numFeatures = featureIndex != null ? featureIndex.size() : 0; - weights = new double[numFeatures][]; - for (int i = 0; i < numFeatures; i++) { - String feature = featureIndex.get(i); - int index = crf.featureIndex.indexOf(feature); - weights[i] = new double[crf.weights[index].length]; - System.arraycopy(crf.weights[index], 0, weights[i], 0, weights[i].length); - } - } - - /** - * Returns the total number of weights associated with this classifier. - * @return number of weights - */ - public int getNumWeights() - { - if (weights == null) return 0; - int numWeights = 0; - for (double[] wts : weights) { - numWeights += wts.length; - } - return numWeights; - } - - /** - * Get index of featureType for feature indexed by i. - * (featureType index is used to index labelIndices to get labels.) - * - * @param i feature index - * @return index of featureType - */ - private int getFeatureTypeIndex(int i) - { - return getFeatureTypeIndex(featureIndex.get(i)); - } - - /** - * Get index of featureType for feature based on the feature string - * (featureType index used to index labelIndices to get labels) - * @param feature feature string - * @return index of featureType - */ - private static int getFeatureTypeIndex(String feature) - { - if (feature.endsWith("|C")) { - return 0; - } else if (feature.endsWith("|CpC")) { - return 1; - } else if (feature.endsWith("|Cp2C")) { - return 2; - } else if (feature.endsWith("|Cp3C")) { - return 3; - } else if (feature.endsWith("|Cp4C")) { - return 4; - } else if (feature.endsWith("|Cp5C")) { - return 5; - } else { - throw new RuntimeException("Unknown feature type " + feature); - } - } - - /** - * Scales the weights of this crfclassifier by the specified weight - * @param scale - */ - public void scaleWeights(double scale) - { - for (int i = 0; i < weights.length; i++) { - for (int j = 0; j < weights[i].length; j++) { - weights[i][j] *= scale; - } - } - } - - /** - * Combines weights from another crf (scaled by weight) into this crf's weights - * (assumes that this crf's indices have already been updated to include - * features/labels from the other crf) - * @param crf Other crf whose weights to combine into this crf - * @param weight amount to scale the other crf's weights by - */ - private void combineWeights(CRFClassifier crf, double weight) - { - int numFeatures = featureIndex.size(); - int oldNumFeatures = weights.length; - - // Create a map of other crf labels to this crf labels - Map crfLabelMap = new HashMap(); - for (int i = 0; i < crf.labelIndices.length; i++) { - for (int j=0; j < crf.labelIndices[i].size(); j++) { - CRFLabel labels = crf.labelIndices[i].get(j); - int[] newLabelIndices = new int[i+1]; - for (int ci=0; ci<=i; ci++) { - String classLabel = crf.classIndex.get(labels.getLabel()[ci]); - newLabelIndices[ci] = this.classIndex.indexOf(classLabel); - } - CRFLabel newLabels = new CRFLabel(newLabelIndices); - crfLabelMap.put(labels, newLabels); - int k = this.labelIndices[i].indexOf(newLabels); // the indexing is needed, even when not printed out! - //System.err.println("LabelIndices " + i + " " + labels + ": " + j + " mapped to " + k); - } - } - - // Create map of featureIndex to featureTypeIndex - map = new int[numFeatures]; - for (int i=0; i < numFeatures; i++) { - map[i] = getFeatureTypeIndex(i); - } - - // Create new weights - double[][] newWeights = new double[numFeatures][]; - for (int i = 0; i < numFeatures; i++) { - int length = labelIndices[map[i]].size(); - newWeights[i] = new double[length]; - if (i < oldNumFeatures) { - assert(length >= weights[i].length); - System.arraycopy(weights[i], 0, newWeights[i], 0, weights[i].length); - } - } - weights = newWeights; - - // Get original weight indices from other crf and weight them in - // depending on the type of the feature, different number of weights is associated with it - for (int i = 0; i < crf.weights.length; i++) { - String feature = crf.featureIndex.get(i); - int newIndex = featureIndex.indexOf(feature); - // Check weights are okay dimension - if (weights[newIndex].length < crf.weights[i].length) { - throw new RuntimeException("Incompatible CRFClassifier: weight length mismatch for feature " - + newIndex + ": " + featureIndex.get(newIndex) - + " (also feature " + i + ": " + crf.featureIndex.get(i) + ") " - + ", len1=" + weights[newIndex].length + ", len2=" + crf.weights[i].length); - } - int featureTypeIndex = map[newIndex]; - for (int j=0; j < crf.weights[i].length; j++) { - CRFLabel labels = crf.labelIndices[featureTypeIndex].get(j); - CRFLabel newLabels = crfLabelMap.get(labels); - int k = this.labelIndices[featureTypeIndex].indexOf(newLabels); - weights[newIndex][k] += crf.weights[i][j]*weight; - } - } - } - - /** - * Combines weighted crf with this crf - * @param crf - * @param weight - */ - public void combine(CRFClassifier crf, double weight) - { - Timing timer = new Timing(); - - // Check the CRFClassifiers are compatible - if (!this.pad.equals(crf.pad)) { - throw new RuntimeException("Incompatible CRFClassifier: pad does not match"); - } - if (this.windowSize != crf.windowSize) { - throw new RuntimeException("Incompatible CRFClassifier: windowSize does not match"); - } - if (this.labelIndices.length != crf.labelIndices.length) { - // Should match since this should be same as the windowSize - throw new RuntimeException("Incompatible CRFClassifier: labelIndices length does not match"); - } - this.classIndex.addAll(crf.classIndex.objectsList()); - - // Combine weights of the other classifier with this classifier, - // weighing the other classifier's weights by weight - // First merge the feature indicies - int oldNumFeatures1 = this.featureIndex.size(); - int oldNumFeatures2 = crf.featureIndex.size(); - int oldNumWeights1 = this.getNumWeights(); - int oldNumWeights2 = crf.getNumWeights(); - this.featureIndex.addAll(crf.featureIndex.objectsList()); - this.knownLCWords.addAll(crf.knownLCWords); - assert(weights.length == oldNumFeatures1); - - // Combine weights of this classifier with other classifier - for (int i = 0; i < labelIndices.length; i++) { - this.labelIndices[i].addAll(crf.labelIndices[i].objectsList()); - } - System.err.println("Combining weights: will automatically match labelIndices"); - combineWeights(crf, weight); - - int numFeatures = featureIndex.size(); - int numWeights = getNumWeights(); - long elapsedMs = timer.stop(); - System.err.println("numFeatures: orig1=" + oldNumFeatures1 + ", orig2=" + oldNumFeatures2 - + ", combined=" + numFeatures); - System.err.println("numWeights: orig1=" + oldNumWeights1 + ", orig2=" + oldNumWeights2 - + ", combined=" + numWeights); - System.err.println("Time to combine CRFClassifier: " + Timing.toSecondsString(elapsedMs) + " seconds"); - } - - public void dropFeaturesBelowThreshold(double threshold) { - Index newFeatureIndex = new HashIndex(); - for (int i = 0; i < weights.length; i++) { - double smallest = weights[i][0]; - double biggest = weights[i][0]; - for (int j = 1; j < weights[i].length; j++) { - if (weights[i][j] > biggest) { - biggest = weights[i][j]; - } - if (weights[i][j] < smallest) { - smallest = weights[i][j]; - } - if (biggest - smallest > threshold) { - newFeatureIndex.add(featureIndex.get(i)); - break; - } - } - } - - int[] newMap = new int[newFeatureIndex.size()]; - for (int i = 0; i < newMap.length; i++) { - int index = featureIndex.indexOf(newFeatureIndex.get(i)); - newMap[i] = map[index]; - } - map = newMap; - featureIndex = newFeatureIndex; - } - - /** - * Convert a document List into arrays storing the data features and labels. - * - * @param document Training documents - * @return A Pair, where the first element is an int[][][] representing the data - * and the second element is an int[] representing the labels - */ - public Pair documentToDataAndLabels(List document) { - - int docSize = document.size(); - // first index is position in the document also the index of the clique/factor table - // second index is the number of elements in the clique/window these features are for (starting with last element) - // third index is position of the feature in the array that holds them - // element in data[j][k][m] is the index of the mth feature occurring in position k of the jth clique - int[][][] data = new int[docSize][windowSize][]; - // index is the position in the document - // element in labels[j] is the index of the correct label (if it exists) at position j of document - int[] labels = new int[docSize]; - - if (flags.useReverse) { - Collections.reverse(document); - } - - //System.err.println("docSize:"+docSize); - for (int j = 0; j < docSize; j++) { - CRFDatum,CRFLabel> d = makeDatum(document, j, featureFactory); - - List> features = d.asFeatures(); - for (int k = 0, fSize = features.size(); k < fSize; k++) { - Collection cliqueFeatures = features.get(k); - data[j][k] = new int[cliqueFeatures.size()]; - int m = 0; - for (String feature : cliqueFeatures) { - int index = featureIndex.indexOf(feature); - if (index >= 0) { - data[j][k][m] = index; - m++; - } else { - // this is where we end up when we do feature threshold cutoffs - } - } - // Reduce memory use when some features were cut out by threshold - if (m < data[j][k].length) { - int[] f = new int[m]; - System.arraycopy(data[j][k], 0, f, 0, m); - data[j][k] = f; - } - } - - IN wi = document.get(j); - labels[j] = classIndex.indexOf(wi.get(AnswerAnnotation.class)); - } - - if (flags.useReverse) { - Collections.reverse(document); - } - - // System.err.println("numClasses: "+classIndex.size()+" "+classIndex); - // System.err.println("numDocuments: 1"); - // System.err.println("numDatums: "+data.length); - // System.err.println("numFeatures: "+featureIndex.size()); - - return new Pair(data, labels); - } - - - public void printLabelInformation(String testFile, - DocumentReaderAndWriter readerAndWriter) - throws Exception - { - ObjectBank> documents = - makeObjectBankFromFile(testFile, readerAndWriter); - for (List document : documents) { - printLabelValue(document); - } - } - - - public void printLabelValue(List document) { - - if (flags.useReverse) { - Collections.reverse(document); - } - - NumberFormat nf = new DecimalFormat(); - - List classes = new ArrayList(); - for (int i = 0; i < classIndex.size(); i++) { - classes.add(classIndex.get(i)); - } - String[] columnHeaders = classes.toArray(new String[classes.size()]); - - //System.err.println("docSize:"+docSize); - for (int j = 0; j < document.size(); j++) { - - System.out.println("--== "+document.get(j).get(TextAnnotation.class)+" ==--"); - - List lines = new ArrayList(); - List rowHeaders = new ArrayList(); - List line = new ArrayList(); - - for (int p = 0; p < labelIndices.length; p++) { - if (j+p >= document.size()) { continue; } - CRFDatum, CRFLabel> d = makeDatum(document, j+p, featureFactory); - - List features = d.asFeatures(); - for (int k = p, fSize = features.size(); k < fSize; k++) { - Collection cliqueFeatures = (Collection) features.get(k); - for (String feature : cliqueFeatures) { - int index = featureIndex.indexOf(feature); - if (index >= 0) { -// line.add(feature+"["+(-p)+"]"); - rowHeaders.add(feature+ '[' + (-p) + ']'); - double[] values = new double[labelIndices[0].size()]; - for (CRFLabel label : labelIndices[k]) { - int[] l = label.getLabel(); - double v = weights[index][labelIndices[k].indexOf(label)]; - values[l[l.length-1-p]] += v; - } - for (double value : values) { - line.add(nf.format(value)); - } - lines.add(line.toArray(new String[line.size()])); - line = new ArrayList(); - } - } - } -// lines.add(Collections.emptyList()); - System.out.println(StringUtils.makeAsciiTable(lines.toArray(new String[lines.size()][0]), - rowHeaders.toArray(new String[rowHeaders.size()]), - columnHeaders, - 0, 1, true)); - System.out.println(); - } -// System.err.println(edu.stanford.nlp.util.StringUtils.join(lines,"\n")); - } - - if (flags.useReverse) { - Collections.reverse(document); - } - } - - - /** Convert an ObjectBank to arrays of data features and labels. - * - * @return A Pair, where the first element is an int[][][][] representing the data - * and the second element is an int[][] representing the labels. - */ - public Pair documentsToDataAndLabels(Collection> documents) { - - // first index is the number of the document - // second index is position in the document also the index of the clique/factor table - // third index is the number of elements in the clique/window thase features are for (starting with last element) - // fourth index is position of the feature in the array that holds them - // element in data[i][j][k][m] is the index of the mth feature occurring in position k of the jth clique of the ith document - // int[][][][] data = new int[documentsSize][][][]; - List data = new ArrayList(); - - // first index is the number of the document - // second index is the position in the document - // element in labels[i][j] is the index of the correct label (if it exists) at position j in document i - // int[][] labels = new int[documentsSize][]; - List labels = new ArrayList(); - - int numDatums = 0; - - for (List doc : documents) { - Pair docPair = documentToDataAndLabels(doc); - data.add(docPair.first()); - labels.add(docPair.second()); - numDatums += doc.size(); - } - - System.err.println("numClasses: " + classIndex.size() + ' ' + classIndex); - System.err.println("numDocuments: " + data.size()); - System.err.println("numDatums: " + numDatums); - System.err.println("numFeatures: " + featureIndex.size()); - printFeatures(); - - int[][][][] dataA = new int[0][][][]; - int[][] labelsA = new int[0][]; - - return new Pair(data.toArray(dataA), labels.toArray(labelsA)); - } - - /** Convert an ObjectBank to corresponding collection of data features and labels. - * - * @return A List of pairs, one for each document, - * where the first element is an int[][][] representing the data - * and the second element is an int[] representing the labels. - */ - public List> documentsToDataAndLabelsList(Collection> documents) { - int numDatums = 0; - - List> docList = new ArrayList>(); - for (List doc : documents) { - Pair docPair = documentToDataAndLabels(doc); - docList.add(docPair); - numDatums += doc.size(); - } - - System.err.println("numClasses: " + classIndex.size() + ' ' + classIndex); - System.err.println("numDocuments: " + docList.size()); - System.err.println("numDatums: " + numDatums); - System.err.println("numFeatures: " + featureIndex.size()); - return docList; - } - - private void printFeatures() { - if (flags.printFeatures == null) { - return; - } - try { - String enc = flags.inputEncoding; - if (flags.inputEncoding == null) { - System.err.println("flags.inputEncoding doesn't exist, Use UTF-8 as default"); - enc = "UTF-8"; - } - - PrintWriter pw = new PrintWriter(new OutputStreamWriter( - new FileOutputStream("feats-" + flags.printFeatures + ".txt"), enc), true); - for (String feat : featureIndex) { - pw.println(feat); - } - pw.close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - } - - /** This routine builds the labelIndices which give the - * empirically legal label sequences (of length (order) at most - * windowSize) - * and the classIndex, - * which indexes known answer classes. - * - * @param ob The training data: Read from an ObjectBank, each - * item in it is a List. - */ - protected void makeAnswerArraysAndTagIndex(Collection> ob) { - - HashSet[] featureIndices = new HashSet[windowSize]; - for (int i = 0; i < windowSize; i++) { - featureIndices[i] = new HashSet(); - } - - labelIndices = new HashIndex[windowSize]; - for (int i = 0; i < labelIndices.length; i++) { - labelIndices[i] = new HashIndex(); - } - - Index labelIndex = labelIndices[windowSize - 1]; - - classIndex = new HashIndex(); - //classIndex.add("O"); - classIndex.add(flags.backgroundSymbol); - - HashSet[] seenBackgroundFeatures = new HashSet[2]; - seenBackgroundFeatures[0] = new HashSet(); - seenBackgroundFeatures[1] = new HashSet(); - - //int count = 0; - for (List doc : ob) { - //if (count % 100 == 0) { - //System.err.println(count); - //} - //count++; - - if (flags.useReverse) { - Collections.reverse(doc); - } - - int docSize = doc.size(); - //create the full set of labels in classIndex - //note: update to use addAll later - for (int j = 0; j < docSize; j++) { - String ans = doc.get(j).get(AnswerAnnotation.class); - classIndex.add(ans); - } - - for (int j = 0; j < docSize; j++) { - - CRFDatum,CRFLabel> d = makeDatum(doc, j, featureFactory); - labelIndex.add(d.label()); - - List> features = d.asFeatures(); - for (int k = 0, fsize = features.size(); k < fsize; k++) { - Collection cliqueFeatures = features.get(k); - if (k < 2 && flags.removeBackgroundSingletonFeatures) { - String ans = doc.get(j).get(AnswerAnnotation.class); - boolean background = ans.equals(flags.backgroundSymbol); - if (k == 1 && j > 0 && background) { - ans = doc.get(j - 1).get(AnswerAnnotation.class); - background = ans.equals(flags.backgroundSymbol); - } - if (background) { - for (String f : cliqueFeatures) { - if (!featureIndices[k].contains(f)) { - if (seenBackgroundFeatures[k].contains(f)) { - seenBackgroundFeatures[k].remove(f); - featureIndices[k].add(f); - } else { - seenBackgroundFeatures[k].add(f); - } - } - } - } else { - seenBackgroundFeatures[k].removeAll(cliqueFeatures); - featureIndices[k].addAll(cliqueFeatures); - } - } else { - featureIndices[k].addAll(cliqueFeatures); - } - } - } - - if (flags.useReverse) { - Collections.reverse(doc); - } - } - - // String[] fs = new String[featureIndices[0].size()]; - // for (Iterator iter = featureIndices[0].iterator(); iter.hasNext(); ) { - // System.err.println(iter.next()); - // } - - int numFeatures = 0; - for (int i = 0; i < windowSize; i++) { - numFeatures += featureIndices[i].size(); - } - - featureIndex = new HashIndex(); - map = new int[numFeatures]; - for (int i = 0; i < windowSize; i++) { - featureIndex.addAll(featureIndices[i]); - for (String str : featureIndices[i]) { - map[featureIndex.indexOf(str)] = i; - } - } - - if (flags.useObservedSequencesOnly) { - for (int i = 0, liSize = labelIndex.size(); i < liSize; i++) { - CRFLabel label = labelIndex.get(i); - for (int j = windowSize - 2; j >= 0; j--) { - label = label.getOneSmallerLabel(); - labelIndices[j].add(label); - } - } - } else { - for (int i = 0; i < labelIndices.length; i++) { - labelIndices[i] = allLabels(i + 1, classIndex); - } - } - - if (VERBOSE) { - for (int i = 0, fiSize = featureIndex.size(); i < fiSize; i++) { - System.out.println(i + ": " + featureIndex.get(i)); - } - } - } - - protected static Index allLabels(int window, Index classIndex) { - int[] label = new int[window]; - // cdm july 2005: below array initialization isn't necessary: JLS (3rd ed.) 4.12.5 - // Arrays.fill(label, 0); - int numClasses = classIndex.size(); - Index labelIndex = new HashIndex(); - OUTER: while (true) { - CRFLabel l = new CRFLabel(label); - labelIndex.add(l); - int[] label1 = new int[window]; - System.arraycopy(label, 0, label1, 0, label.length); - label = label1; - for (int j = 0; j < label.length; j++) { - label[j]++; - if (label[j] >= numClasses) { - label[j] = 0; - if (j == label.length - 1) { - break OUTER; - } - } else { - break; - } - } - } - return labelIndex; - } - - - /** Makes a CRFDatum by producing features and a label from input data - * at a specific position, using the provided factory. - * @param info The input data - * @param loc The position to build a datum at - * @param featureFactory The FeatureFactory to use to extract features - * @return The constructed CRFDatum - */ - public CRFDatum,CRFLabel> makeDatum(List info, int loc, edu.stanford.nlp.sequences.FeatureFactory featureFactory) { - pad.set(AnswerAnnotation.class, flags.backgroundSymbol); - PaddedList pInfo = new PaddedList(info, pad); - - ArrayList> features = new ArrayList>(); - - // for (int i = 0; i < windowSize; i++) { - // List featuresC = new ArrayList(); - // for (int j = 0; j < FeatureFactory.win[i].length; j++) { - // featuresC.addAll(featureFactory.features(info, loc, FeatureFactory.win[i][j])); - // } - // features.add(featuresC); - // } - - Collection done = new HashSet(); - for (int i = 0; i < windowSize; i++) { - List featuresC = new ArrayList(); - List windowCliques = featureFactory.getCliques(i, 0); - windowCliques.removeAll(done); - done.addAll(windowCliques); - for (Clique c : windowCliques) { - featuresC.addAll(featureFactory.getCliqueFeatures(pInfo, loc, c)); - } - features.add(featuresC); - } - - int[] labels = new int[windowSize]; - - for (int i = 0; i < windowSize; i++) { - String answer = pInfo.get(loc + i - windowSize + 1).get(AnswerAnnotation.class); - - labels[i] = classIndex.indexOf(answer); - } - - printFeatureLists(pInfo.get(loc), features); - - CRFDatum,CRFLabel> d = new CRFDatum,CRFLabel>(features, new CRFLabel(labels)); - //System.err.println(d); - return d; - } - - - public static class TestSequenceModel implements SequenceModel { - - private int window; - private int numClasses; - //private FactorTable[] factorTables; - private CRFCliqueTree cliqueTree; - private int[] tags; - private int[] backgroundTag; - - //public Scorer(FactorTable[] factorTables) { - public TestSequenceModel(CRFCliqueTree cliqueTree) { - //this.factorTables = factorTables; - this.cliqueTree = cliqueTree; - //this.window = factorTables[0].windowSize(); - this.window = cliqueTree.window(); - //this.numClasses = factorTables[0].numClasses(); - this.numClasses = cliqueTree.getNumClasses(); - tags = new int[numClasses]; - for (int i = 0; i < tags.length; i++) { - tags[i] = i; - } - backgroundTag = new int[]{cliqueTree.backgroundIndex()}; - } - - public int length() { - return cliqueTree.length(); - } - - public int leftWindow() { - return window - 1; - } - - public int rightWindow() { - return 0; - } - - public int[] getPossibleValues(int pos) { - if (pos < window - 1) { - return backgroundTag; - } - return tags; - } - - public double scoreOf(int[] tags, int pos) { - int[] previous = new int[window - 1]; - int realPos = pos - window + 1; - for (int i = 0; i < window - 1; i++) { - previous[i] = tags[realPos + i]; - } - return cliqueTree.condLogProbGivenPrevious(realPos, tags[pos], previous); - } - - public double[] scoresOf(int[] tags, int pos) { - int realPos = pos - window + 1; - double[] scores = new double[numClasses]; - int[] previous = new int[window - 1]; - for (int i = 0; i < window - 1; i++) { - previous[i] = tags[realPos + i]; - } - for (int i = 0; i < numClasses; i++) { - scores[i] = cliqueTree.condLogProbGivenPrevious(realPos, i, previous); - } - return scores; - } - - public double scoreOf(int[] sequence) { - throw new UnsupportedOperationException(); - } - - } // end class TestSequenceModel - - - @Override - public List classify(List document) { - if (flags.doGibbs) { - try { - return classifyGibbs(document); - } catch (Exception e) { - System.err.println("Error running testGibbs inference!"); - e.printStackTrace(); - return null; - } - } else if (flags.crfType.equalsIgnoreCase("maxent")) { - return classifyMaxEnt(document); - } else { - throw new RuntimeException("Unsupported inference type: " + flags.crfType); - } - } - - private List classify(List document, Pair documentDataAndLabels) { - if (flags.doGibbs) { - try { - return classifyGibbs(document, documentDataAndLabels); - } catch (Exception e) { - System.err.println("Error running testGibbs inference!"); - e.printStackTrace(); - return null; - } - } else if (flags.crfType.equalsIgnoreCase("maxent")) { - return classifyMaxEnt(document, documentDataAndLabels); - } else { - throw new RuntimeException("Unsupported inference type: " + flags.crfType); - } - } - - public void classifyAndWriteAnswers(Collection> documents, - List> documentDataAndLabels, - OutputStream outStream, - DocumentReaderAndWriter readerAndWriter) - throws IOException - { - Timing timer = new Timing(); - - Counter entityTP = new ClassicCounter(); - Counter entityFP = new ClassicCounter(); - Counter entityFN = new ClassicCounter(); - boolean resultsCounted = true; - - int numWords = 0; - int numDocs = 0; - for (List doc : documents) { - classify(doc, documentDataAndLabels.get(numDocs)); - numWords += doc.size(); - writeAnswers(doc, outStream, readerAndWriter); - resultsCounted = (resultsCounted && - countResults(doc, entityTP, entityFP, entityFN)); - numDocs++; - } - long millis = timer.stop(); - double wordspersec = numWords / (((double) millis) / 1000); - NumberFormat nf = new DecimalFormat("0.00"); // easier way! - System.err.println(StringUtils.getShortClassName(this) + - " tagged " + numWords + " words in " + numDocs + - " documents at " + nf.format(wordspersec) + - " words per second."); - if (resultsCounted) { - printResults(entityTP, entityFP, entityFN); - } - } - - @Override - public SequenceModel getSequenceModel(List doc) { - Pair p = documentToDataAndLabels(doc); - return getSequenceModel(doc, p); - } - - public SequenceModel getSequenceModel(List doc, Pair documentDataAndLabels) { - Pair p = documentDataAndLabels; - int[][][] data = p.first(); - - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(), classIndex, flags.backgroundSymbol); - - //Scorer scorer = new Scorer(factorTables); - return new TestSequenceModel(cliqueTree); - } - - /** Do standard sequence inference, using either Viterbi or Beam inference - * depending on the value of flags.inferenceType. - * - * @param document Document to classify. Classification happens in place. - * This document is modified. - * @return The classified document - */ - public List classifyMaxEnt(List document) { - if (document.isEmpty()) { - return document; - } - - SequenceModel model = getSequenceModel(document); - return classifyMaxEnt(document, model); - } - - private List classifyMaxEnt(List document, Pair documentDataAndLabels) { - if (document.isEmpty()) { - return document; - } - SequenceModel model = getSequenceModel(document, documentDataAndLabels); - return classifyMaxEnt(document, model); - } - - private List classifyMaxEnt(List document, SequenceModel model) { - if (document.isEmpty()) { - return document; - } - - if (flags.inferenceType == null) { flags.inferenceType = "Viterbi"; } - - BestSequenceFinder tagInference; - if (flags.inferenceType.equalsIgnoreCase("Viterbi")) { - tagInference = new ExactBestSequenceFinder(); - } else if (flags.inferenceType.equalsIgnoreCase("Beam")) { - tagInference = new BeamBestSequenceFinder(flags.beamSize); - } else { - throw new RuntimeException("Unknown inference type: "+flags.inferenceType+". Your options are Viterbi|Beam."); - } - - int[] bestSequence = tagInference.bestSequence(model); - - if (flags.useReverse) { - Collections.reverse(document); - } - for (int j = 0, docSize = document.size(); j < docSize; j++) { - IN wi = document.get(j); - String guess = classIndex.get(bestSequence[j + windowSize - 1]); - wi.set(AnswerAnnotation.class, guess); - } - if (flags.useReverse) { - Collections.reverse(document); - } - return document; - } - - - public List classifyGibbs(List document) - throws ClassNotFoundException, SecurityException, NoSuchMethodException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException - { - Pair p = documentToDataAndLabels(document); - return classifyGibbs(document, p); - } - - public List classifyGibbs(List document, Pair documentDataAndLabels) - throws ClassNotFoundException, SecurityException, NoSuchMethodException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException - { - //System.err.println("Testing using Gibbs sampling."); - Pair p = documentDataAndLabels; - int[][][] data = p.first(); - List newDocument = document; // reversed if necessary - if (flags.useReverse) { - Collections.reverse(document); - newDocument = new ArrayList(document); - Collections.reverse(document); - } - - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(), classIndex, flags.backgroundSymbol); - - SequenceModel model = cliqueTree; - SequenceListener listener = cliqueTree; - - SequenceModel priorModel = null; - SequenceListener priorListener = null; - - if (flags.useNERPrior) { - EntityCachingAbstractSequencePrior prior = new EmpiricalNERPrior(flags.backgroundSymbol, classIndex, newDocument); - // SamplingNERPrior prior = new SamplingNERPrior(flags.backgroundSymbol, classIndex, newDocument); - priorModel = prior; - priorListener = prior; - } else if (flags.useAcqPrior) { - EntityCachingAbstractSequencePrior prior = new AcquisitionsPrior(flags.backgroundSymbol, classIndex, newDocument); - priorModel = prior; - priorListener = prior; - } else if (flags.useSemPrior) { - EntityCachingAbstractSequencePrior prior = new SeminarsPrior(flags.backgroundSymbol, classIndex, newDocument); - priorModel = prior; - priorListener = prior; - } else if(flags.useUniformPrior) { - //System.err.println("Using uniform prior!"); - UniformPrior uniPrior = new UniformPrior(flags.backgroundSymbol, classIndex, newDocument); - priorModel = uniPrior; - priorListener = uniPrior; - } else { - throw new RuntimeException("no prior specified"); - } - - model = new FactoredSequenceModel(model, priorModel); - listener = new FactoredSequenceListener(listener, priorListener); - - SequenceGibbsSampler sampler = new SequenceGibbsSampler(0, 0, listener); - int[] sequence = new int[cliqueTree.length()]; - - if (flags.initViterbi) { - TestSequenceModel testSequenceModel = new TestSequenceModel(cliqueTree); - ExactBestSequenceFinder tagInference = new ExactBestSequenceFinder(); - int[] bestSequence = tagInference.bestSequence(testSequenceModel); - System.arraycopy(bestSequence, windowSize-1, sequence, 0, sequence.length); - } else { - int[] initialSequence = SequenceGibbsSampler.getRandomSequence(model); - System.arraycopy(initialSequence, 0, sequence, 0, sequence.length); - } - - sampler.verbose = 0; - - if (flags.annealingType.equalsIgnoreCase("linear")) { - sequence = sampler.findBestUsingAnnealing(model, CoolingSchedule.getLinearSchedule(1.0, flags.numSamples), sequence); - } else if (flags.annealingType.equalsIgnoreCase("exp") || flags.annealingType.equalsIgnoreCase("exponential")) { - sequence = sampler.findBestUsingAnnealing(model, CoolingSchedule.getExponentialSchedule(1.0, flags.annealingRate, flags.numSamples), sequence); - } else { - throw new RuntimeException("No annealing type specified"); - } - - //System.err.println(ArrayMath.toString(sequence)); - - if (flags.useReverse) { - Collections.reverse(document); - } - - for (int j = 0, dsize = newDocument.size(); j < dsize; j++) { - IN wi = document.get(j); - if (wi==null) throw new RuntimeException(""); - if (classIndex==null) throw new RuntimeException(""); - wi.set(AnswerAnnotation.class, classIndex.get(sequence[j])); - } - - if (flags.useReverse) { - Collections.reverse(document); - } - - return document; - } - - - /** - * - * @param sentence - * @param priorModels an array of prior models - * @param priorListeners an array of prior listeners - * @param modelWts an array of model weights: IMPORTANT: this includes the weight of CRF clasifier as well at position 0, and therefore is longer than priorListeners/priorModels array by 1. - * @return A list of INs with - * @throws ClassNotFoundException - * @throws SecurityException - * @throws NoSuchMethodException - * @throws IllegalArgumentException - * @throws InstantiationException - * @throws IllegalAccessException - * @throws InvocationTargetException - */ - public List classifyGibbsUsingPrior(List sentence, SequenceModel[] priorModels, SequenceListener[] priorListeners, double[] modelWts) throws ClassNotFoundException, SecurityException, NoSuchMethodException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException { - - if((priorModels.length + 1) != modelWts.length) - throw new RuntimeException("modelWts array should be longer than the priorModels array by 1 unit since it also includes the weight of the CRF model at position 0."); - - //System.err.println("Testing using Gibbs sampling."); - Pair p = documentToDataAndLabels(sentence); - int[][][] data = p.first(); - - List newDocument = sentence; // reversed if necessary - if (flags.useReverse) { - Collections.reverse(sentence); - newDocument = new ArrayList(sentence); - Collections.reverse(sentence); - } - - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(), classIndex, flags.backgroundSymbol); - - SequenceModel model = cliqueTree; - SequenceListener listener = cliqueTree; - - - SequenceModel[] models = new SequenceModel[priorModels.length+1]; - models[0] = model; - for(int i = 1; i < models.length; i++)models[i] = priorModels[i-1]; - model = new FactoredSequenceModel(models,modelWts); - - SequenceListener[] listeners = new SequenceListener[priorListeners.length+1]; - listeners[0] = listener; - for(int i =1; i < listeners.length; i++)listeners[i] = priorListeners[i-1]; - listener = new FactoredSequenceListener(listeners); - - SequenceGibbsSampler sampler = new SequenceGibbsSampler(0, 0, listener); - int[] sequence = new int[cliqueTree.length()]; - - if (flags.initViterbi) { - TestSequenceModel testSequenceModel = new TestSequenceModel(cliqueTree); - ExactBestSequenceFinder tagInference = new ExactBestSequenceFinder(); - int[] bestSequence = tagInference.bestSequence(testSequenceModel); - System.arraycopy(bestSequence, windowSize-1, sequence, 0, sequence.length); - } else { - int[] initialSequence = SequenceGibbsSampler.getRandomSequence(model); - System.arraycopy(initialSequence, 0, sequence, 0, sequence.length); - } - - SequenceGibbsSampler.verbose = 0; - - if (flags.annealingType.equalsIgnoreCase("linear")) { - sequence = sampler.findBestUsingAnnealing(model, CoolingSchedule.getLinearSchedule(1.0, flags.numSamples), sequence); - } else if (flags.annealingType.equalsIgnoreCase("exp") || flags.annealingType.equalsIgnoreCase("exponential")) { - sequence = sampler.findBestUsingAnnealing(model, CoolingSchedule.getExponentialSchedule(1.0, flags.annealingRate, flags.numSamples), sequence); - } else { - throw new RuntimeException("No annealing type specified"); - } - - //System.err.println(ArrayMath.toString(sequence)); - - if (flags.useReverse) { - Collections.reverse(sentence); - } - - for (int j = 0, dsize = newDocument.size(); j < dsize; j++) { - IN wi = sentence.get(j); - if (wi==null) throw new RuntimeException(""); - if (classIndex==null) throw new RuntimeException(""); - wi.set(AnswerAnnotation.class, classIndex.get(sequence[j])); - } - - if (flags.useReverse) { - Collections.reverse(sentence); - } - - return sentence; - } - - - - public List classifyGibbsUsingPrior(List sentence, SequenceModel priorModel, SequenceListener priorListener, double model1Wt, double model2Wt) throws ClassNotFoundException, SecurityException, NoSuchMethodException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException { - //System.err.println("Testing using Gibbs sampling."); - Pair p = documentToDataAndLabels(sentence); - int[][][] data = p.first(); - - List newDocument = sentence; // reversed if necessary - if (flags.useReverse) { - newDocument = new ArrayList(sentence); - Collections.reverse(newDocument); - } - - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(), classIndex, flags.backgroundSymbol); - - SequenceModel model = cliqueTree; - SequenceListener listener = cliqueTree; - - model = new FactoredSequenceModel(model, priorModel,model1Wt,model2Wt); - listener = new FactoredSequenceListener(listener, priorListener); - - SequenceGibbsSampler sampler = new SequenceGibbsSampler(0, 0, listener); - int[] sequence = new int[cliqueTree.length()]; - - if (flags.initViterbi) { - TestSequenceModel testSequenceModel = new TestSequenceModel(cliqueTree); - ExactBestSequenceFinder tagInference = new ExactBestSequenceFinder(); - int[] bestSequence = tagInference.bestSequence(testSequenceModel); - System.arraycopy(bestSequence, windowSize-1, sequence, 0, sequence.length); - } else { - int[] initialSequence = SequenceGibbsSampler.getRandomSequence(model); - System.arraycopy(initialSequence, 0, sequence, 0, sequence.length); - } - - SequenceGibbsSampler.verbose = 0; - - if (flags.annealingType.equalsIgnoreCase("linear")) { - sequence = sampler.findBestUsingAnnealing(model, CoolingSchedule.getLinearSchedule(1.0, flags.numSamples), sequence); - } else if (flags.annealingType.equalsIgnoreCase("exp") || flags.annealingType.equalsIgnoreCase("exponential")) { - sequence = sampler.findBestUsingAnnealing(model, CoolingSchedule.getExponentialSchedule(1.0, flags.annealingRate, flags.numSamples), sequence); - } else { - throw new RuntimeException("No annealing type specified"); - } - - //System.err.println(ArrayMath.toString(sequence)); - - if (flags.useReverse) { - Collections.reverse(sentence); - } - - for (int j = 0, dsize = newDocument.size(); j < dsize; j++) { - IN wi = sentence.get(j); - if (wi==null) throw new RuntimeException(""); - if (classIndex==null) throw new RuntimeException(""); - wi.set(AnswerAnnotation.class, classIndex.get(sequence[j])); - } - - if (flags.useReverse) { - Collections.reverse(sentence); - } - - return sentence; - } - - /** - * Takes a {@link List} of something that extends {@link CoreMap} and prints the likelihood - * of each possible label at each point. - * - * @param document A {@link List} of something that extends CoreMap. - */ - @Override - public void printProbsDocument(List document) { - - Pair p = documentToDataAndLabels(document); - int[][][] data = p.first(); - - //FactorTable[] factorTables = CRFLogConditionalObjectiveFunction.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size()); - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(), classIndex, flags.backgroundSymbol); - - // for (int i = 0; i < factorTables.length; i++) { - for (int i = 0; i < cliqueTree.length(); i++) { - IN wi = document.get(i); - System.out.print(wi.get(CoreAnnotations.TextAnnotation.class) + '\t'); - for (Iterator iter = classIndex.iterator(); iter.hasNext();) { - String label = iter.next(); - int index = classIndex.indexOf(label); - // double prob = Math.pow(Math.E, factorTables[i].logProbEnd(index)); - double prob = cliqueTree.prob(i, index); - System.out.print(label + '=' + prob); - if (iter.hasNext()) { - System.out.print("\t"); - } else { - System.out.print("\n"); - } - } - } - } - - /** - * Takes the file, reads it in, and prints out the likelihood of - * each possible label at each point. This gives a simple way to examine - * the probability distributions of the CRF. See - * getCliqueTrees() for more. - * - * @param filename The path to the specified file - */ - public void printFirstOrderProbs(String filename, - DocumentReaderAndWriter readerAndWriter) { - // only for the OCR data does this matter - flags.ocrTrain = false; - - ObjectBank> docs = - makeObjectBankFromFile(filename, readerAndWriter); - printFirstOrderProbsDocuments(docs); - } - - /** - * Takes a {@link List} of documents and prints the likelihood - * of each possible label at each point. - * - * @param documents A {@link List} of {@link List} of INs. - */ - public void printFirstOrderProbsDocuments(ObjectBank> documents) { - for (List doc : documents) { - printFirstOrderProbsDocument(doc); - System.out.println(); - } - } - - /** - * Want to make arbitrary probability queries? Then this is the method for you. - * Given the filename, it reads it in and breaks it into documents, and then makes - * a CRFCliqueTree for each document. you can then ask the clique tree for marginals - * and conditional probabilities of almost anything you want. - */ - public List getCliqueTrees(String filename, DocumentReaderAndWriter readerAndWriter) { - // only for the OCR data does this matter - flags.ocrTrain = false; - - List cts = new ArrayList(); - ObjectBank> docs = - makeObjectBankFromFile(filename, readerAndWriter); - for (List doc : docs) { - cts.add(getCliqueTree(doc)); - } - - return cts; - } - - - private CRFCliqueTree getCliqueTree(List document) { - - Pair p = documentToDataAndLabels(document); - int[][][] data = p.first(); - - //FactorTable[] factorTables = CRFLogConditionalObjectiveFunction.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size()); - return CRFCliqueTree.getCalibratedCliqueTree(weights, data, labelIndices, classIndex.size(), classIndex, flags.backgroundSymbol); - } - - /** - * Takes a {@link List} of something that extends {@link CoreMap} and prints the likelihood - * of each possible label at each point. - * - * @param document A {@link List} of something that extends {@link CoreMap}. - */ - public void printFirstOrderProbsDocument(List document) { - - CRFCliqueTree cliqueTree = getCliqueTree(document); - - // for (int i = 0; i < factorTables.length; i++) { - for (int i = 0; i < cliqueTree.length(); i++) { - IN wi = document.get(i); - System.out.print(wi.get(CoreAnnotations.TextAnnotation.class) + '\t'); - for (Iterator iter = classIndex.iterator(); iter.hasNext();) { - String label = iter.next(); - int index = classIndex.indexOf(label); - if (i == 0) { - //double prob = Math.pow(Math.E, factorTables[i].logProbEnd(index)); - double prob = cliqueTree.prob(i, index); - System.out.print(label + '=' + prob); - if (iter.hasNext()) { - System.out.print("\t"); - } else { - System.out.print("\n"); - } - } else { - for (Iterator iter1 = classIndex.iterator(); iter1.hasNext();) { - String label1 = iter1.next(); - int index1 = classIndex.indexOf(label1); - //double prob = Math.pow(Math.E, factorTables[i].logProbEnd(new int[]{index1, index})); - double prob = cliqueTree.prob(i, new int[]{index1, index}); - System.out.print(label1 + '_' + label + '=' + prob); - if (iter.hasNext() || iter1.hasNext()) { - System.out.print("\t"); - } else { - System.out.print("\n"); - } - } - } - } - } - } - - /** Train a classifier from documents. - * - * @param docs An objectbank representation of documents. - * Changed this type from ObjectBank to Collection for generality (mihai) - */ - @Override - public void train(Collection> docs, - DocumentReaderAndWriter readerAndWriter) { - Timing timer = new Timing(); - timer.start(); - makeAnswerArraysAndTagIndex(docs); - long elapsedMs = timer.stop(); - System.err.println("Time to convert docs to feature indices: " + Timing.toSecondsString(elapsedMs) + " seconds"); - if (flags.exportFeatures != null) { - timer.start(); - CRFFeatureExporter featureExporter = new CRFFeatureExporter(this); - featureExporter.printFeatures(flags.exportFeatures, docs); - elapsedMs = timer.stop(); - System.err.println("Time to export features: " + Timing.toSecondsString(elapsedMs) + " seconds"); - } - for (int i = 0; i <= flags.numTimesPruneFeatures; i++) { - timer.start(); - Pair dataAndLabels = documentsToDataAndLabels(docs); - elapsedMs = timer.stop(); - System.err.println("Time to convert docs to data/labels: " + Timing.toSecondsString(elapsedMs) + " seconds"); - - Evaluator[] evaluators = null; - if (flags.evaluateIters > 0) { - List evaluatorList = new ArrayList(); - evaluatorList.add(new MemoryEvaluator()); - if (flags.evaluateTrain) { - CRFClassifierEvaluator crfEvaluator = new CRFClassifierEvaluator("Train set", this); - List> trainDataAndLabels = new ArrayList>(); - for (int j = 0; j < dataAndLabels.first().length; j++) { - Pair p = new Pair(dataAndLabels.first()[j], dataAndLabels.second()[j]); - trainDataAndLabels.add(p); - } - crfEvaluator.setTestData(docs, trainDataAndLabels); - crfEvaluator.setEvalCmd(flags.evalCmd); - evaluatorList.add(crfEvaluator); - } - if (flags.testFile != null) { - CRFClassifierEvaluator crfEvaluator = new CRFClassifierEvaluator("Test set (" + flags.testFile + ")", this); - ObjectBank> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter); - List> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); - crfEvaluator.setTestData(testObjBank, testDataAndLabels); - crfEvaluator.setEvalCmd(flags.evalCmd); - evaluatorList.add(crfEvaluator); - } - if (flags.testFiles != null) { - String[] testFiles = flags.testFiles.split(","); - for (String testFile: testFiles) { - CRFClassifierEvaluator crfEvaluator = crfEvaluator = - new CRFClassifierEvaluator("Test set (" + testFile + ")", this); - ObjectBank> testObjBank = - makeObjectBankFromFile(testFile, readerAndWriter); - List> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); - crfEvaluator.setTestData(testObjBank, testDataAndLabels); - crfEvaluator.setEvalCmd(flags.evalCmd); - evaluatorList.add(crfEvaluator); - } - } - evaluators = new Evaluator[evaluatorList.size()]; - evaluatorList.toArray(evaluators); - } - - if (flags.numTimesPruneFeatures == i) { - docs = null; // hopefully saves memory - } - // save feature index to disk and read in later - File featIndexFile = null; - - if (flags.saveFeatureIndexToDisk) { - try { - System.err.println("Writing feature index to temporary file."); - featIndexFile = IOUtils.writeObjectToTempFile(featureIndex, "featIndex" + i+ ".tmp"); - featureIndex = null; - } catch (IOException e) { - throw new RuntimeException("Could not open temporary feature index file for writing."); - } - } - - // first index is the number of the document - // second index is position in the document also the index of the clique/factor table - // third index is the number of elements in the clique/window thase features are for (starting with last element) - // fourth index is position of the feature in the array that holds them - // element in data[i][j][k][m] is the index of the mth feature occurring in position k of the jth clique of the ith document - int[][][][] data = dataAndLabels.first(); - // first index is the number of the document - // second index is the position in the document - // element in labels[i][j] is the index of the correct label (if it exists) at position j in document i - int[][] labels = dataAndLabels.second(); - - - if (flags.loadProcessedData != null) { - List processedData = loadProcessedData(flags.loadProcessedData); - if (processedData != null) { - // enlarge the data and labels array - int[][][][] allData = new int[data.length + processedData.size()][][][]; - int[][] allLabels = new int[labels.length + processedData.size()][]; - System.arraycopy(data, 0, allData, 0, data.length); - System.arraycopy(labels, 0, allLabels, 0, labels.length); - // add to the data and labels array - addProcessedData(processedData, allData, allLabels, data.length); - data = allData; - labels = allLabels; - } - } - - if (flags.useFloat) { - CRFLogConditionalObjectiveFloatFunction func = new CRFLogConditionalObjectiveFloatFunction(data, labels, - featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma); - func.crfType = flags.crfType; - - QNMinimizer minimizer; - if (flags.interimOutputFreq != 0) { - FloatFunction monitor = new ResultStoringFloatMonitor(flags.interimOutputFreq, flags.serializeTo); - minimizer = new QNMinimizer(monitor); - } else { - minimizer = new QNMinimizer(); - } - - if (i == 0) { - minimizer.setM(flags.QNsize); - } else { - minimizer.setM(flags.QNsize2); - } - - float[] initialWeights; - if (flags.initialWeights == null) { - initialWeights = func.initial(); - } else { - try { - System.err.println("Reading initial weights from file " + flags.initialWeights); - DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(flags.initialWeights)))); - initialWeights = Convert.readFloatArr(dis); - } catch (IOException e) { - throw new RuntimeException("Could not read from float initial weight file " + flags.initialWeights); - } - } - System.err.println("numWeights: " + initialWeights.length); - float[] weights = minimizer.minimize(func, (float) flags.tolerance, initialWeights); - this.weights = ArrayMath.floatArrayToDoubleArray(func.to2D(weights)); - - } else { - - /*double[] estimate = null; - - if(flags.estimateInitial){ - int[][][][] approxData = new int[data.length/100][][][]; - int[][] approxLabels = new int[data.length/100][]; - - Random generator = new Random(1); - for(int k=0;k) IOUtils.readObjectFromFile(featIndexFile); - } catch (Exception e) { - throw new RuntimeException("Could not open temporary feature index file for reading."); - } - } - - if (i != flags.numTimesPruneFeatures) { - dropFeaturesBelowThreshold(flags.featureDiffThresh); - System.err.println("Removing features with weight below " + flags.featureDiffThresh + " and retraining..."); - } - - } - } - - protected Minimizer getMinimizer(){ - return getMinimizer(0, null); - } - - protected Minimizer getMinimizer(int featurePruneIteration, Evaluator[] evaluators){ - Minimizer minimizer = null; - if( flags.useQN ){ - - int QNmem; - if (featurePruneIteration == 0) { - QNmem = flags.QNsize; - } else { - QNmem = flags.QNsize2; - } - - if (flags.interimOutputFreq != 0) { - Function monitor = new ResultStoringMonitor(flags.interimOutputFreq, flags.serializeTo); - minimizer = new QNMinimizer(monitor,QNmem,flags.useRobustQN); - } else { - minimizer = new QNMinimizer(QNmem,flags.useRobustQN); - } - } else if( flags.useInPlaceSGD){ - StochasticInPlaceMinimizer sgdMinimizer = new StochasticInPlaceMinimizer(flags.sigma, flags.SGDPasses, flags.tuneSampleSize); - if (flags.useSGDtoQN) { - QNMinimizer qnMinimizer; - int QNmem; - if (featurePruneIteration == 0) { - QNmem = flags.QNsize; - } else { - QNmem = flags.QNsize2; - } - if (flags.interimOutputFreq != 0) { - Function monitor = new ResultStoringMonitor(flags.interimOutputFreq, flags.serializeTo); - qnMinimizer = new QNMinimizer(monitor,QNmem,flags.useRobustQN); - } else { - qnMinimizer = new QNMinimizer(QNmem,flags.useRobustQN); - } - minimizer = new HybridMinimizer(sgdMinimizer, qnMinimizer, flags.SGDPasses); - } else { - minimizer = sgdMinimizer; - } - } else if( flags.useSGDtoQN ) { - minimizer = new SGDToQNMinimizer(flags); - } else if( flags.useSMD){ - minimizer = new SMDMinimizer(flags.initialGain, flags.stochasticBatchSize, flags.stochasticMethod,flags.SGDPasses); - } else if( flags.useSGD){ - minimizer = new SGDMinimizer(flags.initialGain,flags.stochasticBatchSize); - } else if( flags.useScaledSGD){ - minimizer = new ScaledSGDMinimizer(flags.initialGain,flags.stochasticBatchSize,flags.SGDPasses,flags.scaledSGDMethod); - } else if( flags.l1reg > 0.0){ - minimizer = ReflectionLoading.loadByReflection("edu.stanford.nlp.optimization.OWLQNMinimizer", flags.l1reg); - } - - if (minimizer instanceof HasEvaluators) { - ((HasEvaluators) minimizer).setEvaluators(flags.evaluateIters, evaluators); - } - if(minimizer==null){ - throw new RuntimeException("No minimizer assigned!"); - } - - return minimizer; - } - - /** - * Creates a new CRFDatum from the preprocessed allData format, given the document number, - * position number, and a List of Object labels. - * - * @return A new CRFDatum - */ - protected List extractDatumSequence(int[][][] allData, int beginPosition, int endPosition, List labeledWordInfos) { - List result = new ArrayList(); - int beginContext = beginPosition - windowSize + 1; - if (beginContext < 0) { - beginContext = 0; - } - // for the beginning context, add some dummy datums with no features! - // TODO: is there any better way to do this? - for (int position = beginContext; position < beginPosition; position++) { - List cliqueFeatures = new ArrayList(); - for (int i = 0; i < windowSize; i++) { - // create a feature list - cliqueFeatures.add(Collections.emptySet()); - } - CRFDatum datum = new CRFDatum(cliqueFeatures, labeledWordInfos.get(position).get(AnswerAnnotation.class)); - result.add(datum); - } - // now add the real datums - for (int position = beginPosition; position <= endPosition; position++) { - List cliqueFeatures = new ArrayList(); - for (int i = 0; i < windowSize; i++) { - // create a feature list - Collection features = new ArrayList(); - for (int j = 0; j < allData[position][i].length; j++) { - features.add(featureIndex.get(allData[position][i][j])); - } - cliqueFeatures.add(features); - } - CRFDatum datum = new CRFDatum(cliqueFeatures, labeledWordInfos.get(position).get(AnswerAnnotation.class)); - result.add(datum); - } - return result; - } - - /** - * Adds the List of Lists of CRFDatums to the data and labels arrays, treating each datum as if - * it were its own document. - * Adds context labels in addition to the target label for each datum, meaning that for a particular - * document, the number of labels will be windowSize-1 greater than the number of datums. - * - * @param processedData a List of Lists of CRFDatums - */ - protected void addProcessedData(List,String>>> processedData, int[][][][] data, int[][] labels, int offset) { - for (int i = 0, pdSize = processedData.size(); i < pdSize; i++) { - int dataIndex = i + offset; - List,String>> document = processedData.get(i); - int dsize = document.size(); - labels[dataIndex] = new int[dsize]; - data[dataIndex] = new int[dsize][][]; - for (int j = 0; j < dsize; j++) { - CRFDatum,String> crfDatum = document.get(j); - // add label, they are offset by extra context - labels[dataIndex][j] = classIndex.indexOf(crfDatum.label()); - // add features - List> cliques = crfDatum.asFeatures(); - int csize = cliques.size(); - data[dataIndex][j] = new int[csize][]; - for (int k = 0; k < csize; k++) { - Collection features = cliques.get(k); - - // Debug only: Remove - // if (j < windowSize) { - // System.err.println("addProcessedData: Features Size: " + features.size()); - // } - - data[dataIndex][j][k] = new int[features.size()]; - - int m = 0; - try { - for (String feature : features) { - //System.err.println("feature " + feature); - // if (featureIndex.indexOf(feature)) ; - if (featureIndex == null) { - System.out.println("Feature is NULL!"); - } - data[dataIndex][j][k][m] = featureIndex.indexOf(feature); - m++; - } - } catch (Exception e) { - e.printStackTrace(); - System.err.printf("[index=%d, j=%d, k=%d, m=%d]\n", dataIndex, j, k, m); - System.err.println("data.length " + data.length); - System.err.println("data[dataIndex].length " + data[dataIndex].length); - System.err.println("data[dataIndex][j].length " + data[dataIndex][j].length); - System.err.println("data[dataIndex][j][k].length " + data[dataIndex][j].length); - System.err.println("data[dataIndex][j][k][m] " + data[dataIndex][j][k][m]); - return; - } - } - } - } - } - - protected static void saveProcessedData(List datums, String filename) { - System.err.print("Saving processsed data of size " + datums.size() + " to serialized file..."); - ObjectOutputStream oos = null; - try { - oos = new ObjectOutputStream(new FileOutputStream(filename)); - oos.writeObject(datums); - } catch (IOException e) { - // do nothing - } finally { - IOUtils.closeIgnoringExceptions(oos); - } - System.err.println("done."); - } - - protected static List loadProcessedData(String filename) { - System.err.print("Loading processed data from serialized file..."); - ObjectInputStream ois = null; - List result = Collections.emptyList(); - try { - ois = new ObjectInputStream(new FileInputStream(filename)); - result = (List) ois.readObject(); - } catch (Exception e) { - e.printStackTrace(); - } finally { - IOUtils.closeIgnoringExceptions(ois); - } - System.err.println("done. Got " + result.size() + " datums."); - return result; - } - - public void loadTextClassifier(String text, Properties props) throws ClassCastException, IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { - //System.err.println("DEBUG: in loadTextClassifier"); - System.err.println("Loading Text Classifier from "+text); - BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(text)))); - - String line = br.readLine(); - // first line should be this format: - // labelIndices.length=\t%d - String[] toks = line.split("\\t"); - if (!toks[0].equals("labelIndices.length=")) { throw new RuntimeException("format error"); } - int size = Integer.parseInt(toks[1]); - labelIndices = new HashIndex[size]; - for (int labelIndicesIdx = 0; labelIndicesIdx < size; labelIndicesIdx++) { - line = br.readLine(); - // first line should be this format: - // labelIndices.length=\t%d - // labelIndices[0].size()=\t%d - toks = line.split("\\t"); - if (! (toks[0].startsWith("labelIndices[") && toks[0].endsWith("].size()="))) { - throw new RuntimeException("format error"); - } - int labelIndexSize = Integer.parseInt(toks[1]); - labelIndices[labelIndicesIdx] = new HashIndex(); - int count = 0; - while(count list = new ArrayList(); - for(int l : label) { - list.add(l); - } - System.err.printf("DEBUG: %d\t%s\n", j, StringUtils.join(list, " ")); - } - } - /**************************************/ - - - line = br.readLine(); - toks = line.split("\\t"); - if (!toks[0].equals("classIndex.size()=")) { throw new RuntimeException("format error"); } - int classIndexSize = Integer.parseInt(toks[1]); - classIndex = new HashIndex(); - int count = 0; - while(count(); - count = 0; - while(count")) { throw new RuntimeException("format error"); } - Properties p = new Properties(); - line = br.readLine(); - - while(!line.equals("")) { - //System.err.println("DEBUG: flags line: "+line); - String[] keyValue = line.split("="); - //System.err.printf("DEBUG: p.setProperty(%s,%s)\n", keyValue[0], keyValue[1]); - p.setProperty(keyValue[0], keyValue[1]); - line = br.readLine(); - } - - //System.err.println("DEBUG: out from flags"); - flags = new SeqClassifierFlags(p); - System.err.println("DEBUG: "); - System.err.print(flags.toString()); - System.err.println("DEBUG: "); - - // edu.stanford.nlp.wordseg.Gale2007ChineseSegmenterFeatureFactory - line = br.readLine(); - - String[] featureFactoryName = line.split(" "); - if (!featureFactoryName[0].equals("") - || !featureFactoryName[2].equals("")) { - throw new RuntimeException("format error"); - } - featureFactory = (edu.stanford.nlp.sequences.FeatureFactory) Class.forName(featureFactoryName[1]).newInstance(); - featureFactory.init(flags); - - - reinit(); - - // 2 - line = br.readLine(); - - String[] windowSizeName = line.split(" "); - if (!windowSizeName[0].equals("") - || !windowSizeName[2].equals("")) { - throw new RuntimeException("format error"); - } - windowSize = Integer.parseInt(windowSizeName[1]); - - // weights.length= 2655170 - line = br.readLine(); - - toks = line.split("\\t"); - if (!toks[0].equals("weights.length=")) { throw new RuntimeException("format error"); } - int weightsLength = Integer.parseInt(toks[1]); - weights = new double[weightsLength][]; - count = 0; - while(count < weightsLength) { - line = br.readLine(); - - toks = line.split("\\t"); - int weights2Length = Integer.parseInt(toks[0]); - weights[count] = new double[weights2Length]; - String[] weightsValue = toks[1].split(" "); - if (weights2Length != weightsValue.length) - { throw new RuntimeException("weights format error"); } - - for(int i2 = 0; i2 < weights2Length; i2++) { - weights[count][i2] = Double.parseDouble(weightsValue[i2]); - } - count++; - } - System.err.printf("DEBUG: double[%d][] weights loaded\n", weightsLength); - line = br.readLine(); - - if (line != null) - { throw new RuntimeException("weights format error"); } - } - - /** - * Serialize the model to a human readable format. - * It's not yet complete. It should now work for Chinese segmenter though. - * TODO: check things in serializeClassifier and add other necessary serialization back - * - * @param serializePath File to write text format of classifier to. - */ - public void serializeTextClassifier(String serializePath) { - System.err.print("Serializing Text classifier to " + serializePath + "..."); - try { - PrintWriter pw = new PrintWriter(new GZIPOutputStream(new FileOutputStream(serializePath))); - - pw.printf("labelIndices.length=\t%d\n",labelIndices.length); - for(int i = 0; i < labelIndices.length; i++) { - pw.printf("labelIndices[%d].size()=\t%d\n", i, labelIndices[i].size()); - for(int j = 0; j < labelIndices[i].size(); j++) { - int[] label = labelIndices[i].get(j).getLabel(); - List list = new ArrayList(); - for(int l : label) { - list.add(l); - } - pw.printf("%d\t%s\n", j, StringUtils.join(list, " ")); - } - } - - pw.printf("classIndex.size()=\t%d\n", classIndex.size()); - for(int i = 0; i < classIndex.size(); i++) { - pw.printf("%d\t%s\n", i, classIndex.get(i)); - } - //pw.printf("\n"); - - pw.printf("featureIndex.size()=\t%d\n", featureIndex.size()); - for(int i = 0; i < featureIndex.size(); i++) { - pw.printf("%d\t%s\n", i, featureIndex.get(i)); - } - //pw.printf("\n"); - - pw.println(""); - pw.print(flags.toString()); - pw.println(""); - - pw.printf(" %s \n",featureFactory.getClass().getName()); - - pw.printf(" %d \n", windowSize); - - pw.printf("weights.length=\t%d\n", weights.length); - for (double[] ws : weights) { - ArrayList list = new ArrayList(); - for (double w : ws) { - list.add(w); - } - pw.printf("%d\t%s\n", ws.length, StringUtils.join(list, " ")); - } - - pw.close(); - System.err.println("done."); - - } catch (Exception e) { - System.err.println("Failed"); - e.printStackTrace(); - // don't actually exit in case they're testing too - //System.exit(1); - } - } - - - /** {@inheritDoc} - */ - @Override - public void serializeClassifier(String serializePath) { - System.err.print("Serializing classifier to " + serializePath + "..."); - - ObjectOutputStream oos = null; - try { - oos = IOUtils.writeStreamFromString(serializePath); - - oos.writeObject(labelIndices); - oos.writeObject(classIndex); - oos.writeObject(featureIndex); - oos.writeObject(flags); - oos.writeObject(featureFactory); - oos.writeInt(windowSize); - oos.writeObject(weights); - //oos.writeObject(WordShapeClassifier.getKnownLowerCaseWords()); - - oos.writeObject(knownLCWords); - - System.err.println("done."); - - } catch (Exception e) { - System.err.println("Failed"); - e.printStackTrace(); - // don't actually exit in case they're testing too - //System.exit(1); - } finally { - IOUtils.closeIgnoringExceptions(oos); - } - } - - - /** - * Loads a classifier from the specified InputStream. - * This version works quietly (unless VERBOSE is true). - * If props is non-null then any properties it specifies override - * those in the serialized file. However, only some properties are - * sensible to change (you shouldn't change how features are defined). - *

- * Note: This method does not close the ObjectInputStream. (But - * earlier versions of the code used to, so beware....) - */ - @Override - @SuppressWarnings({"unchecked"}) // can't have right types in deserialization - public void loadClassifier(ObjectInputStream ois, Properties props) throws ClassCastException, IOException, ClassNotFoundException { - labelIndices = (Index[]) ois.readObject(); - classIndex = (Index) ois.readObject(); - featureIndex = (Index) ois.readObject(); - flags = (SeqClassifierFlags) ois.readObject(); - featureFactory = (edu.stanford.nlp.sequences.FeatureFactory) ois.readObject(); - - if (props != null) { - flags.setProperties(props, false); - } - reinit(); - - windowSize = ois.readInt(); - weights = (double[][]) ois.readObject(); - - //WordShapeClassifier.setKnownLowerCaseWords((Set) ois.readObject()); - knownLCWords = (Set) ois.readObject(); - - if (VERBOSE) { - System.err.println("windowSize=" + windowSize); - System.err.println("flags=\n" + flags); - } - } - - /** - * This is used to load the default supplied classifier stored within - * the jar file. - * THIS FUNCTION WILL ONLY WORK IF THE CODE WAS LOADED FROM A JAR FILE - * WHICH HAS A SERIALIZED CLASSIFIER STORED INSIDE IT. - */ - public void loadDefaultClassifier() { - loadJarClassifier(DEFAULT_CLASSIFIER, null); - } - - - /** - * This is used to load the default supplied classifier stored within - * the jar file. - * THIS FUNCTION WILL ONLY WORK IF THE CODE WAS LOADED FROM A JAR FILE - * WHICH HAS A SERIALIZED CLASSIFIER STORED INSIDE IT. - */ - public void loadDefaultClassifier(Properties props) { - loadJarClassifier(DEFAULT_CLASSIFIER, props); - } - - - /** - * Used to get the default supplied classifier inside the jar file. - * THIS FUNCTION WILL ONLY WORK IF THE CODE WAS LOADED FROM A JAR FILE - * WHICH HAS A SERIALIZED CLASSIFIER STORED INSIDE IT. - * - * @return The default CRFClassifier in the jar file (if there is one) - */ - public static CRFClassifier getDefaultClassifier() { - CRFClassifier crf = new CRFClassifier(); - crf.loadDefaultClassifier(); - return crf; - } - - /** - * Used to get the default supplied classifier inside the jar file. - * THIS FUNCTION WILL ONLY WORK IF THE CODE WAS LOADED FROM A JAR FILE - * WHICH HAS A SERIALIZED CLASSIFIER STORED INSIDE IT. - * - * @return The default CRFClassifier in the jar file (if there is one) - */ - public static CRFClassifier getDefaultClassifier(Properties props) { - CRFClassifier crf = new CRFClassifier(); - crf.loadDefaultClassifier(props); - return crf; - } - - /** - * Used to load a classifier stored as a resource inside a jar file. - * THIS FUNCTION WILL ONLY WORK IF THE CODE WAS LOADED FROM A JAR FILE - * WHICH HAS A SERIALIZED CLASSIFIER STORED INSIDE IT. - * - * @param resourceName Name of clasifier resource inside the jar file. - * @return A CRFClassifier stored in the jar file - */ - public static CRFClassifier getJarClassifier(String resourceName, Properties props) { - CRFClassifier crf = new CRFClassifier(); - crf.loadJarClassifier(resourceName, props); - return crf; - } - - - /** Loads a CRF classifier from a filepath, and returns it. - * - * @param file File to load classifier from - * @return The CRF classifier - * - * @throws IOException If there are problems accessing the input stream - * @throws ClassCastException If there are problems interpreting the serialized data - * @throws ClassNotFoundException If there are problems interpreting the serialized data - */ - public static CRFClassifier getClassifier(File file) throws IOException, ClassCastException, ClassNotFoundException { - CRFClassifier crf = new CRFClassifier(); - crf.loadClassifier(file); - return crf; - } - - /** Loads a CRF classifier from an InputStream, and returns it. This method - * does not buffer the InputStream, so you should have buffered it before - * calling this method. - * - * @param in InputStream to load classifier from - * @return The CRF classifier - * - * @throws IOException If there are problems accessing the input stream - * @throws ClassCastException If there are problems interpreting the serialized data - * @throws ClassNotFoundException If there are problems interpreting the serialized data - */ - public static CRFClassifier getClassifier(InputStream in) throws IOException, ClassCastException, ClassNotFoundException { - CRFClassifier crf = new CRFClassifier(); - crf.loadClassifier(in); - return crf; - } - - public static CRFClassifier getClassifierNoExceptions(String loadPath) { - CRFClassifier crf = new CRFClassifier(); - crf.loadClassifierNoExceptions(loadPath); - return crf; - } - - public static CRFClassifier getClassifier(String loadPath) throws IOException, ClassCastException, ClassNotFoundException { - CRFClassifier crf = new CRFClassifier(); - crf.loadClassifier(loadPath); - return crf; - } - - public static CRFClassifier getClassifier(String loadPath, Properties props) throws IOException, ClassCastException, ClassNotFoundException { - CRFClassifier crf = new CRFClassifier(); - crf.loadClassifier(loadPath, props); - return crf; - } - - - /** The main method. See the class documentation. */ - public static void main(String[] args) throws Exception { - StringUtils.printErrInvocationString("CRFClassifier", args); - - Properties props = StringUtils.argsToProperties(args); - CRFClassifier crf = new CRFClassifier(props); - String testFile = crf.flags.testFile; - String textFile = crf.flags.textFile; - String loadPath = crf.flags.loadClassifier; - String loadTextPath = crf.flags.loadTextClassifier; - String serializeTo = crf.flags.serializeTo; - String serializeToText = crf.flags.serializeToText; - - if (loadPath != null) { - crf.loadClassifierNoExceptions(loadPath, props); - } else if (loadTextPath != null) { - System.err.println("Warning: this is now only tested for Chinese Segmenter"); - System.err.println("(Sun Dec 23 00:59:39 2007) (pichuan)"); - try { - crf.loadTextClassifier(loadTextPath, props); - //System.err.println("DEBUG: out from crf.loadTextClassifier"); - } catch (Exception e) { - throw new RuntimeException("error loading "+loadTextPath, e); - } - } else if (crf.flags.loadJarClassifier != null) { - crf.loadJarClassifier(crf.flags.loadJarClassifier, props); - } else if (crf.flags.trainFile != null || crf.flags.trainFileList != null) { - crf.train(); - } else { - crf.loadDefaultClassifier(); - } - - // System.err.println("Using " + crf.flags.featureFactory); - // System.err.println("Using " + StringUtils.getShortClassName(crf.readerAndWriter)); - - if (serializeTo != null) { - crf.serializeClassifier(serializeTo); - } - - if (serializeToText != null) { - crf.serializeTextClassifier(serializeToText); - } - - if (testFile != null) { - DocumentReaderAndWriter readerAndWriter = crf.makeReaderAndWriter(); - if (crf.flags.searchGraphPrefix != null) { - crf.classifyAndWriteViterbiSearchGraph(testFile,crf.flags.searchGraphPrefix,crf.makeReaderAndWriter()); - } else if (crf.flags.printFirstOrderProbs) { - crf.printFirstOrderProbs(testFile, readerAndWriter); - } else if (crf.flags.printProbs) { - crf.printProbs(testFile, readerAndWriter); - } else if (crf.flags.useKBest) { - int k = crf.flags.kBest; - crf.classifyAndWriteAnswersKBest(testFile, k, readerAndWriter); - } else if (crf.flags.printLabelValue) { - crf.printLabelInformation(testFile, readerAndWriter); - } else { - crf.classifyAndWriteAnswers(testFile, readerAndWriter); - } - } - - if (textFile != null) { - // todo: This is at present hardwired to PTBTokenizer. - // Generalize for other uses (like Chinese NER) - DocumentReaderAndWriter readerAndWriter = - new PlainTextDocumentReaderAndWriter(); - readerAndWriter.init(crf.flags); - crf.classifyAndWriteAnswers(textFile, readerAndWriter); - } - } // end main - - @Override - public List classifyWithGlobalInformation(List tokenSeq, final CoreMap doc, final CoreMap sent) { - return classify(tokenSeq); - } - -} // end class CRFClassifier diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFClassifierEvaluator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFClassifierEvaluator.java deleted file mode 100644 index d2d5ebf..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFClassifierEvaluator.java +++ /dev/null @@ -1,139 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.optimization.CmdEvaluator; -import edu.stanford.nlp.sequences.DocumentReaderAndWriter; -import edu.stanford.nlp.stats.MultiClassChunkEvalStats; -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.Pair; - -import java.io.*; -import java.util.Collection; -import java.util.List; - -/** - * Evaluates CRFClassifier on a set of data - * - called by QNMinimizer periodically - * - If evalCmd is set, runs command line specified by evalCmd - * otherwise does evaluation internally - * NOTE: when running conlleval with exec on Linux, linux will first - * fork process by duplicating memory of current process. So if - * JVM has lots of memory, it will all be duplicated when - * child process is initially forked. - * @author Angel Chang - */ -public class CRFClassifierEvaluator extends CmdEvaluator { - private CRFClassifier classifier; - private CRFLogConditionalObjectiveFunction func; - // NOTE: Defalt uses -r, specify without -r if IOB - private String cmdStr = "/u/nlp/bin/conlleval -r"; - private String[] cmd; - - // TODO: Use data structure to hold data + features - // Cache already featurized documents - // Original object bank - Collection> data; - // Featurized data - List> featurizedData; - - public CRFClassifierEvaluator(String description, - CRFClassifier classifier, - CRFLogConditionalObjectiveFunction func, - Collection> data, - List> featurizedData) - { - this.description = description; - this.classifier = classifier; - this.func = func; - this.data = data; - this.featurizedData = featurizedData; - cmd = getCmd(cmdStr); - } - - public CRFClassifierEvaluator(String description, - CRFClassifier classifier) - { - this.description = description; - this.classifier = classifier; - } - - /** - * Set helper function - */ - public void setHelperFunction(CRFLogConditionalObjectiveFunction func) - { - this.func = func; - } - - /** - * Set the data to test on - */ - public void setTestData(Collection> data, List> featurizedData) - { - this.data = data; - this.featurizedData = featurizedData; - } - - /** - * Set the evaluation command (set to null to skip evaluation using command line) - * @param evalCmd - */ - public void setEvalCmd(String evalCmd) - { - this.cmdStr = evalCmd; - if (cmdStr != null) { - cmdStr = cmdStr.trim(); - if (cmdStr.length() == 0) { cmdStr = null; } - } - cmd = getCmd(cmdStr); - } - - public void setValues(double[] x) - { - // TODO: Avoid this conversion of weights from 1D to 2D and usage of the - // CRFLogConditionalObjectiveFunction - // (unnecessary and expensive if weights are large vectors - like say 100 million) - classifier.weights = func.to2D(x); - } - - public String[] getCmd() - { - return cmd; - } - - public void outputToCmd(OutputStream outputStream) - { - try { - classifier.classifyAndWriteAnswers(data, featurizedData, outputStream, - classifier.makeReaderAndWriter()); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - public double evaluate(double[] x) { - double score = 0; - setValues(x); - if (getCmd() != null) { - evaluateCmd(getCmd()); - } else { - try { - // TODO: Classify in memory instead of writing to tmp file - File f = File.createTempFile("CRFClassifierEvaluator","txt"); - f.deleteOnExit(); - OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(f)); - classifier.classifyAndWriteAnswers(data, featurizedData, outputStream, - classifier.makeReaderAndWriter()); - outputStream.close(); - BufferedReader br = new BufferedReader(new FileReader(f)); - MultiClassChunkEvalStats stats = new MultiClassChunkEvalStats("O"); - score = stats.score(br, "\t"); - System.err.println(stats.getConllEvalString()); - f.delete(); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - return score; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFCliqueTree.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFCliqueTree.java deleted file mode 100644 index ffaacb0..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFCliqueTree.java +++ /dev/null @@ -1,651 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.sequences.SequenceListener; -import edu.stanford.nlp.sequences.SequenceModel; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.GeneralizedCounter; -import edu.stanford.nlp.util.Index; - -import java.util.Arrays; -import java.util.List; - - -/** Builds a CliqueTree (an array of FactorTable) and does message passing - * inference along it. - * - * @author Jenny Finkel - */ -public class CRFCliqueTree implements SequenceModel, SequenceListener { - - private FactorTable[] factorTables; - private double z; // norm constant - private Index classIndex; - private String backgroundSymbol; - private int backgroundIndex; - // the window size, which is also the clique size - private int windowSize; - // the number of possible classes for each label - private int numClasses; - private int[] possibleValues; - - private CRFCliqueTree() {} - - private CRFCliqueTree(FactorTable[] factorTables, Index classIndex, String backgroundSymbol) { - this.factorTables = factorTables; - this.classIndex = classIndex; - this.backgroundSymbol = backgroundSymbol; - backgroundIndex = classIndex.indexOf(backgroundSymbol); - z = factorTables[0].totalMass(); - windowSize = factorTables[0].windowSize(); - numClasses = classIndex.size(); - possibleValues = new int[numClasses]; - for (int i=0; i=factorTables.length) throw new RuntimeException("Index out of bounds: " + position); -// DecimalFormat nf = new DecimalFormat("#0.000"); - // if (position>0 && position=length()) { - nextLength = length()-position-1; - } - FactorTable nextFactorTable = factorTables[position+nextLength]; - if (nextLength != windowSize - 1) { - for (int j = 0; j < windowSize - 1 - nextLength; j++) { - nextFactorTable = nextFactorTable.sumOutFront(); - } - } - if (nextLength==0) { // we are asking about the prob of no sequence - Arrays.fill(probNextGivenThis, 1.0); - } else { - int[] next = new int[nextLength]; - System.arraycopy(sequence, position+1, next, 0, nextLength); - for (int label=0; label prevLabels.length + 1) { - ft = ft.sumOutFront(); - } - return ft.conditionalLogProbGivenPrevious(prevLabels, label); - } else { - int[] p = new int[windowSize-1]; - System.arraycopy(prevLabels, prevLabels.length - p.length, p, 0, p.length); - return factorTables[position].conditionalLogProbGivenPrevious(p, label); - } - } - - public double condLogProbGivenPrevious(int position, Object label, Object[] prevLabels) { - return condLogProbGivenPrevious(position, classIndex.indexOf(label), objectArrayToIntArray(prevLabels)); - } - - public double condProbGivenPrevious(int position, int label, int[] prevLabels) { - return Math.exp(condLogProbGivenPrevious(position, label, prevLabels)); - } - - public double condProbGivenPrevious(int position, Object label, Object[] prevLabels) { - return Math.exp(condLogProbGivenPrevious(position, label, prevLabels)); - } - - public ClassicCounter condLogProbsGivenPrevious(int position, int[] prevlabels) { - ClassicCounter c = new ClassicCounter(); - for (int i = 0; i < classIndex.size(); i++) { - Object label = classIndex.get(i); - c.incrementCount(label, condLogProbGivenPrevious(position, i, prevlabels)); - } - return c; - } - - public ClassicCounter condLogProbsGivenPrevious(int position, Object[] prevlabels) { - ClassicCounter c = new ClassicCounter(); - for (int i = 0; i < classIndex.size(); i++) { - Object label = classIndex.get(i); - c.incrementCount(label, condLogProbGivenPrevious(position, i, prevlabels)); - } - return c; - } - - // - // PROB OF TAG AT SINGLE POSITION CONDITIONED ON FOLLOWING SEQUENCE OF LABELS - // - - public double condLogProbGivenNext(int position, int label, int[] nextLabels) { - position = position+nextLabels.length; - if (nextLabels.length + 1 == windowSize) { - return factorTables[position].conditionalLogProbGivenNext(nextLabels, label); - } else if (nextLabels.length + 1 < windowSize) { - FactorTable ft = factorTables[position].sumOutFront(); - while (ft.windowSize() > nextLabels.length + 1) { - ft = ft.sumOutFront(); - } - return ft.conditionalLogProbGivenPrevious(nextLabels, label); - } else { - int[] p = new int[windowSize-1]; - System.arraycopy(nextLabels, 0, p, 0, p.length); - return factorTables[position].conditionalLogProbGivenPrevious(p, label); - } - } - - public double condLogProbGivenNext(int position, Object label, Object[] nextLabels) { - return condLogProbGivenNext(position, classIndex.indexOf(label), objectArrayToIntArray(nextLabels)); - } - - public double condProbGivenNext(int position, int label, int[] nextLabels) { - return Math.exp(condLogProbGivenNext(position, label, nextLabels)); - } - - public double condProbGivenNext(int position, Object label, Object[] nextLabels) { - return Math.exp(condLogProbGivenNext(position, label, nextLabels)); - } - - public ClassicCounter condLogProbsGivenNext(int position, int[] nextlabels) { - ClassicCounter c = new ClassicCounter(); - for (int i = 0; i < classIndex.size(); i++) { - Object label = classIndex.get(i); - c.incrementCount(label, condLogProbGivenNext(position, i, nextlabels)); - } - return c; - } - - public ClassicCounter condLogProbsGivenNext(int position, Object[] nextlabels) { - ClassicCounter c = new ClassicCounter(); - for (int i = 0; i < classIndex.size(); i++) { - Object label = classIndex.get(i); - c.incrementCount(label, condLogProbGivenNext(position, i, nextlabels)); - } - return c; - } - - // - // PROB OF TAG AT SINGLE POSITION CONDITIONED ON PREVIOUS AND FOLLOWING SEQUENCE OF LABELS - // - -// public double condProbGivenPreviousAndNext(int position, int label, int[] prevLabels, int[] nextLabels) { - -// } - - - // - // JOINT CONDITIONAL PROBS - // - - /** - * @return a new CRFCliqueTree for the weights on the data - */ - public static CRFCliqueTree getCalibratedCliqueTree(double[][] weights, int[][][] data, Index[] labelIndices, int numClasses, Index classIndex, String backgroundSymbol) { - - - FactorTable[] factorTables = new FactorTable[data.length]; - FactorTable[] messages = new FactorTable[data.length - 1]; - - for (int i = 0; i < data.length; i++) { - - factorTables[i] = getFactorTable(weights, data[i], labelIndices, numClasses); - - if (i > 0) { - messages[i - 1] = factorTables[i - 1].sumOutFront(); - factorTables[i].multiplyInFront(messages[i - 1]); - } - } - - for (int i = factorTables.length - 2; i >= 0; i--) { - - FactorTable summedOut = factorTables[i + 1].sumOutEnd(); - summedOut.divideBy(messages[i]); - factorTables[i].multiplyInEnd(summedOut); - } - - return new CRFCliqueTree(factorTables, classIndex, backgroundSymbol); - } - - /** - * @return a new CRFCliqueTree for the weights on the data - */ - public static CRFCliqueTree getCalibratedCliqueTree(double[] weights, - double wscale, - int[][] weightIndices, - int[][][] data, - Index[] labelIndices, - int numClasses, - Index classIndex, - String backgroundSymbol) { - - - FactorTable[] factorTables = new FactorTable[data.length]; - FactorTable[] messages = new FactorTable[data.length - 1]; - - for (int i = 0; i < data.length; i++) { - - factorTables[i] = getFactorTable(weights, wscale, weightIndices, data[i], labelIndices, numClasses); - - if (i > 0) { - messages[i - 1] = factorTables[i - 1].sumOutFront(); - factorTables[i].multiplyInFront(messages[i - 1]); - } - } - - for (int i = factorTables.length - 2; i >= 0; i--) { - - FactorTable summedOut = factorTables[i + 1].sumOutEnd(); - summedOut.divideBy(messages[i]); - factorTables[i].multiplyInEnd(summedOut); - } - - return new CRFCliqueTree(factorTables, classIndex, backgroundSymbol); - } - - private static FactorTable getFactorTable(double[] weights, double wscale, int[][] weightIndices, int[][] data, Index[] labelIndices, int numClasses) { - - FactorTable factorTable = null; - - for (int j = 0; j < labelIndices.length; j++) { - Index labelIndex = labelIndices[j]; - FactorTable ft = new FactorTable(numClasses, j + 1); - - // ... and each possible labeling for that clique - for (int k = 0, liSize = labelIndex.size(); k < liSize; k++) { - int[] label = ((CRFLabel) labelIndex.get(k)).getLabel(); - double weight = 0.0; - for (int m = 0; m < data[j].length; m++) { - int wi = weightIndices[data[j][m]][k]; - weight += wscale*weights[wi]; - } - // try{ - ft.setValue(label, weight); - // } catch (Exception e) { -// System.out.println("CRFCliqueTree::getFactorTable"); -// System.out.println("NumClasses: " + numClasses + " j+1: " + (j+1)); -// System.out.println("k: " + k+" label: " +label+" labelIndexSize: " + labelIndex.size()); -// throw new RunTimeException(e.toString()); -// } - - } - if (j > 0) { - ft.multiplyInEnd(factorTable); - } - factorTable = ft; - - } - - return factorTable; - } - - private static FactorTable getFactorTable(double[][] weights, int[][] data, Index[] labelIndices, int numClasses) { - - FactorTable factorTable = null; - - for (int j = 0; j < labelIndices.length; j++) { - Index labelIndex = labelIndices[j]; - FactorTable ft = new FactorTable(numClasses, j + 1); - - // ... and each possible labeling for that clique - for (int k = 0, liSize = labelIndex.size(); k < liSize; k++) { - int[] label = ((CRFLabel) labelIndex.get(k)).getLabel(); - double weight = 0.0; - for (int m = 0; m < data[j].length; m++) { - weight += weights[data[j][m]][k]; - } - // try{ - ft.setValue(label, weight); - // } catch (Exception e) { -// System.out.println("CRFCliqueTree::getFactorTable"); -// System.out.println("NumClasses: " + numClasses + " j+1: " + (j+1)); -// System.out.println("k: " + k+" label: " +label+" labelIndexSize: " + labelIndex.size()); -// throw new RunTimeException(e.toString()); -// } - - } - if (j > 0) { - ft.multiplyInEnd(factorTable); - } - factorTable = ft; - - } - - return factorTable; - - } - - // SEQUENCE MODEL METHODS - - /** - * Computes the distribution over values of the element at position pos in the sequence, - * conditioned on the values of the elements in all other positions of the provided sequence. - * - * @param sequence the sequence containing the rest of the values to condition on - * @param position the position of the element to give a distribution for - * @return an array of type double, representing a probability distribution; sums to 1.0 - */ - public double[] getConditionalDistribution(int[] sequence, int position) { - double[] result = scoresOf(sequence, position); - ArrayMath.logNormalize(result); - // System.out.println("marginal: " + ArrayMath.toString(marginal, nf)); - // System.out.println("conditional: " + ArrayMath.toString(result, nf)); - result = ArrayMath.exp(result); - // System.out.println("conditional: " + ArrayMath.toString(result, nf)); - return result; - } - - /** - * Informs this sequence model that the value of the element at position pos has changed. - * This allows this sequence model to update its internal model if desired. - * - */ - public void updateSequenceElement(int[] sequence, int pos, int oldVal) { - // do nothing; we don't change this model - } - - /** - * Informs this sequence model that the value of the whole sequence is initialized to sequence - * - */ - public void setInitialSequence(int[] sequence) { - // do nothing - } - - /** - * @return the number of possible values for each element; it is assumed - * to be the same for the element at each position - */ - public int getNumValues() { - return numClasses; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFDatum.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFDatum.java deleted file mode 100644 index c483dcb..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFDatum.java +++ /dev/null @@ -1,110 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.ling.Datum; - -import java.io.Serializable; -import java.util.List; - - -/** - * The representation of Datums used internally in CRFClassifier. - * - * @author Jenny Finkel - */ - -public class CRFDatum implements Serializable { - - /** - * Features for this Datum. - */ - private final List features; - private LAB label; // = null; - - /** - * Constructs a new BasicDatum with the given features and label. - * - * @param features The features of the CRFDatum - * @param label The label of the CRFDatum - */ - public CRFDatum(List features, LAB label) { - this(features); - setLabel(label); - } - - /** - * Constructs a new BasicDatum with the given features and no labels. - * @param features The features of the CRFDatum - */ - public CRFDatum(List features) { - this.features = features; - } - - /** - * Returns the collection that this BasicDatum was constructed with. - * - * @return the collection that this BasicDatum was constructed with. - */ - public List asFeatures() { - return features; - } - - /** - * Returns the label for this Datum, or null if none have been set. - * @return The label for this Datum, or null if none have been set. - */ - - public LAB label() { - return label; - } - - /** - * Removes all currently assigned Labels for this Datum then adds the - * given Label. - * Calling setLabel(null) effectively clears all labels. - * @param label New label for CRFDatum - */ - public void setLabel(LAB label) { - this.label = label; - } - - /** - * Returns a String representation of this BasicDatum (lists features and labels). - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder("CRFDatum[\n"); - sb.append(" label=").append(label).append('\n'); - for (int i = 0, sz = features.size(); i < sz; i++) { - sb.append(" features(").append(i).append("):").append(features.get(i)).append('\n'); - } - sb.append(']'); - return sb.toString(); - } - - - /** - * Returns whether the given Datum contains the same features as this Datum. - * Doesn't check the labels, should we change this? - * - * @param o The object to test equality with - * @return Whether it is equal to this CRFDatum in terms of features - */ - @Override - public boolean equals(Object o) { - if (!(o instanceof Datum)) { - return (false); - } - - Datum d = (Datum) o; - return features.equals(d.asFeatures()); - } - - @Override - public int hashCode() { - return features.hashCode(); - } - - private static final long serialVersionUID = -8345554365027671190L; - -} - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFFeatureExporter.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFFeatureExporter.java deleted file mode 100644 index 43849fd..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFFeatureExporter.java +++ /dev/null @@ -1,176 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.ling.CoreAnnotations; -import edu.stanford.nlp.ling.CoreLabel; -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.StringUtils; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -/** - * Exports CRF features for use with other programs - * - Usage: CRFFeatureExporter -prop -trainFile -exportFeatures - * - Output file is automatically gzipped/b2zipped if ending in gz/bz2 - * - bzip2 requires that bzip2 is availaible via command line - * - Currently exports features in a format that can be read by a modified crfsgd - * (crfsgd assumes features are gzipped) - * TODO: Support other formats (like crfsuite) - * @author Angel Chang - */ -public class CRFFeatureExporter { - private char delimiter = '\t'; - private static String eol = System.getProperty("line.separator"); - private CRFClassifier classifier; - - public CRFFeatureExporter(CRFClassifier classifier) - { - this.classifier = classifier; - } - - /** - * Prefix features with U- (for unigram) features - * or B- (for bigram) features - * @param feat String representing the feature - * @return new prefixed feature string - */ - private String ubPrefixFeatureString(String feat) - { - if (feat.endsWith("|C")) { - return "U-" + feat; - } else if (feat.endsWith("|CpC")) { - return "B-" + feat; - } else { - return feat; - } - } - - /** - * Constructs a big string representing the input list of CoreLabel, - * with one line per token using the following format - * word label feat1 feat2 ... - * (where each space is actually a tab) - * Assume that CoreLabel has both TextAnnotation and AnswerAnnotation - * @param document List of CoreLabel - * (does not have to represent a "document", just a sequence of text, - * like a sentence or a paragraph) - * @return String representation of features - */ - private String getFeatureString(List document) { - int docSize = document.size(); - if (classifier.flags.useReverse) { - Collections.reverse(document); - } - - StringBuilder sb = new StringBuilder(); - for (int j = 0; j < docSize; j++) { - IN token = document.get(j); - sb.append(token.get(CoreAnnotations.TextAnnotation.class)); - sb.append(delimiter); - sb.append(token.get(CoreAnnotations.AnswerAnnotation.class)); - - CRFDatum d = classifier.makeDatum(document, j, classifier.featureFactory); - - List features = d.asFeatures(); - for (int k = 0, fSize = features.size(); k < fSize; k++) { - Collection cliqueFeatures = (Collection) features.get(k); - for (String feat: cliqueFeatures) { - feat = ubPrefixFeatureString(feat); - sb.append(delimiter); - sb.append(feat); - } - } - sb.append(eol); - } - if (classifier.flags.useReverse) { - Collections.reverse(document); - } - return sb.toString(); - } - - /** - * Output features that have already been converted into features - * (using documentToDataAndLabels) in format suitable for CRFSuite - * Format is with one line per token using the following format - * label feat1 feat2 ... - * (where each space is actually a tab) - * Each document is separated by an empty line - * @param exportFile file to export the features to - * @param docsData array of document features - * @param labels correct labels indexed by document, and position within document - */ - public void printFeatures(String exportFile, int[][][][] docsData, int[][] labels) { - try { - PrintWriter pw = IOUtils.getPrintWriter(exportFile); - for (int i = 0; i < docsData.length; i++) { - for (int j = 0; j < docsData[i].length; j++) { - StringBuilder sb = new StringBuilder(); - int label = labels[i][j]; - sb.append(classifier.classIndex.get(label)); - for (int k = 0; k < docsData[i][j].length; k++) { - for (int m = 0; m < docsData[i][j][k].length; m++) { - String feat = classifier.featureIndex.get(docsData[i][j][k][m]); - feat = ubPrefixFeatureString(feat); - sb.append(delimiter); - sb.append(feat); - } - } - pw.println(sb.toString()); - } - pw.println(); - } - pw.close(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - /** - * Output features from a collection of documents to a file - * Format is with one line per token using the following format - * word label feat1 feat2 ... - * (where each space is actually a tab) - * Each document is separated by an empty line - * This format is suitable for modified crfsgd - * @param exportFile file to export the features to - * @param documents input collection of documents - */ - public void printFeatures(String exportFile, Collection> documents) { - try { - PrintWriter pw = IOUtils.getPrintWriter(exportFile); - for (List doc:documents) { - String str = getFeatureString(doc); - pw.println(str); - } - pw.close(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - public static void main(String[] args) throws Exception { - StringUtils.printErrInvocationString("CRFFeatureExporter", args); - Properties props = StringUtils.argsToProperties(args); - CRFClassifier crf = new CRFClassifier(props); - String inputFile = crf.flags.trainFile; - if (inputFile == null) { - System.err.println("Please provide input file using -trainFile"); - System.exit(-1); - } - String outputFile = crf.flags.exportFeatures; - if (outputFile == null) { - System.err.println("Please provide output file using -exportFeatures"); - System.exit(-1); - } - CRFFeatureExporter featureExporter = new CRFFeatureExporter(crf); - Collection> docs = - crf.makeObjectBankFromFile(inputFile, crf.makeReaderAndWriter()); - crf.makeAnswerArraysAndTagIndex(docs); - featureExporter.printFeatures(outputFile, docs); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFLabel.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFLabel.java deleted file mode 100644 index 9aa5ed6..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFLabel.java +++ /dev/null @@ -1,91 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.util.Index; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - - -/** - * @author Jenny Finkel - */ - -public class CRFLabel implements Serializable { - - /** - * - */ - private static final long serialVersionUID = 7403010868396790276L; - private int[] label; - int hashCode = -1; - - public static int maxNumClasses = 10; - - public CRFLabel(int[] label) { - this.label = label; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof CRFLabel)) { - return false; - } - CRFLabel other = (CRFLabel) o; - - if (other.label.length != label.length) { - return false; - } - for (int i = 0; i < label.length; i++) { - if (label[i] != other.label[i]) { - return false; - } - } - - return true; - } - - public CRFLabel getSmallerLabel(int size) { - int[] newLabel = new int[size]; - System.arraycopy(label, label.length - size, newLabel, 0, size); - return new CRFLabel(newLabel); - } - - public CRFLabel getOneSmallerLabel() { - return getSmallerLabel(label.length - 1); - } - - public int[] getLabel() { - return label; - } - - public String toString(Index classIndex) { - List l = new ArrayList(); - for (int i = 0; i < label.length; i++) { - l.add(classIndex.get(label[i])); - } - return l.toString(); - } - - @Override - public String toString() { - List l = new ArrayList(); - for (int i = 0; i < label.length; i++) { - l.add(Integer.valueOf(label[i])); - } - return l.toString(); - } - - @Override - public int hashCode() { - if (hashCode < 0) { - hashCode = 0; - for (int i = 0; i < label.length; i++) { - hashCode *= maxNumClasses; - hashCode += label[i]; - } - } - return hashCode; - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFLogConditionalObjectiveFloatFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFLogConditionalObjectiveFloatFunction.java deleted file mode 100644 index b1cd27d..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/CRFLogConditionalObjectiveFloatFunction.java +++ /dev/null @@ -1,604 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.optimization.AbstractCachingDiffFloatFunction; -import edu.stanford.nlp.util.Index; - -import java.util.Arrays; - - -/** - * @author Jenny Finkel - */ - -public class CRFLogConditionalObjectiveFloatFunction extends AbstractCachingDiffFloatFunction { - - public static final int NO_PRIOR = 0; - public static final int QUADRATIC_PRIOR = 1; - /* Use a Huber robust regression penalty (L1 except very near 0) not L2 */ - public static final int HUBER_PRIOR = 2; - public static final int QUARTIC_PRIOR = 3; - - protected int prior; - protected float sigma; - protected float epsilon; - - Index[] labelIndices; - Index classIndex; - Index featureIndex; - float[][] Ehat; // empirical counts of all the features [feature][class] - int window; - int numClasses; - int[] map; - int[][][][] data; - int[][] labels; - int domainDimension = -1; - - String crfType = "maxent"; - String backgroundSymbol; - - public static boolean VERBOSE = false; - - CRFLogConditionalObjectiveFloatFunction(int[][][][] data, int[][] labels, Index featureIndex, int window, Index classIndex, Index[] labelIndices, int[] map, String backgroundSymbol) { - this(data, labels, featureIndex, window, classIndex, labelIndices, map, QUADRATIC_PRIOR, backgroundSymbol); - } - - CRFLogConditionalObjectiveFloatFunction(int[][][][] data, int[][] labels, Index featureIndex, int window, Index classIndex, Index[] labelIndices, int[] map, String backgroundSymbol, double sigma) { - this(data, labels, featureIndex, window, classIndex, labelIndices, map, QUADRATIC_PRIOR, backgroundSymbol, sigma); - } - - CRFLogConditionalObjectiveFloatFunction(int[][][][] data, int[][] labels, Index featureIndex, int window, Index classIndex, Index[] labelIndices, int[] map, int prior, String backgroundSymbol) { - this(data, labels, featureIndex, window, classIndex, labelIndices, map, prior, backgroundSymbol, 1.0f); - } - - CRFLogConditionalObjectiveFloatFunction(int[][][][] data, int[][] labels, Index featureIndex, int window, Index classIndex, Index[] labelIndices, int[] map, int prior, String backgroundSymbol, double sigma) { - this.featureIndex = featureIndex; - this.window = window; - this.classIndex = classIndex; - this.numClasses = classIndex.size(); - this.labelIndices = labelIndices; - this.map = map; - this.data = data; - this.labels = labels; - this.prior = prior; - this.backgroundSymbol = backgroundSymbol; - this.sigma = (float) sigma; - empiricalCounts(data, labels); - } - - @Override - public int domainDimension() { - if (domainDimension < 0) { - domainDimension = 0; - for (int i = 0; i < map.length; i++) { - domainDimension += labelIndices[map[i]].size(); - } - } - return domainDimension; - } - - public float[][] to2D(float[] weights) { - float[][] newWeights = new float[map.length][]; - int index = 0; - for (int i = 0; i < map.length; i++) { - newWeights[i] = new float[labelIndices[map[i]].size()]; - System.arraycopy(weights, index, newWeights[i], 0, labelIndices[map[i]].size()); - index += labelIndices[map[i]].size(); - } - return newWeights; - } - - public float[] to1D(float[][] weights) { - float[] newWeights = new float[domainDimension()]; - int index = 0; - for (int i = 0; i < weights.length; i++) { - System.arraycopy(weights[i], 0, newWeights, index, weights[i].length); - index += weights[i].length; - } - return newWeights; - } - - public float[][] empty2D() { - float[][] d = new float[map.length][]; - int index = 0; - for (int i = 0; i < map.length; i++) { - d[i] = new float[labelIndices[map[i]].size()]; - Arrays.fill(d[i], 0); - index += labelIndices[map[i]].size(); - } - return d; - } - - private void empiricalCounts(int[][][][] data, int[][] labels) { - Ehat = empty2D(); - - for (int m = 0; m < data.length; m++) { - int[][][] dataDoc = data[m]; - int[] labelsDoc = labels[m]; - int[] label = new int[window]; - //Arrays.fill(label, classIndex.indexOf("O")); - Arrays.fill(label, classIndex.indexOf(backgroundSymbol)); - for (int i = 0; i < dataDoc.length; i++) { - System.arraycopy(label, 1, label, 0, window - 1); - label[window - 1] = labelsDoc[i]; - for (int j = 0; j < dataDoc[i].length; j++) { - int[] cliqueLabel = new int[j + 1]; - System.arraycopy(label, window - 1 - j, cliqueLabel, 0, j + 1); - CRFLabel crfLabel = new CRFLabel(cliqueLabel); - int labelIndex = labelIndices[j].indexOf(crfLabel); - //System.err.println(crfLabel + " " + labelIndex); - for (int k = 0; k < dataDoc[i][j].length; k++) { - Ehat[dataDoc[i][j][k]][labelIndex]++; - } - } - } - } - } - - public static FloatFactorTable getFloatFactorTable(float[][] weights, int[][] data, Index[] labelIndices, int numClasses) { - - FloatFactorTable factorTable = null; - - for (int j = 0; j < labelIndices.length; j++) { - Index labelIndex = labelIndices[j]; - FloatFactorTable ft = new FloatFactorTable(numClasses, j + 1); - - // ...and each possible labeling for that clique - for (int k = 0; k < labelIndex.size(); k++) { - int[] label = ((CRFLabel) labelIndex.get(k)).getLabel(); - float weight = 0.0f; - for (int m = 0; m < data[j].length; m++) { - //System.err.println("**"+weights[data[j][m]][k]); - weight += weights[data[j][m]][k]; - } - ft.setValue(label, weight); - //System.err.println(">>"+ft); - } - //System.err.println("::"+ft); - if (j > 0) { - ft.multiplyInEnd(factorTable); - } - //System.err.println("::"+ft); - factorTable = ft; - - } - - return factorTable; - - } - - - - public static FloatFactorTable[] getCalibratedCliqueTree(float[][] weights, int[][][] data, Index[] labelIndices, int numClasses) { - - // for (int i = 0; i < weights.length; i++) { - // for (int j = 0; j < weights[i].length; j++) { - // System.err.println(i+" "+j+": "+weights[i][j]); - // } - // } - - //System.err.println("calibrating clique tree"); - - FloatFactorTable[] factorTables = new FloatFactorTable[data.length]; - FloatFactorTable[] messages = new FloatFactorTable[data.length - 1]; - - for (int i = 0; i < data.length; i++) { - - factorTables[i] = getFloatFactorTable(weights, data[i], labelIndices, numClasses); - if (VERBOSE) { - System.err.println(i + ": " + factorTables[i]); - } - - if (i > 0) { - messages[i - 1] = factorTables[i - 1].sumOutFront(); - if (VERBOSE) { - System.err.println(messages[i - 1]); - } - factorTables[i].multiplyInFront(messages[i - 1]); - if (VERBOSE) { - System.err.println(factorTables[i]); - if (i == data.length - 1) { - System.err.println(i + ": " + factorTables[i].toProbString()); - } - } - } - } - - for (int i = factorTables.length - 2; i >= 0; i--) { - - FloatFactorTable summedOut = factorTables[i + 1].sumOutEnd(); - if (VERBOSE) { - System.err.println((i + 1) + "-->" + i + ": " + summedOut); - } - summedOut.divideBy(messages[i]); - if (VERBOSE) { - System.err.println((i + 1) + "-->" + i + ": " + summedOut); - } - factorTables[i].multiplyInEnd(summedOut); - if (VERBOSE) { - System.err.println(i + ": " + factorTables[i]); - System.err.println(i + ": " + factorTables[i].toProbString()); - } - - - } - - return factorTables; - } - - @Override - public void calculate(float[] x) { - - if (crfType.equalsIgnoreCase("weird")) { - calculateWeird(x); - return; - } - - float[][] weights = to2D(x); - float prob = 0; - - float[][] E = empty2D(); - - for (int m = 0; m < data.length; m++) { - - FloatFactorTable[] factorTables = getCalibratedCliqueTree(weights, data[m], labelIndices, numClasses); - // System.err.println("calibrated:"); - // for (int i = 0; i < factorTables.length; i++) { - // System.out.println(factorTables[i]); - // System.out.println("+++++++++++++++++++++++++++++"); - - // } - // System.exit(0); - float z = factorTables[0].totalMass(); - - int[] given = new int[window - 1]; - Arrays.fill(given, classIndex.indexOf(backgroundSymbol)); - for (int i = 0; i < data[m].length; i++) { - float p = factorTables[i].conditionalLogProb(given, labels[m][i]); - if (VERBOSE) { - System.err.println("P(" + labels[m][i] + "|" + Arrays.toString(given) + ")=" + p); - } - prob += p; - System.arraycopy(given, 1, given, 0, given.length - 1); - given[given.length - 1] = labels[m][i]; - } - - // get predicted count - for (int i = 0; i < data[m].length; i++) { - // go through each clique... - for (int j = 0; j < data[m][i].length; j++) { - Index labelIndex = labelIndices[j]; - // ...and each possible labeling for that clique - for (int k = 0; k < labelIndex.size(); k++) { - int[] label = ((CRFLabel) labelIndex.get(k)).getLabel(); - - // float p = Math.pow(Math.E, factorTables[i].logProbEnd(label)); - float p = (float) Math.exp(factorTables[i].unnormalizedLogProbEnd(label) - z); - for (int n = 0; n < data[m][i][j].length; n++) { - E[data[m][i][j][n]][k] += p; - } - } - } - } - } - - if (Float.isNaN(prob)) { - System.exit(0); - } - value = -prob; - - // compute the partial derivative for each feature - int index = 0; - for (int i = 0; i < E.length; i++) { - for (int j = 0; j < E[i].length; j++) { - derivative[index++] = (E[i][j] - Ehat[i][j]); - if (VERBOSE) { - System.err.println("deriv(" + i + "," + j + ") = " + E[i][j] + " - " + Ehat[i][j] + " = " + derivative[index - 1]); - } - } - } - - - // priors - if (prior == QUADRATIC_PRIOR) { - float sigmaSq = sigma * sigma; - for (int i = 0; i < x.length; i++) { - float k = 1.0f; - float w = x[i]; - value += k * w * w / 2.0 / sigmaSq; - derivative[i] += k * w / sigmaSq; - } - } else if (prior == HUBER_PRIOR) { - float sigmaSq = sigma * sigma; - for (int i = 0; i < x.length; i++) { - float w = x[i]; - float wabs = Math.abs(w); - if (wabs < epsilon) { - value += w * w / 2.0 / epsilon / sigmaSq; - derivative[i] += w / epsilon / sigmaSq; - } else { - value += (wabs - epsilon / 2) / sigmaSq; - derivative[i] += ((w < 0.0) ? -1.0 : 1.0) / sigmaSq; - } - } - } else if (prior == QUARTIC_PRIOR) { - float sigmaQu = sigma * sigma * sigma * sigma; - for (int i = 0; i < x.length; i++) { - float k = 1.0f; - float w = x[i]; - value += k * w * w * w * w / 2.0 / sigmaQu; - derivative[i] += k * w / sigmaQu; - } - } - - - } - - public void calculateWeird1(float[] x) { - - float[][] weights = to2D(x); - float[][] E = empty2D(); - - value = 0.0f; - Arrays.fill(derivative, 0.0f); - float[][] sums = new float[labelIndices.length][]; - float[][] probs = new float[labelIndices.length][]; - float[][] counts = new float[labelIndices.length][]; - - for (int i = 0; i < sums.length; i++) { - int size = labelIndices[i].size(); - sums[i] = new float[size]; - probs[i] = new float[size]; - counts[i] = new float[size]; - Arrays.fill(counts[i], 0.0f); - } - - for (int d = 0; d < data.length; d++) { - int[] llabels = labels[d]; - for (int e = 0; e < data[d].length; e++) { - int[][] ddata = this.data[d][e]; - - for (int cl = 0; cl < ddata.length; cl++) { - int[] features = ddata[cl]; - // activation - Arrays.fill(sums[cl], 0.0f); - int numClasses = labelIndices[cl].size(); - for (int c = 0; c < numClasses; c++) { - for (int f = 0; f < features.length; f++) { - sums[cl][c] += weights[features[f]][c]; - } - } - } - - - for (int cl = 0; cl < ddata.length; cl++) { - - int[] label = new int[cl + 1]; - //Arrays.fill(label, classIndex.indexOf("O")); - Arrays.fill(label, classIndex.indexOf(backgroundSymbol)); - int index1 = label.length - 1; - for (int pos = e; pos >= 0 && index1 >= 0; pos--) { - //System.err.println(index1+" "+pos); - label[index1--] = llabels[pos]; - } - CRFLabel crfLabel = new CRFLabel(label); - int labelIndex = labelIndices[cl].indexOf(crfLabel); - - float total = ArrayMath.logSum(sums[cl]); - // int[] features = ddata[cl]; - int numClasses = labelIndices[cl].size(); - for (int c = 0; c < numClasses; c++) { - probs[cl][c] = (float) Math.exp(sums[cl][c] - total); - } - // for (int f=0; f[] labelIndices; - Index classIndex; // really assumed to be IndexdocData.length) { // only true for self-training - // fill the windowLabel array with the extra docLabels - System.arraycopy(docLabels, 0, windowLabels, 0, windowLabels.length); - // shift the docLabels array left - int[] newDocLabels = new int[docData.length]; - System.arraycopy(docLabels, docLabels.length-newDocLabels.length, newDocLabels, 0, newDocLabels.length); - docLabels = newDocLabels; - } - for (int i = 0; i < docData.length; i++) { - System.arraycopy(windowLabels, 1, windowLabels, 0, window - 1); - windowLabels[window - 1] = docLabels[i]; - for (int j = 0; j < docData[i].length; j++) { - int[] cliqueLabel = new int[j + 1]; - System.arraycopy(windowLabels, window - 1 - j, cliqueLabel, 0, j + 1); - CRFLabel crfLabel = new CRFLabel(cliqueLabel); - int labelIndex = labelIndices[j].indexOf(crfLabel); - //System.err.println(crfLabel + " " + labelIndex); - for (int k = 0; k < docData[i][j].length; k++) { - Ehat[docData[i][j][k]][labelIndex]++; - } - } - } - } - } - - // todo [cdm]: Below data[m] --> docData - /** - * Calculates both value and partial derivatives at the point x, and save them internally. - */ - @Override - public void calculate(double[] x) { - - double prob = 0.0; // the log prob of the sequence given the model, which is the negation of value at this point - double[][] weights = to2D(x); - - // the expectations over counts - // first index is feature index, second index is of possible labeling - double[][] E = empty2D(); - - // iterate over all the documents - for (int m = 0; m < data.length; m++) { - int[][][] docData = data[m]; - int[] docLabels = labels[m]; - - // make a clique tree for this document - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, docData, labelIndices, numClasses, classIndex, backgroundSymbol); - - // compute the log probability of the document given the model with the parameters x - int[] given = new int[window - 1]; - Arrays.fill(given, classIndex.indexOf(backgroundSymbol)); - if (docLabels.length>docData.length) { // only true for self-training - // fill the given array with the extra docLabels - System.arraycopy(docLabels, 0, given, 0, given.length); - // shift the docLabels array left - int[] newDocLabels = new int[docData.length]; - System.arraycopy(docLabels, docLabels.length-newDocLabels.length, newDocLabels, 0, newDocLabels.length); - docLabels = newDocLabels; - } - // iterate over the positions in this document - for (int i = 0; i < docData.length; i++) { - int label = docLabels[i]; - double p = cliqueTree.condLogProbGivenPrevious(i, label, given); - if (VERBOSE) { - System.err.println("P(" + label + "|" + ArrayMath.toString(given) + ")=" + p); - } - prob += p; - System.arraycopy(given, 1, given, 0, given.length - 1); - given[given.length - 1] = label; - } - - // compute the expected counts for this document, which we will need to compute the derivative - // iterate over the positions in this document - for (int i = 0; i < data[m].length; i++) { - // for each possible clique at this position - for (int j = 0; j < data[m][i].length; j++) { - Index labelIndex = labelIndices[j]; - // for each possible labeling for that clique - for (int k = 0; k < labelIndex.size(); k++) { - int[] label = labelIndex.get(k).getLabel(); - double p = cliqueTree.prob(i, label); // probability of these labels occurring in this clique with these features - for (int n = 0; n < data[m][i][j].length; n++) { - E[data[m][i][j][n]][k] += p; - } - } - } - } - } - - if (Double.isNaN(prob)) { // shouldn't be the case - throw new RuntimeException("Got NaN for prob in CRFLogConditionalObjectiveFunction.calculate()"); - } - - value = -prob; - - // compute the partial derivative for each feature by comparing expected counts to empirical counts - int index = 0; - for (int i = 0; i < E.length; i++) { - for (int j = 0; j < E[i].length; j++) { - derivative[index++] = (E[i][j] - Ehat[i][j]); - if (VERBOSE) { - System.err.println("deriv(" + i + "," + j + ") = " + E[i][j] + " - " + Ehat[i][j] + " = " + derivative[index - 1]); - } - } - } - - - // incorporate priors - if (prior == QUADRATIC_PRIOR) { - double sigmaSq = sigma * sigma; - for (int i = 0; i < x.length; i++) { - double k = 1.0; - double w = x[i]; - value += k * w * w / 2.0 / sigmaSq; - derivative[i] += k * w / sigmaSq; - } - } else if (prior == HUBER_PRIOR) { - double sigmaSq = sigma * sigma; - for (int i = 0; i < x.length; i++) { - double w = x[i]; - double wabs = Math.abs(w); - if (wabs < epsilon) { - value += w * w / 2.0 / epsilon / sigmaSq; - derivative[i] += w / epsilon / sigmaSq; - } else { - value += (wabs - epsilon / 2) / sigmaSq; - derivative[i] += ((w < 0.0) ? -1.0 : 1.0) / sigmaSq; - } - } - } else if (prior == QUARTIC_PRIOR) { - double sigmaQu = sigma * sigma * sigma * sigma; - for (int i = 0; i < x.length; i++) { - double k = 1.0; - double w = x[i]; - value += k * w * w * w * w / 2.0 / sigmaQu; - derivative[i] += k * w / sigmaQu; - } - } - - - } - - @Override - public void calculateStochastic(double[] x, double [] v, int[] batch){ - calculateStochasticGradientOnly(x,batch); - } - - @Override - public int dataDimension(){ - return data.length; - } - - - - public void calculateStochasticGradientOnly(double[] x, int[] batch) { - - double prob = 0; // the log prob of the sequence given the model, which is the negation of value at this point - double[][] weights = to2D(x); - - double batchScale = ((double) batch.length)/((double) this.dataDimension()); - - // the expectations over counts - // first index is feature index, second index is of possible labeling - double[][] E = empty2D(); - - // iterate over all the documents - for (int m = 0; m < batch.length; m++) { - int ind = batch[m]; - int[][][] docData = data[ind]; - int[] docLabels = labels[ind]; - - // make a clique tree for this document - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, docData, labelIndices, numClasses, classIndex, backgroundSymbol); - - // compute the log probability of the document given the model with the parameters x - int[] given = new int[window - 1]; - Arrays.fill(given, classIndex.indexOf(backgroundSymbol)); - if (docLabels.length>docData.length) { // only true for self-training - // fill the given array with the extra docLabels - System.arraycopy(docLabels, 0, given, 0, given.length); - // shift the docLabels array left - int[] newDocLabels = new int[docData.length]; - System.arraycopy(docLabels, docLabels.length-newDocLabels.length, newDocLabels, 0, newDocLabels.length); - docLabels = newDocLabels; - } - // iterate over the positions in this document - for (int i = 0; i < docData.length; i++) { - int label = docLabels[i]; - double p = cliqueTree.condLogProbGivenPrevious(i, label, given); - if (VERBOSE) { - System.err.println("P(" + label + "|" + ArrayMath.toString(given) + ")=" + p); - } - prob += p; - System.arraycopy(given, 1, given, 0, given.length - 1); - given[given.length - 1] = label; - } - - // compute the expected counts for this document, which we will need to compute the derivative - // iterate over the positions in this document - for (int i = 0; i < data[ind].length; i++) { - // for each possible clique at this position - for (int j = 0; j < data[ind][i].length; j++) { - Index labelIndex = labelIndices[j]; - // for each possible labeling for that clique - for (int k = 0; k < labelIndex.size(); k++) { - int[] label = ((CRFLabel) labelIndex.get(k)).getLabel(); - double p = cliqueTree.prob(i, label); // probability of these labels occurring in this clique with these features - for (int n = 0; n < data[ind][i][j].length; n++) { - E[data[ind][i][j][n]][k] += p; - } - } - } - } - } - - if (Double.isNaN(prob)) { // shouldn't be the case - throw new RuntimeException("Got NaN for prob in CRFLogConditionalObjectiveFunction.calculate()"); - } - - value = -prob; - - // compute the partial derivative for each feature by comparing expected counts to empirical counts - int index = 0; - for (int i = 0; i < E.length; i++) { - for (int j = 0; j < E[i].length; j++) { - derivative[index++] = (E[i][j] - batchScale*Ehat[i][j]); - if (VERBOSE) { - System.err.println("deriv(" + i + "," + j + ") = " + E[i][j] + " - " + Ehat[i][j] + " = " + derivative[index - 1]); - } - } - } - - - // incorporate priors - if (prior == QUADRATIC_PRIOR) { - double sigmaSq = sigma * sigma; - for (int i = 0; i < x.length; i++) { - double k = 1.0; - double w = x[i]; - value += batchScale*k * w * w / 2.0 / sigmaSq; - derivative[i] += batchScale*k * w / sigmaSq; - } - } else if (prior == HUBER_PRIOR) { - double sigmaSq = sigma * sigma; - for (int i = 0; i < x.length; i++) { - double w = x[i]; - double wabs = Math.abs(w); - if (wabs < epsilon) { - value += batchScale*w * w / 2.0 / epsilon / sigmaSq; - derivative[i] += batchScale*w / epsilon / sigmaSq; - } else { - value += batchScale*(wabs - epsilon / 2) / sigmaSq; - derivative[i] += batchScale*((w < 0.0) ? -1.0 : 1.0) / sigmaSq; - } - } - } else if (prior == QUARTIC_PRIOR) { - double sigmaQu = sigma * sigma * sigma * sigma; - for (int i = 0; i < x.length; i++) { - double k = 1.0; - double w = x[i]; - value += batchScale*k * w * w * w * w / 2.0 / sigmaQu; - derivative[i] += batchScale*k * w / sigmaQu; - } - } - } - - /** - * Performs stochastic update of weights x (scaled by xscale) based - * on samples indexed by batch - * NOTE: This function does not do regularization (regularization is done by minimizer) - * @param x - unscaled weights - * @param xscale - how much to scale x by when performing calculations - * @param batch - indices of which samples to compute function over - * @param gscale - how much to scale adjustments to x - * @return value of function at specified x (scaled by xscale) for samples - */ - public double calculateStochasticUpdate(double[] x, double xscale, int[] batch, double gscale) { - double prob = 0; // the log prob of the sequence given the model, which is the negation of value at this point - double[] weights = x; - int[][] wis = getWeightIndices(); - - // Adjust weight by -gscale*gradient - // gradient is expected count - empirical count - // so we adjust by + gscale(empirical count - expected count) - - int[] given = new int[window - 1]; - int[][] docCliqueLabels = new int[window][]; - for (int j = 0; j < window; j++) { - docCliqueLabels[j] = new int[j+1]; - } - // iterate over all the documents - for (int m = 0; m < batch.length; m++) { - int ind = batch[m]; - int[][][] docData = data[ind]; - int[] docLabels = labels[ind]; - - // make a clique tree for this document - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, xscale, wis, docData, - labelIndices, numClasses, classIndex, backgroundSymbol); - - // compute the log probability of the document given the model with the parameters x - Arrays.fill(given, classIndex.indexOf(backgroundSymbol)); - if (docLabels.length>docData.length) { // only true for self-training - // fill the given array with the extra docLabels - System.arraycopy(docLabels, 0, given, 0, given.length); - // shift the docLabels array left - int[] newDocLabels = new int[docData.length]; - System.arraycopy(docLabels, docLabels.length-newDocLabels.length, newDocLabels, 0, newDocLabels.length); - docLabels = newDocLabels; - } - // iterate over the positions in this document - for (int i = 0; i < docData.length; i++) { - int label = docLabels[i]; - double p = cliqueTree.condLogProbGivenPrevious(i, label, given); - if (VERBOSE) { - System.err.println("P(" + label + "|" + ArrayMath.toString(given) + ")=" + p); - } - prob += p; - - // Empirical count - for (int j = 0; j < data[ind][i].length; j++) { - if (j > 0) { - System.arraycopy(given, window - j - 1, docCliqueLabels[j], 0, j); - } - docCliqueLabels[j][j] = label; - // TODO: We can eliminate this lookup by saving the correctLabelIndex (or marking it in CRFLabel) - CRFLabel crfLabel = new CRFLabel(docCliqueLabels[j]); - int correctLabelIndex = labelIndices[j].indexOf(crfLabel); - for (int n = 0; n < data[ind][i][j].length; n++) { - // Adjust by gscale (empirical count) - x[wis[data[ind][i][j][n]][correctLabelIndex]] += gscale; - } - } - // Shift window over - System.arraycopy(given, 1, given, 0, given.length - 1); - given[given.length - 1] = label; - } - - // compute the expected counts for this document, which we will need to compute the derivative - // iterate over the positions in this document - for (int i = 0; i < data[ind].length; i++) { - // for each possible clique at this position - for (int j = 0; j < data[ind][i].length; j++) { - // Expected count - Index labelIndex = labelIndices[j]; - // for each possible labeling for that clique - for (int k = 0; k < labelIndex.size(); k++) { - int[] label = ((CRFLabel) labelIndex.get(k)).getLabel(); - double p = cliqueTree.prob(i, label); // probability of these labels occurring in this clique with these features - for (int n = 0; n < data[ind][i][j].length; n++) { - // Adjust weight by -p*gscale (expected count scaled) - x[wis[docData[i][j][n]][k]] -= p*gscale; - } - } - } - } - } - - if (Double.isNaN(prob)) { // shouldn't be the case - throw new RuntimeException("Got NaN for prob in CRFLogConditionalObjectiveFunction.calculate()"); - } - - value = -prob; - return value; - } - - /** - * Computes value of function for specified value of x (scaled by xscale) - * only over samples indexed by batch - * NOTE: This function does not do regularization (regularization is done by minimizer) - * @param x - unscaled weights - * @param xscale - how much to scale x by when performing calculations - * @param batch - indices of which samples to compute function over - * @return value of function at specified x (scaled by xscale) for samples - */ - public double valueAt(double[] x, double xscale, int[] batch) { - double prob = 0; // the log prob of the sequence given the model, which is the negation of value at this point - double[] weights = x; - int[][] wis = getWeightIndices(); - - int[] given = new int[window - 1]; - int[][] docCliqueLabels = new int[window][]; - for (int j = 0; j < window; j++) { - docCliqueLabels[j] = new int[j+1]; - } - // iterate over all the documents - for (int m = 0; m < batch.length; m++) { - int ind = batch[m]; - int[][][] docData = data[ind]; - int[] docLabels = labels[ind]; - - // make a clique tree for this document - CRFCliqueTree cliqueTree = CRFCliqueTree.getCalibratedCliqueTree(weights, xscale, wis, docData, - labelIndices, numClasses, classIndex, backgroundSymbol); - - // compute the log probability of the document given the model with the parameters x - Arrays.fill(given, classIndex.indexOf(backgroundSymbol)); - if (docLabels.length>docData.length) { // only true for self-training - // fill the given array with the extra docLabels - System.arraycopy(docLabels, 0, given, 0, given.length); - // shift the docLabels array left - int[] newDocLabels = new int[docData.length]; - System.arraycopy(docLabels, docLabels.length-newDocLabels.length, newDocLabels, 0, newDocLabels.length); - docLabels = newDocLabels; - } - // iterate over the positions in this document - for (int i = 0; i < docData.length; i++) { - int label = docLabels[i]; - double p = cliqueTree.condLogProbGivenPrevious(i, label, given); - if (VERBOSE) { - System.err.println("P(" + label + "|" + ArrayMath.toString(given) + ")=" + p); - } - prob += p; - - // Shift window over - System.arraycopy(given, 1, given, 0, given.length - 1); - given[given.length - 1] = label; - } - - } - - if (Double.isNaN(prob)) { // shouldn't be the case - throw new RuntimeException("Got NaN for prob in CRFLogConditionalObjectiveFunction.calculate()"); - } - - value = -prob; - return value; - } -// public void calculateWeird1(double[] x) { - -// double[][] weights = to2D(x); -// double[][] E = empty2D(); - -// value = 0.0; -// Arrays.fill(derivative, 0.0); -// double[][] sums = new double[labelIndices.length][]; -// double[][] probs = new double[labelIndices.length][]; -// double[][] counts = new double[labelIndices.length][]; - -// for (int i = 0; i < sums.length; i++) { -// int size = labelIndices[i].size(); -// sums[i] = new double[size]; -// probs[i] = new double[size]; -// counts[i] = new double[size]; -// Arrays.fill(counts[i], 0.0); -// } - -// for (int d = 0; d < data.length; d++) { -// int[] llabels = labels[d]; -// for (int e = 0; e < data[d].length; e++) { -// int[][] ddata = this.data[d][e]; - -// for (int cl = 0; cl < ddata.length; cl++) { -// int[] features = ddata[cl]; -// // activation -// Arrays.fill(sums[cl], 0.0); -// int numClasses = labelIndices[cl].size(); -// for (int c = 0; c < numClasses; c++) { -// for (int f = 0; f < features.length; f++) { -// sums[cl][c] += weights[features[f]][c]; -// } -// } -// } - - -// for (int cl = 0; cl < ddata.length; cl++) { - -// int[] label = new int[cl + 1]; -// //Arrays.fill(label, classIndex.indexOf("O")); -// Arrays.fill(label, classIndex.indexOf(backgroundSymbol)); -// int index1 = label.length - 1; -// for (int pos = e; pos >= 0 && index1 >= 0; pos--) { -// //System.err.println(index1+" "+pos); -// label[index1--] = llabels[pos]; -// } -// CRFLabel crfLabel = new CRFLabel(label); -// int labelIndex = labelIndices[cl].indexOf(crfLabel); - -// double total = SloppyMath.logSum(sums[cl]); -// // int[] features = ddata[cl]; -// int numClasses = labelIndices[cl].size(); -// for (int c = 0; c < numClasses; c++) { -// probs[cl][c] = Math.exp(sums[cl][c] - total); -// } -// // for (int f=0; f String toString(Index classIndex) { - StringBuilder sb = new StringBuilder("{\n"); - for (int i = 0; i < table.length; i++) { - sb.append(toString(toArray(i), classIndex)); - sb.append(": "); - sb.append(getValue(i)); - sb.append("\n"); - } - sb.append("}"); - return sb.toString(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("{\n"); - for (int i = 0; i < table.length; i++) { - sb.append(Arrays.toString(toArray(i))); - sb.append(": "); - sb.append(getValue(i)); - sb.append("\n"); - } - sb.append("}"); - return sb.toString(); - } - - private static String toString(int[] array, Index classIndex) { - List l = new ArrayList(); - for (int i = 0; i < array.length; i++) { - l.add(classIndex.get(array[i])); - } - return l.toString(); - } - - private int[] toArray(int index) { - int[] indices = new int[windowSize]; - for (int i = indices.length - 1; i >= 0; i--) { - indices[i] = index % numClasses; - index /= numClasses; - } - return indices; - } - - private int indexOf(int[] entry) { - int index = 0; - for (int i = 0; i < entry.length; i++) { - index *= numClasses; - index += entry[i]; - } - if (index<0) throw new RuntimeException("index=" + index + " entry=" + Arrays.toString(entry)); - return index; - } - - private int indexOf(int[] front, int end) { - int index = 0; - for (int i = 0; i < front.length; i++) { - index *= numClasses; - index += front[i]; - } - index *= numClasses; - index += end; - return index; - } - - private int indexOf(int front, int[] end) { - int index = front; - for (int i = 0; i < end.length; i++) { - index *= numClasses; - index += end[i]; - } - return index; - } - - private int[] indicesEnd(int[] entries) { - int[] indices = new int[SloppyMath.intPow(numClasses, windowSize - entries.length)]; - int offset = SloppyMath.intPow(numClasses, entries.length); - int index = 0; - for (int i = 0; i < entries.length; i++) { - index *= numClasses; - index += entries[i]; - } - for (int i = 0; i < indices.length; i++) { - indices[i] = index; - index += offset; - } - return indices; - } - - private int[] indicesFront(int[] entries) { - int[] indices = new int[SloppyMath.intPow(numClasses, windowSize - entries.length)]; - int offset = SloppyMath.intPow(numClasses, windowSize - entries.length); - int start = 0; - for (int i = 0; i < entries.length; i++) { - start *= numClasses; - start += entries[i]; - } - start *= offset; - int end = 0; - for (int i = 0; i < entries.length; i++) { - end *= numClasses; - end += entries[i]; - if (i == entries.length - 1) { - end += 1; - } - } - end *= offset; - for (int i = start; i < end; i++) { - indices[i - start] = i; - } - return indices; - } - - public int windowSize() { - return windowSize; - } - - public int numClasses() { - return numClasses; - } - - private int size() { - return table.length; - } - - public double totalMass() { - return ArrayMath.logSum(table); - } - - public double unnormalizedLogProb(int[] label) { - return getValue(label); - } - - public double logProb(int[] label) { - return unnormalizedLogProb(label) - totalMass(); - } - - - public double prob(int[] label) { - return Math.pow(Math.E, unnormalizedLogProb(label) - totalMass()); - } - - /** - * Computes the probability of the tag OF being at the end of the table - * given that the previous tag sequence in table is GIVEN. - * given is at the begining, of is at the end - * @return the probability of the tag OF being at the end of the table - */ - public double conditionalLogProbGivenPrevious(int[] given, int of) { - if (given.length != windowSize - 1) { - System.err.println("error computing conditional log prob"); - System.exit(0); - } - int[] label = indicesFront(given); - double[] masses = new double[label.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[label[i]]; - } - double z = ArrayMath.logSum(masses); - - int i = indexOf(given, of); - -// if (SloppyMath.isDangerous(z) || SloppyMath.isDangerous(table[i])) { -// System.err.println("z="+z); -// System.err.println("t="+table[i]); -// } - - return table[i] - z; - } - - /** - * Computes the probabilities of the tag at the end of the table - * given that the previous tag sequence in table is GIVEN. - * given is at the begining, position in question is at the end - * @return the probabilities of the tag at the end of the table - */ - public double[] conditionalLogProbsGivenPrevious(int[] given) { - if (given.length != windowSize - 1) { - System.err.println("error computing conditional log prob"); - System.exit(0); - } - double[] result = new double[numClasses]; - for (int i = 0; i < numClasses; i++) { - int index = indexOf(given, i); - result[i] = table[index]; - } - ArrayMath.logNormalize(result); - return result; - } - - /** - * Computes the probability of the sequence OF being at the end of the table - * given that the first tag in table is GIVEN. - * given is at the begining, of is at the end - * @return the probability of the sequence of being at the end of the table - */ - public double conditionalLogProbGivenFirst(int given, int[] of) { - if (of.length != windowSize - 1) { - System.err.println("error computing conditional log prob"); - System.exit(0); - } - // compute P(given, of) - int[] labels = new int[windowSize]; - labels[0] = given; - System.arraycopy(of, 0, labels, 1, windowSize-1); - //double probAll = logProb(labels); - double probAll = unnormalizedLogProb(labels); - - // compute P(given) - //double probGiven = logProbFront(given); - double probGiven = unnormalizedLogProbFront(given); - - // compute P(given, of) / P(given) - return probAll - probGiven; - } - - /** - * Computes the probability of the sequence OF being at the end of the table - * given that the first tag in table is GIVEN. - * given is at the begining, of is at the end - * @return the probability of the sequence of being at the end of the table - */ - public double unnormalizedConditionalLogProbGivenFirst(int given, int[] of) { - if (of.length != windowSize - 1) { - System.err.println("error computing conditional log prob"); - System.exit(0); - } - // compute P(given, of) - int[] labels = new int[windowSize]; - labels[0] = given; - System.arraycopy(of, 0, labels, 1, windowSize-1); - //double probAll = logProb(labels); - double probAll = unnormalizedLogProb(labels); - - // compute P(given) - //double probGiven = logProbFront(given); - //double probGiven = unnormalizedLogProbFront(given); - - // compute P(given, of) / P(given) - //return probAll - probGiven; - return probAll; - } - - /** - * Computes the probability of the tag OF being at the beginning of the table - * given that the tag sequence GIVEN is at the end of the table. - * given is at the end, of is at the begining - * @return the probability of the tag of being at the beginning of the table - */ - public double conditionalLogProbGivenNext(int[] given, int of) { - if (given.length != windowSize - 1) { - System.err.println("error computing conditional log prob"); - System.exit(0); - } - int[] label = indicesEnd(given); - double[] masses = new double[label.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[label[i]]; - } - double z = ArrayMath.logSum(masses); - - return table[indexOf(of, given)] - z; - } - - public double unnormalizedLogProbFront(int[] labels) { - labels = indicesFront(labels); - double[] masses = new double[labels.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[labels[i]]; - } - return ArrayMath.logSum(masses); - } - - public double logProbFront(int[] label) { - return unnormalizedLogProbFront(label) - totalMass(); - } - - public double unnormalizedLogProbFront(int label) { - int[] labels = {label}; - return unnormalizedLogProbFront(labels); - } - - public double logProbFront(int label) { - return unnormalizedLogProbFront(label) - totalMass(); - } - - public double unnormalizedLogProbEnd(int[] labels) { - labels = indicesEnd(labels); - double[] masses = new double[labels.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[labels[i]]; - } - return ArrayMath.logSum(masses); - } - - public double logProbEnd(int[] labels) { - return unnormalizedLogProbEnd(labels) - totalMass(); - } - - public double unnormalizedLogProbEnd(int label) { - int[] labels = {label}; - return unnormalizedLogProbEnd(labels); - } - - public double logProbEnd(int label) { - return unnormalizedLogProbEnd(label) - totalMass(); - } - - private double getValue(int index) { - return table[index]; - } - - public double getValue(int[] label) { - return table[indexOf(label)]; - } - - @SuppressWarnings("unused") - private void setValue(int index, double value) { - table[index] = value; - } - - public void setValue(int[] label, double value) { - // try{ - table[indexOf(label)] = value; - // } catch (Exception e) { -// e.printStackTrace(); -// System.err.println("Table length: " + table.length + " indexOf(label): " + indexOf(label)); -// throw new ArrayIndexOutOfBoundsException(e.toString()); -// // System.exit(1); -// } - } - - public void incrementValue(int[] label, double value) { - table[indexOf(label)] += value; - } - - private void logIncrementValue(int index, double value) { - table[index] = SloppyMath.logAdd(table[index], value); - } - - public void logIncrementValue(int[] label, double value) { - int index = indexOf(label); - table[index] = SloppyMath.logAdd(table[index], value); - } - - public void multiplyInFront(FactorTable other) { - int divisor = SloppyMath.intPow(numClasses, windowSize - other.windowSize()); - for (int i = 0; i < table.length; i++) { - table[i] += other.getValue(i / divisor); - } - } - - public void multiplyInEnd(FactorTable other) { - int divisor = SloppyMath.intPow(numClasses, other.windowSize()); - for (int i = 0; i < table.length; i++) { - table[i] += other.getValue(i % divisor); - } - } - - public FactorTable sumOutEnd() { - FactorTable ft = new FactorTable(numClasses, windowSize - 1); - for (int i = 0; i < table.length; i++) { - ft.logIncrementValue(i / numClasses, table[i]); - } - return ft; - } - - public FactorTable sumOutFront() { - FactorTable ft = new FactorTable(numClasses, windowSize - 1); - int mod = SloppyMath.intPow(numClasses, windowSize - 1); - for (int i = 0; i < table.length; i++) { - ft.logIncrementValue(i % mod, table[i]); - } - return ft; - } - - public void divideBy(FactorTable other) { - for (int i = 0; i < table.length; i++) { - if (table[i] != Double.NEGATIVE_INFINITY || other.table[i] != Double.NEGATIVE_INFINITY) { - table[i] -= other.table[i]; - } - } - } - - public static void main(String[] args) { - FactorTable ft = new FactorTable(6, 3); - - /** - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - for (int k = 0; k < 2; k++) { - int[] a = new int[]{i, j, k}; - System.out.print(ft.toString(a)+": "+ft.indexOf(a)); - } - } - } - for (int i = 0; i < 2; i++) { - int[] b = new int[]{i}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesFront(b))); - } - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int[] b = new int[]{i, j}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesFront(b))); - } - } - for (int i = 0; i < 2; i++) { - int[] b = new int[]{i}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesBack(b))); - } for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int[] b = new int[]{i, j}; - ft2.setValue(b, (i*2)+j); - } - } - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int[] b = new int[]{i, j}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesBack(b))); - } - } - - System.out.println("##########################################"); - - **/ - - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 6; j++) { - for (int k = 0; k < 6; k++) { - int[] b = new int[]{i, j, k}; - ft.setValue(b, (i * 4) + (j * 2) + k); - } - } - } - - //System.out.println(ft); - //System.out.println(ft.sumOutFront()); - - FactorTable ft2 = new FactorTable(6, 2); - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 6; j++) { - int[] b = new int[]{i, j}; - ft2.setValue(b, i * 6 + j); - } - } - - System.out.println(ft); - //FactorTable ft3 = ft2.sumOutFront(); - //System.out.println(ft3); - - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 6; j++) { - int[] b = new int[]{i, j}; - double t = 0; - for (int k = 0; k < 6; k++) { - t += Math.pow(Math.E, ft.conditionalLogProbGivenPrevious(b, k)); - System.err.println(k + "|" + i + "," + j + " : " + Math.pow(Math.E, ft.conditionalLogProbGivenPrevious(b, k))); - } - System.out.println(t); - } - } - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/FloatFactorTable.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/FloatFactorTable.java deleted file mode 100644 index a515e4a..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/FloatFactorTable.java +++ /dev/null @@ -1,386 +0,0 @@ -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.math.ArrayMath; -import edu.stanford.nlp.math.SloppyMath; -import edu.stanford.nlp.util.Index; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - - -/** Stores a factor table as a one dimensional array of floats. - * - * @author Jenny Finkel - */ - -public class FloatFactorTable { - - private int numClasses; - private int windowSize; - - private float[] table; - - public FloatFactorTable(int numClasses, int windowSize) { - this.numClasses = numClasses; - this.windowSize = windowSize; - - table = new float[SloppyMath.intPow(numClasses, windowSize)]; - Arrays.fill(table, Float.NEGATIVE_INFINITY); - } - - public boolean containsNaN() { - for (int i = 0; i < table.length; i++) { - if (Float.isNaN(table[i])) { - return true; - } - } - return false; - } - - - public String toProbString() { - StringBuilder sb = new StringBuilder("{\n"); - for (int i = 0; i < table.length; i++) { - sb.append(Arrays.toString(toArray(i))); - sb.append(": "); - sb.append(prob(toArray(i))); - sb.append("\n"); - } - sb.append("}"); - return sb.toString(); - } - - public String toString(Index classIndex) { - StringBuilder sb = new StringBuilder("{\n"); - for (int i = 0; i < table.length; i++) { - sb.append(toString(toArray(i), classIndex)); - sb.append(": "); - sb.append(getValue(i)); - sb.append("\n"); - } - sb.append("}"); - return sb.toString(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("{\n"); - for (int i = 0; i < table.length; i++) { - sb.append(Arrays.toString(toArray(i))); - sb.append(": "); - sb.append(getValue(i)); - sb.append("\n"); - } - sb.append("}"); - return sb.toString(); - } - - private String toString(int[] array, Index classIndex) { - List l = new ArrayList(); - for (int i = 0; i < array.length; i++) { - l.add(classIndex.get(array[i])); - } - return l.toString(); - } - - private int[] toArray(int index) { - int[] indices = new int[windowSize]; - for (int i = indices.length - 1; i >= 0; i--) { - indices[i] = index % numClasses; - index /= numClasses; - } - return indices; - } - - private int indexOf(int[] entry) { - int index = 0; - for (int i = 0; i < entry.length; i++) { - index *= numClasses; - index += entry[i]; - } - return index; - } - - private int indexOf(int[] front, int end) { - int index = 0; - for (int i = 0; i < front.length; i++) { - index *= numClasses; - index += front[i]; - } - index *= numClasses; - index += end; - return index; - } - - private int[] indicesEnd(int[] entries) { - int[] indices = new int[SloppyMath.intPow(numClasses, windowSize - entries.length)]; - int offset = SloppyMath.intPow(numClasses, entries.length); - int index = 0; - for (int i = 0; i < entries.length; i++) { - index *= numClasses; - index += entries[i]; - } - for (int i = 0; i < indices.length; i++) { - indices[i] = index; - index += offset; - } - return indices; - } - - private int[] indicesFront(int[] entries) { - int[] indices = new int[SloppyMath.intPow(numClasses, windowSize - entries.length)]; - int offset = SloppyMath.intPow(numClasses, windowSize - entries.length); - int start = 0; - for (int i = 0; i < entries.length; i++) { - start *= numClasses; - start += entries[i]; - } - start *= offset; - int end = 0; - for (int i = 0; i < entries.length; i++) { - end *= numClasses; - end += entries[i]; - if (i == entries.length - 1) { - end += 1; - } - } - end *= offset; - for (int i = start; i < end; i++) { - indices[i - start] = i; - } - return indices; - } - - public int windowSize() { - return windowSize; - } - - public int numClasses() { - return numClasses; - } - - private int size() { - return table.length; - } - - public float totalMass() { - return ArrayMath.logSum(table); - } - - public float unnormalizedLogProb(int[] label) { - return getValue(label); - } - - public float logProb(int[] label) { - return unnormalizedLogProb(label) - totalMass(); - } - - - public float prob(int[] label) { - return (float) Math.exp(unnormalizedLogProb(label) - totalMass()); - } - - // given is at the begining, of is at the end - public float conditionalLogProb(int[] given, int of) { - if (given.length != windowSize - 1) { - System.err.println("error computing conditional log prob"); - System.exit(0); - } - int[] label = indicesFront(given); - float[] masses = new float[label.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[label[i]]; - } - float z = ArrayMath.logSum(masses); - - return table[indexOf(given, of)] - z; - } - - public float unnormalizedLogProbFront(int[] label) { - label = indicesFront(label); - float[] masses = new float[label.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[label[i]]; - } - return ArrayMath.logSum(masses); - } - - public float logProbFront(int[] label) { - return unnormalizedLogProbFront(label) - totalMass(); - } - - public float unnormalizedLogProbEnd(int[] label) { - label = indicesEnd(label); - float[] masses = new float[label.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[label[i]]; - } - return ArrayMath.logSum(masses); - } - - public float logProbEnd(int[] label) { - return unnormalizedLogProbEnd(label) - totalMass(); - } - - public float unnormalizedLogProbEnd(int label) { - int[] l = {label}; - l = indicesEnd(l); - float[] masses = new float[l.length]; - for (int i = 0; i < masses.length; i++) { - masses[i] = table[l[i]]; - } - return ArrayMath.logSum(masses); - } - - public float logProbEnd(int label) { - return unnormalizedLogProbEnd(label) - totalMass(); - } - - private float getValue(int index) { - return table[index]; - } - - public float getValue(int[] label) { - return table[indexOf(label)]; - } - - private void setValue(int index, float value) { - table[index] = value; - } - - public void setValue(int[] label, float value) { - table[indexOf(label)] = value; - } - - public void incrementValue(int[] label, float value) { - table[indexOf(label)] += value; - } - - private void logIncrementValue(int index, float value) { - table[index] = SloppyMath.logAdd(table[index], value); - } - - public void logIncrementValue(int[] label, float value) { - int index = indexOf(label); - table[index] = SloppyMath.logAdd(table[index], value); - } - - public void multiplyInFront(FloatFactorTable other) { - int divisor = SloppyMath.intPow(numClasses, windowSize - other.windowSize()); - for (int i = 0; i < table.length; i++) { - table[i] += other.getValue(i / divisor); - } - } - - public void multiplyInEnd(FloatFactorTable other) { - int divisor = SloppyMath.intPow(numClasses, other.windowSize()); - for (int i = 0; i < table.length; i++) { - table[i] += other.getValue(i % divisor); - } - } - - public FloatFactorTable sumOutEnd() { - FloatFactorTable ft = new FloatFactorTable(numClasses, windowSize - 1); - for (int i = 0; i < table.length; i++) { - ft.logIncrementValue(i / numClasses, table[i]); - } - return ft; - } - - public FloatFactorTable sumOutFront() { - FloatFactorTable ft = new FloatFactorTable(numClasses, windowSize - 1); - int mod = SloppyMath.intPow(numClasses, windowSize - 1); - for (int i = 0; i < table.length; i++) { - ft.logIncrementValue(i % mod, table[i]); - } - return ft; - } - - public void divideBy(FloatFactorTable other) { - for (int i = 0; i < table.length; i++) { - if (table[i] != Float.NEGATIVE_INFINITY || other.table[i] != Float.NEGATIVE_INFINITY) { - table[i] -= other.table[i]; - } - } - } - - public static void main(String[] args) { - FloatFactorTable ft = new FloatFactorTable(6, 3); - - /** - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - for (int k = 0; k < 2; k++) { - int[] a = new int[]{i, j, k}; - System.out.print(ft.toString(a)+": "+ft.indexOf(a)); - } - } - } - for (int i = 0; i < 2; i++) { - int[] b = new int[]{i}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesFront(b))); - } - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int[] b = new int[]{i, j}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesFront(b))); - } - } - for (int i = 0; i < 2; i++) { - int[] b = new int[]{i}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesBack(b))); - } for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int[] b = new int[]{i, j}; - ft2.setValue(b, (i*2)+j); - } - } - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int[] b = new int[]{i, j}; - System.out.print(ft.toString(b)+": "+ft.toString(ft.indicesBack(b))); - } - } - - System.out.println("##########################################"); - - **/ - - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 6; j++) { - for (int k = 0; k < 6; k++) { - int[] b = new int[]{i, j, k}; - ft.setValue(b, (i * 4) + (j * 2) + k); - } - } - } - - //System.out.println(ft); - //System.out.println(ft.sumOutFront()); - - FloatFactorTable ft2 = new FloatFactorTable(6, 2); - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 6; j++) { - int[] b = new int[]{i, j}; - ft2.setValue(b, i * 6 + j); - } - } - - System.out.println(ft); - //FloatFactorTable ft3 = ft2.sumOutFront(); - //System.out.println(ft3); - - for (int i = 0; i < 6; i++) { - for (int j = 0; j < 6; j++) { - int[] b = new int[]{i, j}; - float t = 0; - for (int k = 0; k < 6; k++) { - t += Math.exp(ft.conditionalLogProb(b, k)); - System.err.println(k + "|" + i + "," + j + " : " + Math.exp(ft.conditionalLogProb(b, k))); - } - System.out.println(t); - } - } - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/NERGUI.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/NERGUI.java deleted file mode 100644 index 28281ec..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/NERGUI.java +++ /dev/null @@ -1,773 +0,0 @@ -// NERGUI -- a GUI for a probabilistic (CRF) sequence model for NER. -// Copyright (c) 2002-2008 The Board of Trustees of -// The Leland Stanford Junior University. All Rights Reserved. -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -// -// For more information, bug reports, fixes, contact: -// Christopher Manning -// Dept of Computer Science, Gates 1A -// Stanford CA 94305-9010 -// USA -// Support/Questions: java-nlp-user@lists.stanford.edu -// Licensing: java-nlp-support@lists.stanford.edu -// http://nlp.stanford.edu/downloads/crf-classifier.shtml - -package edu.stanford.nlp.ie.crf; - -import edu.stanford.nlp.ie.AbstractSequenceClassifier; -import edu.stanford.nlp.util.StringUtils; - -import javax.swing.*; -import javax.swing.text.*; -import java.awt.*; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.KeyEvent; -import java.awt.event.KeyListener; -import java.io.File; -import java.util.HashMap; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** A GUI for Named Entity sequence classifiers. - * This version only supports the CRF. - * - * @author Jenny Finkel - * @author Christopher Manning - */ -public class NERGUI { - - private AbstractSequenceClassifier classifier; - - private JFrame frame; - private JEditorPane editorPane; - private JToolBar tagPanel; - private static int HEIGHT = 600; - private static int WIDTH = 650; - private HashMap tagToColorMap; - private JFileChooser fileChooser = new JFileChooser(); - private MutableAttributeSet defaultAttrSet = new SimpleAttributeSet(); - private ActionListener actor = new ActionPerformer(); - private File loadedFile; - private String taggedContents = null; - private String htmlContents = null; - - private JMenuItem saveUntagged = null; - private JMenuItem saveTaggedAs = null; - - private JButton extractButton = null; - private JMenuItem extract = null; - - private void createAndShowGUI() { - //Make sure we have nice window decorations. - JFrame.setDefaultLookAndFeelDecorated(true); - - //Create and set up the window. - frame = new JFrame("Stanford Named Entity Recognizer"); - frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); - frame.getContentPane().setLayout(new BorderLayout()); - frame.getContentPane().setPreferredSize(new Dimension(WIDTH, HEIGHT)); - - frame.setJMenuBar(addMenuBar()); - - buildTagPanel(); - buildContentPanel(); - buildExtractButton(); - extractButton.setEnabled(false); - extract.setEnabled(false); - - //Display the window. - frame.pack(); - frame.setVisible(true); - } - - private JMenuBar addMenuBar() { - JMenuBar menubar = new JMenuBar(); - - JMenu fileMenu = new JMenu("File"); - menubar.add(fileMenu); - - JMenu editMenu = new JMenu("Edit"); - menubar.add(editMenu); - - JMenu classifierMenu = new JMenu("Classifier"); - menubar.add(classifierMenu); - - /** - * FILE MENU - */ - - JMenuItem openFile = new JMenuItem("Open File"); - openFile.setMnemonic('O'); - openFile.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_F, java.awt.Event.CTRL_MASK)); - openFile.addActionListener(actor); - fileMenu.add(openFile); - - JMenuItem loadURL = new JMenuItem("Load URL"); - loadURL.setMnemonic('L'); - loadURL.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_U, java.awt.Event.CTRL_MASK)); - loadURL.addActionListener(actor); - fileMenu.add(loadURL); - - fileMenu.add(new JSeparator()); - - saveUntagged = new JMenuItem("Save Untagged File"); - saveUntagged.setMnemonic('S'); - saveUntagged.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_S, java.awt.Event.CTRL_MASK)); - saveUntagged.addActionListener(actor); - saveUntagged.setEnabled(false); - fileMenu.add(saveUntagged); - - JMenuItem saveUntaggedAs = new JMenuItem("Save Untagged File As ..."); - saveUntaggedAs.setMnemonic('U'); - saveUntaggedAs.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_U, java.awt.Event.CTRL_MASK)); - saveUntaggedAs.addActionListener(actor); - fileMenu.add(saveUntaggedAs); - - saveTaggedAs = new JMenuItem("Save Tagged File As ..."); - saveTaggedAs.setMnemonic('T'); - saveTaggedAs.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_T, java.awt.Event.CTRL_MASK)); - saveTaggedAs.addActionListener(actor); - saveTaggedAs.setEnabled(false); - fileMenu.add(saveTaggedAs); - - fileMenu.add(new JSeparator()); - - JMenuItem exit = new JMenuItem("Exit"); - exit.setMnemonic('x'); - exit.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_Q, java.awt.Event.CTRL_MASK)); - exit.addActionListener(actor); - fileMenu.add(exit); - - - /** - * EDIT MENU - */ - - JMenuItem cut = new JMenuItem("Cut"); - cut.setMnemonic('X'); - cut.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_X, java.awt.Event.CTRL_MASK)); - cut.addActionListener(actor); - editMenu.add(cut); - - JMenuItem copy = new JMenuItem("Copy"); - copy.setMnemonic('C'); - copy.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_C, java.awt.Event.CTRL_MASK)); - copy.addActionListener(actor); - editMenu.add(copy); - - JMenuItem paste = new JMenuItem("Paste"); - paste.setMnemonic('V'); - paste.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_V, java.awt.Event.CTRL_MASK)); - paste.addActionListener(actor); - editMenu.add(paste); - - JMenuItem clear = new JMenuItem("Clear"); - clear.setMnemonic('C'); - clear.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_L, java.awt.Event.CTRL_MASK)); - clear.addActionListener(actor); - editMenu.add(clear); - - - /** - * CLASSIFIER MENU - */ - - JMenuItem loadCRF = new JMenuItem("Load CRF From File"); - loadCRF.setMnemonic('R'); - loadCRF.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_R, java.awt.Event.CTRL_MASK)); - loadCRF.addActionListener(actor); - classifierMenu.add(loadCRF); - - JMenuItem loadDefaultCRF = new JMenuItem("Load Default CRF"); - loadDefaultCRF.setMnemonic('L'); - loadDefaultCRF.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_L, java.awt.Event.CTRL_MASK)); - loadDefaultCRF.addActionListener(actor); - classifierMenu.add(loadDefaultCRF); - - extract = new JMenuItem("Run NER"); - extract.setMnemonic('N'); - extract.setAccelerator(KeyStroke.getKeyStroke(java.awt.event.KeyEvent.VK_N, java.awt.Event.CTRL_MASK)); - extract.addActionListener(actor); - classifierMenu.add(extract); - - return menubar; - } - - - private class InputListener implements KeyListener { - public void keyPressed(KeyEvent e) { - - } - - public void keyReleased(KeyEvent e) { - - } - - public void keyTyped(KeyEvent e) { - saveTaggedAs.setEnabled(false); - } - - } - - - private class ActionPerformer implements ActionListener { - - public void actionPerformed(ActionEvent e) { - String com = e.getActionCommand(); - - if (com.equals("Open File")) { - File file = getFile(true); - if (file != null) { - openFile(file); - } - } else if (com.equals("Load URL")) { - String url = getURL(); - if (url != null) { - openURL(url); - } - } else if (com.equals("Exit")) { - exit(); - } else if (com.equals("Clear")) { - clearDocument(); - } else if (com.equals("Cut")) { - cutDocument(); - } else if (com.equals("Copy")) { - copyDocument(); - } else if (com.equals("Paste")) { - pasteDocument(); - } else if (com.equals("Load CRF From File")) { - File file = getFile(true); - if (file != null) { - loadClassifier(file); - } - } else if (com.equals("Load Default CRF")) { - loadClassifier(null); - } else if (com.equals("Run NER")) { - extract(); - } else if (com.equals("Save Untagged File")) { - saveUntaggedContents(loadedFile); - } else if (com.equals("Save Untagged File As ...")) { - saveUntaggedContents(getFile(false)); - } else if (com.equals("Save Tagged File As ...")) { - File f = getFile(false); - if (f != null) { - // i.e., they didn't cancel out of the file dialog - saveFile (f, taggedContents); - } - } else { - System.err.println("Unknown Action: "+e); - } - } - } - - public File getFile(boolean open) { - File file = null; - int returnVal; - if (open) { - returnVal = fileChooser.showOpenDialog(frame); - } else { - returnVal = fileChooser.showSaveDialog(frame); - } - if(returnVal == JFileChooser.APPROVE_OPTION) { - file = fileChooser.getSelectedFile(); - if (open && !checkFile(file)) { file = null; } - } - return file; - } - - public void saveUntaggedContents(File file) { - try { - String contents; - if (editorPane.getContentType().equals("text/html")) { - contents = editorPane.getText(); - } else { - Document doc = editorPane.getDocument(); - contents = doc.getText(0, doc.getLength()); - } - saveFile(file, contents); - saveUntagged.setEnabled(true); - loadedFile = file; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static void saveFile(File file, String contents) { - StringUtils.printToFile(file, contents); - } - - public String getURL() { - String url = JOptionPane.showInputDialog(frame, "URL: ", "Load URL", JOptionPane.QUESTION_MESSAGE); - return url; - } - - public boolean checkFile(File file) { - if (file.isFile()) { - fileChooser.setCurrentDirectory(file.getParentFile()); - return true; - } else { - String message = "File Not Found: "+file.getAbsolutePath(); - displayError("File Not Found Error", message); - return false; - } - } - - public void displayError(String title, String message) { - JOptionPane.showMessageDialog(frame, message, title, JOptionPane.ERROR_MESSAGE); - } - - /** Load a classifier from a file or the default. - * The default is specified by passing in null. - */ - public void loadClassifier(File file) { - try { - if (file != null) { - classifier = CRFClassifier.getClassifier(file); - } else { - // default classifier in jar - classifier = CRFClassifier.getDefaultClassifier(); - } - } catch (Throwable e) { - // we catch Throwable, since we'd also like to be able to get an - // OutOfMemoryError - String message; - if (file != null) { - message = "Error loading CRF: " + file.getAbsolutePath(); - } else { - message = "Error loading default CRF"; - } - System.err.println(message); - String title = "CRF Load Error"; - String msg = e.toString(); - if (msg != null) { - message += "\n" + msg; - } - displayError(title, message); - return; - } - removeTags(); - buildTagPanel(); - // buildExtractButton(); - extractButton.setEnabled(true); - extract.setEnabled(true); - } - - public void openFile(File file) { - openURL(file.toURI().toString()); - loadedFile = file; - saveUntagged.setEnabled(true); - } - - public void openURL(String url) { - try { - editorPane.setPage(url); - } catch (Exception e) { - System.err.println("Error loading |" + url + "|"); - e.printStackTrace(); - displayError("Error Loading URL " + url, "Message: " + e.toString()); - return; - } - loadedFile = null; - String text = editorPane.getText(); - taggedContents = null; - if ( ! editorPane.getContentType().equals("text/html")) { - editorPane.setContentType("text/rtf"); - Document doc = editorPane.getDocument(); - try { - doc.insertString(0, text, defaultAttrSet); - } catch (Exception e) { - throw new RuntimeException(e); - } - editorPane.revalidate(); - editorPane.repaint(); - editorPane.setEditable(true); - htmlContents = null; - } else { - editorPane.setEditable(false); - htmlContents = editorPane.getText(); - } - - saveUntagged.setEnabled(false); - saveTaggedAs.setEnabled(false); - } - - private void removeTags() { - if (editorPane.getContentType().equals("text/html")) { - if (htmlContents != null) { - editorPane.setText(htmlContents); - } - editorPane.revalidate(); - editorPane.repaint(); - } else { - DefaultStyledDocument doc = (DefaultStyledDocument)editorPane.getDocument(); - SimpleAttributeSet attr = new SimpleAttributeSet(); - StyleConstants.setForeground(attr, Color.BLACK); - StyleConstants.setBackground(attr, Color.WHITE); - doc.setCharacterAttributes(0, doc.getLength(), attr, false); - } - saveTaggedAs.setEnabled(false); - } - - - private void extract() { - System.err.println("content type: "+editorPane.getContentType()); - if ( ! editorPane.getContentType().equals("text/html")) { - - DefaultStyledDocument doc = (DefaultStyledDocument)editorPane.getDocument(); - String text = null; - try { - text = doc.getText(0, doc.getLength()); - } catch (Exception e) { - e.printStackTrace(); - } - String labeledText = classifier.classifyWithInlineXML(text); - taggedContents = labeledText; - - Set tags = classifier.labels(); - String background = classifier.backgroundSymbol(); - String tagPattern = ""; - for (String tag : tags) { - if (tag.equals(background)) { continue; } - if (tagPattern.length() > 0) { tagPattern += "|"; } - tagPattern += tag; - } - - Pattern startPattern = Pattern.compile("<("+tagPattern+")>"); - Pattern endPattern = Pattern.compile(""); - - String finalText = labeledText; - - Matcher m = startPattern.matcher(finalText); - while (m.find()) { - int start = m.start(); - finalText = m.replaceFirst(""); - m = endPattern.matcher(finalText); - if (m.find()) { - int end = m.start(); - String tag = m.group(1); - finalText = m.replaceFirst(""); - AttributeSet attSet = getAttributeSet(tag); - try { - String entity = finalText.substring(start, end); - doc.setCharacterAttributes(start, entity.length(), attSet, false); - } catch (Exception ex) { - ex.printStackTrace(); - System.exit(-1); - } - System.err.println(tag+": "+ finalText.substring(start, end)); - } else { - // print error message - } - m = startPattern.matcher(finalText); - } - editorPane.revalidate(); - editorPane.repaint(); - } else { - String untaggedContents = editorPane.getText(); - if (untaggedContents == null) { untaggedContents = ""; } - taggedContents = classifier.classifyWithInlineXML(untaggedContents); - - Set tags = classifier.labels(); - String background = classifier.backgroundSymbol(); - String tagPattern = ""; - for (String tag : tags) { - if (tag.equals(background)) { continue; } - if (tagPattern.length() > 0) { tagPattern += "|"; } - tagPattern += tag; - } - - Pattern startPattern = Pattern.compile("<("+tagPattern+")>"); - Pattern endPattern = Pattern.compile(""); - - String finalText = taggedContents; - - Matcher m = startPattern.matcher(finalText); - while (m.find()) { - String tag = m.group(1); - String color = colorToHTML(tagToColorMap.get(tag)); - String newTag = ""; - finalText = m.replaceFirst(newTag); - int start = m.start()+newTag.length(); - Matcher m1 = endPattern.matcher(finalText); - m1.find(m.end()); - String entity = finalText.substring(start, m1.start()); - System.err.println(tag+": "+ entity); - finalText = m1.replaceFirst(""); - m = startPattern.matcher(finalText); - } - // System.out.println(finalText); - editorPane.setText(finalText); - editorPane.revalidate(); - editorPane.repaint(); - - // System.err.println(finalText); - } - saveTaggedAs.setEnabled(true); - } - - private AttributeSet getAttributeSet(String tag) { - MutableAttributeSet attr = new SimpleAttributeSet(); - Color color = tagToColorMap.get(tag); - StyleConstants.setBackground(attr, color); - StyleConstants.setForeground(attr, Color.WHITE); - return attr; - } - - public void clearDocument() { - editorPane.setContentType("text/rtf"); - Document doc = new DefaultStyledDocument(); - editorPane.setDocument(doc); - // defaultAttrSet = ((StyledEditorKit)editorPane.getEditorKit()).getInputAttributes(); - // StyleConstants.setFontFamily(defaultAttrSet, "Lucinda Sans Unicode"); - System.err.println("attr: "+defaultAttrSet); - - try { - doc.insertString(0, " ", defaultAttrSet); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - - editorPane.setEditable(true); - editorPane.revalidate(); - editorPane.repaint(); - - saveUntagged.setEnabled(false); - saveTaggedAs.setEnabled(false); - - taggedContents = null; - htmlContents = null; - loadedFile = null; - - } - - public void cutDocument() { - editorPane.cut(); - saveTaggedAs.setEnabled(false); - } - - public void copyDocument() { - editorPane.copy(); - } - - public void pasteDocument() { - editorPane.paste(); - saveTaggedAs.setEnabled(false); - } - - - - public void exit() { - // ask if they're sure? - System.exit(-1); - } - - private String initText = "In bringing his distinct vision to the Western genre, writer-director Jim Jarmusch has created a quasi-mystical avant-garde drama that remains a deeply spiritual viewing experience. After losing his parents and fianc\u00E9e, a Cleveland accountant named William Blake (a remarkable Johnny Depp) spends all his money and takes a train to the frontier town of Machine in order to work at a factory. Upon arriving in Machine, he is denied his expected job and finds himself a fugitive after murdering a man in self-defense. Wounded and helpless, Blake is befriended by Nobody (Gary Farmer), a wandering Native American who considers him to be a ghostly manifestation of the famous poet. Nobody aids Blake in his flight from three bumbling bounty hunters, preparing him for his final journey--a return to the world of the spirits."; - // private String initText = "In"; - - private void buildContentPanel() { - - editorPane = new JEditorPane (); - editorPane.setContentType("text/rtf"); - editorPane.addKeyListener(new InputListener()); - - // defaultAttrSet = ((StyledEditorKit)editorPane.getEditorKit()).getInputAttributes(); - StyleConstants.setFontFamily(defaultAttrSet, "Lucida Sans"); - - Document doc = new DefaultStyledDocument(); - editorPane.setDocument(doc); - try { - doc.insertString(0, initText, defaultAttrSet); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - - JScrollPane scrollPane = new JScrollPane(editorPane); - frame.getContentPane().add(scrollPane, BorderLayout.CENTER); - - editorPane.setEditable(true); - } - - - private static String colorToHTML(Color color) { - String r = Integer.toHexString(color.getRed()); - if (r.length() == 0) { r = "00"; } - else if (r.length() == 1) { r = "0" + r; } - else if (r.length() > 2) { System.err.println("invalid hex color for red"+r); } - - String g = Integer.toHexString(color.getGreen()); - if (g.length() == 0) { g = "00"; } - else if (g.length() == 1) { g = "0" + g; } - else if (g.length() > 2) { System.err.println("invalid hex color for green"+g); } - - String b = Integer.toHexString(color.getBlue()); - if (b.length() == 0) { b = "00"; } - else if (b.length() == 1) { b = "0" + b; } - else if (b.length() > 2) { System.err.println("invalid hex color for blue"+b); } - - return "#"+r+g+b; - } - - static class ColorIcon implements Icon { - Color color; - - public ColorIcon(Color c) { - color = c; - } - - public void paintIcon(Component c, Graphics g, int x, int y) { - g.setColor(color); - g.fillRect(x, y, getIconWidth(), getIconHeight()); - } - - public int getIconWidth() { - return 10; - } - - public int getIconHeight() { - return 10; - } - } - - private void buildExtractButton() { - if (extractButton == null) { - JPanel buttonPanel = new JPanel(); - extractButton = new JButton("Run NER"); - buttonPanel.add(extractButton); - frame.getContentPane().add(buttonPanel, BorderLayout.SOUTH); - extractButton.addActionListener(actor); - } - } - - private void buildTagPanel() { - - if (tagPanel == null) { - tagPanel = new JToolBar(SwingConstants.VERTICAL); - tagPanel.setFloatable(false); - frame.getContentPane().add(tagPanel, BorderLayout.EAST); - } else { - tagPanel.removeAll(); - } - - if (classifier != null) { - - makeTagMaps(); - - Set tags = classifier.labels(); - String backgroundSymbol = classifier.backgroundSymbol(); - - for (String tag : tags) { - if (tag.equals(backgroundSymbol)) { continue; } - Color color = tagToColorMap.get(tag); - JButton b = new JButton(tag, new ColorIcon(color)); - tagPanel.add(b); - } - } - tagPanel.revalidate(); - tagPanel.repaint(); - } - - private void makeTagMaps() { - - Set tags = classifier.labels(); - String backgroundSymbol = classifier.backgroundSymbol(); - int numColors = tags.size() - 1; - Color[] colors = getNColors(numColors); - tagToColorMap = new HashMap(); - - int i = 0; - for (String tag : tags) { - if (tag.equals(backgroundSymbol)) { continue; } - if (tagToColorMap.get(tag) != null) { continue; } - tagToColorMap.put(tag, colors[i++]); - } - } - - - private static Color[] basicColors = new Color[]{new Color(204, 102, 0), - new Color(102, 0, 102), - new Color(204, 0, 102), - new Color(153, 0, 0), - new Color(153, 0, 204), - new Color(255, 102, 0), - new Color(255, 102, 153), - new Color(204, 152, 255), - new Color(102, 102, 255), - new Color(153, 102, 0), - new Color(51, 102, 51), - new Color(0, 102, 255)}; - -// private static Color[] basicColors = new Color[]{new Color(153, 102, 153), -// new Color(102, 153, 153), -// new Color(153, 153, 102), -// new Color(102, 102, 102), -// new Color(102, 153, 102), -// new Color(153, 102, 102), -// new Color(204, 153, 51), -// new Color(204, 51, 102), -// new Color(255, 204, 0), -// new Color(153, 0, 255), -// new Color(204, 204, 204), -// new Color(0, 255, 153)}; - -// private static Color[] basicColors = new Color[]{Color.BLUE, -// Color.GREEN, -// Color.RED, -// Color.ORANGE, -// Color.LIGHT_GRAY, -// Color.CYAN, -// Color.MAGENTA, -// Color.YELLOW, -// Color.RED, -// Color.GRAY, -// Color.PINK, -// Color.DARK_GRAY}; - - - public static Color[] getNColors(int n) { - Color[] colors = new Color[n]; - if (n <= basicColors.length) { - System.arraycopy(basicColors, 0, colors, 0, n); - } else { - int s = 255 / (int)Math.ceil(Math.pow(n, (1.0 / 3.0))); - int index = 0; - OUTER: for (int i = 0; i < 256; i += s) { - for (int j = 0; j < 256; j += s) { - for (int k = 0; k < 256; k += s) { - colors[index++] = new Color(i,j,k); - if (index == n) { break OUTER; } - } - } - } - } - return colors; - } - - /** Run the GUI. This program accepts no command-line arguments. - * Everything is entered into the GUI. - */ - public static void main(String[] args) { - //Schedule a job for the event-dispatching thread: - //creating and showing this application's GUI. - SwingUtilities.invokeLater(new Runnable() { - public void run() { - NERGUI gui = new NERGUI(); - gui.createAndShowGUI(); - } - }); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/ner-manifest.txt b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/ner-manifest.txt deleted file mode 100644 index a84faf4..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/ner-manifest.txt +++ /dev/null @@ -1,3 +0,0 @@ -Manifest-Version: 1.0 -Created-By: Stanford JavaNLP -Main-Class: edu.stanford.nlp.ie.crf.NERGUI diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/package.html b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/package.html deleted file mode 100644 index 709d676..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/crf/package.html +++ /dev/null @@ -1,22 +0,0 @@ - - - -A package for doing inference with conditional random fields. This -implements the common, standard case of a linear chain CRF of arbitrary -order (though usually first order in practice) where each position can -be labeled with one of a fixed set of classes. -

-Through the use of different -edu.stanford.nlp.ie.FeatureFactory classes and different -edu.stanford.nlp.sequences.DocumentReaderAndWriter classes, -it can read data in various formats, and be customized for various -sequence inference tasks. Most of its use has been for Named Entity -Recognition, but it has also been used for other applications such as -Chinese word segmentation and OCR. -

-For more usage information, consult the Javadoc of the -CRFClassifier class. - -@author Jenny Finkel - - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/package.html b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/package.html deleted file mode 100644 index 44c8faa..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/package.html +++ /dev/null @@ -1,224 +0,0 @@ - - - -

-This package implements various subpackages for information extraction. -Some examples of use appear later in this description. -At the moment, three types of information extraction are supported - (where some of these have internal variants): -

-
    -
  1. Regular expression based matching: These extractors are hand-written - and match whatever the regular expression matches.
  2. -
  3. Hidden Markov model based extractors: These can be either single - field extractors or two level HMMs where the individual - component models and how they are glued together is trained - separately. These models are trained automatically, but require tagged - training data.
  4. -
  5. Description extractor: This does higher level NLP analysis of - sentences (using a POS tagger and chunker) to find sentences - that describe an object. This might be a biography of a person, - or a description of an animal. This module is fixed: there is - nothing to write or train (unless one wants to start to change - its internal behavior). -
-

-There are some demonstrations of the stuff here which you can run (and several - other classes have main() methods which exhibit their - functionality): -

-
    -
  1. ExtractDemo is a simple GUI front-end to the information - extraction components. It isn't intended for serious use, but -illustrates loading and using various extractors, and includes -an interface to web search. This can be done via Google (using the - Google API) or Altavista or other engines (using screen scraping). - The web search -functionality is provided by the edu.stanford.nlp.web -package. You should try this out first (see the examples below).
  2. -
  3. edu.stanford.nlp.ie.test.ExtractorTest is a simple -command line test of the -information extraction components. It extracts only from a file.
  4. -
- -

Usage examples

-

-0. Setup: For all of these examples except 3., you need to be -connected to the Internet, and for the application's web search module -to be -able to connect to search engines. The web search -functionality is provided by the supplied edu.stanford.nlp.web -package. How web search works is controlled -by a websearch.init file in your current directory (or if - none is present, you will get search results from AltaVista). If - you are registered to use the GoogleAPI, you should probably edit - this file so web queries can be done to Google using their SOAP - interface. Even if not, you can specify additional or different - search engines to access in websearch.init. - A copy of this file is supplied in the distribution. The -DescExtractor in 4. also requires another init file so that -it can use the include part-of-speech tagger. -

-1. Corporate Contact Information. This illustrates simple information -extraction from a web page. -Using the included - ExtractDemo.bat or by hand run: -java edu.stanford.nlp.ie.ExtractDemo -

-
    -
  • Select as Extractor Directory the folder: -serialized-extractors/companycontact
  • -
  • Select as an Ontology the one in -serialized-extractors/companycontact/Corporation-Information.kaon -
  • -
  • Enter Corporation as the Concept to extract.
  • -
  • You can then do various searches: -
      -
    • You can enter a URL, click Extract, and look at the results: -
        -
      • http://www.ziatech.com/
      • -
      • http://www.cs.stanford.edu/
      • -
      • http://www.ananova.com/business/story/sm_635565.html
      • -
      -The components will work reasonably well on clean-ish text pages like - this. They work even better on text such as newswire or press -releases, as one can demonstrate either over the web or using the - command line extractor
    • -
    • You can do a search for a term and get extraction from the top -search hits, by entering a term in the "Search for words" box and - pressing "Extract": -
        -
      • Audiovox Corporation -
      -Extraction is done over a number of pages from a search engine, and the - results from each are shown. Typically some of these pages - will have suitable content to extract, and some just won't. -
    -
- -

2. Corporate Contact Information merged. This illustrates the addition -of information merger across web pages. Using the included - MergeExtractDemo.bat or similarly do:

-
java edu.stanford.nlp.ie.ExtractDemo -m
-

-The ExtractDemo screen is similar, but adds a button to - Select a Merger. -

-
    -
  • Select an Extractor Directory and Ontology as - above.
  • -
  • Click on "Select Merger" and then navigate to -serialized-extractors/mergers and Select the file -unscoredmerger.obj.
  • -
  • Enter the concept "Corporation" as before. -
  • One can now do search as above, by URL or search, but Merger is only - appropriate to a word search with multiple results. Try Search - for words: -
      -
    • Audiovox Corporation
    • -
    -and press "Extract". Results gradually appear. After all results have - been processed (this may take a few seconds), a Merged best - extracted information result will be produced and displayed as - the first of the results. "Merged Instance" will appear on the - bottom line corresponding to it, rather than a URL. -
- - -

3. Company names via direct use of an HMM information extractor. -One can also train, load, and use HMM information extractors directly, - without using any of the RDF-based KAON framework -(http://kaon.semanticweb.org/) used by ExtractDemo. -

-
    -
  • The file edu.stanford.nlp.ie.hmm.Tester illustrates the use - of a pretrained HMM on data via the command line interface: -
      -
    • cd serialized-extractors/companycontact/
    • -
    • java edu.stanford.nlp.ie.hmm.Tester cisco.txt company - company-name.hmm
    • -
    • java edu.stanford.nlp.ie.hmm.Tester EarningsReports.txt - company company-name.hmm
    • -
    • java edu.stanford.nlp.ie.hmm.Tester companytest.txt - company company-name.hmm
    • -
    -

    -The first shows the HMM running on an unmarked up file with a single - document. The second shows a Corpus of several - documents, separated with ENDOFDOC, used as a document delimiter - inside a Corpus. This second use of Tester expects to -normally have an annotated corpus on which it can score its answers. -Here, the corpus is unannotated, and so some of the output is - inappropriate, but it shows what is selected as the company name - for each document (it's mostly correct...). -The final example shows it running on a corpus that does have answers -marked in it. It does the testing with the XML elements stripped, but - then uses them to evaluate correctness. -

    -
  • -
  • To train one's own HMM, one needs data where one or - more fields is annotated in the data in the style of an XML - element, with all the documents in one file, separated by - lines with ENDOFDOC on them. Then one can - train (and then test) as follows. Training an HMM - (optimizing all its probabilities) takes a long time - (it depends on the speed of the computer, but 10 minutes or - so to adjust probabilities for a fixed structure, and often - hours if one additionally attempts structure learning). -
      -
    1. cd edu/stanford/nlp/ie/training/
    2. -
    3. java -server edu.stanford.nlp.ie.hmm.Trainer companydata.txt - company mycompany.hmm
    4. -
    5. java edu.stanford.nlp.ie.hmm.HMMSingleFieldExtractor Company - mycompany.hmm mycompany.obj
    6. -
    7. java edu.stanford.nlp.ie.hmm.Tester testdoc.txt company - mycompany.hmm
    8. -
    -The third step converts a serialized HMM into the serialized objects used - in ExtractDemo. Note that company - in the second line must match the element name in the - marked-up data that you will train on, while - Company in the third line must match the - relation name in the ontology over which you will extract with - mycompany.obj. These two names need not be the - same. The last step then runs the trained HMM on a file. -
  • -
- -

4. Extraction of descriptions (such as biographical information about - a person or a description of an animal). -This does extraction of such descriptions -from a web page. This component uses a POS tagger, and looks for where - to find a path to it in the file - descextractor.init in the current directory. So, - you should be in the root directory of the current archive, - which has such a file. Double click on the included - MergeExtractDemo.bat in that directory, or by hand - one can equivalently do: -java edu.stanford.nlp.ie.ExtractDemo -m -

-
    -
  • Select as Extractor Directory the folder: -serialized-extractors/description
  • -
  • Select as an Ontology the one in -serialized-extractors/description/Entity-NameDescription.kaon -
  • -
  • Click on "Select Merger" and then navigate to -serialized-extractors/mergers and Select the file -unscoredmerger.obj.
  • -
  • Enter Entity as the Concept to extract. -
  • You can then do various searches for people or animals by entering - words in the "Search for words" box and pressing Extract: -
      -
    • Gareth Evans
    • -
    • Tawny Frogmouth
    • -
    • Christopher Manning
    • -
    • Joshua Nkomo
    • -
    -The first search will be slower than subsequent searches, as it takes a - while to load the part of speech tagger. -
  • -
- - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/AcronymModel.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/AcronymModel.java deleted file mode 100644 index a00bfa1..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/AcronymModel.java +++ /dev/null @@ -1,666 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import java.util.*; -import java.io.*; - /** - * Scores Pascal challenge workshop information templates. - * This score reflects which fields are present/absent, how well acronyms - * agree with the names and URLs they correspond to. - * - * @author Jamie Nicolson - */ -public class AcronymModel implements RelationalModel { - - - private static final double HIGH_PROB = 1.0; - private static final double LOW_PROB = 0.0; - private static boolean DEBUG= false; - - private static final String acronymStatistics = - "workshopname workshopacronym workshophomepage conferencename conferenceacronym conferencehomepage\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00549450549450549\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.0521978021978022\n" + - "0.00274725274725275\n" + - "0.0357142857142857\n" + - "0.00549450549450549\n" + - "0.021978021978022\n" + - "0.010989010989011\n" + - "0.0357142857142857\n" + - "0.0302197802197802\n" + - "0.0824175824175824\n" + - "0.00549450549450549\n" + - "0.043956043956044\n" + - "0.010989010989011\n" + - "0.021978021978022\n" + - "0.00549450549450549\n" + - "0.0521978021978022\n" + - "0.0412087912087912\n" + - "0.0467032967032967\n" + - "0.00274725274725275\n" + - "0.010989010989011\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.00274725274725275\n" + - "0.0137362637362637\n" + - "0.00824175824175824\n" + - "0.167582417582418\n" + - "0.00549450549450549\n" + - "0.0494505494505494\n" + - "0.00824175824175824\n" + - "0.0164835164835165\n" + - "0.00549450549450549\n" + - "0.0604395604395604\n" + - "0.0467032967032967\n"; - - private Prior priors; - - /** - * Scores the partial template containing only the fields relevant to the score. - * @param temp the {@link InfoTemplate} to be scored. - * @return the model's score - */ - public double computeProb(InfoTemplate temp){ - return computeProb(temp.wname,temp.wacronym,temp.cname,temp.cacronym, - temp.whomepage, temp.chomepage); - } -/** - * Scores the {@link PascalTemplate} using the fields it contains which are relevant to the score. - * (Ignores location and date fields.) - * @param temp the full {@link PascalTemplate} to be scored - * @return the model's score - */ - - public double computeProb(PascalTemplate temp) { - double prob = 1.0; - - String wsname = temp.getValue("workshopname"); - String confname = temp.getValue("conferencename"); - String wsacronym = temp.getValue("workshopacronym"); - String confacronym = temp.getValue("conferenceacronym"); - String wsurl = temp.getValue("workshophomepage"); - String confurl = temp.getValue("conferencehomepage"); - return computeProb(wsname, wsacronym,confname,confacronym, wsurl, confurl); - } - - /** - * @throws IOException if the acronym statistics/weights can't be read from file. - */ - public AcronymModel() throws IOException { - priors = new Prior(new BufferedReader(new StringReader(acronymStatistics))); - features = new Feature[]{new AcronymModel.LettersAligned(), new AcronymModel.BegWord(), new AcronymModel.EndWord(), new AcronymModel.AfterAligned(), new AcronymModel.AlignedPerWord(), new AcronymModel.WordsSkipped(), new AcronymModel.SyllableBoundary()}; - weights = new double[]{// here's weights from a bunch of training examples - //-4.1004, 18.4127, 0.1789, 16.3189, 0.8818, -0.0725, -0.6550 - //-12.4082, 18.3893, 2.1826, 18.8487, 0.5042, -0.1231, 1.8876 - -11.8880, 14.4534, -2.6316, 24.1838, -2.2320, -0.2508, 4.3501 - - }; - //intercept = -14.1449; - //intercept = -7.4882; - intercept = -2.2062; - } - - private double computeProb(String wsname, String wsacronym, String confname, - String confacronym, String wsurl, String confurl){ - - HashSet presentFields = new HashSet(); - if( wsname != null && !wsname.equals("null") && !wsname.equals("") ) - presentFields.add("workshopname"); - if( wsacronym != null && !wsacronym.equals("null") && !wsacronym.equals("")) - presentFields.add("workshopacronym"); - if( confname != null && !confname.equals("null") - && !confname.equals("")) - presentFields.add("conferencename"); - if( confacronym != null && !confacronym.equals("null") - && !confacronym.equals("")) - presentFields.add("conferenceacronym"); - if( wsurl != null && !wsurl.equals("null") && !wsurl.equals("")) - presentFields.add("workshophomepage"); - if( confurl != null && !confurl.equals("null") && !confurl.equals("")) - presentFields.add("conferencehomepage"); - - //if the workshop and conference have the same acronym we return 0. - if(presentFields.contains("conferenceacronym") && - presentFields.contains("workshopacronym") && - confacronym.equals(wsacronym)){ - - return 0.0; - } - - double prob = priors.get(presentFields); - //System.out.println("Setting prior to " + prob + " based on the following "+ - // "fields being present: " + presentFields.toString()); - if( wsname != null && wsacronym != null ) { - if(DEBUG)System.err.println("computing similarity for workshop"); - prob *= similarity(wsname, wsacronym); - } else { - if(DEBUG)System.err.println("NOT computing similarity for workshop"); - } - - if( confname != null && confacronym != null ) { - if(DEBUG)System.err.println("computing similarity for conference"); - prob *= similarity(confname, confacronym); - } else { - if(DEBUG)System.err.println("NOT computing similarity for conference"); - } - - if( confacronym != null && confurl != null ) { - if( acronymMatchesURL(confacronym, confurl) ) { - prob *= probMatchFromAcronymAndURLMatch; - } else { - prob *= probMatchFromAcronymAndURLNoMatch; - } - } - - if( wsacronym != null && wsurl != null ) { - if( acronymMatchesURL(wsacronym, wsurl) ) { - prob *= probMatchFromAcronymAndURLMatch; - } else { - prob *= probMatchFromAcronymAndURLNoMatch; - } - } - return prob; - } - - private static boolean acronymMatchesURL(String ac, String url) { - String lowerURL = url.toLowerCase(); - String strippedAc = (new String(AcronymModel.stripAcronym(ac))).toLowerCase(); - - return lowerURL.indexOf(strippedAc) != -1; - } - - private static final double probMatchFromAcronymAndURLMatch = .23934426; - private static final double probMatchFromAcronymAndURLNoMatch = .052516411378; - - /** - * Finds longest subsequent string of digits. Returns empty string - * if there aren't any digits. - */ - private static String acronymNumber(String acronym) { - return ""; - } - - public static double URLSimilarity(String URL, String acronym) { - String strippedAc = new String(stripAcronym(acronym)); - String acNumber = acronymNumber(acronym); - return 0.0; - } - - /** - * @return the "rich similarity" score - */ - public double similarity(String name, String acronym) { - return RichSimilarity(name, acronym); - } - /** - * - * @return the "naive similarity" score - */ - public double naiveSimilarity(String name, String acronym) { - double similarity = LOW_PROB; - String[] nameWords = splitOnWhitespace(name); - String[] acronymWords = splitOnWhitespace(acronym); - - // first put together the letters in the acronym - char[] acLetters = allLetters(acronymWords); - - // first let's try pulling the first letters from the name, and combining them to get the acronym - char[] nameFirstLetters = firstLetters(nameWords); - - if (firstLetterInOrderMatch(nameFirstLetters, acLetters)) { - // the letters in acronym can be constructed from the first letters in the name, in order - similarity = HIGH_PROB; - } - - if (DEBUG) { - System.err.println("Similarity between (" + name + ") and (" + acronym + ") is " + similarity); - } - return similarity; - } - - /** - * - * @return the Hearst similarity score - */ - public double HearstSimilarity(String name, String acronym) { - char[] namechars = name.toLowerCase().toCharArray(); - char[] acrochars = acronym.toLowerCase().toCharArray(); - - int nindex = namechars.length - 1; - for (int aindex = acrochars.length - 1; aindex >= 0; --aindex) { - if (!Character.isLetter(acrochars[aindex])) { - continue; - } - while ((nindex >= 0 && namechars[nindex] != acrochars[aindex]) || (aindex == 0 && nindex > 0 && Character.isLetterOrDigit(namechars[nindex - 1]))) { - nindex--; - } - if (nindex < 0) { - // System.err.println("\"" + name + "\" does NOT match \"" + - // acronym + "\"\n"); - return 0; - } - - nindex--; - } - - //System.err.println("\"" + name + "\" matches \"" + acronym + "\"\n"); - return 1.0; - } - - public static interface Feature { - public double value(Alignment alignment); - - public String toString(); - } - - public static class LettersAligned implements Feature { - public String toString() { - return "LettersAligned"; - }; - public double value(Alignment alignment) { - int numAligned = 0; - for (int i = 0; i < alignment.pointers.length; ++i) { - if (alignment.pointers[i] != -1) { - numAligned++; - } - } - double pct = (double) numAligned / (double) alignment.pointers.length; - if (DEBUG) - System.out.println("LettersAligned=" + pct); - return pct; - } - } - - public static class BegWord implements Feature { - public String toString() { return "BegWord"; }; - public double value(Alignment alignment) { - int begAligned = 0; - for( int s = 0; s < alignment.pointers.length; ++s) { - int idx = alignment.pointers[s]; - if( idx == 0 ) { - begAligned++; - } else if( idx > 0) { - char cur = alignment.longForm[idx]; - char prev = alignment.longForm[idx-1]; - if( !Character.isLetterOrDigit(prev) && - Character.isLetterOrDigit(cur) ) - { - begAligned++; - } - } - } - return (double)begAligned / (double)alignment.shortForm.length; - } - } - - public static class EndWord implements Feature { - public String toString() { return "EndWord"; }; - public double value(Alignment alignment) { - int endAligned = 0; - for( int s = 0; s < alignment.pointers.length; ++s) { - int idx = alignment.pointers[s]; - if( idx == alignment.longForm.length-1 ) { - endAligned++; - } else if( idx >= 0) { - char cur = alignment.longForm[idx]; - char next = alignment.longForm[idx+1]; - if( !Character.isLetterOrDigit(next) && - Character.isLetterOrDigit(cur) ) - { - endAligned++; - } - } - } - return (double)endAligned / (double)alignment.shortForm.length; - } - } - - /** - * Percent of letters aligned immediately after another aligned letter. - */ - public static class AfterAligned implements Feature { - public String toString() { return "AfterAligned"; } - - public double value(Alignment alignment) { - int numAfter = 0; - for( int i = 1; i < alignment.pointers.length; ++i) { - if( alignment.pointers[i] == alignment.pointers[i-1] + 1 ) { - numAfter++; - } - } - return (double)numAfter / (double)alignment.shortForm.length; - } - } - - private static class RunningAverage { - double average; - int numSamples; - - public RunningAverage() { - average = 0.0; - numSamples = 0; - } - - public void addSample(double sample) { - average = (numSamples * average) + sample; - numSamples++; - average /= numSamples; - } - - public double getAverage() { - return average; - } - - public double getNumSammples() { - return numSamples; - } - } - - /** - * Average number of aligned letters per word. - */ - public static class AlignedPerWord implements Feature { - public String toString() { return "AlignedPerWord"; } - - public double value(Alignment alignment) { -/* - RunningAverage alignedPerWord = new RunningAverage(); - boolean inWord = false; - int alignCount = 0; - int sidx = 0; - for(int lidx = 0; lidx < alignment.longForm.length; ++lidx ) { - char cur = alignment.longForm[lidx]; - if( Character.isLetterOrDigit(cur) && !inWord ) { - // beginning of word - inWord = true; - } else if( inWord && !Character.isLetterOrDigit(cur) ) { - // end of word - alignedPerWord.addSample(alignCount); - alignCount = 0; - inWord = false; - } - - while( sidx < alignment.pointers.length && - alignment.pointers[sidx] < lidx ) - sidx++; - - if( sidx < alignment.pointers.length && - alignment.pointers[sidx] == lidx && inWord) - { - alignCount++; - } - } - if( inWord ) { - // end of last word - alignedPerWord.addSample(alignCount); - } - - return alignedPerWord.getAverage(); -*/ - boolean inWord = false; - int wordCount = 0; - for(int lidx = 0; lidx < alignment.longForm.length; ++lidx ) { - char cur = alignment.longForm[lidx]; - if( Character.isLetterOrDigit(cur) && !inWord ) { - // beginning of word - ++wordCount; - inWord = true; - } else if( inWord && !Character.isLetterOrDigit(cur) ) { - // end of word - inWord = false; - } - } - int alignCount = 0; - for( int sidx = 0; sidx < alignment.pointers.length; ++sidx) { - if( alignment.pointers[sidx] != -1 ) { - ++alignCount; - } - } - if( wordCount == 0 ) { - return 0; - } else { - return (double)alignCount / (double)wordCount; - } - } - } - - public static class WordsSkipped implements Feature { - public String toString() { return "WordsSkipped"; }; - public double value(Alignment alignment) { - int wordsSkipped = 0; - int wordsAligned = 0; - boolean inWord = false; - boolean gotAlignedChar = false; - boolean []isAligned = new boolean[alignment.longForm.length]; - for( int s = 0; s < alignment.pointers.length; ++s ) { - if( alignment.pointers[s] != -1 ) { - isAligned[alignment.pointers[s]] = true; - } - } - for( int l = 0; l < alignment.longForm.length; ++l ) { - char cur = alignment.longForm[l]; - if( inWord ) { - if( !Character.isLetterOrDigit(cur)) { - // just finished a word - if( gotAlignedChar ) { - wordsAligned++; - } else { - wordsSkipped++; - } - inWord = false; - } - } else { - if( Character.isLetterOrDigit(cur)) { - inWord = true; - gotAlignedChar = false; - } - } - if( isAligned[l] ) gotAlignedChar = true; - } - if( inWord ) { - if( gotAlignedChar ) { - wordsAligned++; - } else { - wordsSkipped++; - } - } - if(DEBUG)System.out.println("Words skipped: " + wordsSkipped + "/" + - (wordsSkipped + wordsAligned) ); - return wordsSkipped; - } - } - - public static class SyllableBoundary implements Feature { - public String toString() { return "SyllableBoundary"; }; - TeXHyphenator teXHyphenator = new TeXHyphenator(); - public SyllableBoundary() throws IOException { - teXHyphenator.loadDefault(); - } - public double value(Alignment alignment) { - char [] lcLongForm = - (new String(alignment.longForm)).toLowerCase().toCharArray(); - boolean [] breakPoints = teXHyphenator.findBreakPoints(lcLongForm); - int numSylAligned = 0; - for( int i = 0; i < alignment.pointers.length; ++i ) { - if( alignment.pointers[i] != -1 && - breakPoints[alignment.pointers[i]] ) - { - numSylAligned++; - } - } - return (double)numSylAligned / (double)alignment.pointers.length; - } - } - - private final Feature[] features; - - private final double[] weights; - private final double intercept; - - public static char[] stripAcronym(String acronym) { - char [] raw = acronym.toCharArray(); - char [] firstTry = new char[raw.length]; - int outIdx = 0; - for( int inIdx = 0; inIdx < raw.length; ++inIdx) { - if( Character.isLetter(raw[inIdx]) ) { - firstTry[outIdx++] = raw[inIdx]; - } - } - if( outIdx == firstTry.length ) { - if(DEBUG) System.out.println("Converted \"" + acronym + "\" to \"" + - (new String(firstTry)) + "\"\n"); - return firstTry; - } else { - char [] polished = new char[outIdx]; - System.arraycopy(firstTry, 0, polished, 0, outIdx); - if(DEBUG) System.out.println("Converted \"" + acronym + "\" to \"" + - (new String(polished)) + "\"\n"); - return polished; - } - } - - - public double RichSimilarity(String name, String acronym) { - AlignmentFactory fact = new AlignmentFactory( - name.toCharArray(), stripAcronym(acronym) ); - - double maxprob = 0.0; - Iterator iter = fact.getAlignments(); - while(iter.hasNext()) { - Alignment align = (Alignment) iter.next(); - - double [] featureVals = new double[features.length]; - for( int f = 0; f < features.length; ++f) { - featureVals[f] = features[f].value(align); - } - - // compute dotproduct and sigmoid - double dotprod = dotproduct(weights, featureVals) + intercept; - double exp = Math.exp(dotprod); - double prob = exp / (1 + exp); - - // align.print(); - //System.out.println("Prob: " + prob + "\n-----------\n"); - - if( prob > maxprob ){ - maxprob = prob; - } - } - - return maxprob; - } - - private static double dotproduct(double[] one, double[]two) { - double sum = 0.0; - for( int i = 0; i < one.length; ++i) { - double product = one[i] * two[i]; - if(DEBUG)System.out.println("product: " + product); - sum += product; - } - if(DEBUG)System.out.println("sum: " + sum); - return sum; - } - - private static final String[] stringArrayType = new String[0]; - - private static String[] splitOnWhitespace(String words) { - String[] firstCut = words.split("\\s+"); - - ArrayList wordList = new ArrayList(firstCut.length); - for( int i = 0; i < firstCut.length; ++i ) { - if( firstCut[i].length() > 0 ) { - wordList.add(firstCut[i]); - } - } - return wordList.toArray(stringArrayType); - } - - private static boolean firstLetterInOrderMatch(char[] nameFirstLetters, char[] acLetters) { - int nameIdx = 0; - int acIdx = 0; - - for( ; acIdx < acLetters.length; ++acIdx) { - while( nameIdx < nameFirstLetters.length && nameFirstLetters[nameIdx] != acLetters[acIdx] ) { - ++nameIdx; - } - if( nameIdx == nameFirstLetters.length ) { - return false; - } - } - return true; - } - - private static char[] allLetters(String[] acronym) { - StringBuffer sb = new StringBuffer(); - for( int s = 0; s < acronym.length; ++s ) { - String acr = acronym[s]; - for(int c = 0; c < acr.length(); ++c ) { - char ch = acr.charAt(c); - if( Character.isLetter( ch ) ) { - sb.append(ch); - } - } - } - return sbToChars(sb); - } - - private static char[] firstLetters(String[] name) { - StringBuffer sb = new StringBuffer(name.length); - for( int s = 0; s < name.length; ++s) { - char c = name[s].charAt(0); - if( Character.isLetter(c) ) { - sb.append(c); - } - } - return sbToChars(sb); - } - - private static char[] sbToChars(StringBuffer sb) { - char[] letters = new char[sb.length()]; - sb.getChars(0, sb.length(), letters, 0); - return letters; - } - - public static void main(String[] args) throws Exception { - - AcronymModel am = new AcronymModel(); - String s1 = args[0]; - String s2 = args[1]; - System.out.println("Hearst: "+am.HearstSimilarity(s1, s2)); - System.out.println("naive: "+am.naiveSimilarity(s1, s2)); - System.out.println("Rich: "+am.RichSimilarity(s1, s2)); - System.out.println("default: "+am.similarity(s1, s2)); - - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/Alignment.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/Alignment.java deleted file mode 100644 index 7d26608..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/Alignment.java +++ /dev/null @@ -1,113 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Arrays; - -/** - * Container class for aligning acronyms. - * - * @author Jamie Nicolson - */ - -public class Alignment { - public char[] longForm; - public char[] shortForm; - public int[] pointers; - - public Alignment(char[] longForm, char[] shortForm, int[] pointers) { - this.longForm = longForm; - this.shortForm = shortForm; - this.pointers = pointers; - } - - public void serialize(PrintWriter writer) { - writer.println(new String(longForm)); - writer.println(new String(shortForm)); - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < pointers.length; ++i) { - sb.append(pointers[i] + " "); - } - writer.println(sb.toString()); - } - - public Alignment(BufferedReader reader) throws IOException { - String line; - line = reader.readLine(); - if (line == null) { - throw new IOException(); - } - longForm = line.toCharArray(); - line = reader.readLine(); - if (line == null) { - throw new IOException(); - } - shortForm = line.toCharArray(); - line = reader.readLine(); - if (line == null) { - throw new IOException(); - } - String[] pstrings = line.split("\\s+"); - if (pstrings.length != shortForm.length) { - throw new IOException("Number of pointers != size of short form"); - } - pointers = new int[pstrings.length]; - for (int i = 0; i < pointers.length; ++i) { - pointers[i] = Integer.parseInt(pstrings[i]); - } - } - - public void print() { - System.out.println(toString()); - } - - @Override - public String toString() { - return toString(""); - } - - private static final char[] spaces = " ".toCharArray(); - - public String toString(String prefix) { - StringBuffer buf = new StringBuffer(); - buf.append(prefix); - buf.append(longForm); - buf.append("\n"); - buf.append(spaces, 0, prefix.length()); - int l = 0; - for (int s = 0; s < shortForm.length; ++s) { - if (pointers[s] == -1) { - continue; - } - for (; l < longForm.length && pointers[s] != l; ++l) { - buf.append(" "); - } - if (l < longForm.length) { - buf.append(shortForm[s]); - ++l; - } - } - return buf.toString(); - } - - @Override - public boolean equals(Object o) { - if (o == null || !(o instanceof Alignment)) { - return false; - } - Alignment cmp = (Alignment) o; - - return Arrays.equals(longForm, cmp.longForm) && Arrays.equals(shortForm, cmp.shortForm) && Arrays.equals(pointers, cmp.pointers); - } - - @Override - public int hashCode() { - int code = 0; - for (int i = 0; i < pointers.length; ++i) { - code += pointers[i]; - code *= 31; - } - return code; - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/AlignmentFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/AlignmentFactory.java deleted file mode 100644 index 611f560..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/AlignmentFactory.java +++ /dev/null @@ -1,216 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import java.util.*; -/** - * Generates {@link Alignment} objects for acronym alignment. - * - * @author Jamie Nicolson - */ -public class AlignmentFactory { - - public static final byte SHIFT_LONG = 1; - public static final byte SHIFT_SHORT = 2; - public static final byte SHIFT_BOTH = 4; - - private char[] longForm; - private char[] lcLongForm; - private char[] shortForm; - private char[] lcShortForm; - private int [][]alignMatrix; - private byte [][]backMatrix; - private HashSet alignments; - - public AlignmentFactory(String longForm, String shortForm) { - this(longForm.toCharArray(), shortForm.toCharArray()); - } - - public static char[] toLower(char []in) { - char[] out = new char[in.length]; - for(int i = 0; i < in.length; ++i) { - out[i] = Character.toLowerCase(in[i]); - } - return out; - } - - public AlignmentFactory(char[] longForm, char[] shortForm) { - this.longForm = longForm; - this.lcLongForm = toLower(longForm); - this.shortForm = shortForm; - this.lcShortForm = toLower(shortForm); - - alignMatrix = new int[lcLongForm.length][lcShortForm.length]; - backMatrix = new byte[lcLongForm.length][lcShortForm.length]; - for( int l = 0; l < lcLongForm.length; ++l) { - for( int s = 0; s < lcShortForm.length; ++s) { - int match = (lcLongForm[l] == lcShortForm[s]) ? 1 : 0; - int froml = (l == 0) ? 0 : alignMatrix[l-1][s]; - int froms = (s == 0) ? 0 : alignMatrix[l][s-1]; - int frommatch = - ((l==0 || s==0) ? 0 : alignMatrix[l-1][s-1]) + match; - int max = Math.max(froml, Math.max(froms, frommatch)); - byte backp = 0; - if( froml == max ) backp |= SHIFT_LONG; - if( froms == max ) backp |= SHIFT_SHORT; - if( match == 1 && frommatch == max ) backp |= SHIFT_BOTH; - backMatrix[l][s] = backp; - alignMatrix[l][s] = max; - } - } - - alignments = new HashSet(); - int[] pointers = new int[lcShortForm.length]; - Arrays.fill(pointers, -1); - - if( lcLongForm.length > 0 && lcShortForm.length > 0 ) { - addCount = 0; - //initListMatrix(); - findAlignments(pointers, lcLongForm.length-1, lcShortForm.length-1); - //listMatrix = null; - - } - } - - public Iterator getAlignments() { - return alignments.iterator(); - } - - public ArrayList getAlignmentsList() { - return new ArrayList(alignments); - } - - public static String dumpIntArray(int []a) { - StringBuilder buf = new StringBuilder(); - buf.append('['); - for (int anA : a) { - buf.append(anA).append(' '); - } - buf.append(']'); - return buf.toString(); - } - - int addCount; - -/* - LinkedList [] [] listMatrix; - - private void initListMatrix() { - listMatrix = new LinkedList[][lcLongForm.length]; - for( int i = 0; i < lcLongForm.length; i++) { - listMatrix[i] = new LinkedList[lcShortForm.length]; - } - } -*/ - -/* - private void findAlignments(int l, int s) { - if( listMatrix[l][s] != null ) - return; - - byte backp = backMatrix[l][s]; - - listMatrix[l][s] = new LinkedList(); - - if( alignMatrix[l][s] == 0 ) { - listMatrix[l][s].add( new int[shortForm.length] ); - return; - } - - if( (backp & SHIFT_BOTH) != 0 ) { - assert( lcLongForm[l] == lcShortForm[s] ); - findAlignments(l-1,s-1); - LinkedList from = listMatrix[l-1][s-1]; - Iterator iter = from.iterator(); - while(iter.hasNext()) { - int[] ref = (int[]) iter.next(); - int[] cpy = ref.clone(); - cpy[s] = l; - listMatrix[l][s].add(cpy); - } - } - - if( (backp & SHIFT_LONG) != 0 ) { - if( l != 0 ) { - findAlignments(l-1, s); - Iterator iter = listMatrix[l-1][s]; - while(iter.hasNext()) { - listMatrix[l][s].add( iter.next() ); - } - } else { - listMatrix[l][s].add( new int[shortForm.length] ); - } - } - - if( (backp & SHIFT_SHORT) != 0 ) { - backp &= ~SHIFT_SHORT; - int[] ptrcpy = (int[]) ((backp == 0) ? pointers : pointers.clone()); - if( s == 0 ) { - ++addCount; - alignments.add( new Alignment(longForm, shortForm, ptrcpy) ); - } else { - findAlignments(ptrcpy, l, s-1); - } - } - - if( lcLongForm[l] == lcShortForm[s] ) - assert( (backMatrix[l][s] & SHIFT_BOTH) != 0); -*/ - - private void findAlignments(int[]pointers, int lg, int s) - { - byte backp = backMatrix[lg][s]; - - if( alignMatrix[lg][s] == 0 ) { - ++addCount; - alignments.add( new Alignment(longForm, shortForm, pointers) ); - return; - } - - if( (backp & SHIFT_LONG)!= 0 ) { - backp &= ~SHIFT_LONG; - int[] ptrcpy = ((backp == 0) ? pointers : pointers.clone()); - if( lg == 0 ) { - ++addCount; - alignments.add( new Alignment(longForm, shortForm, ptrcpy) ); - } else { - findAlignments(ptrcpy, lg-1, s); - } - } - - if( (backp & SHIFT_SHORT) != 0 ) { - backp &= ~SHIFT_SHORT; - int[] ptrcpy = ((backp == 0) ? pointers : pointers.clone()); - if( s == 0 ) { - ++addCount; - alignments.add( new Alignment(longForm, shortForm, ptrcpy) ); - } else { - findAlignments(ptrcpy, lg, s-1); - } - } - - if( lcLongForm[lg] == lcShortForm[s] ) - assert( (backMatrix[lg][s] & SHIFT_BOTH) != 0); - - if( (backp & SHIFT_BOTH) != 0 ) { - assert( lcLongForm[lg] == lcShortForm[s] ); - pointers[s] = lg; - if( lg == 0 || s == 0 ) { - ++addCount; - alignments.add( new Alignment(longForm, shortForm, pointers) ); - } else { - findAlignments(pointers, lg-1, s-1); - } - } - } - - public static void main(String[] args) throws Exception { - AlignmentFactory fact = new AlignmentFactory(args[0].toCharArray(), - AcronymModel.stripAcronym(args[1])); - - Iterator iter = fact.getAlignments(); - while( iter.hasNext() ) { - Alignment a = iter.next(); - a.print(); - } - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/CliqueTemplates.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/CliqueTemplates.java deleted file mode 100644 index 100f093..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/CliqueTemplates.java +++ /dev/null @@ -1,31 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import edu.stanford.nlp.stats.ClassicCounter; - -import java.net.URL; -import java.util.HashMap; -import java.util.ArrayList; - -/** - * Template information and counters corresponding to sampling on one document. - * - * As an alternative to reading a document labelling into a full {@link PascalTemplate} - * we can read it into partial templates which contain only strictly related information, - * (See {@link DateTemplate} and {@link InfoTemplate}). - * - * @author Chris Cox - */ - -public class CliqueTemplates { - - public HashMap stemmedAcronymIndex = new HashMap(); - public HashMap inverseAcronymMap = new HashMap(); - - public ArrayList urls = null; - - public ClassicCounter dateCliqueCounter = new ClassicCounter(); - public ClassicCounter locationCliqueCounter = new ClassicCounter(); - public ClassicCounter workshopInfoCliqueCounter = new ClassicCounter(); - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/DateTemplate.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/DateTemplate.java deleted file mode 100644 index 6e3a25e..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/DateTemplate.java +++ /dev/null @@ -1,49 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -/** - * A partial {@link PascalTemplate}. Holds date fields only. - * - * @author Chris Cox - */ -public class DateTemplate{ - - public String subdate="1/1/1000"; - public String noadate="1/1/1000"; - public String crcdate="1/1/1000"; - public String workdate="1/1/1000"; - - public DateTemplate(String subdate,String noadate,String crcdate,String workdate) { - if(subdate!=null)this.subdate=subdate; - if(noadate!=null)this.noadate=noadate; - if(crcdate!=null)this.crcdate=crcdate; - if(workdate!=null)this.workdate=workdate; - } - - @Override - public int hashCode() { - int tally = 31; - int n = 3; - tally = tally+n*subdate.hashCode()+n*n*noadate.hashCode()+ - n*n*n*crcdate.hashCode()+n*workdate.hashCode(); - return tally; - } - - @Override - public boolean equals(Object obj) { - if(obj==null)return false; - if(! (obj instanceof DateTemplate)) return false; - - DateTemplate d = (DateTemplate)obj; - return (subdate.equals(d.subdate) && - noadate.equals(d.noadate) && - crcdate.equals(d.crcdate) && - workdate.equals(d.workdate)); - } - - @Override - public String toString() { - return (" Sub:" + subdate + " Noa:" + noadate + " Crc:" + crcdate + " Wrk:" + workdate); - } - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/DefaultTeXHyphenData.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/DefaultTeXHyphenData.java deleted file mode 100644 index 029912f..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/DefaultTeXHyphenData.java +++ /dev/null @@ -1,4461 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -/** - * Default TeX hyphenation data, as borrowed from the TeX distribution. - * - * THIS FILE IS AUTOMATICALLY GENERATED from DefaultTeXHyphenData.java.template - * and buildDefaultTeXHyphenData.pl. - * - * DO NOT EDIT DefaultTeXHyphenData.java; YOUR CHANGES WILL BE DISCARDED. - */ -public class DefaultTeXHyphenData { - public static final String hyphenData = - ".ach4\n" + - ".ad4der\n" + - ".af1t\n" + - ".al3t\n" + - ".am5at\n" + - ".an5c\n" + - ".ang4\n" + - ".ani5m\n" + - ".ant4\n" + - ".an3te\n" + - ".anti5s\n" + - ".ar5s\n" + - ".ar4tie\n" + - ".ar4ty\n" + - ".as3c\n" + - ".as1p\n" + - ".as1s\n" + - ".aster5\n" + - ".atom5\n" + - ".au1d\n" + - ".av4i\n" + - ".awn4\n" + - ".ba4g\n" + - ".ba5na\n" + - ".bas4e\n" + - ".ber4\n" + - ".be5ra\n" + - ".be3sm\n" + - ".be5sto\n" + - ".bri2\n" + - ".but4ti\n" + - ".cam4pe\n" + - ".can5c\n" + - ".capa5b\n" + - ".car5ol\n" + - ".ca4t\n" + - ".ce4la\n" + - ".ch4\n" + - ".chill5i\n" + - ".ci2\n" + - ".cit5r\n" + - ".co3e\n" + - ".co4r\n" + - ".cor5ner\n" + - ".de4moi\n" + - ".de3o\n" + - ".de3ra\n" + - ".de3ri\n" + - ".des4c\n" + - ".dictio5\n" + - ".do4t\n" + - ".du4c\n" + - ".dumb5\n" + - ".earth5\n" + - ".eas3i\n" + - ".eb4\n" + - ".eer4\n" + - ".eg2\n" + - ".el5d\n" + - ".el3em\n" + - ".enam3\n" + - ".en3g\n" + - ".en3s\n" + - ".eq5ui5t\n" + - ".er4ri\n" + - ".es3\n" + - ".eu3\n" + - ".eye5\n" + - ".fes3\n" + - ".for5mer\n" + - ".ga2\n" + - ".ge2\n" + - ".gen3t4\n" + - ".ge5og\n" + - ".gi5a\n" + - ".gi4b\n" + - ".go4r\n" + - ".hand5i\n" + - ".han5k\n" + - ".he2\n" + - ".hero5i\n" + - ".hes3\n" + - ".het3\n" + - ".hi3b\n" + - ".hi3er\n" + - ".hon5ey\n" + - ".hon3o\n" + - ".hov5\n" + - ".id4l\n" + - ".idol3\n" + - ".im3m\n" + - ".im5pin\n" + - ".in1\n" + - ".in3ci\n" + - ".ine2\n" + - ".in2k\n" + - ".in3s\n" + - ".ir5r\n" + - ".is4i\n" + - ".ju3r\n" + - ".la4cy\n" + - ".la4m\n" + - ".lat5er\n" + - ".lath5\n" + - ".le2\n" + - ".leg5e\n" + - ".len4\n" + - ".lep5\n" + - ".lev1\n" + - ".li4g\n" + - ".lig5a\n" + - ".li2n\n" + - ".li3o\n" + - ".li4t\n" + - ".mag5a5\n" + - ".mal5o\n" + - ".man5a\n" + - ".mar5ti\n" + - ".me2\n" + - ".mer3c\n" + - ".me5ter\n" + - ".mis1\n" + - ".mist5i\n" + - ".mon3e\n" + - ".mo3ro\n" + - ".mu5ta\n" + - ".muta5b\n" + - ".ni4c\n" + - ".od2\n" + - ".odd5\n" + - ".of5te\n" + - ".or5ato\n" + - ".or3c\n" + - ".or1d\n" + - ".or3t\n" + - ".os3\n" + - ".os4tl\n" + - ".oth3\n" + - ".out3\n" + - ".ped5al\n" + - ".pe5te\n" + - ".pe5tit\n" + - ".pi4e\n" + - ".pio5n\n" + - ".pi2t\n" + - ".pre3m\n" + - ".ra4c\n" + - ".ran4t\n" + - ".ratio5na\n" + - ".ree2\n" + - ".re5mit\n" + - ".res2\n" + - ".re5stat\n" + - ".ri4g\n" + - ".rit5u\n" + - ".ro4q\n" + - ".ros5t\n" + - ".row5d\n" + - ".ru4d\n" + - ".sci3e\n" + - ".self5\n" + - ".sell5\n" + - ".se2n\n" + - ".se5rie\n" + - ".sh2\n" + - ".si2\n" + - ".sing4\n" + - ".st4\n" + - ".sta5bl\n" + - ".sy2\n" + - ".ta4\n" + - ".te4\n" + - ".ten5an\n" + - ".th2\n" + - ".ti2\n" + - ".til4\n" + - ".tim5o5\n" + - ".ting4\n" + - ".tin5k\n" + - ".ton4a\n" + - ".to4p\n" + - ".top5i\n" + - ".tou5s\n" + - ".trib5ut\n" + - ".un1a\n" + - ".un3ce\n" + - ".under5\n" + - ".un1e\n" + - ".un5k\n" + - ".un5o\n" + - ".un3u\n" + - ".up3\n" + - ".ure3\n" + - ".us5a\n" + - ".ven4de\n" + - ".ve5ra\n" + - ".wil5i\n" + - ".ye4\n" + - "4ab.\n" + - "a5bal\n" + - "a5ban\n" + - "abe2\n" + - "ab5erd\n" + - "abi5a\n" + - "ab5it5ab\n" + - "ab5lat\n" + - "ab5o5liz\n" + - "4abr\n" + - "ab5rog\n" + - "ab3ul\n" + - "a4car\n" + - "ac5ard\n" + - "ac5aro\n" + - "a5ceou\n" + - "ac1er\n" + - "a5chet\n" + - "4a2ci\n" + - "a3cie\n" + - "ac1in\n" + - "a3cio\n" + - "ac5rob\n" + - "act5if\n" + - "ac3ul\n" + - "ac4um\n" + - "a2d\n" + - "ad4din\n" + - "ad5er.\n" + - "2adi\n" + - "a3dia\n" + - "ad3ica\n" + - "adi4er\n" + - "a3dio\n" + - "a3dit\n" + - "a5diu\n" + - "ad4le\n" + - "ad3ow\n" + - "ad5ran\n" + - "ad4su\n" + - "4adu\n" + - "a3duc\n" + - "ad5um\n" + - "ae4r\n" + - "aeri4e\n" + - "a2f\n" + - "aff4\n" + - "a4gab\n" + - "aga4n\n" + - "ag5ell\n" + - "age4o\n" + - "4ageu\n" + - "ag1i\n" + - "4ag4l\n" + - "ag1n\n" + - "a2go\n" + - "3agog\n" + - "ag3oni\n" + - "a5guer\n" + - "ag5ul\n" + - "a4gy\n" + - "a3ha\n" + - "a3he\n" + - "ah4l\n" + - "a3ho\n" + - "ai2\n" + - "a5ia\n" + - "a3ic.\n" + - "ai5ly\n" + - "a4i4n\n" + - "ain5in\n" + - "ain5o\n" + - "ait5en\n" + - "a1j\n" + - "ak1en\n" + - "al5ab\n" + - "al3ad\n" + - "a4lar\n" + - "4aldi\n" + - "2ale\n" + - "al3end\n" + - "a4lenti\n" + - "a5le5o\n" + - "al1i\n" + - "al4ia.\n" + - "ali4e\n" + - "al5lev\n" + - "4allic\n" + - "4alm\n" + - "a5log.\n" + - "a4ly.\n" + - "4alys\n" + - "5a5lyst\n" + - "5alyt\n" + - "3alyz\n" + - "4ama\n" + - "am5ab\n" + - "am3ag\n" + - "ama5ra\n" + - "am5asc\n" + - "a4matis\n" + - "a4m5ato\n" + - "am5era\n" + - "am3ic\n" + - "am5if\n" + - "am5ily\n" + - "am1in\n" + - "ami4no\n" + - "a2mo\n" + - "a5mon\n" + - "amor5i\n" + - "amp5en\n" + - "a2n\n" + - "an3age\n" + - "3analy\n" + - "a3nar\n" + - "an3arc\n" + - "anar4i\n" + - "a3nati\n" + - "4and\n" + - "ande4s\n" + - "an3dis\n" + - "an1dl\n" + - "an4dow\n" + - "a5nee\n" + - "a3nen\n" + - "an5est.\n" + - "a3neu\n" + - "2ang\n" + - "ang5ie\n" + - "an1gl\n" + - "a4n1ic\n" + - "a3nies\n" + - "an3i3f\n" + - "an4ime\n" + - "a5nimi\n" + - "a5nine\n" + - "an3io\n" + - "a3nip\n" + - "an3ish\n" + - "an3it\n" + - "a3niu\n" + - "an4kli\n" + - "5anniz\n" + - "ano4\n" + - "an5ot\n" + - "anoth5\n" + - "an2sa\n" + - "an4sco\n" + - "an4sn\n" + - "an2sp\n" + - "ans3po\n" + - "an4st\n" + - "an4sur\n" + - "antal4\n" + - "an4tie\n" + - "4anto\n" + - "an2tr\n" + - "an4tw\n" + - "an3ua\n" + - "an3ul\n" + - "a5nur\n" + - "4ao\n" + - "apar4\n" + - "ap5at\n" + - "ap5ero\n" + - "a3pher\n" + - "4aphi\n" + - "a4pilla\n" + - "ap5illar\n" + - "ap3in\n" + - "ap3ita\n" + - "a3pitu\n" + - "a2pl\n" + - "apoc5\n" + - "ap5ola\n" + - "apor5i\n" + - "apos3t\n" + - "aps5es\n" + - "a3pu\n" + - "aque5\n" + - "2a2r\n" + - "ar3act\n" + - "a5rade\n" + - "ar5adis\n" + - "ar3al\n" + - "a5ramete\n" + - "aran4g\n" + - "ara3p\n" + - "ar4at\n" + - "a5ratio\n" + - "ar5ativ\n" + - "a5rau\n" + - "ar5av4\n" + - "araw4\n" + - "arbal4\n" + - "ar4chan\n" + - "ar5dine\n" + - "ar4dr\n" + - "ar5eas\n" + - "a3ree\n" + - "ar3ent\n" + - "a5ress\n" + - "ar4fi\n" + - "ar4fl\n" + - "ar1i\n" + - "ar5ial\n" + - "ar3ian\n" + - "a3riet\n" + - "ar4im\n" + - "ar5inat\n" + - "ar3io\n" + - "ar2iz\n" + - "ar2mi\n" + - "ar5o5d\n" + - "a5roni\n" + - "a3roo\n" + - "ar2p\n" + - "ar3q\n" + - "arre4\n" + - "ar4sa\n" + - "ar2sh\n" + - "4as.\n" + - "as4ab\n" + - "as3ant\n" + - "ashi4\n" + - "a5sia.\n" + - "a3sib\n" + - "a3sic\n" + - "5a5si4t\n" + - "ask3i\n" + - "as4l\n" + - "a4soc\n" + - "as5ph\n" + - "as4sh\n" + - "as3ten\n" + - "as1tr\n" + - "asur5a\n" + - "a2ta\n" + - "at3abl\n" + - "at5ac\n" + - "at3alo\n" + - "at5ap\n" + - "ate5c\n" + - "at5ech\n" + - "at3ego\n" + - "at3en.\n" + - "at3era\n" + - "ater5n\n" + - "a5terna\n" + - "at3est\n" + - "at5ev\n" + - "4ath\n" + - "ath5em\n" + - "a5then\n" + - "at4ho\n" + - "ath5om\n" + - "4ati.\n" + - "a5tia\n" + - "at5i5b\n" + - "at1ic\n" + - "at3if\n" + - "ation5ar\n" + - "at3itu\n" + - "a4tog\n" + - "a2tom\n" + - "at5omiz\n" + - "a4top\n" + - "a4tos\n" + - "a1tr\n" + - "at5rop\n" + - "at4sk\n" + - "at4tag\n" + - "at5te\n" + - "at4th\n" + - "a2tu\n" + - "at5ua\n" + - "at5ue\n" + - "at3ul\n" + - "at3ura\n" + - "a2ty\n" + - "au4b\n" + - "augh3\n" + - "au3gu\n" + - "au4l2\n" + - "aun5d\n" + - "au3r\n" + - "au5sib\n" + - "aut5en\n" + - "au1th\n" + - "a2va\n" + - "av3ag\n" + - "a5van\n" + - "ave4no\n" + - "av3era\n" + - "av5ern\n" + - "av5ery\n" + - "av1i\n" + - "avi4er\n" + - "av3ig\n" + - "av5oc\n" + - "a1vor\n" + - "3away\n" + - "aw3i\n" + - "aw4ly\n" + - "aws4\n" + - "ax4ic\n" + - "ax4id\n" + - "ay5al\n" + - "aye4\n" + - "ays4\n" + - "azi4er\n" + - "azz5i\n" + - "5ba.\n" + - "bad5ger\n" + - "ba4ge\n" + - "bal1a\n" + - "ban5dag\n" + - "ban4e\n" + - "ban3i\n" + - "barbi5\n" + - "bari4a\n" + - "bas4si\n" + - "1bat\n" + - "ba4z\n" + - "2b1b\n" + - "b2be\n" + - "b3ber\n" + - "bbi4na\n" + - "4b1d\n" + - "4be.\n" + - "beak4\n" + - "beat3\n" + - "4be2d\n" + - "be3da\n" + - "be3de\n" + - "be3di\n" + - "be3gi\n" + - "be5gu\n" + - "1bel\n" + - "be1li\n" + - "be3lo\n" + - "4be5m\n" + - "be5nig\n" + - "be5nu\n" + - "4bes4\n" + - "be3sp\n" + - "be5str\n" + - "3bet\n" + - "bet5iz\n" + - "be5tr\n" + - "be3tw\n" + - "be3w\n" + - "be5yo\n" + - "2bf\n" + - "4b3h\n" + - "bi2b\n" + - "bi4d\n" + - "3bie\n" + - "bi5en\n" + - "bi4er\n" + - "2b3if\n" + - "1bil\n" + - "bi3liz\n" + - "bina5r4\n" + - "bin4d\n" + - "bi5net\n" + - "bi3ogr\n" + - "bi5ou\n" + - "bi2t\n" + - "3bi3tio\n" + - "bi3tr\n" + - "3bit5ua\n" + - "b5itz\n" + - "b1j\n" + - "bk4\n" + - "b2l2\n" + - "blath5\n" + - "b4le.\n" + - "blen4\n" + - "5blesp\n" + - "b3lis\n" + - "b4lo\n" + - "blun4t\n" + - "4b1m\n" + - "4b3n\n" + - "bne5g\n" + - "3bod\n" + - "bod3i\n" + - "bo4e\n" + - "bol3ic\n" + - "bom4bi\n" + - "bon4a\n" + - "bon5at\n" + - "3boo\n" + - "5bor.\n" + - "4b1ora\n" + - "bor5d\n" + - "5bore\n" + - "5bori\n" + - "5bos4\n" + - "b5ota\n" + - "both5\n" + - "bo4to\n" + - "bound3\n" + - "4bp\n" + - "4brit\n" + - "broth3\n" + - "2b5s2\n" + - "bsor4\n" + - "2bt\n" + - "bt4l\n" + - "b4to\n" + - "b3tr\n" + - "buf4fer\n" + - "bu4ga\n" + - "bu3li\n" + - "bumi4\n" + - "bu4n\n" + - "bunt4i\n" + - "bu3re\n" + - "bus5ie\n" + - "buss4e\n" + - "5bust\n" + - "4buta\n" + - "3butio\n" + - "b5uto\n" + - "b1v\n" + - "4b5w\n" + - "5by.\n" + - "bys4\n" + - "1ca\n" + - "cab3in\n" + - "ca1bl\n" + - "cach4\n" + - "ca5den\n" + - "4cag4\n" + - "2c5ah\n" + - "ca3lat\n" + - "cal4la\n" + - "call5in\n" + - "4calo\n" + - "can5d\n" + - "can4e\n" + - "can4ic\n" + - "can5is\n" + - "can3iz\n" + - "can4ty\n" + - "cany4\n" + - "ca5per\n" + - "car5om\n" + - "cast5er\n" + - "cas5tig\n" + - "4casy\n" + - "ca4th\n" + - "4cativ\n" + - "cav5al\n" + - "c3c\n" + - "ccha5\n" + - "cci4a\n" + - "ccompa5\n" + - "ccon4\n" + - "ccou3t\n" + - "2ce.\n" + - "4ced.\n" + - "4ceden\n" + - "3cei\n" + - "5cel.\n" + - "3cell\n" + - "1cen\n" + - "3cenc\n" + - "2cen4e\n" + - "4ceni\n" + - "3cent\n" + - "3cep\n" + - "ce5ram\n" + - "4cesa\n" + - "3cessi\n" + - "ces5si5b\n" + - "ces5t\n" + - "cet4\n" + - "c5e4ta\n" + - "cew4\n" + - "2ch\n" + - "4ch.\n" + - "4ch3ab\n" + - "5chanic\n" + - "ch5a5nis\n" + - "che2\n" + - "cheap3\n" + - "4ched\n" + - "che5lo\n" + - "3chemi\n" + - "ch5ene\n" + - "ch3er.\n" + - "ch3ers\n" + - "4ch1in\n" + - "5chine.\n" + - "ch5iness\n" + - "5chini\n" + - "5chio\n" + - "3chit\n" + - "chi2z\n" + - "3cho2\n" + - "ch4ti\n" + - "1ci\n" + - "3cia\n" + - "ci2a5b\n" + - "cia5r\n" + - "ci5c\n" + - "4cier\n" + - "5cific.\n" + - "4cii\n" + - "ci4la\n" + - "3cili\n" + - "2cim\n" + - "2cin\n" + - "c4ina\n" + - "3cinat\n" + - "cin3em\n" + - "c1ing\n" + - "c5ing.\n" + - "5cino\n" + - "cion4\n" + - "4cipe\n" + - "ci3ph\n" + - "4cipic\n" + - "4cista\n" + - "4cisti\n" + - "2c1it\n" + - "cit3iz\n" + - "5ciz\n" + - "ck1\n" + - "ck3i\n" + - "1c4l4\n" + - "4clar\n" + - "c5laratio\n" + - "5clare\n" + - "cle4m\n" + - "4clic\n" + - "clim4\n" + - "cly4\n" + - "c5n\n" + - "1co\n" + - "co5ag\n" + - "coe2\n" + - "2cog\n" + - "co4gr\n" + - "coi4\n" + - "co3inc\n" + - "col5i\n" + - "5colo\n" + - "col3or\n" + - "com5er\n" + - "con4a\n" + - "c4one\n" + - "con3g\n" + - "con5t\n" + - "co3pa\n" + - "cop3ic\n" + - "co4pl\n" + - "4corb\n" + - "coro3n\n" + - "cos4e\n" + - "cov1\n" + - "cove4\n" + - "cow5a\n" + - "coz5e\n" + - "co5zi\n" + - "c1q\n" + - "cras5t\n" + - "5crat.\n" + - "5cratic\n" + - "cre3at\n" + - "5cred\n" + - "4c3reta\n" + - "cre4v\n" + - "cri2\n" + - "cri5f\n" + - "c4rin\n" + - "cris4\n" + - "5criti\n" + - "cro4pl\n" + - "crop5o\n" + - "cros4e\n" + - "cru4d\n" + - "4c3s2\n" + - "2c1t\n" + - "cta4b\n" + - "ct5ang\n" + - "c5tant\n" + - "c2te\n" + - "c3ter\n" + - "c4ticu\n" + - "ctim3i\n" + - "ctu4r\n" + - "c4tw\n" + - "cud5\n" + - "c4uf\n" + - "c4ui\n" + - "cu5ity\n" + - "5culi\n" + - "cul4tis\n" + - "3cultu\n" + - "cu2ma\n" + - "c3ume\n" + - "cu4mi\n" + - "3cun\n" + - "cu3pi\n" + - "cu5py\n" + - "cur5a4b\n" + - "cu5ria\n" + - "1cus\n" + - "cuss4i\n" + - "3c4ut\n" + - "cu4tie\n" + - "4c5utiv\n" + - "4cutr\n" + - "1cy\n" + - "cze4\n" + - "1d2a\n" + - "5da.\n" + - "2d3a4b\n" + - "dach4\n" + - "4daf\n" + - "2dag\n" + - "da2m2\n" + - "dan3g\n" + - "dard5\n" + - "dark5\n" + - "4dary\n" + - "3dat\n" + - "4dativ\n" + - "4dato\n" + - "5dav4\n" + - "dav5e\n" + - "5day\n" + - "d1b\n" + - "d5c\n" + - "d1d4\n" + - "2de.\n" + - "deaf5\n" + - "deb5it\n" + - "de4bon\n" + - "decan4\n" + - "de4cil\n" + - "de5com\n" + - "2d1ed\n" + - "4dee.\n" + - "de5if\n" + - "deli4e\n" + - "del5i5q\n" + - "de5lo\n" + - "d4em\n" + - "5dem.\n" + - "3demic\n" + - "dem5ic.\n" + - "de5mil\n" + - "de4mons\n" + - "demor5\n" + - "1den\n" + - "de4nar\n" + - "de3no\n" + - "denti5f\n" + - "de3nu\n" + - "de1p\n" + - "de3pa\n" + - "depi4\n" + - "de2pu\n" + - "d3eq\n" + - "d4erh\n" + - "5derm\n" + - "dern5iz\n" + - "der5s\n" + - "des2\n" + - "d2es.\n" + - "de1sc\n" + - "de2s5o\n" + - "des3ti\n" + - "de3str\n" + - "de4su\n" + - "de1t\n" + - "de2to\n" + - "de1v\n" + - "dev3il\n" + - "4dey\n" + - "4d1f\n" + - "d4ga\n" + - "d3ge4t\n" + - "dg1i\n" + - "d2gy\n" + - "d1h2\n" + - "5di.\n" + - "1d4i3a\n" + - "dia5b\n" + - "di4cam\n" + - "d4ice\n" + - "3dict\n" + - "3did\n" + - "5di3en\n" + - "d1if\n" + - "di3ge\n" + - "di4lato\n" + - "d1in\n" + - "1dina\n" + - "3dine.\n" + - "5dini\n" + - "di5niz\n" + - "1dio\n" + - "dio5g\n" + - "di4pl\n" + - "dir2\n" + - "di1re\n" + - "dirt5i\n" + - "dis1\n" + - "5disi\n" + - "d4is3t\n" + - "d2iti\n" + - "1di1v\n" + - "d1j\n" + - "d5k2\n" + - "4d5la\n" + - "3dle.\n" + - "3dled\n" + - "3dles.\n" + - "4dless\n" + - "2d3lo\n" + - "4d5lu\n" + - "2dly\n" + - "d1m\n" + - "4d1n4\n" + - "1do\n" + - "3do.\n" + - "do5de\n" + - "5doe\n" + - "2d5of\n" + - "d4og\n" + - "do4la\n" + - "doli4\n" + - "do5lor\n" + - "dom5iz\n" + - "do3nat\n" + - "doni4\n" + - "doo3d\n" + - "dop4p\n" + - "d4or\n" + - "3dos\n" + - "4d5out\n" + - "do4v\n" + - "3dox\n" + - "d1p\n" + - "1dr\n" + - "drag5on\n" + - "4drai\n" + - "dre4\n" + - "drea5r\n" + - "5dren\n" + - "dri4b\n" + - "dril4\n" + - "dro4p\n" + - "4drow\n" + - "5drupli\n" + - "4dry\n" + - "2d1s2\n" + - "ds4p\n" + - "d4sw\n" + - "d4sy\n" + - "d2th\n" + - "1du\n" + - "d1u1a\n" + - "du2c\n" + - "d1uca\n" + - "duc5er\n" + - "4duct.\n" + - "4ducts\n" + - "du5el\n" + - "du4g\n" + - "d3ule\n" + - "dum4be\n" + - "du4n\n" + - "4dup\n" + - "du4pe\n" + - "d1v\n" + - "d1w\n" + - "d2y\n" + - "5dyn\n" + - "dy4se\n" + - "dys5p\n" + - "e1a4b\n" + - "e3act\n" + - "ead1\n" + - "ead5ie\n" + - "ea4ge\n" + - "ea5ger\n" + - "ea4l\n" + - "eal5er\n" + - "eal3ou\n" + - "eam3er\n" + - "e5and\n" + - "ear3a\n" + - "ear4c\n" + - "ear5es\n" + - "ear4ic\n" + - "ear4il\n" + - "ear5k\n" + - "ear2t\n" + - "eart3e\n" + - "ea5sp\n" + - "e3ass\n" + - "east3\n" + - "ea2t\n" + - "eat5en\n" + - "eath3i\n" + - "e5atif\n" + - "e4a3tu\n" + - "ea2v\n" + - "eav3en\n" + - "eav5i\n" + - "eav5o\n" + - "2e1b\n" + - "e4bel.\n" + - "e4bels\n" + - "e4ben\n" + - "e4bit\n" + - "e3br\n" + - "e4cad\n" + - "ecan5c\n" + - "ecca5\n" + - "e1ce\n" + - "ec5essa\n" + - "ec2i\n" + - "e4cib\n" + - "ec5ificat\n" + - "ec5ifie\n" + - "ec5ify\n" + - "ec3im\n" + - "eci4t\n" + - "e5cite\n" + - "e4clam\n" + - "e4clus\n" + - "e2col\n" + - "e4comm\n" + - "e4compe\n" + - "e4conc\n" + - "e2cor\n" + - "ec3ora\n" + - "eco5ro\n" + - "e1cr\n" + - "e4crem\n" + - "ec4tan\n" + - "ec4te\n" + - "e1cu\n" + - "e4cul\n" + - "ec3ula\n" + - "2e2da\n" + - "4ed3d\n" + - "e4d1er\n" + - "ede4s\n" + - "4edi\n" + - "e3dia\n" + - "ed3ib\n" + - "ed3ica\n" + - "ed3im\n" + - "ed1it\n" + - "edi5z\n" + - "4edo\n" + - "e4dol\n" + - "edon2\n" + - "e4dri\n" + - "e4dul\n" + - "ed5ulo\n" + - "ee2c\n" + - "eed3i\n" + - "ee2f\n" + - "eel3i\n" + - "ee4ly\n" + - "ee2m\n" + - "ee4na\n" + - "ee4p1\n" + - "ee2s4\n" + - "eest4\n" + - "ee4ty\n" + - "e5ex\n" + - "e1f\n" + - "e4f3ere\n" + - "1eff\n" + - "e4fic\n" + - "5efici\n" + - "efil4\n" + - "e3fine\n" + - "ef5i5nite\n" + - "3efit\n" + - "efor5es\n" + - "e4fuse.\n" + - "4egal\n" + - "eger4\n" + - "eg5ib\n" + - "eg4ic\n" + - "eg5ing\n" + - "e5git5\n" + - "eg5n\n" + - "e4go.\n" + - "e4gos\n" + - "eg1ul\n" + - "e5gur\n" + - "5egy\n" + - "e1h4\n" + - "eher4\n" + - "ei2\n" + - "e5ic\n" + - "ei5d\n" + - "eig2\n" + - "ei5gl\n" + - "e3imb\n" + - "e3inf\n" + - "e1ing\n" + - "e5inst\n" + - "eir4d\n" + - "eit3e\n" + - "ei3th\n" + - "e5ity\n" + - "e1j\n" + - "e4jud\n" + - "ej5udi\n" + - "eki4n\n" + - "ek4la\n" + - "e1la\n" + - "e4la.\n" + - "e4lac\n" + - "elan4d\n" + - "el5ativ\n" + - "e4law\n" + - "elaxa4\n" + - "e3lea\n" + - "el5ebra\n" + - "5elec\n" + - "e4led\n" + - "el3ega\n" + - "e5len\n" + - "e4l1er\n" + - "e1les\n" + - "el2f\n" + - "el2i\n" + - "e3libe\n" + - "e4l5ic.\n" + - "el3ica\n" + - "e3lier\n" + - "el5igib\n" + - "e5lim\n" + - "e4l3ing\n" + - "e3lio\n" + - "e2lis\n" + - "el5ish\n" + - "e3liv3\n" + - "4ella\n" + - "el4lab\n" + - "ello4\n" + - "e5loc\n" + - "el5og\n" + - "el3op.\n" + - "el2sh\n" + - "el4ta\n" + - "e5lud\n" + - "el5ug\n" + - "e4mac\n" + - "e4mag\n" + - "e5man\n" + - "em5ana\n" + - "em5b\n" + - "e1me\n" + - "e2mel\n" + - "e4met\n" + - "em3ica\n" + - "emi4e\n" + - "em5igra\n" + - "em1in2\n" + - "em5ine\n" + - "em3i3ni\n" + - "e4mis\n" + - "em5ish\n" + - "e5miss\n" + - "em3iz\n" + - "5emniz\n" + - "emo4g\n" + - "emoni5o\n" + - "em3pi\n" + - "e4mul\n" + - "em5ula\n" + - "emu3n\n" + - "e3my\n" + - "en5amo\n" + - "e4nant\n" + - "ench4er\n" + - "en3dic\n" + - "e5nea\n" + - "e5nee\n" + - "en3em\n" + - "en5ero\n" + - "en5esi\n" + - "en5est\n" + - "en3etr\n" + - "e3new\n" + - "en5ics\n" + - "e5nie\n" + - "e5nil\n" + - "e3nio\n" + - "en3ish\n" + - "en3it\n" + - "e5niu\n" + - "5eniz\n" + - "4enn\n" + - "4eno\n" + - "eno4g\n" + - "e4nos\n" + - "en3ov\n" + - "en4sw\n" + - "ent5age\n" + - "4enthes\n" + - "en3ua\n" + - "en5uf\n" + - "e3ny.\n" + - "4en3z\n" + - "e5of\n" + - "eo2g\n" + - "e4oi4\n" + - "e3ol\n" + - "eop3ar\n" + - "e1or\n" + - "eo3re\n" + - "eo5rol\n" + - "eos4\n" + - "e4ot\n" + - "eo4to\n" + - "e5out\n" + - "e5ow\n" + - "e2pa\n" + - "e3pai\n" + - "ep5anc\n" + - "e5pel\n" + - "e3pent\n" + - "ep5etitio\n" + - "ephe4\n" + - "e4pli\n" + - "e1po\n" + - "e4prec\n" + - "ep5reca\n" + - "e4pred\n" + - "ep3reh\n" + - "e3pro\n" + - "e4prob\n" + - "ep4sh\n" + - "ep5ti5b\n" + - "e4put\n" + - "ep5uta\n" + - "e1q\n" + - "equi3l\n" + - "e4q3ui3s\n" + - "er1a\n" + - "era4b\n" + - "4erand\n" + - "er3ar\n" + - "4erati.\n" + - "2erb\n" + - "er4bl\n" + - "er3ch\n" + - "er4che\n" + - "2ere.\n" + - "e3real\n" + - "ere5co\n" + - "ere3in\n" + - "er5el.\n" + - "er3emo\n" + - "er5ena\n" + - "er5ence\n" + - "4erene\n" + - "er3ent\n" + - "ere4q\n" + - "er5ess\n" + - "er3est\n" + - "eret4\n" + - "er1h\n" + - "er1i\n" + - "e1ria4\n" + - "5erick\n" + - "e3rien\n" + - "eri4er\n" + - "er3ine\n" + - "e1rio\n" + - "4erit\n" + - "er4iu\n" + - "eri4v\n" + - "e4riva\n" + - "er3m4\n" + - "er4nis\n" + - "4ernit\n" + - "5erniz\n" + - "er3no\n" + - "2ero\n" + - "er5ob\n" + - "e5roc\n" + - "ero4r\n" + - "er1ou\n" + - "er1s\n" + - "er3set\n" + - "ert3er\n" + - "4ertl\n" + - "er3tw\n" + - "4eru\n" + - "eru4t\n" + - "5erwau\n" + - "e1s4a\n" + - "e4sage.\n" + - "e4sages\n" + - "es2c\n" + - "e2sca\n" + - "es5can\n" + - "e3scr\n" + - "es5cu\n" + - "e1s2e\n" + - "e2sec\n" + - "es5ecr\n" + - "es5enc\n" + - "e4sert.\n" + - "e4serts\n" + - "e4serva\n" + - "4esh\n" + - "e3sha\n" + - "esh5en\n" + - "e1si\n" + - "e2sic\n" + - "e2sid\n" + - "es5iden\n" + - "es5igna\n" + - "e2s5im\n" + - "es4i4n\n" + - "esis4te\n" + - "esi4u\n" + - "e5skin\n" + - "es4mi\n" + - "e2sol\n" + - "es3olu\n" + - "e2son\n" + - "es5ona\n" + - "e1sp\n" + - "es3per\n" + - "es5pira\n" + - "es4pre\n" + - "2ess\n" + - "es4si4b\n" + - "estan4\n" + - "es3tig\n" + - "es5tim\n" + - "4es2to\n" + - "e3ston\n" + - "2estr\n" + - "e5stro\n" + - "estruc5\n" + - "e2sur\n" + - "es5urr\n" + - "es4w\n" + - "eta4b\n" + - "eten4d\n" + - "e3teo\n" + - "ethod3\n" + - "et1ic\n" + - "e5tide\n" + - "etin4\n" + - "eti4no\n" + - "e5tir\n" + - "e5titio\n" + - "et5itiv\n" + - "4etn\n" + - "et5ona\n" + - "e3tra\n" + - "e3tre\n" + - "et3ric\n" + - "et5rif\n" + - "et3rog\n" + - "et5ros\n" + - "et3ua\n" + - "et5ym\n" + - "et5z\n" + - "4eu\n" + - "e5un\n" + - "e3up\n" + - "eu3ro\n" + - "eus4\n" + - "eute4\n" + - "euti5l\n" + - "eu5tr\n" + - "eva2p5\n" + - "e2vas\n" + - "ev5ast\n" + - "e5vea\n" + - "ev3ell\n" + - "evel3o\n" + - "e5veng\n" + - "even4i\n" + - "ev1er\n" + - "e5verb\n" + - "e1vi\n" + - "ev3id\n" + - "evi4l\n" + - "e4vin\n" + - "evi4v\n" + - "e5voc\n" + - "e5vu\n" + - "e1wa\n" + - "e4wag\n" + - "e5wee\n" + - "e3wh\n" + - "ewil5\n" + - "ew3ing\n" + - "e3wit\n" + - "1exp\n" + - "5eyc\n" + - "5eye.\n" + - "eys4\n" + - "1fa\n" + - "fa3bl\n" + - "fab3r\n" + - "fa4ce\n" + - "4fag\n" + - "fain4\n" + - "fall5e\n" + - "4fa4ma\n" + - "fam5is\n" + - "5far\n" + - "far5th\n" + - "fa3ta\n" + - "fa3the\n" + - "4fato\n" + - "fault5\n" + - "4f5b\n" + - "4fd\n" + - "4fe.\n" + - "feas4\n" + - "feath3\n" + - "fe4b\n" + - "4feca\n" + - "5fect\n" + - "2fed\n" + - "fe3li\n" + - "fe4mo\n" + - "fen2d\n" + - "fend5e\n" + - "fer1\n" + - "5ferr\n" + - "fev4\n" + - "4f1f\n" + - "f4fes\n" + - "f4fie\n" + - "f5fin.\n" + - "f2f5is\n" + - "f4fly\n" + - "f2fy\n" + - "4fh\n" + - "1fi\n" + - "fi3a\n" + - "2f3ic.\n" + - "4f3ical\n" + - "f3ican\n" + - "4ficate\n" + - "f3icen\n" + - "fi3cer\n" + - "fic4i\n" + - "5ficia\n" + - "5ficie\n" + - "4fics\n" + - "fi3cu\n" + - "fi5del\n" + - "fight5\n" + - "fil5i\n" + - "fill5in\n" + - "4fily\n" + - "2fin\n" + - "5fina\n" + - "fin2d5\n" + - "fi2ne\n" + - "f1in3g\n" + - "fin4n\n" + - "fis4ti\n" + - "f4l2\n" + - "f5less\n" + - "flin4\n" + - "flo3re\n" + - "f2ly5\n" + - "4fm\n" + - "4fn\n" + - "1fo\n" + - "5fon\n" + - "fon4de\n" + - "fon4t\n" + - "fo2r\n" + - "fo5rat\n" + - "for5ay\n" + - "fore5t\n" + - "for4i\n" + - "fort5a\n" + - "fos5\n" + - "4f5p\n" + - "fra4t\n" + - "f5rea\n" + - "fres5c\n" + - "fri2\n" + - "fril4\n" + - "frol5\n" + - "2f3s\n" + - "2ft\n" + - "f4to\n" + - "f2ty\n" + - "3fu\n" + - "fu5el\n" + - "4fug\n" + - "fu4min\n" + - "fu5ne\n" + - "fu3ri\n" + - "fusi4\n" + - "fus4s\n" + - "4futa\n" + - "1fy\n" + - "1ga\n" + - "gaf4\n" + - "5gal.\n" + - "3gali\n" + - "ga3lo\n" + - "2gam\n" + - "ga5met\n" + - "g5amo\n" + - "gan5is\n" + - "ga3niz\n" + - "gani5za\n" + - "4gano\n" + - "gar5n4\n" + - "gass4\n" + - "gath3\n" + - "4gativ\n" + - "4gaz\n" + - "g3b\n" + - "gd4\n" + - "2ge.\n" + - "2ged\n" + - "geez4\n" + - "gel4in\n" + - "ge5lis\n" + - "ge5liz\n" + - "4gely\n" + - "1gen\n" + - "ge4nat\n" + - "ge5niz\n" + - "4geno\n" + - "4geny\n" + - "1geo\n" + - "ge3om\n" + - "g4ery\n" + - "5gesi\n" + - "geth5\n" + - "4geto\n" + - "ge4ty\n" + - "ge4v\n" + - "4g1g2\n" + - "g2ge\n" + - "g3ger\n" + - "gglu5\n" + - "ggo4\n" + - "gh3in\n" + - "gh5out\n" + - "gh4to\n" + - "5gi.\n" + - "1gi4a\n" + - "gia5r\n" + - "g1ic\n" + - "5gicia\n" + - "g4ico\n" + - "gien5\n" + - "5gies.\n" + - "gil4\n" + - "g3imen\n" + - "3g4in.\n" + - "gin5ge\n" + - "5g4ins\n" + - "5gio\n" + - "3gir\n" + - "gir4l\n" + - "g3isl\n" + - "gi4u\n" + - "5giv\n" + - "3giz\n" + - "gl2\n" + - "gla4\n" + - "glad5i\n" + - "5glas\n" + - "1gle\n" + - "gli4b\n" + - "g3lig\n" + - "3glo\n" + - "glo3r\n" + - "g1m\n" + - "g4my\n" + - "gn4a\n" + - "g4na.\n" + - "gnet4t\n" + - "g1ni\n" + - "g2nin\n" + - "g4nio\n" + - "g1no\n" + - "g4non\n" + - "1go\n" + - "3go.\n" + - "gob5\n" + - "5goe\n" + - "3g4o4g\n" + - "go3is\n" + - "gon2\n" + - "4g3o3na\n" + - "gondo5\n" + - "go3ni\n" + - "5goo\n" + - "go5riz\n" + - "gor5ou\n" + - "5gos.\n" + - "gov1\n" + - "g3p\n" + - "1gr\n" + - "4grada\n" + - "g4rai\n" + - "gran2\n" + - "5graph.\n" + - "g5rapher\n" + - "5graphic\n" + - "4graphy\n" + - "4gray\n" + - "gre4n\n" + - "4gress.\n" + - "4grit\n" + - "g4ro\n" + - "gruf4\n" + - "gs2\n" + - "g5ste\n" + - "gth3\n" + - "gu4a\n" + - "3guard\n" + - "2gue\n" + - "5gui5t\n" + - "3gun\n" + - "3gus\n" + - "4gu4t\n" + - "g3w\n" + - "1gy\n" + - "2g5y3n\n" + - "gy5ra\n" + - "h3ab4l\n" + - "hach4\n" + - "hae4m\n" + - "hae4t\n" + - "h5agu\n" + - "ha3la\n" + - "hala3m\n" + - "ha4m\n" + - "han4ci\n" + - "han4cy\n" + - "5hand.\n" + - "han4g\n" + - "hang5er\n" + - "hang5o\n" + - "h5a5niz\n" + - "han4k\n" + - "han4te\n" + - "hap3l\n" + - "hap5t\n" + - "ha3ran\n" + - "ha5ras\n" + - "har2d\n" + - "hard3e\n" + - "har4le\n" + - "harp5en\n" + - "har5ter\n" + - "has5s\n" + - "haun4\n" + - "5haz\n" + - "haz3a\n" + - "h1b\n" + - "1head\n" + - "3hear\n" + - "he4can\n" + - "h5ecat\n" + - "h4ed\n" + - "he5do5\n" + - "he3l4i\n" + - "hel4lis\n" + - "hel4ly\n" + - "h5elo\n" + - "hem4p\n" + - "he2n\n" + - "hena4\n" + - "hen5at\n" + - "heo5r\n" + - "hep5\n" + - "h4era\n" + - "hera3p\n" + - "her4ba\n" + - "here5a\n" + - "h3ern\n" + - "h5erou\n" + - "h3ery\n" + - "h1es\n" + - "he2s5p\n" + - "he4t\n" + - "het4ed\n" + - "heu4\n" + - "h1f\n" + - "h1h\n" + - "hi5an\n" + - "hi4co\n" + - "high5\n" + - "h4il2\n" + - "himer4\n" + - "h4ina\n" + - "hion4e\n" + - "hi4p\n" + - "hir4l\n" + - "hi3ro\n" + - "hir4p\n" + - "hir4r\n" + - "his3el\n" + - "his4s\n" + - "hith5er\n" + - "hi2v\n" + - "4hk\n" + - "4h1l4\n" + - "hlan4\n" + - "h2lo\n" + - "hlo3ri\n" + - "4h1m\n" + - "hmet4\n" + - "2h1n\n" + - "h5odiz\n" + - "h5ods\n" + - "ho4g\n" + - "hoge4\n" + - "hol5ar\n" + - "3hol4e\n" + - "ho4ma\n" + - "home3\n" + - "hon4a\n" + - "ho5ny\n" + - "3hood\n" + - "hoon4\n" + - "hor5at\n" + - "ho5ris\n" + - "hort3e\n" + - "ho5ru\n" + - "hos4e\n" + - "ho5sen\n" + - "hos1p\n" + - "1hous\n" + - "house3\n" + - "hov5el\n" + - "4h5p\n" + - "4hr4\n" + - "hree5\n" + - "hro5niz\n" + - "hro3po\n" + - "4h1s2\n" + - "h4sh\n" + - "h4tar\n" + - "ht1en\n" + - "ht5es\n" + - "h4ty\n" + - "hu4g\n" + - "hu4min\n" + - "hun5ke\n" + - "hun4t\n" + - "hus3t4\n" + - "hu4t\n" + - "h1w\n" + - "h4wart\n" + - "hy3pe\n" + - "hy3ph\n" + - "hy2s\n" + - "2i1a\n" + - "i2al\n" + - "iam4\n" + - "iam5ete\n" + - "i2an\n" + - "4ianc\n" + - "ian3i\n" + - "4ian4t\n" + - "ia5pe\n" + - "iass4\n" + - "i4ativ\n" + - "ia4tric\n" + - "i4atu\n" + - "ibe4\n" + - "ib3era\n" + - "ib5ert\n" + - "ib5ia\n" + - "ib3in\n" + - "ib5it.\n" + - "ib5ite\n" + - "i1bl\n" + - "ib3li\n" + - "i5bo\n" + - "i1br\n" + - "i2b5ri\n" + - "i5bun\n" + - "4icam\n" + - "5icap\n" + - "4icar\n" + - "i4car.\n" + - "i4cara\n" + - "icas5\n" + - "i4cay\n" + - "iccu4\n" + - "4iceo\n" + - "4ich\n" + - "2ici\n" + - "i5cid\n" + - "ic5ina\n" + - "i2cip\n" + - "ic3ipa\n" + - "i4cly\n" + - "i2c5oc\n" + - "4i1cr\n" + - "5icra\n" + - "i4cry\n" + - "ic4te\n" + - "ictu2\n" + - "ic4t3ua\n" + - "ic3ula\n" + - "ic4um\n" + - "ic5uo\n" + - "i3cur\n" + - "2id\n" + - "i4dai\n" + - "id5anc\n" + - "id5d\n" + - "ide3al\n" + - "ide4s\n" + - "i2di\n" + - "id5ian\n" + - "idi4ar\n" + - "i5die\n" + - "id3io\n" + - "idi5ou\n" + - "id1it\n" + - "id5iu\n" + - "i3dle\n" + - "i4dom\n" + - "id3ow\n" + - "i4dr\n" + - "i2du\n" + - "id5uo\n" + - "2ie4\n" + - "ied4e\n" + - "5ie5ga\n" + - "ield3\n" + - "ien5a4\n" + - "ien4e\n" + - "i5enn\n" + - "i3enti\n" + - "i1er.\n" + - "i3esc\n" + - "i1est\n" + - "i3et\n" + - "4if.\n" + - "if5ero\n" + - "iff5en\n" + - "if4fr\n" + - "4ific.\n" + - "i3fie\n" + - "i3fl\n" + - "4ift\n" + - "2ig\n" + - "iga5b\n" + - "ig3era\n" + - "ight3i\n" + - "4igi\n" + - "i3gib\n" + - "ig3il\n" + - "ig3in\n" + - "ig3it\n" + - "i4g4l\n" + - "i2go\n" + - "ig3or\n" + - "ig5ot\n" + - "i5gre\n" + - "igu5i\n" + - "ig1ur\n" + - "i3h\n" + - "4i5i4\n" + - "i3j\n" + - "4ik\n" + - "i1la\n" + - "il3a4b\n" + - "i4lade\n" + - "i2l5am\n" + - "ila5ra\n" + - "i3leg\n" + - "il1er\n" + - "ilev4\n" + - "il5f\n" + - "il1i\n" + - "il3ia\n" + - "il2ib\n" + - "il3io\n" + - "il4ist\n" + - "2ilit\n" + - "il2iz\n" + - "ill5ab\n" + - "4iln\n" + - "il3oq\n" + - "il4ty\n" + - "il5ur\n" + - "il3v\n" + - "i4mag\n" + - "im3age\n" + - "ima5ry\n" + - "imenta5r\n" + - "4imet\n" + - "im1i\n" + - "im5ida\n" + - "imi5le\n" + - "i5mini\n" + - "4imit\n" + - "im4ni\n" + - "i3mon\n" + - "i2mu\n" + - "im3ula\n" + - "2in.\n" + - "i4n3au\n" + - "4inav\n" + - "incel4\n" + - "in3cer\n" + - "4ind\n" + - "in5dling\n" + - "2ine\n" + - "i3nee\n" + - "iner4ar\n" + - "i5ness\n" + - "4inga\n" + - "4inge\n" + - "in5gen\n" + - "4ingi\n" + - "in5gling\n" + - "4ingo\n" + - "4ingu\n" + - "2ini\n" + - "i5ni.\n" + - "i4nia\n" + - "in3io\n" + - "in1is\n" + - "i5nite.\n" + - "5initio\n" + - "in3ity\n" + - "4ink\n" + - "4inl\n" + - "2inn\n" + - "2i1no\n" + - "i4no4c\n" + - "ino4s\n" + - "i4not\n" + - "2ins\n" + - "in3se\n" + - "insur5a\n" + - "2int.\n" + - "2in4th\n" + - "in1u\n" + - "i5nus\n" + - "4iny\n" + - "2io\n" + - "4io.\n" + - "ioge4\n" + - "io2gr\n" + - "i1ol\n" + - "io4m\n" + - "ion3at\n" + - "ion4ery\n" + - "ion3i\n" + - "io5ph\n" + - "ior3i\n" + - "i4os\n" + - "io5th\n" + - "i5oti\n" + - "io4to\n" + - "i4our\n" + - "2ip\n" + - "ipe4\n" + - "iphras4\n" + - "ip3i\n" + - "ip4ic\n" + - "ip4re4\n" + - "ip3ul\n" + - "i3qua\n" + - "iq5uef\n" + - "iq3uid\n" + - "iq3ui3t\n" + - "4ir\n" + - "i1ra\n" + - "ira4b\n" + - "i4rac\n" + - "ird5e\n" + - "ire4de\n" + - "i4ref\n" + - "i4rel4\n" + - "i4res\n" + - "ir5gi\n" + - "ir1i\n" + - "iri5de\n" + - "ir4is\n" + - "iri3tu\n" + - "5i5r2iz\n" + - "ir4min\n" + - "iro4g\n" + - "5iron.\n" + - "ir5ul\n" + - "2is.\n" + - "is5ag\n" + - "is3ar\n" + - "isas5\n" + - "2is1c\n" + - "is3ch\n" + - "4ise\n" + - "is3er\n" + - "3isf\n" + - "is5han\n" + - "is3hon\n" + - "ish5op\n" + - "is3ib\n" + - "isi4d\n" + - "i5sis\n" + - "is5itiv\n" + - "4is4k\n" + - "islan4\n" + - "4isms\n" + - "i2so\n" + - "iso5mer\n" + - "is1p\n" + - "is2pi\n" + - "is4py\n" + - "4is1s\n" + - "is4sal\n" + - "issen4\n" + - "is4ses\n" + - "is4ta.\n" + - "is1te\n" + - "is1ti\n" + - "ist4ly\n" + - "4istral\n" + - "i2su\n" + - "is5us\n" + - "4ita.\n" + - "ita4bi\n" + - "i4tag\n" + - "4ita5m\n" + - "i3tan\n" + - "i3tat\n" + - "2ite\n" + - "it3era\n" + - "i5teri\n" + - "it4es\n" + - "2ith\n" + - "i1ti\n" + - "4itia\n" + - "4i2tic\n" + - "it3ica\n" + - "5i5tick\n" + - "it3ig\n" + - "it5ill\n" + - "i2tim\n" + - "2itio\n" + - "4itis\n" + - "i4tism\n" + - "i2t5o5m\n" + - "4iton\n" + - "i4tram\n" + - "it5ry\n" + - "4itt\n" + - "it3uat\n" + - "i5tud\n" + - "it3ul\n" + - "4itz.\n" + - "i1u\n" + - "2iv\n" + - "iv3ell\n" + - "iv3en.\n" + - "i4v3er.\n" + - "i4vers.\n" + - "iv5il.\n" + - "iv5io\n" + - "iv1it\n" + - "i5vore\n" + - "iv3o3ro\n" + - "i4v3ot\n" + - "4i5w\n" + - "ix4o\n" + - "4iy\n" + - "4izar\n" + - "izi4\n" + - "5izont\n" + - "5ja\n" + - "jac4q\n" + - "ja4p\n" + - "1je\n" + - "jer5s\n" + - "4jestie\n" + - "4jesty\n" + - "jew3\n" + - "jo4p\n" + - "5judg\n" + - "3ka.\n" + - "k3ab\n" + - "k5ag\n" + - "kais4\n" + - "kal4\n" + - "k1b\n" + - "k2ed\n" + - "1kee\n" + - "ke4g\n" + - "ke5li\n" + - "k3en4d\n" + - "k1er\n" + - "kes4\n" + - "k3est.\n" + - "ke4ty\n" + - "k3f\n" + - "kh4\n" + - "k1i\n" + - "5ki.\n" + - "5k2ic\n" + - "k4ill\n" + - "kilo5\n" + - "k4im\n" + - "k4in.\n" + - "kin4de\n" + - "k5iness\n" + - "kin4g\n" + - "ki4p\n" + - "kis4\n" + - "k5ish\n" + - "kk4\n" + - "k1l\n" + - "4kley\n" + - "4kly\n" + - "k1m\n" + - "k5nes\n" + - "1k2no\n" + - "ko5r\n" + - "kosh4\n" + - "k3ou\n" + - "kro5n\n" + - "4k1s2\n" + - "k4sc\n" + - "ks4l\n" + - "k4sy\n" + - "k5t\n" + - "k1w\n" + - "lab3ic\n" + - "l4abo\n" + - "laci4\n" + - "l4ade\n" + - "la3dy\n" + - "lag4n\n" + - "lam3o\n" + - "3land\n" + - "lan4dl\n" + - "lan5et\n" + - "lan4te\n" + - "lar4g\n" + - "lar3i\n" + - "las4e\n" + - "la5tan\n" + - "4lateli\n" + - "4lativ\n" + - "4lav\n" + - "la4v4a\n" + - "2l1b\n" + - "lbin4\n" + - "4l1c2\n" + - "lce4\n" + - "l3ci\n" + - "2ld\n" + - "l2de\n" + - "ld4ere\n" + - "ld4eri\n" + - "ldi4\n" + - "ld5is\n" + - "l3dr\n" + - "l4dri\n" + - "le2a\n" + - "le4bi\n" + - "left5\n" + - "5leg.\n" + - "5legg\n" + - "le4mat\n" + - "lem5atic\n" + - "4len.\n" + - "3lenc\n" + - "5lene.\n" + - "1lent\n" + - "le3ph\n" + - "le4pr\n" + - "lera5b\n" + - "ler4e\n" + - "3lerg\n" + - "3l4eri\n" + - "l4ero\n" + - "les2\n" + - "le5sco\n" + - "5lesq\n" + - "3less\n" + - "5less.\n" + - "l3eva\n" + - "lev4er.\n" + - "lev4era\n" + - "lev4ers\n" + - "3ley\n" + - "4leye\n" + - "2lf\n" + - "l5fr\n" + - "4l1g4\n" + - "l5ga\n" + - "lgar3\n" + - "l4ges\n" + - "lgo3\n" + - "2l3h\n" + - "li4ag\n" + - "li2am\n" + - "liar5iz\n" + - "li4as\n" + - "li4ato\n" + - "li5bi\n" + - "5licio\n" + - "li4cor\n" + - "4lics\n" + - "4lict.\n" + - "l4icu\n" + - "l3icy\n" + - "l3ida\n" + - "lid5er\n" + - "3lidi\n" + - "lif3er\n" + - "l4iff\n" + - "li4fl\n" + - "5ligate\n" + - "3ligh\n" + - "li4gra\n" + - "3lik\n" + - "4l4i4l\n" + - "lim4bl\n" + - "lim3i\n" + - "li4mo\n" + - "l4im4p\n" + - "l4ina\n" + - "1l4ine\n" + - "lin3ea\n" + - "lin3i\n" + - "link5er\n" + - "li5og\n" + - "4l4iq\n" + - "lis4p\n" + - "l1it\n" + - "l2it.\n" + - "5litica\n" + - "l5i5tics\n" + - "liv3er\n" + - "l1iz\n" + - "4lj\n" + - "lka3\n" + - "l3kal\n" + - "lka4t\n" + - "l1l\n" + - "l4law\n" + - "l2le\n" + - "l5lea\n" + - "l3lec\n" + - "l3leg\n" + - "l3lel\n" + - "l3le4n\n" + - "l3le4t\n" + - "ll2i\n" + - "l2lin4\n" + - "l5lina\n" + - "ll4o\n" + - "lloqui5\n" + - "ll5out\n" + - "l5low\n" + - "2lm\n" + - "l5met\n" + - "lm3ing\n" + - "l4mod\n" + - "lmon4\n" + - "2l1n2\n" + - "3lo.\n" + - "lob5al\n" + - "lo4ci\n" + - "4lof\n" + - "3logic\n" + - "l5ogo\n" + - "3logu\n" + - "lom3er\n" + - "5long\n" + - "lon4i\n" + - "l3o3niz\n" + - "lood5\n" + - "5lope.\n" + - "lop3i\n" + - "l3opm\n" + - "lora4\n" + - "lo4rato\n" + - "lo5rie\n" + - "lor5ou\n" + - "5los.\n" + - "los5et\n" + - "5losophiz\n" + - "5losophy\n" + - "los4t\n" + - "lo4ta\n" + - "loun5d\n" + - "2lout\n" + - "4lov\n" + - "2lp\n" + - "lpa5b\n" + - "l3pha\n" + - "l5phi\n" + - "lp5ing\n" + - "l3pit\n" + - "l4pl\n" + - "l5pr\n" + - "4l1r\n" + - "2l1s2\n" + - "l4sc\n" + - "l2se\n" + - "l4sie\n" + - "4lt\n" + - "lt5ag\n" + - "ltane5\n" + - "l1te\n" + - "lten4\n" + - "ltera4\n" + - "lth3i\n" + - "l5ties.\n" + - "ltis4\n" + - "l1tr\n" + - "ltu2\n" + - "ltur3a\n" + - "lu5a\n" + - "lu3br\n" + - "luch4\n" + - "lu3ci\n" + - "lu3en\n" + - "luf4\n" + - "lu5id\n" + - "lu4ma\n" + - "5lumi\n" + - "l5umn.\n" + - "5lumnia\n" + - "lu3o\n" + - "luo3r\n" + - "4lup\n" + - "luss4\n" + - "lus3te\n" + - "1lut\n" + - "l5ven\n" + - "l5vet4\n" + - "2l1w\n" + - "1ly\n" + - "4lya\n" + - "4lyb\n" + - "ly5me\n" + - "ly3no\n" + - "2lys4\n" + - "l5yse\n" + - "1ma\n" + - "2mab\n" + - "ma2ca\n" + - "ma5chine\n" + - "ma4cl\n" + - "mag5in\n" + - "5magn\n" + - "2mah\n" + - "maid5\n" + - "4mald\n" + - "ma3lig\n" + - "ma5lin\n" + - "mal4li\n" + - "mal4ty\n" + - "5mania\n" + - "man5is\n" + - "man3iz\n" + - "4map\n" + - "ma5rine.\n" + - "ma5riz\n" + - "mar4ly\n" + - "mar3v\n" + - "ma5sce\n" + - "mas4e\n" + - "mas1t\n" + - "5mate\n" + - "math3\n" + - "ma3tis\n" + - "4matiza\n" + - "4m1b\n" + - "mba4t5\n" + - "m5bil\n" + - "m4b3ing\n" + - "mbi4v\n" + - "4m5c\n" + - "4me.\n" + - "2med\n" + - "4med.\n" + - "5media\n" + - "me3die\n" + - "m5e5dy\n" + - "me2g\n" + - "mel5on\n" + - "mel4t\n" + - "me2m\n" + - "mem1o3\n" + - "1men\n" + - "men4a\n" + - "men5ac\n" + - "men4de\n" + - "4mene\n" + - "men4i\n" + - "mens4\n" + - "mensu5\n" + - "3ment\n" + - "men4te\n" + - "me5on\n" + - "m5ersa\n" + - "2mes\n" + - "3mesti\n" + - "me4ta\n" + - "met3al\n" + - "me1te\n" + - "me5thi\n" + - "m4etr\n" + - "5metric\n" + - "me5trie\n" + - "me3try\n" + - "me4v\n" + - "4m1f\n" + - "2mh\n" + - "5mi.\n" + - "mi3a\n" + - "mid4a\n" + - "mid4g\n" + - "mig4\n" + - "3milia\n" + - "m5i5lie\n" + - "m4ill\n" + - "min4a\n" + - "3mind\n" + - "m5inee\n" + - "m4ingl\n" + - "min5gli\n" + - "m5ingly\n" + - "min4t\n" + - "m4inu\n" + - "miot4\n" + - "m2is\n" + - "mis4er.\n" + - "mis5l\n" + - "mis4ti\n" + - "m5istry\n" + - "4mith\n" + - "m2iz\n" + - "4mk\n" + - "4m1l\n" + - "m1m\n" + - "mma5ry\n" + - "4m1n\n" + - "mn4a\n" + - "m4nin\n" + - "mn4o\n" + - "1mo\n" + - "4mocr\n" + - "5mocratiz\n" + - "mo2d1\n" + - "mo4go\n" + - "mois2\n" + - "moi5se\n" + - "4mok\n" + - "mo5lest\n" + - "mo3me\n" + - "mon5et\n" + - "mon5ge\n" + - "moni3a\n" + - "mon4ism\n" + - "mon4ist\n" + - "mo3niz\n" + - "monol4\n" + - "mo3ny.\n" + - "mo2r\n" + - "4mora.\n" + - "mos2\n" + - "mo5sey\n" + - "mo3sp\n" + - "moth3\n" + - "m5ouf\n" + - "3mous\n" + - "mo2v\n" + - "4m1p\n" + - "mpara5\n" + - "mpa5rab\n" + - "mpar5i\n" + - "m3pet\n" + - "mphas4\n" + - "m2pi\n" + - "mpi4a\n" + - "mp5ies\n" + - "m4p1in\n" + - "m5pir\n" + - "mp5is\n" + - "mpo3ri\n" + - "mpos5ite\n" + - "m4pous\n" + - "mpov5\n" + - "mp4tr\n" + - "m2py\n" + - "4m3r\n" + - "4m1s2\n" + - "m4sh\n" + - "m5si\n" + - "4mt\n" + - "1mu\n" + - "mula5r4\n" + - "5mult\n" + - "multi3\n" + - "3mum\n" + - "mun2\n" + - "4mup\n" + - "mu4u\n" + - "4mw\n" + - "1na\n" + - "2n1a2b\n" + - "n4abu\n" + - "4nac.\n" + - "na4ca\n" + - "n5act\n" + - "nag5er.\n" + - "nak4\n" + - "na4li\n" + - "na5lia\n" + - "4nalt\n" + - "na5mit\n" + - "n2an\n" + - "nanci4\n" + - "nan4it\n" + - "nank4\n" + - "nar3c\n" + - "4nare\n" + - "nar3i\n" + - "nar4l\n" + - "n5arm\n" + - "n4as\n" + - "nas4c\n" + - "nas5ti\n" + - "n2at\n" + - "na3tal\n" + - "nato5miz\n" + - "n2au\n" + - "nau3se\n" + - "3naut\n" + - "nav4e\n" + - "4n1b4\n" + - "ncar5\n" + - "n4ces.\n" + - "n3cha\n" + - "n5cheo\n" + - "n5chil\n" + - "n3chis\n" + - "nc1in\n" + - "nc4it\n" + - "ncour5a\n" + - "n1cr\n" + - "n1cu\n" + - "n4dai\n" + - "n5dan\n" + - "n1de\n" + - "nd5est.\n" + - "ndi4b\n" + - "n5d2if\n" + - "n1dit\n" + - "n3diz\n" + - "n5duc\n" + - "ndu4r\n" + - "nd2we\n" + - "2ne.\n" + - "n3ear\n" + - "ne2b\n" + - "neb3u\n" + - "ne2c\n" + - "5neck\n" + - "2ned\n" + - "ne4gat\n" + - "neg5ativ\n" + - "5nege\n" + - "ne4la\n" + - "nel5iz\n" + - "ne5mi\n" + - "ne4mo\n" + - "1nen\n" + - "4nene\n" + - "3neo\n" + - "ne4po\n" + - "ne2q\n" + - "n1er\n" + - "nera5b\n" + - "n4erar\n" + - "n2ere\n" + - "n4er5i\n" + - "ner4r\n" + - "1nes\n" + - "2nes.\n" + - "4nesp\n" + - "2nest\n" + - "4nesw\n" + - "3netic\n" + - "ne4v\n" + - "n5eve\n" + - "ne4w\n" + - "n3f\n" + - "n4gab\n" + - "n3gel\n" + - "nge4n4e\n" + - "n5gere\n" + - "n3geri\n" + - "ng5ha\n" + - "n3gib\n" + - "ng1in\n" + - "n5git\n" + - "n4gla\n" + - "ngov4\n" + - "ng5sh\n" + - "n1gu\n" + - "n4gum\n" + - "n2gy\n" + - "4n1h4\n" + - "nha4\n" + - "nhab3\n" + - "nhe4\n" + - "3n4ia\n" + - "ni3an\n" + - "ni4ap\n" + - "ni3ba\n" + - "ni4bl\n" + - "ni4d\n" + - "ni5di\n" + - "ni4er\n" + - "ni2fi\n" + - "ni5ficat\n" + - "n5igr\n" + - "nik4\n" + - "n1im\n" + - "ni3miz\n" + - "n1in\n" + - "5nine.\n" + - "nin4g\n" + - "ni4o\n" + - "5nis.\n" + - "nis4ta\n" + - "n2it\n" + - "n4ith\n" + - "3nitio\n" + - "n3itor\n" + - "ni3tr\n" + - "n1j\n" + - "4nk2\n" + - "n5kero\n" + - "n3ket\n" + - "nk3in\n" + - "n1kl\n" + - "4n1l\n" + - "n5m\n" + - "nme4\n" + - "nmet4\n" + - "4n1n2\n" + - "nne4\n" + - "nni3al\n" + - "nni4v\n" + - "nob4l\n" + - "no3ble\n" + - "n5ocl\n" + - "4n3o2d\n" + - "3noe\n" + - "4nog\n" + - "noge4\n" + - "nois5i\n" + - "no5l4i\n" + - "5nologis\n" + - "3nomic\n" + - "n5o5miz\n" + - "no4mo\n" + - "no3my\n" + - "no4n\n" + - "non4ag\n" + - "non5i\n" + - "n5oniz\n" + - "4nop\n" + - "5nop5o5li\n" + - "nor5ab\n" + - "no4rary\n" + - "4nosc\n" + - "nos4e\n" + - "nos5t\n" + - "no5ta\n" + - "1nou\n" + - "3noun\n" + - "nov3el3\n" + - "nowl3\n" + - "n1p4\n" + - "npi4\n" + - "npre4c\n" + - "n1q\n" + - "n1r\n" + - "nru4\n" + - "2n1s2\n" + - "ns5ab\n" + - "nsati4\n" + - "ns4c\n" + - "n2se\n" + - "n4s3es\n" + - "nsid1\n" + - "nsig4\n" + - "n2sl\n" + - "ns3m\n" + - "n4soc\n" + - "ns4pe\n" + - "n5spi\n" + - "nsta5bl\n" + - "n1t\n" + - "nta4b\n" + - "nter3s\n" + - "nt2i\n" + - "n5tib\n" + - "nti4er\n" + - "nti2f\n" + - "n3tine\n" + - "n4t3ing\n" + - "nti4p\n" + - "ntrol5li\n" + - "nt4s\n" + - "ntu3me\n" + - "nu1a\n" + - "nu4d\n" + - "nu5en\n" + - "nuf4fe\n" + - "n3uin\n" + - "3nu3it\n" + - "n4um\n" + - "nu1me\n" + - "n5umi\n" + - "3nu4n\n" + - "n3uo\n" + - "nu3tr\n" + - "n1v2\n" + - "n1w4\n" + - "nym4\n" + - "nyp4\n" + - "4nz\n" + - "n3za\n" + - "4oa\n" + - "oad3\n" + - "o5a5les\n" + - "oard3\n" + - "oas4e\n" + - "oast5e\n" + - "oat5i\n" + - "ob3a3b\n" + - "o5bar\n" + - "obe4l\n" + - "o1bi\n" + - "o2bin\n" + - "ob5ing\n" + - "o3br\n" + - "ob3ul\n" + - "o1ce\n" + - "och4\n" + - "o3chet\n" + - "ocif3\n" + - "o4cil\n" + - "o4clam\n" + - "o4cod\n" + - "oc3rac\n" + - "oc5ratiz\n" + - "ocre3\n" + - "5ocrit\n" + - "octor5a\n" + - "oc3ula\n" + - "o5cure\n" + - "od5ded\n" + - "od3ic\n" + - "odi3o\n" + - "o2do4\n" + - "odor3\n" + - "od5uct.\n" + - "od5ucts\n" + - "o4el\n" + - "o5eng\n" + - "o3er\n" + - "oe4ta\n" + - "o3ev\n" + - "o2fi\n" + - "of5ite\n" + - "ofit4t\n" + - "o2g5a5r\n" + - "og5ativ\n" + - "o4gato\n" + - "o1ge\n" + - "o5gene\n" + - "o5geo\n" + - "o4ger\n" + - "o3gie\n" + - "1o1gis\n" + - "og3it\n" + - "o4gl\n" + - "o5g2ly\n" + - "3ogniz\n" + - "o4gro\n" + - "ogu5i\n" + - "1ogy\n" + - "2ogyn\n" + - "o1h2\n" + - "ohab5\n" + - "oi2\n" + - "oic3es\n" + - "oi3der\n" + - "oiff4\n" + - "oig4\n" + - "oi5let\n" + - "o3ing\n" + - "oint5er\n" + - "o5ism\n" + - "oi5son\n" + - "oist5en\n" + - "oi3ter\n" + - "o5j\n" + - "2ok\n" + - "o3ken\n" + - "ok5ie\n" + - "o1la\n" + - "o4lan\n" + - "olass4\n" + - "ol2d\n" + - "old1e\n" + - "ol3er\n" + - "o3lesc\n" + - "o3let\n" + - "ol4fi\n" + - "ol2i\n" + - "o3lia\n" + - "o3lice\n" + - "ol5id.\n" + - "o3li4f\n" + - "o5lil\n" + - "ol3ing\n" + - "o5lio\n" + - "o5lis.\n" + - "ol3ish\n" + - "o5lite\n" + - "o5litio\n" + - "o5liv\n" + - "olli4e\n" + - "ol5ogiz\n" + - "olo4r\n" + - "ol5pl\n" + - "ol2t\n" + - "ol3ub\n" + - "ol3ume\n" + - "ol3un\n" + - "o5lus\n" + - "ol2v\n" + - "o2ly\n" + - "om5ah\n" + - "oma5l\n" + - "om5atiz\n" + - "om2be\n" + - "om4bl\n" + - "o2me\n" + - "om3ena\n" + - "om5erse\n" + - "o4met\n" + - "om5etry\n" + - "o3mia\n" + - "om3ic.\n" + - "om3ica\n" + - "o5mid\n" + - "om1in\n" + - "o5mini\n" + - "5ommend\n" + - "omo4ge\n" + - "o4mon\n" + - "om3pi\n" + - "ompro5\n" + - "o2n\n" + - "on1a\n" + - "on4ac\n" + - "o3nan\n" + - "on1c\n" + - "3oncil\n" + - "2ond\n" + - "on5do\n" + - "o3nen\n" + - "on5est\n" + - "on4gu\n" + - "on1ic\n" + - "o3nio\n" + - "on1is\n" + - "o5niu\n" + - "on3key\n" + - "on4odi\n" + - "on3omy\n" + - "on3s\n" + - "onspi4\n" + - "onspir5a\n" + - "onsu4\n" + - "onten4\n" + - "on3t4i\n" + - "ontif5\n" + - "on5um\n" + - "onva5\n" + - "oo2\n" + - "ood5e\n" + - "ood5i\n" + - "oo4k\n" + - "oop3i\n" + - "o3ord\n" + - "oost5\n" + - "o2pa\n" + - "ope5d\n" + - "op1er\n" + - "3opera\n" + - "4operag\n" + - "2oph\n" + - "o5phan\n" + - "o5pher\n" + - "op3ing\n" + - "o3pit\n" + - "o5pon\n" + - "o4posi\n" + - "o1pr\n" + - "op1u\n" + - "opy5\n" + - "o1q\n" + - "o1ra\n" + - "o5ra.\n" + - "o4r3ag\n" + - "or5aliz\n" + - "or5ange\n" + - "ore5a\n" + - "o5real\n" + - "or3ei\n" + - "ore5sh\n" + - "or5est.\n" + - "orew4\n" + - "or4gu\n" + - "4o5ria\n" + - "or3ica\n" + - "o5ril\n" + - "or1in\n" + - "o1rio\n" + - "or3ity\n" + - "o3riu\n" + - "or2mi\n" + - "orn2e\n" + - "o5rof\n" + - "or3oug\n" + - "or5pe\n" + - "3orrh\n" + - "or4se\n" + - "ors5en\n" + - "orst4\n" + - "or3thi\n" + - "or3thy\n" + - "or4ty\n" + - "o5rum\n" + - "o1ry\n" + - "os3al\n" + - "os2c\n" + - "os4ce\n" + - "o3scop\n" + - "4oscopi\n" + - "o5scr\n" + - "os4i4e\n" + - "os5itiv\n" + - "os3ito\n" + - "os3ity\n" + - "osi4u\n" + - "os4l\n" + - "o2so\n" + - "os4pa\n" + - "os4po\n" + - "os2ta\n" + - "o5stati\n" + - "os5til\n" + - "os5tit\n" + - "o4tan\n" + - "otele4g\n" + - "ot3er.\n" + - "ot5ers\n" + - "o4tes\n" + - "4oth\n" + - "oth5esi\n" + - "oth3i4\n" + - "ot3ic.\n" + - "ot5ica\n" + - "o3tice\n" + - "o3tif\n" + - "o3tis\n" + - "oto5s\n" + - "ou2\n" + - "ou3bl\n" + - "ouch5i\n" + - "ou5et\n" + - "ou4l\n" + - "ounc5er\n" + - "oun2d\n" + - "ou5v\n" + - "ov4en\n" + - "over4ne\n" + - "over3s\n" + - "ov4ert\n" + - "o3vis\n" + - "oviti4\n" + - "o5v4ol\n" + - "ow3der\n" + - "ow3el\n" + - "ow5est\n" + - "ow1i\n" + - "own5i\n" + - "o4wo\n" + - "oy1a\n" + - "1pa\n" + - "pa4ca\n" + - "pa4ce\n" + - "pac4t\n" + - "p4ad\n" + - "5pagan\n" + - "p3agat\n" + - "p4ai\n" + - "pain4\n" + - "p4al\n" + - "pan4a\n" + - "pan3el\n" + - "pan4ty\n" + - "pa3ny\n" + - "pa1p\n" + - "pa4pu\n" + - "para5bl\n" + - "par5age\n" + - "par5di\n" + - "3pare\n" + - "par5el\n" + - "p4a4ri\n" + - "par4is\n" + - "pa2te\n" + - "pa5ter\n" + - "5pathic\n" + - "pa5thy\n" + - "pa4tric\n" + - "pav4\n" + - "3pay\n" + - "4p1b\n" + - "pd4\n" + - "4pe.\n" + - "3pe4a\n" + - "pear4l\n" + - "pe2c\n" + - "2p2ed\n" + - "3pede\n" + - "3pedi\n" + - "pedia4\n" + - "ped4ic\n" + - "p4ee\n" + - "pee4d\n" + - "pek4\n" + - "pe4la\n" + - "peli4e\n" + - "pe4nan\n" + - "p4enc\n" + - "pen4th\n" + - "pe5on\n" + - "p4era.\n" + - "pera5bl\n" + - "p4erag\n" + - "p4eri\n" + - "peri5st\n" + - "per4mal\n" + - "perme5\n" + - "p4ern\n" + - "per3o\n" + - "per3ti\n" + - "pe5ru\n" + - "per1v\n" + - "pe2t\n" + - "pe5ten\n" + - "pe5tiz\n" + - "4pf\n" + - "4pg\n" + - "4ph.\n" + - "phar5i\n" + - "phe3no\n" + - "ph4er\n" + - "ph4es.\n" + - "ph1ic\n" + - "5phie\n" + - "ph5ing\n" + - "5phisti\n" + - "3phiz\n" + - "ph2l\n" + - "3phob\n" + - "3phone\n" + - "5phoni\n" + - "pho4r\n" + - "4phs\n" + - "ph3t\n" + - "5phu\n" + - "1phy\n" + - "pi3a\n" + - "pian4\n" + - "pi4cie\n" + - "pi4cy\n" + - "p4id\n" + - "p5ida\n" + - "pi3de\n" + - "5pidi\n" + - "3piec\n" + - "pi3en\n" + - "pi4grap\n" + - "pi3lo\n" + - "pi2n\n" + - "p4in.\n" + - "pind4\n" + - "p4ino\n" + - "3pi1o\n" + - "pion4\n" + - "p3ith\n" + - "pi5tha\n" + - "pi2tu\n" + - "2p3k2\n" + - "1p2l2\n" + - "3plan\n" + - "plas5t\n" + - "pli3a\n" + - "pli5er\n" + - "4plig\n" + - "pli4n\n" + - "ploi4\n" + - "plu4m\n" + - "plum4b\n" + - "4p1m\n" + - "2p3n\n" + - "po4c\n" + - "5pod.\n" + - "po5em\n" + - "po3et5\n" + - "5po4g\n" + - "poin2\n" + - "5point\n" + - "poly5t\n" + - "po4ni\n" + - "po4p\n" + - "1p4or\n" + - "po4ry\n" + - "1pos\n" + - "pos1s\n" + - "p4ot\n" + - "po4ta\n" + - "5poun\n" + - "4p1p\n" + - "ppa5ra\n" + - "p2pe\n" + - "p4ped\n" + - "p5pel\n" + - "p3pen\n" + - "p3per\n" + - "p3pet\n" + - "ppo5site\n" + - "pr2\n" + - "pray4e\n" + - "5preci\n" + - "pre5co\n" + - "pre3em\n" + - "pref5ac\n" + - "pre4la\n" + - "pre3r\n" + - "p3rese\n" + - "3press\n" + - "pre5ten\n" + - "pre3v\n" + - "5pri4e\n" + - "prin4t3\n" + - "pri4s\n" + - "pris3o\n" + - "p3roca\n" + - "prof5it\n" + - "pro3l\n" + - "pros3e\n" + - "pro1t\n" + - "2p1s2\n" + - "p2se\n" + - "ps4h\n" + - "p4sib\n" + - "2p1t\n" + - "pt5a4b\n" + - "p2te\n" + - "p2th\n" + - "pti3m\n" + - "ptu4r\n" + - "p4tw\n" + - "pub3\n" + - "pue4\n" + - "puf4\n" + - "pul3c\n" + - "pu4m\n" + - "pu2n\n" + - "pur4r\n" + - "5pus\n" + - "pu2t\n" + - "5pute\n" + - "put3er\n" + - "pu3tr\n" + - "put4ted\n" + - "put4tin\n" + - "p3w\n" + - "qu2\n" + - "qua5v\n" + - "2que.\n" + - "3quer\n" + - "3quet\n" + - "2rab\n" + - "ra3bi\n" + - "rach4e\n" + - "r5acl\n" + - "raf5fi\n" + - "raf4t\n" + - "r2ai\n" + - "ra4lo\n" + - "ram3et\n" + - "r2ami\n" + - "rane5o\n" + - "ran4ge\n" + - "r4ani\n" + - "ra5no\n" + - "rap3er\n" + - "3raphy\n" + - "rar5c\n" + - "rare4\n" + - "rar5ef\n" + - "4raril\n" + - "r2as\n" + - "ration4\n" + - "rau4t\n" + - "ra5vai\n" + - "rav3el\n" + - "ra5zie\n" + - "r1b\n" + - "r4bab\n" + - "r4bag\n" + - "rbi2\n" + - "rbi4f\n" + - "r2bin\n" + - "r5bine\n" + - "rb5ing.\n" + - "rb4o\n" + - "r1c\n" + - "r2ce\n" + - "rcen4\n" + - "r3cha\n" + - "rch4er\n" + - "r4ci4b\n" + - "rc4it\n" + - "rcum3\n" + - "r4dal\n" + - "rd2i\n" + - "rdi4a\n" + - "rdi4er\n" + - "rdin4\n" + - "rd3ing\n" + - "2re.\n" + - "re1al\n" + - "re3an\n" + - "re5arr\n" + - "5reav\n" + - "re4aw\n" + - "r5ebrat\n" + - "rec5oll\n" + - "rec5ompe\n" + - "re4cre\n" + - "2r2ed\n" + - "re1de\n" + - "re3dis\n" + - "red5it\n" + - "re4fac\n" + - "re2fe\n" + - "re5fer.\n" + - "re3fi\n" + - "re4fy\n" + - "reg3is\n" + - "re5it\n" + - "re1li\n" + - "re5lu\n" + - "r4en4ta\n" + - "ren4te\n" + - "re1o\n" + - "re5pin\n" + - "re4posi\n" + - "re1pu\n" + - "r1er4\n" + - "r4eri\n" + - "rero4\n" + - "re5ru\n" + - "r4es.\n" + - "re4spi\n" + - "ress5ib\n" + - "res2t\n" + - "re5stal\n" + - "re3str\n" + - "re4ter\n" + - "re4ti4z\n" + - "re3tri\n" + - "reu2\n" + - "re5uti\n" + - "rev2\n" + - "re4val\n" + - "rev3el\n" + - "r5ev5er.\n" + - "re5vers\n" + - "re5vert\n" + - "re5vil\n" + - "rev5olu\n" + - "re4wh\n" + - "r1f\n" + - "rfu4\n" + - "r4fy\n" + - "rg2\n" + - "rg3er\n" + - "r3get\n" + - "r3gic\n" + - "rgi4n\n" + - "rg3ing\n" + - "r5gis\n" + - "r5git\n" + - "r1gl\n" + - "rgo4n\n" + - "r3gu\n" + - "rh4\n" + - "4rh.\n" + - "4rhal\n" + - "ri3a\n" + - "ria4b\n" + - "ri4ag\n" + - "r4ib\n" + - "rib3a\n" + - "ric5as\n" + - "r4ice\n" + - "4rici\n" + - "5ricid\n" + - "ri4cie\n" + - "r4ico\n" + - "rid5er\n" + - "ri3enc\n" + - "ri3ent\n" + - "ri1er\n" + - "ri5et\n" + - "rig5an\n" + - "5rigi\n" + - "ril3iz\n" + - "5riman\n" + - "rim5i\n" + - "3rimo\n" + - "rim4pe\n" + - "r2ina\n" + - "5rina.\n" + - "rin4d\n" + - "rin4e\n" + - "rin4g\n" + - "ri1o\n" + - "5riph\n" + - "riph5e\n" + - "ri2pl\n" + - "rip5lic\n" + - "r4iq\n" + - "r2is\n" + - "r4is.\n" + - "ris4c\n" + - "r3ish\n" + - "ris4p\n" + - "ri3ta3b\n" + - "r5ited.\n" + - "rit5er.\n" + - "rit5ers\n" + - "rit3ic\n" + - "ri2tu\n" + - "rit5ur\n" + - "riv5el\n" + - "riv3et\n" + - "riv3i\n" + - "r3j\n" + - "r3ket\n" + - "rk4le\n" + - "rk4lin\n" + - "r1l\n" + - "rle4\n" + - "r2led\n" + - "r4lig\n" + - "r4lis\n" + - "rl5ish\n" + - "r3lo4\n" + - "r1m\n" + - "rma5c\n" + - "r2me\n" + - "r3men\n" + - "rm5ers\n" + - "rm3ing\n" + - "r4ming.\n" + - "r4mio\n" + - "r3mit\n" + - "r4my\n" + - "r4nar\n" + - "r3nel\n" + - "r4ner\n" + - "r5net\n" + - "r3ney\n" + - "r5nic\n" + - "r1nis4\n" + - "r3nit\n" + - "r3niv\n" + - "rno4\n" + - "r4nou\n" + - "r3nu\n" + - "rob3l\n" + - "r2oc\n" + - "ro3cr\n" + - "ro4e\n" + - "ro1fe\n" + - "ro5fil\n" + - "rok2\n" + - "ro5ker\n" + - "5role.\n" + - "rom5ete\n" + - "rom4i\n" + - "rom4p\n" + - "ron4al\n" + - "ron4e\n" + - "ro5n4is\n" + - "ron4ta\n" + - "1room\n" + - "5root\n" + - "ro3pel\n" + - "rop3ic\n" + - "ror3i\n" + - "ro5ro\n" + - "ros5per\n" + - "ros4s\n" + - "ro4the\n" + - "ro4ty\n" + - "ro4va\n" + - "rov5el\n" + - "rox5\n" + - "r1p\n" + - "r4pea\n" + - "r5pent\n" + - "rp5er.\n" + - "r3pet\n" + - "rp4h4\n" + - "rp3ing\n" + - "r3po\n" + - "r1r4\n" + - "rre4c\n" + - "rre4f\n" + - "r4reo\n" + - "rre4st\n" + - "rri4o\n" + - "rri4v\n" + - "rron4\n" + - "rros4\n" + - "rrys4\n" + - "4rs2\n" + - "r1sa\n" + - "rsa5ti\n" + - "rs4c\n" + - "r2se\n" + - "r3sec\n" + - "rse4cr\n" + - "rs5er.\n" + - "rs3es\n" + - "rse5v2\n" + - "r1sh\n" + - "r5sha\n" + - "r1si\n" + - "r4si4b\n" + - "rson3\n" + - "r1sp\n" + - "r5sw\n" + - "rtach4\n" + - "r4tag\n" + - "r3teb\n" + - "rten4d\n" + - "rte5o\n" + - "r1ti\n" + - "rt5ib\n" + - "rti4d\n" + - "r4tier\n" + - "r3tig\n" + - "rtil3i\n" + - "rtil4l\n" + - "r4tily\n" + - "r4tist\n" + - "r4tiv\n" + - "r3tri\n" + - "rtroph4\n" + - "rt4sh\n" + - "ru3a\n" + - "ru3e4l\n" + - "ru3en\n" + - "ru4gl\n" + - "ru3in\n" + - "rum3pl\n" + - "ru2n\n" + - "runk5\n" + - "run4ty\n" + - "r5usc\n" + - "ruti5n\n" + - "rv4e\n" + - "rvel4i\n" + - "r3ven\n" + - "rv5er.\n" + - "r5vest\n" + - "r3vey\n" + - "r3vic\n" + - "rvi4v\n" + - "r3vo\n" + - "r1w\n" + - "ry4c\n" + - "5rynge\n" + - "ry3t\n" + - "sa2\n" + - "2s1ab\n" + - "5sack\n" + - "sac3ri\n" + - "s3act\n" + - "5sai\n" + - "salar4\n" + - "sal4m\n" + - "sa5lo\n" + - "sal4t\n" + - "3sanc\n" + - "san4de\n" + - "s1ap\n" + - "sa5ta\n" + - "5sa3tio\n" + - "sat3u\n" + - "sau4\n" + - "sa5vor\n" + - "5saw\n" + - "4s5b\n" + - "scan4t5\n" + - "sca4p\n" + - "scav5\n" + - "s4ced\n" + - "4scei\n" + - "s4ces\n" + - "sch2\n" + - "s4cho\n" + - "3s4cie\n" + - "5scin4d\n" + - "scle5\n" + - "s4cli\n" + - "scof4\n" + - "4scopy\n" + - "scour5a\n" + - "s1cu\n" + - "4s5d\n" + - "4se.\n" + - "se4a\n" + - "seas4\n" + - "sea5w\n" + - "se2c3o\n" + - "3sect\n" + - "4s4ed\n" + - "se4d4e\n" + - "s5edl\n" + - "se2g\n" + - "seg3r\n" + - "5sei\n" + - "se1le\n" + - "5self\n" + - "5selv\n" + - "4seme\n" + - "se4mol\n" + - "sen5at\n" + - "4senc\n" + - "sen4d\n" + - "s5ened\n" + - "sen5g\n" + - "s5enin\n" + - "4sentd\n" + - "4sentl\n" + - "sep3a3\n" + - "4s1er.\n" + - "s4erl\n" + - "ser4o\n" + - "4servo\n" + - "s1e4s\n" + - "se5sh\n" + - "ses5t\n" + - "5se5um\n" + - "5sev\n" + - "sev3en\n" + - "sew4i\n" + - "5sex\n" + - "4s3f\n" + - "2s3g\n" + - "s2h\n" + - "2sh.\n" + - "sh1er\n" + - "5shev\n" + - "sh1in\n" + - "sh3io\n" + - "3ship\n" + - "shiv5\n" + - "sho4\n" + - "sh5old\n" + - "shon3\n" + - "shor4\n" + - "short5\n" + - "4shw\n" + - "si1b\n" + - "s5icc\n" + - "3side.\n" + - "5sides\n" + - "5sidi\n" + - "si5diz\n" + - "4signa\n" + - "sil4e\n" + - "4sily\n" + - "2s1in\n" + - "s2ina\n" + - "5sine.\n" + - "s3ing\n" + - "1sio\n" + - "5sion\n" + - "sion5a\n" + - "si2r\n" + - "sir5a\n" + - "1sis\n" + - "3sitio\n" + - "5siu\n" + - "1siv\n" + - "5siz\n" + - "sk2\n" + - "4ske\n" + - "s3ket\n" + - "sk5ine\n" + - "sk5ing\n" + - "s1l2\n" + - "s3lat\n" + - "s2le\n" + - "slith5\n" + - "2s1m\n" + - "s3ma\n" + - "small3\n" + - "sman3\n" + - "smel4\n" + - "s5men\n" + - "5smith\n" + - "smol5d4\n" + - "s1n4\n" + - "1so\n" + - "so4ce\n" + - "soft3\n" + - "so4lab\n" + - "sol3d2\n" + - "so3lic\n" + - "5solv\n" + - "3som\n" + - "3s4on.\n" + - "sona4\n" + - "son4g\n" + - "s4op\n" + - "5sophic\n" + - "s5ophiz\n" + - "s5ophy\n" + - "sor5c\n" + - "sor5d\n" + - "4sov\n" + - "so5vi\n" + - "2spa\n" + - "5spai\n" + - "spa4n\n" + - "spen4d\n" + - "2s5peo\n" + - "2sper\n" + - "s2phe\n" + - "3spher\n" + - "spho5\n" + - "spil4\n" + - "sp5ing\n" + - "4spio\n" + - "s4ply\n" + - "s4pon\n" + - "spor4\n" + - "4spot\n" + - "squal4l\n" + - "s1r\n" + - "2ss\n" + - "s1sa\n" + - "ssas3\n" + - "s2s5c\n" + - "s3sel\n" + - "s5seng\n" + - "s4ses.\n" + - "s5set\n" + - "s1si\n" + - "s4sie\n" + - "ssi4er\n" + - "ss5ily\n" + - "s4sl\n" + - "ss4li\n" + - "s4sn\n" + - "sspend4\n" + - "ss2t\n" + - "ssur5a\n" + - "ss5w\n" + - "2st.\n" + - "s2tag\n" + - "s2tal\n" + - "stam4i\n" + - "5stand\n" + - "s4ta4p\n" + - "5stat.\n" + - "s4ted\n" + - "stern5i\n" + - "s5tero\n" + - "ste2w\n" + - "stew5a\n" + - "s3the\n" + - "st2i\n" + - "s4ti.\n" + - "s5tia\n" + - "s1tic\n" + - "5stick\n" + - "s4tie\n" + - "s3tif\n" + - "st3ing\n" + - "5stir\n" + - "s1tle\n" + - "5stock\n" + - "stom3a\n" + - "5stone\n" + - "s4top\n" + - "3store\n" + - "st4r\n" + - "s4trad\n" + - "5stratu\n" + - "s4tray\n" + - "s4trid\n" + - "4stry\n" + - "4st3w\n" + - "s2ty\n" + - "1su\n" + - "su1al\n" + - "su4b3\n" + - "su2g3\n" + - "su5is\n" + - "suit3\n" + - "s4ul\n" + - "su2m\n" + - "sum3i\n" + - "su2n\n" + - "su2r\n" + - "4sv\n" + - "sw2\n" + - "4swo\n" + - "s4y\n" + - "4syc\n" + - "3syl\n" + - "syn5o\n" + - "sy5rin\n" + - "1ta\n" + - "3ta.\n" + - "2tab\n" + - "ta5bles\n" + - "5taboliz\n" + - "4taci\n" + - "ta5do\n" + - "4taf4\n" + - "tai5lo\n" + - "ta2l\n" + - "ta5la\n" + - "tal5en\n" + - "tal3i\n" + - "4talk\n" + - "tal4lis\n" + - "ta5log\n" + - "ta5mo\n" + - "tan4de\n" + - "tanta3\n" + - "ta5per\n" + - "ta5pl\n" + - "tar4a\n" + - "4tarc\n" + - "4tare\n" + - "ta3riz\n" + - "tas4e\n" + - "ta5sy\n" + - "4tatic\n" + - "ta4tur\n" + - "taun4\n" + - "tav4\n" + - "2taw\n" + - "tax4is\n" + - "2t1b\n" + - "4tc\n" + - "t4ch\n" + - "tch5et\n" + - "4t1d\n" + - "4te.\n" + - "tead4i\n" + - "4teat\n" + - "tece4\n" + - "5tect\n" + - "2t1ed\n" + - "te5di\n" + - "1tee\n" + - "teg4\n" + - "te5ger\n" + - "te5gi\n" + - "3tel.\n" + - "teli4\n" + - "5tels\n" + - "te2ma2\n" + - "tem3at\n" + - "3tenan\n" + - "3tenc\n" + - "3tend\n" + - "4tenes\n" + - "1tent\n" + - "ten4tag\n" + - "1teo\n" + - "te4p\n" + - "te5pe\n" + - "ter3c\n" + - "5ter3d\n" + - "1teri\n" + - "ter5ies\n" + - "ter3is\n" + - "teri5za\n" + - "5ternit\n" + - "ter5v\n" + - "4tes.\n" + - "4tess\n" + - "t3ess.\n" + - "teth5e\n" + - "3teu\n" + - "3tex\n" + - "4tey\n" + - "2t1f\n" + - "4t1g\n" + - "2th.\n" + - "than4\n" + - "th2e\n" + - "4thea\n" + - "th3eas\n" + - "the5at\n" + - "the3is\n" + - "3thet\n" + - "th5ic.\n" + - "th5ica\n" + - "4thil\n" + - "5think\n" + - "4thl\n" + - "th5ode\n" + - "5thodic\n" + - "4thoo\n" + - "thor5it\n" + - "tho5riz\n" + - "2ths\n" + - "1tia\n" + - "ti4ab\n" + - "ti4ato\n" + - "2ti2b\n" + - "4tick\n" + - "t4ico\n" + - "t4ic1u\n" + - "5tidi\n" + - "3tien\n" + - "tif2\n" + - "ti5fy\n" + - "2tig\n" + - "5tigu\n" + - "till5in\n" + - "1tim\n" + - "4timp\n" + - "tim5ul\n" + - "2t1in\n" + - "t2ina\n" + - "3tine.\n" + - "3tini\n" + - "1tio\n" + - "ti5oc\n" + - "tion5ee\n" + - "5tiq\n" + - "ti3sa\n" + - "3tise\n" + - "tis4m\n" + - "ti5so\n" + - "tis4p\n" + - "5tistica\n" + - "ti3tl\n" + - "ti4u\n" + - "1tiv\n" + - "tiv4a\n" + - "1tiz\n" + - "ti3za\n" + - "ti3zen\n" + - "2tl\n" + - "t5la\n" + - "tlan4\n" + - "3tle.\n" + - "3tled\n" + - "3tles.\n" + - "t5let.\n" + - "t5lo\n" + - "4t1m\n" + - "tme4\n" + - "2t1n2\n" + - "1to\n" + - "to3b\n" + - "to5crat\n" + - "4todo\n" + - "2tof\n" + - "to2gr\n" + - "to5ic\n" + - "to2ma\n" + - "tom4b\n" + - "to3my\n" + - "ton4ali\n" + - "to3nat\n" + - "4tono\n" + - "4tony\n" + - "to2ra\n" + - "to3rie\n" + - "tor5iz\n" + - "tos2\n" + - "5tour\n" + - "4tout\n" + - "to3war\n" + - "4t1p\n" + - "1tra\n" + - "tra3b\n" + - "tra5ch\n" + - "traci4\n" + - "trac4it\n" + - "trac4te\n" + - "tras4\n" + - "tra5ven\n" + - "trav5es5\n" + - "tre5f\n" + - "tre4m\n" + - "trem5i\n" + - "5tria\n" + - "tri5ces\n" + - "5tricia\n" + - "4trics\n" + - "2trim\n" + - "tri4v\n" + - "tro5mi\n" + - "tron5i\n" + - "4trony\n" + - "tro5phe\n" + - "tro3sp\n" + - "tro3v\n" + - "tru5i\n" + - "trus4\n" + - "4t1s2\n" + - "t4sc\n" + - "tsh4\n" + - "t4sw\n" + - "4t3t2\n" + - "t4tes\n" + - "t5to\n" + - "ttu4\n" + - "1tu\n" + - "tu1a\n" + - "tu3ar\n" + - "tu4bi\n" + - "tud2\n" + - "4tue\n" + - "4tuf4\n" + - "5tu3i\n" + - "3tum\n" + - "tu4nis\n" + - "2t3up.\n" + - "3ture\n" + - "5turi\n" + - "tur3is\n" + - "tur5o\n" + - "tu5ry\n" + - "3tus\n" + - "4tv\n" + - "tw4\n" + - "4t1wa\n" + - "twis4\n" + - "4two\n" + - "1ty\n" + - "4tya\n" + - "2tyl\n" + - "type3\n" + - "ty5ph\n" + - "4tz\n" + - "tz4e\n" + - "4uab\n" + - "uac4\n" + - "ua5na\n" + - "uan4i\n" + - "uar5ant\n" + - "uar2d\n" + - "uar3i\n" + - "uar3t\n" + - "u1at\n" + - "uav4\n" + - "ub4e\n" + - "u4bel\n" + - "u3ber\n" + - "u4bero\n" + - "u1b4i\n" + - "u4b5ing\n" + - "u3ble.\n" + - "u3ca\n" + - "uci4b\n" + - "uc4it\n" + - "ucle3\n" + - "u3cr\n" + - "u3cu\n" + - "u4cy\n" + - "ud5d\n" + - "ud3er\n" + - "ud5est\n" + - "udev4\n" + - "u1dic\n" + - "ud3ied\n" + - "ud3ies\n" + - "ud5is\n" + - "u5dit\n" + - "u4don\n" + - "ud4si\n" + - "u4du\n" + - "u4ene\n" + - "uens4\n" + - "uen4te\n" + - "uer4il\n" + - "3ufa\n" + - "u3fl\n" + - "ugh3en\n" + - "ug5in\n" + - "2ui2\n" + - "uil5iz\n" + - "ui4n\n" + - "u1ing\n" + - "uir4m\n" + - "uita4\n" + - "uiv3\n" + - "uiv4er.\n" + - "u5j\n" + - "4uk\n" + - "u1la\n" + - "ula5b\n" + - "u5lati\n" + - "ulch4\n" + - "5ulche\n" + - "ul3der\n" + - "ul4e\n" + - "u1len\n" + - "ul4gi\n" + - "ul2i\n" + - "u5lia\n" + - "ul3ing\n" + - "ul5ish\n" + - "ul4lar\n" + - "ul4li4b\n" + - "ul4lis\n" + - "4ul3m\n" + - "u1l4o\n" + - "4uls\n" + - "uls5es\n" + - "ul1ti\n" + - "ultra3\n" + - "4ultu\n" + - "u3lu\n" + - "ul5ul\n" + - "ul5v\n" + - "um5ab\n" + - "um4bi\n" + - "um4bly\n" + - "u1mi\n" + - "u4m3ing\n" + - "umor5o\n" + - "um2p\n" + - "unat4\n" + - "u2ne\n" + - "un4er\n" + - "u1ni\n" + - "un4im\n" + - "u2nin\n" + - "un5ish\n" + - "uni3v\n" + - "un3s4\n" + - "un4sw\n" + - "unt3ab\n" + - "un4ter.\n" + - "un4tes\n" + - "unu4\n" + - "un5y\n" + - "un5z\n" + - "u4ors\n" + - "u5os\n" + - "u1ou\n" + - "u1pe\n" + - "uper5s\n" + - "u5pia\n" + - "up3ing\n" + - "u3pl\n" + - "up3p\n" + - "upport5\n" + - "upt5ib\n" + - "uptu4\n" + - "u1ra\n" + - "4ura.\n" + - "u4rag\n" + - "u4ras\n" + - "ur4be\n" + - "urc4\n" + - "ur1d\n" + - "ure5at\n" + - "ur4fer\n" + - "ur4fr\n" + - "u3rif\n" + - "uri4fic\n" + - "ur1in\n" + - "u3rio\n" + - "u1rit\n" + - "ur3iz\n" + - "ur2l\n" + - "url5ing.\n" + - "ur4no\n" + - "uros4\n" + - "ur4pe\n" + - "ur4pi\n" + - "urs5er\n" + - "ur5tes\n" + - "ur3the\n" + - "urti4\n" + - "ur4tie\n" + - "u3ru\n" + - "2us\n" + - "u5sad\n" + - "u5san\n" + - "us4ap\n" + - "usc2\n" + - "us3ci\n" + - "use5a\n" + - "u5sia\n" + - "u3sic\n" + - "us4lin\n" + - "us1p\n" + - "us5sl\n" + - "us5tere\n" + - "us1tr\n" + - "u2su\n" + - "usur4\n" + - "uta4b\n" + - "u3tat\n" + - "4ute.\n" + - "4utel\n" + - "4uten\n" + - "uten4i\n" + - "4u1t2i\n" + - "uti5liz\n" + - "u3tine\n" + - "ut3ing\n" + - "ution5a\n" + - "u4tis\n" + - "5u5tiz\n" + - "u4t1l\n" + - "ut5of\n" + - "uto5g\n" + - "uto5matic\n" + - "u5ton\n" + - "u4tou\n" + - "uts4\n" + - "u3u\n" + - "uu4m\n" + - "u1v2\n" + - "uxu3\n" + - "uz4e\n" + - "1va\n" + - "5va.\n" + - "2v1a4b\n" + - "vac5il\n" + - "vac3u\n" + - "vag4\n" + - "va4ge\n" + - "va5lie\n" + - "val5o\n" + - "val1u\n" + - "va5mo\n" + - "va5niz\n" + - "va5pi\n" + - "var5ied\n" + - "3vat\n" + - "4ve.\n" + - "4ved\n" + - "veg3\n" + - "v3el.\n" + - "vel3li\n" + - "ve4lo\n" + - "v4ely\n" + - "ven3om\n" + - "v5enue\n" + - "v4erd\n" + - "5vere.\n" + - "v4erel\n" + - "v3eren\n" + - "ver5enc\n" + - "v4eres\n" + - "ver3ie\n" + - "vermi4n\n" + - "3verse\n" + - "ver3th\n" + - "v4e2s\n" + - "4ves.\n" + - "ves4te\n" + - "ve4te\n" + - "vet3er\n" + - "ve4ty\n" + - "vi5ali\n" + - "5vian\n" + - "5vide.\n" + - "5vided\n" + - "4v3iden\n" + - "5vides\n" + - "5vidi\n" + - "v3if\n" + - "vi5gn\n" + - "vik4\n" + - "2vil\n" + - "5vilit\n" + - "v3i3liz\n" + - "v1in\n" + - "4vi4na\n" + - "v2inc\n" + - "vin5d\n" + - "4ving\n" + - "vio3l\n" + - "v3io4r\n" + - "vi1ou\n" + - "vi4p\n" + - "vi5ro\n" + - "vis3it\n" + - "vi3so\n" + - "vi3su\n" + - "4viti\n" + - "vit3r\n" + - "4vity\n" + - "3viv\n" + - "5vo.\n" + - "voi4\n" + - "3vok\n" + - "vo4la\n" + - "v5ole\n" + - "5volt\n" + - "3volv\n" + - "vom5i\n" + - "vor5ab\n" + - "vori4\n" + - "vo4ry\n" + - "vo4ta\n" + - "4votee\n" + - "4vv4\n" + - "v4y\n" + - "w5abl\n" + - "2wac\n" + - "wa5ger\n" + - "wag5o\n" + - "wait5\n" + - "w5al.\n" + - "wam4\n" + - "war4t\n" + - "was4t\n" + - "wa1te\n" + - "wa5ver\n" + - "w1b\n" + - "wea5rie\n" + - "weath3\n" + - "wed4n\n" + - "weet3\n" + - "wee5v\n" + - "wel4l\n" + - "w1er\n" + - "west3\n" + - "w3ev\n" + - "whi4\n" + - "wi2\n" + - "wil2\n" + - "will5in\n" + - "win4de\n" + - "win4g\n" + - "wir4\n" + - "3wise\n" + - "with3\n" + - "wiz5\n" + - "w4k\n" + - "wl4es\n" + - "wl3in\n" + - "w4no\n" + - "1wo2\n" + - "wom1\n" + - "wo5ven\n" + - "w5p\n" + - "wra4\n" + - "wri4\n" + - "writa4\n" + - "w3sh\n" + - "ws4l\n" + - "ws4pe\n" + - "w5s4t\n" + - "4wt\n" + - "wy4\n" + - "x1a\n" + - "xac5e\n" + - "x4ago\n" + - "xam3\n" + - "x4ap\n" + - "xas5\n" + - "x3c2\n" + - "x1e\n" + - "xe4cuto\n" + - "x2ed\n" + - "xer4i\n" + - "xe5ro\n" + - "x1h\n" + - "xhi2\n" + - "xhil5\n" + - "xhu4\n" + - "x3i\n" + - "xi5a\n" + - "xi5c\n" + - "xi5di\n" + - "x4ime\n" + - "xi5miz\n" + - "x3o\n" + - "x4ob\n" + - "x3p\n" + - "xpan4d\n" + - "xpecto5\n" + - "xpe3d\n" + - "x1t2\n" + - "x3ti\n" + - "x1u\n" + - "xu3a\n" + - "xx4\n" + - "y5ac\n" + - "3yar4\n" + - "y5at\n" + - "y1b\n" + - "y1c\n" + - "y2ce\n" + - "yc5er\n" + - "y3ch\n" + - "ych4e\n" + - "ycom4\n" + - "ycot4\n" + - "y1d\n" + - "y5ee\n" + - "y1er\n" + - "y4erf\n" + - "yes4\n" + - "ye4t\n" + - "y5gi\n" + - "4y3h\n" + - "y1i\n" + - "y3la\n" + - "ylla5bl\n" + - "y3lo\n" + - "y5lu\n" + - "ymbol5\n" + - "yme4\n" + - "ympa3\n" + - "yn3chr\n" + - "yn5d\n" + - "yn5g\n" + - "yn5ic\n" + - "5ynx\n" + - "y1o4\n" + - "yo5d\n" + - "y4o5g\n" + - "yom4\n" + - "yo5net\n" + - "y4ons\n" + - "y4os\n" + - "y4ped\n" + - "yper5\n" + - "yp3i\n" + - "y3po\n" + - "y4poc\n" + - "yp2ta\n" + - "y5pu\n" + - "yra5m\n" + - "yr5ia\n" + - "y3ro\n" + - "yr4r\n" + - "ys4c\n" + - "y3s2e\n" + - "ys3ica\n" + - "ys3io\n" + - "3ysis\n" + - "y4so\n" + - "yss4\n" + - "ys1t\n" + - "ys3ta\n" + - "ysur4\n" + - "y3thin\n" + - "yt3ic\n" + - "y1w\n" + - "za1\n" + - "z5a2b\n" + - "zar2\n" + - "4zb\n" + - "2ze\n" + - "ze4n\n" + - "ze4p\n" + - "z1er\n" + - "ze3ro\n" + - "zet4\n" + - "2z1i\n" + - "z4il\n" + - "z4is\n" + - "5zl\n" + - "4zm\n" + - "1zo\n" + - "zo4m\n" + - "zo5ol\n" + - "zte4\n" + - "4z1z2\n" + - "z4zy\n" - ; -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/InfoTemplate.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/InfoTemplate.java deleted file mode 100644 index 3dc0f01..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/InfoTemplate.java +++ /dev/null @@ -1,58 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -/** - * A partial {@link PascalTemplate}. - * Holds URL, acronym, and name template fields. - * - * @author Chris Cox - */ -public class InfoTemplate{ - String whomepage="null"; - String wacronym="null"; - String wname="null"; - String chomepage="null"; - String cacronym="null"; - String cname="null"; - - public InfoTemplate(String whomepage, String wacronym, String wname, - String chomepage, String cacronym, String cname, - CliqueTemplates ct) { - - if(whomepage!=null)this.whomepage=whomepage; - if(wacronym!=null)this.wacronym=PascalTemplate.stemAcronym(wacronym,ct); - if(wname!=null)this.wname=wname; - if(chomepage!=null)this.chomepage=chomepage; - if(cacronym!=null)this.cacronym=PascalTemplate.stemAcronym(cacronym,ct); - if(cname!=null)this.cname=cname; - } - - @Override - public int hashCode() { - int tally=31; - int n=7; - tally = whomepage.hashCode()+n*wacronym.hashCode()+n*n*wname.hashCode(); - tally += (chomepage.hashCode() + - n*cacronym.hashCode()+ n*n*cname.hashCode()); - return tally; - } - - @Override - public boolean equals(Object obj){ - if(obj==null)return false; - if(!( obj instanceof InfoTemplate)) return false; - InfoTemplate i = (InfoTemplate)obj; - - return(whomepage.equals(i.whomepage)&& - wacronym.equals(i.wacronym) && - wname.equals(i.wname) && - chomepage.equals(i.chomepage)&& - cacronym.equals(i.cacronym) && - cname.equals(i.cname)); - } - - @Override - public String toString(){ - return ("W_URL: "+whomepage+" W_ACRO: "+wacronym+" W_NAME: "+wname+ - "\nC_URL: "+chomepage+" C_ACRO: "+cacronym+" C_NAME: "+cname); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/PascalTemplate.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/PascalTemplate.java deleted file mode 100644 index 32ab58a..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/PascalTemplate.java +++ /dev/null @@ -1,283 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.util.Index; -import edu.stanford.nlp.util.HashIndex; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Maps non-background Pascal fields to strings. - * - * @author Chris Cox - */ - - -public class PascalTemplate { - - public static final String fields[] = { - //dates - "workshoppapersubmissiondate", - "workshopnotificationofacceptancedate", - "workshopcamerareadycopydate", - "workshopdate", - //location - "workshoplocation", - //workshop info - "workshopacronym", - "workshophomepage", - "workshopname", - //conference info - "conferenceacronym", - "conferencehomepage", - "conferencename", - //background symbol - "0" - }; - - public static final String BACKGROUND_SYMBOL = "0"; - - private static final Index fieldIndices; - - static { - fieldIndices = new HashIndex(); - for (int i = 0; i < fields.length; i++) { - fieldIndices.add(fields[i]); - } - } - - private String[] values = null; - - - public PascalTemplate() { - values = new String[fields.length]; - for (int i = 0; i < values.length; i++) { - values[i] = null; - } - } - - //copy constructor - public PascalTemplate(PascalTemplate pt) { - this.values = new String[fields.length]; - for (int i = 0; i < values.length; i++) { - if (pt.values[i] == null) { - this.values[i] = null; - } else { - this.values[i] = pt.values[i]; - } - } - }; - - /* - * Acronym stemming and matching fields - */ - private static Pattern acronymPattern = Pattern.compile("([ \r-/a-zA-Z]+?)(?:[ -'*\t\r\n\f0-9]*)", Pattern.DOTALL); - - /** - * - */ - public static boolean acronymMatch(String s1, String s2, HashMap stemmedAcronymIndex) { - System.err.println("Testing match:" + s1 + " : " + s2); - String stem1 = (String) stemmedAcronymIndex.get(s1); - String stem2 = (String) stemmedAcronymIndex.get(s2); - System.err.println("Got stems:" + s1 + " : " + s2); - if (stem1.equals(stem2)) { - return true; - } else { - return false; - } - } - /** - * - */ - public static String stemAcronym(String s, CliqueTemplates ct) { - if (ct.stemmedAcronymIndex.containsKey(s)) { - return (String) ct.stemmedAcronymIndex.get(s); - } - Matcher matcher = acronymPattern.matcher(s); - if (!matcher.matches() || s.equalsIgnoreCase("www")) { - System.err.println("Not a valid acronym: " + s); - return "null"; - } - - String stemmed = matcher.group(1).toLowerCase(); - if (stemmed.endsWith("-")) { - stemmed = stemmed.substring(0, stemmed.length() - 1); - } - - ct.stemmedAcronymIndex.put(s, stemmed); - System.err.println("Stemmed: " + s + " to: " + stemmed); - if (ct.inverseAcronymMap.containsKey(stemmed)) { - HashSet set = (HashSet) ct.inverseAcronymMap.get(stemmed); - set.add(s); - } else { - HashSet set = new HashSet(); - set.add(s); - ct.inverseAcronymMap.put(stemmed, set); - } - return stemmed; - } - -/** - * Merges partial (clique) templates into a full one. - * - * @param dt date template - * @param location location - * @param wi workshop/conference info template - * @return the {@link PascalTemplate} resulting from this merge. - */ - - public static PascalTemplate mergeCliqueTemplates(DateTemplate dt, String location, InfoTemplate wi) { - PascalTemplate pt = new PascalTemplate(); - pt.setValue("workshopnotificationofacceptancedate", dt.noadate); - pt.setValue("workshopcamerareadycopydate", dt.crcdate); - pt.setValue("workshopdate", dt.workdate); - pt.setValue("workshoppapersubmissiondate", dt.subdate); - pt.setValue("workshoplocation", location); - pt.setValue("workshopacronym", wi.wacronym); - pt.setValue("workshophomepage", wi.whomepage); - pt.setValue("workshopname", wi.wname); - pt.setValue("conferenceacronym", wi.cacronym); - pt.setValue("conferencehomepage", wi.chomepage); - pt.setValue("conferencename", wi.cname); - return pt; - } - -/** - * Sets template values. - * @param fieldName (i.e. workshopname, workshopdate) - */ - public void setValue(String fieldName, String value) { - int index = getFieldIndex(fieldName); - assert(index != -1); - values[index] = value; - } - - public void setValue(int index, String value) { - if (index != values.length - 1) { - values[index] = value; - } - } - - public String getValue(String fieldName) { - int i = getFieldIndex(fieldName); - if (i == -1 || i == values.length - 1) { - return null; - } else { - return values[i]; - } - } - - @Override - public boolean equals(Object obj) { - - if (obj == null) { - return false; - } - if (!(obj instanceof PascalTemplate)) { - return false; - } - - PascalTemplate pt = (PascalTemplate) obj; - String[] values2 = pt.values; - - if (values.length != values2.length) { - return false; - } - - for (int i = 0; i < values.length - 1; i++) { - if (values[i] == null) { - if (values2[i] != null) { - return false; - } - } else { - if (values2[i] == null) { - return false; - } - if (!values2[i].equals(values[i])) { - return false; - } - } - } - return true; - } - - @Override - public int hashCode() { - int tally = 37; - int n; - for (int i = 0; i < values.length - 1; i++) { - if (values[i] == null) { - n = 11; - } else { - n = values[i].hashCode(); - } - tally = 17 * tally + n; - } - return tally; - } - - /** - * - * @param tag field name (i.e. workshopdate, workshoplocation) - * @return the reference of that field in the underlying {@link edu.stanford.nlp.util.Index} - */ - static public int getFieldIndex(String tag) { - return (fieldIndices.indexOf(tag)); - } - - /** - * Should be passed a Counter[], each entry of which - * keeps scores for possibilities in that template slot. The counter - * for each template value is incremented by the corresponding score of - * this PascalTemplate. - * - * @param fieldValueCounter an array of counters, each of which holds label possibilities for one field - * @param score increment counts by this much. - */ - - public void writeToFieldValueCounter(ClassicCounter[] fieldValueCounter, double score) { - for (int i = 0; i < fields.length; i++) { - if ((values[i] != null) && !values[i].equals("NULL")) { - fieldValueCounter[i].incrementCount(values[i], score); - } - } - } -/** - * Divides this template into partial templates, and updates the counts of these - * partial templates in the {@link CliqueTemplates} object. - * - * @param ct the partial templates counter object - * @param score increment counts by this much - */ - public void unpackToCliqueTemplates(CliqueTemplates ct, double score) { - - ct.dateCliqueCounter.incrementCount(new DateTemplate(values[0], values[1], values[2], values[3]), score); - if (values[4] != null) { - ct.locationCliqueCounter.incrementCount(values[4], score); - } - - ct.workshopInfoCliqueCounter.incrementCount(new InfoTemplate(values[6], values[5], values[7], values[9], values[8], values[10], ct), score); - } - - public void print() { - System.err.println("PascalTemplate: "); - System.err.println(this.toString()); - } - - @Override - public String toString() { - String str = new String("\n====================\n"); - for (int i = 0; i < values.length; i++) { - if (values[i] != null) { - if (!(values[i].equalsIgnoreCase("NULL"))) { - str = str.concat(fields[i] + " : " + values[i] + "\n"); - } - } - } - return str; - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/Prior.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/Prior.java deleted file mode 100644 index 0a1b96b..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/Prior.java +++ /dev/null @@ -1,77 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * @author Jamie Nicolson - */ -public class Prior { - // Map maps field names to indexes in the matrix - private Map fieldIndices; - private String[] indexFields; - - // n-dimensional boolean matrix. There will be 2^n entries in the matrix. - private double[] matrix; - - public Prior(BufferedReader reader) throws IOException { - String line; - line = reader.readLine(); - if (line == null) { - throw new IOException(); - } - indexFields = line.split("\\s+"); - fieldIndices = new HashMap(); - for (int i = 0; i < indexFields.length; ++i) { - fieldIndices.put(indexFields[i], Integer.valueOf(i)); - } - if (indexFields.length < 1 || indexFields.length > 31) { - throw new IOException("Invalid number of fields, should be >=1 and <= 31"); - } - int matrixSize = 1 << indexFields.length; - matrix = new double[matrixSize]; - int matrixIdx = 0; - while (matrixIdx < matrix.length && (line = reader.readLine()) != null) { - String[] tokens = line.split("\\s+"); - for (int t = 0; matrixIdx < matrix.length && t < tokens.length; ++t) { - matrix[matrixIdx++] = Double.parseDouble(tokens[t]); - } - } - } - - /** - * Map - */ - public double get(Set presentFields) { - int index = 0; - for (int f = 0; f < indexFields.length; ++f) { - String field = indexFields[f]; - index *= 2; - if (presentFields.contains(field)) { - ++index; - } - } - return matrix[index]; - } - - public static void main(String args[]) throws Exception { - - BufferedReader br = new BufferedReader(new FileReader("/tmp/acstats")); - - Prior p = new Prior(br); - - HashSet hs = new HashSet(); - hs.add("workshopname"); - //hs.add("workshopacronym"); - - double d = p.get(hs); - System.out.println("d is " + d); - - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/RelationalModel.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/RelationalModel.java deleted file mode 100644 index 5bcfb77..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/RelationalModel.java +++ /dev/null @@ -1,18 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - - -/** - * An interface for the relational models in phase 2 of the pascal system. - * - * @author Jamie Nicolson - */ -public interface RelationalModel { - /** - * - * @param temp template to be scored - * @return its score - */ - public double computeProb(PascalTemplate temp); - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/TeXHyphenator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/TeXHyphenator.java deleted file mode 100644 index 1f8b9cb..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ie/pascal/TeXHyphenator.java +++ /dev/null @@ -1,192 +0,0 @@ -package edu.stanford.nlp.ie.pascal; - -import java.io.*; -import java.util.*; -import edu.stanford.nlp.util.StringUtils; - -/** - * Hyphenates words according to the TeX algorithm. - * @author Jamie Nicolson (nicolson@cs.stanford.edu) - */ -public class TeXHyphenator { - - private static class Node { - HashMap children = new HashMap(); - - int [] pattern = null; - }; - - /** - * Loads the default hyphenation rules in DefaultTeXHyphenator. - */ - public void loadDefault() { - try { - load( new BufferedReader(new StringReader( - DefaultTeXHyphenData.hyphenData) ) ); - } catch(IOException e) { - // shouldn't happen - throw new RuntimeException(e); - } - } - - /** - * Loads custom hyphenation rules. You probably want to use - * loadDefault() instead. - * - */ - public void load(BufferedReader input) throws IOException { - String line; - while( (line=input.readLine()) != null ) { - if( StringUtils.matches(line, "\\s*(%.*)?") ) { - // comment or blank line - System.err.println("Skipping: " + line); - continue; - } - char [] linechars = line.toCharArray(); - int [] pattern = new int[linechars.length]; - char [] chars = new char[linechars.length]; - int c = 0; - for( int i = 0; i < linechars.length; ++i) { - if( Character.isDigit(linechars[i]) ) { - pattern[c] = Character.digit(linechars[i], 10); - } else { - chars[c++] = linechars[i]; - } - } - char[] shortchars = new char[c]; - int [] shortpattern = new int[c+1]; - System.arraycopy(chars, 0, shortchars, 0, c); - System.arraycopy(pattern, 0, shortpattern, 0, c+1); - insertHyphPattern(shortchars, shortpattern); - } - } - - private Node head = new Node(); - - public static String toString(int[]i) { - StringBuffer sb = new StringBuffer(); - for(int j = 0; j < i.length; ++j) { - sb.append(i[j]); - } - return sb.toString(); - } - - private void insertHyphPattern(char [] chars, int [] pattern) { - // find target node, building as we go - Node cur = head; - for( int c = 0; c < chars.length; ++c) { - Character curchar = new Character(chars[c]); - Node next = (Node) cur.children.get(curchar); - if( next == null ) { - next = new Node(); - cur.children.put( curchar, next ); - } - cur = next; - } - assert( cur.pattern == null ); - cur.pattern = pattern; - } - - private List getMatchingPatterns( char[] chars, int startingIdx ) { - Node cur = head; - LinkedList matchingPatterns = new LinkedList(); - if( cur.pattern != null ) { - matchingPatterns.add(cur.pattern); - } - for(int c = startingIdx; cur != null && c < chars.length; ++c ) { - Character curchar = new Character(chars[c]); - Node next = (Node) cur.children.get(curchar); - cur = next; - if( cur != null && cur.pattern != null ) { - matchingPatterns.add(cur.pattern); - } - } - return matchingPatterns; - } - - - private void labelWordBreakPoints( char [] phrase, int start, int end, - boolean[] breakPoints) - { - - char [] word = new char[end-start+2]; - System.arraycopy(phrase, start, word, 1, end-start); - word[0] = '.'; - word[word.length-1] = '.'; - - // breakScore[i] is the score for breaking before word[i] - int [] breakScore = new int [word.length + 1]; - - for( int c = 0; c < word.length; ++c ) { - List patterns = getMatchingPatterns(word, c); - Iterator iter = patterns.iterator(); - while(iter.hasNext()) { - int [] pattern = (int[]) iter.next(); - for( int i = 0; i < pattern.length; ++i ) { - if( breakScore[c+i] < pattern[i] ) { - breakScore[c+i] = pattern[i]; - } - } - } - } - - breakPoints[start] = true; - for( int i = start+1; i < end; i++) { - // remember that breakPoints is offset by one because we introduced - // the leading "." - breakPoints[i-1] |= (breakScore[i-start] % 2 == 1 ); - } - } - - /** - * @param lcphrase Some English text in lowercase. - * @return An array of booleans, one per character of the input, - * indicating whether it would be OK to insert a hyphen before that - * character. - */ - public boolean[] findBreakPoints(char [] lcphrase) { - - boolean [] breakPoints = new boolean[lcphrase.length]; - - boolean inWord = false; - int wordStart = 0; - int c = 0; - for(; c < lcphrase.length; ++c) { - if( !inWord && Character.isLetter(lcphrase[c]) ) { - wordStart = c; - inWord = true; - } else if( inWord && !Character.isLetter(lcphrase[c]) ) { - inWord = false; - labelWordBreakPoints(lcphrase, wordStart, c, breakPoints); - } - } - if( inWord ) { - labelWordBreakPoints(lcphrase, wordStart, c, breakPoints); - } - - return breakPoints; - } - - public static void main(String[] args) throws Exception { - - TeXHyphenator hyphenator = new TeXHyphenator(); - hyphenator.loadDefault(); - - for( int a = 0; a < args.length; ++a) { - char[] chars = args[a].toLowerCase().toCharArray(); - boolean [] breakPoints = hyphenator.findBreakPoints(chars); - System.out.println(args[a]); - StringBuffer sb = new StringBuffer(); - for(int i = 0; i < breakPoints.length; ++i) { - if( breakPoints[i] ) { - sb.append("^"); - } else { - sb.append("-"); - } - } - System.out.println(sb.toString()); - } - } - - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/international/morph/MorphoFeatureSpecification.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/international/morph/MorphoFeatureSpecification.java deleted file mode 100644 index 74ff953..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/international/morph/MorphoFeatureSpecification.java +++ /dev/null @@ -1,46 +0,0 @@ -package edu.stanford.nlp.international.morph; - -import java.io.Serializable; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Morphological feature specification for surface forms in a given language. - * Currently supported feature names are the values of MorphFeatureType. - * - * @author Spence Green - * - */ -public abstract class MorphoFeatureSpecification implements Serializable { - - private static final long serialVersionUID = -5720683653931585664L; - - //Delimiter for associating a surface form with a morphological analysis, e.g., - // - // his~#PRP_3ms - // - public static final String MORPHO_MARK = "~#"; - - public static enum MorphoFeatureType {TENSE,DEF,ASP,MOOD,NUM,GEN,CASE,PER,POSS,VOICE,OTHER}; - - protected final Set activeFeatures; - - public MorphoFeatureSpecification() { - activeFeatures = new HashSet(); - } - - public void activate(MorphoFeatureType feat) { - activeFeatures.add(feat); - } - - public boolean isActive(MorphoFeatureType feat) { return activeFeatures.contains(feat); } - - public abstract List getValues(MorphoFeatureType feat); - - public abstract MorphoFeatures strToFeatures(String spec); - - @Override - public String toString() { return activeFeatures.toString(); } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/international/morph/MorphoFeatures.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/international/morph/MorphoFeatures.java deleted file mode 100644 index e4bd28b..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/international/morph/MorphoFeatures.java +++ /dev/null @@ -1,126 +0,0 @@ -package edu.stanford.nlp.international.morph; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import edu.stanford.nlp.international.morph.MorphoFeatureSpecification.MorphoFeatureType; - -/** - * Holds a set of morphosyntactic features for a given surface form - * - * @author Spence Green - * - */ -public class MorphoFeatures implements Serializable { - - private static final long serialVersionUID = -3893316324305154940L; - - public static final String KEY_VAL_DELIM = ":"; - - protected final Map fSpec; - protected String altTag; - - public MorphoFeatures() { - fSpec = new HashMap(); - } - - public MorphoFeatures(MorphoFeatures other) { - this(); - for(Map.Entry entry : other.fSpec.entrySet()) - this.fSpec.put(entry.getKey(), entry.getValue()); - this.altTag = other.altTag; - } - - public void addFeature(MorphoFeatureType feat, String val) { - fSpec.put(feat, val); - } - - public boolean hasFeature(MorphoFeatureType feat) { - return fSpec.containsKey(feat); - } - - public String getValue(MorphoFeatureType feat) { - if(fSpec.containsKey(feat)) - return fSpec.get(feat); - throw new IllegalArgumentException("Value requested for non-existent feature: " + feat.toString()); - } - - public int numFeatureMatches(MorphoFeatures other) { - int nMatches = 0; - for(Map.Entry fPair : fSpec.entrySet()) { - if(other.hasFeature(fPair.getKey()) && other.getValue(fPair.getKey()).equals(fPair.getValue())) - nMatches++; - } - - return nMatches; - } - - public int numActiveFeatures() { return fSpec.keySet().size(); } - - /** - * Build a POS tag consisting of a base category plus inflectional features. - * - * @param baseTag - * @return the tag - */ - public String getTag(String baseTag) { - return baseTag + toString(); - } - - public void setAltTag(String tag) { altTag = tag; } - - - /** - * An alternate tag form than the one produced by getTag(). Subclasses - * may want to use this form to implement someone else's tagset (e.g., CC, ERTS, etc.) - * - * @return the tag - */ - public String getAltTag() { - return altTag; - } - - /** - * Assumes that the tag string has been formed using a call to getTag(). As such, - * it removes the basic category from the feature string. - *

- * Note that this method returns a new MorphoFeatures object. As a result, it - * behaves like a static method, but is non-static so that subclasses can override - * this method. - * - * @param str - * @return - */ - public MorphoFeatures fromTagString(String str) { - List feats = Arrays.asList(str.split("\\-")); - MorphoFeatures mFeats = new MorphoFeatures(); - for(String fPair : feats) { - String[] keyValue = fPair.split(KEY_VAL_DELIM); - if(keyValue.length != 2)//Manual state split annotations - continue; - MorphoFeatureType fName = MorphoFeatureType.valueOf(keyValue[0].trim()); - mFeats.addFeature(fName, keyValue[1].trim()); - } - - return mFeats; - } - - /** - * values() returns the values in the order in which they are declared. Thus we will not have - * the case where two feature types can yield two strings: - * -feat1:A-feat2:B - * -feat2:B-feat1:A - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - for(MorphoFeatureType feat : MorphoFeatureType.values()) - if(fSpec.containsKey(feat)) - sb.append(String.format("-%s%s%s",feat.toString(),KEY_VAL_DELIM,fSpec.get(feat))); - - return sb.toString(); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/BZip2PipedOutputStream.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/BZip2PipedOutputStream.java deleted file mode 100644 index bae738e..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/BZip2PipedOutputStream.java +++ /dev/null @@ -1,64 +0,0 @@ -package edu.stanford.nlp.io; - -import edu.stanford.nlp.util.ByteStreamGobbler; -import edu.stanford.nlp.util.StreamGobbler; - -import java.io.*; - -/** -* Opens a outputstream for writing into a bzip2 file by piping into the bzip2 command. -* Output from bzip2 command is written into the specified file. -* -* @author Angel Chang -*/ -public class BZip2PipedOutputStream extends OutputStream -{ - private String filename; - private Process process; - private ByteStreamGobbler outGobbler; - private StreamGobbler errGobbler; - private PrintWriter errWriter; - - public BZip2PipedOutputStream(String filename) throws IOException { - this(filename, System.err); - } - - public BZip2PipedOutputStream(String filename, OutputStream err) throws IOException { - String bzip2 = System.getProperty("bzip2", "bzip2"); - String cmd = bzip2; // + " > " + filename; - //System.err.println("getBZip2PipedOutputStream: Running command: "+cmd); - ProcessBuilder pb = new ProcessBuilder(); - pb.command(cmd); - this.process = pb.start(); - this.filename = filename; - OutputStream outStream = new FileOutputStream(filename); - errWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(err))); - outGobbler = new ByteStreamGobbler("Output stream gobbler: " + cmd + " " + filename, - process.getInputStream(), outStream); - errGobbler = new StreamGobbler(process.getErrorStream(), errWriter); - outGobbler.start(); - errGobbler.start(); - } - - public void flush() throws IOException - { - process.getOutputStream().flush(); - } - - public void write(int b) throws IOException - { - process.getOutputStream().write(b); - } - - public void close() throws IOException - { - process.getOutputStream().close(); - try { - outGobbler.join(); - errGobbler.join(); - outGobbler.getOutputStream().close(); - process.waitFor(); - } catch (InterruptedException ex) {} - //System.err.println("getBZip2PipedOutputStream: Closed. "); - } -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/EncodingFileReader.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/EncodingFileReader.java deleted file mode 100644 index eb8d1d0..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/EncodingFileReader.java +++ /dev/null @@ -1,92 +0,0 @@ -package edu.stanford.nlp.io; - -import java.io.*; - -/** - * This is a convenience class which works almost exactly like - * FileReader - * but allows for the specification of input encoding. - * - * @author Alex Kleeman - */ - -public class EncodingFileReader extends InputStreamReader { - - private static final String DEFAULT_ENCODING = "UTF-8"; - /** - * Creates a new EncodingFileReader, given the name of the - * file to read from. - * - * @param fileName the name of the file to read from - * @exception java.io.FileNotFoundException if the named file does not - * exist, is a directory rather than a regular file, - * or for some other reason cannot be opened for - * reading. - * @exception java.io.UnsupportedEncodingException if the encoding does not exist. - * - */ - public EncodingFileReader(String fileName) throws UnsupportedEncodingException, FileNotFoundException { - super(new FileInputStream(fileName), DEFAULT_ENCODING); - } - - /** - * Creates a new EncodingFileReader, given the name of the - * file to read from and an encoding - * - * @param fileName the name of the file to read from - * @param encoding String specifying the encoding to be used - * @exception java.io.UnsupportedEncodingException if the encoding does not exist. - * @exception java.io.FileNotFoundException if the named file does not exist, - * is a directory rather than a regular file, - * or for some other reason cannot be opened for - * reading. - * - */ - public EncodingFileReader(String fileName, String encoding) throws UnsupportedEncodingException, FileNotFoundException { - super(new FileInputStream(fileName), - encoding == null ? DEFAULT_ENCODING: encoding); - } - - /** - * Creates a new EncodingFileReader, given the File - * to read from, and using default of utf-8. - * - * @param file the File to read from - * @exception FileNotFoundException if the file does not exist, - * is a directory rather than a regular file, - * or for some other reason cannot be opened for - * reading. - * @exception java.io.UnsupportedEncodingException if the encoding does not exist. - */ - public EncodingFileReader(File file) throws UnsupportedEncodingException, FileNotFoundException { - super(new FileInputStream(file), DEFAULT_ENCODING); - } - - /** - * Creates a new FileReader, given the File - * to read from and encoding. - * - * @param file the File to read from - * @param encoding String specifying the encoding to be used - * @exception FileNotFoundException if the file does not exist, - * is a directory rather than a regular file, - * or for some other reason cannot be opened for - * reading. - * @exception java.io.UnsupportedEncodingException if the encoding does not exist. - */ - public EncodingFileReader(File file,String encoding) throws UnsupportedEncodingException, FileNotFoundException { - super(new FileInputStream(file), - encoding == null ? DEFAULT_ENCODING: encoding); - } - - /** - * Creates a new FileReader, given the - * FileDescriptor to read from. - * - * @param fd the FileDescriptor to read from - */ - public EncodingFileReader(FileDescriptor fd) { - super(new FileInputStream(fd)); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/EncodingPrintWriter.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/EncodingPrintWriter.java deleted file mode 100644 index a088789..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/EncodingPrintWriter.java +++ /dev/null @@ -1,125 +0,0 @@ -package edu.stanford.nlp.io; - -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.UnsupportedEncodingException; - -/** - * A convenience IO class with print and println statements to - * standard output and standard error allowing encoding in an - * arbitrary character set. It also provides methods which use UTF-8 - * always, overriding the system default charset. - * - * @author Roger Levy - * @author Christopher Manning - */ - -public class EncodingPrintWriter { - - private static final String DEFAULT_ENCODING = "UTF-8"; - - private static PrintWriter cachedErrWriter; - private static String cachedErrEncoding = ""; - - private static PrintWriter cachedOutWriter; - private static String cachedOutEncoding = ""; - - // uninstantiable - private EncodingPrintWriter() {} - - /** - * Print methods wrapped around System.err - */ - public static class err { - - private err() {} // uninstantiable - - private static void setupErrWriter(String encoding) { - if (encoding == null) { - encoding = DEFAULT_ENCODING; - } - if (cachedErrWriter == null || ! cachedErrEncoding.equals(encoding)) { - try { - cachedErrWriter = new PrintWriter(new OutputStreamWriter(System.err, encoding), true); - cachedErrEncoding = encoding; - } catch (UnsupportedEncodingException e) { - System.err.println("Error " + e + "Printing as default encoding."); - cachedErrWriter = new PrintWriter(new OutputStreamWriter(System.err), true); - cachedErrEncoding = ""; - } - } - } - - public static void println(String o, String encoding) { - setupErrWriter(encoding); - cachedErrWriter.println(o); - } - - public static void print(String o, String encoding) { - setupErrWriter(encoding); - cachedErrWriter.print(o); - cachedErrWriter.flush(); - } - - public static void println(String o) { - println(o, null); - } - - public static void print(String o) { - print(o, null); - } - - } // end static class err - - - /** - * Print methods wrapped around System.out - */ - public static class out { - - private out() {} // uninstantiable - - private static void setupOutWriter(String encoding) { - if (encoding == null) { - encoding = DEFAULT_ENCODING; - } - if (cachedOutWriter == null || ! cachedOutEncoding.equals(encoding)) { - try { - cachedOutWriter = new PrintWriter(new OutputStreamWriter(System.out, encoding), true); - cachedOutEncoding = encoding; - } catch (UnsupportedEncodingException e) { - System.err.println("Error " + e + "Printing as default encoding."); - cachedOutWriter = new PrintWriter(new OutputStreamWriter(System.out), true); - cachedOutEncoding = ""; - } - } - } - - public static void println(String o, String encoding) { - setupOutWriter(encoding); - cachedOutWriter.println(o); - - } - - public static void print(String o, String encoding) { - setupOutWriter(encoding); - cachedOutWriter.print(o); - cachedOutWriter.flush(); - } - - /** Print the argument plus a NEWLINE in UTF-8, regardless of - * the platform default. - * - * @param o String to print - */ - public static void println(String o) { - println(o, null); - } - - public static void print(String o) { - print(o, null); - } - - } // end static class out - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/ExtensionFileFilter.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/ExtensionFileFilter.java deleted file mode 100644 index 4196b51..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/ExtensionFileFilter.java +++ /dev/null @@ -1,75 +0,0 @@ -package edu.stanford.nlp.io; - -import java.io.File; - -/** - * Implements a file filter that uses file extensions to filter files. - * - * @author cmanning 2000/01/24 - */ -public class ExtensionFileFilter extends javax.swing.filechooser.FileFilter implements java.io.FileFilter { - - private String extension; // = null - private boolean recursively; - - /** - * Sets up Extension file filter by specifying an extension - * to accept (currently only 1) and whether to also display - * folders for recursive search. - * The passed extension may be null, in which case the filter - * will pass all files (passing an empty String does not have the same - * effect -- this would look for file names ending in a period). - * - * @param ext File extension (not including period) or null for any - * @param recurse go into folders - */ - public ExtensionFileFilter(String ext, boolean recurse) { - if (ext != null) { - if (ext.startsWith(".")) { - extension = ext; - } else { - extension = '.' + ext; - } - } - recursively = recurse; - } - - /** - * Sets up an extension file filter that will recurse into sub directories. - * @param ext The extension to accept (with or without a leading period). - */ - public ExtensionFileFilter(String ext) { - this(ext, true); - } - - /** - * Checks whether a file satisfies the selection filter. - * - * @param file The file - * @return true if the file is acceptable - */ - @Override - public boolean accept(File file) { - if (file.isDirectory()) { - return recursively; - } else if (extension == null) { - return true; - } else { - return file.getName().endsWith(extension); - } - } - - /** - * Returns a description of what extension is being used (for file choosers). - * For example, if the suffix is "xml", the description will be - * "XML Files (*.xml)". - * - * @return description of this file filter - */ - @Override - public String getDescription() { - String ucExt = extension.substring(1).toUpperCase(); - return (ucExt + " Files (*" + extension + ')'); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/IOUtils.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/IOUtils.java deleted file mode 100644 index 9f2f7d2..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/IOUtils.java +++ /dev/null @@ -1,1148 +0,0 @@ -package edu.stanford.nlp.io; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.Reader; -import java.io.Writer; -import java.lang.reflect.InvocationTargetException; -import java.net.InetAddress; -import java.net.SocketTimeoutException; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Queue; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -import edu.stanford.nlp.util.AbstractIterator; -import edu.stanford.nlp.util.ErasureUtils; -import edu.stanford.nlp.util.Generics; -import edu.stanford.nlp.util.StreamGobbler; -import edu.stanford.nlp.util.StringUtils; - -/** - * Helper Class for storing serialized objects to disk. - * - * @author Kayur Patel, Teg Grenager - */ - -public class IOUtils { - - private static final int SLURPBUFFSIZE = 16000; - - public static final String eolChar = System.getProperty("line.separator"); - private static final String defaultEnc = "utf-8"; - - // A class of static methods - private IOUtils() { - } - - /** - * Write object to a file with the specified name. - * - * @param o - * object to be written to file - * @param filename - * name of the temp file - * @throws IOException - * If can't write file. - * @return File containing the object - */ - public static File writeObjectToFile(Object o, String filename) - throws IOException { - return writeObjectToFile(o, new File(filename)); - } - - /** - * Write an object to a specified File. - * - * @param o - * object to be written to file - * @param file - * The temp File - * @throws IOException - * If File cannot be written - * @return File containing the object - */ - public static File writeObjectToFile(Object o, File file) throws IOException { - return writeObjectToFile(o, file, false); - } - - /** - * Write an object to a specified File. - * - * @param o - * object to be written to file - * @param file - * The temp File - * @param append If true, append to this file instead of overwriting it - * @throws IOException - * If File cannot be written - * @return File containing the object - */ - public static File writeObjectToFile(Object o, File file, boolean append) throws IOException { - // file.createNewFile(); // cdm may 2005: does nothing needed - ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream( - new GZIPOutputStream(new FileOutputStream(file, append)))); - oos.writeObject(o); - oos.close(); - return file; - } - - /** - * Write object to a file with the specified name. - * - * @param o - * object to be written to file - * @param filename - * name of the temp file - * - * @return File containing the object, or null if an exception was caught - */ - public static File writeObjectToFileNoExceptions(Object o, String filename) { - File file = null; - ObjectOutputStream oos = null; - try { - file = new File(filename); - // file.createNewFile(); // cdm may 2005: does nothing needed - oos = new ObjectOutputStream(new BufferedOutputStream( - new GZIPOutputStream(new FileOutputStream(file)))); - oos.writeObject(o); - oos.close(); - } catch (Exception e) { - e.printStackTrace(); - } finally { - closeIgnoringExceptions(oos); - } - return file; - } - - /** - * Write object to temp file which is destroyed when the program exits. - * - * @param o - * object to be written to file - * @param filename - * name of the temp file - * @throws IOException - * If file cannot be written - * @return File containing the object - */ - public static File writeObjectToTempFile(Object o, String filename) - throws IOException { - File file = File.createTempFile(filename, ".tmp"); - file.deleteOnExit(); - ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream( - new GZIPOutputStream(new FileOutputStream(file)))); - oos.writeObject(o); - oos.close(); - return file; - } - - /** - * Write object to a temp file and ignore exceptions. - * - * @param o - * object to be written to file - * @param filename - * name of the temp file - * @return File containing the object - */ - public static File writeObjectToTempFileNoExceptions(Object o, String filename) { - try { - return writeObjectToTempFile(o, filename); - } catch (Exception e) { - System.err.println("Error writing object to file " + filename); - e.printStackTrace(); - return null; - } - } - - - /** - * Read an object from a stored file. - * - * @param file - * the file pointing to the object to be retrived - * @throws IOException - * If file cannot be read - * @throws ClassNotFoundException - * If reading serialized object fails - * @return the object read from the file. - */ - public static T readObjectFromFile(File file) throws IOException, - ClassNotFoundException { - ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream( - new GZIPInputStream(new FileInputStream(file)))); - Object o = ois.readObject(); - ois.close(); - return ErasureUtils. uncheckedCast(o); - } - - public static T readObjectFromObjectStream(ObjectInputStream ois) throws IOException, - ClassNotFoundException { - Object o = ois.readObject(); - return ErasureUtils. uncheckedCast(o); - } - - /** - * Read an object from a stored file. - * - * @param filename - * The filename of the object to be retrieved - * @throws IOException - * If file cannot be read - * @throws ClassNotFoundException - * If reading serialized object fails - * @return The object read from the file. - */ - public static T readObjectFromFile(String filename) throws IOException, - ClassNotFoundException { - return ErasureUtils - . uncheckedCast(readObjectFromFile(new File(filename))); - } - - /** - * Read an object from a stored file without throwing exceptions. - * - * @param file - * the file pointing to the object to be retrieved - * @return the object read from the file, or null if an exception occurred. - */ - public static T readObjectFromFileNoExceptions(File file) { - Object o = null; - try { - ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream( - new GZIPInputStream(new FileInputStream(file)))); - o = ois.readObject(); - ois.close(); - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - return ErasureUtils. uncheckedCast(o); - } - - public static int lineCount(File textFile) throws IOException { - BufferedReader r = new BufferedReader(new FileReader(textFile)); - int numLines = 0; - while (r.readLine() != null) { - numLines++; - } - return numLines; - } - - public static ObjectOutputStream writeStreamFromString(String serializePath) - throws IOException { - ObjectOutputStream oos; - if (serializePath.endsWith(".gz")) { - oos = new ObjectOutputStream(new BufferedOutputStream( - new GZIPOutputStream(new FileOutputStream(serializePath)))); - } else { - oos = new ObjectOutputStream(new BufferedOutputStream( - new FileOutputStream(serializePath))); - } - - return oos; - } - - public static ObjectInputStream readStreamFromString(String filenameOrUrl) - throws IOException { - ObjectInputStream in; - InputStream is = getInputStreamFromURLOrClasspathOrFileSystem(filenameOrUrl); - in = new ObjectInputStream(is); - return in; - } - - /** - * Locates this file either in the CLASSPATH or in the file system. The CLASSPATH takes priority - * @param fn - * @return - * @throws FileNotFoundException - * if the file does not exist - */ - private static InputStream findStreamInClasspathOrFileSystem(String fn) throws FileNotFoundException { - // ms 10-04-2010: - // - even though this may look like a regular file, it may be a path inside a jar in the CLASSPATH - // - check for this first. This takes precedence over the file system. - InputStream is = IOUtils.class.getClassLoader().getResourceAsStream(fn); - // if not found in the CLASSPATH, load from the file system - if (is == null) is = new FileInputStream(fn); - return is; - } - - /** - * Locates this file either using the given URL, or in the CLASSPATH, or in the file system - * The CLASSPATH takes priority over the file system! - * This stream is buffered and gzipped (if necessary) - * @param textFileOrUrl - * @return An InputStream for loading a resource - * @throws IOException - */ - public static InputStream - getInputStreamFromURLOrClasspathOrFileSystem(String textFileOrUrl) - throws IOException - { - InputStream in; - if (textFileOrUrl.matches("https?://.*")) { - URL u = new URL(textFileOrUrl); - URLConnection uc = u.openConnection(); - in = uc.getInputStream(); - } else { - try { - in = findStreamInClasspathOrFileSystem(textFileOrUrl); - } catch (FileNotFoundException e) { - try { - // Maybe this happens to be some other format of URL? - URL u = new URL(textFileOrUrl); - URLConnection uc = u.openConnection(); - in = uc.getInputStream(); - } catch (IOException e2) { - // TODO: freaking Java 1.5 didn't have an IOException that - // could take a throwable as a cause - throw new IOException("Unable to resolve \"" + - textFileOrUrl + "\" as either " + - "class path, filename or URL"); - } - } - } - - // buffer this stream - in = new BufferedInputStream(in); - - // gzip it if necessary - if(textFileOrUrl.endsWith(".gz")) - in = new GZIPInputStream(in); - - return in; - } - - public static BufferedReader readReaderFromString(String textFileOrUrl) - throws IOException { - return new BufferedReader(new InputStreamReader( - getInputStreamFromURLOrClasspathOrFileSystem(textFileOrUrl))); - } - - /** - * Open a BufferedReader to a file or URL specified by a String name. If the - * String starts with https?://, then it is interpreted as a URL, otherwise it - * is interpreted as a local file. If the String ends in .gz, it is - * interpreted as a gzipped file (and uncompressed), else it is interpreted as - * a regular text file in the given encoding. - * - * @param textFileOrUrl - * What to read from - * @param encoding - * CharSet encoding - * @return The BufferedReader - * @throws IOException - * If there is an I/O problem - */ - public static BufferedReader readReaderFromString(String textFileOrUrl, - String encoding) throws IOException { - InputStream is = getInputStreamFromURLOrClasspathOrFileSystem(textFileOrUrl); - return new BufferedReader(new InputStreamReader(is, encoding)); - } - - /** - * Returns an Iterable of the lines in the file. - * - * The file reader will be closed when the iterator is exhausted. IO errors - * will throw an (unchecked) RuntimeIOException - * - * @param path - * The file whose lines are to be read. - * @return An Iterable containing the lines from the file. - */ - public static Iterable readLines(String path) { - return readLines(new File(path)); - } - - /** - * Returns an Iterable of the lines in the file. - * - * The file reader will be closed when the iterator is exhausted. - * - * @param file - * The file whose lines are to be read. - * @return An Iterable containing the lines from the file. - */ - public static Iterable readLines(final File file) { - return readLines(file, null); - } - - /** - * Returns an Iterable of the lines in the file, wrapping the generated - * FileInputStream with an instance of the supplied class. IO errors will - * throw an (unchecked) RuntimeIOException - * - * @param file - * The file whose lines are to be read. - * @param fileInputStreamWrapper - * The class to wrap the InputStream with, e.g. GZIPInputStream. Note - * that the class must have a constructor that accepts an - * InputStream. - * @return An Iterable containing the lines from the file. - */ - public static Iterable readLines(final File file, - final Class fileInputStreamWrapper) { - - return new Iterable() { - public Iterator iterator() { - return new Iterator() { - - protected BufferedReader reader = this.getReader(); - protected String line = this.getLine(); - - public boolean hasNext() { - return this.line != null; - } - - public String next() { - String nextLine = this.line; - if (nextLine == null) { - throw new NoSuchElementException(); - } - line = getLine(); - return nextLine; - } - - protected String getLine() { - try { - String result = this.reader.readLine(); - if (result == null) { - this.reader.close(); - } - return result; - } catch (IOException e) { - throw new RuntimeIOException(e); - } - } - - protected BufferedReader getReader() { - try { - InputStream stream = new FileInputStream(file); - if (fileInputStreamWrapper != null) { - stream = fileInputStreamWrapper.getConstructor( - InputStream.class).newInstance(stream); - } - return new BufferedReader(new InputStreamReader(stream)); - } catch (Exception e) { - throw new RuntimeIOException(e); - } - } - - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - }; - } - - /** - * Quietly opens a File. If the file ends with a ".gz" extension, - * automatically opens a GZIPInputStream to wrap the constructed - * FileInputStream. - */ - public static InputStream openFile(File file) throws RuntimeIOException { - try { - InputStream is = new BufferedInputStream(new FileInputStream(file)); - if (file.getName().endsWith(".gz")) { - is = new GZIPInputStream(is); - } - return is; - } catch (Exception e) { - throw new RuntimeIOException(e); - } - } - - /** - * Provides an implementation of closing a file for use in a finally block so - * you can correctly close a file without even more exception handling stuff. - * From a suggestion in a talk by Josh Bloch. - * - * @param c - * The IO resource to close (e.g., a Stream/Reader) - */ - public static void closeIgnoringExceptions(Closeable c) { - if (c != null) { - try { - c.close(); - } catch (IOException ioe) { - // ignore - } - } - } - - /** - * Iterate over all the files in the directory, recursively. - * - * @param dir - * The root directory. - * @return All files within the directory. - */ - public static Iterable iterFilesRecursive(final File dir) { - return iterFilesRecursive(dir, (Pattern) null); - } - - /** - * Iterate over all the files in the directory, recursively. - * - * @param dir - * The root directory. - * @param ext - * A string that must be at the end of all files (e.g. ".txt") - * @return All files within the directory ending in the given extension. - */ - public static Iterable iterFilesRecursive(final File dir, - final String ext) { - return iterFilesRecursive(dir, Pattern.compile(Pattern.quote(ext) + "$")); - } - - /** - * Iterate over all the files in the directory, recursively. - * - * @param dir - * The root directory. - * @param pattern - * A regular expression that the file path must match. This uses - * Matcher.find(), so use ^ and $ to specify endpoints. - * @return All files within the directory. - */ - public static Iterable iterFilesRecursive(final File dir, - final Pattern pattern) { - return new Iterable() { - public Iterator iterator() { - return new AbstractIterator() { - private final Queue files = new LinkedList(Collections - .singleton(dir)); - private File file = this.findNext(); - - @Override - public boolean hasNext() { - return this.file != null; - } - - @Override - public File next() { - File result = this.file; - if (result == null) { - throw new NoSuchElementException(); - } - this.file = this.findNext(); - return result; - } - - private File findNext() { - File next = null; - while (!this.files.isEmpty() && next == null) { - next = this.files.remove(); - if (next.isDirectory()) { - files.addAll(Arrays.asList(next.listFiles())); - next = null; - } else if (pattern != null) { - if (!pattern.matcher(next.getPath()).find()) { - next = null; - } - } - } - return next; - } - }; - } - }; - } - - /** - * Returns all the text in the given File. - */ - public static String slurpFile(File file) throws IOException { - Reader r = new FileReader(file); - return IOUtils.slurpReader(r); - } - - /** - * Returns all the text in the given File. - * - * @param file The file to read from - * @param encoding The character encoding to assume. This may be null, and - * the platform default character encoding is used. - */ - public static String slurpFile(File file, String encoding) throws IOException { - Reader r; - // InputStreamReader doesn't allow encoding to be null; - if (encoding == null) { - r = new InputStreamReader(new FileInputStream(file)); - } else { - r = new InputStreamReader(new FileInputStream(file), encoding); - } - return IOUtils.slurpReader(r); - } - - /** - * Returns all the text in the given File. - */ - public static String slurpGZippedFile(String filename) throws IOException { - Reader r = new InputStreamReader(new GZIPInputStream(new FileInputStream( - filename))); - return IOUtils.slurpReader(r); - } - - /** - * Returns all the text in the given File. - */ - public static String slurpGZippedFile(File file) throws IOException { - Reader r = new InputStreamReader(new GZIPInputStream(new FileInputStream( - file))); - return IOUtils.slurpReader(r); - } - - public static String slurpGBFileNoExceptions(String filename) { - return IOUtils.slurpFileNoExceptions(filename, "GB18030"); - } - - /** - * Returns all the text in the given file with the given encoding. - */ - public static String slurpFile(String filename, String encoding) - throws IOException { - Reader r = new InputStreamReader(new FileInputStream(filename), encoding); - return IOUtils.slurpReader(r); - } - - /** - * Returns all the text in the given file with the given encoding. If the file - * cannot be read (non-existent, etc.), then and only then the method returns - * null. - */ - public static String slurpFileNoExceptions(String filename, String encoding) { - try { - return slurpFile(filename, encoding); - } catch (Exception e) { - throw new RuntimeIOException("slurpFile IO problem", e); - } - } - - public static String slurpGBFile(String filename) throws IOException { - return slurpFile(filename, "GB18030"); - } - - /** - * Returns all the text in the given file - * - * @return The text in the file. - */ - public static String slurpFile(String filename) throws IOException { - return IOUtils.slurpReader(new FileReader(filename)); - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpGBURL(URL u) throws IOException { - return IOUtils.slurpURL(u, "GB18030"); - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpGBURLNoExceptions(URL u) { - try { - return slurpGBURL(u); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpURLNoExceptions(URL u, String encoding) { - try { - return IOUtils.slurpURL(u, encoding); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpURL(URL u, String encoding) throws IOException { - String lineSeparator = System.getProperty("line.separator"); - URLConnection uc = u.openConnection(); - uc.setReadTimeout(30000); - InputStream is; - try { - is = uc.getInputStream(); - } catch (SocketTimeoutException e) { - // e.printStackTrace(); - System.err.println("Time out. Return empty string"); - return ""; - } - BufferedReader br = new BufferedReader(new InputStreamReader(is, encoding)); - String temp; - StringBuilder buff = new StringBuilder(16000); // make biggish - while ((temp = br.readLine()) != null) { - buff.append(temp); - buff.append(lineSeparator); - } - br.close(); - return buff.toString(); - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpURL(URL u) throws IOException { - String lineSeparator = System.getProperty("line.separator"); - URLConnection uc = u.openConnection(); - InputStream is = uc.getInputStream(); - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - String temp; - StringBuilder buff = new StringBuilder(16000); // make biggish - while ((temp = br.readLine()) != null) { - buff.append(temp); - buff.append(lineSeparator); - } - br.close(); - return buff.toString(); - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpURLNoExceptions(URL u) { - try { - return slurpURL(u); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - /** - * Returns all the text at the given URL. - */ - public static String slurpURL(String path) throws Exception { - return slurpURL(new URL(path)); - } - - /** - * Returns all the text at the given URL. If the file cannot be read - * (non-existent, etc.), then and only then the method returns - * null. - */ - public static String slurpURLNoExceptions(String path) { - try { - return slurpURL(path); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - /** - * Returns all the text in the given File. - * - * @return The text in the file. May be an empty string if the file is empty. - * If the file cannot be read (non-existent, etc.), then and only then - * the method returns null. - */ - public static String slurpFileNoExceptions(File file) { - try { - return IOUtils.slurpReader(new FileReader(file)); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - /** - * Returns all the text in the given File. - * - * @return The text in the file. May be an empty string if the file is empty. - * If the file cannot be read (non-existent, etc.), then and only then - * the method returns null. - */ - public static String slurpFileNoExceptions(String filename) { - try { - return slurpFile(filename); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - /** - * Returns all the text from the given Reader. - * - * @return The text in the file. - */ - public static String slurpReader(Reader reader) { - BufferedReader r = new BufferedReader(reader); - StringBuilder buff = new StringBuilder(); - try { - char[] chars = new char[SLURPBUFFSIZE]; - while (true) { - int amountRead = r.read(chars, 0, SLURPBUFFSIZE); - if (amountRead < 0) { - break; - } - buff.append(chars, 0, amountRead); - } - r.close(); - } catch (Exception e) { - throw new RuntimeIOException("slurpReader IO problem", e); - } - return buff.toString(); - } - - /** - * Send all bytes from the input stream to the output stream. - * - * @param input - * The input bytes. - * @param output - * Where the bytes should be written. - */ - public static void writeStreamToStream(InputStream input, OutputStream output) - throws IOException { - byte[] buffer = new byte[4096]; - while (true) { - int len = input.read(buffer); - if (len == -1) { - break; - } - output.write(buffer, 0, len); - } - } - - /** - * Read in a CSV formatted file with a header row - * @param path - path to CSV file - * @param quoteChar - character for enclosing strings, defaults to " - * @param escapeChar - character for escaping quotes appearing in quoted strings; defaults to " (i.e. "" is used for " inside quotes, consistent with Excel) - * @return a list of maps representing the rows of the csv. The maps' keys are the header strings and their values are the row contents - * @throws IOException - */ - public static List> readCSVWithHeader(String path, char quoteChar, char escapeChar) throws IOException { - String[] labels = null; - List> rows = Generics.newArrayList(); - for (String line : IOUtils.readLines(path)) { - System.out.println("Splitting "+line); - if (labels == null) { - labels = StringUtils.splitOnCharWithQuoting(line,',','"',escapeChar); - } else { - String[] cells = StringUtils.splitOnCharWithQuoting(line,',',quoteChar,escapeChar); - assert(cells.length == labels.length); - Map cellMap = new HashMap(); - for (int i=0; i> readCSVWithHeader(String path) throws IOException { - return readCSVWithHeader(path,'"','"'); - } - - /** - * Get a input file stream (automatically gunzip/bunzip2 depending on file extension) - * @param filename Name of file to open - * @return Input stream that can be used to read from the file - * @throws IOException if there are exceptions opening the file - */ - public static InputStream getFileInputStream(String filename) throws IOException { - InputStream in = new FileInputStream(filename); - if (filename.endsWith(".gz")) { - in = new GZIPInputStream(in); - } else if (filename.endsWith(".bz2")) { - //in = new CBZip2InputStream(in); - in = getBZip2PipedInputStream(filename); - } - return in; - } - - /** - * Get a output file stream (automatically gzip/bzip2 depending on file extension) - * @param filename Name of file to open - * @return Output stream that can be used to write to the file - * @throws IOException if there are exceptions opening the file - */ - public static OutputStream getFileOutputStream(String filename) throws IOException { - OutputStream out = new FileOutputStream(filename); - if (filename.endsWith(".gz")) { - out = new GZIPOutputStream(out); - } else if (filename.endsWith(".bz2")) { - //out = new CBZip2OutputStream(out); - out = getBZip2PipedOutputStream(filename); - } - return out; - } - - public static BufferedReader getBufferedFileReader(String filename) throws IOException { - return getBufferedFileReader(filename, defaultEnc); - } - - public static BufferedReader getBufferedFileReader(String filename, String encoding) throws IOException { - InputStream in = getFileInputStream(filename); - return new BufferedReader(new InputStreamReader(in, encoding)); - } - - public static PrintWriter getPrintWriter(File textFile) throws IOException { - File f = textFile.getAbsoluteFile(); - return new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f)))); - } - - public static PrintWriter getPrintWriter(String filename) throws IOException { - return getPrintWriter(filename, defaultEnc); - } - - public static PrintWriter getPrintWriter(String filename, String encoding) throws IOException { - OutputStream out = getFileOutputStream(filename); - return new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, encoding))); - } - - public static InputStream getBZip2PipedInputStream(String filename) throws IOException - { - String bzcat = System.getProperty("bzcat", "bzcat"); - Runtime rt = Runtime.getRuntime(); - String cmd = bzcat + " " + filename; - //System.err.println("getBZip2PipedInputStream: Running command: "+cmd); - Process p = rt.exec(cmd); - Writer errWriter = new BufferedWriter(new OutputStreamWriter(System.err)); - StreamGobbler errGobler = new StreamGobbler(p.getErrorStream(), errWriter); - errGobler.start(); - return p.getInputStream(); - } - - public static OutputStream getBZip2PipedOutputStream(String filename) throws IOException - { - return new BZip2PipedOutputStream(filename); - } - - private static final Pattern tab = Pattern.compile("\t"); - /** - * Read column as set - * @param infile - filename - * @param field index of field to read - * @return a set of the entries in column field - * @throws IOException - */ - public static Set readColumnSet(String infile, int field) throws IOException - { - BufferedReader br = IOUtils.getBufferedFileReader(infile); - String line; - Set set = new HashSet(); - while ((line = br.readLine()) != null) { - line = line.trim(); - if (line.length() > 0) { - if (field < 0) { - set.add(line); - } else { - String[] fields = tab.split(line); - if (field < fields.length) { - set.add(fields[field]); - } - } - } - } - br.close(); - return set; - } - - public static List readObjectFromColumns(Class objClass, String filename, String[] fieldNames, String delimiter) - throws IOException, InstantiationException, IllegalAccessException, - NoSuchFieldException, NoSuchMethodException, InvocationTargetException - { - Pattern delimiterPattern = Pattern.compile(delimiter); - List list = new ArrayList(); - BufferedReader br = IOUtils.getBufferedFileReader(filename); - String line; - while ((line = br.readLine()) != null) { - line = line.trim(); - if (line.length() > 0) { - C item = StringUtils.columnStringToObject(objClass, line, delimiterPattern, fieldNames); - list.add(item); - } - } - br.close(); - return list; - } - - public static Map readMap(String filename) throws IOException - { - Map map = new HashMap(); - try { - BufferedReader br = IOUtils.getBufferedFileReader(filename); - String line; - while ((line = br.readLine()) != null) { - String[] fields = tab.split(line,2); - map.put(fields[0], fields[1]); - } - } catch (IOException ex) { - throw new RuntimeException(ex); - } - return map; - } - - - /** - * Returns the contents of a file as a single string. The string may be - * empty, if the file is empty. If there is an IOException, it is caught - * and null is returned. - */ - public static String stringFromFile(String filename) { - return stringFromFile(filename,defaultEnc); - } - - /** - * Returns the contents of a file as a single string. The string may be - * empty, if the file is empty. If there is an IOException, it is caught - * and null is returned. Encoding can also be specified. - */ - public static String stringFromFile(String filename, String encoding) { - try { - StringBuilder sb = new StringBuilder(); - BufferedReader in = new BufferedReader(new EncodingFileReader(filename,encoding)); - String line; - while ((line = in.readLine()) != null) { - sb.append(line); - sb.append(eolChar); - } - in.close(); - return sb.toString(); - } - catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - - /** - * Returns the contents of a file as a list of strings. The list may be - * empty, if the file is empty. If there is an IOException, it is caught - * and null is returned. - */ - public static List linesFromFile(String filename) { - return linesFromFile(filename,defaultEnc); - } - - /** - * Returns the contents of a file as a list of strings. The list may be - * empty, if the file is empty. If there is an IOException, it is caught - * and null is returned. Encoding can also be specified - */ - public static List linesFromFile(String filename,String encoding) { - try { - List lines = new ArrayList(); - BufferedReader in = new BufferedReader(new EncodingFileReader(filename,encoding)); - String line; - while ((line = in.readLine()) != null) { - lines.add(line); - } - in.close(); - return lines; - } - catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - public static String backupName(String filename) { - return backupFile(new File(filename)).toString(); - } - - public static File backupFile(File file) { - int max = 1000; - String filename = file.toString(); - File backup = new File(filename + "~"); - if (!backup.exists()) { return backup; } - for (int i = 1; i <= max; i++) { - backup = new File(filename + ".~" + i + ".~"); - if (!backup.exists()) { return backup; } - } - return null; - } - - public static boolean renameToBackupName(File file) { - return file.renameTo(backupFile(file)); - } - - - /** - * A JavaNLP specific convenience routine for obtaining the current - * scratch directory for the machine you're currently running on. - */ - public static File getJNLPLocalScratch() { - try { - String machineName = InetAddress.getLocalHost().getHostName().split("\\.")[0]; - String username = System.getProperty("user.name"); - return new File("/"+machineName+"/scr1/"+username); - } catch (Exception e) { - return new File("./scr/"); // default scratch - } - } - - /** - * Given a filepath, makes sure a directory exists there. If not, creates and returns it. - * Same as ENSURE-DIRECTORY in CL. - * @throws Exception - */ - public static File ensureDir(File tgtDir) throws Exception { - if (tgtDir.exists()) { - if (tgtDir.isDirectory()) return tgtDir; - else - throw new Exception("Could not create directory "+tgtDir.getAbsolutePath()+", as a file already exists at that path."); - } else { - tgtDir.mkdirs(); - return tgtDir; - } - } - - public static void main(String[] args) { - System.out.println(backupName(args[0])); - } - - public static String getExtension(String fileName) { - if(!fileName.contains(".")) - return null; - int idx = fileName.lastIndexOf("."); - return fileName.substring(idx+1); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/NumberRangesFileFilter.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/NumberRangesFileFilter.java deleted file mode 100644 index e84414c..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/NumberRangesFileFilter.java +++ /dev/null @@ -1,169 +0,0 @@ -package edu.stanford.nlp.io; - -import edu.stanford.nlp.util.Pair; - -import java.io.File; -import java.io.FileFilter; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -/** - * Implements a file filter that examines a number in a filename to - * determine acceptance. This is useful for wanting to process ranges - * of numbered files in collections where each file has some name, part - * of which is alphabetic and constant, and part of which is numeric. - * The test is evaluated based on the rightmost natural number found in - * the filename string. (It only looks in the final filename, not in other - * components of the path.) Number ranges are inclusive. - *

- * This filter can select multiple discontinuous ranges based on a format - * similar to page selection ranges in various formatting software, such as - * "34,52-65,67,93-95". The constructor takes a String of this sort and - * deconstructs it into a list of ranges. The accepted syntax is:

- * ranges = range
- * ranges = range "," ranges
- * range = integer
- * range = integer "-" integer

- * Whitespace will be ignored. If the filter constructor is passed anything - * that is not a list of numeric ranges of this sort, including being passed - * an empty String, then an IllegalArgumentException will be - * thrown. - * - * @author Christopher Manning - * @version 2003/03/31 - */ -public class NumberRangesFileFilter implements FileFilter { - - private List> ranges = new ArrayList>(); - private boolean recursively; - - - /** - * Sets up a NumberRangesFileFilter by specifying the ranges of numbers - * to accept, and whether to also traverse - * folders for recursive search. - * - * @param ranges The ranges of numbers to accept (see class documentation) - * @param recurse Whether to go into subfolders - * @throws IllegalArgumentException If the String ranges does not - * contain a suitable ranges format - */ - public NumberRangesFileFilter(String ranges, boolean recurse) { - recursively = recurse; - try { - String[] ra = ranges.split(","); - for (String range : ra) { - String[] one = range.split("-"); - if (one.length > 2) { - throw new IllegalArgumentException("Constructor argument not valid list of number ranges (too many hyphens): "); - } else { - int low = Integer.parseInt(one[0].trim()); - int high; - if (one.length == 2) { - high = Integer.parseInt(one[1].trim()); - } else { - high = low; - } - Pair p = new Pair(Integer.valueOf(low), Integer.valueOf(high)); - this.ranges.add(p); - } - } - } catch (Exception e) { - IllegalArgumentException iae = new IllegalArgumentException("Constructor argument not valid list of number ranges: " + ranges); - iae.initCause(e); - throw iae; - } - } - - - /** - * Checks whether a file satisfies the number range selection filter. - * The test is evaluated based on the rightmost natural number found in - * the filename string (proper, not including directories in a path). - * - * @param file The file - * @return true If the file is within the ranges filtered for - */ - public boolean accept(File file) { - if (file.isDirectory()) { - return recursively; - } else { - String filename = file.getName(); - return accept(filename); - } - } - - - /** - * Checks whether a String satisfies the number range selection filter. - * The test is evaluated based on the rightmost natural number found in - * the String. Note that this is just evaluated on the String as given. - * It is not trying to interpret it as a filename and to decide whether - * the file exists, is a directory or anything like that. - * - * @param str The String to check for a number in - * @return true If the String is within the ranges filtered for - */ - public boolean accept(String str) { - int k = str.length() - 1; - char c = str.charAt(k); - while (k >= 0 && !Character.isDigit(c)) { - k--; - if (k >= 0) { - c = str.charAt(k); - } - } - if (k < 0) { - return false; - } - int j = k; - c = str.charAt(j); - while (j >= 0 && Character.isDigit(c)) { - j--; - if (j >= 0) { - c = str.charAt(j); - } - } - j++; - k++; - String theNumber = str.substring(j, k); - int number = Integer.parseInt(theNumber); - for (Pair p : ranges) { - int low = p.first().intValue(); - int high = p.second().intValue(); - if (number >= low && number <= high) { - return true; - } - } - return false; - } - - - @Override - public String toString() { - StringBuilder sb; - if (recursively) { - sb = new StringBuilder("recursively "); - } else { - sb = new StringBuilder(); - } - for (Iterator> it = ranges.iterator(); it.hasNext(); ) { - Pair p = it.next(); - int low = p.first().intValue(); - int high = p.second().intValue(); - if (low == high) { - sb.append(low); - } else { - sb.append(low); - sb.append('-'); - sb.append(high); - } - if (it.hasNext()) { - sb.append(','); - } - } - return sb.toString(); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/RegExFileFilter.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/RegExFileFilter.java deleted file mode 100644 index bc6b4cf..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/RegExFileFilter.java +++ /dev/null @@ -1,36 +0,0 @@ -package edu.stanford.nlp.io; - -import java.io.*; -import java.util.regex.*; -/** - * Implements a file filter that filters based on a passed in {@link java.util.regex.Pattern}. - * Preciesly, it will accept exactly those {@link java.io.File}s for which - * the matches() method of the Pattern returns true on the output of the getName() - * method of the File. - * - * @author Jenny Finkel - */ -public class RegExFileFilter implements FileFilter { - - private Pattern pattern = null; - - /** - * Sets up a RegExFileFilter which checks if the file name (not the - * entire path) matches the passed in {@link java.util.regex.Pattern}. - */ - public RegExFileFilter(Pattern pattern) { - this.pattern = pattern; - } - - /** - * Checks whether a file satisfies the selection filter. - * - * @param file The file - * @return true if the file is acceptable - */ - public boolean accept(File file) { - Matcher m = pattern.matcher(file.getName()); - return m.matches(); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/RuntimeIOException.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/RuntimeIOException.java deleted file mode 100644 index 505779c..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/io/RuntimeIOException.java +++ /dev/null @@ -1,51 +0,0 @@ -package edu.stanford.nlp.io; - - -/** - * An unchecked version of {@link java.io.IOException}. Thrown by - * {@link edu.stanford.nlp.process.Tokenizer} implementing classes, - * among other things. - * - * @author Roger Levy - * @author Christopher Manning - */ -public class RuntimeIOException extends RuntimeException { - - private static final long serialVersionUID = -8572218999165094626L; - - /** - * Creates a new exception. - */ - public RuntimeIOException() { - } - - - /** - * Creates a new exception with a message. - * - * @param message the message for the exception - */ - public RuntimeIOException(String message) { - super(message); - } - - /** - * Creates a new exception with an embedded cause. - * - * @param cause The cause for the exception - */ - public RuntimeIOException(Throwable cause) { - super(cause); - } - - /** - * Creates a new exception with a message and an embedded cause. - * - * @param message the message for the exception - * @param cause The cause for the exception - */ - public RuntimeIOException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/AnnotationLookup.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/AnnotationLookup.java deleted file mode 100644 index 27d4c7e..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/AnnotationLookup.java +++ /dev/null @@ -1,230 +0,0 @@ -package edu.stanford.nlp.ling; - -import java.util.HashMap; -import java.util.Map; - -import edu.stanford.nlp.ling.CoreAnnotations.AfterAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.AnswerAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ArgumentAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.BeforeAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.CharacterOffsetBeginAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.CategoryAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.CategoryFunctionalTagAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ChineseCharAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ChineseOrigSegAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ChineseSegAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ChunkAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.OriginalTextAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.DocIDAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.CharacterOffsetEndAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.FeaturesAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.GazetteerAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.GoldAnswerAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.IDFAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.IndexAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.InterpretationAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.LeftTermAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.LemmaAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.MarkingAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.MorphoCaseAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.MorphoGenAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.MorphoNumAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.MorphoPersAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.NamedEntityTagAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.NormalizedNamedEntityTagAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ParentAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.PolarityAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ProjectedCategoryAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ProtoAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.RoleAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.SemanticHeadTagAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.SemanticHeadWordAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.SentenceIndexAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ShapeAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.SpanAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.StemAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.ValueAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.VerbSenseAnnotation; -import edu.stanford.nlp.ling.CoreAnnotations.WordnetSynAnnotation; -import edu.stanford.nlp.ling.CoreLabel.GenericAnnotation; -import edu.stanford.nlp.util.ErasureUtils; - -/** @author Anna Rafferty */ -public class AnnotationLookup { - - private AnnotationLookup() {} - - public enum KeyLookup { - VALUE_KEY(ValueAnnotation.class, OldFeatureLabelKeys.VALUE_KEY), - TAG_KEY(PartOfSpeechAnnotation.class, OldFeatureLabelKeys.TAG_KEY), - WORD_KEY(TextAnnotation.class, OldFeatureLabelKeys.WORD_KEY), - LEMMA_KEY(LemmaAnnotation.class, OldFeatureLabelKeys.LEMMA_KEY), - CATEGORY_KEY(CategoryAnnotation.class, OldFeatureLabelKeys.CATEGORY_KEY), - PROJ_CAT_KEY(ProjectedCategoryAnnotation.class, OldFeatureLabelKeys.PROJ_CAT_KEY), - HEAD_WORD_KEY("edu.stanford.nlp.ling.TreeCoreAnnotations.HeadWordAnnotation", OldFeatureLabelKeys.HEAD_WORD_KEY), - HEAD_TAG_KEY("edu.stanford.nlp.ling.TreeCoreAnnotations.HeadTagAnnotation", OldFeatureLabelKeys.HEAD_TAG_KEY), - INDEX_KEY(IndexAnnotation.class, OldFeatureLabelKeys.INDEX_KEY), - ARG_KEY(ArgumentAnnotation.class, OldFeatureLabelKeys.ARG_KEY), - MARKING_KEY(MarkingAnnotation.class, OldFeatureLabelKeys.MARKING_KEY), - SEMANTIC_HEAD_WORD_KEY(SemanticHeadWordAnnotation.class, OldFeatureLabelKeys.SEMANTIC_HEAD_WORD_KEY), - SEMANTIC_HEAD_POS_KEY(SemanticHeadTagAnnotation.class, OldFeatureLabelKeys.SEMANTIC_HEAD_POS_KEY), - VERB_SENSE_KEY(VerbSenseAnnotation.class, OldFeatureLabelKeys.VERB_SENSE_KEY), - CATEGORY_FUNCTIONAL_TAG_KEY(CategoryFunctionalTagAnnotation.class, OldFeatureLabelKeys.CATEGORY_FUNCTIONAL_TAG_KEY), - NER_KEY(NamedEntityTagAnnotation.class, OldFeatureLabelKeys.NER_KEY), - SHAPE_KEY(ShapeAnnotation.class, OldFeatureLabelKeys.SHAPE_KEY), - LEFT_TERM_KEY(LeftTermAnnotation.class, OldFeatureLabelKeys.LEFT_TERM_KEY), - PARENT_KEY(ParentAnnotation.class, OldFeatureLabelKeys.PARENT_KEY), - SPAN_KEY(SpanAnnotation.class, OldFeatureLabelKeys.SPAN_KEY), - BEFORE_KEY(BeforeAnnotation.class, OldFeatureLabelKeys.BEFORE_KEY), - AFTER_KEY(AfterAnnotation.class, OldFeatureLabelKeys.AFTER_KEY), - CURRENT_KEY(OriginalTextAnnotation.class, OldFeatureLabelKeys.CURRENT_KEY), - ANSWER_KEY(AnswerAnnotation.class, OldFeatureLabelKeys.ANSWER_KEY), - GOLDANSWER_Key(GoldAnswerAnnotation.class, OldFeatureLabelKeys.GOLDANSWER_KEY), - FEATURES_KEY(FeaturesAnnotation.class, OldFeatureLabelKeys.FEATURES_KEY), - INTERPRETATION_KEY(InterpretationAnnotation.class, OldFeatureLabelKeys.INTERPRETATION_KEY), - ROLE_KEY(RoleAnnotation.class, OldFeatureLabelKeys.ROLE_KEY), - GAZETTEER_KEY(GazetteerAnnotation.class, OldFeatureLabelKeys.GAZETTEER_KEY), - STEM_KEY(StemAnnotation.class, OldFeatureLabelKeys.STEM_KEY), - POLARITY_KEY(PolarityAnnotation.class, OldFeatureLabelKeys.POLARITY_KEY), - CH_CHAR_KEY(ChineseCharAnnotation.class, OldFeatureLabelKeys.CH_CHAR_KEY), - CH_ORIG_SEG_KEY(ChineseOrigSegAnnotation.class, OldFeatureLabelKeys.CH_ORIG_SEG_KEY), - CH_SEG_KEY(ChineseSegAnnotation.class, OldFeatureLabelKeys.CH_SEG_KEY), - BEGIN_POSITION_KEY(CharacterOffsetBeginAnnotation.class, OldFeatureLabelKeys.BEGIN_POSITION_KEY), - END_POSITION_KEY(CharacterOffsetEndAnnotation.class, OldFeatureLabelKeys.END_POSITION_KEY), - DOCID_KEY(DocIDAnnotation.class, OldFeatureLabelKeys.DOCID_KEY), - SENTINDEX_KEY(SentenceIndexAnnotation.class, OldFeatureLabelKeys.SENTINDEX_KEY), - IDF_KEY(IDFAnnotation.class, "idf"), - END_POSITION_KEY2(CharacterOffsetEndAnnotation.class, "endPosition"), - CHUNK_KEY(ChunkAnnotation.class, "chunk"), - NORMALIZED_NER_KEY(NormalizedNamedEntityTagAnnotation.class, "normalized"), - MORPHO_NUM_KEY(MorphoNumAnnotation.class,"num"), - MORPHO_PERS_KEY(MorphoPersAnnotation.class,"pers"), - MORPHO_GEN_KEY(MorphoGenAnnotation.class,"gen"), - MORPHO_CASE_KEY(MorphoCaseAnnotation.class,"case"), - WORDNET_SYN_KEY(WordnetSynAnnotation.class,"wordnetsyn"), - PROTO_SYN_KEY(ProtoAnnotation.class,"proto"); - - public final Class> coreKey; - public final String oldKey; - - private KeyLookup(Class> coreKey, String oldKey) { - this.coreKey = coreKey; - this.oldKey = oldKey; - } - - /** - * This constructor allows us to use reflection for loading old class keys. - * This is useful because we can then create distributions that do not have - * all of the classes required for all the old keys (such as trees package classes). - */ - private KeyLookup(String className, String oldKey) { - Class keyClass; - try { - keyClass = Class.forName(className); - } catch(ClassNotFoundException e) { - GenericAnnotation newKey = new GenericAnnotation() { - public Class getType() { return Object.class;} }; - keyClass = newKey.getClass(); - } - this.coreKey = ErasureUtils.uncheckedCast(keyClass); - this.oldKey = oldKey; - } - - - } - - /** - * Returns a CoreAnnotation class key for the given old-style FeatureLabel - * key if one exists; null otherwise. - */ - public static KeyLookup getCoreKey(String oldKey) { - for (KeyLookup lookup : KeyLookup.values()) { - if (lookup.oldKey.equals(oldKey)) { - return lookup; - } - } - return null; - } - - private static Map>,Class> valueCache - = new HashMap>,Class>(); - - /** - * Returns the runtime value type associated with the given key. Caches - * results. - */ - @SuppressWarnings("unchecked") - public static Class getValueType(Class key) { - Class type = valueCache.get(key); - if (type == null) { - try { - type = key.newInstance().getType(); - } catch (Exception e) { - throw new RuntimeException("Unexpected failure to instantiate - is your key class fancy?", e); - } - valueCache.put((Class)key, type); - } - return type; - } - - /** - * Lookup table for mapping between old-style *Label keys and classes - * the provide comparable backings in the core. - */ -//OLD keys kept around b/c we're kill IndexedFeatureLabel and these keys used to live there - private static class OldFeatureLabelKeys { - - public static final String DOCID_KEY = "docID"; - public static final String SENTINDEX_KEY = "sentIndex"; - public static final Object WORD_FORMAT = "WORD_FORMAT"; - public static final Object WORD_TAG_FORMAT = "WORD_TAG_FORMAT"; - public static final Object WORD_TAG_INDEX_FORMAT = "WORD_TAG_INDEX_FORMAT"; - public static final Object VALUE_FORMAT = "VALUE_FORMAT"; - public static final Object COMPLETE_FORMAT = "COMPLETE_FORMAT"; - public static final String VALUE_KEY = "value"; - public static final String TAG_KEY = "tag"; - public static final String WORD_KEY = "word"; - public static final String LEMMA_KEY = "lemma"; - public static final String CATEGORY_KEY = "cat"; - public static final String PROJ_CAT_KEY = "pcat"; - public static final String HEAD_WORD_KEY = "hw"; - public static final String HEAD_TAG_KEY = "ht"; - public static final String INDEX_KEY = "idx"; - public static final String ARG_KEY = "arg"; - public static final String MARKING_KEY = "mark"; - public static final String SEMANTIC_HEAD_WORD_KEY = "shw"; - public static final String SEMANTIC_HEAD_POS_KEY = "shp"; - public static final String VERB_SENSE_KEY = "vs"; - public static final String CATEGORY_FUNCTIONAL_TAG_KEY = "cft"; - public static final String NER_KEY = "ner"; - public static final String SHAPE_KEY = "shape"; - public static final String LEFT_TERM_KEY = "LEFT_TERM"; - public static final String PARENT_KEY = "PARENT"; - public static final String SPAN_KEY = "SPAN"; - public static final String BEFORE_KEY = "before"; - public static final String AFTER_KEY = "after"; - public static final String CURRENT_KEY = "current"; - public static final String ANSWER_KEY = "answer"; - public static final String GOLDANSWER_KEY = "goldAnswer"; - public static final String FEATURES_KEY = "features"; - public static final String INTERPRETATION_KEY = "interpretation"; - public static final String ROLE_KEY = "srl"; - public static final String GAZETTEER_KEY = "gazetteer"; - public static final String STEM_KEY = "stem"; - public static final String POLARITY_KEY = "polarity"; - public static final String CH_CHAR_KEY = "char"; - public static final String CH_ORIG_SEG_KEY = "orig_seg"; // the segmentation info existing in the original text - public static final String CH_SEG_KEY = "seg"; // the segmentation information from the segmenter - public static final String BEGIN_POSITION_KEY = "BEGIN_POS"; - public static final String END_POSITION_KEY = "END_POS"; - - - private OldFeatureLabelKeys() { - } - - } // end static class OldFeatureLabelKeys - -} - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/BasicDatum.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/BasicDatum.java deleted file mode 100644 index 5da9a3a..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/BasicDatum.java +++ /dev/null @@ -1,146 +0,0 @@ -package edu.stanford.nlp.ling; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Basic implementation of Datum interface that can be constructed with a - * Collection of features and one more more labels. The features must be - * specified - * at construction, but the labels can be set and/or changed later. - * - * @author Joseph Smarr (jsmarr@stanford.edu) - * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization) - * - * @param The type of the labels in the Dataset - * @param The type of the features in the Dataset - */ -public class BasicDatum implements Datum { - - /** - * features for this Datum - */ - @SuppressWarnings({"NonSerializableFieldInSerializableClass"}) - private final Collection features; - - /** - * labels for this Datum. Invariant: always non-null - */ - @SuppressWarnings({"NonSerializableFieldInSerializableClass"}) - private final List labels = new ArrayList(); - - /** - * Constructs a new BasicDatum with the given features and labels. - */ - public BasicDatum(Collection features, Collection labels) { - this(features); - setLabels(labels); - } - - /** - * Constructs a new BasicDatum with the given features and label. - */ - public BasicDatum(Collection features, LabelType label) { - this(features); - setLabel(label); - } - - /** - * Constructs a new BasicDatum with the given features and no labels. - */ - public BasicDatum(Collection features) { - this.features = features; - } - - /** - * Constructs a new BasicDatum with no features or labels. - */ - public BasicDatum() { - this(null); - } - - /** - * Returns the collection that this BasicDatum was constructed with. - */ - public Collection asFeatures() { - return (features); - } - - /** - * Returns the first label for this Datum, or null if none have been set. - */ - public LabelType label() { - return ((labels.size() > 0) ? labels.get(0) : null); - } - - /** - * Returns the complete List of labels for this Datum, which may be empty. - */ - public Collection labels() { - return labels; - } - - /** - * Removes all currently assigned Labels for this Datum then adds the - * given Label. - * Calling setLabel(null) effectively clears all labels. - */ - public void setLabel(LabelType label) { - labels.clear(); - addLabel(label); - } - - /** - * Removes all currently assigned labels for this Datum then adds all - * of the given Labels. - */ - public void setLabels(Collection labels) { - this.labels.clear(); - if (labels != null) { - this.labels.addAll(labels); - } - } - - /** - * Adds the given Label to the List of labels for this Datum if it is not - * null. - */ - public void addLabel(LabelType label) { - if (label != null) { - labels.add(label); - } - } - - /** - * Returns a String representation of this BasicDatum (lists features and labels). - */ - @Override - public String toString() { - return ("BasicDatum[features=" + asFeatures() + ",labels=" + labels() + "]"); - } - - - /** - * Returns whether the given Datum contains the same features as this Datum. - * Doesn't check the labels, should we change this? - */ - @SuppressWarnings("unchecked") - @Override - public boolean equals(Object o) { - if (!(o instanceof Datum)) { - return (false); - } - - Datum d = (Datum) o; - return features.equals(d.asFeatures()); - } - - public int hashCode() { - return features.hashCode(); - } - - private static final long serialVersionUID = -4857004070061779966L; - -} - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CategoryWordTag.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CategoryWordTag.java deleted file mode 100644 index 18f6eaa..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CategoryWordTag.java +++ /dev/null @@ -1,182 +0,0 @@ -package edu.stanford.nlp.ling; - - -/** - * A CategoryWordTag object acts as a complex Label - * which contains a category, a head word, and a tag. - * The category label is the primary value - * - * @author Christopher Manning - */ -public class CategoryWordTag extends StringLabel implements HasCategory, HasWord, HasTag { - - private static final long serialVersionUID = -745085381666943254L; - - protected String word; - protected String tag; - - /** - * If this is false, the tag and word are never printed in toString() - * calls. - */ - public static boolean printWordTag = true; - - /** - * If set to true, when a terminal or preterminal has as its category - * something that is also the word or tag value, the latter are - * suppressed. - */ - public static boolean suppressTerminalDetails; // = false; - - - public CategoryWordTag() { - super(); - } - - /** - * This one argument constructor sets just the value. - * - * @param label the string that will become the category/value - */ - public CategoryWordTag(String label) { - super(label); - } - - public CategoryWordTag(String category, String word, String tag) { - super(category); - this.word = word; - this.tag = tag; - } - - /** - * Creates a new CategoryWordTag label from an existing label. - * The oldLabel value() -- i.e., category -- is used for the new label. - * The tag and word - * are initialized iff the current label implements HasTag and HasWord - * respectively. - * - * @param oldLabel The label to use as a basis of this Label - */ - public CategoryWordTag(Label oldLabel) { - super(oldLabel); - if (oldLabel instanceof HasTag) { - this.tag = ((HasTag) oldLabel).tag(); - } - if (oldLabel instanceof HasWord) { - this.word = ((HasWord) oldLabel).word(); - } - } - - public String category() { - return value(); - } - - public void setCategory(String category) { - setValue(category); - } - - public String word() { - return word; - } - - public void setWord(String word) { - this.word = word; - } - - public String tag() { - return tag; - } - - public void setTag(String tag) { - this.tag = tag; - } - - public void setCategoryWordTag(String category, String word, String tag) { - setCategory(category); - setWord(word); - setTag(tag); - } - - - /** - * Returns a String representation of the label. - * This attempts to be somewhat clever in choosing to print or - * suppress null components and the details of words or categories - * depending on the setting of printWordTag and - * suppressTerminalDetails. - * - * @return The label as a string - */ - @Override - public String toString() { - if (category() != null) { - if ((word() == null || tag() == null) || !printWordTag || (suppressTerminalDetails && (word().equals(category()) || tag().equals(category())))) { - return category(); - } else { - return category() + "[" + word() + "/" + tag() + "]"; - } - } else { - if (tag() == null) { - return word(); - } else { - return word() + "/" + tag(); - } - } - } - - - /** - * Returns a String representation of the label. - * If the argument String is "full" then all components of the label - * are returned, and otherwise the normal toString() is returned. - * - * @return The label as a string - */ - public String toString(String mode) { - if ("full".equals(mode)) { - return category() + "[" + word() + "/" + tag() + "]"; - } - return toString(); - } - - - /** - * Set everything by reversing a toString operation. - * This should be added at some point. - */ - @Override - public void setFromString(String labelStr) { - throw new UnsupportedOperationException(); - } - - - // extra class guarantees correct lazy loading (Bloch p.194) - private static class LabelFactoryHolder { - private LabelFactoryHolder() {} - private static final LabelFactory lf = new CategoryWordTagFactory(); - } - - /** - * Return a factory for this kind of label - * (i.e., CategoryWordTag). - * The factory returned is always the same one (a singleton). - * - * @return The label factory - */ - @Override - public LabelFactory labelFactory() { - return LabelFactoryHolder.lf; - } - - - /** - * Return a factory for this kind of label - * - * @return The label factory - */ - public static LabelFactory factory() { - return LabelFactoryHolder.lf; - } - -} - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CategoryWordTagFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CategoryWordTagFactory.java deleted file mode 100644 index c11da31..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CategoryWordTagFactory.java +++ /dev/null @@ -1,75 +0,0 @@ -package edu.stanford.nlp.ling; - - -/** - * A CategoryWordTagFactory is a factory that makes - * a Label which is a CategoryWordTag triplet. - * - * @author Christopher Manning - */ -public class CategoryWordTagFactory implements LabelFactory { - - /** - * Make a new label with this String as the "name". - * - * @param labelStr The string to use as a label - * @return The newly created Label - */ - public Label newLabel(String labelStr) { - return new CategoryWordTag(labelStr); - } - - /** - * Make a new label with this String as the value. - * This implementation ignores the options - * - * @param labelStr The String that will be used for balue - * @param options This argument is ignored - * @return The newly created Label - */ - public Label newLabel(String labelStr, int options) { - return new CategoryWordTag(labelStr); - } - - /** - * Make a new label with this String as the "name". - * - * @param labelStr The string to use as a label - * @return The newly created Label - */ - public Label newLabelFromString(String labelStr) { - CategoryWordTag cwt = new CategoryWordTag(); - cwt.setFromString(labelStr); - return cwt; - } - - /** - * Create a new CategoryWordTag label, where the label is formed from - * the various String objects passed in. - * - * @param word The word part of the label - * @param tag The tag part of the label - * @param category The category part of the label - * @return The newly created Label - */ - public Label newLabel(String word, String tag, String category) { - // System.out.println("Making new CWT label: " + category + " | " + - // word + " | " + tag); - return new CategoryWordTag(category, word, tag); - } - - /** - * Create a new CategoryWordTag Label, where the label is - * formed from - * the Label object passed in. Depending on what fields - * each label has, other things will be null. - * - * @param oldLabel The Label that the new label is being created from - * @return a new label of a particular type - */ - public Label newLabel(Label oldLabel) { - return new CategoryWordTag(oldLabel); - } - -} - diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CoreAnnotation.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CoreAnnotation.java deleted file mode 100644 index 41c85ac..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CoreAnnotation.java +++ /dev/null @@ -1,25 +0,0 @@ -package edu.stanford.nlp.ling; - -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.TypesafeMap; - -/** - * The base class for any annotation that can be marked on a {@link CoreMap}, - * parameterized by the type of the value associated with the annotation. - * Subclasses of this class are the keys in the {@link CoreMap}, so they are - * instantiated only by utility methods in {@link CoreAnnotations}. - * - * @author dramage - * @author rafferty - */ -public interface CoreAnnotation - extends TypesafeMap.Key { - - /** - * Returns the type associated with this annotation. This method must - * return the same class type as its value type parameter. It feels like - * one should be able to get away without this method, but because Java - * erases the generic type signature, that info disappears at runtime. - */ - public Class getType(); -} diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CoreAnnotations.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CoreAnnotations.java deleted file mode 100644 index dbb6486..0000000 --- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/ling/CoreAnnotations.java +++ /dev/null @@ -1,1457 +0,0 @@ -package edu.stanford.nlp.ling; - -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.SortedSet; - -import edu.stanford.nlp.util.CoreMap; -import edu.stanford.nlp.util.ErasureUtils; -import edu.stanford.nlp.util.IntPair; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.Triple; - -/** - *

- * Set of common annotations for {@link CoreMap}s. The classes - * defined here are typesafe keys for getting and setting annotation - * values. These classes need not be instantiated outside of this - * class. e.g {@link TextAnnotation}.class serves as the key and a - * String serves as the value containing the - * corresponding word. - *

- * - *

- * New types of {@link CoreAnnotation} can be defined anywhere that is - * convenient in the source tree - they are just classes. This file exists to - * hold widely used "core" annotations and others inherited from the - * {@link Label} family. In general, most keys should be placed in this file as - * they may often be reused throughout the code. This architecture allows for - * flexibility, but in many ways it should be considered as equivalent to an - * enum in which everything should be defined - *

- * - *

- * The getType method required by CoreAnnotation must return the same class type - * as its value type parameter. It feels like one should be able to get away - * without that method, but because Java erases the generic type signature, that - * info disappears at runtime. See {@link ValueAnnotation} for an example. - *

- * - * @author dramage - * @author rafferty - * @author bethard - */ -public class CoreAnnotations { - - private CoreAnnotations() { - } // only static members - - /** - * The CoreMap key identifying the annotation's text. - * - * Note that this key is intended to be used with many different kinds of - * annotations - documents, sentences and tokens all have their own text. - */ - public static class TextAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - - /** - * The CoreMap key for getting the lemma (morphological stem) of a token. - * - * This key is typically set on token annotations. - * - * TODO: merge with StemAnnotation? - */ - public static class LemmaAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The CoreMap key for getting the Penn part of speech of a token. - * - * This key is typically set on token annotations. - */ - public static class PartOfSpeechAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The CoreMap key for getting the token-level named entity tag (e.g., DATE, - * PERSON, etc.) - * - * This key is typically set on token annotations. - */ - public static class NamedEntityTagAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The CoreMap key for getting the token-level named entity tag (e.g., DATE, - * PERSON, etc.) from a previous NER tagger. NERFeatureFactory is sensitive to - * this tag and will turn the annotations from the previous NER tagger into - * new features. This is currently used to implement one level of stacking -- - * we may later change it to take a list as needed. - * - * This key is typically set on token annotations. - */ - public static class StackedNamedEntityTagAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The CoreMap key for getting the token-level true case annotation (e.g., - * INIT_UPPER) - * - * This key is typically set on token annotations. - */ - public static class TrueCaseAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The CoreMap key identifying the annotation's true-cased text. - * - * Note that this key is intended to be used with many different kinds of - * annotations - documents, sentences and tokens all have their own text. - */ - public static class TrueCaseTextAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The CoreMap key for getting the tokens contained by an annotation. - * - * This key should be set for any annotation that contains tokens. It can be - * done without much memory overhead using List.subList. - */ - public static class TokensAnnotation implements CoreAnnotation> { - public Class> getType() { - return ErasureUtils.>> uncheckedCast(List.class); - } - } - - /** - * The CoreMap key for getting the tokens (can be words, phrases or anything that are of type CoreMap) contained by an annotation. - * - * This key should be set for any annotation that contains tokens (words, phrases etc). It can be - * done without much memory overhead using List.subList. - */ - public static class GenericTokensAnnotation implements CoreAnnotation> { - public Class> getType() { - return ErasureUtils.>> uncheckedCast(List.class); - } - } - - /** - * The CoreMap key for getting the sentences contained by an annotation. - * - * This key is typically set only on document annotations. - */ - public static class SentencesAnnotation implements CoreAnnotation> { - public Class> getType() { - return ErasureUtils.uncheckedCast(List.class); - } - } - - /** - * The CoreMap key for getting the paragraphs contained by an annotation. - * - * This key is typically set only on document annotations. - */ - public static class ParagraphsAnnotation implements CoreAnnotation> { - public Class> getType() { - return ErasureUtils.uncheckedCast(List.class); - } - } - - /** - * The CoreMap key identifying the first token included in an annotation. The - * token with index 0 is the first token in the document. - * - * This key should be set for any annotation that contains tokens. - */ - public static class TokenBeginAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * The CoreMap key identifying the last token after the end of an annotation. - * The token with index 0 is the first token in the document. - * - * This key should be set for any annotation that contains tokens. - */ - public static class TokenEndAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * The CoreMap key identifying the date and time associated with an - * annotation. - * - * This key is typically set on document annotations. - */ - public static class CalendarAnnotation implements CoreAnnotation { - public Class getType() { - return Calendar.class; - } - } - - /** - * These are the keys hashed on by IndexedWord - */ - /** - * This refers to the unique identifier for a "document", where document may - * vary based on your application. - */ - public static class DocIDAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * This indexes a token number inside a sentence. Standardly, tokens are - * indexed within a sentence starting at 1 (not 0: we follow common parlance - * whereby we speak of the first word of a sentence). - * This is generally an individual word or feature index - it is local, and - * may not be uniquely identifying without other identifiers such as sentence - * and doc. However, if these are the same, the index annotation should be a - * unique identifier for differentiating objects. - */ - public static class IndexAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * This indexes the beginning of a span of words, e.g., a constituent in a - * tree. See {@link edu.stanford.nlp.trees.Tree#indexSpans(int)}. This annotation counts tokens. - * It standardly indexes from 1 (like IndexAnnotation). - */ - public static class BeginIndexAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * This indexes the end of a span of words, e.g., a constituent in a - * tree. See {@link edu.stanford.nlp.trees.Tree#indexSpans(int)}. This annotation counts - * tokens. It standardly indexes from 1 (like IndexAnnotation). - * The end index is not a fencepost: its value is equal to the - * IndexAnnotation of the last word in the span. - */ - public static class EndIndexAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * This indicates the sentence should end at this token. Used to - * force the ssplit annotator (eg the WordToSentenceProcessor) to - * start a new sentence at the next token. - */ - public static class ForcedSentenceEndAnnotation - implements CoreAnnotation { - public Class getType() { - return Boolean.class; - } - } - - /** - * Unique identifier within a document for a given sentence. - */ - public static class SentenceIndexAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * Contains the "value" - an ill-defined string used widely in MapLabel. - */ - public static class ValueAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class CategoryAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The exact original surface form of a token. This is created in the - * invertible PTBTokenizer. The tokenizer may normalize the token form to - * match what appears in the PTB, but this key will hold the original characters. - */ - public static class OriginalTextAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * Annotation for the whitespace characters appearing before this word. This - * can be filled in by the tokenizer so that the original text string can be - * reconstructed. - */ - public static class BeforeAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * Annotation for the whitespace characters appear after this word. This can - * be filled in by the tokenizer so that the original text string can be - * reconstructed. - */ - public static class AfterAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * CoNLL dep parsing - coarser POS tags. - */ - public static class CoarseTagAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * CoNLL dep parsing - the dependency type - */ - public static class CoNLLDepAnnotation implements CoreAnnotation { - public Class getType() { - return CoreMap.class; - } - } - - /** - * CoNLL SRL/dep parsing - whether the word is a predicate - */ - public static class CoNLLPredicateAnnotation implements CoreAnnotation { - public Class getType() { - return Boolean.class; - } - } - - /** - * CoNLL SRL/dep parsing - map which, for the current word, specifies its - * specific role for each predicate - */ - @SuppressWarnings("unchecked") - public static class CoNLLSRLAnnotation implements CoreAnnotation { - public Class getType() { - return Map.class; - } - } - - /** - * CoNLL dep parsing - the dependency type - */ - public static class CoNLLDepTypeAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * CoNLL dep parsing - the index of the word which is the parent of this word - * in the dependency tree - */ - public static class CoNLLDepParentIndexAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * Inverse document frequency of the word this label represents - */ - public static class IDFAnnotation implements CoreAnnotation { - public Class getType() { - return Double.class; - } - } - - /** - * Keys from AbstractMapLabel (descriptions taken from that class) - */ - /** - * The standard key for storing a projected category in the map, as a String. - * For any word (leaf node), the projected category is the syntactic category - * of the maximal constituent headed by the word. Used in SemanticGraph. - */ - public static class ProjectedCategoryAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for a propbank label which is of type Argument - */ - public static class ArgumentAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * Another key used for propbank - to signify core arg nodes or predicate - * nodes - */ - public static class MarkingAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for Semantic Head Word which is a String - */ - public static class SemanticHeadWordAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for Semantic Head Word POS which is a String - */ - public static class SemanticHeadTagAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * Probank key for the Verb sense given in the Propbank Annotation, should - * only be in the verbnode - */ - public static class VerbSenseAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for storing category with functional tags. - */ - public static class CategoryFunctionalTagAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * This is an NER ID annotation (in case the all caps parsing didn't work out - * for you...) - */ - public static class NERIDAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The key for the normalized value of numeric named entities. - */ - public static class NormalizedNamedEntityTagAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public enum SRL_ID { - ARG, NO, ALL_NO, REL - } - - /** - * The key for semantic role labels (Note: please add to this description if - * you use this key) - */ - public static class SRLIDAnnotation implements CoreAnnotation { - public Class getType() { - return SRL_ID.class; - } - } - - /** - * The standard key for the "shape" of a word: a String representing the type - * of characters in a word, such as "Xx" for a capitalized word. See - * {@link edu.stanford.nlp.process.WordShapeClassifier} for functions for - * making shape strings. - */ - public static class ShapeAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The Standard key for storing the left terminal number relative to the root - * of the tree of the leftmost terminal dominated by the current node - */ - public static class LeftTermAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * The standard key for the parent which is a String - */ - public static class ParentAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class INAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for span which is an IntPair - */ - public static class SpanAnnotation implements CoreAnnotation { - public Class getType() { - return IntPair.class; - } - } - - /** - * The standard key for the answer which is a String - */ - public static class AnswerAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for gold answer which is a String - */ - public static class GoldAnswerAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for the features which is a Collection - */ - public static class FeaturesAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for the semantic interpretation - */ - public static class InterpretationAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for the semantic role label of a phrase. - */ - public static class RoleAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * The standard key for the gazetteer information - */ - public static class GazetteerAnnotation implements CoreAnnotation> { - @SuppressWarnings( { "unchecked" }) - public Class> getType() { - return (Class) List.class; - } - } - - /** - * Morphological stem of the word this label represents - */ - public static class StemAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class PolarityAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class MorphoNumAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class MorphoPersAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class MorphoGenAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class MorphoCaseAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * for Chinese: character level information, segmentation - */ - public static class ChineseCharAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class ChineseOrigSegAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - public static class ChineseSegAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - /** - * Not sure exactly what this is, but it is different from - * ChineseSegAnnotation and seems to indicate if the text is segmented - */ - public static class ChineseIsSegmentedAnnotation implements CoreAnnotation { - public Class getType() { - return Boolean.class; - } - } - - /** - * The CoreMap key identifying the offset of the first character of an - * annotation. The character with index 0 is the first character in the - * document. - * - * This key should be set for any annotation that represents a span of text. - */ - public static class CharacterOffsetBeginAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * The CoreMap key identifying the offset of the last character after the end - * of an annotation. The character with index 0 is the first character in the - * document. - * - * This key should be set for any annotation that represents a span of text. - */ - public static class CharacterOffsetEndAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * Key for relative value of a word - used in RTE - */ - public static class CostMagnificationAnnotation implements CoreAnnotation { - public Class getType() { - return Double.class; - } - } - - public static class WordSenseAnnotation implements CoreAnnotation { - public Class getType() { - return String.class; - } - } - - @SuppressWarnings( { "unchecked" }) - public static class SRLInstancesAnnotation implements CoreAnnotation>>> { - public Class>>> getType() { - return (Class) List.class; - } - } - - /** - * Used by RTE to track number of text sentences, to determine when hyp - * sentences begin. - */ - public static class NumTxtSentencesAnnotation implements CoreAnnotation { - public Class getType() { - return Integer.class; - } - } - - /** - * Used in Trees - */ - public static class TagLabelAnnotation implements CoreAnnotation