diff --git a/README.md b/README.md
index ebc7eec..7f50f42 100644
--- a/README.md
+++ b/README.md
@@ -121,12 +121,12 @@ Then [re]start Solr, e.g.:
Now start the Stanford Named Entity Recognition server, which is used to pull
names, places etc. out of the source data:
- $ cd stanford-ner-2011-09-14
+ $ cd stanford-ner-2014-06-16
$ ./server.sh &
or on Windows:
- C:\ cd stanford-ner-2011-09-14
+ C:\ cd stanford-ner-2014-06-16
C:\ server.bat
Finally, add the example documents (which are provided as plaintext files):
diff --git a/go2.bat b/go2.bat
index 6751533..91c6351 100644
--- a/go2.bat
+++ b/go2.bat
@@ -4,5 +4,5 @@ rem Each must run in its own command line shell
call paths.bat
-cd stanford-ner-2011-09-14
+cd stanford-ner-2014-06-16
server.bat
diff --git a/stanford-ner-2011-09-14/Makefile b/stanford-ner-2011-09-14/Makefile
deleted file mode 100644
index 2f8d594..0000000
--- a/stanford-ner-2011-09-14/Makefile
+++ /dev/null
@@ -1,12 +0,0 @@
-# This is a rudimentary Makefile for rebuilding the CRF NER distribution.
-# We actually use ant (q.v.) or a Java IDE.
-
-JAVAC = javac
-JAVAFLAGS = -O -d classes -encoding utf-8
-
-ner:
- mkdir -p classes
- $(JAVAC) $(JAVAFLAGS) src/edu/stanford/nlp/*/*.java src/edu/stanford/nlp/*/*/*.java
- cd classes ; jar -cfm ../stanford-ner-`date +%Y-%m-%d`.jar ../src/edu/stanford/nlp/ie/crf/ner-manifest.txt edu ; cd ..
- cp stanford-ner-`date +%Y-%m-%d`.jar stanford-ner.jar
-
diff --git a/stanford-ner-2011-09-14/NERDemo.java b/stanford-ner-2011-09-14/NERDemo.java
deleted file mode 100644
index bc7bc50..0000000
--- a/stanford-ner-2011-09-14/NERDemo.java
+++ /dev/null
@@ -1,77 +0,0 @@
-import edu.stanford.nlp.ie.crf.*;
-import edu.stanford.nlp.ie.AbstractSequenceClassifier;
-import edu.stanford.nlp.io.IOUtils;
-import edu.stanford.nlp.ling.CoreLabel;
-import edu.stanford.nlp.ling.CoreAnnotations.AnswerAnnotation;
-
-import java.util.List;
-import java.io.IOException;
-
-
-
-/** This is a demo of calling CRFClassifier programmatically.
- *
- * Usage: java -cp "stanford-ner.jar:." NERDemo [serializedClassifier [fileName]]
- *
- * If arguments aren't specified, they default to
- * ner-eng-ie.crf-3-all2006.ser.gz and some hardcoded sample text.
- *
- * To use CRFClassifier from the command line:
- * java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier
- * [classifier] -textFile [file]
- * Or if the file is already tokenized and one word per line, perhaps in
- * a tab-separated value format with extra columns for part-of-speech tag,
- * etc., use the version below (note the 's' instead of the 'x'):
- * java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier
- * [classifier] -testFile [file]
- *
- * @author Jenny Finkel
- * @author Christopher Manning
- */
-
-public class NERDemo {
-
- public static void main(String[] args) throws IOException {
-
- String serializedClassifier = "classifiers/ner-eng-ie.crf-3-all2008.ser.gz";
-
- if (args.length > 0) {
- serializedClassifier = args[0];
- }
-
- AbstractSequenceClassifier classifier = CRFClassifier.getClassifierNoExceptions(serializedClassifier);
-
- /* For either a file to annotate or for the hardcoded text example,
- this demo file shows two ways to process the output, for teaching
- purposes. For the file, it shows both how to run NER on a String
- and how to run it on a whole file. For the hard-coded String,
- it shows how to run it on a single sentence, and how to do this
- and produce an inline XML output format.
- */
- if (args.length > 1) {
- String fileContents = IOUtils.slurpFile(args[1]);
- List> out = classifier.classify(fileContents);
- for (List sentence : out) {
- for (CoreLabel word : sentence) {
- System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' ');
- }
- System.out.println();
- }
- out = classifier.classifyFile(args[1]);
- for (List sentence : out) {
- for (CoreLabel word : sentence) {
- System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' ');
- }
- System.out.println();
- }
-
- } else {
- String s1 = "Good afternoon Rajat Raina, how are you today?";
- String s2 = "I go to school at Stanford University, which is located in California.";
- System.out.println(classifier.classifyToString(s1));
- System.out.println(classifier.classifyWithInlineXML(s2));
- System.out.println(classifier.classifyToString(s2, "xml", true));
- }
- }
-
-}
diff --git a/stanford-ner-2011-09-14/ner-gui.sh b/stanford-ner-2011-09-14/ner-gui.sh
deleted file mode 100755
index ea2cb5e..0000000
--- a/stanford-ner-2011-09-14/ner-gui.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-java -mx500m -cp "stanford-ner.jar:" edu.stanford.nlp.ie.crf.NERGUI
diff --git a/stanford-ner-2011-09-14/ner.bat b/stanford-ner-2011-09-14/ner.bat
deleted file mode 100755
index e87b3c8..0000000
--- a/stanford-ner-2011-09-14/ner.bat
+++ /dev/null
@@ -1 +0,0 @@
-java -mx1500m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers\all.3class.distsim.crf.ser.gz -textFile %1
diff --git a/stanford-ner-2011-09-14/ner.sh b/stanford-ner-2011-09-14/ner.sh
deleted file mode 100755
index 56827f1..0000000
--- a/stanford-ner-2011-09-14/ner.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-java -mx700m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers/all.3class.distsim.crf.ser.gz -textFile $1
diff --git a/stanford-ner-2011-09-14/server.bat b/stanford-ner-2011-09-14/server.bat
deleted file mode 100644
index 81f810a..0000000
--- a/stanford-ner-2011-09-14/server.bat
+++ /dev/null
@@ -1 +0,0 @@
-java -mx1000m -cp stanford-ner.jar edu.stanford.nlp.ie.NERServer -loadClassifier classifiers/all.3class.distsim.crf.ser.gz -port 9000
\ No newline at end of file
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AbstractLinearClassifierFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AbstractLinearClassifierFactory.java
deleted file mode 100644
index a671b16..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AbstractLinearClassifierFactory.java
+++ /dev/null
@@ -1,91 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import edu.stanford.nlp.ling.Datum;
-import edu.stanford.nlp.ling.RVFDatum;
-import edu.stanford.nlp.util.Index;
-import edu.stanford.nlp.util.HashIndex;
-
-import java.lang.ref.Reference;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Shared methods for training a {@link LinearClassifier}.
- * Inheriting classes need to implement the
- * trainWeights method.
- *
- * @author Dan Klein
- *
- * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization)
- *
- * @param The type of the labels in the Dataset and Datum
- * @param The type of the features in the Dataset and Datum
- */
-
-public abstract class AbstractLinearClassifierFactory implements ClassifierFactory> {
-
- private static final long serialVersionUID = 1L;
-
- Index labelIndex = new HashIndex();
- Index featureIndex = new HashIndex();
-
- public AbstractLinearClassifierFactory() {
- }
-
- int numFeatures() {
- return featureIndex.size();
- }
-
- int numClasses() {
- return labelIndex.size();
- }
-
- public Classifier trainClassifier(List> examples) {
- Dataset dataset = new Dataset();
- dataset.addAll(examples);
- return trainClassifier(dataset);
- }
-
- protected abstract double[][] trainWeights(GeneralDataset dataset) ;
-
- /**
- * Takes a {@link Collection} of {@link Datum} objects and gives you back a
- * {@link Classifier} trained on it.
- *
- * @param examples {@link Collection} of {@link Datum} objects to train the
- * classifier on
- * @return A {@link Classifier} trained on it.
- */
- public LinearClassifier trainClassifier(Collection> examples) {
- Dataset dataset = new Dataset();
- dataset.addAll(examples);
- return trainClassifier(dataset);
- }
-
- /**
- * Takes a {@link Reference} to a {@link Collection} of {@link Datum}
- * objects and gives you back a {@link Classifier} trained on them
- *
- * @param ref {@link Reference} to a {@link Collection} of {@link
- * Datum} objects to train the classifier on
- * @return A Classifier trained on a collection of Datum
- */
- public LinearClassifier trainClassifier(Reference>> ref) {
- Collection> examples = ref.get();
- return trainClassifier(examples);
- }
-
-
- /**
- * Trains a {@link Classifier} on a {@link Dataset}.
- *
- * @return A {@link Classifier} trained on the data.
- */
- public LinearClassifier trainClassifier(GeneralDataset data) {
- labelIndex = data.labelIndex();
- featureIndex = data.featureIndex();
- double[][] weights = trainWeights(data);
- return new LinearClassifier(weights, featureIndex, labelIndex);
- }
-
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AdaptedGaussianPriorObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AdaptedGaussianPriorObjectiveFunction.java
deleted file mode 100644
index 16521e1..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/AdaptedGaussianPriorObjectiveFunction.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import edu.stanford.nlp.math.ArrayMath;
-import java.util.Arrays;
-
-
-/**
- * Adapt the mean of the Gaussian Prior by shifting the mean to the previously trained weights
- * @author Pi-Chuan Chang
- * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization)
- *
- * @param The type of the labels in the Dataset (one can be passed in to the constructor)
- * @param The type of the features in the Dataset
- */
-
-public class AdaptedGaussianPriorObjectiveFunction extends LogConditionalObjectiveFunction {
-
- double[] weights;
-
- /**
- * Calculate the conditional likelihood.
- */
- @Override
- protected void calculate(double[] x) {
- if (useSummedConditionalLikelihood) {
- calculateSCL(x);
- } else {
- calculateCL(x);
- }
- }
-
-
- /**
- */
- private void calculateSCL(double[] x) {
- throw new UnsupportedOperationException();
- }
-
- /**
- */
- private void calculateCL(double[] x) {
- value = 0.0;
- if (derivativeNumerator == null) {
- derivativeNumerator = new double[x.length];
- for (int d = 0; d < data.length; d++) {
- int[] features = data[d];
- for (int f = 0; f < features.length; f++) {
- int i = indexOf(features[f], labels[d]);
- if (dataweights == null) {
- derivativeNumerator[i] -= 1;
- } else {
- derivativeNumerator[i] -= dataweights[d];
- }
- }
- }
- }
- copy(derivative, derivativeNumerator);
-
- double[] sums = new double[numClasses];
- double[] probs = new double[numClasses];
-
- for (int d = 0; d < data.length; d++) {
- int[] features = data[d];
- // activation
- Arrays.fill(sums, 0.0);
-
- for (int c = 0; c < numClasses; c++) {
- for (int f = 0; f < features.length; f++) {
- int i = indexOf(features[f], c);
- sums[c] += x[i];
- }
- }
- double total = ArrayMath.logSum(sums);
- for (int c = 0; c < numClasses; c++) {
- probs[c] = Math.exp(sums[c] - total);
- if (dataweights != null) {
- probs[c] *= dataweights[d];
- }
- for (int f = 0; f < features.length; f++) {
- int i = indexOf(features[f], c);
- derivative[i] += probs[c];
- }
- }
-
- double dV = sums[labels[d]] - total;
- if (dataweights != null) {
- dV *= dataweights[d];
- }
- value -= dV;
- }
- //System.err.println("x length="+x.length);
- //System.err.println("weights length="+weights.length);
- double[] newX = ArrayMath.pairwiseSubtract(x, weights);
- value += prior.compute(newX, derivative);
- }
-
- /**
- */
- @Override
- protected void rvfcalculate(double[] x) {
- throw new UnsupportedOperationException();
- }
-
- public AdaptedGaussianPriorObjectiveFunction(GeneralDataset dataset, LogPrior prior, double weights[][]) {
- super(dataset, prior);
- this.weights = to1D(weights);
- }
-
- public double[] to1D(double[][] x2) {
- double[] x = new double[numFeatures*numClasses];
- for (int i = 0; i < numFeatures; i++) {
- for (int j = 0; j < numClasses; j++) {
- x[indexOf(i, j)] = x2[i][j];
- }
- }
- return x;
- }
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/BiasedLogConditionalObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/BiasedLogConditionalObjectiveFunction.java
deleted file mode 100644
index 5378d14..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/BiasedLogConditionalObjectiveFunction.java
+++ /dev/null
@@ -1,137 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import edu.stanford.nlp.math.ArrayMath;
-import edu.stanford.nlp.optimization.AbstractCachingDiffFunction;
-
-import java.util.Arrays;
-
-
-/**
- * Maximizes the conditional likelihood with a given prior.
- *
- * @author Jenny Finkel
- */
-
-public class BiasedLogConditionalObjectiveFunction extends AbstractCachingDiffFunction {
-
- public void setPrior(LogPrior prior) {
- this.prior = prior;
- }
-
- protected LogPrior prior;
-
- protected int numFeatures = 0;
- protected int numClasses = 0;
-
- protected int[][] data = null;
- protected int[] labels = null;
-
- private double[][] confusionMatrix;
-
- @Override
- public int domainDimension() {
- return numFeatures * numClasses;
- }
-
- int classOf(int index) {
- return index % numClasses;
- }
-
- int featureOf(int index) {
- return index / numClasses;
- }
-
- protected int indexOf(int f, int c) {
- return f * numClasses + c;
- }
-
- public double[][] to2D(double[] x) {
- double[][] x2 = new double[numFeatures][numClasses];
- for (int i = 0; i < numFeatures; i++) {
- for (int j = 0; j < numClasses; j++) {
- x2[i][j] = x[indexOf(i, j)];
- }
- }
- return x2;
- }
-
- @Override
- protected void calculate(double[] x) {
-
- if (derivative == null) {
- derivative = new double[x.length];
- } else {
- Arrays.fill(derivative, 0.0);
- }
-
- value = 0.0;
-
- double[] sums = new double[numClasses];
- double[] probs = new double[numClasses];
- double[] weightedProbs = new double[numClasses];
-
- for (int d = 0; d < data.length; d++) {
- int[] features = data[d];
- int observedLabel = labels[d];
- // activation
- Arrays.fill(sums, 0.0);
-
- for (int c = 0; c < numClasses; c++) {
- for (int f = 0; f < features.length; f++) {
- int i = indexOf(features[f], c);
- sums[c] += x[i];
- }
- }
-
- double total = ArrayMath.logSum(sums);
-
- double[] weightedSums = new double[numClasses];
- for (int trueLabel = 0; trueLabel < numClasses; trueLabel++) {
- weightedSums[trueLabel] = Math.log(confusionMatrix[observedLabel][trueLabel]) + sums[trueLabel];
- }
-
- double weightedTotal = ArrayMath.logSum(weightedSums);
-
- for (int c = 0; c < numClasses; c++) {
- probs[c] = Math.exp(sums[c] - total);
- weightedProbs[c] = Math.exp(weightedSums[c] - weightedTotal);
- for (int f = 0; f < features.length; f++) {
- int i = indexOf(features[f], c);
- derivative[i] += probs[c] - weightedProbs[c];
- }
- }
-
- double tmpValue = 0.0;
- for (int c = 0; c < numClasses; c++) {
- tmpValue += confusionMatrix[observedLabel][c] * Math.exp(sums[c] - total);
- }
- value -= Math.log(tmpValue);
- }
-
- value += prior.compute(x, derivative);
-
- }
-
-
-
- public BiasedLogConditionalObjectiveFunction(GeneralDataset, ?> dataset, double[][] confusionMatrix) {
- this(dataset, confusionMatrix, new LogPrior(LogPrior.LogPriorType.QUADRATIC));
- }
-
- public BiasedLogConditionalObjectiveFunction(GeneralDataset, ?> dataset, double[][] confusionMatrix, LogPrior prior) {
- this(dataset.numFeatures(), dataset.numClasses(), dataset.getDataArray(), dataset.getLabelsArray(), confusionMatrix, prior);
- }
-
- public BiasedLogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, double[][] confusionMatrix) {
- this(numFeatures, numClasses, data, labels, confusionMatrix, new LogPrior(LogPrior.LogPriorType.QUADRATIC));
- }
-
- public BiasedLogConditionalObjectiveFunction(int numFeatures, int numClasses, int[][] data, int[] labels, double[][] confusionMatrix, LogPrior prior) {
- this.numFeatures = numFeatures;
- this.numClasses = numClasses;
- this.data = data;
- this.labels = labels;
- this.prior = prior;
- this.confusionMatrix = confusionMatrix;
- }
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Classifier.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Classifier.java
deleted file mode 100644
index 4998323..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Classifier.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import edu.stanford.nlp.ling.Datum;
-import edu.stanford.nlp.stats.Counter;
-
-import java.io.Serializable;
-import java.util.Collection;
-
-/**
- * A simple interface for classifying and scoring data points, implemented
- * by most of the classifiers in this package. A basic Classifier
- * works over a List of categorical features. For classifiers over
- * real-valued features, see {@link RVFClassifier}.
- *
- * @author Dan Klein
- * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization)
- *
- * @param The type of the label(s) in each Datum
- * @param The type of the features in each Datum
- */
-
-public interface Classifier extends Serializable {
- public L classOf(Datum example);
-
- public Counter scoresOf(Datum example);
-
- public Collection labels();
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierCreator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierCreator.java
deleted file mode 100644
index d58905b..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierCreator.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package edu.stanford.nlp.classify;
-
-/**
- * Creates a classifier with given weights
- *
- * @author Angel Chang
- */
-public interface ClassifierCreator {
- public Classifier createClassifier(double[] weights);
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierFactory.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierFactory.java
deleted file mode 100644
index 4d4cb38..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/ClassifierFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import java.io.Serializable;
-import java.util.List;
-
-import edu.stanford.nlp.ling.RVFDatum;
-
-/**
- * A simple interface for training a Classifier from a Dataset of training
- * examples.
- *
- * @author Dan Klein
- *
- * Templatized by Sarah Spikes (sdspikes@cs.stanford.edu)
- */
-
-public interface ClassifierFactory> extends Serializable {
-
- @Deprecated //ClassifierFactory should implement trainClassifier(GeneralDataset) instead.
- public C trainClassifier(List> examples);
-
- public C trainClassifier(GeneralDataset dataset);
-
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/CrossValidator.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/CrossValidator.java
deleted file mode 100644
index aba5462..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/CrossValidator.java
+++ /dev/null
@@ -1,93 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import edu.stanford.nlp.util.Pair;
-import edu.stanford.nlp.util.Triple;
-import edu.stanford.nlp.util.Function;
-
-import java.util.Iterator;
-
-/**
- * This class is meant to simplify performing cross validation on
- * classifiers for hyper-parameters. It has the ability to save
- * state for each fold (for instance, the weights for a MaxEnt
- * classifier, and the alphas for an SVM).
- *
- * @author Aria Haghighi
- * @author Jenny Finkel
- * @author Sarah Spikes (Templatization)
- */
-
-public class CrossValidator {
- private GeneralDataset originalTrainData;
- private int kfold;
- private SavedState[] savedStates;
-
- public CrossValidator(GeneralDataset trainData) {
- this (trainData,5);
- }
-
- public CrossValidator(GeneralDataset trainData, int kfold) {
- originalTrainData = trainData;
- this.kfold = kfold;
- savedStates = new SavedState[kfold];
- for (int i = 0; i < savedStates.length; i++) {
- savedStates[i] = new SavedState();
- }
- }
-
- /**
- * Returns and Iterator over train/test/saved states
- */
- private Iterator,GeneralDataset,SavedState>> iterator() { return new CrossValidationIterator(); }
-
- /**
- * This computes the average over all folds of the function we're trying to optimize.
- * The input triple contains, in order, the train set, the test set, and the saved state.
- * You don't have to use the saved state if you don't want to.
- */
- public double computeAverage (Function,GeneralDataset,SavedState>,Double> function)
- {
- double sum = 0;
- Iterator,GeneralDataset,SavedState>> foldIt = iterator();
- while (foldIt.hasNext()) {
- sum += function.apply(foldIt.next());
- }
- return sum / kfold;
- }
-
- class CrossValidationIterator implements Iterator,GeneralDataset,SavedState>>
- {
- int iter = 0;
- public boolean hasNext() { return iter < kfold; }
-
- public void remove()
- {
- throw new RuntimeException("CrossValidationIterator doesn't support remove()");
- }
-
- public Triple,GeneralDataset,SavedState> next()
- {
- if (iter == kfold) return null;
- int start = originalTrainData.size() * iter / kfold;
- int end = originalTrainData.size() * (iter + 1) / kfold;
- //System.err.println("##train data size: " + originalTrainData.size() + " start " + start + " end " + end);
- Pair, GeneralDataset> split = originalTrainData.split(start, end);
-
- return new Triple,GeneralDataset,SavedState>(split.first(),split.second(),savedStates[iter++]);
- }
- }
-
- public static class SavedState {
- public Object state;
- }
-
- public static void main(String[] args) {
- Dataset d = Dataset.readSVMLightFormat(args[0]);
- Iterator,GeneralDataset,SavedState>> it = (new CrossValidator(d)).iterator();
- while (it.hasNext())
- {
- it.next();
- break;
- }
- }
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Dataset.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Dataset.java
deleted file mode 100644
index 36d7e52..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/Dataset.java
+++ /dev/null
@@ -1,728 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import edu.stanford.nlp.ling.BasicDatum;
-import edu.stanford.nlp.ling.Datum;
-import edu.stanford.nlp.ling.RVFDatum;
-import edu.stanford.nlp.stats.ClassicCounter;
-import edu.stanford.nlp.stats.Counter;
-import edu.stanford.nlp.stats.TwoDimensionalCounter;
-import edu.stanford.nlp.objectbank.ObjectBank;
-import edu.stanford.nlp.util.Index;
-import edu.stanford.nlp.util.HashIndex;
-import edu.stanford.nlp.util.Pair;
-import edu.stanford.nlp.util.ScoredComparator;
-import edu.stanford.nlp.util.ScoredObject;
-
-
-/**
- * An interfacing class for {@link ClassifierFactory} that incrementally
- * builds a more memory-efficient representation of a {@link List} of
- * {@link Datum} objects for the purposes of training a {@link Classifier}
- * with a {@link ClassifierFactory}.
- *
- * @author Roger Levy (rog@stanford.edu)
- * @author Anna Rafferty (various refactoring with GeneralDataset/RVFDataset)
- * @author Sarah Spikes (sdspikes@cs.stanford.edu) (templatization)
- * @author nmramesh@cs.stanford.edu {@link #getL1NormalizedTFIDFDatum(Datum, Counter) and #getL1NormalizedTFIDFDataset()}
- *
- * @param Label type
- * @param Feature type
- */
-public class Dataset extends GeneralDataset {
-
- private static final long serialVersionUID = -3883164942879961091L;
-
- public Dataset() {
- this(10);
- }
-
-
- public Dataset(int numDatums, Index featureIndex, Index labelIndex) {
- initialize(numDatums);
- this.labelIndex = labelIndex;
- this.featureIndex = featureIndex;
- }
-
- public Dataset(Index featureIndex, Index labelIndex) {
- this(10);
- this.labelIndex = labelIndex;
- this.featureIndex = featureIndex;
- }
-
- public Dataset(int numDatums) {
- initialize(numDatums);
- }
-
- /**
- * Constructor that fully specifies a Dataset. Needed this for MulticlassDataset.
- */
- public Dataset(Index labelIndex, int[] labels, Index featureIndex, int[][] data) {
- this (labelIndex, labels, featureIndex, data, data.length);
- }
-
- /**
- * Constructor that fully specifies a Dataset. Needed this for MulticlassDataset.
- */
- public Dataset(Index labelIndex, int[] labels, Index featureIndex, int[][] data, int size) {
- this.labelIndex = labelIndex;
- this.labels = labels;
- this.featureIndex = featureIndex;
- this.data = data;
- this.size = size;
- }
-
- @Override
- public Pair, GeneralDataset> split(double percentDev) {
- int devSize = (int)(percentDev * size());
- int trainSize = size() - devSize;
-
- int[][] devData = new int[devSize][];
- int[] devLabels = new int[devSize];
-
- int[][] trainData = new int[trainSize][];
- int[] trainLabels = new int[trainSize];
-
- System.arraycopy(data, 0, devData, 0, devSize);
- System.arraycopy(labels, 0, devLabels, 0, devSize);
-
- System.arraycopy(data, devSize, trainData, 0, trainSize);
- System.arraycopy(labels, devSize, trainLabels, 0, trainSize);
-
- if (this instanceof WeightedDataset,?>) {
- float[] trainWeights = new float[trainSize];
- float[] devWeights = new float[devSize];
-
- WeightedDataset w = (WeightedDataset)this;
-
- System.arraycopy(w.weights, 0, devWeights, 0, devSize);
- System.arraycopy(w.weights, devSize, trainWeights, 0, trainSize);
-
- WeightedDataset dev = new WeightedDataset(labelIndex, devLabels, featureIndex, devData, devSize, devWeights);
- WeightedDataset train = new WeightedDataset(labelIndex, trainLabels, featureIndex, trainData, trainSize, trainWeights);
-
- return new Pair,GeneralDataset>(train, dev);
- }
- Dataset dev = new Dataset(labelIndex, devLabels, featureIndex, devData, devSize);
- Dataset train = new Dataset(labelIndex, trainLabels, featureIndex, trainData, trainSize);
-
- return new Pair,GeneralDataset>(train, dev);
- }
-
- @Override
- public Pair,GeneralDataset> split(int start, int end) {
- int devSize = end - start;
- int trainSize = size() - devSize;
-
- int[][] devData = new int[devSize][];
- int[] devLabels = new int[devSize];
-
- int[][] trainData = new int[trainSize][];
- int[] trainLabels = new int[trainSize];
-
- System.arraycopy(data, start, devData, 0, devSize);
- System.arraycopy(labels, start, devLabels, 0, devSize);
-
- System.arraycopy(data, 0, trainData, 0, start);
- System.arraycopy(data, end, trainData, start, size()-end);
- System.arraycopy(labels, 0, trainLabels, 0, start);
- System.arraycopy(labels, end, trainLabels, start, size()-end);
-
- if (this instanceof WeightedDataset,?>) {
- float[] trainWeights = new float[trainSize];
- float[] devWeights = new float[devSize];
-
- WeightedDataset w = (WeightedDataset)this;
-
- System.arraycopy(w.weights, start, devWeights, 0, devSize);
- System.arraycopy(w.weights, 0, trainWeights, 0, start);
- System.arraycopy(w.weights, end, trainWeights, start, size()-end);
-
- WeightedDataset dev = new WeightedDataset(labelIndex, devLabels, featureIndex, devData, devSize, devWeights);
- WeightedDataset train = new WeightedDataset(labelIndex, trainLabels, featureIndex, trainData, trainSize, trainWeights);
-
- return new Pair,GeneralDataset>(train, dev);
- }
- Dataset dev = new Dataset(labelIndex, devLabels, featureIndex, devData, devSize);
- Dataset train = new Dataset(labelIndex, trainLabels, featureIndex, trainData, trainSize);
-
- return new Pair,GeneralDataset>(train, dev);
- }
-
-
- public Dataset getRandomSubDataset(double p, int seed) {
- int newSize = (int)(p * size());
- Set indicesToKeep = new HashSet();
- Random r = new Random(seed);
- int s = size();
- while (indicesToKeep.size() < newSize) {
- indicesToKeep.add(r.nextInt(s));
- }
-
- int[][] newData = new int[newSize][];
- int[] newLabels = new int[newSize];
-
- int i = 0;
- for (int j : indicesToKeep) {
- newData[i] = data[j];
- newLabels[i] = labels[j];
- i++;
- }
-
- return new Dataset(labelIndex, newLabels, featureIndex, newData);
- }
-
- @Override
- public double[][] getValuesArray() {
- return null;
- }
-
- /**
- * Constructs a Dataset by reading in a file in SVM light format.
- */
- public static Dataset readSVMLightFormat(String filename) {
- return readSVMLightFormat(filename, new HashIndex(), new HashIndex());
- }
-
- /**
- * Constructs a Dataset by reading in a file in SVM light format.
- * The lines parameter is filled with the lines of the file for further processing
- * (if lines is null, it is assumed no line information is desired)
- */
- public static Dataset readSVMLightFormat(String filename, List lines) {
- return readSVMLightFormat(filename, new HashIndex(), new HashIndex(), lines);
- }
-
- /**
- * Constructs a Dataset by reading in a file in SVM light format.
- * the created dataset has the same feature and label index as given
- */
- public static Dataset readSVMLightFormat(String filename, Index featureIndex, Index labelIndex) {
- return readSVMLightFormat(filename, featureIndex, labelIndex, null);
- }
- /**
- * Constructs a Dataset by reading in a file in SVM light format.
- * the created dataset has the same feature and label index as given
- */
- public static Dataset readSVMLightFormat(String filename, Index featureIndex, Index labelIndex, List lines) {
- Dataset dataset;
- try {
- dataset = new Dataset(10, featureIndex, labelIndex);
- for (String line : ObjectBank.getLineIterator(new File(filename))) {
- if(lines != null)
- lines.add(line);
- dataset.add(svmLightLineToDatum(line));
- }
-
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- return dataset;
- }
-
- private static int line1 = 0;
-
- public static Datum svmLightLineToDatum(String l) {
- line1++;
- l = l.replaceAll("#.*", ""); // remove any trailing comments
- String[] line = l.split("\\s+");
- Collection features = new ArrayList();
- for (int i = 1; i < line.length; i++) {
- String[] f = line[i].split(":");
- if (f.length != 2) {
- System.err.println("Dataset error: line " + line1);
- }
- int val = (int) Double.parseDouble(f[1]);
- for (int j = 0; j < val; j++) {
- features.add(f[0]);
- }
- }
- features.add(String.valueOf(Integer.MAX_VALUE)); // a constant feature for a class
- Datum d = new BasicDatum(features, line[0]);
- return d;
- }
-
- /**
- * Get Number of datums a given feature appears in.
- */
- public Counter getFeatureCounter()
- {
- Counter featureCounts = new ClassicCounter();
- for (int i=0; i < this.size(); i++)
- {
- BasicDatum datum = (BasicDatum) getDatum(i);
- Set featureSet = new HashSet(datum.asFeatures());
- for (F key : featureSet) {
- featureCounts.incrementCount(key, 1.0);
- }
- }
- return featureCounts;
- }
-
- /**
- * Method to convert features from counts to L1-normalized TFIDF based features
- * @param datum with a collection of features.
- * @param featureDocCounts a counter of doc-count for each feature.
- * @return RVFDatum with l1-normalized tf-idf features.
- */
- public RVFDatum getL1NormalizedTFIDFDatum(Datum datum,Counter featureDocCounts){
- Counter tfidfFeatures = new ClassicCounter();
- for(F feature : datum.asFeatures()){
- if(featureDocCounts.containsKey(feature))
- tfidfFeatures.incrementCount(feature,1.0);
- }
- double l1norm = 0;
- for(F feature: tfidfFeatures.keySet()){
- double idf = Math.log(((double)(this.size()+1))/(featureDocCounts.getCount(feature)+0.5));
- double tf = tfidfFeatures.getCount(feature);
- tfidfFeatures.setCount(feature, tf*idf);
- l1norm += tf*idf;
- }
- for(F feature: tfidfFeatures.keySet()){
- double tfidf = tfidfFeatures.getCount(feature);
- tfidfFeatures.setCount(feature, tfidf/l1norm);
- }
- RVFDatum rvfDatum = new RVFDatum(tfidfFeatures,datum.label());
- return rvfDatum;
- }
-
- /**
- * Method to convert this dataset to RVFDataset using L1-normalized TF-IDF features
- * @return RVFDataset
- */
- public RVFDataset getL1NormalizedTFIDFDataset(){
- RVFDataset rvfDataset = new RVFDataset(this.size(),this.featureIndex,this.labelIndex);
- Counter featureDocCounts = getFeatureCounter();
- for(int i = 0; i < this.size(); i++){
- Datum datum = this.getDatum(i);
- RVFDatum rvfDatum = getL1NormalizedTFIDFDatum(datum,featureDocCounts);
- rvfDataset.add(rvfDatum);
- }
- return rvfDataset;
- }
-
- @Override
- public void add(Datum d) {
- add(d.asFeatures(), d.label());
- }
-
- public void add(Collection features, L label) {
- ensureSize();
- addLabel(label);
- addFeatures(features);
- size++;
- }
-
- protected void ensureSize() {
- if (labels.length == size) {
- int[] newLabels = new int[size * 2];
- System.arraycopy(labels, 0, newLabels, 0, size);
- labels = newLabels;
- int[][] newData = new int[size * 2][];
- System.arraycopy(data, 0, newData, 0, size);
- data = newData;
- }
- }
-
- protected void addLabel(L label) {
- labelIndex.add(label);
- labels[size] = labelIndex.indexOf(label);
- }
-
- protected void addFeatures(Collection features) {
- int[] intFeatures = new int[features.size()];
- int j = 0;
- for (F feature : features) {
- featureIndex.add(feature);
- int index = featureIndex.indexOf(feature);
- if (index >= 0) {
- intFeatures[j] = featureIndex.indexOf(feature);
- j++;
- }
- }
- data[size] = new int[j];
- System.arraycopy(intFeatures, 0, data[size], 0, j);
- }
-
-
-
- @Override
- protected final void initialize(int numDatums) {
- labelIndex = new HashIndex();
- featureIndex = new HashIndex();
- labels = new int[numDatums];
- data = new int[numDatums][];
- size = 0;
- }
-
- /**
- * @return the index-ed datum
- */
- @Override
- public Datum getDatum(int index) {
- return new BasicDatum(featureIndex.objects(data[index]), labelIndex.get(labels[index]));
- }
-
- /**
- * @return the index-ed datum
- */
- @Override
- public RVFDatum getRVFDatum(int index) {
- ClassicCounter c = new ClassicCounter();
- for (F key : featureIndex.objects(data[index])) {
- c.incrementCount(key);
- }
- return new RVFDatum(c, labelIndex.get(labels[index]));
- }
-
- /**
- * Prints some summary statistics to stderr for the Dataset.
- */
- @Override
- public void summaryStatistics() {
- System.err.println(toSummaryStatistics());
- }
-
- public String toSummaryStatistics() {
- StringBuilder sb = new StringBuilder();
- sb.append("numDatums: ").append(size).append('\n');
- sb.append("numLabels: ").append(labelIndex.size()).append(" [");
- Iterator iter = labelIndex.iterator();
- while (iter.hasNext()) {
- sb.append(iter.next());
- if (iter.hasNext()) {
- sb.append(", ");
- }
- }
- sb.append("]\n");
- sb.append("numFeatures (Phi(X) types): ").append(featureIndex.size()).append('\n');
- // List l = new ArrayList(featureIndex);
-// Collections.sort(l);
-// sb.append(l);
- return sb.toString();
- }
-
-
- /**
- * Applies feature count thresholds to the Dataset.
- * Only features that match pattern_i and occur at
- * least threshold_i times (for some i) are kept.
- *
- * @param thresholds a list of pattern, threshold pairs
- */
- public void applyFeatureCountThreshold(List> thresholds) {
-
- // get feature counts
- float[] counts = getFeatureCounts();
-
- // build a new featureIndex
- Index newFeatureIndex = new HashIndex();
- LOOP: for (F f : featureIndex) {
- for (Pair threshold : thresholds) {
- Pattern p = threshold.first();
- Matcher m = p.matcher(f.toString());
- if (m.matches()) {
- if (counts[featureIndex.indexOf(f)] >= threshold.second) {
- newFeatureIndex.add(f);
- }
- continue LOOP;
- }
- }
- // we only get here if it didn't match anything on the list
- newFeatureIndex.add(f);
- }
-
- counts = null;
-
- int[] featMap = new int[featureIndex.size()];
- for (int i = 0; i < featMap.length; i++) {
- featMap[i] = newFeatureIndex.indexOf(featureIndex.get(i));
- }
-
- featureIndex = null;
-
- for (int i = 0; i < size; i++) {
- List featList = new ArrayList(data[i].length);
- for (int j = 0; j < data[i].length; j++) {
- if (featMap[data[i][j]] >= 0) {
- featList.add(featMap[data[i][j]]);
- }
- }
- data[i] = new int[featList.size()];
- for (int j = 0; j < data[i].length; j++) {
- data[i][j] = featList.get(j);
- }
- }
-
- //TODO: continuation of hack from above... hope those features are really strings
- //if you call this function. ~Sarah
- featureIndex = newFeatureIndex;
- }
-
-
- /**
- * prints the full feature matrix in tab-delimited form. These can be BIG
- * matrices, so be careful!
- */
- public void printFullFeatureMatrix(PrintWriter pw) {
- String sep = "\t";
- for (int i = 0; i < featureIndex.size(); i++) {
- pw.print(sep + featureIndex.get(i));
- }
- pw.println();
- for (int i = 0; i < labels.length; i++) {
- pw.print(labelIndex.get(i));
- Set feats = new HashSet();
- for (int j = 0; j < data[i].length; j++) {
- int feature = data[i][j];
- feats.add(Integer.valueOf(feature));
- }
- for (int j = 0; j < featureIndex.size(); j++) {
- if (feats.contains(Integer.valueOf(j))) {
- pw.print(sep + '1');
- } else {
- pw.print(sep + '0');
- }
- }
- }
- }
-
- /**
- * prints the sparse feature matrix using {@link #printSparseFeatureMatrix()}
- * to {@link System#out System.out}.
- */
- public void printSparseFeatureMatrix() {
- printSparseFeatureMatrix(new PrintWriter(System.out, true));
- }
-
- /**
- * prints a sparse feature matrix representation of the Dataset. Prints the actual
- * {@link Object#toString()} representations of features.
- */
- public void printSparseFeatureMatrix(PrintWriter pw) {
- String sep = "\t";
- for (int i = 0; i < size; i++) {
- pw.print(labelIndex.get(labels[i]));
- int[] datum = data[i];
- for (int j : datum) {
- pw.print(sep + featureIndex.get(j));
- }
- pw.println();
- }
- }
-
-
- public void changeLabelIndex(Index newLabelIndex) {
-
- labels = trimToSize(labels);
-
- for (int i = 0; i < labels.length; i++) {
- labels[i] = newLabelIndex.indexOf(labelIndex.get(labels[i]));
- }
- labelIndex = newLabelIndex;
- }
-
- public void changeFeatureIndex(Index newFeatureIndex) {
-
- data = trimToSize(data);
- labels = trimToSize(labels);
-
- int[][] newData = new int[data.length][];
- for (int i = 0; i < data.length; i++) {
- int[] newD = new int[data[i].length];
- int k = 0;
- for (int j = 0; j < data[i].length; j++) {
- int newIndex = newFeatureIndex.indexOf(featureIndex.get(data[i][j]));
- if (newIndex >= 0) {
- newD[k++] = newIndex;
- }
- }
- newData[i] = new int[k];
- System.arraycopy(newD, 0, newData[i], 0, k);
- }
- data = newData;
- featureIndex = newFeatureIndex;
- }
-
- public void selectFeaturesBinaryInformationGain(int numFeatures) {
- double[] scores = getInformationGains();
- selectFeatures(numFeatures,scores);
- }
-
- /**
- * Generic method to select features based on the feature scores vector provided as an argument.
- * @param numFeatures number of features to be selected.
- * @param scores a vector of size total number of features in the data.
- */
- public void selectFeatures(int numFeatures, double[] scores) {
-
- List> scoredFeatures = new ArrayList>();
-
- for (int i = 0; i < scores.length; i++) {
- scoredFeatures.add(new ScoredObject(featureIndex.get(i), scores[i]));
- }
-
- Collections.sort(scoredFeatures, ScoredComparator.DESCENDING_COMPARATOR);
- Index newFeatureIndex = new HashIndex();
- for (int i = 0; i < scoredFeatures.size() && i < numFeatures; i++) {
- newFeatureIndex.add(scoredFeatures.get(i).object());
- //System.err.println(scoredFeatures.get(i));
- }
-
- for (int i = 0; i < size; i++) {
- int[] newData = new int[data[i].length];
- int curIndex = 0;
- for (int j = 0; j < data[i].length; j++) {
- int index;
- if ((index = newFeatureIndex.indexOf(featureIndex.get(data[i][j]))) != -1) {
- newData[curIndex++] = index;
- }
- }
- int[] newDataTrimmed = new int[curIndex];
- System.arraycopy(newData, 0, newDataTrimmed, 0, curIndex);
- data[i] = newDataTrimmed;
- }
- featureIndex = newFeatureIndex;
- }
-
-
- public double[] getInformationGains() {
-
- data = trimToSize(data);
- labels = trimToSize(labels);
-
- // counts the number of times word X is present
- ClassicCounter featureCounter = new ClassicCounter();
-
- // counts the number of time a document has label Y
- ClassicCounter labelCounter = new ClassicCounter();
-
- // counts the number of times the document has label Y given word X is present
- TwoDimensionalCounter condCounter = new TwoDimensionalCounter();
-
- for (int i = 0; i < labels.length; i++) {
- labelCounter.incrementCount(labelIndex.get(labels[i]));
-
- // convert the document to binary feature representation
- boolean[] doc = new boolean[featureIndex.size()];
- //System.err.println(i);
- for (int j = 0; j < data[i].length; j++) {
- doc[data[i][j]] = true;
- }
-
- for (int j = 0; j < doc.length; j++) {
- if (doc[j]) {
- featureCounter.incrementCount(featureIndex.get(j));
- condCounter.incrementCount(featureIndex.get(j), labelIndex.get(labels[i]), 1.0);
- }
- }
- }
-
- double entropy = 0.0;
- for (int i = 0; i < labelIndex.size(); i++) {
- double labelCount = labelCounter.getCount(labelIndex.get(i));
- double p = labelCount / size();
- entropy -= p * (Math.log(p) / Math.log(2));
- }
-
- double[] ig = new double[featureIndex.size()];
- Arrays.fill(ig, entropy);
-
- for (int i = 0; i < featureIndex.size(); i++) {
- F feature = featureIndex.get(i);
-
- double featureCount = featureCounter.getCount(feature);
- double notFeatureCount = size() - featureCount;
-
- double pFeature = featureCount / size();
- double pNotFeature = (1.0 - pFeature);
-
- if (featureCount == 0) { ig[i] = 0; continue; }
- if (notFeatureCount == 0) { ig[i] = 0; continue; }
-
- double sumFeature = 0.0;
- double sumNotFeature = 0.0;
-
- for (int j = 0; j < labelIndex.size(); j++) {
- L label = labelIndex.get(j);
-
- double featureLabelCount = condCounter.getCount(feature, label);
- double notFeatureLabelCount = size() - featureLabelCount;
-
- // yes, these dont sum to 1. that is correct.
- // one is the prob of the label, given that the
- // feature is present, and the other is the prob
- // of the label given that the feature is absent
- double p = featureLabelCount / featureCount;
- double pNot = notFeatureLabelCount / notFeatureCount;
-
- if (featureLabelCount != 0) {
- sumFeature += p * (Math.log(p) / Math.log(2));
- }
-
- if (notFeatureLabelCount != 0) {
- sumNotFeature += pNot * (Math.log(pNot) / Math.log(2));
- }
- //System.out.println(pNot+" "+(Math.log(pNot)/Math.log(2)));
-
- }
-
- //System.err.println(pFeature+" * "+sumFeature+" = +"+);
- //System.err.println("^ "+pNotFeature+" "+sumNotFeature);
-
- ig[i] += pFeature*sumFeature + pNotFeature*sumNotFeature;
- /* earlier the line above used to be: ig[i] = pFeature*sumFeature + pNotFeature*sumNotFeature;
- * This completely ignored the entropy term computed above. So added the "+=" to take that into account.
- * -Ramesh (nmramesh@cs.stanford.edu)
- */
- }
- return ig;
- }
-
-
-
- @Override
- public String toString() {
- return "Dataset of size " + size;
- }
-
- public String toSummaryString() {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- pw.println("Number of data points: " + size());
- pw.println("Number of active feature tokens: " + numFeatureTokens());
- pw.println("Number of active feature types:" + numFeatureTypes());
- return pw.toString();
- }
-
- /**
- * Need to sort the counter by feature keys and dump it
- *
- */
- public static void printSVMLightFormat(PrintWriter pw, ClassicCounter c, int classNo) {
- Integer[] features = c.keySet().toArray(new Integer[c.keySet().size()]);
- Arrays.sort(features);
- StringBuilder sb = new StringBuilder();
- sb.append(classNo);
- sb.append(' ');
- for (int f: features) {
- sb.append(f + 1).append(':').append(c.getCount(f)).append(' ');
- }
- pw.println(sb.toString());
- }
-
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralDataset.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralDataset.java
deleted file mode 100644
index e25adb4..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralDataset.java
+++ /dev/null
@@ -1,488 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import java.io.PrintWriter;
-import java.io.Serializable;
-import java.util.*;
-
-import edu.stanford.nlp.ling.BasicDatum;
-import edu.stanford.nlp.ling.Datum;
-import edu.stanford.nlp.ling.RVFDatum;
-import edu.stanford.nlp.stats.ClassicCounter;
-import edu.stanford.nlp.stats.Counter;
-import edu.stanford.nlp.util.HashIndex;
-import edu.stanford.nlp.util.Index;
-import edu.stanford.nlp.util.Pair;
-
-/**
- * The purpose of this interface is to unify {@link Dataset} and {@link RVFDataset}.
- * @author Kristina Toutanova (kristina@cs.stanford.edu)
- * @author Anna Rafferty (various refactoring with subclasses)
- * @author Sarah Spikes (sdspikes@cs.stanford.edu) (Templatization)
- *
- * @param The type of the labels in the Dataset
- * @param The type of the features in the Dataset
- *
- * @author Ramesh Nallapati (nmramesh@cs.stanford.edu)
- * Added an abstract method getDatum. Juily 17th, 2008.
- */
-public abstract class GeneralDataset implements Serializable, Iterable> {
-
- private static final long serialVersionUID = 19157757130054829L;
-
- public Index labelIndex;
- public Index featureIndex;
-
- protected int[] labels;
- protected int[][] data;
-
- protected int size;
-
- public GeneralDataset() {
- }
-
- public Index labelIndex() { return labelIndex; }
-
- public Index featureIndex() { return featureIndex; }
-
- public int numFeatures() { return featureIndex.size(); }
-
- public int numClasses() { return labelIndex.size(); }
-
- public int[] getLabelsArray() {
- labels = trimToSize(labels);
- return labels;
- }
-
- public int[][] getDataArray() {
- data = trimToSize(data);
- return data;
- }
-
- public abstract double[][] getValuesArray();
-
- /**
- * Resets the Dataset so that it is empty and ready to collect data.
- */
- public void clear() {
- clear(10);
- }
-
- /**
- * Resets the Dataset so that it is empty and ready to collect data.
- * @param numDatums initial capacity of dataset
- */
- public void clear(int numDatums) {
- initialize(numDatums);
- }
-
- /**
- * This method takes care of resetting values of the dataset
- * such that it is empty with an initial capacity of numDatums
- *
- * Should be accessed only by appropriate methods within the class,
- * such as clear(), which take care of other parts of the emptying of data
- *
- * @param numDatums initial capacity of dataset
- */
- protected abstract void initialize(int numDatums);
-
-
- public abstract RVFDatum getRVFDatum(int index);
-
- public abstract Datum getDatum(int index);
-
-
- public abstract void add(Datum d);
-
- /**
- * Get the total count (over all data instances) of each feature
- *
- * @return an array containing the counts (indexed by index)
- */
- public float[] getFeatureCounts() {
- float[] counts = new float[featureIndex.size()];
- for (int i = 0, m = size; i < m; i++) {
- for (int j = 0, n = data[i].length; j < n; j++) {
- counts[data[i][j]] += 1.0;
- }
- }
- return counts;
- }
-
- /**
- * Applies a feature count threshold to the Dataset. All features that
- * occur fewer than k times are expunged.
- */
- public void applyFeatureCountThreshold(int k) {
- float[] counts = getFeatureCounts();
- Index newFeatureIndex = new HashIndex();
-
- int[] featMap = new int[featureIndex.size()];
- for (int i = 0; i < featMap.length; i++) {
- F feat = featureIndex.get(i);
- if (counts[i] >= k) {
- int newIndex = newFeatureIndex.size();
- newFeatureIndex.add(feat);
- featMap[i] = newIndex;
- } else {
- featMap[i] = -1;
- }
- // featureIndex.remove(feat);
- }
-
- featureIndex = newFeatureIndex;
- // counts = null; // This is unnecessary; JVM can clean it up
-
- for (int i = 0; i < size; i++) {
- List featList = new ArrayList(data[i].length);
- for (int j = 0; j < data[i].length; j++) {
- if (featMap[data[i][j]] >= 0) {
- featList.add(featMap[data[i][j]]);
- }
- }
- data[i] = new int[featList.size()];
- for (int j = 0; j < data[i].length; j++) {
- data[i][j] = featList.get(j);
- }
- }
- }
-
-
- /**
- * Applies a max feature count threshold to the Dataset. All features that
- * occur greater than k times are expunged.
- */
- public void applyFeatureMaxCountThreshold(int k) {
- float[] counts = getFeatureCounts();
- HashIndex newFeatureIndex = new HashIndex();
-
- int[] featMap = new int[featureIndex.size()];
- for (int i = 0; i < featMap.length; i++) {
- F feat = featureIndex.get(i);
- if (counts[i] <= k) {
- int newIndex = newFeatureIndex.size();
- newFeatureIndex.add(feat);
- featMap[i] = newIndex;
- } else {
- featMap[i] = -1;
- }
- // featureIndex.remove(feat);
- }
-
- featureIndex = newFeatureIndex;
- // counts = null; // This is unnecessary; JVM can clean it up
-
- for (int i = 0; i < size; i++) {
- List featList = new ArrayList(data[i].length);
- for (int j = 0; j < data[i].length; j++) {
- if (featMap[data[i][j]] >= 0) {
- featList.add(featMap[data[i][j]]);
- }
- }
- data[i] = new int[featList.size()];
- for (int j = 0; j < data[i].length; j++) {
- data[i][j] = featList.get(j);
- }
- }
- }
-
-
- /**
- * returns the number of feature tokens in the Dataset.
- */
- public int numFeatureTokens() {
- int x = 0;
- for (int i = 0, m = size; i < m; i++) {
- x += data[i].length;
- }
- return x;
- }
-
- /**
- * returns the number of distinct feature types in the Dataset.
- */
- public int numFeatureTypes() {
- return featureIndex.size();
- }
-
-
-
- /**
- * Adds all Datums in the given collection of data to this dataset
- * @param data collection of datums you would like to add to the dataset
- */
- public void addAll(Iterable extends Datum> data) {
- for (Datum d : data) {
- add(d);
- }
- }
-
- public abstract Pair, GeneralDataset> split (int start, int end) ;
- public abstract Pair, GeneralDataset> split (double p) ;
-
- /**
- * Returns the number of examples ({@link Datum}s) in the Dataset.
- */
- public int size() { return size; }
-
- protected void trimData() {
- data = trimToSize(data);
- }
-
- protected void trimLabels() {
- labels = trimToSize(labels);
- }
-
- protected int[] trimToSize(int[] i) {
- int[] newI = new int[size];
- System.arraycopy(i, 0, newI, 0, size);
- return newI;
- }
-
- protected int[][] trimToSize(int[][] i) {
- int[][] newI = new int[size][];
- System.arraycopy(i, 0, newI, 0, size);
- return newI;
- }
-
- protected double[][] trimToSize(double[][] i) {
- double[][] newI = new double[size][];
- System.arraycopy(i, 0, newI, 0, size);
- return newI;
- }
-
- /**
- * Randomizes the data array in place
- * Note: this cannot change the values array or the datum weights, so redefine this for RVFDataset and WeightedDataset!
- * @param randomSeed
- */
- public void randomize(int randomSeed) {
- Random rand = new Random(randomSeed);
- for(int j = size - 1; j > 0; j --){
- int randIndex = rand.nextInt(j);
-
- int [] tmp = data[randIndex];
- data[randIndex] = data[j];
- data[j] = tmp;
-
- int tmpl = labels[randIndex];
- labels[randIndex] = labels[j];
- labels[j] = tmpl;
- }
- }
-
- public GeneralDataset sampleDataset(int randomSeed, double sampleFrac, boolean sampleWithReplacement) {
- int sampleSize = (int)(this.size()*sampleFrac);
- Random rand = new Random(randomSeed);
- GeneralDataset subset;
- if(this instanceof RVFDataset)
- subset = new RVFDataset();
- else if (this instanceof Dataset) {
- subset = new Dataset();
- }
- else {
- throw new RuntimeException("Can't handle this type of GeneralDataset.");
- }
- if (sampleWithReplacement) {
- for(int i = 0; i < sampleSize; i++){
- int datumNum = rand.nextInt(this.size());
- subset.add(this.getDatum(datumNum));
- }
- } else {
- Set indicedSampled = new HashSet();
- while (subset.size() < sampleSize) {
- int datumNum = rand.nextInt(this.size());
- if (!indicedSampled.contains(datumNum)) {
- subset.add(this.getDatum(datumNum));
- indicedSampled.add(datumNum);
- }
- }
- }
- return subset;
- }
-
- /**
- * Print some statistics summarizing the dataset
- *
- */
- public abstract void summaryStatistics();
-
- /**
- * Returns an iterator over the class labels of the Dataset
- *
- * @return An iterator over the class labels of the Dataset
- */
- public Iterator labelIterator() {
- return labelIndex.iterator();
- }
-
-
- /**
- *
- * @param dataset
- * @return a new GeneralDataset whose features and ids map exactly to those of this GeneralDataset.
- * Useful when two Datasets are created independently and one wants to train a model on one dataset and test on the other. -Ramesh.
- */
- public GeneralDataset mapDataset(GeneralDataset dataset){
- GeneralDataset newDataset;
- if(dataset instanceof RVFDataset)
- newDataset = new RVFDataset(this.featureIndex,this.labelIndex);
- else newDataset = new Dataset(this.featureIndex,this.labelIndex);
- this.featureIndex.lock();
- this.labelIndex.lock();
- //System.out.println("inside mapDataset: dataset size:"+dataset.size());
- for(int i = 0; i < dataset.size(); i++)
- //System.out.println("inside mapDataset: adding datum number"+i);
- newDataset.add(dataset.getDatum(i));
-
- //System.out.println("old Dataset stats: numData:"+dataset.size()+" numfeatures:"+dataset.featureIndex().size()+" numlabels:"+dataset.labelIndex.size());
- //System.out.println("new Dataset stats: numData:"+newDataset.size()+" numfeatures:"+newDataset.featureIndex().size()+" numlabels:"+newDataset.labelIndex.size());
- //System.out.println("this dataset stats: numData:"+size()+" numfeatures:"+featureIndex().size()+" numlabels:"+labelIndex.size());
-
- this.featureIndex.unlock();
- this.labelIndex.unlock();
- return newDataset;
- }
-
- public static Datum mapDatum(Datum d, Map labelMapping, L2 defaultLabel)
- {
- Datum d2 = null;
- // TODO: How to copy datum?
- L2 newLabel = labelMapping.get(d.label());
- if (newLabel == null) {
- newLabel = defaultLabel;
- }
- if (d instanceof RVFDatum) {
- d2 = new RVFDatum( ((RVFDatum) d).asFeaturesCounter(), newLabel );
- } else {
- d2 = new BasicDatum( d.asFeatures(), newLabel );
- }
- return d2;
- }
-
-
- /**
- *
- * @param dataset
- * @return a new GeneralDataset whose features and ids map exactly to those of this GeneralDataset. But labels are converted to be another set of labels
- */
- public GeneralDataset mapDataset(GeneralDataset dataset, Index newLabelIndex, Map labelMapping, L2 defaultLabel)
- {
- GeneralDataset newDataset;
- if(dataset instanceof RVFDataset)
- newDataset = new RVFDataset(this.featureIndex, newLabelIndex);
- else newDataset = new Dataset(this.featureIndex, newLabelIndex);
- this.featureIndex.lock();
- this.labelIndex.lock();
- //System.out.println("inside mapDataset: dataset size:"+dataset.size());
- for(int i = 0; i < dataset.size(); i++) {
- //System.out.println("inside mapDataset: adding datum number"+i);
- Datum d = dataset.getDatum(i);
- Datum d2 = mapDatum(d, labelMapping, defaultLabel);
- newDataset.add(d2);
- }
- //System.out.println("old Dataset stats: numData:"+dataset.size()+" numfeatures:"+dataset.featureIndex().size()+" numlabels:"+dataset.labelIndex.size());
- //System.out.println("new Dataset stats: numData:"+newDataset.size()+" numfeatures:"+newDataset.featureIndex().size()+" numlabels:"+newDataset.labelIndex.size());
- //System.out.println("this dataset stats: numData:"+size()+" numfeatures:"+featureIndex().size()+" numlabels:"+labelIndex.size());
-
- this.featureIndex.unlock();
- this.labelIndex.unlock();
- return newDataset;
- }
-
- /**
- * Dumps the Dataset as a training/test file for SVMLight.
- * class [fno:val]+
- * The features must occur in consecutive order.
- */
- public void printSVMLightFormat() {
- printSVMLightFormat(new PrintWriter(System.out));
- }
-
- /**
- * Maps our labels to labels that are compatible with svm_light
- * @return array of strings
- */
- public String[] makeSvmLabelMap() {
- String[] labelMap = new String[numClasses()];
- if (numClasses() > 2) {
- for (int i = 0; i < labelMap.length; i++) {
- labelMap[i] = String.valueOf((i + 1));
- }
- } else {
- labelMap = new String[]{"+1", "-1"};
- }
- return labelMap;
- }
-
- /**
- * Print SVM Light Format file.
- *
- * The following comments are no longer applicable because I am
- * now printing out the exact labelID for each example. -Ramesh (nmramesh@cs.stanford.edu) 12/17/2009.
- *
- * If the Dataset has more than 2 classes, then it
- * prints using the label index (+1) (for svm_struct). If it is 2 classes, then the labelIndex.get(0)
- * is mapped to +1 and labelIndex.get(1) is mapped to -1 (for svm_light).
- */
-
- public void printSVMLightFormat(PrintWriter pw) {
- //assumes each data item has a few features on, and sorts the feature keys while collecting the values in a counter
-
- // old comment:
- // the following code commented out by Ramesh (nmramesh@cs.stanford.edu) 12/17/2009.
- // why not simply print the exact id of the label instead of mapping to some values??
- // new comment:
- // mihai: we NEED this, because svm_light has special conventions not supported by default by our labels,
- // e.g., in a multiclass setting it assumes that labels start at 1 whereas our labels start at 0 (08/31/2010)
- String[] labelMap = makeSvmLabelMap();
-
- for (int i = 0; i < size; i++) {
- RVFDatum d = getRVFDatum(i);
- Counter c = d.asFeaturesCounter();
- ClassicCounter printC = new ClassicCounter();
- for (F f : c.keySet()) {
- printC.setCount(featureIndex.indexOf(f), c.getCount(f));
- }
- Integer[] features = printC.keySet().toArray(new Integer[printC.keySet().size()]);
- Arrays.sort(features);
- StringBuilder sb = new StringBuilder();
- sb.append(labelMap[labels[i]]).append(' ');
- // sb.append(labels[i]).append(' '); // commented out by mihai: labels[i] breaks svm_light conventions!
-
- /* Old code: assumes that F is Integer....
- *
- for (int f: features) {
- sb.append((f + 1)).append(":").append(c.getCount(f)).append(" ");
- }
- */
- //I think this is what was meant (using printC rather than c), but not sure
- // ~Sarah Spikes (sdspikes@cs.stanford.edu)
- for (int f: features) {
- sb.append((f + 1)).append(':').append(printC.getCount(f)).append(' ');
- }
- pw.println(sb.toString());
- }
- }
-
-
- public Iterator> iterator() {
- return new Iterator>() {
- int id = 0;
-
- public boolean hasNext() {
- return id < size();
- }
-
- public RVFDatum next() {
- return getRVFDatum(id++);
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
-}
diff --git a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralizedExpectationObjectiveFunction.java b/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralizedExpectationObjectiveFunction.java
deleted file mode 100644
index 8c1ce5b..0000000
--- a/stanford-ner-2011-09-14/src/edu/stanford/nlp/classify/GeneralizedExpectationObjectiveFunction.java
+++ /dev/null
@@ -1,217 +0,0 @@
-package edu.stanford.nlp.classify;
-
-import edu.stanford.nlp.ling.Datum;
-import edu.stanford.nlp.ling.RVFDatum;
-import edu.stanford.nlp.math.ArrayMath;
-import edu.stanford.nlp.optimization.AbstractCachingDiffFunction;
-import edu.stanford.nlp.stats.ClassicCounter;
-import edu.stanford.nlp.stats.Counter;
-import edu.stanford.nlp.util.Triple;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-
-/**
- * Implementation of Generalized Expectation Objective function for an I.I.D. log-linear model. See Mann and McCallum, ACL 2008.
- * IMPORTANT : the current implementation is only correct as long as the labeled features passed to GE are binary.
- * However, other features are allowed to be real valued.
- * The original paper also discusses GE only for binary features.
- * @author Ramesh Nallapati (nmramesh@cs.stanford.edu)
- */
-
-public class GeneralizedExpectationObjectiveFunction extends AbstractCachingDiffFunction {
-
-
- private GeneralDataset labeledDataset;
- private List extends Datum