diff --git a/.gitignore b/.gitignore
index 9a954b8..1d81a7b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ bin/
*/.tmp
*.log
+/target/
diff --git a/pom.xml b/pom.xml
index f8e0514..5e71b07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,123 +1,128 @@
- 4.0.0
- com.magnetic
- erd
- 0.0.1-SNAPSHOT
- erd-contest
- Entity Linking Challenge
-
-
-
-
-
- org.apache.lucene
- lucene-suggest
- 4.3.0
-
-
-
- com.fasterxml.jackson.core
- jackson-databind
- 2.6.2
-
-
-
- com.fasterxml.jackson.core
- jackson-annotations
- 2.6.2
-
-
-
- com.fasterxml.jackson.core
- jackson-core
- 2.6.2
-
-
-
- junit
- junit
- 4.11
-
-
-
- com.sun.jersey
- jersey-servlet
- 1.17
-
-
-
- com.sun.jersey.contribs
- jersey-multipart
- 1.17
-
-
-
- com.sun.jersey
- jersey-json
- 1.17
-
-
- com.sun.jersey
- jersey-bundle
- 1.17
-
-
-
- org.jvnet
- mimepull
- 1.6
-
-
-
- org.apache.lucene
- lucene-core
- 4.3.0
-
-
-
-
- org.apache.lucene
- lucene-queryparser
- 4.3.0
-
-
-
- org.apache.lucene
- lucene-analyzers-common
- 4.3.0
-
-
-
- log4j
- log4j
- 1.2.16
-
-
-
-
- com.ibm.icu
- icu4j
- 53.1
-
-
-
- commons-lang
- commons-lang
- 2.3
-
-
-
-
-
-
- org.apache.commons
- commons-compress
- 1.5
-
-
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
+ com.magnetic
+ erd
+ 0.0.1-SNAPSHOT
+ erd-contest
+ Entity Linking Challenge
+
+
+
+
+
+ org.apache.lucene
+ lucene-suggest
+ 4.3.0
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ 2.6.2
+
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+ 2.6.2
+
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+ 2.6.2
+
+
+
+ junit
+ junit
+ 4.11
+
+
+
+ com.sun.jersey
+ jersey-servlet
+ 1.17
+
+
+
+ com.sun.jersey.contribs
+ jersey-multipart
+ 1.17
+
+
+
+ com.sun.jersey
+ jersey-json
+ 1.17
+
+
+ com.sun.jersey
+ jersey-bundle
+ 1.17
+
+
+
+ org.jvnet
+ mimepull
+ 1.6
+
+
+
+ org.apache.lucene
+ lucene-core
+ 4.3.0
+
+
+
+
+ org.apache.lucene
+ lucene-queryparser
+ 4.3.0
+
+
+
+ org.apache.lucene
+ lucene-analyzers-common
+ 4.3.0
+
+
+
+ log4j
+ log4j
+ 1.2.16
+
+
+
+
+ com.ibm.icu
+ icu4j
+ 53.1
+
+
+
+ commons-lang
+ commons-lang
+ 2.3
+
+
+
+
+
+
+ org.apache.commons
+ commons-compress
+ 1.5
+
+
+
+ commons-cli
+ commons-cli
+ 1.2
+
commons-cli
commons-cli
@@ -128,67 +133,82 @@
poi
3.11
-
-
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- 2.3.2
-
- 1.7
- 1.7
- UTF-8
-
-
-
-
-
- maven-assembly-plugin
-
-
-
- jar-with-dependencies
-
-
-
-
-
-
+
+ org.apache.commons
+ commons-lang3
+
+
+ commons-io
+ commons-io
+
+
+ org.jblas
+ jblas
+
+
+ me.lemire.integercompression
+ JavaFastPFOR
+
+
+ com.esotericsoftware.kryo
+ kryo
+
+
+ edu.stanford.nlp
+ stanford-corenlp
+ 3.6.0
+
+
+ edu.stanford.nlp
+ stanford-corenlp
+ 3.6.0
+ models
+
+
+ edu.washington.cs.knowitall.openie
+ openie_2.10
+ 4.2.1
+
+
+ net.sf.jwordnet
+ jwnl
+ 1.4_rc3
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 2.3.2
+
+ 1.7
+ 1.7
+ UTF-8
+
+
+
+
+
+ maven-assembly-plugin
+
+
+
+ jar-with-dependencies
+
+
+
+
+
+
org.apache.maven.plugins
maven-assembly-plugin
@@ -216,7 +236,46 @@
-
-
+
+
+
+
+
+ org.apache.commons
+ commons-math3
+ 3.5
+
+
+ org.apache.commons
+ commons-lang3
+ 3.3.2
+
+
+ commons-io
+ commons-io
+ 2.4
+
+
+ org.jblas
+ jblas
+ 1.2.3
+
+
+ me.lemire.integercompression
+ JavaFastPFOR
+ 0.0.12
+
+
+ com.esotericsoftware.kryo
+ kryo
+ 2.24.0
+
+
+ edu.stanford.nlp
+ stanford-corenlp
+ 3.6.0
+
+
+
diff --git a/src/main/java/DP_entity_linking/Main.java b/src/main/java/DP_entity_linking/Main.java
index 793f442..9ea37a1 100644
--- a/src/main/java/DP_entity_linking/Main.java
+++ b/src/main/java/DP_entity_linking/Main.java
@@ -23,7 +23,7 @@ public class Main {
public void normalStart() throws IOException, ParseException {
- DataSet dataset = new DataSet();
+ DataSet dataset = new DataSet("C:\\workspace\\webquestions.json");
List records = dataset.loadWebquestions();
//records = records.subList(0, 3700);
records = records.subList(0, 5);
diff --git a/src/main/java/DP_entity_linking/dataset/DataSet.java b/src/main/java/DP_entity_linking/dataset/DataSet.java
index 3077e77..2beea89 100644
--- a/src/main/java/DP_entity_linking/dataset/DataSet.java
+++ b/src/main/java/DP_entity_linking/dataset/DataSet.java
@@ -13,21 +13,25 @@
* Created by miroslav.kudlac on 11/22/2015.
*/
public class DataSet {
- private static Logger LOGGER = Logger.getLogger(Search.class);
- //private static final File JSON_FILE = new File("data/webquestions.train");
- private static final File JSON_FILE = new File("data/data.json");
- public static class Records extends ArrayList {
- public Records() {
- }
- }
+ private static Logger LOGGER = Logger.getLogger(Search.class);
+ private static File JSON_FILE;
- /**
- * @return
- * @throws IOException
- */
- public List loadWebquestions() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return (List)mapper.readValue(JSON_FILE, Records.class);
- }
+ public DataSet(String path) {
+ JSON_FILE = new File(path);
+ }
+
+ public static class Records extends ArrayList {
+ public Records() {
+ }
+ }
+
+ /**
+ * @return
+ * @throws IOException
+ */
+ public List loadWebquestions() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ return (List) mapper.readValue(JSON_FILE, Records.class);
+ }
}
diff --git a/src/main/java/DP_entity_linking/dataset/Record.java b/src/main/java/DP_entity_linking/dataset/Record.java
index 57fba4c..e108601 100644
--- a/src/main/java/DP_entity_linking/dataset/Record.java
+++ b/src/main/java/DP_entity_linking/dataset/Record.java
@@ -1,58 +1,64 @@
package DP_entity_linking.dataset;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Map;
+
/**
* Created by miroslav.kudlac on 10/3/2015.
*/
public class Record {
- private String utterance;
- private String url;
- private String targetValue;
-
- public String getUrl() {
- return url;
- }
-
- public void setUrl(String url) {
- this.url = url;
- }
-
- public void setTargetValue(String targetValue) {
- this.targetValue = targetValue;
- }
-
- public String getTargetValue() {
- return targetValue;
- }
-
- public String getUtterance() {
- return utterance;
- }
-
- public void setUtterance(String utterance) {
- this.utterance = utterance;
- }
-
- public String getAnswer() {
- if (getUrl() == null) {
- return " ";
- }
- return getUrl();
- // String path = this.getUrl().getPath();
- // path = path.substring(path.lastIndexOf('/') + 1);
- // String answer = path.replace("_", " ");
- // return answer;
- }
-
- public String getQuestion() {
- return getUtterance();
- }
-
- @Override
- public String toString() {
- return "Record{" +
- "utterance='" + utterance + '\'' +
- ", url=" + url +
- ", targetValue='" + targetValue + '\'' +
- '}';
- }
+ private String utterance;
+ private URI url;
+ private String targetValue;
+ private Map> relations;
+
+ public URI getUrl() {
+ return url;
+ }
+
+ public void setUrl(URI url) {
+ this.url = url;
+ }
+
+ public void setTargetValue(String targetValue) {
+ this.targetValue = targetValue;
+ }
+
+ public String getTargetValue() {
+ return targetValue;
+ }
+
+ public String getUtterance() {
+ return utterance;
+ }
+
+ public Map> getRelations() {
+ return relations;
+ }
+
+ public void setUtterance(String utterance) {
+ this.utterance = utterance;
+ }
+
+ public String getAnswer() {
+ String path = this.getUrl().getPath();
+ path = path.substring(path.lastIndexOf('/') + 1);
+ String answer = path.replace("_", " ");
+ return answer;
+ }
+
+ public String getQuestion() {
+ return getUtterance();
+ }
+
+ public void setRelations(Map> relations) {
+ this.relations = relations;
+ }
+
+ @Override
+ public String toString() {
+ return "Record{" + "utterance='" + utterance + '\'' + ", url=" + url + ", targetValue='" + targetValue + '\''
+ + '}';
+ }
}
diff --git a/src/main/java/DP_entity_linking/geneticAlgorithm/GeneticClass.java b/src/main/java/DP_entity_linking/geneticAlgorithm/GeneticClass.java
index 2b8972f..c857543 100644
--- a/src/main/java/DP_entity_linking/geneticAlgorithm/GeneticClass.java
+++ b/src/main/java/DP_entity_linking/geneticAlgorithm/GeneticClass.java
@@ -34,7 +34,7 @@ public GeneticClass(Random rnd) {
* @throws ParseException
*/
public void doJob() throws IOException, ParseException {
- DataSet dataset = new DataSet();
+ DataSet dataset = new DataSet("C:\\workspace\\webquestions.json");
List records = dataset.loadWebquestions();
records = records.subList(0, 1000);
Search search = new Search();
diff --git a/src/main/java/relation_linking/DBPediaOntologyExtractor.java b/src/main/java/relation_linking/DBPediaOntologyExtractor.java
new file mode 100644
index 0000000..c525cf4
--- /dev/null
+++ b/src/main/java/relation_linking/DBPediaOntologyExtractor.java
@@ -0,0 +1,86 @@
+package relation_linking;
+
+import java.io.*;
+import java.util.*;
+
+public class DBPediaOntologyExtractor {
+
+ private ArrayList listOfRelations = new ArrayList();
+ private Map listOfCleanRelations = new HashMap();
+ private ArrayList lowerCaseListOfRelations = new ArrayList();
+
+ @SuppressWarnings("unchecked")
+ public DBPediaOntologyExtractor(String filePath) throws FileNotFoundException, IOException, ClassNotFoundException {
+
+ System.out.println("Initializing DBPedia Ontology extractor...");
+
+ File dbPediaStore = new File("src/main/resources/data/DBPediaStore");
+
+ if (!dbPediaStore.exists() || dbPediaStore.isDirectory()) {
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ String relation = new String();
+ while ((line = br.readLine()) != null) {
+ if (line.indexOf(") ois.readObject();
+ ois.close();
+ }
+ cleanDBPediaTypes();
+ toLowerCaseTypes();
+ }
+
+ private String getRelation(String line) {
+ String ontologyLink = line.substring(line.indexOf("<"), line.indexOf(">") + 1);
+ return ontologyLink.substring(ontologyLink.lastIndexOf("/") + 1, ontologyLink.indexOf(">"));
+ }
+
+ public ArrayList getDBPediaRelations() {
+ return this.listOfRelations;
+ }
+
+ public ArrayList getLowerDBPediaRelations(){
+ return this.lowerCaseListOfRelations;
+ }
+
+ public String[] splitKey(String key) {
+ String[] r = key.split("(?=\\p{Upper})");
+ return r;
+ }
+
+ private void cleanDBPediaTypes(){
+ for (String relation : listOfRelations){
+ if(!relation.equals(relation.toLowerCase())){
+ String[] r = splitKey(relation);
+ for (int i=0;i getCleanDBPediaTypes(){
+ return this.listOfCleanRelations;
+ }
+
+}
diff --git a/src/main/java/relation_linking/DirectSearchEngine.java b/src/main/java/relation_linking/DirectSearchEngine.java
new file mode 100644
index 0000000..4cc4b88
--- /dev/null
+++ b/src/main/java/relation_linking/DirectSearchEngine.java
@@ -0,0 +1,111 @@
+package relation_linking;
+
+import java.io.*;
+import java.util.*;
+
+import edu.stanford.nlp.ling.HasWord;
+import edu.stanford.nlp.process.DocumentPreprocessor;
+
+public class DirectSearchEngine {
+
+ private DBPediaOntologyExtractor doe;
+ private FBCategoriesExtractor fce;
+ private int matchedInSentence;
+
+ public DirectSearchEngine() {
+
+ System.out.println("Initializing Direct search engine...");
+
+ this.doe = RelationLinkingEngine.getDBPediaOntologyExtractor();
+ this.fce = RelationLinkingEngine.getFBCategoriesExtractor();
+ }
+
+ protected Map getRelations(String sentence)
+ throws FileNotFoundException, UnsupportedEncodingException {
+
+ System.out.println("Getting direct search relations...");
+
+ Map results = new HashMap();
+
+ Reader reader = new StringReader(sentence);
+
+ for (Iterator> iterator = new DocumentPreprocessor(reader).iterator(); iterator.hasNext();) {
+ List word = iterator.next();
+
+ matchedInSentence = 0;
+
+ for (int i = 0; i < word.size(); i++) {
+ String sWord = word.get(i).toString();
+ if (isDBPediaRelation(sWord)) {
+ results.put(sWord, new Double(1.00));
+ matchedInSentence++;
+ }
+ if (isFBCategory(sWord)) {
+ results.put(sWord, new Double(1.00));
+ matchedInSentence++;
+ }
+
+ String matched = isInComposedDBPediaRelations(word.get(i), word);
+ if (matched != null) {
+ results.put(matched, new Double(1.00));
+ matchedInSentence++;
+ }
+
+ matched = isInComposedFBRelations(word.get(i), word);
+ if (matched != null) {
+ results.put(matched, new Double(1.00));
+ matchedInSentence++;
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private boolean isDBPediaRelation(String word) {
+ return doe.getLowerDBPediaRelations().contains(word.toLowerCase());
+ }
+
+ private boolean isFBCategory(String word) {
+ return fce.getCategories().contains(word);
+ }
+
+ private String findComposedRelation(HasWord word, List sentence, boolean Freebase,
+ Map cleanTypes) {
+ boolean matched = false;
+ String key = new String();
+
+ for (Map.Entry entry : cleanTypes.entrySet()) {
+ if (entry.getValue().toLowerCase().equals(word.toString().toLowerCase())) {
+ int wordIndex = sentence.indexOf(word);
+ key = entry.getKey();
+ key = key.substring(0, key.length() - 1);
+ String[] r = Freebase ? fce.splitKey(key) : doe.splitKey(key);
+ if (word.toString().toLowerCase().equals(r[0].toLowerCase())) {
+ matched = true;
+ for (int i = 1; i < r.length; i++) {
+ if (sentence.size() < r.length + wordIndex) {
+ matched = false;
+ break;
+ } else if (!r[i].toLowerCase().equals(sentence.get(wordIndex + i).toString().toLowerCase())) {
+ matched = false;
+ break;
+ }
+ }
+ }
+ if (matched) {
+ return key;
+ }
+ }
+ }
+ return null;
+ }
+
+ private String isInComposedDBPediaRelations(HasWord word, List sentence) {
+ return findComposedRelation(word, sentence, false, doe.getCleanDBPediaTypes());
+ }
+
+ private String isInComposedFBRelations(HasWord word, List sentence) {
+ return findComposedRelation(word, sentence, true, fce.getCleanFBCategories());
+ }
+}
diff --git a/src/main/java/relation_linking/FBCategoriesExtractor.java b/src/main/java/relation_linking/FBCategoriesExtractor.java
new file mode 100644
index 0000000..7536658
--- /dev/null
+++ b/src/main/java/relation_linking/FBCategoriesExtractor.java
@@ -0,0 +1,83 @@
+package relation_linking;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.BytesRef;
+
+public class FBCategoriesExtractor {
+
+ ArrayList fbCategories = new ArrayList();
+ private Map listOfCleanCategories = new HashMap();
+
+ @SuppressWarnings("unchecked")
+ public FBCategoriesExtractor() throws IOException, ClassNotFoundException {
+
+ System.out.println("Initializing FBCategories extractor...");
+
+ File fbStore = new File("src/main/resources/data/FBStore");
+ if (!fbStore.exists() || fbStore.isDirectory()) {
+ Directory directory = new MMapDirectory(new File("/workspace/erd/index_wikipedia"));
+ @SuppressWarnings("deprecation")
+ IndexReader indexReader = IndexReader.open(directory);
+ Fields fields = MultiFields.getFields(indexReader);
+ Terms terms = fields.terms("fb_category");
+ TermsEnum iterator = terms.iterator(null);
+ BytesRef byteRef = null;
+
+ while ((byteRef = iterator.next()) != null) {
+ String term = new String(byteRef.bytes, byteRef.offset, byteRef.length);
+ term = term.substring(term.lastIndexOf(".") + 1, term.length());
+ if (!fbCategories.contains(term) && !term.isEmpty()) {
+ fbCategories.add(term);
+ }
+ }
+
+ ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream("src/main/resources/data/FBStore"));
+ oos.writeObject(fbCategories);
+ oos.flush();
+ oos.close();
+ } else {
+ ObjectInputStream ois = new ObjectInputStream(new FileInputStream("src/main/resources/data/FBStore"));
+ fbCategories = (ArrayList) ois.readObject();
+ ois.close();
+ }
+
+ cleanCategories();
+ }
+
+ public ArrayList getCategories() {
+ return this.fbCategories;
+ }
+
+ public String[] splitKey(String key) {
+ String[] r = key.split("_");
+ return r;
+ }
+
+ private void cleanCategories(){
+ for (String category : fbCategories){
+ if (category.contains("_")){
+ String[] r = splitKey(category);
+ for (int i=0;i getCleanFBCategories(){
+ return this.listOfCleanCategories;
+ }
+
+}
diff --git a/src/main/java/relation_linking/GloVeEngine.java b/src/main/java/relation_linking/GloVeEngine.java
new file mode 100644
index 0000000..c71691c
--- /dev/null
+++ b/src/main/java/relation_linking/GloVeEngine.java
@@ -0,0 +1,332 @@
+package relation_linking;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.*;
+
+import edu.stanford.nlp.ling.HasWord;
+import edu.stanford.nlp.process.DocumentPreprocessor;
+import relation_linking.RelationLinkingEngine.METHOD_DETECTION_TYPE;
+import word2vec.*;
+
+public class GloVeEngine {
+
+ private DBPediaOntologyExtractor doe = null;
+ private FBCategoriesExtractor fce = null;
+ private LexicalParsingEngine lpe = null;
+ private OpenIEEngine openIE = null;
+ private QueryStrippingEngine qse = null;
+
+ private boolean allOverSimilarity;
+ private double similarity;
+
+ private GloVeSpace model = null;
+
+ private static GloVeEngine instance = null;
+
+ public static GloVeEngine getInstance() {
+ if (instance != null)
+ return instance;
+ else
+ return new GloVeEngine();
+ }
+
+ public void init(String modelPath, double similarity, boolean allOverSimilarity) {
+
+ System.out.println("Initializing Glove search engine...");
+
+ if (model == null) {
+ model = new GloVeSpace();
+ model = GloVeSpace.load(modelPath, true, false);
+ }
+
+ if (this.doe == null)
+ this.doe = RelationLinkingEngine.getDBPediaOntologyExtractor();
+ if (this.fce == null)
+ this.fce = RelationLinkingEngine.getFBCategoriesExtractor();
+ this.similarity = similarity;
+ this.allOverSimilarity = allOverSimilarity;
+ }
+
+ public void init(String modelPath, double similarity, LexicalParsingEngine lpe, boolean allOverSimilarity) {
+ System.out.println("Initializing Glove search engine with lexical parser...");
+
+ if (model == null) {
+ model = new GloVeSpace();
+ model = GloVeSpace.load(modelPath, true, false);
+ }
+
+ if (this.lpe == null)
+ this.lpe = lpe;
+ if (this.doe == null)
+ this.doe = RelationLinkingEngine.getDBPediaOntologyExtractor();
+ if (this.fce == null)
+ this.fce = RelationLinkingEngine.getFBCategoriesExtractor();
+ this.similarity = similarity;
+ this.allOverSimilarity = allOverSimilarity;
+ }
+
+ public void init(String modelPath, double similarity, OpenIEEngine openIE, boolean allOverSimilarity) {
+ System.out.println("Initializing Glove search engine with OpenIE...");
+
+ if (model == null) {
+ model = new GloVeSpace();
+ model = GloVeSpace.load(modelPath, true, false);
+ }
+ if (this.openIE == null)
+ this.openIE = openIE;
+ if (this.doe == null)
+ this.doe = RelationLinkingEngine.getDBPediaOntologyExtractor();
+ if (this.fce == null)
+ this.fce = RelationLinkingEngine.getFBCategoriesExtractor();
+ this.similarity = similarity;
+ this.allOverSimilarity = allOverSimilarity;
+ }
+
+ public void init(String modelPath, double similarity, QueryStrippingEngine qse, boolean allOverSimilarity) {
+ System.out.println("Initializing Glove search engine with Query stripping...");
+
+ if (model == null) {
+ model = new GloVeSpace();
+ model = GloVeSpace.load(modelPath, true, false);
+ }
+ if (this.qse == null)
+ this.qse = qse;
+ if (this.doe == null)
+ this.doe = RelationLinkingEngine.getDBPediaOntologyExtractor();
+ if (this.fce == null)
+ this.fce = RelationLinkingEngine.getFBCategoriesExtractor();
+ this.similarity = similarity;
+ this.allOverSimilarity = allOverSimilarity;
+ }
+
+ private Map getComposedRelations(ArrayList sentenceParts) {
+ Map results = new HashMap();
+
+ for (String sentencePart : sentenceParts) {
+
+ Map relations = isDBPediaRelation(sentencePart);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isFBCategory(sentencePart);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isInComposedDBPediaRelations(sentencePart);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isInComposedFBRelations(sentencePart);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+ }
+
+ return results;
+ }
+
+ private Map getOpenIERelations(String sentence) {
+ return getComposedRelations(openIE.getRelations(sentence));
+ }
+
+ private Map getLexicalizedRelations(String sentence) {
+ return getComposedRelations(lpe.getPairsFromSentence(sentence));
+ }
+
+ private Map getStrippedRelations(String sentence) {
+ StringBuilder sb = new StringBuilder();
+ ArrayList words = qse.getRelations(sentence);
+
+ for (String word : words) {
+ sb.append(word);
+ sb.append(" ");
+ }
+
+ words.clear();
+ words.add(sb.toString());
+
+ return getComposedRelations(words);
+ }
+
+ public Map getRelations(String sentence, METHOD_DETECTION_TYPE methodType) {
+ System.out.println("Getting glove relations...");
+
+ Map results = new HashMap();
+
+ switch (methodType) {
+ case ALL: {
+ Reader reader = new StringReader(sentence);
+
+ for (Iterator> iterator = new DocumentPreprocessor(reader).iterator(); iterator.hasNext();) {
+ List word = iterator.next();
+
+ for (int i = 0; i < word.size(); i++) {
+ String sWord = word.get(i).toString();
+
+ Map relations = isDBPediaRelation(sWord);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isFBCategory(sWord);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isInComposedDBPediaRelations(sWord);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isInComposedFBRelations(sWord);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+ }
+ }
+ }
+ break;
+ case OPENIE:
+ results.putAll(getOpenIERelations(sentence));
+ break;
+ case LEXICALPARSER:
+ results.putAll(getLexicalizedRelations(sentence));
+ break;
+ case QUERYSTRIPPING:
+ results.putAll(getStrippedRelations(sentence));
+ break;
+ default:
+ break;
+ }
+
+ return results;
+ }
+
+ private double getSentencesSimilarity(String sentence, String composedRelation) {
+ double similarity = 0;
+ if (canBeSentenceVectorized(sentence) && canBeSentenceVectorized(composedRelation)) {
+ similarity = model.cosineSimilarity(model.sentenceVector(sentence), model.sentenceVector(composedRelation));
+ }
+ return similarity;
+ }
+
+ private double getSimilarity(String sentence, String word) {
+ double similarity = 0;
+ if (isWordInModel(word) && canBeSentenceVectorized(sentence))
+ similarity = model.cosineSimilarity(model.sentenceVector(sentence), model.vector(word.toLowerCase()));
+ return similarity;
+ }
+
+ private double getWordsSimilarity(String word1, String word2) {
+ double similarity = model.cosineSimilarity(word1.toLowerCase(), word2.toLowerCase());
+ return similarity;
+ }
+
+ private boolean isWordInModel(String word) {
+ return model.contains(word.toLowerCase());
+ }
+
+ private boolean canBeSentenceVectorized(String sentence) {
+ return model.sentenceVector(sentence) == null ? false : true;
+ }
+
+ private String makeSentenceFromSequence(String[] r) {
+ StringBuilder sentence = new StringBuilder();
+ for (int i = 0; i < r.length; i++) {
+ sentence.append(r[i]);
+ sentence.append(" ");
+ }
+ return sentence.toString();
+ }
+
+ private Map findComposedRelation(String word, boolean Freebase, Map cleanTypes) {
+ double maxSimilarity = 0;
+ String maxRelation = null;
+ String key;
+ Map foundRelations = new HashMap();
+
+ for (Map.Entry entry : cleanTypes.entrySet()) {
+ key = entry.getKey();
+ key = key.substring(0, key.length() - 1);
+ String[] r = Freebase ? fce.splitKey(key) : doe.splitKey(key);
+ String sentence = makeSentenceFromSequence(r);
+ double tSim = 0;
+ if (lpe != null || qse != null || openIE != null) {
+ tSim = getSentencesSimilarity(sentence, word);
+ } else {
+ tSim = getSimilarity(sentence, word);
+ }
+ if (tSim > similarity) {
+ if (Double.isFinite(tSim)) {
+ if (allOverSimilarity) {
+ foundRelations.put(key, tSim);
+ } else if (tSim > maxSimilarity) {
+ maxRelation = key;
+ maxSimilarity = tSim;
+ }
+ }
+ }
+
+ }
+
+ if (!allOverSimilarity)
+ foundRelations.put(maxRelation, maxSimilarity);
+
+ return foundRelations;
+ }
+
+ private Map isInComposedDBPediaRelations(String word) {
+ return findComposedRelation(word, false, doe.getCleanDBPediaTypes());
+ }
+
+ private Map isInComposedFBRelations(String word) {
+ return findComposedRelation(word, true, fce.getCleanFBCategories());
+ }
+
+ private Map findRelation(String word, ArrayList relations) {
+ double maxSimilarity = 0;
+ String maxRelation = null;
+ Map foundResults = new HashMap();
+
+ for (String relation : relations) {
+
+ double tSim = 0;
+ if (lpe != null || qse != null || openIE != null) {
+ tSim = getSimilarity(word, relation);
+ } else {
+ if (isWordInModel(word) && isWordInModel(relation)) {
+ tSim = getWordsSimilarity(word, relation);
+ }
+ }
+
+ if (tSim > similarity) {
+ if (Double.isFinite(tSim)) {
+
+ if (allOverSimilarity) {
+ foundResults.put(relation, tSim);
+ } else if (tSim > maxSimilarity) {
+ maxRelation = relation;
+ maxSimilarity = tSim;
+ }
+ }
+ }
+ }
+
+ if (!allOverSimilarity)
+ foundResults.put(maxRelation, maxSimilarity);
+
+ return foundResults;
+ }
+
+ private Map isDBPediaRelation(String word) {
+ return findRelation(word, doe.getLowerDBPediaRelations());
+ }
+
+ private Map isFBCategory(String word) {
+ return findRelation(word, fce.getCategories());
+ }
+}
diff --git a/src/main/java/relation_linking/LexicalParsingEngine.java b/src/main/java/relation_linking/LexicalParsingEngine.java
new file mode 100644
index 0000000..5dce947
--- /dev/null
+++ b/src/main/java/relation_linking/LexicalParsingEngine.java
@@ -0,0 +1,87 @@
+package relation_linking;
+
+import edu.stanford.nlp.process.*;
+
+import java.io.*;
+import java.util.*;
+
+import edu.stanford.nlp.ling.*;
+import edu.stanford.nlp.trees.*;
+import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
+
+public class LexicalParsingEngine {
+
+ LexicalizedParser lp;
+
+ public LexicalParsingEngine(String parserModel) throws FileNotFoundException, UnsupportedEncodingException {
+
+ System.out.println("Initializing Lexical Parser...");
+ lp = LexicalizedParser.loadModel(parserModel);
+ }
+
+ private Collection parseSentenceTDL(String text) {
+ System.out.println("Parsing sentence...");
+
+ Collection tdl = null;
+ TreebankLanguagePack tlp = lp.treebankLanguagePack();
+ GrammaticalStructureFactory gsf = null;
+ if (tlp.supportsGrammaticalStructures()) {
+ gsf = tlp.grammaticalStructureFactory();
+ }
+
+ Reader reader = new StringReader(text);
+
+ for (List sentence : new DocumentPreprocessor(reader)) {
+ Tree parse = lp.apply(sentence);
+ if (gsf != null) {
+ GrammaticalStructure gs = gsf.newGrammaticalStructure(parse);
+ tdl = gs.allTypedDependencies();
+ }
+ }
+ return tdl;
+ }
+
+ private ArrayList parseSentenceTD(String text) {
+ System.out.println("Parsing sentence...");
+
+ ArrayList tw = new ArrayList();
+
+ Reader reader = new StringReader(text);
+
+ for (List sentence : new DocumentPreprocessor(reader)) {
+
+ Tree parse = lp.apply(sentence);
+
+ tw = parse.taggedYield();
+ }
+ return tw;
+ }
+
+ public ArrayList getPairsFromSentence(String sentence) {
+ Collection tdl = parseSentenceTDL(sentence);
+ ArrayList pairs = new ArrayList();
+
+ for (TypedDependency td : tdl) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(td.gov().originalText());
+ sb.append(" ");
+ sb.append(td.dep().originalText());
+ pairs.add(sb.toString());
+ }
+
+ return pairs;
+ }
+
+ public ArrayList getNounsFromSentence(String sentence) {
+ ArrayList tw = parseSentenceTD(sentence);
+ ArrayList nouns = new ArrayList();
+
+ for (TaggedWord t : tw) {
+ if (t.tag().startsWith("N")) {
+ nouns.add(t.value());
+ }
+ }
+
+ return nouns;
+ }
+}
diff --git a/src/main/java/relation_linking/OpenIEEngine.java b/src/main/java/relation_linking/OpenIEEngine.java
new file mode 100644
index 0000000..0456917
--- /dev/null
+++ b/src/main/java/relation_linking/OpenIEEngine.java
@@ -0,0 +1,37 @@
+package relation_linking;
+
+import java.util.*;
+
+import scala.collection.*;
+import edu.knowitall.openie.*;
+import edu.knowitall.tool.parse.ClearParser;
+import edu.knowitall.tool.postag.ClearPostagger;
+import edu.knowitall.tool.srl.ClearSrl;
+import edu.knowitall.tool.tokenize.ClearTokenizer;
+
+public class OpenIEEngine {
+
+ private OpenIE openIE;
+
+ public OpenIEEngine(){
+ System.out.println("Starting openIE Engine...");
+ openIE = new OpenIE(new ClearParser(new ClearPostagger(new ClearTokenizer())), new ClearSrl(), true, true);
+ }
+
+ public ArrayList getRelations(String sentence){
+
+ System.out.println("Getting openIE relations...");
+
+ ArrayList results = new ArrayList();
+
+ Seq extractions = openIE.extract(sentence);
+ List list_extractions = JavaConversions.seqAsJavaList(extractions);
+
+ for(Instance instance : list_extractions) {
+ results.add(instance.extr().rel().text());
+ }
+
+ return results;
+ }
+
+}
diff --git a/src/main/java/relation_linking/QueryStrippingEngine.java b/src/main/java/relation_linking/QueryStrippingEngine.java
new file mode 100644
index 0000000..807fe75
--- /dev/null
+++ b/src/main/java/relation_linking/QueryStrippingEngine.java
@@ -0,0 +1,91 @@
+package relation_linking;
+
+import java.io.*;
+import java.util.*;
+
+import util.StopWords;
+
+public class QueryStrippingEngine {
+
+ private String filePath;
+ private Map entities = null;
+
+ public QueryStrippingEngine(String filePath) throws FileNotFoundException, IOException, ClassNotFoundException {
+ this.filePath = filePath;
+ if (entities == null)
+ getEntities();
+ }
+
+ @SuppressWarnings("unchecked")
+ private void getEntities() throws FileNotFoundException, IOException, ClassNotFoundException {
+ File entitiesFile = new File("src/main/resources/data/entitiesStore");
+ entities = new HashMap();
+
+ if (entitiesFile.exists()) {
+ ObjectInputStream ois = new ObjectInputStream(new FileInputStream("src/main/resources/data/entitiesStore"));
+ entities = (Map) ois.readObject();
+ ois.close();
+ } else {
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.indexOf("QUESTION:") != -1) {
+ String question = line.substring(line.indexOf(":") + 2);
+ line = br.readLine();
+ String entity = line.substring(line.indexOf(":") + 2);
+ entity = entity.replaceAll("_", " ");
+ System.out.println(question + ":" + entity);
+ entities.put(question, entity);
+ }
+ }
+ }
+ ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream("src/main/resources/data/entitiesStore"));
+ oos.writeObject(entities);
+ oos.flush();
+ oos.close();
+ }
+ }
+
+ public ArrayList getRelations(String sentence) {
+ String entity = entities.get(sentence);
+
+ String[] words = sentence.split("\\s+");
+ for (int i = 0; i < words.length; i++) {
+ words[i] = words[i].replaceAll("[^\\w]", "");
+ }
+
+ ArrayList listOfWords = new ArrayList();
+ for (String word : words) {
+ if (!word.isEmpty())
+ listOfWords.add(word);
+ }
+
+ words = entity.split("\\s+");
+ for (String word : words) {
+ listOfWords.remove(word);
+ }
+
+ StopWords stopWords = new StopWords(
+ new File("src/main/resources/data/stop-words_long.txt"));
+ Set stopW = stopWords.getStopWords();
+
+ ArrayList copy = new ArrayList(listOfWords);
+ for (String word : copy) {
+ if (stopW.contains(word)) {
+ listOfWords.remove(word);
+ }
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (String word : listOfWords) {
+ sb.append(word);
+ sb.append(" ");
+ }
+
+ listOfWords.clear();
+ listOfWords.add(sb.toString());
+
+ return listOfWords;
+ }
+
+}
diff --git a/src/main/java/relation_linking/RelationLinkingEngine.java b/src/main/java/relation_linking/RelationLinkingEngine.java
new file mode 100644
index 0000000..977ed85
--- /dev/null
+++ b/src/main/java/relation_linking/RelationLinkingEngine.java
@@ -0,0 +1,599 @@
+package relation_linking;
+
+import java.io.*;
+import java.util.*;
+import java.util.Map.Entry;
+
+import com.ibm.icu.text.DecimalFormat;
+
+import DP_entity_linking.dataset.*;
+import net.didion.jwnl.JWNLException;
+
+public class RelationLinkingEngine {
+
+ public enum METHOD_MAPPING_TYPE {
+ DIRECT, GLOVE, WORDNET;
+ }
+
+ public enum METHOD_DETECTION_TYPE {
+ ALL, OPENIE, LEXICALPARSER, QUERYSTRIPPING;
+ }
+
+ private boolean directCheck = true;
+ private boolean checkGlove = true;
+ private boolean checkWordNet = true;
+
+ private boolean withOpenIE = true;
+ private boolean withLexicalParser = true;
+ private boolean withQueryStripping = true;
+ private boolean withEveryWord = true;
+ private boolean allOverSimilarity = true;
+
+ private double similarity = 0.1;
+ private int firstResults = 3;
+
+ private String datasetPath = "src/main/resources/data/webquestionsRelation.json";
+ private String dbPediaOntologyPath = "src/main/resources/data/dbpedia_2015-04.nt";
+ private String gloveModelPath = "/Users/fjuras/OneDriveBusiness/DPResources/glove.6B/glove.6B.300d.txt";
+ private String lexicalParserModel = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
+ private String JWNLPropertiesPath = "src/main/resources/data/file_properties.xml";
+ private String entitySearchResultsFilePath = "src/main/resources/data/resultsWebquestions.txt";
+
+ private String csvOutputPath = "/Users/fjuras/OneDriveBusiness/DPResources/Relations.csv";
+ private String trainOutputPath = "/Users/fjuras/OneDriveBusiness/DPResources/trainSet";
+ private String testOutputPath = "/Users/fjuras/OneDriveBusiness/DPResources/testSet";
+
+ private String outputUtteranceKey = "utterance";
+ private String outputRelationKey = "relation";
+ private String outputDetectedKey = "detected";
+ private String outputFoundRelationsKey = "number of found";
+ private String outputDetectedRelationsKey = "number of detected";
+ private String outputFromDetectedKey = "detected from";
+ private String outputDetectedForKey = "detected for";
+ private String outputFromDetectedAllKey = "detected from complete";
+ private String outputSeparator = ";";
+ private String outputDirectKey = "Direct";
+ private String outputGloveLexicalKey = "GloVe_Lexical";
+ private String outputGloveOpenIEKey = "GloVe_OpenIE";
+ private String outputGloveStrippingKey = "GloVe_QuerryStripping";
+ private String outputGloveAllKey = "GloVe_All";
+ private String outputWordNetLexicalKey = "WordNet_Lexical";
+ private String outputWordNetOpenIEKey = "WordNet_OpenIE";
+ private String outputWordNetStrippingKey = "WordNet_QuerryStripping";
+ private String outputWordNetAllKey = "WordNet_All";
+ private String outputTrueValue = "1";
+ private String outputFalseValue = "0";
+ private String outputNotFoundValue = "-1";
+ private String outputNewLine = "\n";
+
+ private String outputTrainSeparator = " ";
+ private String outputCategory = "|a";
+ private String outputTrainValueSeparator = ":";
+
+ private static DBPediaOntologyExtractor doe = null;
+ private static FBCategoriesExtractor fce = null;
+
+ private FileWriter csvOutput;
+ private FileWriter trainOutput;
+ private FileWriter testOutput;
+
+ private DirectSearchEngine dse = null;
+ private GloVeEngine glove = null;
+ private WordNetEngine wordnet = null;
+
+ boolean testStarted = false;
+ private double precision = 0;
+ private double recall = 0;
+ private int TP = 0;
+ private int tTP = 0;
+ private int FP = 0;
+ private int tFP = 0;
+ private int FN = 0;
+ private int tFN = 0;
+
+ public RelationLinkingEngine() {
+ }
+
+ public static void main(String[] args)
+ throws ClassNotFoundException, IOException, JWNLException, InterruptedException {
+
+ RelationLinkingEngine rle = new RelationLinkingEngine();
+ rle.runDetection();
+ rle.calculateXGBoostStatistics();
+ }
+
+ private void runDetection() throws IOException, ClassNotFoundException, JWNLException, InterruptedException {
+ System.out.println("Reading dataset...");
+ DataSet dataset = new DataSet(datasetPath);
+ List records = dataset.loadWebquestions();
+
+ csvOutput = new FileWriter(csvOutputPath);
+ trainOutput = new FileWriter(trainOutputPath);
+ testOutput = new FileWriter(testOutputPath);
+ printCSVRow(outputUtteranceKey, outputRelationKey, outputDirectKey, outputGloveLexicalKey, outputGloveOpenIEKey,
+ outputGloveStrippingKey, outputGloveAllKey, outputWordNetLexicalKey, outputWordNetOpenIEKey,
+ outputWordNetStrippingKey, outputWordNetAllKey, outputDetectedKey, outputDetectedRelationsKey,
+ outputFoundRelationsKey, outputFromDetectedKey, outputDetectedForKey, outputFromDetectedAllKey);
+
+ doe = new DBPediaOntologyExtractor(dbPediaOntologyPath);
+ fce = new FBCategoriesExtractor();
+
+ LexicalParsingEngine lpe = null;
+ OpenIEEngine openIE = null;
+ QueryStrippingEngine qse = null;
+ if (withLexicalParser)
+ lpe = new LexicalParsingEngine(lexicalParserModel);
+ if (withOpenIE)
+ openIE = new OpenIEEngine();
+ if (withQueryStripping)
+ qse = new QueryStrippingEngine(entitySearchResultsFilePath);
+
+ if (directCheck)
+ dse = new DirectSearchEngine();
+
+ if (checkGlove) {
+ glove = GloVeEngine.getInstance();
+ if (withLexicalParser) {
+ glove.init(gloveModelPath, similarity, lpe, allOverSimilarity);
+ }
+ if (withOpenIE) {
+ glove.init(gloveModelPath, similarity, openIE, allOverSimilarity);
+ }
+ if (withQueryStripping) {
+ glove.init(gloveModelPath, similarity, qse, allOverSimilarity);
+ }
+ if (withEveryWord) {
+ glove.init(gloveModelPath, similarity, allOverSimilarity);
+ }
+ }
+
+ if (checkWordNet) {
+ wordnet = WordNetEngine.getInstance();
+ if (withLexicalParser) {
+ wordnet.init(JWNLPropertiesPath, lpe, similarity);
+ }
+ if (withOpenIE) {
+ wordnet.init(JWNLPropertiesPath, openIE, similarity);
+ }
+ if (withQueryStripping) {
+ wordnet.init(JWNLPropertiesPath, qse, similarity);
+ }
+
+ if (withEveryWord) {
+ wordnet.init(JWNLPropertiesPath, similarity);
+ }
+ }
+
+ int r = 0;
+ for (Record record : records) {
+ System.out.println(r + ":Processing utterance: " + record.getUtterance());
+
+ Map results = new HashMap();
+ if (directCheck)
+ results.putAll(addFoundRelations(dse.getRelations(record.getUtterance()), results,
+ METHOD_MAPPING_TYPE.DIRECT, null, record));
+
+ if (checkGlove) {
+ if (withLexicalParser) {
+ results.putAll(addFoundRelations(
+ glove.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.LEXICALPARSER), results,
+ METHOD_MAPPING_TYPE.GLOVE, METHOD_DETECTION_TYPE.LEXICALPARSER, record));
+ }
+ if (withOpenIE) {
+ results.putAll(
+ addFoundRelations(glove.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.OPENIE),
+ results, METHOD_MAPPING_TYPE.GLOVE, METHOD_DETECTION_TYPE.OPENIE, record));
+ }
+ if (withQueryStripping) {
+ results.putAll(addFoundRelations(
+ glove.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.QUERYSTRIPPING), results,
+ METHOD_MAPPING_TYPE.GLOVE, METHOD_DETECTION_TYPE.QUERYSTRIPPING, record));
+ }
+
+ if (withEveryWord) {
+ results.putAll(
+ addFoundRelations(glove.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.ALL),
+ results, METHOD_MAPPING_TYPE.GLOVE, METHOD_DETECTION_TYPE.ALL, record));
+ }
+ }
+ if (checkWordNet) {
+ if (withLexicalParser) {
+ results.putAll(addFoundRelations(
+ wordnet.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.LEXICALPARSER), results,
+ METHOD_MAPPING_TYPE.WORDNET, METHOD_DETECTION_TYPE.LEXICALPARSER, record));
+ }
+ if (withOpenIE) {
+ results.putAll(
+ addFoundRelations(wordnet.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.OPENIE),
+ results, METHOD_MAPPING_TYPE.WORDNET, METHOD_DETECTION_TYPE.OPENIE, record));
+ }
+ if (withQueryStripping) {
+ results.putAll(addFoundRelations(
+ wordnet.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.QUERYSTRIPPING), results,
+ METHOD_MAPPING_TYPE.WORDNET, METHOD_DETECTION_TYPE.QUERYSTRIPPING, record));
+ }
+
+ if (withEveryWord) {
+ results.putAll(
+ addFoundRelations(wordnet.getRelations(record.getUtterance(), METHOD_DETECTION_TYPE.ALL),
+ results, METHOD_MAPPING_TYPE.WORDNET, METHOD_DETECTION_TYPE.ALL, record));
+ }
+ }
+
+ if (r < 3 * records.size() / 4) {
+ printFoundRelations(results, record.getUtterance(), trainOutput);
+ } else {
+ if (!testStarted) {
+ tTP = 0;
+ tFP = 0;
+ tFN = 0;
+ testStarted = true;
+ }
+ printFoundRelations(results, record.getUtterance(), testOutput);
+ }
+
+ r++;
+ }
+
+ System.out.println();
+ precision = ((double) TP / ((double) TP + (double) FP));
+ recall = ((double) TP / ((double) TP + (double) FN));
+ System.out.println("Precision = " + precision);
+ System.out.println("Recall = " + recall);
+ System.out.println("F1 = " + (2 * ((precision * recall) / (precision + recall))));
+ System.out.println();
+ precision = ((double) tTP / ((double) tTP + (double) tFP));
+ recall = ((double) tTP / ((double) tTP + (double) tFN));
+ System.out.println("Test Precision = " + precision);
+ System.out.println("Test Recall = " + recall);
+ System.out.println("Test F1 = " + (2 * ((precision * recall) / (precision + recall))));
+
+ csvOutput.flush();
+ csvOutput.close();
+ trainOutput.flush();
+ trainOutput.close();
+ testOutput.flush();
+ testOutput.close();
+
+ }
+
+ private void calculateXGBoostStatistics() throws IOException {
+ FileReader test = new FileReader("/Users/fjuras/Downloads/xgboost-0.47/testedTrain");
+ FileReader pred = new FileReader("/Users/fjuras/Downloads/xgboost-0.47/predicted.txt");
+
+ @SuppressWarnings("resource")
+ BufferedReader brT = new BufferedReader(test);
+ @SuppressWarnings("resource")
+ BufferedReader brP = new BufferedReader(pred);
+ String lineT;
+ String lineP;
+ while ((lineT = brT.readLine()) != null) {
+ lineP = brP.readLine();
+ if (lineT.startsWith("1")) {
+ if (lineP.startsWith("0")) {
+ FN++;
+ } else {
+ TP++;
+ }
+ } else {
+ if (lineP.startsWith("1")) {
+ FP++;
+ }
+ }
+ }
+ System.out.println();
+ precision = ((double) TP / ((double) TP + (double) FP));
+ recall = ((double) TP / ((double) TP + (double) FN));
+ System.out.println("Precision = " + precision);
+ System.out.println("Recall = " + recall);
+ System.out.println("F1 = " + (2 * ((precision * recall) / (precision + recall))));
+ System.out.println();
+ }
+
+ private void printCSVRow(String utteranceValue, String relationValue, String directValue, String gloveLexicalValue,
+ String gloveOpenIEValue, String gloveStrippingValue, String gloveAllValue, String wordNetLexicalValue,
+ String wordNetOpenIEValue, String wordNetStrippingValue, String wordNetAllValue, String detectedValue,
+ String foundValue, String detectedNumberValue, String outputFromDetectedValue,
+ String outputDetectedForValue, String outputFromDetectedAllValue) throws IOException {
+
+ csvOutput.append(utteranceValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(relationValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(directValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(gloveLexicalValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(gloveOpenIEValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(gloveStrippingValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(gloveAllValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(wordNetLexicalValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(wordNetOpenIEValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(wordNetStrippingValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(wordNetAllValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(detectedValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(foundValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(detectedNumberValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(outputFromDetectedAllValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(outputFromDetectedValue);
+ csvOutput.append(outputSeparator);
+ csvOutput.append(outputDetectedForValue);
+ csvOutput.append(outputNewLine);
+ }
+
+ private void printTrainRow(boolean found, String relationName, Double direct, Double gloveLexical,
+ Double gloveOpenIE, Double gloveStripping, Double gloveAll, Double wordnetLexical, Double wordnetOpenIE,
+ Double wordnetStripping, Double wordnetAll, FileWriter output) throws IOException {
+
+ DecimalFormat formatter = new DecimalFormat("#0.00");
+
+ if (found)
+ output.append(outputTrueValue);
+ else
+ output.append(outputNotFoundValue);
+ output.append(outputTrainSeparator);
+ output.append(outputCategory);
+ output.append(outputTrainSeparator);
+ output.append(relationName);
+ output.append(outputTrainSeparator);
+ output.append(outputDirectKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(direct));
+ output.append(outputTrainSeparator);
+ output.append(outputGloveLexicalKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(gloveLexical));
+ output.append(outputTrainSeparator);
+ output.append(outputGloveOpenIEKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(gloveOpenIE));
+ output.append(outputTrainSeparator);
+ output.append(outputGloveStrippingKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(gloveStripping));
+ output.append(outputTrainSeparator);
+ output.append(outputGloveAllKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(gloveAll));
+ output.append(outputTrainSeparator);
+ output.append(outputWordNetLexicalKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(wordnetLexical));
+ output.append(outputTrainSeparator);
+ output.append(outputWordNetOpenIEKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(wordnetOpenIE));
+ output.append(outputTrainSeparator);
+ output.append(outputWordNetStrippingKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(wordnetStripping));
+ output.append(outputTrainSeparator);
+ output.append(outputWordNetAllKey);
+ output.append(outputTrainValueSeparator);
+ output.append(formatter.format(wordnetAll));
+ output.append(outputNewLine);
+ }
+
+ private int getNumberOfDetected(Map results) {
+ int detected = 0;
+ for (Entry result : results.entrySet()) {
+ if (result.getValue().isDetected()) {
+ detected++;
+ }
+ }
+ return detected;
+ }
+
+ private String valueForBool(boolean bool) {
+ return bool ? outputTrueValue : outputFalseValue;
+ }
+
+ private void printFoundRelations(Map results, String utterance, FileWriter output)
+ throws IOException {
+ System.out.println("Printing relations...");
+
+ int numberOfDetected = getNumberOfDetected(results);
+ int numberOfFound = results.size();
+
+ if (results.isEmpty()) {
+ printCSVRow(utterance, outputNotFoundValue, outputNotFoundValue, outputNotFoundValue, outputNotFoundValue,
+ outputNotFoundValue, outputNotFoundValue, outputNotFoundValue, outputNotFoundValue,
+ outputNotFoundValue, outputNotFoundValue, outputNotFoundValue, String.valueOf(numberOfDetected),
+ String.valueOf(numberOfFound), outputNotFoundValue, outputNotFoundValue, outputNotFoundValue);
+ } else {
+ for (Entry relation : results.entrySet()) {
+ Result result = relation.getValue();
+ DecimalFormat formatter = new DecimalFormat("#0.00");
+ printCSVRow(utterance, result.getName(), formatter.format(result.getDirectSearch()),
+ formatter.format(result.getGloveLexicalParserSimilarity()),
+ formatter.format(result.getGloveOpenIESimilarity()),
+ formatter.format(result.getGloveStrippingSimilarity()),
+ formatter.format(result.getGloveAllSimilarity()),
+ formatter.format(result.getWordnetLexicalParserSimilarity()),
+ formatter.format(result.getWordnetOpenIESimilarity()),
+ formatter.format(result.getWordnetStrippingSimilarity()),
+ formatter.format(result.getWordnetAllSimilarity()), valueForBool(result.isDetected()),
+ String.valueOf(numberOfDetected), String.valueOf(numberOfFound),
+ result.getNumberOfRelations().toString(), result.getNumberOfAllRelations().toString(),
+ result.getDetectedFor());
+ printTrainRow(result.isDetected(), result.getName(), result.getDirectSearch(),
+ result.getGloveLexicalParserSimilarity(), result.getGloveOpenIESimilarity(),
+ result.getGloveStrippingSimilarity(), result.getGloveAllSimilarity(),
+ result.getWordnetLexicalParserSimilarity(), result.getWordnetOpenIESimilarity(),
+ result.getWordnetStrippingSimilarity(), result.getWordnetAllSimilarity(), output);
+ }
+ }
+ }
+
+ private boolean isRelationDetected(String relation, Record record) {
+ Map> relations = record.getRelations();
+
+ for (Entry> rel : relations.entrySet()) {
+ for (String r : rel.getValue())
+ if (r.toLowerCase().compareTo(relation.toLowerCase()) == 0) {
+ return true;
+ }
+ }
+ FP++;
+ tFP++;
+ return false;
+ }
+
+ private Integer getNumberOfRelations(Record record, boolean all) {
+ Map> relations = record.getRelations();
+ if (all)
+ return new Integer(relations.size());
+
+ Integer number = new Integer(0);
+ for (Entry> entry : relations.entrySet()) {
+ number += entry.getValue().size();
+ }
+ return number;
+ }
+
+ private String getDetectedFor(Record record, String relation) {
+ Map> relations = record.getRelations();
+
+ for (Entry> entry : relations.entrySet()) {
+ for (String rel : entry.getValue())
+ if (rel.toLowerCase().equals(relation.toLowerCase()))
+ return entry.getKey();
+ }
+ return null;
+ }
+
+ private ArrayList getRelationsToDetect(Record record) {
+ ArrayList rel = new ArrayList();
+ for (Entry> r : record.getRelations().entrySet()) {
+ rel.add(r.getKey().toString());
+ }
+ return rel;
+ }
+
+ private int getUndetected(ArrayList rel) {
+ return rel.size();
+ }
+
+ @SuppressWarnings("hiding")
+ > List> entriesSortedByValues(
+ Map map) {
+
+ List> sortedEntries = new ArrayList>(map.entrySet());
+
+ Collections.sort(sortedEntries, new Comparator>() {
+ @Override
+ public int compare(Entry e1, Entry e2) {
+ return e2.getValue().compareTo(e1.getValue());
+ }
+ });
+
+ return sortedEntries;
+ }
+
+ private Map addFoundRelations(Map relations, Map results,
+ METHOD_MAPPING_TYPE mappingType, METHOD_DETECTION_TYPE detectionType, Record record) {
+
+ List> sortedRelations = entriesSortedByValues(relations);
+
+ ArrayList relationsToDetect = getRelationsToDetect(record);
+
+ Result result;
+ int r = 0;
+
+ for (Entry relation : sortedRelations) {
+ if (r == firstResults || relationsToDetect.isEmpty())
+ break;
+ if (relation.getKey() == null)
+ continue;
+ if (!results.isEmpty() && results.containsKey(relation.getKey().toLowerCase())) {
+ result = results.get(relation.getKey().toLowerCase());
+ switch (mappingType) {
+ case DIRECT:
+ result.setDirectSearch(relation.getValue());
+ break;
+ case GLOVE: {
+ switch (detectionType) {
+ case ALL:
+ result.setGloveAllSimilarity(relation.getValue());
+ break;
+ case OPENIE:
+ result.setGloveOpenIESimilarity(relation.getValue());
+ break;
+ case LEXICALPARSER:
+ result.setGloveLexicalParserSimilarity(relation.getValue());
+ break;
+ case QUERYSTRIPPING:
+ result.setGloveStrippingSimilarity(relation.getValue());
+ break;
+ default:
+ break;
+ }
+ }
+ break;
+ case WORDNET:
+ switch (detectionType) {
+ case ALL:
+ result.setWordnetAllSimilarity(relation.getValue());
+ break;
+ case OPENIE:
+ result.setWordnetOpenIESimilarity(relation.getValue());
+ break;
+ case LEXICALPARSER:
+ result.setWordnetLexicalParserSimilarity(relation.getValue());
+ break;
+ case QUERYSTRIPPING:
+ result.setWordnetStrippingSimilarity(relation.getValue());
+ break;
+ default:
+ break;
+ }
+ break;
+ }
+ } else {
+ result = new Result(relation.getKey(), mappingType, detectionType, relation.getValue());
+ boolean detected = isRelationDetected(relation.getKey(), record);
+ result.setDetected(detected);
+ String detFor = getDetectedFor(record, relation.getKey());
+ if (detected) {
+ TP++;
+ tTP++;
+ if (!relationsToDetect.remove(detFor)) {
+ TP--;
+ tTP--;
+ }
+ }
+ result.setDetectedFor(detFor);
+ results.put(relation.getKey().toLowerCase(), result);
+ }
+
+ result.setNumberOfRelations(getNumberOfRelations(record, false));
+ result.setNumberOfAllRelations(getNumberOfRelations(record, true));
+ results.replace(relation.getKey().toLowerCase(), result);
+ r++;
+ }
+
+ FN += getUndetected(relationsToDetect);
+ tFN += getUndetected(relationsToDetect);
+
+ return results;
+ }
+
+ public static DBPediaOntologyExtractor getDBPediaOntologyExtractor() {
+ return doe;
+ }
+
+ public static FBCategoriesExtractor getFBCategoriesExtractor() {
+ return fce;
+ }
+}
diff --git a/src/main/java/relation_linking/Result.java b/src/main/java/relation_linking/Result.java
new file mode 100644
index 0000000..84c1ce1
--- /dev/null
+++ b/src/main/java/relation_linking/Result.java
@@ -0,0 +1,186 @@
+package relation_linking;
+
+import relation_linking.RelationLinkingEngine.METHOD_DETECTION_TYPE;
+import relation_linking.RelationLinkingEngine.METHOD_MAPPING_TYPE;
+
+public class Result {
+
+ private String name;
+ private Double directSearchSimilarity = 0.00;
+ private Double gloveAllSimilarity = 0.00;
+ private Double gloveOpenIESimilarity = 0.00;
+ private Double gloveLexicalParserSimilarity = 0.00;
+ private Double gloveStrippingSimilarity = 0.00;
+ private Double wordnetAllSimilarity = 0.00;
+ private Double wordnetOpenIESimilarity = 0.00;
+ private Double wordnetLexicalParserSimilarity = 0.00;
+ private Double wordnetStrippingSimilarity = 0.00;
+ private Integer numberOfRelations = 0;
+ private Integer numberOfAllRelations = 0;
+ private String detectedFor;
+ private boolean detected = false;
+
+ public Result(String name, METHOD_MAPPING_TYPE mappingType, METHOD_DETECTION_TYPE detectionType,
+ Double similarity) {
+ switch (mappingType) {
+ case DIRECT:
+ directSearchSimilarity = similarity;
+ break;
+ case GLOVE: {
+ switch (detectionType) {
+ case ALL:
+ gloveAllSimilarity = similarity;
+ break;
+ case OPENIE:
+ gloveOpenIESimilarity = similarity;
+ break;
+ case LEXICALPARSER:
+ gloveLexicalParserSimilarity = similarity;
+ break;
+ case QUERYSTRIPPING:
+ gloveStrippingSimilarity = similarity;
+ break;
+ default:
+ break;
+ }
+ }
+ break;
+ case WORDNET: {
+ switch (detectionType) {
+ case ALL:
+ wordnetAllSimilarity = similarity;
+ break;
+ case OPENIE:
+ wordnetOpenIESimilarity = similarity;
+ break;
+ case LEXICALPARSER:
+ wordnetLexicalParserSimilarity = similarity;
+ break;
+ case QUERYSTRIPPING:
+ wordnetStrippingSimilarity = similarity;
+ break;
+ default:
+ break;
+ }
+ }
+ break;
+ default:
+ break;
+ }
+
+ this.setName(name);
+ }
+
+ public Double getDirectSearch() {
+ return directSearchSimilarity;
+ }
+
+ public void setDirectSearch(Double directSearch) {
+ this.directSearchSimilarity = directSearch;
+ }
+
+ public boolean isDetected() {
+ return detected;
+ }
+
+ public void setDetected(boolean detected) {
+ this.detected = detected;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Double getGloveAllSimilarity() {
+ return gloveAllSimilarity;
+ }
+
+ public void setGloveAllSimilarity(Double gloveAllSimilarity) {
+ this.gloveAllSimilarity = gloveAllSimilarity;
+ }
+
+ public Double getGloveOpenIESimilarity() {
+ return gloveOpenIESimilarity;
+ }
+
+ public void setGloveOpenIESimilarity(Double gloveOpenIESimilarity) {
+ this.gloveOpenIESimilarity = gloveOpenIESimilarity;
+ }
+
+ public Double getGloveLexicalParserSimilarity() {
+ return gloveLexicalParserSimilarity;
+ }
+
+ public void setGloveLexicalParserSimilarity(Double gloveLexicalParserSimilarity) {
+ this.gloveLexicalParserSimilarity = gloveLexicalParserSimilarity;
+ }
+
+ public Double getGloveStrippingSimilarity() {
+ return gloveStrippingSimilarity;
+ }
+
+ public void setGloveStrippingSimilarity(Double gloveStrippingSimilarity) {
+ this.gloveStrippingSimilarity = gloveStrippingSimilarity;
+ }
+
+ public Double getWordnetAllSimilarity() {
+ return wordnetAllSimilarity;
+ }
+
+ public void setWordnetAllSimilarity(Double wordnetAllSimilarity) {
+ this.wordnetAllSimilarity = wordnetAllSimilarity;
+ }
+
+ public Double getWordnetOpenIESimilarity() {
+ return wordnetOpenIESimilarity;
+ }
+
+ public void setWordnetOpenIESimilarity(Double wordnetOpenIESimilarity) {
+ this.wordnetOpenIESimilarity = wordnetOpenIESimilarity;
+ }
+
+ public Double getWordnetLexicalParserSimilarity() {
+ return wordnetLexicalParserSimilarity;
+ }
+
+ public void setWordnetLexicalParserSimilarity(Double wordnetLexicalParserSimilarity) {
+ this.wordnetLexicalParserSimilarity = wordnetLexicalParserSimilarity;
+ }
+
+ public Double getWordnetStrippingSimilarity() {
+ return wordnetStrippingSimilarity;
+ }
+
+ public void setWordnetStrippingSimilarity(Double wordnetStrippingSimilarity) {
+ this.wordnetStrippingSimilarity = wordnetStrippingSimilarity;
+ }
+
+ public Integer getNumberOfRelations() {
+ return numberOfRelations;
+ }
+
+ public void setNumberOfRelations(Integer numberOfRelations) {
+ this.numberOfRelations = numberOfRelations;
+ }
+
+ public Integer getNumberOfAllRelations() {
+ return numberOfAllRelations;
+ }
+
+ public void setNumberOfAllRelations(Integer numberOfAllRelations) {
+ this.numberOfAllRelations = numberOfAllRelations;
+ }
+
+ public String getDetectedFor() {
+ return detectedFor;
+ }
+
+ public void setDetectedFor(String detectedFor) {
+ this.detectedFor = detectedFor;
+ }
+
+}
diff --git a/src/main/java/relation_linking/WordNetEngine.java b/src/main/java/relation_linking/WordNetEngine.java
new file mode 100644
index 0000000..0bb4ab2
--- /dev/null
+++ b/src/main/java/relation_linking/WordNetEngine.java
@@ -0,0 +1,314 @@
+package relation_linking;
+
+import java.io.*;
+import java.util.*;
+import java.util.Map.Entry;
+
+import edu.stanford.nlp.ling.HasWord;
+import edu.stanford.nlp.process.DocumentPreprocessor;
+import net.didion.jwnl.JWNL;
+import net.didion.jwnl.JWNLException;
+import net.didion.jwnl.data.*;
+import net.didion.jwnl.dictionary.Dictionary;
+import relation_linking.RelationLinkingEngine.METHOD_DETECTION_TYPE;
+
+public class WordNetEngine {
+
+ Dictionary wordnet = null;
+ private Map> DBPediaSynsets = null;
+ private Map> FreebaseSynsets = null;
+ private LexicalParsingEngine lpe = null;
+ private OpenIEEngine openIE = null;
+ private QueryStrippingEngine qse = null;
+ private double similarity;
+
+ private static WordNetEngine instance = null;
+
+ public static WordNetEngine getInstance() {
+ if (instance != null)
+ return instance;
+ else
+ return new WordNetEngine();
+ }
+
+ public void init(String path, double similarity) throws JWNLException, ClassNotFoundException, IOException {
+ System.out.println("Initializing WordNet Search engine...");
+
+ if (wordnet == null) {
+ JWNL.initialize(new FileInputStream(path));
+ wordnet = Dictionary.getInstance();
+ }
+
+ if (DBPediaSynsets == null)
+ DBPediaSynsets = getSynsetsForDBPedia();
+ if (FreebaseSynsets == null)
+ FreebaseSynsets = getSynsetsForFreebase();
+ this.similarity = similarity;
+ }
+
+ public void init(String path, LexicalParsingEngine lpe, double similarity)
+ throws JWNLException, ClassNotFoundException, IOException {
+ System.out.println("Initializing WordNet Search engine with lexical parser...");
+
+ if (wordnet == null) {
+ JWNL.initialize(new FileInputStream(path));
+ wordnet = Dictionary.getInstance();
+ }
+
+ if (DBPediaSynsets == null)
+ DBPediaSynsets = getSynsetsForDBPedia();
+ if (FreebaseSynsets == null)
+ FreebaseSynsets = getSynsetsForFreebase();
+ if (this.lpe == null)
+ this.lpe = lpe;
+ this.similarity = similarity;
+ }
+
+ public void init(String path, OpenIEEngine openIE, double similarity)
+ throws JWNLException, ClassNotFoundException, IOException {
+ System.out.println("Initializing WordNet Search engine with OpenIE...");
+
+ if (wordnet == null) {
+ JWNL.initialize(new FileInputStream(path));
+ wordnet = Dictionary.getInstance();
+ }
+
+ if (DBPediaSynsets == null)
+ DBPediaSynsets = getSynsetsForDBPedia();
+ if (FreebaseSynsets == null)
+ FreebaseSynsets = getSynsetsForFreebase();
+ if (this.openIE == null)
+ this.openIE = openIE;
+ this.similarity = similarity;
+ }
+
+ public void init(String path, QueryStrippingEngine qse, double similarity)
+ throws JWNLException, ClassNotFoundException, IOException {
+ System.out.println("Initializing WordNet Search engine with lexical parser...");
+
+ if (wordnet == null) {
+ JWNL.initialize(new FileInputStream(path));
+ wordnet = Dictionary.getInstance();
+ }
+
+ if (DBPediaSynsets == null)
+ DBPediaSynsets = getSynsetsForDBPedia();
+ if (FreebaseSynsets == null)
+ FreebaseSynsets = getSynsetsForFreebase();
+ if (this.qse == null)
+ this.qse = qse;
+ this.similarity = similarity;
+ }
+
+ private ArrayList getSynsetsFromWord(String relation) throws JWNLException {
+ ArrayList results = new ArrayList();
+
+ IndexWordSet indexWordSet = wordnet.lookupAllIndexWords(relation);
+ IndexWord[] indexWords = indexWordSet.getIndexWordArray();
+
+ for (IndexWord indexWord : indexWords) {
+ for (Synset synset : indexWord.getSenses()) {
+ Word[] words = synset.getWords();
+ for (Word word : words) {
+ if (!results.contains(word.getLemma()))
+ results.add(word.getLemma());
+ }
+ }
+ }
+
+ return results;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map> getSynsets(String filename, boolean freebase)
+ throws JWNLException, FileNotFoundException, IOException, ClassNotFoundException {
+ File wordnetStore = new File(filename);
+
+ Map> map = new HashMap>();
+
+ if (!wordnetStore.exists()) {
+ ArrayList relations;
+ if (freebase) {
+ FBCategoriesExtractor fbe = RelationLinkingEngine.getFBCategoriesExtractor();
+ relations = fbe.getCategories();
+ } else {
+ DBPediaOntologyExtractor doe = RelationLinkingEngine.getDBPediaOntologyExtractor();
+ relations = doe.getDBPediaRelations();
+ }
+ for (String relation : relations) {
+ ArrayList synsets = getSynsetsFromWord(relation);
+ map.put(relation, synsets);
+ }
+
+ System.out.println("Writing synsets to file...");
+ ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(filename));
+ oos.writeObject(map);
+ oos.flush();
+ oos.close();
+ } else {
+ System.out.println("Reading synsets from file...");
+ ObjectInputStream ois = new ObjectInputStream(new FileInputStream(filename));
+ map = (HashMap>) ois.readObject();
+ ois.close();
+ }
+
+ return map;
+ }
+
+ private Map> getSynsetsForDBPedia()
+ throws JWNLException, FileNotFoundException, IOException, ClassNotFoundException {
+ System.out.println("Getting synsets for DBPedia...");
+ return getSynsets("src/main/resources/data/DBPediaSynsets", false);
+ }
+
+ private Map> getSynsetsForFreebase()
+ throws FileNotFoundException, ClassNotFoundException, JWNLException, IOException {
+ System.out.println("Getting synsets for Freebase...");
+ return getSynsets("src/main/resources/data/FreebaseSynsets", true);
+ }
+
+ private Map getLexicalizedRelations(String sentence) throws JWNLException {
+ ArrayList nouns = lpe.getNounsFromSentence(sentence);
+
+ Map results = new HashMap();
+
+ for (String word : nouns) {
+ ArrayList synsets = getSynsetsFromWord(word);
+
+ Map relations = isDBPediaRelation(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isFBCategory(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+ }
+
+ return results;
+ }
+
+ private String[] splitRelation(String relation) {
+ return relation.split("\\s+");
+ }
+
+ private Map getOpenIERelations(String sentence) throws JWNLException {
+
+ ArrayList openIERelations = openIE.getRelations(sentence);
+ Map results = new HashMap();
+
+ for (String relation : openIERelations) {
+ String[] words = splitRelation(relation);
+ for (String word : words) {
+ ArrayList synsets = getSynsetsFromWord(word);
+
+ Map relations = isDBPediaRelation(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isFBCategory(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private Map getQueryStrippedRelations(String sentence) throws JWNLException {
+ ArrayList strippedWords = qse.getRelations(sentence);
+ Map results = new HashMap();
+
+ for (String relation : strippedWords) {
+ ArrayList synsets = getSynsetsFromWord(relation);
+
+ Map relations = isDBPediaRelation(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isFBCategory(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+ }
+
+ return results;
+ }
+
+ private Map getRelations(ArrayList synsets, Map> map) {
+ Map results = new HashMap();
+
+ for (Entry> mapEntry : map.entrySet()) {
+ @SuppressWarnings("unchecked")
+ ArrayList relSynsets = (ArrayList) synsets.clone();
+ relSynsets.removeAll((Collection>) mapEntry.getValue());
+ if (relSynsets.size() != synsets.size()) {
+ double number = (double) relSynsets.size() / (double) synsets.size();
+ if (((double) (1 - number)) > similarity) {
+ results.put(mapEntry.getKey(), new Double(1 - number));
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private Map isDBPediaRelation(ArrayList synsets) {
+ return getRelations(synsets, DBPediaSynsets);
+ }
+
+ private Map isFBCategory(ArrayList synsets) {
+ return getRelations(synsets, FreebaseSynsets);
+ }
+
+ public Map getRelations(String sentence, METHOD_DETECTION_TYPE methodType) throws JWNLException {
+ System.out.println("Getting WordNet relations...");
+
+ Map results = new HashMap();
+
+ switch (methodType) {
+ case ALL: {
+ Reader reader = new StringReader(sentence);
+
+ for (Iterator> iterator = new DocumentPreprocessor(reader).iterator(); iterator.hasNext();) {
+ List word = iterator.next();
+
+ for (int i = 0; i < word.size(); i++) {
+ String sWord = word.get(i).toString();
+
+ ArrayList synsets = getSynsetsFromWord(sWord);
+
+ Map relations = isDBPediaRelation(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+
+ relations = isFBCategory(synsets);
+ if (relations != null) {
+ results.putAll(relations);
+ }
+ }
+ }
+ }
+ break;
+ case OPENIE:
+ results.putAll(getOpenIERelations(sentence));
+ break;
+ case LEXICALPARSER:
+ results.putAll(getLexicalizedRelations(sentence));
+ break;
+ case QUERYSTRIPPING:
+ results.putAll(getQueryStrippedRelations(sentence));
+ break;
+ default:
+ break;
+ }
+
+ return results;
+ }
+
+}
diff --git a/src/main/java/util/CollectionChanges.java b/src/main/java/util/CollectionChanges.java
new file mode 100755
index 0000000..afd68f4
--- /dev/null
+++ b/src/main/java/util/CollectionChanges.java
@@ -0,0 +1,106 @@
+package util;
+
+import java.util.Collection;
+import java.util.HashSet;
+
+public class CollectionChanges {
+
+ public String summaryString() {
+
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("Additions: ");
+ sb.append("" + addedToSet + "\n");
+
+ sb.append("Removals: ");
+ sb.append("" + removedFromSet + "\n");
+
+ return sb.toString();
+
+ }
+
+ Collection addedToSet = new HashSet();
+ Collection removedFromSet = new HashSet();
+
+ public void setAddition(String setItem) {
+ addedToSet.add(setItem);
+ }
+
+ public void setRemoval(String setItem) {
+ removedFromSet.add(setItem);
+ }
+
+ public Collection additions() {
+ return addedToSet;
+ }
+
+ public Collection removals() {
+ return removedFromSet;
+ }
+
+ public boolean isEmpty() {
+ return addedToSet.isEmpty() && removedFromSet.isEmpty();
+ }
+
+ /**
+ * Compute the changes between the two sets.
+ *
+ * @param previous the previous set
+ * @param next the current set
+ * @return the changes
+ */
+
+ public static CollectionChanges findAdditions(Collection previous, Collection next) {
+
+ CollectionChanges chg = new CollectionChanges();
+
+ // Check for added terms
+ for (String setElement : next) {
+ if (!previous.contains(setElement)) {
+ chg.setAddition(setElement);
+ }
+ }
+
+ return chg;
+
+ }
+
+ public static CollectionChanges findAdditions(Collection previous, Collection next, boolean ignore_mentions) {
+
+ CollectionChanges chg = new CollectionChanges();
+
+ // Check for added terms
+ for (String setElement : next) {
+
+ if (setElement.startsWith("@")) {
+ continue;
+ }
+
+ if (!previous.contains(setElement)) {
+ chg.setAddition(setElement);
+ }
+ }
+
+ return chg;
+
+ }
+
+ public CollectionChanges findChanges(Collection previous, Collection next) {
+
+ // Check for added users
+ for (String setElement : next) {
+ if (!previous.contains(setElement)) {
+ this.setAddition(setElement);
+ }
+ }
+
+ // Check for removed
+ for (String setElement : previous) {
+ if (!next.contains(setElement)) {
+ this.setRemoval(setElement);
+ }
+ }
+
+ return this;
+ }
+}
diff --git a/src/main/java/util/Compression.java b/src/main/java/util/Compression.java
new file mode 100755
index 0000000..21d4d6a
--- /dev/null
+++ b/src/main/java/util/Compression.java
@@ -0,0 +1,50 @@
+package util;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import me.lemire.integercompression.Composition;
+import me.lemire.integercompression.FastPFOR;
+import me.lemire.integercompression.IntWrapper;
+import me.lemire.integercompression.IntegerCODEC;
+import me.lemire.integercompression.VariableByte;
+
+public class Compression {
+
+ static final IntegerCODEC codec = new Composition(new FastPFOR(), new VariableByte());
+
+ public static HashMap compressSemantics(HashMap input) {
+ HashMap compressed = new HashMap();
+ System.out.println("Compress...");
+ for (Entry e : input.entrySet()) {
+ compressed.put(e.getKey(), compress(e.getValue()));
+ }
+ System.out.println("...Done.");
+ return compressed;
+ }
+
+ public static int[] compress(int[] data) {
+ int[] compressed = new int[data.length + 1024];// could need more
+ // compressing
+ IntWrapper inputoffset = new IntWrapper(0);
+ IntWrapper outputoffset = new IntWrapper(0);
+ codec.compress(data, inputoffset, data.length, compressed, outputoffset);
+ // System.out.println("compressed unsorted integers from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");
+ // we can repack the data: (optional)
+ compressed = Arrays.copyOf(compressed, outputoffset.intValue());
+ return compressed;
+ }
+
+ public static int[] decompress(int[] compressed, int vectorSize) {
+ int[] recovered = new int[vectorSize];
+ IntWrapper recoffset = new IntWrapper(0);
+ codec.uncompress(compressed, new IntWrapper(0), compressed.length, recovered, recoffset);
+ // if(Arrays.equals(data,recovered))
+ // System.out.println("data is recovered without loss");
+ // else
+ // throw new RuntimeException("bug"); // could use assert
+ return recovered;
+ }
+
+}
diff --git a/src/main/java/util/ConvertFormats.java b/src/main/java/util/ConvertFormats.java
new file mode 100755
index 0000000..69cd5ed
--- /dev/null
+++ b/src/main/java/util/ConvertFormats.java
@@ -0,0 +1,21 @@
+package util;
+
+import java.io.File;
+
+import word2vec.W2vSpace;
+
+public class ConvertFormats {
+
+ public static void main(String[] args) {
+
+ W2vSpace txt = W2vSpace.loadText("/home/igor/git/word2vec-java/src/test/resources/small_vectors.txt");
+
+ W2vSpace gz = W2vSpace.loadText("/home/igor/git/word2vec-java/src/test/resources/small_vectors.txt.gz");
+
+
+ txt.saveAsText(new File("/home/igor/git/word2vec-java/src/test/resources/small_vectors.txt1"));
+ gz.saveAsText(new File("/home/igor/git/word2vec-java/src/test/resources/small_vectors.txt2"));
+
+ }
+
+}
diff --git a/src/main/java/util/Filters.java b/src/main/java/util/Filters.java
new file mode 100755
index 0000000..cb5ed83
--- /dev/null
+++ b/src/main/java/util/Filters.java
@@ -0,0 +1,211 @@
+package util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+
+/*
+ * Apply Various Filters to words:
+ */
+public class Filters {
+
+ public WordFilter ccomp = (new PrefixFilter()).with("ccomp");
+ public WordFilter nsubj = (new PrefixFilter()).with("nsubj");
+ public WordFilter agent = (new PrefixFilter()).with("agent");
+ public WordFilter advcl = (new PrefixFilter()).with("advcl");
+
+
+
+ public WordFilter removeHashtags = (new PrefixFilter()).with("#");
+ public WordFilter removeMentions = (new PrefixFilter()).with("@");
+
+ public WordFilter removeWords = new RemoveWords();
+
+ public WordFilter removeShort = new RemoveShortWords();
+
+
+ /*
+ * Remove default nltk + twitter stopwords
+ */
+ public WordFilter removeStopwords = new RemoveWords() {
+ {
+ try (InputStream in = this.getClass().getResourceAsStream("/data/nltk_en_stopwords.txt")) {
+ String text = IOUtils.toString(in, "UTF-8");
+ remove.addAll(Arrays.asList(StringUtils.split(text, ' ')));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ };
+
+ public WordFilter removeShortWords = new RemoveWords() {
+ {
+ try (InputStream in = this.getClass().getResourceAsStream("/data/nltk_en_stopwords.txt")) {
+ String text = IOUtils.toString(in, "UTF-8");
+ remove.addAll(Arrays.asList(StringUtils.split(text, ' ')));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ };
+
+ /*
+ * Common filter:
+ */
+ public WordFilter[] wordsOnly(WordFilter... moreFilters) {
+ WordFilter[] filters = new WordFilter[3 + moreFilters.length];
+ filters[0] = removeStopwords;
+ filters[1] = removeHashtags;
+ filters[2] = removeMentions;
+ int i = 3;
+ for (WordFilter f : moreFilters) {
+ filters[i++] = f;
+ }
+ return filters;
+ }
+
+ /*
+ * Apply filters to words:
+ */
+ public static boolean apply(WordSim wordSim, WordFilter... filters) {
+ return apply(wordSim.getString(), filters);
+ }
+
+ public static boolean apply(String word, WordFilter... filters) {
+ boolean keep = true;
+ //Remove Duplicate Filters:
+ final Set applyFilters = new HashSet(Arrays.asList(filters));
+ // XOR Results:
+ for (WordFilter f : applyFilters) {
+ keep = keep ^ f.evaluate(word);
+ // System.out.println("Eval: " + word + " " + f.evaluate(word) + " still keep? " + keep);
+ }
+ return keep;
+ }
+
+ public interface WordFilter {
+ public boolean evaluate(String word);
+
+ public boolean evaluate(WordSim wordSim);
+
+ public WordFilter with(String... words);
+
+ public WordFilter with(Collection words);
+ }
+
+ /*
+ * Default: treat WordSim as String:
+ */
+ private abstract class DefaultFilter implements WordFilter {
+ @Override
+ public abstract boolean evaluate(String word);
+
+ @Override
+ public boolean evaluate(WordSim wordSim) {
+ return evaluate(wordSim.getString());
+ }
+
+ @Override
+ public WordFilter with(String... words) {
+ return this;
+ }
+ }
+
+ /*
+ * Remove words specified with init(...)
+ */
+ private class RemoveWords extends DefaultFilter {
+ Set remove;
+
+ public RemoveWords() {
+ remove = new HashSet();
+ }
+
+ @Override
+ public boolean evaluate(String word) {
+ return remove.contains(word);
+ }
+
+ @Override
+ public WordFilter with(String... words) {
+ remove.addAll(Arrays.asList(words));
+ return this;
+ }
+ @Override
+ public WordFilter with(Collection words) {
+ remove.addAll(words);
+ return this;
+ }
+ }
+
+ private class RemoveShortWords extends DefaultFilter {
+ public RemoveShortWords() {
+ }
+
+ @Override
+ public boolean evaluate(String word) {
+ return (word.length() < 3);
+ }
+
+ @Override
+ public WordFilter with(Collection words) {
+ return this;
+ }
+
+ }
+
+
+ /*
+ * Useful to add if you want to keep a word that gets filtered.
+ *
+ * Invert removal of specific words: Filters.invert.with("is", "#is")
+ *
+ * Invert ALL removals from all other filters: Filters.invert
+ */
+ public WordFilter invertRemove = new DefaultFilter() {
+ Set keep = new HashSet();
+
+ @Override
+ public boolean evaluate(String word) {
+ if (keep.size() > 0) {
+ return keep.contains(word);
+ } else {
+ return !keep.contains(word);
+ }
+ }
+
+ @Override
+ public WordFilter with(Collection words) {
+ return this;
+ }
+ };
+
+ public class PrefixFilter extends DefaultFilter {
+ String[] pref = new String[0];
+
+ @Override
+ public boolean evaluate(String word) {
+ for (String p : pref) {
+ return word.startsWith(p);
+ }
+ return false;
+ }
+
+ @Override
+ public WordFilter with(String... pref) {
+ this.pref = pref;
+ return this;
+ }
+
+ @Override
+ public WordFilter with(Collection words) {
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/util/VectorMath.java b/src/main/java/util/VectorMath.java
new file mode 100755
index 0000000..04b8bf7
--- /dev/null
+++ b/src/main/java/util/VectorMath.java
@@ -0,0 +1,78 @@
+package util;
+
+import java.util.List;
+
+import org.jblas.DoubleMatrix;
+import org.jblas.FloatMatrix;
+
+public class VectorMath {
+
+ /*
+ * Vectors are normalized on load, just need dot product:
+ */
+ public static double cosineSimilarity(FloatMatrix vec1, FloatMatrix vec2) {
+ double dotSim = vec1.dot(vec2);
+ double norm = vec1.norm2() * vec2.norm2();
+ return norm == 0 ? 1 : dotSim / norm;
+ }
+
+ public static double cosineSimilarity(DoubleMatrix vec1, DoubleMatrix vec2) {
+ double dotSim = vec1.dot(vec2);
+ double norm = vec1.norm2() * vec2.norm2();
+ return norm == 0 ? 1 : dotSim / norm;
+ }
+
+ public static double distance(FloatMatrix vec1, final FloatMatrix vec2) {
+ return vec1.distance2(vec2);
+ }
+
+ public static double distanceSimilarity(FloatMatrix vec1, final FloatMatrix vec2) {
+ return 1 / (1 + vec1.distance2(vec2));
+ }
+
+ public static double distanceSimilarity(DoubleMatrix vec1, final DoubleMatrix vec2) {
+ return 1 / (1 + vec1.distance2(vec2));
+ }
+
+ /*
+ * Utility functions:
+ */
+
+ public static FloatMatrix normalize(FloatMatrix f) {
+ return f.divi(f.norm2());
+ }
+
+ public static DoubleMatrix normalize(DoubleMatrix d) {
+ return d.divi(d.norm2());
+ }
+
+ public static FloatMatrix addFloatMatrix(List vectors) {
+ if (vectors.size() < 1) {
+ return null;
+ }
+ FloatMatrix vec = vectors.remove(0);
+ for (FloatMatrix f : vectors) {
+ vec.addi(f);
+ }
+ return vec;
+ }
+
+ public static DoubleMatrix addDoubleMatrix(List vectors) {
+ if (vectors.size() < 1) {
+ return null;
+ }
+ DoubleMatrix vec = vectors.remove(0);
+ for (DoubleMatrix d : vectors) {
+ vec.addi(d);
+ }
+ return vec;
+ }
+
+ /*
+ * Only used by GloveModelLoader
+ */
+ public static DoubleMatrix addDoubleMatrix(DoubleMatrix vector1, DoubleMatrix vector2) {
+ return vector1.addi(vector2);
+ }
+
+}
diff --git a/src/main/java/util/WordSim.java b/src/main/java/util/WordSim.java
new file mode 100755
index 0000000..850373d
--- /dev/null
+++ b/src/main/java/util/WordSim.java
@@ -0,0 +1,32 @@
+package util;
+
+public class WordSim implements Comparable {
+
+ private final String s;
+ private final Double d;
+
+ public WordSim(String s, Double d) {
+ this.s = s;
+ this.d = d;
+ }
+
+ public String getString() {
+ return this.s;
+ }
+
+ public Double getDouble() {
+ return this.d;
+ }
+
+ @Override
+ public int compareTo(WordSim other) {
+ // Reverse:
+ return (-1 * getDouble().compareTo(other.getDouble()));
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s : %.2f", this.s, this.d);
+ }
+
+}
diff --git a/src/main/java/util/WordSpaceUtils.java b/src/main/java/util/WordSpaceUtils.java
new file mode 100755
index 0000000..ec29cd8
--- /dev/null
+++ b/src/main/java/util/WordSpaceUtils.java
@@ -0,0 +1,23 @@
+package util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import word2vec.GenericWordSpace;
+
+public class WordSpaceUtils {
+
+ /*
+ * Remove vectors from model:
+ */
+ public static GenericWordSpace reduceModelVocab(GenericWordSpace model, Set keyVectors) {
+ Set vocab = new HashSet(model.store.keySet());
+ for (String word : vocab) {
+ if (!keyVectors.contains(word)) {
+ model.store.remove(word);
+ }
+ }
+ return model;
+ }
+
+}
diff --git a/src/main/java/word2vec/GenericWordSpace.java b/src/main/java/word2vec/GenericWordSpace.java
new file mode 100755
index 0000000..dae6b10
--- /dev/null
+++ b/src/main/java/word2vec/GenericWordSpace.java
@@ -0,0 +1,166 @@
+package word2vec;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.PriorityQueue;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import util.Filters;
+import util.Filters.WordFilter;
+import util.WordSim;
+
+public abstract class GenericWordSpace implements WordSpace {
+
+ public Filters f = new Filters();
+ /*
+ * Store vectors & vocab in memory:
+ */
+ public Map store = new HashMap();
+
+ @Override
+ public boolean contains(String word) {
+ return store.containsKey(word);
+ }
+
+ @Override
+ public T vector(String word) {
+ return store.get(word);
+ }
+
+ @Override
+ public double cosineSimilarity(String w1, String w2) {
+ return cosineSimilarity(vector(w1), vector(w2));
+ }
+
+ public abstract double cosineSimilarity(T vec1, T vec2);
+
+ @Override
+ public double distanceSimilarity(String w1, String w2) {
+ return distanceSimilarity(vector(w1), vector(w2));
+ }
+
+ public abstract double distanceSimilarity(T vec1, T vec2);
+
+ /*
+ * Linear search for k Nearest Neighbours of a vector:
+ */
+ @Override
+ public List knn(T vec, int k, WordFilter... filters) {
+ PriorityQueue kSimilarWords = new PriorityQueue(k * 2);
+ for (Entry e : store.entrySet()) {
+ if (Filters.apply(e.getKey(), filters)) {
+ double dot = cosineSimilarity(vec, e.getValue());
+ if (Double.isFinite(dot)) {
+ kSimilarWords.add(new WordSim(e.getKey(), dot));
+ }
+ }
+ }
+ List col = new ArrayList();
+ for (int i = 0; i < k; i++) {
+ WordSim ws = kSimilarWords.poll();
+ if (ws != null) {
+ col.add(ws);
+ }
+ }
+ return col;
+ }
+
+ public List dist_knn(T vec, int k, WordFilter... filters) {
+ PriorityQueue kSimilarWords = new PriorityQueue(k * 2);
+ for (Entry e : store.entrySet()) {
+ if (Filters.apply(e.getKey(), filters)) {
+ double dot = distanceSimilarity(vec, e.getValue());
+ if (Double.isFinite(dot)) {
+ kSimilarWords.add(new WordSim(e.getKey(), dot));
+ }
+ }
+ }
+ List col = new ArrayList();
+ for (int i = 0; i < k; i++) {
+ WordSim ws = kSimilarWords.poll();
+ if (ws != null) {
+ col.add(ws);
+ }
+ }
+ return col;
+ }
+
+ @Override
+ public List knn(String word, int k, WordFilter... filters) {
+ return knn(vector(word), k, filters);
+ }
+
+ @Override
+ public List knnWords(String word, int k, WordFilter... filters) {
+ return knnWords(vector(word), k, filters);
+ }
+
+ @Override
+ public List knnWords(T vec, int k, WordFilter... filters) {
+ List sims = knn(vec, k, filters);
+ List words = new ArrayList();
+ for (WordSim sim : sims) {
+ words.add(sim.getString());
+ }
+ return words;
+ }
+
+ /*
+ * Additive representation of several words:
+ */
+ @Override
+ public T sentenceVector(String sentence) {
+ return sentenceVector(sentence, f.removeStopwords);
+ }
+
+ @Override
+ public T sentenceVector(String sentence, WordFilter... filters) {
+ List vectors = new ArrayList();
+ for (String word : StringUtils.split(sentence, ' ')) {
+ if (Filters.apply(word, filters) && contains(word)) {
+ vectors.add(vector(word));
+ }
+ }
+ return additiveSentenceVector(vectors);
+ }
+
+ @Override
+ public abstract T additiveSentenceVector(List vectors);
+
+ /*
+ * Debug:
+ */
+ public void printSims(String w, List sims) {
+ for (WordSim s : sims) {
+ String str = String.format("%s %s %.4f", w, s.getString(), s.getDouble());
+ System.out.println(str);
+ }
+ }
+
+ /*
+ * Save Model as text file:
+ */
+ public boolean saveAsText(File output) {
+ for (Entry entry : store.entrySet()) {
+ String word = String.format("%s %s\n", entry.getKey(), StringUtils.join(entry.getValue(), ' '));
+ // TODO:
+ word = word.replaceAll("\\[", "");
+ word = word.replaceAll(";", "");
+ word = word.replaceAll("\\]", "");
+ try {
+ FileUtils.writeStringToFile(output, word, true);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
diff --git a/src/main/java/word2vec/GloVeSpace.java b/src/main/java/word2vec/GloVeSpace.java
new file mode 100755
index 0000000..06d3128
--- /dev/null
+++ b/src/main/java/word2vec/GloVeSpace.java
@@ -0,0 +1,172 @@
+package word2vec;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import util.VectorMath;
+import org.jblas.DoubleMatrix;
+
+/*
+ * A Java wrapper for GloVe - Only Reads a pre trained model!
+ */
+public class GloVeSpace extends GenericWordSpace {
+
+ /*
+ * Read .txt or .txt.gz model:
+ */
+ public static GloVeSpace load(String gloVeModel, boolean norm, boolean header) {
+ GloVeSpace model = new GloVeSpace();
+ try {
+ Reader decoder;
+ if (gloVeModel.endsWith("gz")) {
+ decoder = new InputStreamReader(new GZIPInputStream(new FileInputStream(gloVeModel)), "UTF-8");
+ } else {
+ decoder = new InputStreamReader(new FileInputStream(gloVeModel), "UTF-8");
+ }
+ BufferedReader r = new BufferedReader(decoder);
+
+ long numWords = 0;
+ String line;
+
+ if (header) {
+ String h = r.readLine();
+ System.out.println(h);
+ }
+
+ while ((line = r.readLine()) != null) {
+ // Split into words:
+ String[] wordvec = StringUtils.split(line, ' ');
+ if (wordvec.length < 2) {
+ break;
+ }
+ double[] vec = readDoubleVector(wordvec);
+ if (norm) {
+ model.store.put(wordvec[0], VectorMath.normalize(new DoubleMatrix(vec)));
+ } else {
+ model.store.put(wordvec[0], new DoubleMatrix(vec));
+ }
+ numWords++;
+ }
+ decoder.close();
+ r.close();
+ int vecSize = model.store.entrySet().iterator().next().getValue().length;
+ System.out.println(String.format("Loaded %s words, vector size %s", numWords, vecSize));
+
+ } catch (IOException e) {
+ System.err.println("ERROR: Failed to load model: " + gloVeModel);
+ e.printStackTrace();
+ }
+ return model;
+ }
+
+ /*
+ * Equivalent to Text model: With Contexts, no bias. norm = unit vector
+ */
+ public static GloVeSpace load(String vocabFile, String gloVeModel, boolean norm) {
+ return load(vocabFile, gloVeModel, true, false, norm);
+ }
+
+ /*
+ * Read binary model, includes bias term, context vectors:
+ */
+ public static GloVeSpace load(String vocabFile, String gloVeModel, boolean withContexts, boolean bias, boolean norm) {
+ GloVeSpace model = new GloVeSpace();
+ try {
+ FileInputStream in = new FileInputStream(gloVeModel);
+ DataInputStream ds = new DataInputStream(new BufferedInputStream(in, 131072));
+ List vocab = FileUtils.readLines(new File(vocabFile));
+ long numWords = vocab.size();
+ // Vector Size = num of bytes in total / 16 / vocab
+ int vecSize = (int) (in.getChannel().size() / 16 / numWords) - 1;
+ // Word Vectors:
+ for (int i = 0; i < numWords; i++) {
+ String word = StringUtils.split(vocab.get(i), ' ')[0];
+ double[] vector = readDoubleVector(ds, vecSize, bias);
+ model.store.put(word, new DoubleMatrix(vector));
+ }
+ // Context Vectors:
+ if (withContexts) {
+ for (int i = 0; i < numWords; i++) {
+ String word = StringUtils.split(vocab.get(i), ' ')[0];
+ double[] vector = readDoubleVector(ds, vecSize, bias);
+ model.store.put(word, VectorMath.addDoubleMatrix(model.store.get(word), new DoubleMatrix(vector)));
+ }
+ }
+ // Unit Vectors:
+ if (norm) {
+ for (Entry e : model.store.entrySet()) {
+ model.store.put(e.getKey(), VectorMath.normalize(e.getValue()));
+ }
+ }
+ System.out.println(String.format("Loaded %s words, vector size %s", numWords, vecSize));
+ } catch (IOException e) {
+ System.err.println("ERROR: Failed to load model: " + gloVeModel);
+ e.printStackTrace();
+ }
+ return model;
+ }
+
+ /*
+ * Read a Vector - Array from text file:
+ */
+ private static double[] readDoubleVector(String[] line) throws IOException {
+ int vectorSize = line.length;
+ double[] vector = new double[vectorSize - 1];
+ for (int j = 1; j < vectorSize; j++) {
+ try {
+ double d = Double.parseDouble(line[j]);
+ vector[j - 1] = d;
+ } catch (NumberFormatException e) {
+ System.err.println("ERROR Parsing: " + line + " " + e.getMessage());
+ vector[j - 1] = 0.0D;
+ }
+ }
+ return vector;
+ }
+
+ /*
+ * Read a Vector - Array from binary file:
+ */
+ private static double[] readDoubleVector(DataInputStream ds, int vectorSize, boolean bias) throws IOException {
+ if (bias) {
+ vectorSize += 1; // Include Bias
+ }
+ double[] vector = new double[vectorSize];
+ for (int j = 0; j < vectorSize; j++) {
+ long l = ds.readLong();
+ double d = Double.longBitsToDouble(Long.reverseBytes(l));
+ vector[j] = d;
+ }
+ if (!bias) {
+ ds.readLong(); // Skip Bias
+ }
+ return vector;
+ }
+
+ @Override
+ public double cosineSimilarity(DoubleMatrix vec1, DoubleMatrix vec2) {
+ return VectorMath.cosineSimilarity(vec1, vec2);
+ }
+
+ @Override
+ public double distanceSimilarity(DoubleMatrix vec1, DoubleMatrix vec2) {
+ return VectorMath.distanceSimilarity(vec1, vec2);
+ }
+
+ @Override
+ public DoubleMatrix additiveSentenceVector(List vectors) {
+ return VectorMath.addDoubleMatrix(vectors);
+ }
+
+}
diff --git a/src/main/java/word2vec/RidxSpace.java b/src/main/java/word2vec/RidxSpace.java
new file mode 100755
index 0000000..5e1b0e5
--- /dev/null
+++ b/src/main/java/word2vec/RidxSpace.java
@@ -0,0 +1,108 @@
+package word2vec;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map.Entry;
+
+import util.Compression;
+import util.VectorMath;
+import org.jblas.FloatMatrix;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.UnsafeMemoryInput;
+
+/*
+ * Random Indexing
+ */
+public class RidxSpace extends GenericWordSpace {
+
+ private static int vectorSize;
+
+ public RidxSpace(int vectorSize) {
+ RidxSpace.vectorSize = vectorSize;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static RidxSpace load(String ridxSpaceModel, int vectorSize) {
+ RidxSpace model = new RidxSpace(vectorSize);
+ System.out.println("Loading from file: " + ridxSpaceModel);
+ try {
+ // KRYO Store:
+ HashMap clazz = new HashMap();
+ Kryo kryo = new Kryo();
+ kryo.register(clazz.getClass());
+ UnsafeMemoryInput input = new UnsafeMemoryInput(new FileInputStream(ridxSpaceModel));
+ clazz = kryo.readObject(input, clazz.getClass());
+ for (Entry e : clazz.entrySet()) {
+ model.store.put(e.getKey(), Compression.decompress(e.getValue(), vectorSize));
+ }
+ input.close();
+ } catch (FileNotFoundException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return model;
+ }
+
+
+ public static FloatMatrix convertVector(int[] input, boolean decompress) {
+ FloatMatrix f = new FloatMatrix(convertVector(Compression.decompress(input, vectorSize)));
+ return f;
+ }
+
+ public static float[] convertVector(int[] input) {
+ if (input == null) {
+ return new float[vectorSize]; // Or throw an exception?
+ }
+ float[] output = new float[input.length];
+ for (int i = 0; i < input.length; i++) {
+ output[i] = input[i];
+ }
+ return output;
+ }
+
+
+ @Override
+ public double cosineSimilarity(int[] vec1, int[] vec2) {
+
+ FloatMatrix d1 = new FloatMatrix(copyFromIntArray(vec1));
+ FloatMatrix d2 = new FloatMatrix(copyFromIntArray(vec2));
+
+ d1 = VectorMath.normalize(d1);
+ d2 = VectorMath.normalize(d2);
+
+ return VectorMath.cosineSimilarity(d1, d2);
+
+ }
+
+
+ @Override
+ public double distanceSimilarity(int[] vec1, int[] vec2) {
+ //EuclideanDistance e = new EuclideanDistance();
+ //return e.compute(copyFromIntArray(vec1), copyFromIntArray(vec2));
+ return 0;
+ }
+
+
+ @Override
+ public int[] additiveSentenceVector(List vectors) {
+ final int[] result = new int[vectorSize];
+ for (final int[] v : vectors) {
+ Arrays.parallelSetAll(result, i -> result[i] + v[i]);
+ }
+ return result;
+ }
+
+ public static float[] copyFromIntArray(int[] source) {
+ float[] dest = new float[source.length];
+ for (int i = 0; i < source.length; i++) {
+ dest[i] = source[i];
+ }
+ return dest;
+ }
+
+
+}
diff --git a/src/main/java/word2vec/W2vSpace.java b/src/main/java/word2vec/W2vSpace.java
new file mode 100755
index 0000000..698c5eb
--- /dev/null
+++ b/src/main/java/word2vec/W2vSpace.java
@@ -0,0 +1,185 @@
+package word2vec;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.lang3.StringUtils;
+import util.VectorMath;
+import org.jblas.FloatMatrix;
+
+/*
+ * A Java wrapper for W2v - Only Reads a pre trained model!
+ */
+public class W2vSpace extends GenericWordSpace {
+
+ /*
+ * Load vectors from a text file - 1 word per line.
+ */
+ public static W2vSpace loadText(String word2vecModel) {
+ return loadText(word2vecModel, true, false);
+ }
+
+ public static W2vSpace loadText(String word2vecModel, boolean norm, boolean header) {
+ W2vSpace model = new W2vSpace();
+ try {
+ Reader decoder;
+ if (word2vecModel.endsWith("gz")) {
+ decoder = new InputStreamReader(new GZIPInputStream(new FileInputStream(word2vecModel)), "UTF-8");
+ } else {
+ decoder = new InputStreamReader(new FileInputStream(word2vecModel), "UTF-8");
+ }
+ BufferedReader r = new BufferedReader(decoder);
+
+ long numWords = 0;
+ String line;
+
+ if (header) {
+ String h = r.readLine();
+ System.out.println(h);
+ }
+
+ while ((line = r.readLine()) != null) {
+ // Split into words:
+ String[] wordvec = StringUtils.split(line, ' ');
+ if (wordvec.length < 2) {
+ break;
+ }
+ float[] vec = readFloatVector(wordvec);
+ if (norm) {
+ model.store.put(wordvec[0], VectorMath.normalize(new FloatMatrix(vec)));
+ } else {
+ model.store.put(wordvec[0], new FloatMatrix(vec));
+ }
+ numWords++;
+ }
+ int vecSize = model.store.entrySet().iterator().next().getValue().length;
+ System.out.println(String.format(word2vecModel + " Loaded %s words, vector size %s", numWords, vecSize));
+ } catch (FileNotFoundException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return model;
+ }
+
+ /*
+ * Read a Vector - Array from text file:
+ */
+ private static float[] readFloatVector(String[] line) throws IOException {
+ int vectorSize = line.length;
+ float[] vector = new float[vectorSize - 1];
+ for (int j = 1; j < vectorSize; j++) {
+ try {
+ float d = Float.parseFloat(line[j]);
+ vector[j - 1] = d;
+ } catch (NumberFormatException e) {
+ System.err.println("ERROR Parsing: " + line + " " + e.getMessage());
+ vector[j - 1] = 0.0f;
+ }
+ }
+ return vector;
+ }
+
+ /*
+ * Load vectors from w2v C binary file
+ */
+ public static W2vSpace load(String word2vecModel) {
+ return load(word2vecModel, true);
+ }
+
+ public static W2vSpace load(String word2vecModel, boolean norm) {
+ W2vSpace model = new W2vSpace();
+ try (DataInputStream ds = new DataInputStream(new BufferedInputStream(new FileInputStream(word2vecModel), 131072))) {
+ // Read header:
+ int numWords = Integer.parseInt(readString(ds));
+ int vecSize = Integer.parseInt(readString(ds));
+ for (int i = 0; i < numWords; i++) {
+ // Word:
+ String word = readString(ds);
+ // Unit Vector
+ FloatMatrix f = new FloatMatrix(readFloatVector(ds, vecSize));
+ if (norm) {
+ f = VectorMath.normalize(f);
+ }
+ model.store.put(word, f);
+ }
+ //System.out.println(String.format("Loaded %s words, vector size %s", numWords, vecSize));
+ } catch (IOException e) {
+ System.err.println("ERROR: Failed to load model: " + word2vecModel);
+ e.printStackTrace();
+ }
+ return model;
+ }
+
+ /*
+ * Read a string from the binary model (System default should be UTF-8):
+ */
+ public static String readString(DataInputStream ds) throws IOException {
+ ByteArrayOutputStream byteBuffer = new ByteArrayOutputStream();
+ while (true) {
+ byte byteValue = ds.readByte();
+ if ((byteValue != 32) && (byteValue != 10)) {
+ byteBuffer.write(byteValue);
+ } else if (byteBuffer.size() > 0) {
+ break;
+ }
+ }
+ String word = byteBuffer.toString();
+ byteBuffer.close();
+ return word;
+ }
+
+ /*
+ * Read a Vector - Array of Floats from the binary model:
+ */
+ public static float[] readFloatVector(DataInputStream ds, int vectorSize) throws IOException {
+ // Vector is an Array of Floats...
+ float[] vector = new float[vectorSize];
+ // Floats stored as 4 bytes
+ byte[] vectorBuffer = new byte[4 * vectorSize];
+ // Read the full vector in a single chunk:
+ ds.read(vectorBuffer);
+ // Parse bytes into floats
+ for (int i = 0; i < vectorSize; i++) {
+ // & with 0xFF to get unsigned byte value as int
+ int byte1 = (vectorBuffer[(i * 4) + 0] & 0xFF) << 0;
+ int byte2 = (vectorBuffer[(i * 4) + 1] & 0xFF) << 8;
+ int byte3 = (vectorBuffer[(i * 4) + 2] & 0xFF) << 16;
+ int byte4 = (vectorBuffer[(i * 4) + 3] & 0xFF) << 24;
+ // Encode the 4 byte values (0-255) above into a single int
+ // Reverse bytes for endian compatibility
+ int reverseBytes = (byte1 | byte2 | byte3 | byte4);
+ vector[i] = Float.intBitsToFloat(reverseBytes);
+ }
+ return vector;
+ }
+
+ @Override
+ public double cosineSimilarity(FloatMatrix vec1, FloatMatrix vec2) {
+ return VectorMath.cosineSimilarity(vec1, vec2);
+ }
+
+ @Override
+ public double distanceSimilarity(FloatMatrix vec1, FloatMatrix vec2) {
+ return VectorMath.distanceSimilarity(vec1, vec2);
+ }
+
+ @Override
+ public FloatMatrix additiveSentenceVector(List vectors) {
+ // return VectorMath.addFloatMatrix(vectors);
+ return VectorMath.normalize(VectorMath.addFloatMatrix(vectors));
+
+ }
+
+}
diff --git a/src/main/java/word2vec/WordSpace.java b/src/main/java/word2vec/WordSpace.java
new file mode 100755
index 0000000..3c33372
--- /dev/null
+++ b/src/main/java/word2vec/WordSpace.java
@@ -0,0 +1,32 @@
+package word2vec;
+
+import java.util.List;
+
+import util.Filters.WordFilter;
+import util.WordSim;
+
+public interface WordSpace {
+
+ public boolean contains(String word);
+
+ public T vector(String word);
+
+ public T sentenceVector(String sentence);
+
+ public T sentenceVector(String sentence, WordFilter... filters);
+
+ public T additiveSentenceVector(List vectors);
+
+ public double cosineSimilarity(String w1, String w2);
+
+ public double distanceSimilarity(String w1, String w2);
+
+ public List