From 3b73946e62967c80d8de14783de5638c623d2205 Mon Sep 17 00:00:00 2001 From: Peter Seipel Date: Thu, 17 Dec 2015 11:20:15 +0100 Subject: [PATCH 1/2] ang.generator added to pom --- ang.parent/pom.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ang.parent/pom.xml b/ang.parent/pom.xml index 4ef407c..cd4b16f 100644 --- a/ang.parent/pom.xml +++ b/ang.parent/pom.xml @@ -11,6 +11,8 @@ ../ang.htmlunit ../ang.twitter ../ang.utils + ../ang.corpusreader + ../ang.generator @@ -31,8 +33,10 @@ maven-compiler-plugin 3.3 - 1.8 - 1.8 + true + C:\Program Files\Java\jdk1.8.0_51\bin\javac.exe + 1.7 + 1.7 From a74d86ce5076269e21b42c3b52ec474e650aa794 Mon Sep 17 00:00:00 2001 From: Peter Seipel Date: Thu, 17 Dec 2015 11:21:12 +0100 Subject: [PATCH 2/2] generator renamed as ang.generator --- generator/pom.xml | 14 +- .../java/ang/generator/generator/App.java | 18 ++- .../ang/generator/generator/Generator.java | 133 ++++++++++-------- .../java/ang/generator/generator/ToText.java | 2 +- 4 files changed, 98 insertions(+), 69 deletions(-) diff --git a/generator/pom.xml b/generator/pom.xml index 1ebdef8..2f34ce6 100644 --- a/generator/pom.xml +++ b/generator/pom.xml @@ -2,12 +2,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - ang.generator - generator - 0.0.1-SNAPSHOT + + de.uni_koeln.spinfo.ang + ang.parent + 0.0.1-SNAPSHOT + ../ang.parent + + + ang.generator jar - generator http://maven.apache.org @@ -15,6 +19,8 @@ + + junit junit diff --git a/generator/src/main/java/ang/generator/generator/App.java b/generator/src/main/java/ang/generator/generator/App.java index 95a4ffd..e747d1f 100644 --- a/generator/src/main/java/ang/generator/generator/App.java +++ b/generator/src/main/java/ang/generator/generator/App.java @@ -6,14 +6,20 @@ public class App { public static void main( String[] args ) throws ClassNotFoundException, IOException { - +/* english Tokens - 109583 + * davon Staemme - 46801 + * prafixe - 144 + * suffixe - 117 + * ConcatList Praefixe - 6,7 Millionen + * ConcatList Sufffixe - 5,5 Millionen + * Zirkum - 788 Millionen + * */ // Generator g = new Generator(); // g.generateList(); -// g.generateSufPrae(); -// +// g.generateSufPrae(); ToText t = new ToText(); - t.writeToText("anglizismenSuf"); - t.writeToText("anglizismenPrae"); - + t.writeToText("anglizismenStemms"); +// t.writeToText("anglizismenPrae"); +// t.writeToText("anglizismenSuf"); } } diff --git a/generator/src/main/java/ang/generator/generator/Generator.java b/generator/src/main/java/ang/generator/generator/Generator.java index 64ee66b..8783619 100644 --- a/generator/src/main/java/ang/generator/generator/Generator.java +++ b/generator/src/main/java/ang/generator/generator/Generator.java @@ -7,143 +7,151 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; import org.tartarus.snowball.SnowballStemmer; import org.tartarus.snowball.ext.englishStemmer; +import org.tartarus.snowball.ext.germanStemmer; import de.uni_koeln.spinfo.ang.utils.FileUtils; public class Generator { -// liste der englischen Tokens + // liste der englischen Tokens private List englInput; -// Liste der deutschen Präfixe + // Liste der deutschen Präfixe private List praefix; -// Liste der deutschen Suffixe + // Liste der deutschen Suffixe private List suffix; - private List stopWords; - private List englStemms; +// private List stopWords; + private HashSet englStemms; private List anglizismenPrae = new ArrayList(); private List anglizismenSuf = new ArrayList(); private List anglizismenZirk = new LinkedList(); - /* - * Füllt die Listen mit Inhalt. - * Liest Input Text-Files mit Präfixen, Suffixen, - * englischen Tokens und stopwords aus und serialisiert diese. - * */ - + * Füllt die Listen mit Inhalt. Liest Input Text-Files mit Präfixen, + * Suffixen, englischen Tokens und stopwords aus und serialisiert diese. + */ + public void generateList() throws IOException { HashSet p = new HashSet(); - p.addAll(FileUtils.fileToList(FileUtils.inputPath - + "angGenerator/affix/praefix.txt")); + p.addAll(FileUtils.fileToList(FileUtils.inputPath + "angGenerator/affix/praefix.txt")); System.out.println("Anzahl der Praefixe: " + p.size()); HashSet s = new HashSet(); - s.addAll(FileUtils.fileToList(FileUtils.inputPath - + "angGenerator/affix/suffixe.txt")); + s.addAll(FileUtils.fileToList(FileUtils.inputPath + "angGenerator/affix/suffixe.txt")); System.out.println("Anzahl der Suffixe: " + s.size()); HashSet e = new HashSet(); - e.addAll(FileUtils.fileToList(FileUtils.inputPath - + "angGenerator/corncob_lowercase.txt")); - System.out.println("Anzahl der englischen Worte: " + e.size()); - - stopWords = FileUtils.fileToList(FileUtils.inputPath - + "angGenerator/stopwords.txt"); - System.out.println("Anzahl der Stopwords: " + stopWords.size()); - - e.removeAll(stopWords); - System.out.println("nach Entfernung der StopWords hat englInput: " - + e.size()); + e.addAll(FileUtils.fileToList(FileUtils.inputPath + "angGenerator/silwordsEn.txt")); + System.out.println("Anzahl der englischen Worteintraege: " + e.size()); + + // stopWords = FileUtils.fileToList(FileUtils.inputPath + // + "angGenerator/stopwords.txt"); + // System.out.println("Anzahl der Stopwords: " + stopWords.size()); + // + // e.removeAll(stopWords); + // System.out.println("nach Entfernung der StopWords hat englInput: " + // + e.size()); + List englInput = new ArrayList(e); List praefix = new ArrayList(p); List suffix = new ArrayList(s); - FileUtils.serializeList(englInput, "generator/lists/englishInput"); FileUtils.serializeList(praefix, "generator/lists/praefix"); FileUtils.serializeList(suffix, "generator/lists/suffix"); - } @SuppressWarnings("unchecked") private void read() throws ClassNotFoundException, IOException { - FileInputStream fis = new FileInputStream(FileUtils.outputPath - + "/generator/lists/englishInput"); + FileInputStream fis = new FileInputStream(FileUtils.outputPath + "/generator/lists/englishInput"); ObjectInputStream ois = new ObjectInputStream(fis); englInput = (List) ois.readObject(); ois.close(); - FileInputStream fis1 = new FileInputStream(FileUtils.outputPath - + "/generator/lists/praefix"); + FileInputStream fis1 = new FileInputStream(FileUtils.outputPath + "/generator/lists/praefix"); ObjectInputStream ois1 = new ObjectInputStream(fis1); praefix = (List) ois1.readObject(); ois1.close(); - FileInputStream fis2 = new FileInputStream(FileUtils.outputPath - + "/generator/lists/suffix"); + FileInputStream fis2 = new FileInputStream(FileUtils.outputPath + "/generator/lists/suffix"); ObjectInputStream ois2 = new ObjectInputStream(fis2); suffix = (List) ois2.readObject(); ois2.close(); - } - private List getStems() { + private HashSet getStems() throws IOException { SnowballStemmer stemmer = new englishStemmer(); - List result = new ArrayList(); + HashSet result = new HashSet(); for (String t : englInput) { - stemmer.setCurrent(t); - stemmer.stem(); - result.add(stemmer.getCurrent()); + if (t.length() > 1) { + stemmer.setCurrent(t); + stemmer.stem(); + String string = stemmer.getCurrent(); + if(string.length()>4 && !string.isEmpty() && string != null){ + result.add(string);} + } + } +// System.out.println("Anzahl der englischen WortStaemme: " + result.size()); + List list = new ArrayList(); + for (String string : result) { + list.add(string); } - System.out.println(result.size()); + + FileUtils.serializeList(list, "anglizismenStemms"); + System.out.println("serialisierte Staemme: " + list.size()); return result; } - /*Setzt Stämme und Suffixe/Praefixe zusammen und fügt diese der Liste anglizismenSuf/anglizismenPrae hinzu*/ - + /* + * Setzt Stämme und Suffixe/Praefixe zusammen und fügt diese der Liste + * anglizismenSuf/anglizismenPrae hinzu + */ + public void generateSufPrae() throws ClassNotFoundException, IOException { read(); - + System.out.println("Anzahl der Suffixe " + suffix.size()); anglizismenSuf = generateConcatList(suffix, false); +// System.out.println("Anzahl der Praefixe" + praefix.size()); anglizismenPrae = generateConcatList(praefix, true); FileUtils.serializeList(anglizismenSuf, "anglizismenSuf"); FileUtils.serializeList(anglizismenPrae, "anglizismenPrae"); } - - private List generateConcatList(List affixList, boolean putInFront){ + + private List generateConcatList(List affixList, boolean putInFront) throws IOException { List anglizismenAff = new ArrayList(); - + englStemms = getStems(); - + for (String englStrig : englStemms) { - + for (String praString : affixList) { String angP = new String(); - if(putInFront){ - angP = praString + englStrig;} - else{ + if(!praString.isEmpty() && praString != null){ + if (putInFront) { + + angP = praString + englStrig; + } else { angP = englStrig + praString; } anglizismenAff.add(angP); - } + }} } - + return anglizismenAff; } - public void generateZirkumfix() throws IOException, ClassNotFoundException { read(); anglizismenSuf = generateConcatList(suffix, false); - int i=0; - + int i = 0; + for (String stringSuf : anglizismenSuf) { System.out.println(stringSuf); for (String prae : praefix) { @@ -154,10 +162,19 @@ public void generateZirkumfix() throws IOException, ClassNotFoundException { } System.out.println(i); i++; - + } System.out.println(anglizismenZirk.size()); FileUtils.serializeList(anglizismenZirk, "anglizismenZirk"); } + public void testStemm(String string) { + SnowballStemmer germStemmer = new germanStemmer(); + System.out.println(string); + germStemmer.setCurrent(string); + germStemmer.stem(); + String result = germStemmer.getCurrent(); + System.out.println(result); + } + } diff --git a/generator/src/main/java/ang/generator/generator/ToText.java b/generator/src/main/java/ang/generator/generator/ToText.java index 9aa1e0d..b375654 100644 --- a/generator/src/main/java/ang/generator/generator/ToText.java +++ b/generator/src/main/java/ang/generator/generator/ToText.java @@ -22,7 +22,7 @@ public void writeToText(String input) throws IOException, ClassNotFoundException text = (List) ois.readObject(); ois.close(); - StringBuilder sb = new StringBuilder(); + StringBuilder sb = new StringBuilder(""); sb.append(" \n "); for (String string : text) { sb.append(string);