diff --git a/.gitignore b/.gitignore index e3b6fa2..de7c187 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea /input /output* +/target \ No newline at end of file diff --git a/2017-big-data.iml b/2017-big-data.iml new file mode 100644 index 0000000..bbfee7d --- /dev/null +++ b/2017-big-data.iml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/answers.txt b/answers.txt index 00fa802..cee0e51 100644 --- a/answers.txt +++ b/answers.txt @@ -1,3 +1,3 @@ -2. is 126420 -3. 41.602 -4. french 5742 +2. was 18391755 +3. 37.164249490179486 +4. October 1072615 diff --git a/pom.xml b/pom.xml index 16acaa5..d454bf2 100644 --- a/pom.xml +++ b/pom.xml @@ -10,13 +10,13 @@ UTF-8 - 2.6.0 + 0.20.2 org.apache.hadoop - hadoop-client + hadoop-core ${hadoop.version} @@ -29,7 +29,7 @@ - pritykovskaya.WordCount + afilippo.Tasks @@ -38,8 +38,8 @@ org.apache.maven.plugins maven-compiler-plugin - 1.6 - 1.6 + 1.8 + 1.8 diff --git a/report.txt b/report.txt new file mode 100644 index 0000000..5678cd8 --- /dev/null +++ b/report.txt @@ -0,0 +1,14 @@ +Задание 1 +2900,292s - Считаем wordcount + +Задание 2 +36,356s - Сортируем слова +6,076s - Вывод только 7-го слова + +Задание 3 +78,431s - Считаем процент стоп-слов + +Задание 4 +78,385s - Находим имена и кладем их в папку +6,072s - Сортируем имена по убыванию частоты +1,048s - Выводим только 5-е имя (5 => 0 to 4) \ No newline at end of file diff --git a/src/main/java/afilippo/NValue.java b/src/main/java/afilippo/NValue.java new file mode 100644 index 0000000..b289fd0 --- /dev/null +++ b/src/main/java/afilippo/NValue.java @@ -0,0 +1,117 @@ +package afilippo; + +import java.io.IOException; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class NValue extends Configured implements Tool { + private static int valueNumber; + private static IntWritable ONE = new IntWritable(1); + + static class MyMapper extends Mapper{ + private int count; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + super.setup(context); + count = 0; + } + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + int inputCount = Integer.valueOf(line.substring(0, pos)); + String inputString = line.substring(pos+1); + + if (count <= valueNumber){ + context.write(ONE, new TextWithCountWriteble(inputString, inputCount)); + count++; + } + } + } + + static class MyReducer extends Reducer{ + private SortedSet setOfTextWithCount; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + super.setup(context); + setOfTextWithCount = new TreeSet<>(); + } + + @Override + protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException { + values.forEach((textWithCountWriteble -> { + setOfTextWithCount.add(textWithCountWriteble.clone()); + if (setOfTextWithCount.size() > valueNumber + 1){ + setOfTextWithCount.remove(setOfTextWithCount.last()); + } + })); + } + + @Override + protected void cleanup(Context context) throws IOException, InterruptedException { + super.cleanup(context); + + int i = 0; + for (TextWithCountWriteble textWithCountWriteble : setOfTextWithCount){ + if (i == valueNumber){ + context.write(new Text(textWithCountWriteble.getText()), new IntWritable(textWithCountWriteble.getCount())); + break; + } + i++; + } + } + } + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "NValue"); + + valueNumber = Integer.valueOf(args[2]); + + job.setJarByClass(NValue.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapperClass(MyMapper.class); + job.setReducerClass(MyReducer.class); + + job.setMapOutputKeyClass(IntWritable.class); + job.setMapOutputValueClass(TextWithCountWriteble.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(String[] args) throws Exception{ + final int returnCode = ToolRunner.run(new Configuration(), new NValue(), args); + System.exit(returnCode); + + } +} diff --git a/src/main/java/afilippo/NamesPercent.java b/src/main/java/afilippo/NamesPercent.java new file mode 100644 index 0000000..a83915f --- /dev/null +++ b/src/main/java/afilippo/NamesPercent.java @@ -0,0 +1,99 @@ +package afilippo; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class NamesPercent extends Configured implements Tool { + private static final Pattern namePattern = Pattern.compile("^[A-Z][a-z0-9]*$"); + + static class MyMapper extends Mapper{ + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + String inputString = line.substring(0, pos); + int inputCount = Integer.valueOf(line.substring(pos+1)); + + context.write(new Text(inputString.toLowerCase()), new TextWithCountWriteble(inputString, inputCount)); + } + } + + static class MyReducer extends Reducer{ + + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + int sumAllForms = 0; + int rightFormCount = 0; + String rightFormText = null; + + for (final TextWithCountWriteble value : values){ + sumAllForms += value.getCount(); + + if (rightFormText == null){ + Matcher matcher = namePattern.matcher(value.getText()); + if (matcher.matches()){ + rightFormText = value.getText(); + rightFormCount = value.getCount(); + } + } + } + + if (rightFormText == null){ + return; + } + + if (rightFormCount / (double)sumAllForms >= 0.995){ + context.write(new Text(rightFormText), new IntWritable(rightFormCount)); + } + } + } + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "NamesPercent"); + job.setJarByClass(NamesPercent.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapperClass(MyMapper.class); + job.setReducerClass(MyReducer.class); + + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(TextWithCountWriteble.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(String[] args) throws Exception{ + final int returnCode = ToolRunner.run(new Configuration(), new NamesPercent(), args); + System.exit(returnCode); + + } +} diff --git a/src/main/java/afilippo/StopWordsCount.java b/src/main/java/afilippo/StopWordsCount.java new file mode 100644 index 0000000..bf801f5 --- /dev/null +++ b/src/main/java/afilippo/StopWordsCount.java @@ -0,0 +1,139 @@ +package afilippo; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class StopWordsCount extends Configured implements Tool { + private static Set stopWords; + + private static void setStopWords(String stopWordsFile){ + stopWords = new HashSet<>(); + + try { + Files + .lines(Paths.get(stopWordsFile), StandardCharsets.UTF_8) + .forEach(stopWords::add); + } catch (IOException e) { + throw new ExceptionInInitializerError(e); + } + } + + enum MY_COUNTERS { + WORDS_COUNT, + STOP_WORDS_COUNT + } + + + static class MyMapper extends Mapper{ + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + String inputString = line.substring(0, pos); + int inputCount = Integer.valueOf(line.substring(pos+1)); + + context.write(new Text(inputString.toLowerCase()), new IntWritable(inputCount)); + } + } + + static class MyCombiner extends Reducer{ + + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + int valuesCount = 0; + + for (final IntWritable value : values) { + valuesCount += value.get(); + } + + context.write(key, new IntWritable(valuesCount)); + } + } + + static class MyReducer extends Reducer{ + + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + boolean isStopWord = stopWords.contains(key.toString()); + + for (final IntWritable value : values) { + context.getCounter(MY_COUNTERS.WORDS_COUNT).increment(value.get()); + + if (isStopWord){ + context.getCounter(MY_COUNTERS.STOP_WORDS_COUNT).increment(value.get()); + } + } + } + } + + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "StopWordsCount"); + + // третьим аргументом приходит файл стоп-слов + setStopWords(args[2]); + + job.setJarByClass(StopWordsCount.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapperClass(MyMapper.class); + job.setReducerClass(MyReducer.class); + job.setCombinerClass(MyCombiner.class); + + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(IntWritable.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + // читаем из выходной папки wordcount + FileInputFormat.addInputPath(job, new Path(args[0])); + + // в выходную папку положится пустой файл (в context ничего не пишем) + // и в этой же папке создастся файл "output", в котором будет результат работы программы + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + boolean success = job.waitForCompletion(true); + + long stopWordsCount = job.getCounters().findCounter(MY_COUNTERS.STOP_WORDS_COUNT).getValue(); + long wordsCount = job.getCounters().findCounter(MY_COUNTERS.WORDS_COUNT).getValue(); + double percent = stopWordsCount / (double)wordsCount * 100; + + File file = new File(args[1] + "/output"); + Files.write(file.toPath(), String.valueOf(percent).getBytes()); + + return success ? 0 : 1; + } + + public static void main(String[] args) throws Exception{ + final int returnCode = ToolRunner.run(new Configuration(), new StopWordsCount(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/afilippo/Tasks.java b/src/main/java/afilippo/Tasks.java new file mode 100644 index 0000000..b8c117f --- /dev/null +++ b/src/main/java/afilippo/Tasks.java @@ -0,0 +1,143 @@ +package afilippo; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class Tasks { + private static final String FILE_REPORT = "report.txt"; + public static void main(String[] args) throws Exception { + String inputDirectory = "input"; + String wordcountDirectory = "output-wordcount"; + String orderedDirectory = "output-ordered"; + String task2OutputDirectory = "output-7-word"; + String task3OutputDirectory = "output-stopwordscount"; + String namesDirectory = "output-names"; + String namesOrderedDirectory = "output-ordered-names"; + String task4OutputDirectory = "output-5-name"; + + // + // ─█───███─████──███──████─█──█─█──█─███ + // ██─────█─█──█──█─█──█──█─█──█─█──█─█ + // ─█───███─████──█─█──████─████─█─██─███ + // ─█─────█─█──█─█████─█──█─█──█─██─█─█ + // ─█───███─█──█─█───█─█──█─█──█─█──█─███ + // + // Считаем wordcount + // Это база для остальных заданий + + Report task1 = new Report("Задание 1") + .addAction( + "Считаем wordcount", + calculateTaskTime(new WordCount(), inputDirectory, wordcountDirectory) + ); + + // + // ████───███─████──███──████─█──█─█──█─███ + // █──█─────█─█──█──█─█──█──█─█──█─█──█─█ + // ──██───███─████──█─█──████─████─█─██─███ + // ██───────█─█──█─█████─█──█─█──█─██─█─█ + // ████───███─█──█─█───█─█──█─█──█─█──█─███ + // + // Вывести 7-ое по популярности слово + + Report task2 = new Report("Задание 2") + .addAction( + "Сортируем слова", + calculateTaskTime(new WordsOrder(), wordcountDirectory, orderedDirectory) + ) + .addAction( + "Вывод только 7-го слова", + calculateTaskTime(new NValue(), orderedDirectory, task2OutputDirectory, "6") + ); + + // + // ███───███─████──███──████─█──█─█──█─███ + // ──█─────█─█──█──█─█──█──█─█──█─█──█─█ + // ███───███─████──█─█──████─████─█─██─███ + // ──█─────█─█──█─█████─█──█─█──█─██─█─█ + // ███───███─█──█─█───█─█──█─█──█─█──█─███ + // + // Посчитать процент стоп-слов + + Report task3 = new Report("Задание 3") + .addAction( + "Считаем процент стоп-слов", + calculateTaskTime(new StopWordsCount(), wordcountDirectory, task3OutputDirectory, "stop_words_en.txt") + ); + + // + // █──────███─████──███──████─█──█─█──█─███ + // █──█─────█─█──█──█─█──█──█─█──█─█──█─█ + // ████───███─████──█─█──████─████─█─██─███ + // ───█─────█─█──█─█████─█──█─█──█─██─█─█ + // ───█───███─█──█─█───█─█──█─█──█─█──█─███ + // + // Посчитать имена и вывести пятое по популярности + // Находим имена и кладем их куда нибудь + Report task4 = new Report("Задание 4") + .addAction( + "Находим имена и кладем их в папку", + calculateTaskTime(new NamesPercent(), wordcountDirectory, namesDirectory) + ) + .addAction( + "Сортируем имена по убыванию частоты", + calculateTaskTime(new WordsOrder(), namesDirectory, namesOrderedDirectory) + ) + .addAction( + "Выводим только 5-е имя (5 => 0 to 4)", + calculateTaskTime(new NValue(), namesOrderedDirectory, task4OutputDirectory, "4") + ); + + + Files.write( + Paths.get(FILE_REPORT), + String.join( + "\r\n\r\n", + task1.getString(), + task2.getString(), + task3.getString(), + task4.getString() + ).getBytes() + ); + } + + private static double calculateTaskTime(Tool tool, String... parameters) throws Exception{ + long from = System.currentTimeMillis(); + + ToolRunner.run(new Configuration(), tool, parameters); + + long to = System.currentTimeMillis(); + return (to - from)/(double)1000; + } + + private static class Report { + private List stringList; + + Report(String name) { + stringList = new ArrayList<>(); + stringList.add(name); + } + + Report addAction(String description, double time){ + stringList.add(formatTimeWithText(time, description)); + + return this; + } + + private String formatTimeWithText(double time, String text){ + return String.format("%.3f", time) + "s" + " - " + text; + } + + public String getString() throws IOException { + return String.join("\r\n", stringList); + } + } +} \ No newline at end of file diff --git a/src/main/java/afilippo/TextWithCountWriteble.java b/src/main/java/afilippo/TextWithCountWriteble.java new file mode 100644 index 0000000..a7b1d2a --- /dev/null +++ b/src/main/java/afilippo/TextWithCountWriteble.java @@ -0,0 +1,76 @@ +package afilippo; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.WritableComparable; + +class TextWithCountWriteble implements WritableComparable, Cloneable { + private String text; + private int count; + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeInt(count); + dataOutput.writeUTF(text); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + count = dataInput.readInt(); + text = dataInput.readUTF(); + } + + public String getText() { + return text; + } + + public int getCount() { + return count; + } + + TextWithCountWriteble(){ + // should be + } + + TextWithCountWriteble(String text, int count) { + this.text = text; + this.count = count; + } + + @Override + public boolean equals(Object other) { + if (other == null){ + return false; + } + if (other == this){ + return true; + } + if (!(other instanceof TextWithCountWriteble)){ + return false; + } + TextWithCountWriteble otherMyClass = (TextWithCountWriteble)other; + if (otherMyClass.count != count){ + return false; + } + if (!otherMyClass.text.equals(text)){ + return false; + } + return true; + } + + @Override + protected TextWithCountWriteble clone() { + return new TextWithCountWriteble(text, count); + } + + @Override + public int compareTo(TextWithCountWriteble o) { + if (equals(o)){ + return 0; + } + int intCompare = Integer.compare(count, o.count); + return (intCompare == 0) ? Integer.compare(this.hashCode(), o.hashCode()) : -intCompare; + } +} \ No newline at end of file diff --git a/src/main/java/afilippo/WordCount.java b/src/main/java/afilippo/WordCount.java new file mode 100644 index 0000000..7492bb3 --- /dev/null +++ b/src/main/java/afilippo/WordCount.java @@ -0,0 +1,77 @@ +package afilippo; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class WordCount extends Configured implements Tool { + + static class MyMapper extends Mapper{ + private static final IntWritable ONE = new IntWritable(1); + private final transient Text word = new Text(); + + @Override + protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + final String[] line = value.toString().trim().split("\t", 2); + String text = line[1].replaceAll("^\\W+|\\W+$", ""); + String[] words = text.split("\\W*\\s+\\W*"); + for (int i = 0; i < words.length; i++){ + word.set(words[i]); + context.write(word, ONE); + } + } + } + + static class MyReducer extends Reducer{ + + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + int sum = 0; + for (final IntWritable val : values){ + sum += val.get(); + } + context.write(key, new IntWritable(sum)); + } + } + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "Word Count"); + job.setJarByClass(WordCount.class); + + job.setMapperClass(MyMapper.class); + job.setReducerClass(MyReducer.class); + job.setCombinerClass(MyReducer.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(String[] args) throws Exception{ + final int returnCode = ToolRunner.run(new Configuration(), new WordCount(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/afilippo/WordsOrder.java b/src/main/java/afilippo/WordsOrder.java new file mode 100644 index 0000000..897cd88 --- /dev/null +++ b/src/main/java/afilippo/WordsOrder.java @@ -0,0 +1,86 @@ +package afilippo; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class WordsOrder extends Configured implements Tool { + + static class MyMapper extends Mapper{ + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + context.write(new IntWritable(Integer.valueOf(line.substring(pos+1))), new Text(line.substring(0, pos))); + } + } + + static class MyReducer extends Reducer{ + + @Override + protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException { + for (final Text value : values){ + context.write(key, value); + } + } + } + + static class MyDescFreqComparator extends WritableComparator{ + protected MyDescFreqComparator() { + super(IntWritable.class); + } + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + return -Integer.compare(readInt(b1, s1), readInt(b2, s2)); + } + } + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "Words Order"); + job.setJarByClass(WordsOrder.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapperClass(MyMapper.class); + job.setReducerClass(MyReducer.class); + + job.setMapOutputKeyClass(IntWritable.class); + job.setMapOutputValueClass(Text.class); + + job.setOutputKeyClass(IntWritable.class); + job.setOutputValueClass(Text.class); + + job.setSortComparatorClass(MyDescFreqComparator.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(String[] args) throws Exception{ + final int returnCode = ToolRunner.run(new Configuration(), new WordsOrder(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/pritykovskaya/WordCount.java b/src/main/java/pritykovskaya/WordCount.java deleted file mode 100644 index b047511..0000000 --- a/src/main/java/pritykovskaya/WordCount.java +++ /dev/null @@ -1,80 +0,0 @@ -package pritykovskaya; - - -import java.io.IOException; -import java.util.StringTokenizer; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; - - -public class WordCount extends Configured implements Tool { - - public static class MyMapper extends Mapper { - private static final IntWritable ONE = new IntWritable(1); - private final transient Text word = new Text(); - - @Override public void map(final LongWritable key, final Text value, final Context context) - throws IOException, InterruptedException { - final String line = value.toString(); - final StringTokenizer tokenizer = new StringTokenizer(line); - while (tokenizer.hasMoreTokens()) { - word.set(tokenizer.nextToken()); - context.write(word, ONE); - } - } - } - - - public static class MyReducer extends Reducer { - - @Override - public void reduce(final Text key, final Iterable values, final Context context) - throws IOException, InterruptedException { - int sum = 0; - for (final IntWritable val : values) { - sum += val.get(); - } - context.write(key, new IntWritable(sum)); - } - } - - - @Override public int run(final String[] args) throws Exception { - final Configuration conf = this.getConf(); - final Job job = Job.getInstance(conf, "Word Count"); - job.setJarByClass(WordCount.class); - - job.setMapperClass(MyMapper.class); - job.setReducerClass(MyReducer.class); - - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(IntWritable.class); - - job.setInputFormatClass(TextInputFormat.class); - job.setOutputFormatClass(TextOutputFormat.class); - - FileInputFormat.addInputPath(job, new Path(args[0])); - FileOutputFormat.setOutputPath(job, new Path(args[1])); - - return job.waitForCompletion(true) ? 0 : 1; - } - - public static void main(final String[] args) throws Exception { - final int returnCode = ToolRunner.run(new Configuration(), new WordCount(), args); - System.exit(returnCode); - } -} diff --git a/stop_words_en.txt b/stop_words_en.txt new file mode 100644 index 0000000..b7454b0 --- /dev/null +++ b/stop_words_en.txt @@ -0,0 +1,319 @@ +a +about +above +across +after +afterwards +again +against +all +almost +alone +along +already +also +although +always +am +among +amongst +amoungst +amount +an +and +another +any +anyhow +anyone +anything +anyway +anywhere +are +around +as +at +back +be +became +because +become +becomes +becoming +been +before +beforehand +behind +being +below +beside +besides +between +beyond +bill +both +bottom +but +by +call +can +cannot +cant +co +computer +con +could +couldnt +cry +de +describe +detail +do +done +down +due +during +each +eg +eight +either +eleven +else +elsewhere +empty +enough +etc +even +ever +every +everyone +everything +everywhere +except +few +fifteen +fify +fill +find +fire +first +five +for +former +formerly +forty +found +four +from +front +full +further +get +give +go +had +has +hasnt +have +he +hence +her +here +hereafter +hereby +herein +hereupon +hers +herse" +him +himse" +his +how +however +hundred +i +ie +if +in +inc +indeed +interest +into +is +it +its +itse" +keep +last +latter +latterly +least +less +ltd +made +many +may +me +meanwhile +might +mill +mine +more +moreover +most +mostly +move +much +must +my +myse" +name +namely +neither +never +nevertheless +next +nine +no +nobody +none +noone +nor +not +nothing +now +nowhere +of +off +often +on +once +one +only +onto +or +other +others +otherwise +our +ours +ourselves +out +over +own +part +per +perhaps +please +put +rather +re +same +see +seem +seemed +seeming +seems +serious +several +she +should +show +side +since +sincere +six +sixty +so +some +somehow +someone +something +sometime +sometimes +somewhere +still +such +system +take +ten +than +that +the +their +them +themselves +then +thence +there +thereafter +thereby +therefore +therein +thereupon +these +they +thick +thin +third +this +those +though +three +through +throughout +thru +thus +to +together +too +top +toward +towards +twelve +twenty +two +un +under +until +up +upon +us +very +via +was +we +well +were +what +whatever +when +whence +whenever +where +whereafter +whereas +whereby +wherein +whereupon +wherever +whether +which +while +whither +who +whoever +whole +whom +whose +why +will +with +within +without +would +yet +you +your +yours +yourself +yourselves