diff --git a/pom.xml b/pom.xml index 16acaa5..81f6910 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ 4.0.0 - pritykovskaya + mburmistrov 2017-big-data jar 1.0-SNAPSHOT @@ -29,7 +29,7 @@ - pritykovskaya.WordCount + mburmistrov.task1.WordCount @@ -38,8 +38,8 @@ org.apache.maven.plugins maven-compiler-plugin - 1.6 - 1.6 + 1.8 + 1.8 diff --git a/resources/stop_words_en.txt b/resources/stop_words_en.txt new file mode 100644 index 0000000..b7454b0 --- /dev/null +++ b/resources/stop_words_en.txt @@ -0,0 +1,319 @@ +a +about +above +across +after +afterwards +again +against +all +almost +alone +along +already +also +although +always +am +among +amongst +amoungst +amount +an +and +another +any +anyhow +anyone +anything +anyway +anywhere +are +around +as +at +back +be +became +because +become +becomes +becoming +been +before +beforehand +behind +being +below +beside +besides +between +beyond +bill +both +bottom +but +by +call +can +cannot +cant +co +computer +con +could +couldnt +cry +de +describe +detail +do +done +down +due +during +each +eg +eight +either +eleven +else +elsewhere +empty +enough +etc +even +ever +every +everyone +everything +everywhere +except +few +fifteen +fify +fill +find +fire +first +five +for +former +formerly +forty +found +four +from +front +full +further +get +give +go +had +has +hasnt +have +he +hence +her +here +hereafter +hereby +herein +hereupon +hers +herse" +him +himse" +his +how +however +hundred +i +ie +if +in +inc +indeed +interest +into +is +it +its +itse" +keep +last +latter +latterly +least +less +ltd +made +many +may +me +meanwhile +might +mill +mine +more +moreover +most +mostly +move +much +must +my +myse" +name +namely +neither +never +nevertheless +next +nine +no +nobody +none +noone +nor +not +nothing +now +nowhere +of +off +often +on +once +one +only +onto +or +other +others +otherwise +our +ours +ourselves +out +over +own +part +per +perhaps +please +put +rather +re +same +see +seem +seemed +seeming +seems +serious +several +she +should +show +side +since +sincere +six +sixty +so +some +somehow +someone +something +sometime +sometimes +somewhere +still +such +system +take +ten +than +that +the +their +them +themselves +then +thence +there +thereafter +thereby +therefore +therein +thereupon +these +they +thick +thin +third +this +those +though +three +through +throughout +thru +thus +to +together +too +top +toward +towards +twelve +twenty +two +un +under +until +up +upon +us +very +via +was +we +well +were +what +whatever +when +whence +whenever +where +whereafter +whereas +whereby +wherein +whereupon +wherever +whether +which +while +whither +who +whoever +whole +whom +whose +why +will +with +within +without +would +yet +you +your +yours +yourself +yourselves diff --git a/src/main/java/mburmistrov/TaskManager.java b/src/main/java/mburmistrov/TaskManager.java new file mode 100644 index 0000000..f7e9ec1 --- /dev/null +++ b/src/main/java/mburmistrov/TaskManager.java @@ -0,0 +1,25 @@ +package mburmistrov; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class TaskManager { + private List stringList; + + TaskManager(String taskName) { + stringList = new ArrayList<>(); + + stringList.add(taskName); + } + + TaskManager performTask(String description, double time){ + stringList.add(String.format("%.3f", time) + "s" + " - " + description); + + return this; + } + + public String getString() throws IOException { + return String.join("\r\n", stringList); + } +} \ No newline at end of file diff --git a/src/main/java/mburmistrov/TaskRunner.java b/src/main/java/mburmistrov/TaskRunner.java new file mode 100644 index 0000000..c5317da --- /dev/null +++ b/src/main/java/mburmistrov/TaskRunner.java @@ -0,0 +1,47 @@ +package mburmistrov; + +import mburmistrov.task1.WordCount; +import mburmistrov.task2.WordCertainPosition; +import mburmistrov.task2.WordSort; +import mburmistrov.task3.StopWordProportion; +import mburmistrov.task4.NameWordProportion; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.conf.Configuration; + +import java.nio.file.Files; +import java.nio.file.Paths; + +public class TaskRunner { + private static double getTaskTime(Tool tool, String... parameters) throws Exception{ + long timeFrom = System.currentTimeMillis(); + + ToolRunner.run(new Configuration(), tool, parameters); + + long timeTo = System.currentTimeMillis(); + return (timeTo - timeFrom) / (double) 1000; + } + + public static void main(String[] args) throws Exception { + + // task 1 + TaskManager t1 = new TaskManager("task 1").performTask("word count", getTaskTime(new WordCount(),"input", "output/1_wordCount")); + + // task 2 + TaskManager t2 = new TaskManager("task 2") + .performTask("word sort", getTaskTime(new WordSort(),"output/1_wordCount", "output/2_wordSort")) + .performTask("output seventh word", getTaskTime(new WordCertainPosition(),"output/2_wordSort", "output/2_seventhWord", "6")); + + // task 3 + TaskManager t3 = new TaskManager("task 3").performTask("stop word proportion", getTaskTime(new StopWordProportion(),"output/1_wordCount", "output/3_stopWordProportion", "resources/stop_words_en.txt")); + + // task 4 + TaskManager t4 = new TaskManager("task 4") + .performTask("name word proportion", getTaskTime(new NameWordProportion(),"output/1_wordCount", "output/4_nameWordProportion")) + .performTask("name word sort", getTaskTime(new WordSort(),"output/4_nameWordProportion", "output/4_nameWordSort")) + .performTask("name fifth", getTaskTime(new WordCertainPosition(),"output/4_nameWordSort", "output/4_nameFifth", "4")); + + Files.write(Paths.get("output/timeReport"), String.join("\r\n\r\n", t1.getString(), t2.getString(), t3.getString(), t4.getString()).getBytes()); + } + +} diff --git a/src/main/java/pritykovskaya/WordCount.java b/src/main/java/mburmistrov/task1/WordCount.java similarity index 51% rename from src/main/java/pritykovskaya/WordCount.java rename to src/main/java/mburmistrov/task1/WordCount.java index b047511..7145312 100644 --- a/src/main/java/pritykovskaya/WordCount.java +++ b/src/main/java/mburmistrov/task1/WordCount.java @@ -1,19 +1,13 @@ -package pritykovskaya; - - -import java.io.IOException; -import java.util.StringTokenizer; +package mburmistrov.task1; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; + import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; @@ -22,44 +16,14 @@ public class WordCount extends Configured implements Tool { - - public static class MyMapper extends Mapper { - private static final IntWritable ONE = new IntWritable(1); - private final transient Text word = new Text(); - - @Override public void map(final LongWritable key, final Text value, final Context context) - throws IOException, InterruptedException { - final String line = value.toString(); - final StringTokenizer tokenizer = new StringTokenizer(line); - while (tokenizer.hasMoreTokens()) { - word.set(tokenizer.nextToken()); - context.write(word, ONE); - } - } - } - - - public static class MyReducer extends Reducer { - - @Override - public void reduce(final Text key, final Iterable values, final Context context) - throws IOException, InterruptedException { - int sum = 0; - for (final IntWritable val : values) { - sum += val.get(); - } - context.write(key, new IntWritable(sum)); - } - } - - @Override public int run(final String[] args) throws Exception { final Configuration conf = this.getConf(); final Job job = Job.getInstance(conf, "Word Count"); job.setJarByClass(WordCount.class); - job.setMapperClass(MyMapper.class); - job.setReducerClass(MyReducer.class); + job.setMapperClass(WordCountMapper.class); + job.setReducerClass(WordCountReducer.class); + job.setCombinerClass(WordCountReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); diff --git a/src/main/java/mburmistrov/task1/WordCountMapper.java b/src/main/java/mburmistrov/task1/WordCountMapper.java new file mode 100644 index 0000000..90bb14e --- /dev/null +++ b/src/main/java/mburmistrov/task1/WordCountMapper.java @@ -0,0 +1,30 @@ +package mburmistrov.task1; + +import java.io.IOException; +import java.util.StringTokenizer; + +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; + + +public class WordCountMapper extends Mapper { + private static final IntWritable ONE = new IntWritable(1); + private final transient Text word = new Text(); + + @Override public void map(final LongWritable key, final Text value, final Context context) throws IOException, InterruptedException { + final String line = value.toString(); + final StringTokenizer tokenizer = new StringTokenizer(line, " \t\n\r.,:;-[]()_?!'\""); + + String cToken; + + while (tokenizer.hasMoreTokens()) { + cToken = tokenizer.nextToken(); + if ((cToken.charAt(0) >= 'a' && cToken.charAt(0) <= 'z') || (cToken.charAt(0) >= 'A' && cToken.charAt(0) <= 'Z')) { + word.set(cToken); + context.write(word, ONE); + } + } + } + } \ No newline at end of file diff --git a/src/main/java/mburmistrov/task1/WordCountReducer.java b/src/main/java/mburmistrov/task1/WordCountReducer.java new file mode 100644 index 0000000..bc73a64 --- /dev/null +++ b/src/main/java/mburmistrov/task1/WordCountReducer.java @@ -0,0 +1,20 @@ +package mburmistrov.task1; + +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Reducer; + +import java.io.IOException; + +public class WordCountReducer extends Reducer { + + @Override + public void reduce(final Text key, final Iterable values, final Context context) + throws IOException, InterruptedException { + int sum = 0; + for (final IntWritable val : values) { + sum += val.get(); + } + context.write(key, new IntWritable(sum)); + } +} \ No newline at end of file diff --git a/src/main/java/mburmistrov/task2/TextWCount.java b/src/main/java/mburmistrov/task2/TextWCount.java new file mode 100644 index 0000000..f181e60 --- /dev/null +++ b/src/main/java/mburmistrov/task2/TextWCount.java @@ -0,0 +1,92 @@ +package mburmistrov.task2; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.WritableComparable; + +public class TextWCount implements WritableComparable, Cloneable { + + private int count; + private String text; + + + @Override + public void readFields(DataInput dInput) throws IOException { + count = dInput.readInt(); + + text = dInput.readUTF(); + } + + @Override + public void write(DataOutput dOutput) throws IOException { + dOutput.writeInt(count); + + dOutput.writeUTF(text); + } + + + public TextWCount(String text, int count) { + this.text = text; + + this.count = count; + } + + @Override + public boolean equals(Object other) { + + if (other == null) { + return false; + } + + if (other == this) { + return true; + } + + if (!(other instanceof TextWCount)) { + return false; + } + + TextWCount otherMyClass = (TextWCount) other; + + if (!otherMyClass.text.equals(text)) { + return false; + } + + if (otherMyClass.count != count) { + return false; + } + + return true; + } + + @Override + protected TextWCount clone() { + return new TextWCount(text, count); + } + + TextWCount() { + } + + public String getText() { + return text; + } + + public int getCount() { + return count; + } + + @Override + public int compareTo(TextWCount o) { + if (equals(o)) { + return 0; + } + int intCompare = Integer.compare(count, o.count); + if (intCompare == 0) { + return Integer.compare(this.hashCode(), o.hashCode()); + } else { + return -intCompare; + } + } +} \ No newline at end of file diff --git a/src/main/java/mburmistrov/task2/WordCertainPosition.java b/src/main/java/mburmistrov/task2/WordCertainPosition.java new file mode 100644 index 0000000..4268b9a --- /dev/null +++ b/src/main/java/mburmistrov/task2/WordCertainPosition.java @@ -0,0 +1,120 @@ +package mburmistrov.task2; + +import java.io.IOException; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class WordCertainPosition extends Configured implements Tool { + private static int vNum; + + private static IntWritable ONE = new IntWritable(1); + + static class WordCertainPositionMapper extends Mapper { + private int count; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + super.setup(context); + count = 0; + } + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + if (pos >= 0) { + int inputCount = Integer.valueOf(line.substring(0, pos)); + String inputString = line.substring(pos + 1); + + if (count <= vNum) { + context.write(ONE, new TextWCount(inputString, inputCount)); + count++; + } + } + } + } + + static class WordCertainPositionReducer extends Reducer { + private SortedSet setOfTextWithCount; + + @Override + protected void setup(Context context) throws IOException, InterruptedException { + super.setup(context); + setOfTextWithCount = new TreeSet<>(); + } + + @Override + protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException { + values.forEach((textWCount -> { + setOfTextWithCount.add(textWCount.clone()); + if (setOfTextWithCount.size() > vNum + 1){ + setOfTextWithCount.remove(setOfTextWithCount.last()); + } + })); + } + + @Override + protected void cleanup(Context context) throws IOException, InterruptedException { + super.cleanup(context); + + int i = 0; + + for (TextWCount textWCount : setOfTextWithCount) { + if (i == vNum) { + context.write(new Text(textWCount.getText()), new IntWritable(textWCount.getCount())); + break; + } + i++; + } + } + } + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "WordCertainPosition"); + + vNum = Integer.valueOf(args[2]); + + job.setJarByClass(WordCertainPosition.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapperClass(WordCertainPositionMapper.class); + job.setReducerClass(WordCertainPositionReducer.class); + + job.setMapOutputKeyClass(IntWritable.class); + job.setMapOutputValueClass(TextWCount.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new WordCertainPosition(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/mburmistrov/task2/WordSort.java b/src/main/java/mburmistrov/task2/WordSort.java new file mode 100644 index 0000000..3958c38 --- /dev/null +++ b/src/main/java/mburmistrov/task2/WordSort.java @@ -0,0 +1,60 @@ +package mburmistrov.task2; + +import mburmistrov.task1.WordCount; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import java.io.IOException; + +public class WordSort extends Configured implements Tool { + static class Comparator extends WritableComparator { + protected Comparator() { + super(IntWritable.class); + } + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + return -Integer.compare(readInt(b1, s1), readInt(b2, s2)); + } + } + + @Override + public int run(final String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = Job.getInstance(conf, "Word Sort"); + job.setJarByClass(WordSort.class); + + job.setMapperClass(WordSortMapper.class); + job.setReducerClass(WordSortReducer.class); + + job.setOutputKeyClass(IntWritable.class); + job.setOutputValueClass(Text.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + job.setSortComparatorClass(Comparator.class); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(final String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new WordCount(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/mburmistrov/task2/WordSortMapper.java b/src/main/java/mburmistrov/task2/WordSortMapper.java new file mode 100644 index 0000000..706145f --- /dev/null +++ b/src/main/java/mburmistrov/task2/WordSortMapper.java @@ -0,0 +1,19 @@ +package mburmistrov.task2; + +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; + +import java.io.IOException; + +public class WordSortMapper extends Mapper { + + @Override + protected void map(final LongWritable key, final Text value, final Context context) throws IOException, InterruptedException { + final String line = value.toString(); + final String[] splitLine = line.split("\t"); + + context.write(new IntWritable(Integer.valueOf(splitLine[1])), new Text(splitLine[0])); + } +} \ No newline at end of file diff --git a/src/main/java/mburmistrov/task2/WordSortReducer.java b/src/main/java/mburmistrov/task2/WordSortReducer.java new file mode 100644 index 0000000..4d87f9d --- /dev/null +++ b/src/main/java/mburmistrov/task2/WordSortReducer.java @@ -0,0 +1,16 @@ +package mburmistrov.task2; + +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Reducer; + +import java.io.IOException; + +public class WordSortReducer extends Reducer { + @Override + protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException { + for (final Text value : values) { + context.write(key, value); + } + } +} diff --git a/src/main/java/mburmistrov/task3/StopWordProportion.java b/src/main/java/mburmistrov/task3/StopWordProportion.java new file mode 100644 index 0000000..e5d140e --- /dev/null +++ b/src/main/java/mburmistrov/task3/StopWordProportion.java @@ -0,0 +1,131 @@ +package mburmistrov.task3; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class StopWordProportion extends Configured implements Tool { + private static Set stopWordSet; + + private static void setStopWords(String stopWordsFile) { + stopWordSet = new HashSet<>(); + try { + Files.lines(Paths.get(stopWordsFile), StandardCharsets.UTF_8).forEach(stopWordSet::add); + } catch (IOException e) { + throw new ExceptionInInitializerError(e); + } + } + + static class StopWordProportionMapper extends Mapper { + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + if (pos >= 0) { + String inputString = line.substring(0, pos); + int inputCount = Integer.valueOf(line.substring(pos + 1)); + + context.write(new Text(inputString.toLowerCase()), new IntWritable(inputCount)); + } + } + } + + static class StopWordProportionReducer extends Reducer { + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + boolean isStopWord = stopWordSet.contains(key.toString()); + + for (final IntWritable value : values) { + + context.getCounter(COUNTERS.WORD_COUNT).increment(value.get()); + + if (isStopWord) { + context.getCounter(COUNTERS.STOP_WORD_COUNT).increment(value.get()); + } + } + } + } + + static class StopWordProportionCombiner extends Reducer{ + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + int vCount = 0; + for (final IntWritable value : values) { + vCount += value.get(); + } + context.write(key, new IntWritable(vCount)); + } + } + + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "StopWordProportion"); + + setStopWords(args[2]); + + job.setJarByClass(StopWordProportion.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(IntWritable.class); + + job.setMapperClass(StopWordProportionMapper.class); + job.setReducerClass(StopWordProportionReducer.class); + job.setCombinerClass(StopWordProportionCombiner.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + boolean success = job.waitForCompletion(true); + + long stopWordCount = job.getCounters().findCounter(COUNTERS.STOP_WORD_COUNT).getValue(); + long wordCount = job.getCounters().findCounter(COUNTERS.WORD_COUNT).getValue(); + + double percentProportion = stopWordCount / (double) wordCount * 100; + + String outputPath = args[1] + "/output"; + File file = new File(outputPath); + Files.write(file.toPath(), String.valueOf(percentProportion).getBytes()); + + return success ? 0 : 1; + } + + enum COUNTERS { + WORD_COUNT, + STOP_WORD_COUNT + } + + public static void main(String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new StopWordProportion(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/mburmistrov/task4/NameWordProportion.java b/src/main/java/mburmistrov/task4/NameWordProportion.java new file mode 100644 index 0000000..9b4709d --- /dev/null +++ b/src/main/java/mburmistrov/task4/NameWordProportion.java @@ -0,0 +1,48 @@ +package mburmistrov.task4; + +import mburmistrov.task2.TextWCount; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class NameWordProportion extends Configured implements Tool { + + @Override + public int run(String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = new Job(conf, "NameWordProportion"); + job.setJarByClass(NameWordProportion.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + job.setMapperClass(NameWordProportionMapper.class); + job.setReducerClass(NameWordProportionReducer.class); + + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(TextWCount.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path(args[1])); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new NameWordProportion(), args); + System.exit(returnCode); + + } +} diff --git a/src/main/java/mburmistrov/task4/NameWordProportionMapper.java b/src/main/java/mburmistrov/task4/NameWordProportionMapper.java new file mode 100644 index 0000000..0a4ef10 --- /dev/null +++ b/src/main/java/mburmistrov/task4/NameWordProportionMapper.java @@ -0,0 +1,25 @@ +package mburmistrov.task4; + +import mburmistrov.task2.TextWCount; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Mapper; + +import java.io.IOException; + +public class NameWordProportionMapper extends Mapper { + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + final String line = value.toString(); + + int pos = line.indexOf(0x09); + + if (pos >= 0) { + + String inputString = line.substring(0, pos); + int inputCount = Integer.valueOf(line.substring(pos + 1)); + + context.write(new Text(inputString.toLowerCase()), new TextWCount(inputString, inputCount)); + } + } +} \ No newline at end of file diff --git a/src/main/java/mburmistrov/task4/NameWordProportionReducer.java b/src/main/java/mburmistrov/task4/NameWordProportionReducer.java new file mode 100644 index 0000000..27ad483 --- /dev/null +++ b/src/main/java/mburmistrov/task4/NameWordProportionReducer.java @@ -0,0 +1,45 @@ +package mburmistrov.task4; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import mburmistrov.task2.TextWCount; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Reducer; + +public class NameWordProportionReducer extends Reducer { + private static final Pattern nameWordPattern = Pattern.compile("^[A-Z][a-z0-9]*$"); + + @Override + protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + int allCount = 0; + int correctCount = 0; + String correctText = null; + double correctPercent = 0.995; + + for (final TextWCount value : values) { + + allCount += value.getCount(); + + if (correctText == null) { + Matcher matcher = nameWordPattern.matcher(value.getText()); + + if (matcher.matches()) { + correctText = value.getText(); + + correctCount = value.getCount(); + } + } + } + + if (correctText == null) { + return; + } + + if (correctCount / (double) allCount >= correctPercent) { + context.write(new Text(correctText), new IntWritable(correctCount)); + } + } +} \ No newline at end of file diff --git a/timeReport.txt b/timeReport.txt new file mode 100644 index 0000000..5155df3 --- /dev/null +++ b/timeReport.txt @@ -0,0 +1,27 @@ +Подсчет до добавления комбайнеров: + +Задание №1 +34047,536 сек. + +Задание №2 +54,232 сек. + +Задание №3 +92,131 сек. + +Задание №4 +158,679 сек. + +Подсчет после добавления комбайнеров: + +Задание №1 +4205,871 сек. + +Задание №2 +60,588 сек. + +Задание №3 +110,276 сек. + +Задание №4 +172,679 сек.