diff --git a/.gitignore b/.gitignore index e3b6fa2..f1ec51b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea /input /output* +*/target/** \ No newline at end of file diff --git a/2017-big-data.iml b/2017-big-data.iml new file mode 100644 index 0000000..ce21085 --- /dev/null +++ b/2017-big-data.iml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MEASUREMENTS.md b/MEASUREMENTS.md new file mode 100644 index 0000000..c055ba4 --- /dev/null +++ b/MEASUREMENTS.md @@ -0,0 +1,11 @@ +Выполнялись замеры на ноутбуке Xiaomi MI Air 13.3 2016 в режиме высокой производительности. + +Какие имеются результаты: + + WordCount job'a (без combiner): + + 1 час 54 минуты (грубый замер по аналоговым часам) + + WordCount job'a (используя combiner): + + 1 час 17 минут (грубый замер по аналоговым часам) diff --git a/pom.xml b/pom.xml index 16acaa5..8f4567d 100644 --- a/pom.xml +++ b/pom.xml @@ -38,8 +38,8 @@ org.apache.maven.plugins maven-compiler-plugin - 1.6 - 1.6 + 1.8 + 1.8 diff --git a/src/main/java/clhost/task1/WordCount.java b/src/main/java/clhost/task1/WordCount.java new file mode 100644 index 0000000..f474603 --- /dev/null +++ b/src/main/java/clhost/task1/WordCount.java @@ -0,0 +1,82 @@ +package clhost.task1; + +import java.io.IOException; +import java.util.StringTokenizer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + + +public class WordCount extends Configured implements Tool { + + public static class WordCountMapper extends Mapper { + private static final IntWritable ONE = new IntWritable(1); + private final transient Text word = new Text(); + + @Override + public void map(final LongWritable key, final Text value, final Context context) throws IOException, InterruptedException { + System.out.println("#map"); + final String line = value.toString(); + final StringTokenizer tokenizer = new StringTokenizer(line); + + while (tokenizer.hasMoreTokens()) { + word.set(tokenizer.nextToken()); + context.write(word, ONE); + } + } + } + + + public static class WordCountReducer extends Reducer { + + @Override + public void reduce(final Text key, final Iterable values, final Context context) throws IOException, InterruptedException { + int sum = 0; + for (final IntWritable val : values) { + sum += val.get(); + } + context.write(key, new IntWritable(sum)); + } + } + + + @Override + public int run(final String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = Job.getInstance(conf, "Word Count"); + job.setJarByClass(WordCount.class); + + job.setMapperClass(WordCount.WordCountMapper.class); + job.setReducerClass(WordCount.WordCountReducer.class); + job.setCombinerClass(WordCount.WordCountReducer.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path("/home/clhost/proj/2017-big-data/wordcountoutput")); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(final String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new WordCount(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/clhost/task2/FrequencyWordCounter.java b/src/main/java/clhost/task2/FrequencyWordCounter.java new file mode 100644 index 0000000..fad6632 --- /dev/null +++ b/src/main/java/clhost/task2/FrequencyWordCounter.java @@ -0,0 +1,85 @@ +package clhost.task2; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import java.io.IOException; + +// подаем на вход результат работы джобы WordCount +public class FrequencyWordCounter extends Configured implements Tool { + + public static class FrequencyMapper extends Mapper { + private final Text word = new Text(); + private final IntWritable count = new IntWritable(); + + @Override + protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + String[] tokens = value.toString().split("\t"); + + word.set(tokens[0]); + count.set((-1) * Integer.parseInt(tokens[1])); + context.write(count, word); + } + } + + + public static class FrequencyReducer extends Reducer { + private final IntWritable count = new IntWritable(); + private static final int SEEK_POSITION = 7; + private int counter = 0; + + @Override + protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException { + System.out.println("#reduce"); + count.set(Integer.parseInt(key.toString()) * (-1)); + + for (Text value : values) { + if (counter == SEEK_POSITION - 1) { + context.write(value, count); + } + counter++; + } + } + } + + + @Override + public int run(final String[] args) throws Exception { + Job job = new Job(getConf(), "Frequency Word Count"); + job.setJarByClass(getClass()); + + TextInputFormat.addInputPath(job, + new Path("/home/clhost/proj/2017-big-data/wordcountoutput/part-r-00000")); + job.setInputFormatClass(TextInputFormat.class); + + job.setMapperClass(FrequencyMapper.class); + job.setReducerClass(FrequencyReducer.class); + + job.setMapOutputKeyClass(IntWritable.class); + job.setMapOutputValueClass(Text.class); + + TextOutputFormat.setOutputPath(job, + new Path("/home/clhost/proj/2017-big-data/frequencycountoutput")); + job.setOutputFormatClass(TextOutputFormat.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(final String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new FrequencyWordCounter(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/clhost/task3/StopWordCounter.java b/src/main/java/clhost/task3/StopWordCounter.java new file mode 100644 index 0000000..4c65647 --- /dev/null +++ b/src/main/java/clhost/task3/StopWordCounter.java @@ -0,0 +1,112 @@ +package clhost.task3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class StopWordCounter extends Configured implements Tool { + + public static class StopWordMapper extends Mapper { + private HashSet set = initSet(); + + @Override + public void map(final LongWritable key, final Text value, final Context context) throws IOException, InterruptedException { + System.out.println("#map"); + final String line = value.toString(); + final StringTokenizer tokenizer = new StringTokenizer(line); + + int count_valid = 0; + int count_all = 0; + while (tokenizer.hasMoreTokens()) { + String word = tokenizer.nextToken(); + + if (set.contains(word.trim())) { + count_valid++; + } + count_all++; + } + context.write(new IntWritable(count_valid), new IntWritable(count_all)); + } + + private HashSet initSet() { + String path = "/home/clhost/proj/2017-big-data/stop_words_en.txt"; + Set set = new HashSet<>(); + + try (Stream stream = Files.lines(Paths.get(path))) { + set = stream + .collect(Collectors.toSet()); + } catch (IOException e) { + e.printStackTrace(); + } + return (HashSet) set; + } + } + + + public static class StopWordReducer extends Reducer { + + @Override + protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException { + System.out.println("#reduce"); + // приходит 2 значения + int count_valid = key.get(); + int count_all = 0; + + for (IntWritable ci : values) { + count_all = ci.get(); // вернет одно значение + } + + // return (1, %) + context.write(new IntWritable(1), new DoubleWritable(((double) count_valid) / count_all)); + } + } + + + @Override + public int run(final String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = Job.getInstance(conf, "Stop Word Count"); + job.setJarByClass(StopWordCounter.class); + + job.setMapperClass(StopWordCounter.StopWordMapper.class); + job.setReducerClass(StopWordCounter.StopWordReducer.class); + + job.setOutputKeyClass(IntWritable.class); + job.setOutputValueClass(IntWritable.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + FileInputFormat.addInputPath(job, new Path(args[0])); + FileOutputFormat.setOutputPath(job, new Path("/home/clhost/proj/2017-big-data/stopwordcountoutput")); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(final String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new StopWordCounter(), args); + System.exit(returnCode); + } +} diff --git a/src/main/java/clhost/task4/NameWordCounter.java b/src/main/java/clhost/task4/NameWordCounter.java new file mode 100644 index 0000000..22a483b --- /dev/null +++ b/src/main/java/clhost/task4/NameWordCounter.java @@ -0,0 +1,82 @@ +package clhost.task4; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +import java.io.IOException; +import java.util.StringTokenizer; + +public class NameWordCounter extends Configured implements Tool { + + public static class NameWordMapper extends Mapper { + private final static IntWritable ONE = new IntWritable(1); + private final Text word = new Text(); + + @Override + protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + StringTokenizer tokenizer = new StringTokenizer(value.toString()); + + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + if (token.matches("[A-Z].+")) { + word.set(token); + context.write(word, ONE); + } else { + context.getCounter("count", token.toLowerCase()).increment(1); + } + } + } + } + + + public static class NameWordReducer extends Reducer { + @Override + public void reduce(final Text key, final Iterable values, final Context context) throws IOException, InterruptedException { + int sum = 0; + for (IntWritable value : values) { + sum += value.get(); + } + + sum += Math.round(context.getCounter("count", key.toString().toLowerCase()).getValue() / (sum * 0.005)); + context.write(key, new IntWritable(sum)); + } + } + + + @Override + public int run(final String[] args) throws Exception { + final Configuration conf = this.getConf(); + final Job job = Job.getInstance(conf, "Name Word Count"); + job.setJarByClass(NameWordCounter.class); + + job.setMapperClass(NameWordCounter.NameWordMapper.class); + job.setReducerClass(NameWordCounter.NameWordReducer.class); + + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + + job.setInputFormatClass(TextInputFormat.class); + job.setOutputFormatClass(TextOutputFormat.class); + + TextInputFormat.addInputPath(job, new Path(args[0])); + TextOutputFormat.setOutputPath(job, new Path("/home/clhost/proj/2017-big-data/namewordcountoutput")); + + return job.waitForCompletion(true) ? 0 : 1; + } + + public static void main(final String[] args) throws Exception { + final int returnCode = ToolRunner.run(new Configuration(), new NameWordCounter(), args); + System.exit(returnCode); + } +} diff --git a/stop_words_en.txt b/stop_words_en.txt new file mode 100644 index 0000000..b7454b0 --- /dev/null +++ b/stop_words_en.txt @@ -0,0 +1,319 @@ +a +about +above +across +after +afterwards +again +against +all +almost +alone +along +already +also +although +always +am +among +amongst +amoungst +amount +an +and +another +any +anyhow +anyone +anything +anyway +anywhere +are +around +as +at +back +be +became +because +become +becomes +becoming +been +before +beforehand +behind +being +below +beside +besides +between +beyond +bill +both +bottom +but +by +call +can +cannot +cant +co +computer +con +could +couldnt +cry +de +describe +detail +do +done +down +due +during +each +eg +eight +either +eleven +else +elsewhere +empty +enough +etc +even +ever +every +everyone +everything +everywhere +except +few +fifteen +fify +fill +find +fire +first +five +for +former +formerly +forty +found +four +from +front +full +further +get +give +go +had +has +hasnt +have +he +hence +her +here +hereafter +hereby +herein +hereupon +hers +herse" +him +himse" +his +how +however +hundred +i +ie +if +in +inc +indeed +interest +into +is +it +its +itse" +keep +last +latter +latterly +least +less +ltd +made +many +may +me +meanwhile +might +mill +mine +more +moreover +most +mostly +move +much +must +my +myse" +name +namely +neither +never +nevertheless +next +nine +no +nobody +none +noone +nor +not +nothing +now +nowhere +of +off +often +on +once +one +only +onto +or +other +others +otherwise +our +ours +ourselves +out +over +own +part +per +perhaps +please +put +rather +re +same +see +seem +seemed +seeming +seems +serious +several +she +should +show +side +since +sincere +six +sixty +so +some +somehow +someone +something +sometime +sometimes +somewhere +still +such +system +take +ten +than +that +the +their +them +themselves +then +thence +there +thereafter +thereby +therefore +therein +thereupon +these +they +thick +thin +third +this +those +though +three +through +throughout +thru +thus +to +together +too +top +toward +towards +twelve +twenty +two +un +under +until +up +upon +us +very +via +was +we +well +were +what +whatever +when +whence +whenever +where +whereafter +whereas +whereby +wherein +whereupon +wherever +whether +which +while +whither +who +whoever +whole +whom +whose +why +will +with +within +without +would +yet +you +your +yours +yourself +yourselves diff --git a/target/classes/clhost/task1/WordCount$WordCountMapper.class b/target/classes/clhost/task1/WordCount$WordCountMapper.class new file mode 100644 index 0000000..f6c7f2b Binary files /dev/null and b/target/classes/clhost/task1/WordCount$WordCountMapper.class differ diff --git a/target/classes/clhost/task1/WordCount$WordCountReducer.class b/target/classes/clhost/task1/WordCount$WordCountReducer.class new file mode 100644 index 0000000..6f8a164 Binary files /dev/null and b/target/classes/clhost/task1/WordCount$WordCountReducer.class differ diff --git a/target/classes/clhost/task1/WordCount.class b/target/classes/clhost/task1/WordCount.class new file mode 100644 index 0000000..7498b29 Binary files /dev/null and b/target/classes/clhost/task1/WordCount.class differ diff --git a/target/classes/clhost/task2/FrequencyWordCounter$FrequencyMapper.class b/target/classes/clhost/task2/FrequencyWordCounter$FrequencyMapper.class new file mode 100644 index 0000000..cccb27f Binary files /dev/null and b/target/classes/clhost/task2/FrequencyWordCounter$FrequencyMapper.class differ diff --git a/target/classes/clhost/task2/FrequencyWordCounter$FrequencyReducer.class b/target/classes/clhost/task2/FrequencyWordCounter$FrequencyReducer.class new file mode 100644 index 0000000..1d13ed1 Binary files /dev/null and b/target/classes/clhost/task2/FrequencyWordCounter$FrequencyReducer.class differ diff --git a/target/classes/clhost/task2/FrequencyWordCounter.class b/target/classes/clhost/task2/FrequencyWordCounter.class new file mode 100644 index 0000000..651e8ff Binary files /dev/null and b/target/classes/clhost/task2/FrequencyWordCounter.class differ diff --git a/target/classes/clhost/task3/StopWordCounter$StopWordMapper.class b/target/classes/clhost/task3/StopWordCounter$StopWordMapper.class new file mode 100644 index 0000000..0c06c63 Binary files /dev/null and b/target/classes/clhost/task3/StopWordCounter$StopWordMapper.class differ diff --git a/target/classes/clhost/task3/StopWordCounter$StopWordReducer.class b/target/classes/clhost/task3/StopWordCounter$StopWordReducer.class new file mode 100644 index 0000000..7d76416 Binary files /dev/null and b/target/classes/clhost/task3/StopWordCounter$StopWordReducer.class differ diff --git a/target/classes/clhost/task3/StopWordCounter.class b/target/classes/clhost/task3/StopWordCounter.class new file mode 100644 index 0000000..6aa835a Binary files /dev/null and b/target/classes/clhost/task3/StopWordCounter.class differ diff --git a/target/classes/clhost/task4/Name05WordCounter$Name05WordMapper.class b/target/classes/clhost/task4/Name05WordCounter$Name05WordMapper.class new file mode 100644 index 0000000..be81178 Binary files /dev/null and b/target/classes/clhost/task4/Name05WordCounter$Name05WordMapper.class differ diff --git a/target/classes/clhost/task4/Name05WordCounter$Name05WordReducer.class b/target/classes/clhost/task4/Name05WordCounter$Name05WordReducer.class new file mode 100644 index 0000000..beb9a0d Binary files /dev/null and b/target/classes/clhost/task4/Name05WordCounter$Name05WordReducer.class differ diff --git a/target/classes/clhost/task4/Name05WordCounter.class b/target/classes/clhost/task4/Name05WordCounter.class new file mode 100644 index 0000000..540e0bb Binary files /dev/null and b/target/classes/clhost/task4/Name05WordCounter.class differ diff --git a/target/classes/clhost/task4/NameWordCounter$NameWordMapper.class b/target/classes/clhost/task4/NameWordCounter$NameWordMapper.class new file mode 100644 index 0000000..ee1ddeb Binary files /dev/null and b/target/classes/clhost/task4/NameWordCounter$NameWordMapper.class differ diff --git a/target/classes/clhost/task4/NameWordCounter$NameWordReducer.class b/target/classes/clhost/task4/NameWordCounter$NameWordReducer.class new file mode 100644 index 0000000..0c9354c Binary files /dev/null and b/target/classes/clhost/task4/NameWordCounter$NameWordReducer.class differ diff --git a/target/classes/clhost/task4/NameWordCounter.class b/target/classes/clhost/task4/NameWordCounter.class new file mode 100644 index 0000000..86a6558 Binary files /dev/null and b/target/classes/clhost/task4/NameWordCounter.class differ diff --git a/target/classes/pritykovskaya/WordCount$MyMapper.class b/target/classes/pritykovskaya/WordCount$MyMapper.class new file mode 100644 index 0000000..bb2ae2d Binary files /dev/null and b/target/classes/pritykovskaya/WordCount$MyMapper.class differ diff --git a/target/classes/pritykovskaya/WordCount$MyReducer.class b/target/classes/pritykovskaya/WordCount$MyReducer.class new file mode 100644 index 0000000..5a41c1c Binary files /dev/null and b/target/classes/pritykovskaya/WordCount$MyReducer.class differ diff --git a/target/classes/pritykovskaya/WordCount.class b/target/classes/pritykovskaya/WordCount.class new file mode 100644 index 0000000..bcd8495 Binary files /dev/null and b/target/classes/pritykovskaya/WordCount.class differ