Homework

.gitignore

@@ -1,3 +1,4 @@
 .idea
 /input
 /output*
+/target

2017-big-data.iml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-core:0.20.2" level="project" />
+    <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
+    <orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
+    <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.0.1" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.3" level="project" />
+    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.3" level="project" />
+    <orderEntry type="library" name="Maven: commons-net:commons-net:1.4.1" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty:6.1.14" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.14" level="project" />
+    <orderEntry type="library" name="Maven: tomcat:jasper-runtime:5.5.12" level="project" />
+    <orderEntry type="library" name="Maven: tomcat:jasper-compiler:5.5.12" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-api-2.1:6.1.14" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-2.1:6.1.14" level="project" />
+    <orderEntry type="library" name="Maven: ant:ant:1.6.5" level="project" />
+    <orderEntry type="library" name="Maven: commons-el:commons-el:1.0" level="project" />
+    <orderEntry type="library" name="Maven: net.java.dev.jets3t:jets3t:0.7.1" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:servlet-api-2.5:6.1.14" level="project" />
+    <orderEntry type="library" name="Maven: net.sf.kosmosfs:kfs:0.3" level="project" />
+    <orderEntry type="library" name="Maven: junit:junit:4.5" level="project" />
+    <orderEntry type="library" name="Maven: hsqldb:hsqldb:1.8.0.10" level="project" />
+    <orderEntry type="library" name="Maven: oro:oro:2.0.8" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jdt:core:3.1.1" level="project" />
+  </component>
+</module>

answers.txt

@@ -1,3 +1,3 @@
-2. is	126420
-3. 41.602
-4. french	5742
+2. was	18391755
+3. 37.164249490179486
+4. October	1072615

pom.xml

@@ -10,13 +10,13 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <hadoop.version>2.6.0</hadoop.version>
+    <hadoop.version>0.20.2</hadoop.version>
   </properties>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
+      <artifactId>hadoop-core</artifactId>
       <version>${hadoop.version}</version>
     </dependency>
   </dependencies>
@@ -29,7 +29,7 @@
         <configuration>
           <archive>
             <manifest>
-              <mainClass>pritykovskaya.WordCount</mainClass>
+              <mainClass>afilippo.Tasks</mainClass>
             </manifest>
           </archive>
         </configuration>
@@ -38,8 +38,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>1.6</source>
-          <target>1.6</target>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
     </plugins>

report.txt

@@ -0,0 +1,14 @@
+Задание 1
+2900,292s - Считаем wordcount
+
+Задание 2
+36,356s - Сортируем слова
+6,076s - Вывод только 7-го слова
+
+Задание 3
+78,431s - Считаем процент стоп-слов
+
+Задание 4
+78,385s - Находим имена и кладем их в папку
+6,072s - Сортируем имена по убыванию частоты
+1,048s - Выводим только 5-е имя (5 => 0 to 4)

src/main/java/afilippo/NValue.java

@@ -0,0 +1,117 @@
+package afilippo;
+
+import java.io.IOException;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class NValue extends Configured implements Tool {
+    private static int valueNumber;
+    private static IntWritable ONE = new IntWritable(1);
+
+    static class MyMapper extends Mapper<Object, Text, IntWritable, TextWithCountWriteble>{
+        private int count;
+
+        @Override
+        protected void setup(Context context) throws IOException, InterruptedException {
+            super.setup(context);
+            count = 0;
+        }
+
+        @Override
+        protected void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+            final String line = value.toString();
+
+            int pos = line.indexOf(0x09);
+
+            int inputCount = Integer.valueOf(line.substring(0, pos));
+            String inputString = line.substring(pos+1);
+
+            if (count <= valueNumber){
+                context.write(ONE, new TextWithCountWriteble(inputString, inputCount));
+                count++;
+            }
+        }
+    }
+
+    static class MyReducer extends Reducer<IntWritable, TextWithCountWriteble, Text, IntWritable>{
+        private SortedSet<TextWithCountWriteble> setOfTextWithCount;
+
+        @Override
+        protected void setup(Context context) throws IOException, InterruptedException {
+            super.setup(context);
+            setOfTextWithCount = new TreeSet<>();
+        }
+
+        @Override
+        protected void reduce(IntWritable key, Iterable<TextWithCountWriteble> values, Context context) throws IOException, InterruptedException {
+            values.forEach((textWithCountWriteble -> {
+                setOfTextWithCount.add(textWithCountWriteble.clone());
+                if (setOfTextWithCount.size() > valueNumber + 1){
+                    setOfTextWithCount.remove(setOfTextWithCount.last());
+                }
+            }));
+        }
+
+        @Override
+        protected void cleanup(Context context) throws IOException, InterruptedException {
+            super.cleanup(context);
+
+            int i = 0;
+            for (TextWithCountWriteble textWithCountWriteble : setOfTextWithCount){
+                if (i == valueNumber){
+                    context.write(new Text(textWithCountWriteble.getText()), new IntWritable(textWithCountWriteble.getCount()));
+                    break;
+                }
+                i++;
+            }
+        }
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+        final Configuration conf = this.getConf();
+        final Job job = new Job(conf, "NValue");
+
+        valueNumber = Integer.valueOf(args[2]);
+
+        job.setJarByClass(NValue.class);
+
+        job.setInputFormatClass(TextInputFormat.class);
+        job.setOutputFormatClass(TextOutputFormat.class);
+
+        job.setMapperClass(MyMapper.class);
+        job.setReducerClass(MyReducer.class);
+
+        job.setMapOutputKeyClass(IntWritable.class);
+        job.setMapOutputValueClass(TextWithCountWriteble.class);
+
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(IntWritable.class);
+
+        FileInputFormat.addInputPath(job, new Path(args[0]));
+        FileOutputFormat.setOutputPath(job, new Path(args[1]));
+
+        return job.waitForCompletion(true) ? 0 : 1;
+    }
+
+    public static void main(String[] args) throws Exception{
+        final int returnCode = ToolRunner.run(new Configuration(), new NValue(), args);
+        System.exit(returnCode);
+
+    }
+}

src/main/java/afilippo/NamesPercent.java

@@ -0,0 +1,99 @@
+package afilippo;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class NamesPercent extends Configured implements Tool {
+    private static final Pattern namePattern = Pattern.compile("^[A-Z][a-z0-9]*$");
+
+    static class MyMapper extends Mapper<Object, Text, Text, TextWithCountWriteble>{
+
+        @Override
+        protected void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+            final String line = value.toString();
+
+            int pos = line.indexOf(0x09);
+
+            String inputString = line.substring(0, pos);
+            int inputCount = Integer.valueOf(line.substring(pos+1));
+
+            context.write(new Text(inputString.toLowerCase()), new TextWithCountWriteble(inputString, inputCount));
+        }
+    }
+
+    static class MyReducer extends Reducer<Text, TextWithCountWriteble, Text, IntWritable>{
+
+        @Override
+        protected void reduce(Text key, Iterable<TextWithCountWriteble> values, Context context) throws IOException, InterruptedException {
+            int sumAllForms = 0;
+            int rightFormCount = 0;
+            String rightFormText = null;
+
+            for (final TextWithCountWriteble value : values){
+                sumAllForms += value.getCount();
+
+                if (rightFormText == null){
+                    Matcher matcher = namePattern.matcher(value.getText());
+                    if (matcher.matches()){
+                        rightFormText = value.getText();
+                        rightFormCount = value.getCount();
+                    }
+                }
+            }
+
+            if (rightFormText == null){
+                return;
+            }
+
+            if (rightFormCount / (double)sumAllForms >= 0.995){
+                context.write(new Text(rightFormText), new IntWritable(rightFormCount));
+            }
+        }
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+        final Configuration conf = this.getConf();
+        final Job job = new Job(conf, "NamesPercent");
+        job.setJarByClass(NamesPercent.class);
+
+        job.setInputFormatClass(TextInputFormat.class);
+        job.setOutputFormatClass(TextOutputFormat.class);
+
+        job.setMapperClass(MyMapper.class);
+        job.setReducerClass(MyReducer.class);
+
+        job.setMapOutputKeyClass(Text.class);
+        job.setMapOutputValueClass(TextWithCountWriteble.class);
+
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(IntWritable.class);
+
+        FileInputFormat.addInputPath(job, new Path(args[0]));
+        FileOutputFormat.setOutputPath(job, new Path(args[1]));
+
+        return job.waitForCompletion(true) ? 0 : 1;
+    }
+
+    public static void main(String[] args) throws Exception{
+        final int returnCode = ToolRunner.run(new Configuration(), new NamesPercent(), args);
+        System.exit(returnCode);
+
+    }
+}