first DZ done

differences between task with Combainer and without it

@@ -0,0 +1,2 @@
+word count без комбайнера 1 час 40 мин
+word count c комбайнера 1 час 10 мин

pom.xml

@@ -1,47 +1,76 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>pritykovskaya</groupId>
-  <artifactId>2017-big-data</artifactId>
-  <packaging>jar</packaging>
-  <version>1.0-SNAPSHOT</version>
-  <name>2017-big-data</name>
-  <url>http://maven.apache.org</url>
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
 
-  <properties>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <hadoop.version>2.6.0</hadoop.version>
-  </properties>
+    <groupId>MapReduce_wordCounter</groupId>
+    <artifactId>MapReduce_wordCounter</artifactId>
+    <version>1.0-SNAPSHOT</version>
 
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <version>${hadoop.version}</version>
-    </dependency>
-  </dependencies>
+    <!--
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce</artifactId>
+            <version>2.7.0</version>
+            <type>pom</type>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-core</artifactId>
+            <version>0.20.2</version>
+        </dependency>
+    </dependencies>
+-->
+
+      <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <hadoop.version>2.6.0</hadoop.version>
+      </properties>
+
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-client</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+      </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <mainClass>mapReduceTask.core.task_1.WordCountJob</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.7</source>
+                    <target>1.7</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
 
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-        <configuration>
-          <archive>
-            <manifest>
-              <mainClass>pritykovskaya.WordCount</mainClass>
-            </manifest>
-          </archive>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <configuration>
-          <source>1.6</source>
-          <target>1.6</target>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
 </project>

src/main/java/task_1/WordCountJob.java

@@ -0,0 +1,50 @@
+package task_1;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Created by iters on 10/21/17.
+ */
+public class WordCountJob extends Configured implements Tool {
+    public int run(String[] strings) throws Exception {
+        Job job = new Job(getConf(), "WordCount");
+        job.setJarByClass(getClass());
+
+        // TextInputFormat.addInputPath(job, new Path(strings[1]));
+        TextInputFormat.addInputPath(job,
+                new Path(strings[0]));
+
+        job.setInputFormatClass(TextInputFormat.class);
+
+        job.setMapperClass(WordCounterMapper.class);
+        job.setReducerClass(WordCounterReducer.class);
+        job.setCombinerClass(WordCounterReducer.class);
+
+        // TextOutputFormat.setOutputPath(job, new Path(strings[2]));
+        TextOutputFormat.setOutputPath(job,
+                new Path(strings[1]));
+        job.setOutputFormatClass(TextOutputFormat.class);
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(IntWritable.class);
+
+        return job.waitForCompletion(true) ? 0 : 1;
+    }
+
+    public static void main(String[] args) {
+        try {
+            int exitCode = ToolRunner.run(new WordCountJob(), args);
+            System.exit(exitCode);
+        } catch (Exception e) {
+            System.out.println("error on start ;(");
+            e.printStackTrace();
+        }
+    }
+}

src/main/java/task_1/WordCounterMapper.java

@@ -0,0 +1,30 @@
+package task_1;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+/**
+ * Created by iters on 10/22/17.
+ */
+public class WordCounterMapper
+        extends Mapper<LongWritable, Text, Text, IntWritable> {
+
+    private final static IntWritable one = new IntWritable(1);
+    private final Text word = new Text();
+
+    @Override
+    protected void map(LongWritable key, Text value, Context context)
+            throws IOException, InterruptedException {
+        StringTokenizer st = new StringTokenizer(value.toString());
+
+        while (st.hasMoreTokens()) {
+            word.set(st.nextToken());
+            context.write(word, one);
+        }
+    }
+}

src/main/java/task_1/WordCounterReducer.java

@@ -0,0 +1,25 @@
+package task_1;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import java.io.IOException;
+
+/**
+ * Created by iters on 10/22/17.
+ */
+public class WordCounterReducer
+        extends Reducer<Text, IntWritable, Text, IntWritable> {
+
+    @Override
+    protected void reduce(Text key, Iterable<IntWritable> values, Context context)
+            throws IOException, InterruptedException {
+        int sum = 0;
+        for (IntWritable num : values) {
+            sum += num.get();
+        }
+
+        context.write(key, new IntWritable(sum));
+    }
+}

src/main/java/task_2/DescendingSorting.java

@@ -0,0 +1,50 @@
+package task_2;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Created by iters on 10/21/17.
+ */
+
+public class DescendingSorting extends Configured implements Tool {
+    public int run(String[] strings) throws Exception {
+        Job job = new Job(getConf(), "DescendingSortingWords");
+        job.setJarByClass(getClass());
+
+        TextInputFormat.addInputPath(job,
+                new Path(strings[0]));
+        job.setInputFormatClass(TextInputFormat.class);
+
+        job.setMapperClass(WordCounterMapper.class);
+        job.setReducerClass(WordCounterReducer.class);
+
+        job.setMapOutputKeyClass(IntWritable.class);
+        job.setMapOutputValueClass(Text.class);
+
+        TextOutputFormat.setOutputPath(job,
+                new Path(strings[1]));
+        job.setOutputFormatClass(TextOutputFormat.class);
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(IntWritable.class);
+
+        return job.waitForCompletion(true) ? 0 : 1;
+    }
+
+    public static void main(String[] args) {
+        try {
+            int exitCode = ToolRunner.run(new DescendingSorting(), args);
+            System.exit(exitCode);
+        } catch (Exception e) {
+            System.out.println("error on start ;(");
+            e.printStackTrace();
+        }
+    }
+}

src/main/java/task_2/WordCounterMapper.java

@@ -0,0 +1,28 @@
+package task_2;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Created by iters on 10/22/17.
+ */
+public class WordCounterMapper
+        extends Mapper<LongWritable, Text, IntWritable, Text> {
+
+    private final Text word = new Text();
+    private final IntWritable num = new IntWritable();
+
+    @Override
+    protected void map(LongWritable key, Text value, Context context)
+            throws IOException, InterruptedException {
+        String[] parts = value.toString().split("\t");
+
+        word.set(parts[0]);
+        num.set(-1 * Integer.parseInt(parts[1]));
+        context.write(num, word);
+    }
+}