Skip to content

WC #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tFNiYaFF wants to merge 2 commits into
base: master
Choose a base branch
from
Open

WC #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
621b5f5
First
ResearchProjects22 Nov 13, 2017
f7261ea
third hw
tFNiYaFF Nov 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions 2017-big-data.iml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.6.0" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:11.0.2" level="project" />
<orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
<orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
<orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.1" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
<orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
<orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.5" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.4" level="project" />
<orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />
<orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-recipes:2.6.0" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
<orderEntry type="library" name="Maven: org.htrace:htrace-core:3.0.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
<orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty:3.6.2.Final" level="project" />
<orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
<orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.0" level="project" />
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
<orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: javax.servlet:servlet-api:2.5" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.0" level="project" />
</component>
</module>
6 changes: 3 additions & 3 deletions answers.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
2. is 126420
3. 41.602
4. french 5742
2. was 18391755
3. 37.164249490179486
4. October 1072615
8 changes: 4 additions & 4 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>pritykovskaya</groupId>
<groupId>tFNiYaFF</groupId>
<artifactId>2017-big-data</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
Expand Down Expand Up @@ -29,7 +29,7 @@
<configuration>
<archive>
<manifest>
<mainClass>pritykovskaya.WordCount</mainClass>
<mainClass>tFNiYaFF.Main</mainClass>
</manifest>
</archive>
</configuration>
Expand All @@ -38,8 +38,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.6</source>
<target>1.6</target>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
</plugins>
Expand Down
14 changes: 14 additions & 0 deletions report.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
������� 1
2900,292s - ������� wordcount

������� 2
36,356s - ��������� �����
6,076s - ����� ������ 7-�� �����

������� 3
78,431s - ������� ������� ����-����

������� 4
78,385s - ������� ����� � ������ �� � �����
6,072s - ��������� ����� �� �������� �������
1,048s - ������� ������ 5-� ��� (5 => 0 to 4)
80 changes: 0 additions & 80 deletions src/main/java/pritykovskaya/WordCount.java
View file
Open in desktop

This file was deleted.

105 changes: 105 additions & 0 deletions src/main/java/tFNiYaFF/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package tFNiYaFF;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

public class Main {
private static final String FILE_REPORT = "report.txt";

public static void main(String[] args) throws Exception {
String inputDirectory = "input";
String wordcountDirectory = "output-wordcount";
String orderedDirectory = "output-ordered";
String task2OutputDirectory = "output-7-word";
String task3OutputDirectory = "output-stopwordscount";
String namesDirectory = "output-names";
String namesOrderedDirectory = "output-ordered-names";
String task4OutputDirectory = "output-5-name";

Report task1 = new Report("Задание 1")
.addAction(
"Считаем wordcount",
calculateTaskTime(new WordCount(), inputDirectory, wordcountDirectory)
);

Report task2 = new Report("Задание 2")
.addAction(
"Сортируем слова",
calculateTaskTime(new WordsOrder(), wordcountDirectory, orderedDirectory)
)
.addAction(
"Вывод только 7-го слова",
calculateTaskTime(new NValue(), orderedDirectory, task2OutputDirectory, "6")
);

Report task3 = new Report("Задание 3")
.addAction(
"Считаем процент стоп-слов",
calculateTaskTime(new StopWordsCount(), wordcountDirectory, task3OutputDirectory, "stop_words_en.txt")
);
Report task4 = new Report("Задание 4")
.addAction(
"Находим имена и кладем их в папку",
calculateTaskTime(new NamesPercent(), wordcountDirectory, namesDirectory)
)
.addAction(
"Сортируем имена по убыванию частоты",
calculateTaskTime(new WordsOrder(), namesDirectory, namesOrderedDirectory)
)
.addAction(
"Выводим только 5-е имя (5 => 0 to 4)",
calculateTaskTime(new NValue(), namesOrderedDirectory, task4OutputDirectory, "4")
);


Files.write(
Paths.get(FILE_REPORT),
String.join(
"\r\n\r\n",
task1.getString(),
task2.getString(),
task3.getString(),
task4.getString()
).getBytes()
);
}

private static double calculateTaskTime(Tool tool, String... parameters) throws Exception{
long from = System.currentTimeMillis();

ToolRunner.run(new Configuration(), tool, parameters);

long to = System.currentTimeMillis();
return (to - from)/(double)1000;
}

private static class Report {
private List<String> stringList;

Report(String name) {
stringList = new ArrayList<>();
stringList.add(name);
}

Report addAction(String description, double time){
stringList.add(formatTimeWithText(time, description));

return this;
}

private String formatTimeWithText(double time, String text){
return String.format("%.3f", time) + "s" + " - " + text;
}

public String getString() throws IOException {
return String.join("\r\n", stringList);
}
}
}
Loading