Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions java/ru/ifmo/ctddev/solutions/walk/Comments.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
My personal git repository link:
https://github.com/HelenMakarchuk/Java_HomeWork.git

------------------------------------------

Helped links:
https://ru.wikipedia.org/wiki/FNV
https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
https://www.programmingalgorithms.com/algorithm/fnv-hash

------------------------------------------

The FNV-1 hash algorithm:

hash = FNV_offset_basis

for each byte_of_data to be hashed
hash = hash × FNV_prime
hash = hash XOR byte_of_data

return hash

------------------------------------------

FNV parameters which used for this task:
32 bits hash,
FNV prime = 16777619,
FNV offset basis = 2166136261 (0x811c9dc5 in hexadecimal)

------------------------------------------

Modified output file template: [file path] [hash value] [elapsed time on hash calculating]

Example:
Path:"C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/1" Hash:811d69050c5d2e Elapsed time: 0.302625051 seconds

------------------------------------------

Console output template:
[file path]; [file size in bytes]; [chunk size] [chunks number]
[Hash calculating process]: [real time changeable hash calculating percent; 0 - began; 100 - completed]%

Example:
File path: "C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/1"; file size(bytes): 1; chunk size(bytes): 1; chunks number: 1
Hash calculating process: 100%
8 changes: 8 additions & 0 deletions java/ru/ifmo/ctddev/solutions/walk/Input.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/1
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/12
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/123
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/1234
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/1
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/binary
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples/no-such-file
C:/GitHub/java-advanced-2017/java/info/kgeorgiy/java/advanced/walk/samples
44 changes: 44 additions & 0 deletions java/ru/ifmo/ctddev/solutions/walk/Managers/FileManager.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package ru.ifmo.ctddev.solutions.walk.Managers;

import java.io.File;
import java.io.FileInputStream;

public class FileManager {

protected File file;
protected FileInputStream inputStream; //file stream
protected int chunkSize; //size of file chunk in bytes
protected long chunksNumber; //number of file chunks
protected long available = 0; //correct analog of FileInputStream.available() - number of available for reading bytes of file

public FileManager(File file) throws Exception {
System.out.print("\nFile path: \"" + file.getPath() + "\"");
System.out.print("; file size(bytes): " + file.length());

this.file = file;
available = file.length();
setChunkSize();
setChunksNumber();
inputStream = new FileInputStream(file.getPath());
}

protected void setChunkSize() {
long chunkSize = file.length();
long freeMemory = Runtime.getRuntime().freeMemory();

if (chunkSize > freeMemory) {
chunkSize = (long) Math.round(freeMemory / 4);

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

неясно что ты хотела этим добиться, кроме желания забрать четверть свободной памяти у всех остальных программ в операционной системе :)

читать по одному байту (из прошлой версии) неэффективно из-за слишком частого частого обращения в файловой системе, которое не бесплатное
читать огромными кусками, да ещё и на каждый кусок выделять новый кусок памяти также неээфективно, так как файловая система «за раз» много не отдаст.

оптимальное использовать размер блока файловой системы. он разнится, но средние значения около 4 / 8 КБ

https://stackoverflow.com/questions/236861/how-do-you-determine-the-ideal-buffer-size-when-using-fileinputstream/

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

и даже не четверть памяти, а больше.
на чтения каждого чанка у тебя выделяется новый буффер равный четверти свободной памяти, и, если файл больше чем свободная память, то программа может попытаться забрать себе всю доступную память

}

if (chunkSize > Integer.MAX_VALUE)
chunkSize = Integer.MAX_VALUE;

this.chunkSize = (int) chunkSize;
System.out.print("; chunk size(bytes): " + this.chunkSize);
}

protected void setChunksNumber() {
this.chunksNumber = (long) Math.ceil(file.length() / (double) this.chunkSize);
System.out.println("; chunks number: " + this.chunksNumber);
}
}
69 changes: 69 additions & 0 deletions java/ru/ifmo/ctddev/solutions/walk/Managers/HashManager.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package ru.ifmo.ctddev.solutions.walk.Managers;

import java.io.File;

public class HashManager {

protected static final String INCORRECT_FILE_HASH = "00000000";
protected static final long FNV_OFFSET_BASIS = 2166136261L;
protected static final int FNV_PRIME = 16777619;
private long hash; //represents hash value

public HashManager() {
this.hash = FNV_OFFSET_BASIS;
}

public String getHash() {
if (this.hash == -1)
return INCORRECT_FILE_HASH;
else
return String.format("%08x", this.hash);
}

public void setHash(long newHash) {
this.hash = newHash;
}

public void calculate(File file) throws Exception {
FileManager fileManager = new FileManager(file);
showCalculatingProcessStatus(0);

for (int i = 0; i < fileManager.chunksNumber; i++) {
if (fileManager.chunkSize > fileManager.available)
fileManager.chunkSize = (int) fileManager.available;

byte[] fileChunk = new byte[fileManager.chunkSize];
fileManager.inputStream.read(fileChunk, 0, fileChunk.length);
fileManager.available -= fileChunk.length;

@techno-aads techno-aads Apr 21, 2018

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

inputStream.read возвращает количество прочитанных байт в буфер или -1 если поток закончился

https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html#read-byte:A-

поддерживать available избыточно, а каждый раз выделять память под буфер на каждый чанк совсем уж печально.


calculate(fileChunk);
showCalculatingProcessStatus(fileManager);
}
}

public void calculate(byte[] fileBytes) throws Exception {
for (final byte b : fileBytes) {
this.hash *= FNV_PRIME;
this.hash ^= b;
}
}

protected void showCalculatingProcessStatus(String error) {
System.out.print("\rHash calculating process: " + error);
}

protected void showCalculatingProcessStatus(double percent) {
String formatPattern = "%s";

if ((long) percent == percent) {
formatPattern = "%.0f";
}

System.out.print("\rHash calculating process: " + String.format(formatPattern, percent) + "%");
}

protected void showCalculatingProcessStatus(FileManager fileManager) {
double percent = ((fileManager.file.length() - fileManager.available) / (double) fileManager.file.length()) * 100;
showCalculatingProcessStatus(percent);
}
}
12 changes: 12 additions & 0 deletions java/ru/ifmo/ctddev/solutions/walk/Output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\1" Hash:811d69050c5d2e Elapsed time: 0.348205921 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\12" Hash:344658b02076af58 Elapsed time: 0.004420786 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\123" Hash:fade13f272d607bb Elapsed time: 0.003912955 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\1234" Hash:de0c3cb281ee2b55 Elapsed time: 0.0038971 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\1" Hash:811d69050c5d2e Elapsed time: 0.003781917 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\binary" Hash:6af63b5c447241c5 Elapsed time: 0.003753005 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\no-such-file" Hash:00000000 Elapsed time: 0.004173166 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\1" Hash:811d69050c5d2e Elapsed time: 0.006087441 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\12" Hash:344658b02076af58 Elapsed time: 0.005170175 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\123" Hash:fade13f272d607bb Elapsed time: 0.003807099 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\1234" Hash:de0c3cb281ee2b55 Elapsed time: 0.004543897 seconds
Path:"C:\GitHub\java-advanced-2017\java\info\kgeorgiy\java\advanced\walk\samples\binary" Hash:6af63b5c447241c5 Elapsed time: 0.004288816 seconds
11 changes: 8 additions & 3 deletions java/ru/ifmo/ctddev/solutions/walk/RecursiveWalk.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
package ru.ifmo.ctddev.solutions.walk;

public class RecursiveWalk {
//todo
}
/* redundant class*/
public class RecursiveWalk extends Walk {

/* redundant method*/
public static void main(String[] args) {
//Walk.main(args);
}
}
120 changes: 118 additions & 2 deletions java/ru/ifmo/ctddev/solutions/walk/Walk.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,121 @@
package ru.ifmo.ctddev.solutions.walk;

import ru.ifmo.ctddev.solutions.walk.Managers.HashManager;

import java.io.*;
import java.util.Scanner;

public class Walk {
//todo
}

/**
* Main method
*
* @param args [input file path] [output file path]
*/
public static void main(String[] args) {
args = getCorrectProgramInput(args);

try (FileWriter outputFileWriter = new FileWriter(args[1], true)) {
FileReader reader = new FileReader(args[0]);
BufferedReader bufferedReader = new BufferedReader(reader);
String path;

while ((path = bufferedReader.readLine()) != null) {
try {
File file = new File(path);

if (file.isDirectory())

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

дублирование с processDirectoryPath

вообще в Java уже есть способ обхода файловой системы https://docs.oracle.com/javase/tutorial/essential/io/walk.html

processDirectoryPath(file, outputFileWriter);
else
processFilePath(file, outputFileWriter);
} catch (Exception e) {
System.out.println("\n\nAn error occurred while processing file: \"" + path + "\"");
e.printStackTrace();
}
}
} catch (IOException e) {
e.printStackTrace();
}
}

/*
Method is used to process each row of input file which represents directory path.
Method performs execution of processFilePath() for each file in directory.
*/
protected static void processDirectoryPath(File directory, FileWriter outputFileWriter) throws Exception {
for (File file : directory.listFiles()) {
if (file.isDirectory())
processDirectoryPath(file, outputFileWriter);
else
processFilePath(file, outputFileWriter);
}
}

/*
Method is used to process each row of input file which represents file path (not directory).
Method performs file validation, hash calculating, output printing.
*/
protected static void processFilePath(File file, FileWriter outputFileWriter) throws Exception {
long startTime = System.nanoTime();
outputFileWriter.write("Path:" + "\"" + file.getPath() + "\"");
HashManager hashManager = new HashManager();

try {
hashManager.calculate(file);
} catch (Exception e) {
hashManager.setHash(-1);
} finally {
outputFileWriter.write(" Hash:" + hashManager.getHash());
outputFileWriter.write(" Elapsed time: " + getElapsedTimeInSeconds(startTime) + " seconds\n");
}
}

protected static String[] getCorrectProgramInput(String[] args) {
try {
if (args.length != 2 || !areValidPaths(args)) {
getCorrectProgramInput(getProgramInputFromUser());
}
} catch (Exception e) {
e.printStackTrace();
}

return args;
}

protected static String[] getProgramInputFromUser() {
var inputFromUser = new String[]{};

System.out.println("\nObtained input arguments are incorrect. " +
"Please enter correct values according this template: " +
"[input file path] [output file path]\n");

try (Scanner scanner = new Scanner(System.in);) {
System.out.print("Enter:");
inputFromUser = scanner.nextLine().split(" ");
} catch (Exception e) {
System.out.println("An error occurred while processing input program arguments. Error:");
e.printStackTrace();
}

return inputFromUser;
}

protected static Boolean areValidPaths(String paths[]) {
try {
for (int i = 0; i < paths.length; i++) {
var file = new File(paths[i]);

if (!file.exists() || file.isDirectory())
return false;
}

return true;
} catch (Exception e) {
return false;
}
}

protected static double getElapsedTimeInSeconds(long startTime) {
return (System.nanoTime() - startTime) / Math.pow(10, 9);
}
}