Skip to content

Single legacy S3 file support #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dstillman merged 1 commit into from
Jul 13, 2018
Merged

Single legacy S3 file support #76

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 78 additions & 38 deletions model/Storage.inc.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,22 @@ public static function getDownloadURL(Zotero_Item $item, $ttl=60) {
// http://docs.aws.amazon.com/aws-sdk-php/v3/api/api-s3-2006-03-01.html#headobject,
// but returning NotFound
if ($e->getAwsErrorCode() == 'NoSuchKey' || $e->getAwsErrorCode() == 'NotFound') {
// Try legacy key format, with zip flag and filename
// Try to find a legacy file: hash/filename or hash/c/filename
try {
$key = self::getPathPrefix($info['hash'], $info['zip']) . $info['filename'];
$s3Client->headObject([
$result = $s3Client->listObjects([
'Bucket' => Z_CONFIG::$S3_BUCKET,
'Key' => $key
'MaxKeys' => 1,
'Prefix' => $info['hash'],
]);
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey' || $e->getAwsErrorCode() == 'NotFound') {
$contents = $result->get('Contents');
if (!$contents || count($contents) < 1) {
return false;
}

$key = $contents[0]['Key'];
}
catch (\Aws\S3\Exception\S3Exception $e) {
throw $e;
}
}
Expand Down Expand Up @@ -136,19 +140,36 @@ public static function downloadFile(array $localFileItemInfo, $savePath, $filena
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey') {
// Try legacy key format, with zip flag and filename
// Try to find a legacy file: hash/filename or hash/c/filename
try {
return $s3Client->getObject([
$result = $s3Client->listObjects([
'Bucket' => Z_CONFIG::$S3_BUCKET,
'Key' => self::getPathPrefix($localFileItemInfo['hash'], $localFileItemInfo['zip'])
. $localFileItemInfo['filename'],
'SaveAs' => $savePath . "/" . ($filename ? $filename : $localFileItemInfo['filename'])
'MaxKeys' => 1,
'Prefix' => $localFileItemInfo['hash'],
]);
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey') {
$contents = $result->get('Contents');
if (!$contents || count($contents) < 1) {
return false;
}
$key = $contents[0]['Key'];

try {
return $s3Client->getObject([
'Bucket' => Z_CONFIG::$S3_BUCKET,
'Key' => $key,
'SaveAs' => $savePath . "/" . ($filename ? $filename : $localFileItemInfo['filename'])
]);
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey') {
return false;
}
throw $e;
}

}
catch (\Aws\S3\Exception\S3Exception $e) {
throw $e;
}
}
Expand Down Expand Up @@ -412,23 +433,41 @@ public static function duplicateFile($storageFileID, $newName, $zip, $contentTyp
// If file doesn't already exist named with just hash, copy it over
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey' || $e->getAwsErrorCode() == 'NotFound') {
// Try to find a legacy file: hash/filename or hash/c/filename
try {
$s3Client->copyObject([
$result = $s3Client->listObjects([
'Bucket' => Z_CONFIG::$S3_BUCKET,
'CopySource' => Z_CONFIG::$S3_BUCKET . '/'
. urlencode(self::getPathPrefix($localInfo['hash'], $localInfo['zip'])
. $localInfo['filename']),
'Key' => $localInfo['hash'],
'ACL' => 'private'
'MaxKeys' => 1,
'Prefix' => $localInfo['hash'],
]);
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey') {
$contents = $result->get('Contents');
if (!$contents || count($contents) < 1) {
return false;
}
else {
throw $e;

$key = $contents[0]['Key'];

try {
$s3Client->copyObject([
'Bucket' => Z_CONFIG::$S3_BUCKET,
'CopySource' => Z_CONFIG::$S3_BUCKET . '/'
. urlencode($key),
'Key' => $localInfo['hash'],
'ACL' => 'private'
]);
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey') {
return false;
}
else {
throw $e;
}
}
}
catch (\Aws\S3\Exception\S3Exception $e) {
throw $e;
}
}
else {
Expand Down Expand Up @@ -467,20 +506,26 @@ public static function getRemoteFileInfo(Zotero_StorageFileInfo $info) {
'Bucket' => Z_CONFIG::$S3_BUCKET,
'Key' => $info->hash
]);
$size = $result['ContentLength'];
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey' || $e->getAwsErrorCode() == 'NotFound') {
// Try legacy key format, with zip flag and filename
// Try to find a legacy file: hash/filename or hash/c/filename
try {
$result = $s3Client->headObject([
$result = $s3Client->listObjects([
'Bucket' => Z_CONFIG::$S3_BUCKET,
'Key' => self::getPathPrefix($info->hash, $info->zip) . $info->filename
'MaxKeys' => 1,
'Prefix' => $info->hash,
]);
}
catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getAwsErrorCode() == 'NoSuchKey' || $e->getAwsErrorCode() == 'NotFound') {
$contents = $result->get('Contents');
if (!$contents || count($contents) < 1) {
return false;
}

$size = $contents[0]['Size'];
}
catch (\Aws\S3\Exception\S3Exception $e) {
throw $e;
}
}
Expand All @@ -490,7 +535,7 @@ public static function getRemoteFileInfo(Zotero_StorageFileInfo $info) {
}

$storageFileInfo = new Zotero_StorageFileInfo;
$storageFileInfo->size = $result['ContentLength'];
$storageFileInfo->size = $size;

return $storageFileInfo;
}
Expand Down Expand Up @@ -632,11 +677,6 @@ public static function deleteFileLibraryReference($storageFileID, $libraryID) {
}


public static function getPathPrefix($hash, $zip=false) {
return "$hash/" . ($zip ? "c/" : '');
}


public static function getUploadPOSTData($item, Zotero_StorageFileInfo $info) {
$params = self::generateUploadPOSTParams($item, $info);
$boundary = "---------------------------" . md5(uniqid());
Expand Down