From a2fbf7c1a9f0f6dfc36559f8abcff8ccea485c32 Mon Sep 17 00:00:00 2001 From: Mator Date: Sun, 4 Dec 2016 19:58:26 -0800 Subject: [PATCH 1/6] created ssebsa header and cpp file --- src/api/ssebsa.cpp | 481 +++++++++++++++++++++++++++++++++++++++++++++ src/api/ssebsa.h | 106 ++++++++++ 2 files changed, 587 insertions(+) create mode 100644 src/api/ssebsa.cpp create mode 100644 src/api/ssebsa.h diff --git a/src/api/ssebsa.cpp b/src/api/ssebsa.cpp new file mode 100644 index 0000000..be130be --- /dev/null +++ b/src/api/ssebsa.cpp @@ -0,0 +1,481 @@ +/* libbsa + + A library for reading and writing BSA files. + + Copyright (C) 2012-2013 WrinklyNinja + + This file is part of libbsa. + + libbsa is free software: you can redistribute + it and/or modify it under the terms of the GNU General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + libbsa is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libbsa. If not, see + . +*/ + +#include "tes4bsa.h" +#include "error.h" +#include "libbsa/libbsa.h" +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +using namespace std; + +namespace libbsa { + namespace sse { + + BSA::BSA(const std::string& path) + : _bsa_handle_int(path), + archiveFlags(0), + fileFlags(0) { + + //Check if file exists. + if (fs::exists(path)) { + + libbsa::ifstream in(fs::path(path), ios::binary); + in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. + + Header header; + in.seekg(0, ios_base::beg); + in.read((char*)&header, sizeof(Header)); + + if ((header.version != BSA_VERSION_SSE) || header.offset != BSA_FOLDER_RECORD_OFFSET) + throw error(LIBBSA_ERROR_PARSE_FAIL, "Folder offset of \"" + path + "\" is " + std::to_string(header.offset)); + + //Now we get to the real meat of the file. + //Folder records are followed by file records in blocks by folder name, followed by file names. + //File records and file names have the same ordering. + FolderRecord * folderRecords; + uint8_t * fileRecords; + uint8_t * fileNames; //A list of null-terminated filenames, one after another. + uint32_t fileRecordsSize = + header.folderCount + //Folder name string length (in 1 byte). + header.totalFolderNameLength + //Total length of folder name strings. + sizeof(FileRecord) * header.fileCount; //Total size of all file records. + try { + folderRecords = new FolderRecord[header.folderCount]; + in.read((char*)folderRecords, sizeof(FolderRecord) * header.folderCount); + + fileRecords = new uint8_t[fileRecordsSize]; + in.read((char*)fileRecords, sizeof(uint8_t) * fileRecordsSize); + + fileNames = new uint8_t[header.totalFileNameLength]; + in.read((char*)fileNames, sizeof(uint8_t) * header.totalFileNameLength); + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + in.close(); //No longer need the file open. + + /* Loop through the folder records, for each folder looking up the file records associated with it, + and the filenames associated with those records. */ + uint32_t fileNameListPos = 0; + uint32_t startOfFileRecords = sizeof(Header) + sizeof(FolderRecord) * header.folderCount; + for (uint32_t i = 0; i < header.folderCount; i++) { + /* folderRecords[i].count gives the number of file records associated with this folder. + folderRecords[i].offset gives the offset to the file records associated with this folder, + from the beginning of the file, plus the total filenames length. + folderRecords[i].hash can be discarded. */ + + folderRecords[i].offset -= header.totalFileNameLength + startOfFileRecords; //Get rid of this first. + + //Need to get folder name to add before file name in internal data store. + uint8_t folderNameLength = *(fileRecords + folderRecords[i].offset) - 1; + string folderName = ToUTF8(string((char*)(fileRecords + folderRecords[i].offset + 1), folderNameLength)); + + //Now loop through file records for this folder record. + uint32_t startOfFolderFileRecords = folderRecords[i].offset + folderNameLength + 2; + for (uint32_t j = 0; j < folderRecords[i].count; j++) { + BsaAsset fileData; + FileRecord fr = *(FileRecord*)(fileRecords + startOfFolderFileRecords + j * sizeof(FileRecord)); + fileData.hash = fr.nameHash; + fileData.size = fr.size; + fileData.offset = fr.offset; + + //Now we need to build the file path. First: file name. + char * filenameStart = (char*)(fileNames + fileNameListPos); + //Find position of null pointer. + char * nptr = strchr(filenameStart, '\0'); + if (nptr == NULL) + throw error(LIBBSA_ERROR_PARSE_FAIL, "SSEBSA: Structure of \"" + path + "\" is invalid."); + + fileData.path += ToUTF8(string(filenameStart, nptr - filenameStart)); + fileNameListPos += fileData.path.length() + 1; + + if (!folderName.empty()) + fileData.path = folderName + '\\' + fileData.path; + + //Finally, add file path and object to list. + assets.push_back(fileData); + } + } + + //Record the file and archive flags. + fileFlags = header.fileFlags; + archiveFlags = header.archiveFlags; + + delete[] folderRecords; + delete[] fileRecords; + delete[] fileNames; + } + } + + void BSA::Save(std::string path, const uint32_t version, const uint32_t compression) { + //Version and compression have been validated. + + if (path == filePath) + path += ".new"; //Avoid read/write collisions. + + libbsa::ifstream in(fs::path(filePath), ios::binary); + in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. + + libbsa::ofstream out(fs::path(path), ios::binary | ios::trunc); + out.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. + + /////////////////////////////// + // Set header up + /////////////////////////////// + + Header header; + + header.fileId = BSA_MAGIC; + + if (version == LIBBSA_VERSION_SSE) + header.version = BSA_VERSION_SSE; + + header.offset = 36; + + header.archiveFlags = archiveFlags; + if (compression != LIBBSA_COMPRESS_LEVEL_NOCHANGE) { + if (compression == LIBBSA_COMPRESS_LEVEL_0 && header.archiveFlags & BSA_COMPRESSED) + header.archiveFlags ^= BSA_COMPRESSED; + else if (compression != LIBBSA_COMPRESS_LEVEL_0 && !(header.archiveFlags & BSA_COMPRESSED)) + header.archiveFlags |= BSA_COMPRESSED; + } + + //Need to sort folder and file names separately into hash-sorted sets before header.folderCount and name lengths can be set. + list folderHashset; + list fileHashset; + for (list::iterator it = assets.begin(), endIt = assets.end(); it != endIt; ++it) { + BsaAsset folderAsset; + BsaAsset fileAsset; + + //Transcode paths. + folderAsset.path = FromUTF8(fs::path(it->path).parent_path().string()); + fileAsset.path = FromUTF8(it->path); /*fs::path(it->path).filename().string();*/ + + folderAsset.hash = CalcHash(folderAsset.path, ""); + fileAsset.hash = it->hash; + + fileAsset.size = it->size; + fileAsset.offset = it->offset; + + folderHashset.push_back(folderAsset); //Size and offset are zero for now. + fileHashset.push_back(fileAsset); + } + path_comp is_same_file; + folderHashset.unique(is_same_file); + fileHashset.unique(is_same_file); + header.folderCount = folderHashset.size(); + + header.fileCount = assets.size(); + + header.totalFolderNameLength = 0; + for (list::iterator it = folderHashset.begin(), endIt = folderHashset.end(); it != endIt; ++it) { + header.totalFolderNameLength += it->path.length() + 1; + } + + header.totalFileNameLength = 0; + for (list::iterator it = fileHashset.begin(), endIt = fileHashset.end(); it != endIt; ++it) { + header.totalFileNameLength += fs::path(it->path).filename().string().length() + 1; + } + + header.fileFlags = fileFlags; + + ///////////////////////////// + // Set folder record array + ///////////////////////////// + + /* Iterate through the folder hashmap. + For each folder, scan the file hashmap for files with matching parent paths. + For any such files, write out their nameHash, size and the offset at which their data can be found (calculated from the sum of previous sizes). + Also prepend the length of the folder name and the folder name to this file data list. + Once all matching files have been found, add their count and offset to the folder record stream. + */ + + FolderRecord * folderRecords; + uint8_t * fileRecordBlocks; + uint8_t * fileNames; + uint32_t fileRecordBlocksSize = header.folderCount + header.totalFolderNameLength + header.fileCount * sizeof(FileRecord); + try { + folderRecords = new FolderRecord[header.folderCount]; + fileRecordBlocks = new uint8_t[fileRecordBlocksSize]; + fileNames = new uint8_t[header.totalFileNameLength]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + uint32_t startOfFileRecordBlock = sizeof(Header) + header.folderCount * sizeof(FolderRecord) + header.totalFileNameLength; //For some reason offsets include this. + uint32_t fileDataOffset = startOfFileRecordBlock + fileRecordBlocksSize; + list orderedAssets; + uint32_t i = 0; + uint32_t currFileRecordBlockPos = 0; + uint32_t currFileNamePos = 0; + folderHashset.sort(hash_comp); + fileHashset.sort(hash_comp); + for (list::iterator it = folderHashset.begin(), endIt = folderHashset.end(); it != endIt; ++it) { + //Write folder hash and offset, write count later. + folderRecords[i].nameHash = it->hash; + folderRecords[i].offset = startOfFileRecordBlock + currFileRecordBlockPos; + + //Write folder name length, folder name to fileRecordBlocks buffer. + size_t fileCount = 0; + uint8_t nameLength = (uint8_t) it->path.length() + 1; + fileRecordBlocks[currFileRecordBlockPos] = nameLength; + currFileRecordBlockPos++; + strcpy((char*)fileRecordBlocks + currFileRecordBlockPos, (it->path + '\0').data()); + currFileRecordBlockPos += nameLength; + + uint32_t j = 0; + for (list::iterator itr = fileHashset.begin(), endItr = fileHashset.end(); itr != endItr; ++itr) { + if (fs::path(itr->path).parent_path().string() == it->path) { + //Write file hash, size and offset to fileRecordBlocks stream. + memcpy(fileRecordBlocks + currFileRecordBlockPos, &(itr->hash), sizeof(uint64_t)); + currFileRecordBlockPos += sizeof(uint64_t); + memcpy(fileRecordBlocks + currFileRecordBlockPos, &(itr->size), sizeof(uint32_t)); + currFileRecordBlockPos += sizeof(uint32_t); + memcpy(fileRecordBlocks + currFileRecordBlockPos, &fileDataOffset, sizeof(uint32_t)); + currFileRecordBlockPos += sizeof(uint32_t); + //Increment count and data offset. + fileCount++; + fileDataOffset += itr->size; + //Add record data to list for later ordered extraction. + orderedAssets.push_back(*itr); + orderedAssets.back().offset = fileDataOffset; //Can't update the offset in the set. + //Also write out filename to fileNameBlock. + string filename = fs::path(itr->path).filename().string() + '\0'; + strcpy((char*)fileNames + currFileNamePos, filename.data()); + currFileNamePos += filename.length(); + } + } + + folderRecords[i].count = fileCount; + + i++; + } + + //////////////////////// + // Write out + //////////////////////// + + out.write((char*)&header, sizeof(Header)); + out.write((char*)folderRecords, sizeof(FolderRecord) * header.folderCount); + out.write((char*)fileRecordBlocks, fileRecordBlocksSize); + out.write((char*)fileNames, header.totalFileNameLength); + + delete[] folderRecords; + delete[] fileRecordBlocks; + delete[] fileNames; + + //Now write out raw file data in the same order it was listed in the FileRecordBlocks. + for (list::iterator it = orderedAssets.begin(), endIt = orderedAssets.end(); it != endIt; ++it) { + //Allocate memory for this file's data, read it in, write it out, then free memory. + //This doesn't yet support compression level changing or assets that have been added to the BSA. + + uint32_t size = it->size; + if (size & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual size. + size ^= FILE_INVERT_COMPRESSED; + + uint8_t * fileData; + try { + fileData = new uint8_t[size]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + //Get the old BSA's file data offset. + list::iterator itr, endItr; + for (itr = assets.begin(), endItr = assets.end(); itr != endItr; ++itr) { + if (itr->path == it->path) + break; + } + + if (itr == endItr) + throw error(LIBBSA_ERROR_PARSE_FAIL, "SSEBSA: Structure of \"" + path + "\" is invalid."); + + //Read data in. + in.seekg(itr->offset, ios_base::beg); //This is the offset in the old BSA. + in.read((char*)fileData, size); + + //Write data out. + out.write((char*)fileData, size); + + //Free memory. + delete[] fileData; + + //Update the stored offset. + itr->offset = it->offset; + } + + //Update member vars. + filePath = path; + archiveFlags = header.archiveFlags; + fileFlags = header.fileFlags; + + in.close(); + out.close(); + + //Now rename the output file. + /* if (fs::path(path).extension().string() == ".new") { + try { + fs::rename(path, fs::path(path).stem()); + } catch (fs::filesystem_error& e) { + throw error(LIBBSA_ERROR_FILESYSTEM_ERROR, e.what()); + } + }*/ + } + + std::pair BSA::ReadData(libbsa::ifstream& in, const libbsa::BsaAsset& data) { + uint8_t * outBuffer = NULL; + uint32_t outSize = data.size; + //Check if given file is compressed or not. If not, can ofstream straight to path, otherwise need to involve zlib. + /* BSA-TYPE-SPECIFIC CHECK */ + if ((archiveFlags & BSA_COMPRESSED && outSize & FILE_INVERT_COMPRESSED) || (!(archiveFlags & BSA_COMPRESSED) && !(outSize & FILE_INVERT_COMPRESSED))) { + /* BSA-TYPE-SPECIFIC CHECK */ + if (outSize & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual size. + outSize ^= FILE_INVERT_COMPRESSED; + + try { + outBuffer = new uint8_t[outSize]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + in.seekg(data.offset, ios_base::beg); + in.read((char*)outBuffer, outSize); + } + else { + //Use zlib. + if (outSize & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual compressed size. + outSize ^= FILE_INVERT_COMPRESSED; + + //Get the uncompressed size. + uint32_t uncompressedSize; + in.seekg(data.offset, ios_base::beg); + in.read((char*)&uncompressedSize, sizeof(uint32_t)); + + //in and out are now at their starting locations for reading and writing, and we have the compressed and uncompressed size. + //Allocate memory for the compressed file and the uncompressed file. + uint8_t * compressedFile; + uint8_t * uncompressedFile; + outSize -= sizeof(uint32_t); //First uint32_t of data is the size of the uncompressed data. + try { + compressedFile = new uint8_t[outSize]; + uncompressedFile = new uint8_t[uncompressedSize]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + in.read((char*)compressedFile, outSize); + + //We can use a pre-made utility function instead of having to mess around with zlib proper. + int ret = uncompress(uncompressedFile, (uLongf*)&uncompressedSize, compressedFile, outSize); + if (ret != Z_OK) + throw error(LIBBSA_ERROR_ZLIB_ERROR, "Uncompressing of \"" + data.path + "\" failed."); + + outBuffer = uncompressedFile; + outSize = uncompressedSize; + + //Free memory. + delete[] compressedFile; + } + + return pair(outBuffer, outSize); + } + + uint32_t BSA::HashString(const std::string& str) { + uint32_t hash = 0; + for (size_t i = 0, len = str.length(); i < len; i++) { + hash = 0x1003F * hash + (uint8_t)str[i]; + } + return hash; + } + + uint64_t BSA::CalcHash(const std::string& path, const std::string& ext) { + uint64_t hash1 = 0; + uint32_t hash2 = 0; + uint32_t hash3 = 0; + const size_t len = path.length(); + + if (!path.empty()) { + hash1 = (uint64_t)( + ((uint8_t)path[len - 1]) + + (len << 16) + + ((uint8_t)path[0] << 24) + ); + + if (len > 2) { + hash1 += ((uint8_t)path[len - 2] << 8); + if (len > 3) + hash2 = HashString(path.substr(1, len - 3)); + } + } + + if (!ext.empty()) { + if (ext == ".kf") + hash1 += 0x80; + else if (ext == ".nif") + hash1 += 0x8000; + else if (ext == ".dds") + hash1 += 0x8080; + else if (ext == ".wav") + hash1 += 0x80000000; + + hash3 = HashString(ext); + } + + hash2 = hash2 + hash3; + return ((uint64_t)hash2 << 32) + hash1; + } + + bool BSA::hash_comp(const BsaAsset& first, const BsaAsset& second) { + return first.hash < second.hash; + } + + //Check if a given file is a Tes4-type BSA. + bool BSA::IsBSA(const std::string& path) { + //Check if file exists. + if (!fs::exists(path)) + return false; + else { + uint32_t magic; + uint32_t version; + libbsa::ifstream in(fs::path(path), ios::binary); + in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. + + in.read((char*)&magic, sizeof(uint32_t)); + in.read((char*)&version, sizeof(uint32_t)); + in.close(); + + return (magic == BSA_MAGIC) && (version == BSA_VERSION_SSE); + } + } + } +} diff --git a/src/api/ssebsa.h b/src/api/ssebsa.h new file mode 100644 index 0000000..8108c6e --- /dev/null +++ b/src/api/ssebsa.h @@ -0,0 +1,106 @@ +/* libbsa + + A library for reading and writing BSA files. + + Copyright (C) 2012-2013 WrinklyNinja + + This file is part of libbsa. + + libbsa is free software: you can redistribute + it and/or modify it under the terms of the GNU General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + libbsa is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libbsa. If not, see + . +*/ + +#ifndef __LIBBSA_SSESTRUCTS_H__ +#define __LIBBSA_SSESTRUCTS_H__ + +#include "genericbsa.h" +#include +#include +#include + +/* File format infos: + + + This header file defines the constants, structures and functions specific + to the Sse-type BSA, which is used by Skyrim: Special Edition +*/ + +namespace libbsa { + namespace sse { + + const uint32_t BSA_MAGIC = '\0ASB'; + const uint32_t BSA_VERSION_SSE = 0x69; + + const uint32_t BSA_FOLDER_RECORD_OFFSET = 36; //Folder record offset for SSE-type BSAs is constant. + + const uint32_t BSA_COMPRESSED = 0x0004; //If this flag is present in the archiveFlags header field, then the BSA file data is compressed. + + const uint32_t FILE_INVERT_COMPRESSED = 0x40000000; //Inverts the file data compression status for the specific file this flag is set for. + + struct Header { + uint32_t fileId; + uint32_t version; + uint32_t offset; + uint32_t archiveFlags; + uint32_t folderCount; + uint32_t fileCount; + uint32_t totalFolderNameLength; + uint32_t totalFileNameLength; + uint32_t fileFlags; + }; + + struct FolderRecord { + uint64_t nameHash; //Hash of folder name. + uint32_t count; //Number of files in folder. + uint32_t unk; //Unknown + uint64_t offset; //Offset to the fileRecords for this folder, including the folder name, from the beginning of the file. + }; + + struct FileRecord { + uint64_t nameHash; //Hash of the filename. + uint32_t size; //Size of the data. See TES4Mod wiki page for details. + uint32_t offset; //Offset to the raw file data, from byte 0. + }; + + //SSE-type BSA class. + class BSA : public _bsa_handle_int { + public: + BSA(const std::string& path); + void Save(std::string path, const uint32_t version, const uint32_t compression); + private: + std::pair ReadData(libbsa::ifstream& in, const libbsa::BsaAsset& data); + + uint32_t HashString(const std::string& str); + uint64_t CalcHash(const std::string& path, const std::string& ext); + + uint32_t archiveFlags; + uint32_t fileFlags; + }; + + bool hash_comp(const BsaAsset& first, const BsaAsset& second); + + //Comparison class for list::unique. + class path_comp { + public: + bool operator() (const BsaAsset& first, const BsaAsset& second) { + return first.path == second.path; + } + }; + + //Check if a given file is a Tes4-type BSA. + bool IsBSA(const std::string& path); + } +} + +#endif \ No newline at end of file From e412eae6efa1ce9466d439f348ec6124fed18be6 Mon Sep 17 00:00:00 2001 From: Mator Date: Sun, 4 Dec 2016 19:58:53 -0800 Subject: [PATCH 2/6] added ssebsa handling to _bsa_handle_int and libbsa --- src/api/_bsa_handle_int.cpp | 3 +++ src/api/libbsa.cpp | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/api/_bsa_handle_int.cpp b/src/api/_bsa_handle_int.cpp index 4c5aae4..56df516 100644 --- a/src/api/_bsa_handle_int.cpp +++ b/src/api/_bsa_handle_int.cpp @@ -24,6 +24,7 @@ #include "_bsa_handle_int.h" #include "tes3bsa.h" #include "tes4bsa.h" +#include "ssebsa.h" #include @@ -34,6 +35,8 @@ _bsa_handle_int::_bsa_handle_int(const boost::filesystem::path& path) : extAssetsNum(0) { if (tes3::BSA::IsBSA(path)) bsa = new tes3::BSA(path); + else if (sse::BSA::IsBSA(pathc)) + bsa = new sse::BSA(pathc); else bsa = new tes4::BSA(path); } diff --git a/src/api/libbsa.cpp b/src/api/libbsa.cpp index 34c7f6d..9edfa7d 100644 --- a/src/api/libbsa.cpp +++ b/src/api/libbsa.cpp @@ -26,6 +26,7 @@ #include "genericbsa.h" #include "tes3bsa.h" #include "tes4bsa.h" +#include "ssebsa.h" #include "error.h" #include @@ -67,6 +68,7 @@ const unsigned int LIBBSA_RETURN_MAX = LIBBSA_ERROR_PARSE_FAIL; const unsigned int LIBBSA_VERSION_TES3 = 0x00000001; const unsigned int LIBBSA_VERSION_TES4 = 0x00000002; const unsigned int LIBBSA_VERSION_TES5 = 0x00000004; +const unsigned int LIBBSA_VERSION_SSE = 0x00000005; /* Use only one compression flag. */ const unsigned int LIBBSA_COMPRESS_LEVEL_0 = 0x00000010; const unsigned int LIBBSA_COMPRESS_LEVEL_1 = 0x00000020; @@ -173,7 +175,7 @@ LIBBSA unsigned int bsa_save(bsa_handle bh, return c_error(LIBBSA_ERROR_INVALID_ARGS, "Morrowind BSAs cannot be compressed."); //Check that the version flag is valid. - std::bitset<3> version(flags & (LIBBSA_VERSION_TES3 | LIBBSA_VERSION_TES4 | LIBBSA_VERSION_TES5)); + std::bitset<3> version(flags & (LIBBSA_VERSION_TES3 | LIBBSA_VERSION_TES4 | LIBBSA_VERSION_TES5 | LIBBSA_VERSION_SSE)); if (version.none()) return c_error(LIBBSA_ERROR_INVALID_ARGS, "Must specify one version."); if (version.count() > 1) From 6ce1899ce71875c75faf00237b57281d94b8da26 Mon Sep 17 00:00:00 2001 From: Mator Date: Sun, 4 Dec 2016 20:19:57 -0800 Subject: [PATCH 3/6] fixed some issues due to changes in structure of bsa classes --- src/api/ssebsa.cpp | 882 ++++++++++++++++++++++----------------------- src/api/ssebsa.h | 131 +++---- 2 files changed, 505 insertions(+), 508 deletions(-) diff --git a/src/api/ssebsa.cpp b/src/api/ssebsa.cpp index be130be..644cc87 100644 --- a/src/api/ssebsa.cpp +++ b/src/api/ssebsa.cpp @@ -21,7 +21,7 @@ . */ -#include "tes4bsa.h" +#include "ssebsa.h" #include "error.h" #include "libbsa/libbsa.h" #include @@ -34,448 +34,442 @@ namespace fs = boost::filesystem; using namespace std; namespace libbsa { - namespace sse { - - BSA::BSA(const std::string& path) - : _bsa_handle_int(path), - archiveFlags(0), - fileFlags(0) { - - //Check if file exists. - if (fs::exists(path)) { - - libbsa::ifstream in(fs::path(path), ios::binary); - in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. - - Header header; - in.seekg(0, ios_base::beg); - in.read((char*)&header, sizeof(Header)); - - if ((header.version != BSA_VERSION_SSE) || header.offset != BSA_FOLDER_RECORD_OFFSET) - throw error(LIBBSA_ERROR_PARSE_FAIL, "Folder offset of \"" + path + "\" is " + std::to_string(header.offset)); - - //Now we get to the real meat of the file. - //Folder records are followed by file records in blocks by folder name, followed by file names. - //File records and file names have the same ordering. - FolderRecord * folderRecords; - uint8_t * fileRecords; - uint8_t * fileNames; //A list of null-terminated filenames, one after another. - uint32_t fileRecordsSize = - header.folderCount + //Folder name string length (in 1 byte). - header.totalFolderNameLength + //Total length of folder name strings. - sizeof(FileRecord) * header.fileCount; //Total size of all file records. - try { - folderRecords = new FolderRecord[header.folderCount]; - in.read((char*)folderRecords, sizeof(FolderRecord) * header.folderCount); - - fileRecords = new uint8_t[fileRecordsSize]; - in.read((char*)fileRecords, sizeof(uint8_t) * fileRecordsSize); - - fileNames = new uint8_t[header.totalFileNameLength]; - in.read((char*)fileNames, sizeof(uint8_t) * header.totalFileNameLength); - } - catch (bad_alloc& e) { - throw error(LIBBSA_ERROR_NO_MEM, e.what()); - } - - in.close(); //No longer need the file open. - - /* Loop through the folder records, for each folder looking up the file records associated with it, - and the filenames associated with those records. */ - uint32_t fileNameListPos = 0; - uint32_t startOfFileRecords = sizeof(Header) + sizeof(FolderRecord) * header.folderCount; - for (uint32_t i = 0; i < header.folderCount; i++) { - /* folderRecords[i].count gives the number of file records associated with this folder. - folderRecords[i].offset gives the offset to the file records associated with this folder, - from the beginning of the file, plus the total filenames length. - folderRecords[i].hash can be discarded. */ - - folderRecords[i].offset -= header.totalFileNameLength + startOfFileRecords; //Get rid of this first. - - //Need to get folder name to add before file name in internal data store. - uint8_t folderNameLength = *(fileRecords + folderRecords[i].offset) - 1; - string folderName = ToUTF8(string((char*)(fileRecords + folderRecords[i].offset + 1), folderNameLength)); - - //Now loop through file records for this folder record. - uint32_t startOfFolderFileRecords = folderRecords[i].offset + folderNameLength + 2; - for (uint32_t j = 0; j < folderRecords[i].count; j++) { - BsaAsset fileData; - FileRecord fr = *(FileRecord*)(fileRecords + startOfFolderFileRecords + j * sizeof(FileRecord)); - fileData.hash = fr.nameHash; - fileData.size = fr.size; - fileData.offset = fr.offset; - - //Now we need to build the file path. First: file name. - char * filenameStart = (char*)(fileNames + fileNameListPos); - //Find position of null pointer. - char * nptr = strchr(filenameStart, '\0'); - if (nptr == NULL) - throw error(LIBBSA_ERROR_PARSE_FAIL, "SSEBSA: Structure of \"" + path + "\" is invalid."); - - fileData.path += ToUTF8(string(filenameStart, nptr - filenameStart)); - fileNameListPos += fileData.path.length() + 1; - - if (!folderName.empty()) - fileData.path = folderName + '\\' + fileData.path; - - //Finally, add file path and object to list. - assets.push_back(fileData); - } - } - - //Record the file and archive flags. - fileFlags = header.fileFlags; - archiveFlags = header.archiveFlags; - - delete[] folderRecords; - delete[] fileRecords; - delete[] fileNames; - } - } - - void BSA::Save(std::string path, const uint32_t version, const uint32_t compression) { - //Version and compression have been validated. - - if (path == filePath) - path += ".new"; //Avoid read/write collisions. - - libbsa::ifstream in(fs::path(filePath), ios::binary); - in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. - - libbsa::ofstream out(fs::path(path), ios::binary | ios::trunc); - out.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. - - /////////////////////////////// - // Set header up - /////////////////////////////// - - Header header; - - header.fileId = BSA_MAGIC; - - if (version == LIBBSA_VERSION_SSE) - header.version = BSA_VERSION_SSE; - - header.offset = 36; - - header.archiveFlags = archiveFlags; - if (compression != LIBBSA_COMPRESS_LEVEL_NOCHANGE) { - if (compression == LIBBSA_COMPRESS_LEVEL_0 && header.archiveFlags & BSA_COMPRESSED) - header.archiveFlags ^= BSA_COMPRESSED; - else if (compression != LIBBSA_COMPRESS_LEVEL_0 && !(header.archiveFlags & BSA_COMPRESSED)) - header.archiveFlags |= BSA_COMPRESSED; - } - - //Need to sort folder and file names separately into hash-sorted sets before header.folderCount and name lengths can be set. - list folderHashset; - list fileHashset; - for (list::iterator it = assets.begin(), endIt = assets.end(); it != endIt; ++it) { - BsaAsset folderAsset; - BsaAsset fileAsset; - - //Transcode paths. - folderAsset.path = FromUTF8(fs::path(it->path).parent_path().string()); - fileAsset.path = FromUTF8(it->path); /*fs::path(it->path).filename().string();*/ - - folderAsset.hash = CalcHash(folderAsset.path, ""); - fileAsset.hash = it->hash; - - fileAsset.size = it->size; - fileAsset.offset = it->offset; - - folderHashset.push_back(folderAsset); //Size and offset are zero for now. - fileHashset.push_back(fileAsset); - } - path_comp is_same_file; - folderHashset.unique(is_same_file); - fileHashset.unique(is_same_file); - header.folderCount = folderHashset.size(); - - header.fileCount = assets.size(); - - header.totalFolderNameLength = 0; - for (list::iterator it = folderHashset.begin(), endIt = folderHashset.end(); it != endIt; ++it) { - header.totalFolderNameLength += it->path.length() + 1; - } - - header.totalFileNameLength = 0; - for (list::iterator it = fileHashset.begin(), endIt = fileHashset.end(); it != endIt; ++it) { - header.totalFileNameLength += fs::path(it->path).filename().string().length() + 1; - } - - header.fileFlags = fileFlags; - - ///////////////////////////// - // Set folder record array - ///////////////////////////// - - /* Iterate through the folder hashmap. - For each folder, scan the file hashmap for files with matching parent paths. - For any such files, write out their nameHash, size and the offset at which their data can be found (calculated from the sum of previous sizes). - Also prepend the length of the folder name and the folder name to this file data list. - Once all matching files have been found, add their count and offset to the folder record stream. - */ - - FolderRecord * folderRecords; - uint8_t * fileRecordBlocks; - uint8_t * fileNames; - uint32_t fileRecordBlocksSize = header.folderCount + header.totalFolderNameLength + header.fileCount * sizeof(FileRecord); - try { - folderRecords = new FolderRecord[header.folderCount]; - fileRecordBlocks = new uint8_t[fileRecordBlocksSize]; - fileNames = new uint8_t[header.totalFileNameLength]; - } - catch (bad_alloc& e) { - throw error(LIBBSA_ERROR_NO_MEM, e.what()); - } - - uint32_t startOfFileRecordBlock = sizeof(Header) + header.folderCount * sizeof(FolderRecord) + header.totalFileNameLength; //For some reason offsets include this. - uint32_t fileDataOffset = startOfFileRecordBlock + fileRecordBlocksSize; - list orderedAssets; - uint32_t i = 0; - uint32_t currFileRecordBlockPos = 0; - uint32_t currFileNamePos = 0; - folderHashset.sort(hash_comp); - fileHashset.sort(hash_comp); - for (list::iterator it = folderHashset.begin(), endIt = folderHashset.end(); it != endIt; ++it) { - //Write folder hash and offset, write count later. - folderRecords[i].nameHash = it->hash; - folderRecords[i].offset = startOfFileRecordBlock + currFileRecordBlockPos; - - //Write folder name length, folder name to fileRecordBlocks buffer. - size_t fileCount = 0; - uint8_t nameLength = (uint8_t) it->path.length() + 1; - fileRecordBlocks[currFileRecordBlockPos] = nameLength; - currFileRecordBlockPos++; - strcpy((char*)fileRecordBlocks + currFileRecordBlockPos, (it->path + '\0').data()); - currFileRecordBlockPos += nameLength; - - uint32_t j = 0; - for (list::iterator itr = fileHashset.begin(), endItr = fileHashset.end(); itr != endItr; ++itr) { - if (fs::path(itr->path).parent_path().string() == it->path) { - //Write file hash, size and offset to fileRecordBlocks stream. - memcpy(fileRecordBlocks + currFileRecordBlockPos, &(itr->hash), sizeof(uint64_t)); - currFileRecordBlockPos += sizeof(uint64_t); - memcpy(fileRecordBlocks + currFileRecordBlockPos, &(itr->size), sizeof(uint32_t)); - currFileRecordBlockPos += sizeof(uint32_t); - memcpy(fileRecordBlocks + currFileRecordBlockPos, &fileDataOffset, sizeof(uint32_t)); - currFileRecordBlockPos += sizeof(uint32_t); - //Increment count and data offset. - fileCount++; - fileDataOffset += itr->size; - //Add record data to list for later ordered extraction. - orderedAssets.push_back(*itr); - orderedAssets.back().offset = fileDataOffset; //Can't update the offset in the set. - //Also write out filename to fileNameBlock. - string filename = fs::path(itr->path).filename().string() + '\0'; - strcpy((char*)fileNames + currFileNamePos, filename.data()); - currFileNamePos += filename.length(); - } - } - - folderRecords[i].count = fileCount; - - i++; - } - - //////////////////////// - // Write out - //////////////////////// - - out.write((char*)&header, sizeof(Header)); - out.write((char*)folderRecords, sizeof(FolderRecord) * header.folderCount); - out.write((char*)fileRecordBlocks, fileRecordBlocksSize); - out.write((char*)fileNames, header.totalFileNameLength); - - delete[] folderRecords; - delete[] fileRecordBlocks; - delete[] fileNames; - - //Now write out raw file data in the same order it was listed in the FileRecordBlocks. - for (list::iterator it = orderedAssets.begin(), endIt = orderedAssets.end(); it != endIt; ++it) { - //Allocate memory for this file's data, read it in, write it out, then free memory. - //This doesn't yet support compression level changing or assets that have been added to the BSA. - - uint32_t size = it->size; - if (size & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual size. - size ^= FILE_INVERT_COMPRESSED; - - uint8_t * fileData; - try { - fileData = new uint8_t[size]; - } - catch (bad_alloc& e) { - throw error(LIBBSA_ERROR_NO_MEM, e.what()); - } - - //Get the old BSA's file data offset. - list::iterator itr, endItr; - for (itr = assets.begin(), endItr = assets.end(); itr != endItr; ++itr) { - if (itr->path == it->path) - break; - } - - if (itr == endItr) - throw error(LIBBSA_ERROR_PARSE_FAIL, "SSEBSA: Structure of \"" + path + "\" is invalid."); - - //Read data in. - in.seekg(itr->offset, ios_base::beg); //This is the offset in the old BSA. - in.read((char*)fileData, size); - - //Write data out. - out.write((char*)fileData, size); - - //Free memory. - delete[] fileData; - - //Update the stored offset. - itr->offset = it->offset; - } - - //Update member vars. - filePath = path; - archiveFlags = header.archiveFlags; - fileFlags = header.fileFlags; - - in.close(); - out.close(); - - //Now rename the output file. - /* if (fs::path(path).extension().string() == ".new") { - try { - fs::rename(path, fs::path(path).stem()); - } catch (fs::filesystem_error& e) { - throw error(LIBBSA_ERROR_FILESYSTEM_ERROR, e.what()); - } - }*/ - } - - std::pair BSA::ReadData(libbsa::ifstream& in, const libbsa::BsaAsset& data) { - uint8_t * outBuffer = NULL; - uint32_t outSize = data.size; - //Check if given file is compressed or not. If not, can ofstream straight to path, otherwise need to involve zlib. - /* BSA-TYPE-SPECIFIC CHECK */ - if ((archiveFlags & BSA_COMPRESSED && outSize & FILE_INVERT_COMPRESSED) || (!(archiveFlags & BSA_COMPRESSED) && !(outSize & FILE_INVERT_COMPRESSED))) { - /* BSA-TYPE-SPECIFIC CHECK */ - if (outSize & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual size. - outSize ^= FILE_INVERT_COMPRESSED; - - try { - outBuffer = new uint8_t[outSize]; - } - catch (bad_alloc& e) { - throw error(LIBBSA_ERROR_NO_MEM, e.what()); - } - - in.seekg(data.offset, ios_base::beg); - in.read((char*)outBuffer, outSize); - } - else { - //Use zlib. - if (outSize & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual compressed size. - outSize ^= FILE_INVERT_COMPRESSED; - - //Get the uncompressed size. - uint32_t uncompressedSize; - in.seekg(data.offset, ios_base::beg); - in.read((char*)&uncompressedSize, sizeof(uint32_t)); - - //in and out are now at their starting locations for reading and writing, and we have the compressed and uncompressed size. - //Allocate memory for the compressed file and the uncompressed file. - uint8_t * compressedFile; - uint8_t * uncompressedFile; - outSize -= sizeof(uint32_t); //First uint32_t of data is the size of the uncompressed data. - try { - compressedFile = new uint8_t[outSize]; - uncompressedFile = new uint8_t[uncompressedSize]; - } - catch (bad_alloc& e) { - throw error(LIBBSA_ERROR_NO_MEM, e.what()); - } - - in.read((char*)compressedFile, outSize); - - //We can use a pre-made utility function instead of having to mess around with zlib proper. - int ret = uncompress(uncompressedFile, (uLongf*)&uncompressedSize, compressedFile, outSize); - if (ret != Z_OK) - throw error(LIBBSA_ERROR_ZLIB_ERROR, "Uncompressing of \"" + data.path + "\" failed."); - - outBuffer = uncompressedFile; - outSize = uncompressedSize; - - //Free memory. - delete[] compressedFile; - } - - return pair(outBuffer, outSize); - } - - uint32_t BSA::HashString(const std::string& str) { - uint32_t hash = 0; - for (size_t i = 0, len = str.length(); i < len; i++) { - hash = 0x1003F * hash + (uint8_t)str[i]; - } - return hash; - } - - uint64_t BSA::CalcHash(const std::string& path, const std::string& ext) { - uint64_t hash1 = 0; - uint32_t hash2 = 0; - uint32_t hash3 = 0; - const size_t len = path.length(); - - if (!path.empty()) { - hash1 = (uint64_t)( - ((uint8_t)path[len - 1]) - + (len << 16) - + ((uint8_t)path[0] << 24) - ); - - if (len > 2) { - hash1 += ((uint8_t)path[len - 2] << 8); - if (len > 3) - hash2 = HashString(path.substr(1, len - 3)); - } - } - - if (!ext.empty()) { - if (ext == ".kf") - hash1 += 0x80; - else if (ext == ".nif") - hash1 += 0x8000; - else if (ext == ".dds") - hash1 += 0x8080; - else if (ext == ".wav") - hash1 += 0x80000000; - - hash3 = HashString(ext); - } - - hash2 = hash2 + hash3; - return ((uint64_t)hash2 << 32) + hash1; - } - - bool BSA::hash_comp(const BsaAsset& first, const BsaAsset& second) { - return first.hash < second.hash; - } - - //Check if a given file is a Tes4-type BSA. - bool BSA::IsBSA(const std::string& path) { - //Check if file exists. - if (!fs::exists(path)) - return false; - else { - uint32_t magic; - uint32_t version; - libbsa::ifstream in(fs::path(path), ios::binary); - in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. - - in.read((char*)&magic, sizeof(uint32_t)); - in.read((char*)&version, sizeof(uint32_t)); - in.close(); - - return (magic == BSA_MAGIC) && (version == BSA_VERSION_SSE); - } - } + namespace sse { + + BSA::BSA(const boost::filesystem::path& path) : + GenericBsa(path), + archiveFlags(0), + fileFlags(0) { + boost::filesystem::ifstream in(path, ios::binary); + in.exceptions(ios::failbit | ios::badbit | ios::eofbit); + + Header header; + in.seekg(0, ios_base::beg); + in.read((char*)&header, sizeof(Header)); + + if ((header.version != BSA_VERSION_SSE) || header.offset != BSA_FOLDER_RECORD_OFFSET) + throw error(LIBBSA_ERROR_PARSE_FAIL, "Structure of \"" + path.string() + "\" is invalid."); + + //Now we get to the real meat of the file. + //Folder records are followed by file records in blocks by folder name, followed by file names. + //File records and file names have the same ordering. + vector folderRecords(header.folderCount); + uint8_t * fileRecords; + uint8_t * fileNames; //A list of null-terminated filenames, one after another. + uint32_t fileRecordsSize = + header.folderCount + //Folder name string length (in 1 byte). + header.totalFolderNameLength + //Total length of folder name strings. + sizeof(FileRecord) * header.fileCount; //Total size of all file records. + try { + in.read(reinterpret_cast(&folderRecords[0]), sizeof(FolderRecord) * header.folderCount); + + fileRecords = new uint8_t[fileRecordsSize]; + in.read(reinterpret_cast(fileRecords), sizeof(uint8_t) * fileRecordsSize); + + fileNames = new uint8_t[header.totalFileNameLength]; + in.read(reinterpret_cast(fileNames), sizeof(uint8_t) * header.totalFileNameLength); + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + in.close(); //No longer need the file open. + + /* Loop through the folder records, for each folder looking up the file records associated with it, + and the filenames associated with those records. */ + uint32_t fileNameListPos = 0; + const uint32_t folderRecordOffsetBaseline = sizeof(Header) + + sizeof(FolderRecord) * header.folderCount + + header.totalFileNameLength; + for (auto& folderRecord : folderRecords) { + folderRecord.offset -= folderRecordOffsetBaseline; + + //Need to get folder name to add before file name in internal data store. + string folderName = getFolderName(fileRecords, folderRecord.offset); + + //Now loop through file records for this folder record. + uint32_t startOfFolderFileRecords = folderRecord.offset + folderName.length() + 2; + for (uint32_t i = 0; i < folderRecord.count; i++) { + uint8_t * fileRecordOffset = fileRecords + startOfFolderFileRecords + i * sizeof(FileRecord); + FileRecord fileRecord = *reinterpret_cast(fileRecordOffset); + + BsaAsset fileData; + fileData.hash = fileRecord.nameHash; + fileData.size = fileRecord.size; + fileData.offset = fileRecord.offset; + + if (!folderName.empty()) + fileData.path = folderName + '\\'; + + fileData.path += getFileName(fileNames, fileNameListPos); + fileNameListPos += fileData.path.length() + 1; + + //Finally, store file data. + assets.push_back(fileData); + } + } + + //Record the file and archive flags. + fileFlags = header.fileFlags; + archiveFlags = header.archiveFlags; + + delete[] fileRecords; + delete[] fileNames; + } + + void BSA::Save(const boost::filesystem::path& path, const uint32_t version, const uint32_t compression) { + if (fs::exists(path)) + throw error(LIBBSA_ERROR_INVALID_ARGS, path.string() + " already exists"); + + if (!fs::exists(filePath)) + throw error(LIBBSA_ERROR_FILESYSTEM_ERROR, filePath.string() + " no longer exists"); + + boost::filesystem::ifstream in(filePath, ios::binary); + in.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. + + boost::filesystem::ofstream out(path, ios::binary | ios::trunc); + out.exceptions(ios::failbit | ios::badbit | ios::eofbit); //Causes ifstream::failure to be thrown if problem is encountered. + + /////////////////////////////// + // Set header up + /////////////////////////////// + + Header header; + + header.fileId = BSA_MAGIC; + header.version = BSA_VERSION_SSE; + + header.offset = 36; + + header.archiveFlags = archiveFlags; + if (compression != LIBBSA_COMPRESS_LEVEL_NOCHANGE) { + if (compression == LIBBSA_COMPRESS_LEVEL_0 && header.archiveFlags & BSA_COMPRESSED) + header.archiveFlags ^= BSA_COMPRESSED; + else if (compression != LIBBSA_COMPRESS_LEVEL_0 && !(header.archiveFlags & BSA_COMPRESSED)) + header.archiveFlags |= BSA_COMPRESSED; + } + + //Need to sort folder and file names separately into hash-sorted sets before header.folderCount and name lengths can be set. + list folderHashset; + list fileHashset; + for (auto it = assets.begin(), endIt = assets.end(); it != endIt; ++it) { + BsaAsset folderAsset; + BsaAsset fileAsset; + + //Transcode paths. + folderAsset.path = FromUTF8(fs::path(it->path).parent_path().string()); + fileAsset.path = FromUTF8(it->path); /*fs::path(it->path).filename().string();*/ + + folderAsset.hash = CalcHash(folderAsset.path, ""); + fileAsset.hash = it->hash; + + fileAsset.size = it->size; + fileAsset.offset = it->offset; + + folderHashset.push_back(folderAsset); //Size and offset are zero for now. + fileHashset.push_back(fileAsset); + } + folderHashset.unique(path_comp); + fileHashset.unique(path_comp); + header.folderCount = folderHashset.size(); + + header.fileCount = assets.size(); + + header.totalFolderNameLength = 0; + for (list::iterator it = folderHashset.begin(), endIt = folderHashset.end(); it != endIt; ++it) { + header.totalFolderNameLength += it->path.length() + 1; + } + + header.totalFileNameLength = 0; + for (list::iterator it = fileHashset.begin(), endIt = fileHashset.end(); it != endIt; ++it) { + header.totalFileNameLength += fs::path(it->path).filename().string().length() + 1; + } + + header.fileFlags = fileFlags; + + ///////////////////////////// + // Set folder record array + ///////////////////////////// + + /* Iterate through the folder hashmap. + For each folder, scan the file hashmap for files with matching parent paths. + For any such files, write out their nameHash, size and the offset at which their data can be found (calculated from the sum of previous sizes). + Also prepend the length of the folder name and the folder name to this file data list. + Once all matching files have been found, add their count and offset to the folder record stream. + */ + + FolderRecord * folderRecords; + uint8_t * fileRecordBlocks; + uint8_t * fileNames; + uint32_t fileRecordBlocksSize = header.folderCount + header.totalFolderNameLength + header.fileCount * sizeof(FileRecord); + try { + folderRecords = new FolderRecord[header.folderCount]; + fileRecordBlocks = new uint8_t[fileRecordBlocksSize]; + fileNames = new uint8_t[header.totalFileNameLength]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + uint32_t startOfFileRecordBlock = sizeof(Header) + header.folderCount * sizeof(FolderRecord) + header.totalFileNameLength; //For some reason offsets include this. + uint32_t fileDataOffset = startOfFileRecordBlock + fileRecordBlocksSize; + list orderedAssets; + uint32_t i = 0; + uint32_t currFileRecordBlockPos = 0; + uint32_t currFileNamePos = 0; + folderHashset.sort(hash_comp); + fileHashset.sort(hash_comp); + for (list::iterator it = folderHashset.begin(), endIt = folderHashset.end(); it != endIt; ++it) { + //Write folder hash and offset, write count later. + folderRecords[i].nameHash = it->hash; + folderRecords[i].offset = startOfFileRecordBlock + currFileRecordBlockPos; + + //Write folder name length, folder name to fileRecordBlocks buffer. + size_t fileCount = 0; + uint8_t nameLength = it->path.length() + 1; + fileRecordBlocks[currFileRecordBlockPos] = nameLength; + currFileRecordBlockPos++; + strcpy((char*)fileRecordBlocks + currFileRecordBlockPos, (it->path + '\0').data()); + currFileRecordBlockPos += nameLength; + + uint32_t j = 0; + for (list::iterator itr = fileHashset.begin(), endItr = fileHashset.end(); itr != endItr; ++itr) { + if (fs::path(itr->path).parent_path().string() == it->path) { + //Write file hash, size and offset to fileRecordBlocks stream. + memcpy(fileRecordBlocks + currFileRecordBlockPos, &(itr->hash), sizeof(uint64_t)); + currFileRecordBlockPos += sizeof(uint64_t); + memcpy(fileRecordBlocks + currFileRecordBlockPos, &(itr->size), sizeof(uint32_t)); + currFileRecordBlockPos += sizeof(uint32_t); + memcpy(fileRecordBlocks + currFileRecordBlockPos, &fileDataOffset, sizeof(uint32_t)); + currFileRecordBlockPos += sizeof(uint32_t); + //Increment count and data offset. + fileCount++; + fileDataOffset += itr->size; + //Add record data to list for later ordered extraction. + orderedAssets.push_back(*itr); + orderedAssets.back().offset = fileDataOffset; //Can't update the offset in the set. + //Also write out filename to fileNameBlock. + string filename = fs::path(itr->path).filename().string() + '\0'; + strcpy((char*)fileNames + currFileNamePos, filename.data()); + currFileNamePos += filename.length(); + } + } + + folderRecords[i].count = fileCount; + + i++; + } + + //////////////////////// + // Write out + //////////////////////// + + out.write((char*)&header, sizeof(Header)); + out.write((char*)folderRecords, sizeof(FolderRecord) * header.folderCount); + out.write((char*)fileRecordBlocks, fileRecordBlocksSize); + out.write((char*)fileNames, header.totalFileNameLength); + + delete[] folderRecords; + delete[] fileRecordBlocks; + delete[] fileNames; + + //Now write out raw file data in the same order it was listed in the FileRecordBlocks. + for (list::iterator it = orderedAssets.begin(), endIt = orderedAssets.end(); it != endIt; ++it) { + //Allocate memory for this file's data, read it in, write it out, then free memory. + //This doesn't yet support compression level changing or assets that have been added to the BSA. + + uint32_t size = it->size; + if (size & FILE_INVERT_COMPRESSED) //Remove compression flag from size to get actual size. + size ^= FILE_INVERT_COMPRESSED; + + uint8_t * fileData; + try { + fileData = new uint8_t[size]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + //Get the old BSA's file data offset. + list::iterator itr, endItr; + for (itr = assets.begin(), endItr = assets.end(); itr != endItr; ++itr) { + if (itr->path == it->path) + break; + } + + if (itr == endItr) + throw error(LIBBSA_ERROR_PARSE_FAIL, "Structure of \"" + path.string() + "\" is invalid."); + + //Read data in. + in.seekg(itr->offset, ios_base::beg); //This is the offset in the old BSA. + in.read((char*)fileData, size); + + //Write data out. + out.write((char*)fileData, size); + + //Free memory. + delete[] fileData; + + //Update the stored offset. + itr->offset = it->offset; + } + + //Update member vars. + archiveFlags = header.archiveFlags; + fileFlags = header.fileFlags; + + in.close(); + out.close(); + + //Now rename the output file. + /* if (fs::path(path).extension().string() == ".new") { + try { + fs::rename(path, fs::path(path).stem()); + } catch (fs::filesystem_error& e) { + throw error(LIBBSA_ERROR_FILESYSTEM_ERROR, e.what()); + } + }*/ + } + + std::pair BSA::ReadData(std::ifstream& in, const BsaAsset& data) const { + uint8_t * outBuffer = NULL; + uint32_t outSize = data.size; + + // Remove compression flag from size to get actual size. + if (outSize & FILE_INVERT_COMPRESSED) + outSize ^= FILE_INVERT_COMPRESSED; + + try { + outBuffer = new uint8_t[outSize]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + in.seekg(data.offset, ios_base::beg); + in.read(reinterpret_cast(outBuffer), outSize); + + // If file is compressed, need to uncompress it with zlib. + if ((archiveFlags & BSA_COMPRESSED) != (outSize & FILE_INVERT_COMPRESSED)) + return uncompressData(data.path, outBuffer, outSize); + + return make_pair(outBuffer, outSize); + } + + std::pair BSA::uncompressData(const std::string& assetPath, + const uint8_t * data, + size_t size) { + size_t uncompressedSize = *reinterpret_cast(data); + data += sizeof(uint32_t); + size -= sizeof(uint32_t); + + uint8_t * uncompressedData; + try { + uncompressedData = new uint8_t[uncompressedSize]; + } + catch (bad_alloc& e) { + throw error(LIBBSA_ERROR_NO_MEM, e.what()); + } + + // We can use a pre-made utility function instead of having to mess around with zlib proper. + int ret = uncompress(uncompressedData, reinterpret_cast(&uncompressedSize), data, size); + if (ret != Z_OK) + throw error(LIBBSA_ERROR_ZLIB_ERROR, "Uncompressing of \"" + assetPath + "\" failed."); + + // Free memory. + data -= sizeof(uint32_t); + delete[] data; + + return make_pair(uncompressedData, uncompressedSize); + } + + std::string BSA::getFolderName(const uint8_t * fileRecords, uint32_t folderOffset) { + const char * folderName = reinterpret_cast(fileRecords + folderOffset + 1); + uint8_t folderNameLength = *(fileRecords + folderOffset) - 1; + + return ToUTF8(string(folderName, folderNameLength)); + } + + std::string BSA::getFileName(const uint8_t * fileNames, uint32_t offset) { + const char * filename = reinterpret_cast(fileNames + offset); + + //Find position of null character. + char * nullTerminatorPos = strchr(const_cast(filename), '\0'); + if (nullTerminatorPos == NULL) + throw error(LIBBSA_ERROR_PARSE_FAIL, "String at " + to_string(*(size_t*)filename) + "is not null terminated."); + + return ToUTF8(string(filename, nullTerminatorPos - filename)); + } + + uint32_t BSA::HashString(const std::string& str) { + uint32_t hash = 0; + for (size_t i = 0, len = str.length(); i < len; i++) { + hash = 0x1003F * hash + (uint8_t)str[i]; + } + return hash; + } + + uint64_t BSA::CalcHash(const std::string& path, const std::string& ext) { + uint64_t hash1 = 0; + uint32_t hash2 = 0; + uint32_t hash3 = 0; + const size_t len = path.length(); + + if (!path.empty()) { + hash1 = (uint64_t)( + ((uint8_t)path[len - 1]) + + (len << 16) + + ((uint8_t)path[0] << 24) + ); + + if (len > 2) { + hash1 += ((uint8_t)path[len - 2] << 8); + if (len > 3) + hash2 = HashString(path.substr(1, len - 3)); + } + } + + if (!ext.empty()) { + if (ext == ".kf") + hash1 += 0x80; + else if (ext == ".nif") + hash1 += 0x8000; + else if (ext == ".dds") + hash1 += 0x8080; + else if (ext == ".wav") + hash1 += 0x80000000; + + hash3 = HashString(ext); + } + + hash2 = hash2 + hash3; + return ((uint64_t)hash2 << 32) + hash1; + } + + bool BSA::hash_comp(const BsaAsset& first, const BsaAsset& second) { + return first.hash < second.hash; + } + + bool BSA::path_comp(const BsaAsset& first, const BsaAsset& second) { + return first.path == second.path; + } + + //Check if a given file is a Sse-type BSA. + bool BSA::IsBSA(const boost::filesystem::path& path) { + //Check if file exists. + if (!fs::exists(path)) + return false; + + boost::filesystem::ifstream in(fs::path(path), ios::binary); + in.exceptions(ios::failbit | ios::badbit | ios::eofbit); + + uint32_t magic; + uint32_t version; + in.read((char*)&magic, sizeof(uint32_t)); + in.read((char*)&version, sizeof(uint32_t)); + in.close(); + + return (magic == BSA_MAGIC) && (version == BSA_VERSION_SSE); + } } } diff --git a/src/api/ssebsa.h b/src/api/ssebsa.h index 8108c6e..ce2cd15 100644 --- a/src/api/ssebsa.h +++ b/src/api/ssebsa.h @@ -37,70 +37,73 @@ */ namespace libbsa { - namespace sse { - - const uint32_t BSA_MAGIC = '\0ASB'; - const uint32_t BSA_VERSION_SSE = 0x69; - - const uint32_t BSA_FOLDER_RECORD_OFFSET = 36; //Folder record offset for SSE-type BSAs is constant. - - const uint32_t BSA_COMPRESSED = 0x0004; //If this flag is present in the archiveFlags header field, then the BSA file data is compressed. - - const uint32_t FILE_INVERT_COMPRESSED = 0x40000000; //Inverts the file data compression status for the specific file this flag is set for. - - struct Header { - uint32_t fileId; - uint32_t version; - uint32_t offset; - uint32_t archiveFlags; - uint32_t folderCount; - uint32_t fileCount; - uint32_t totalFolderNameLength; - uint32_t totalFileNameLength; - uint32_t fileFlags; - }; - - struct FolderRecord { - uint64_t nameHash; //Hash of folder name. - uint32_t count; //Number of files in folder. - uint32_t unk; //Unknown - uint64_t offset; //Offset to the fileRecords for this folder, including the folder name, from the beginning of the file. - }; - - struct FileRecord { - uint64_t nameHash; //Hash of the filename. - uint32_t size; //Size of the data. See TES4Mod wiki page for details. - uint32_t offset; //Offset to the raw file data, from byte 0. - }; - - //SSE-type BSA class. - class BSA : public _bsa_handle_int { - public: - BSA(const std::string& path); - void Save(std::string path, const uint32_t version, const uint32_t compression); - private: - std::pair ReadData(libbsa::ifstream& in, const libbsa::BsaAsset& data); - - uint32_t HashString(const std::string& str); - uint64_t CalcHash(const std::string& path, const std::string& ext); - - uint32_t archiveFlags; - uint32_t fileFlags; - }; - - bool hash_comp(const BsaAsset& first, const BsaAsset& second); - - //Comparison class for list::unique. - class path_comp { - public: - bool operator() (const BsaAsset& first, const BsaAsset& second) { - return first.path == second.path; - } - }; - - //Check if a given file is a Tes4-type BSA. - bool IsBSA(const std::string& path); - } + namespace sse { + //Sse-type BSA class. + class BSA : public GenericBsa { + public: + static const uint32_t BSA_MAGIC = '\0ASB'; //Also for TES5, FO3 and probably FNV too. + static const uint32_t BSA_VERSION_SSE = 0x69; + + static const uint32_t BSA_FOLDER_RECORD_OFFSET = 36; //Folder record offset for TES4-type BSAs is constant. + + static const uint32_t BSA_COMPRESSED = 0x0004; //If this flag is present in the archiveFlags header field, then the BSA file data is compressed. + + static const uint32_t FILE_INVERT_COMPRESSED = 0x40000000; //Inverts the file data compression status for the specific file this flag is set for. + + BSA(const boost::filesystem::path& path); + void Save(const boost::filesystem::path& path, + const uint32_t version, + const uint32_t compression); + + //Check if a given file is a Tes4-type BSA. + static bool IsBSA(const boost::filesystem::path& path); + private: + std::pair ReadData(std::ifstream& in, + const BsaAsset& data) const; + static std::pair uncompressData(const std::string& assetPath, + const uint8_t * data, + size_t size); + + static std::string getFolderName(const uint8_t * fileRecords, + uint32_t folderOffset); + static std::string getFileName(const uint8_t * fileNames, + uint32_t offset); + + static uint32_t HashString(const std::string& str); + static uint64_t CalcHash(const std::string& assetPath, const std::string& ext); + + uint32_t archiveFlags; + uint32_t fileFlags; + + static bool hash_comp(const BsaAsset& first, const BsaAsset& second); + static bool path_comp(const BsaAsset& first, const BsaAsset& second); + + struct Header { + uint32_t fileId; + uint32_t version; + uint32_t offset; + uint32_t archiveFlags; + uint32_t folderCount; + uint32_t fileCount; + uint32_t totalFolderNameLength; + uint32_t totalFileNameLength; + uint32_t fileFlags; + }; + + struct FolderRecord { + uint64_t nameHash; //Hash of folder name. + uint32_t count; //Number of files in folder. + uint32_t unk; //Unknown + uint64_t offset; //Offset to the fileRecords for this folder, including the folder name, from the beginning of the file. + }; + + struct FileRecord { + uint64_t nameHash; //Hash of the filename. + uint32_t size; //Size of the data. See TES4Mod wiki page for details. + uint32_t offset; //Offset to the raw file data, from byte 0. + }; + }; + } } #endif \ No newline at end of file From 7814521a7ba44ef5ef9085e95195bbaf52f25ddd Mon Sep 17 00:00:00 2001 From: Mator Date: Sun, 4 Dec 2016 20:24:54 -0800 Subject: [PATCH 4/6] fixed typo in _bsa_handle_int --- src/api/_bsa_handle_int.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/_bsa_handle_int.cpp b/src/api/_bsa_handle_int.cpp index 56df516..505fbb3 100644 --- a/src/api/_bsa_handle_int.cpp +++ b/src/api/_bsa_handle_int.cpp @@ -35,8 +35,8 @@ _bsa_handle_int::_bsa_handle_int(const boost::filesystem::path& path) : extAssetsNum(0) { if (tes3::BSA::IsBSA(path)) bsa = new tes3::BSA(path); - else if (sse::BSA::IsBSA(pathc)) - bsa = new sse::BSA(pathc); + else if (sse::BSA::IsBSA(path)) + bsa = new sse::BSA(path); else bsa = new tes4::BSA(path); } From 6094e84d0e4f0d725363a6cd1c08a399e2c9a637 Mon Sep 17 00:00:00 2001 From: Mator Date: Sun, 4 Dec 2016 20:32:03 -0800 Subject: [PATCH 5/6] added ssebsa files to cmakelists --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16967c7..9efc8b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,7 +53,8 @@ set (PROJECT_SRC "${CMAKE_SOURCE_DIR}/src/api/_bsa_handle_int.cpp" "${CMAKE_SOURCE_DIR}/src/api/genericbsa.cpp" "${CMAKE_SOURCE_DIR}/src/api/libbsa.cpp" "${CMAKE_SOURCE_DIR}/src/api/tes3bsa.cpp" - "${CMAKE_SOURCE_DIR}/src/api/tes4bsa.cpp") + "${CMAKE_SOURCE_DIR}/src/api/tes4bsa.cpp" + "${CMAKE_SOURCE_DIR}/src/api/ssebsa.cpp") set (PROJECT_HEADERS "${CMAKE_SOURCE_DIR}/include/libbsa/libbsa.h" "${CMAKE_SOURCE_DIR}/src/api/_bsa_handle_int.h" @@ -61,7 +62,8 @@ set (PROJECT_HEADERS "${CMAKE_SOURCE_DIR}/include/libbsa/libbsa.h" "${CMAKE_SOURCE_DIR}/src/api/error.h" "${CMAKE_SOURCE_DIR}/src/api/genericbsa.h" "${CMAKE_SOURCE_DIR}/src/api/tes3bsa.h" - "${CMAKE_SOURCE_DIR}/src/api/tes4bsa.h") + "${CMAKE_SOURCE_DIR}/src/api/tes4bsa.h" + "${CMAKE_SOURCE_DIR}/src/api/ssebsa.h") set (TEST_SRC "${CMAKE_SOURCE_DIR}/src/test/main.cpp") From a59c534354659a8cf3b1698870f8bba70a1c4364 Mon Sep 17 00:00:00 2001 From: Mator Date: Mon, 5 Dec 2016 14:07:30 -0800 Subject: [PATCH 6/6] fixed a bit of whitespace --- src/api/_bsa_handle_int.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/_bsa_handle_int.cpp b/src/api/_bsa_handle_int.cpp index 505fbb3..a30e9de 100644 --- a/src/api/_bsa_handle_int.cpp +++ b/src/api/_bsa_handle_int.cpp @@ -35,7 +35,7 @@ _bsa_handle_int::_bsa_handle_int(const boost::filesystem::path& path) : extAssetsNum(0) { if (tes3::BSA::IsBSA(path)) bsa = new tes3::BSA(path); - else if (sse::BSA::IsBSA(path)) + else if (sse::BSA::IsBSA(path)) bsa = new sse::BSA(path); else bsa = new tes4::BSA(path);