added 5 changesets to branch 'refs/remotes/weinhold-github/new-hpkg-format' old head: ffba501a8a70da82e12173dcfe64e6b44038b174 new head: ef6c87440805bde7c7da1210d6fcdced1d2a8cb6 overview: https://github.com/weinhold/HaikuPM/compare/ffba501...ef6c874 ---------------------------------------------------------------------------- e914000: package: Suppress version mismatch errors where V1 is supported * Add flags parameter to Init() of BPackageReader and friends. * Introduce flag B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE and don't print a version mismatch error when given. * package extract/list: Use the new flag. f1fa982: packagefs: PackageFile::DataAccessor: Avoid unnecessary locking ... in case of version 2 package files. cc384fd: PackageFileHeapAccessorBase: Fix error output 2395d46: packagefs: Fix attribute indexing Since the package nodes' attributes are indexed before the VFS has accessed any of its nodes, the package wasn't open and reading the attribute data would fail. We do now open the package explicitly in UnpackingAttributeCookie::IndexAttribute(). Moreover, as an optimization, we also open the package in Volume::_AddPackageContent(), so the package file isn't repeatedly opened and closed as its nodes are being registered. ef6c874: packagefs: Add caching for the package file heap reader * ReaderImplBase: - Add virtual CreateCachedHeapReader() which can create a cached reader based on the given heap reader. - Rename HeapReader() to RawHeapReader() and add HeapReader() for the cached heap reader. - Add DetachHeapReader() to allow a clients to remove the heap reader(s) after deleting the ReaderImplBase object. * packagefs: - Add CachedDataReader class, which wraps a given BAbstractBufferedDataReader and provides caching for it using a VMCache. The implementation is based on the IOCache implementation. - Use CachedDataReader to wrap the heap reader. For file data that means they are cached twice -- in the heap reader cache and in the file cache -- but due to the heap reader using a VMCache as well, the pages will be recycled automatically anyway. For attribute data the cache should be very helpful, since they weren't cached at all before. [ Ingo Weinhold <ingo_weinhold@xxxxxx> ] ---------------------------------------------------------------------------- 21 files changed, 841 insertions(+), 75 deletions(-) headers/os/package/hpkg/HPKGDefs.h | 8 + headers/os/package/hpkg/PackageReader.h | 4 +- headers/private/package/hpkg/PackageReaderImpl.h | 8 +- headers/private/package/hpkg/ReaderImplBase.h | 38 +- .../kernel/file_systems/packagefs/Jamfile | 1 + .../packagefs/nodes/UnpackingAttributeCookie.cpp | 9 + .../packagefs/package/CachedDataReader.cpp | 455 +++++++++++++++++++ .../packagefs/package/CachedDataReader.h | 152 +++++++ .../file_systems/packagefs/package/Package.cpp | 84 ++-- .../file_systems/packagefs/package/Package.h | 1 + .../packagefs/package/PackageFile.cpp | 3 +- .../file_systems/packagefs/volume/Volume.cpp | 10 + src/bin/package/command_extract.cpp | 16 +- src/bin/package/command_list.cpp | 16 +- .../package/hpkg/PackageFileHeapAccessorBase.cpp | 4 +- src/kits/package/hpkg/PackageReader.cpp | 8 +- src/kits/package/hpkg/PackageReaderImpl.cpp | 11 +- src/kits/package/hpkg/PackageWriterImpl.cpp | 4 +- src/kits/package/hpkg/ReaderImplBase.cpp | 75 ++- src/kits/package/hpkg/RepositoryReaderImpl.cpp | 5 +- .../loader/file_systems/packagefs/packagefs.cpp | 4 +- ############################################################################ Commit: e9140000b561dff99ba6d74668e55bd0a2a7a3b8 Author: Ingo Weinhold <ingo_weinhold@xxxxxx> Date: Mon May 20 18:15:59 2013 UTC package: Suppress version mismatch errors where V1 is supported * Add flags parameter to Init() of BPackageReader and friends. * Introduce flag B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE and don't print a version mismatch error when given. * package extract/list: Use the new flag. ---------------------------------------------------------------------------- diff --git a/headers/os/package/hpkg/HPKGDefs.h b/headers/os/package/hpkg/HPKGDefs.h index b11a788..9f1029e 100644 --- a/headers/os/package/hpkg/HPKGDefs.h +++ b/headers/os/package/hpkg/HPKGDefs.h @@ -166,6 +166,14 @@ enum { }; +// Reader Init() flags +enum { + B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE = 0x01 + // Fail silently when encountering a package format version mismatch. + // Don't print anything to the error output. +}; + + enum { B_HPKG_COMPRESSION_LEVEL_NONE = 0, B_HPKG_COMPRESSION_LEVEL_FASTEST = 1, diff --git a/headers/os/package/hpkg/PackageReader.h b/headers/os/package/hpkg/PackageReader.h index 3828a4b..e5b8369 100644 --- a/headers/os/package/hpkg/PackageReader.h +++ b/headers/os/package/hpkg/PackageReader.h @@ -31,8 +31,8 @@ public: BPackageReader(BErrorOutput* errorOutput); ~BPackageReader(); - status_t Init(const char* fileName); - status_t Init(int fd, bool keepFD); + status_t Init(const char* fileName, uint32 flags = 0); + status_t Init(int fd, bool keepFD, uint32 flags = 0); status_t ParseContent( BPackageContentHandler* contentHandler); status_t ParseContent(BLowLevelPackageContentHandler* diff --git a/headers/private/package/hpkg/PackageReaderImpl.h b/headers/private/package/hpkg/PackageReaderImpl.h index 7dd0b82..d506b39 100644 --- a/headers/private/package/hpkg/PackageReaderImpl.h +++ b/headers/private/package/hpkg/PackageReaderImpl.h @@ -28,8 +28,8 @@ public: PackageReaderImpl(BErrorOutput* errorOutput); ~PackageReaderImpl(); - status_t Init(const char* fileName); - status_t Init(int fd, bool keepFD); + status_t Init(const char* fileName, uint32 flags); + status_t Init(int fd, bool keepFD, uint32 flags); status_t ParseContent( BPackageContentHandler* contentHandler); status_t ParseContent(BLowLevelPackageContentHandler* diff --git a/headers/private/package/hpkg/ReaderImplBase.h b/headers/private/package/hpkg/ReaderImplBase.h index a8f6830..07fc879 100644 --- a/headers/private/package/hpkg/ReaderImplBase.h +++ b/headers/private/package/hpkg/ReaderImplBase.h @@ -89,7 +89,8 @@ protected: protected: template<typename Header, uint32 kMagic, uint16 kVersion> - status_t Init(int fd, bool keepFD, Header& header); + status_t Init(int fd, bool keepFD, Header& header, + uint32 flags); status_t InitHeapReader(uint32 compression, uint32 chunkSize, off_t offset, uint64 compressedSize, @@ -316,7 +317,7 @@ private: template<typename Header, uint32 kMagic, uint16 kVersion> status_t -ReaderImplBase::Init(int fd, bool keepFD, Header& header) +ReaderImplBase::Init(int fd, bool keepFD, Header& header, uint32 flags) { status_t error = _Init(fd, keepFD); if (error != B_OK) @@ -345,9 +346,11 @@ ReaderImplBase::Init(int fd, bool keepFD, Header& header) // version if (B_BENDIAN_TO_HOST_INT16(header.version) != kVersion) { - ErrorOutput()->PrintError("Error: Invalid/unsupported %s file " - "version (%d)\n", fFileType, - B_BENDIAN_TO_HOST_INT16(header.version)); + if ((flags & B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE) == 0) { + ErrorOutput()->PrintError("Error: Invalid/unsupported %s file " + "version (%d)\n", fFileType, + B_BENDIAN_TO_HOST_INT16(header.version)); + } return B_MISMATCHED_VALUES; } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp index 643aa44..13d5be7 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp @@ -776,14 +776,15 @@ Package::Load() // try current package file format version { PackageReaderImpl packageReader(&errorOutput); - status_t error = packageReader.Init(fd, false); + status_t error = packageReader.Init(fd, false, + BHPKG::B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE); if (error == B_OK) { // parse content LoaderContentHandler handler(this); error = handler.Init(); if (error != B_OK) RETURN_ERROR(error); - + error = packageReader.ParseContent(&handler); if (error != B_OK) RETURN_ERROR(error); diff --git a/src/bin/package/command_extract.cpp b/src/bin/package/command_extract.cpp index ae5a607..41d7fb8 100644 --- a/src/bin/package/command_extract.cpp +++ b/src/bin/package/command_extract.cpp @@ -82,6 +82,12 @@ struct VersionPolicyV1 { return data.UncompressedSize(); } + static inline status_t InitReader(PackageReader& packageReader, + const char* fileName) + { + return packageReader.Init(fileName); + } + static status_t GetHeapReader(PackageReader& packageReader, HeapReaderBase*& _heapReader, bool& _mustDelete) { @@ -128,6 +134,14 @@ struct VersionPolicyV2 { return data.Size(); } + static inline status_t InitReader(PackageReader& packageReader, + const char* fileName) + { + return packageReader.Init(fileName, + BPackageKit::BHPKG + ::B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE); + } + static status_t GetHeapReader(PackageReader& packageReader, HeapReaderBase*& _heapReader, bool& _mustDelete) { @@ -755,7 +769,7 @@ do_extract(const char* packageFileName, const char* changeToDirectory, } typename VersionPolicy::PackageReader packageReader(&errorOutput); - status_t error = packageReader.Init(packageFileName); + status_t error = VersionPolicy::InitReader(packageReader, packageFileName); if (error != B_OK) { if (ignoreVersionError && error == B_MISMATCHED_VALUES) return; diff --git a/src/bin/package/command_list.cpp b/src/bin/package/command_list.cpp index b6f0b41..caebcbf 100644 --- a/src/bin/package/command_list.cpp +++ b/src/bin/package/command_list.cpp @@ -47,6 +47,12 @@ struct VersionPolicyV1 { { return data.UncompressedSize(); } + + static inline status_t InitReader(PackageReader& packageReader, + const char* fileName) + { + return packageReader.Init(fileName); + } }; struct VersionPolicyV2 { @@ -60,6 +66,14 @@ struct VersionPolicyV2 { { return data.Size(); } + + static inline status_t InitReader(PackageReader& packageReader, + const char* fileName) + { + return packageReader.Init(fileName, + BPackageKit::BHPKG + ::B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE); + } }; @@ -370,7 +384,7 @@ do_list(const char* packageFileName, bool listAttributes, bool filePathsOnly, // open package BStandardErrorOutput errorOutput; typename VersionPolicy::PackageReader packageReader(&errorOutput); - status_t error = packageReader.Init(packageFileName); + status_t error = VersionPolicy::InitReader(packageReader, packageFileName); if (error != B_OK) { if (ignoreVersionError && error == B_MISMATCHED_VALUES) return; diff --git a/src/kits/package/hpkg/PackageReader.cpp b/src/kits/package/hpkg/PackageReader.cpp index 8453d25..1ddeb15 100644 --- a/src/kits/package/hpkg/PackageReader.cpp +++ b/src/kits/package/hpkg/PackageReader.cpp @@ -33,22 +33,22 @@ BPackageReader::~BPackageReader() status_t -BPackageReader::Init(const char* fileName) +BPackageReader::Init(const char* fileName, uint32 flags) { if (fImpl == NULL) return B_NO_INIT; - return fImpl->Init(fileName); + return fImpl->Init(fileName, flags); } status_t -BPackageReader::Init(int fd, bool keepFD) +BPackageReader::Init(int fd, bool keepFD, uint32 flags) { if (fImpl == NULL) return B_NO_INIT; - return fImpl->Init(fd, keepFD); + return fImpl->Init(fd, keepFD, flags); } diff --git a/src/kits/package/hpkg/PackageReaderImpl.cpp b/src/kits/package/hpkg/PackageReaderImpl.cpp index bc21926..acee5c1 100644 --- a/src/kits/package/hpkg/PackageReaderImpl.cpp +++ b/src/kits/package/hpkg/PackageReaderImpl.cpp @@ -318,7 +318,7 @@ PackageReaderImpl::~PackageReaderImpl() status_t -PackageReaderImpl::Init(const char* fileName) +PackageReaderImpl::Init(const char* fileName, uint32 flags) { // open file int fd = open(fileName, O_RDONLY); @@ -328,16 +328,16 @@ PackageReaderImpl::Init(const char* fileName) return errno; } - return Init(fd, true); + return Init(fd, true, flags); } status_t -PackageReaderImpl::Init(int fd, bool keepFD) +PackageReaderImpl::Init(int fd, bool keepFD, uint32 flags) { hpkg_header header; status_t error = inherited::Init<hpkg_header, B_HPKG_MAGIC, B_HPKG_VERSION>( - fd, keepFD, header); + fd, keepFD, header, flags); if (error != B_OK) return error; fHeapSize = HeapReader()->UncompressedHeapSize(); diff --git a/src/kits/package/hpkg/PackageWriterImpl.cpp b/src/kits/package/hpkg/PackageWriterImpl.cpp index 204d3b2..7c519c3 100644 --- a/src/kits/package/hpkg/PackageWriterImpl.cpp +++ b/src/kits/package/hpkg/PackageWriterImpl.cpp @@ -624,7 +624,7 @@ PackageWriterImpl::_Init(const char* fileName, // in update mode, parse the TOC if ((Flags() & B_HPKG_WRITER_UPDATE_PACKAGE) != 0) { PackageReaderImpl packageReader(fListener); - result = packageReader.Init(FD(), false); + result = packageReader.Init(FD(), false, 0); if (result != B_OK) return result; diff --git a/src/kits/package/hpkg/RepositoryReaderImpl.cpp b/src/kits/package/hpkg/RepositoryReaderImpl.cpp index 0dc6467..f59cb8f 100644 --- a/src/kits/package/hpkg/RepositoryReaderImpl.cpp +++ b/src/kits/package/hpkg/RepositoryReaderImpl.cpp @@ -74,7 +74,7 @@ RepositoryReaderImpl::Init(int fd, bool keepFD) { hpkg_repo_header header; status_t error = inherited::Init<hpkg_repo_header, B_HPKG_REPO_MAGIC, - B_HPKG_REPO_VERSION>(fd, keepFD, header); + B_HPKG_REPO_VERSION>(fd, keepFD, header, 0); if (error != B_OK) return error; diff --git a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp index 390fe2b..6257d53 100644 --- a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp +++ b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp @@ -759,7 +759,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory) { PackageLoaderErrorOutput errorOutput; PackageReaderImpl packageReader(&errorOutput); - status_t error = packageReader.Init(fd, false); + status_t error = packageReader.Init(fd, false, 0); if (error != B_OK) RETURN_ERROR(error); ############################################################################ Commit: f1fa9826acebd2fb4bc7f12572ce6eee9f2a1efa Author: Ingo Weinhold <ingo_weinhold@xxxxxx> Date: Mon May 20 20:54:59 2013 UTC packagefs: PackageFile::DataAccessor: Avoid unnecessary locking ... in case of version 2 package files. ---------------------------------------------------------------------------- diff --git a/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp b/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp index 0810673..2fa558c 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp @@ -105,7 +105,8 @@ struct PackageFile::DataAccessor { if (toRead > 0) { IORequestOutput output(request); - MutexLocker locker(fLock); + MutexLocker locker(fLock, false, fData->Version() == 1); + // V2 readers are reentrant status_t error = fReader->ReadDataToOutput(offset, toRead, &output); if (error != B_OK) RETURN_ERROR(error); ############################################################################ Commit: cc384fd837cc8dc191f23c4e9ddb9a76ea752dcf Author: Ingo Weinhold <ingo_weinhold@xxxxxx> Date: Mon May 20 22:36:51 2013 UTC PackageFileHeapAccessorBase: Fix error output ---------------------------------------------------------------------------- diff --git a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp index a556dbe..78461ad 100644 --- a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp +++ b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp @@ -237,8 +237,8 @@ PackageFileHeapAccessorBase::ReadFileData(uint64 offset, void* buffer, { ssize_t bytesRead = pread(fFD, buffer, size, fHeapOffset + (off_t)offset); if (bytesRead < 0) { - fErrorOutput->PrintError("ReadFileData(%" B_PRIu64 "%p, %zu) failed to " - "read data: %s\n", offset, buffer, size, strerror(errno)); + fErrorOutput->PrintError("ReadFileData(%" B_PRIu64 ", %p, %zu) failed " + "to read data: %s\n", offset, buffer, size, strerror(errno)); return errno; } if ((size_t)bytesRead != size) { ############################################################################ Commit: 2395d46941ab6fcd3772ed48981420762e98f10e Author: Ingo Weinhold <ingo_weinhold@xxxxxx> Date: Mon May 20 22:41:31 2013 UTC packagefs: Fix attribute indexing Since the package nodes' attributes are indexed before the VFS has accessed any of its nodes, the package wasn't open and reading the attribute data would fail. We do now open the package explicitly in UnpackingAttributeCookie::IndexAttribute(). Moreover, as an optimization, we also open the package in Volume::_AddPackageContent(), so the package file isn't repeatedly opened and closed as its nodes are being registered. ---------------------------------------------------------------------------- diff --git a/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp b/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp index 81ec8cb..867380f 100644 --- a/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp @@ -163,6 +163,15 @@ UnpackingAttributeCookie::IndexAttribute(PackageNode* packageNode, // read the attribute if (toRead > 0) { + // The package must be open or otherwise reading the attribute data + // may fail. + int fd = packageNode->GetPackage()->Open(); + if (fd < 0) { + indexer->DeleteCookie(); + return fd; + } + PackageCloser packageCloser(packageNode->GetPackage()); + error = ReadAttribute(packageNode, attribute, 0, data, &toRead); if (error != B_OK) { indexer->DeleteCookie(); diff --git a/src/add-ons/kernel/file_systems/packagefs/volume/Volume.cpp b/src/add-ons/kernel/file_systems/packagefs/volume/Volume.cpp index 4a53765..fc24afe 100644 --- a/src/add-ons/kernel/file_systems/packagefs/volume/Volume.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/volume/Volume.cpp @@ -949,6 +949,16 @@ Volume::_FindPackage(const char* fileName) const status_t Volume::_AddPackageContent(Package* package, bool notify) { + // Open the package. We don't need the FD here, but this is an optimization. + // The attribute indices may want to read the package nodes' attributes and + // the package file would be opened and closed for each attribute instance. + // Since Package keeps and shares the FD as long as at least one party has + // the package open, we prevent that. + int fd = package->Open(); + if (fd < 0) + RETURN_ERROR(fd); + PackageCloser packageCloser(package); + status_t error = fPackageFSRoot->AddPackage(package); if (error != B_OK) RETURN_ERROR(error); ############################################################################ Commit: ef6c87440805bde7c7da1210d6fcdced1d2a8cb6 Author: Ingo Weinhold <ingo_weinhold@xxxxxx> Date: Tue May 21 17:42:57 2013 UTC packagefs: Add caching for the package file heap reader * ReaderImplBase: - Add virtual CreateCachedHeapReader() which can create a cached reader based on the given heap reader. - Rename HeapReader() to RawHeapReader() and add HeapReader() for the cached heap reader. - Add DetachHeapReader() to allow a clients to remove the heap reader(s) after deleting the ReaderImplBase object. * packagefs: - Add CachedDataReader class, which wraps a given BAbstractBufferedDataReader and provides caching for it using a VMCache. The implementation is based on the IOCache implementation. - Use CachedDataReader to wrap the heap reader. For file data that means they are cached twice -- in the heap reader cache and in the file cache -- but due to the heap reader using a VMCache as well, the pages will be recycled automatically anyway. For attribute data the cache should be very helpful, since they weren't cached at all before. ---------------------------------------------------------------------------- diff --git a/headers/private/package/hpkg/PackageReaderImpl.h b/headers/private/package/hpkg/PackageReaderImpl.h index d506b39..5ef4e08 100644 --- a/headers/private/package/hpkg/PackageReaderImpl.h +++ b/headers/private/package/hpkg/PackageReaderImpl.h @@ -40,7 +40,9 @@ public: uint64 HeapOffset() const; uint64 HeapSize() const; - PackageFileHeapReader* HeapReader() const + PackageFileHeapReader* RawHeapReader() const + { return inherited::RawHeapReader(); } + BAbstractBufferedDataReader* HeapReader() const { return inherited::HeapReader(); } protected: diff --git a/headers/private/package/hpkg/ReaderImplBase.h b/headers/private/package/hpkg/ReaderImplBase.h index 07fc879..5324812 100644 --- a/headers/private/package/hpkg/ReaderImplBase.h +++ b/headers/private/package/hpkg/ReaderImplBase.h @@ -26,6 +26,7 @@ namespace BPackageKit { namespace BHPKG { +class BAbstractBufferedDataReader; class BErrorOutput; @@ -71,8 +72,23 @@ protected: BErrorOutput* ErrorOutput() const; - PackageFileHeapReader* HeapReader() const + uint64 UncompressedHeapSize() const; + + PackageFileHeapReader* RawHeapReader() const + { return fRawHeapReader; } + BAbstractBufferedDataReader* HeapReader() const { return fHeapReader; } + // equals RawHeapReader(), if uncached + + BAbstractBufferedDataReader* DetachHeapReader( + PackageFileHeapReader** _rawHeapReader + = NULL); + // Detaches both raw and (if applicable) + // cached heap reader. The called gains + // ownership. The FD may need to be set on + // the raw heap reader, if it shall be used + // after destroying this object and Init() + // has been called with keepFD == true. protected: class AttributeHandlerContext; @@ -95,6 +111,10 @@ protected: uint32 chunkSize, off_t offset, uint64 compressedSize, uint64 uncompressedSize); + virtual status_t CreateCachedHeapReader( + PackageFileHeapReader* heapReader, + BAbstractBufferedDataReader*& + _cachedReader); status_t InitSection(PackageFileSection& section, uint64 endOffset, uint64 length, uint64 maxSaneLength, uint64 stringsLength, @@ -156,7 +176,8 @@ private: int fFD; bool fOwnsFD; - PackageFileHeapReader* fHeapReader; + PackageFileHeapReader* fRawHeapReader; + BAbstractBufferedDataReader* fHeapReader; PackageFileSection* fCurrentSection; diff --git a/src/add-ons/kernel/file_systems/packagefs/Jamfile b/src/add-ons/kernel/file_systems/packagefs/Jamfile index e2061a8..5102cd4 100644 --- a/src/add-ons/kernel/file_systems/packagefs/Jamfile +++ b/src/add-ons/kernel/file_systems/packagefs/Jamfile @@ -24,6 +24,7 @@ HAIKU_PACKAGE_FS_SOURCES = AttributeIndex.cpp AutoPackageAttributes.cpp BlockBufferPoolKernel.cpp + CachedDataReader.cpp DebugSupport.cpp Dependency.cpp Directory.cpp diff --git a/src/add-ons/kernel/file_systems/packagefs/package/CachedDataReader.cpp b/src/add-ons/kernel/file_systems/packagefs/package/CachedDataReader.cpp new file mode 100644 index 0000000..fa6f3fc --- /dev/null +++ b/src/add-ons/kernel/file_systems/packagefs/package/CachedDataReader.cpp @@ -0,0 +1,455 @@ +/* + * Copyright 2010-2013, Ingo Weinhold, ingo_weinhold@xxxxxx. + * Distributed under the terms of the MIT License. + */ + + +#include "CachedDataReader.h" + +#include <algorithm> + +#include <package/hpkg/DataOutput.h> + +#include <util/AutoLock.h> +#include <vm/VMCache.h> +#include <vm/vm_page.h> + +#include "DebugSupport.h" + + +using BPackageKit::BHPKG::BBufferDataReader; +using BPackageKit::BHPKG::BBufferDataOutput; + + +static inline bool +page_physical_number_less(const vm_page* a, const vm_page* b) +{ + return a->physical_page_number < b->physical_page_number; +} + + +// #pragma mark - PagesDataOutput + + +struct CachedDataReader::PagesDataOutput : public BDataOutput { + PagesDataOutput(vm_page** pages, size_t pageCount) + : + fPages(pages), + fPageCount(pageCount), + fInPageOffset(0) + { + } + + virtual status_t WriteData(const void* buffer, size_t size) + { + while (size > 0) { + if (fPageCount == 0) + return B_BAD_VALUE; + + size_t toCopy = std::min(size, B_PAGE_SIZE - fInPageOffset); + status_t error = vm_memcpy_to_physical( + fPages[0]->physical_page_number * B_PAGE_SIZE + fInPageOffset, + buffer, toCopy, false); + if (error != B_OK) + return error; + + fInPageOffset += toCopy; + if (fInPageOffset == B_PAGE_SIZE) { + fInPageOffset = 0; + fPages++; + fPageCount--; + } + + buffer = (const char*)buffer + toCopy; + size -= toCopy; + } + + return B_OK; + } + +private: + vm_page** fPages; + size_t fPageCount; + size_t fInPageOffset; +}; + + +// #pragma mark - CachedDataReader + + +CachedDataReader::CachedDataReader() + : + fReader(NULL), + fCache(NULL), + fCacheLineLockers() +{ + mutex_init(&fLock, "packagefs cached reader"); +} + + +CachedDataReader::~CachedDataReader() +{ + if (fCache != NULL) { + fCache->Lock(); + fCache->ReleaseRefAndUnlock(); + } + + mutex_destroy(&fLock); +} + + +status_t +CachedDataReader::Init(BAbstractBufferedDataReader* reader, off_t size) +{ + fReader = reader; + + status_t error = fCacheLineLockers.Init(); + if (error != B_OK) + RETURN_ERROR(error); + + error = VMCacheFactory::CreateNullCache(VM_PRIORITY_SYSTEM, + fCache); + if (error != B_OK) + RETURN_ERROR(error); + + AutoLocker<VMCache> locker(fCache); + + error = fCache->Resize(size, VM_PRIORITY_SYSTEM); + if (error != B_OK) + RETURN_ERROR(error); + + return B_OK; +} + + +status_t +CachedDataReader::ReadData(off_t offset, void* buffer, size_t size) +{ + BBufferDataOutput output(buffer, size); + return ReadDataToOutput(offset, size, &output); +} + + +status_t +CachedDataReader::ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output) +{ + if (offset > fCache->virtual_end + || (off_t)size > fCache->virtual_end - offset) { + return B_BAD_VALUE; + } + + if (size == 0) + return B_OK; + + while (size > 0) { + // the start of the current cache line + off_t lineOffset = (offset / kCacheLineSize) * kCacheLineSize; + + // intersection of request and cache line + off_t cacheLineEnd = std::min(lineOffset + (off_t)kCacheLineSize, + fCache->virtual_end); + size_t requestLineLength + = std::min(cacheLineEnd - offset, (off_t)size); + + // transfer the data of the cache line + status_t error = _ReadCacheLine(lineOffset, cacheLineEnd - lineOffset, + offset, requestLineLength, output); + if (error != B_OK) + return error; + + offset = cacheLineEnd; + size -= requestLineLength; + } + + return B_OK; +} + + +status_t +CachedDataReader::_ReadCacheLine(off_t lineOffset, size_t lineSize, + off_t requestOffset, size_t requestLength, BDataOutput* output) +{ + PRINT("CachedDataReader::_ReadCacheLine(%" B_PRIdOFF ", %zu, %" B_PRIdOFF + ", %zu, %p\n", lineOffset, lineSize, requestOffset, requestLength, + output); + + CacheLineLocker cacheLineLocker(this, lineOffset); + + // check whether there are pages of the cache line and the mark them used + page_num_t firstPageOffset = lineOffset / B_PAGE_SIZE; + page_num_t linePageCount = (lineSize + B_PAGE_SIZE - 1) / B_PAGE_SIZE; + vm_page* pages[kPagesPerCacheLine] = {}; + + AutoLocker<VMCache> cacheLocker(fCache); + + page_num_t firstMissing = 0; + page_num_t lastMissing = 0; + page_num_t missingPages = 0; + page_num_t pageOffset = firstPageOffset; + + VMCachePagesTree::Iterator it = fCache->pages.GetIterator(pageOffset, true, + true); + while (pageOffset < firstPageOffset + linePageCount) { + vm_page* page = it.Next(); + page_num_t currentPageOffset; + if (page == NULL + || page->cache_offset >= firstPageOffset + linePageCount) { + page = NULL; + currentPageOffset = firstPageOffset + linePageCount; + } else + currentPageOffset = page->cache_offset; + + if (pageOffset < currentPageOffset) { + // pages are missing + if (missingPages == 0) + firstMissing = pageOffset; + lastMissing = currentPageOffset - 1; + missingPages += currentPageOffset - pageOffset; + + for (; pageOffset < currentPageOffset; pageOffset++) + pages[pageOffset - firstPageOffset] = NULL; + } + + if (page != NULL) { + pages[pageOffset++ - firstPageOffset] = page; + DEBUG_PAGE_ACCESS_START(page); + vm_page_set_state(page, PAGE_STATE_UNUSED); + DEBUG_PAGE_ACCESS_END(page); + } + } + + cacheLocker.Unlock(); + + if (missingPages > 0) { +// TODO: If the missing pages range doesn't intersect with the request, just +// satisfy the request and don't read anything at all. + // There are pages of the cache line missing. We have to allocate fresh + // ones. + + // reserve + vm_page_reservation reservation; + if (!vm_page_try_reserve_pages(&reservation, missingPages, + VM_PRIORITY_SYSTEM)) { + _DiscardPages(pages, firstMissing - firstPageOffset, missingPages); + + // fall back to uncached transfer + return fReader->ReadDataToOutput(requestOffset, requestLength, + output); + } + + // Allocate the missing pages and remove the already existing pages in + // the range from the cache. We're going to read/write the whole range + // anyway. + for (pageOffset = firstMissing; pageOffset <= lastMissing; + pageOffset++) { + page_num_t index = pageOffset - firstPageOffset; + if (pages[index] == NULL) { + pages[index] = vm_page_allocate_page(&reservation, + PAGE_STATE_UNUSED); + DEBUG_PAGE_ACCESS_END(pages[index]); + } else { + cacheLocker.Lock(); + fCache->RemovePage(pages[index]); + cacheLocker.Unlock(); + } + } + + missingPages = lastMissing - firstMissing + 1; + + // add the pages to the cache + cacheLocker.Lock(); + + for (pageOffset = firstMissing; pageOffset <= lastMissing; + pageOffset++) { + page_num_t index = pageOffset - firstPageOffset; + fCache->InsertPage(pages[index], (off_t)pageOffset * B_PAGE_SIZE); + } + + cacheLocker.Unlock(); + + // read in the missing pages + status_t error = _ReadIntoPages(pages, firstMissing - firstPageOffset, + missingPages); + if (error != B_OK) { + ERROR("CachedDataReader::_ReadCacheLine(): Failed to read into " + "cache (offset: %" B_PRIdOFF ", length: %" B_PRIuSIZE "), " + "trying uncached read (offset: %" B_PRIdOFF ", length: %" + B_PRIuSIZE ")\n", (off_t)firstMissing * B_PAGE_SIZE, + (size_t)missingPages * B_PAGE_SIZE, requestOffset, + requestLength); + + _DiscardPages(pages, firstMissing - firstPageOffset, missingPages); + + // Try again using an uncached transfer + return fReader->ReadDataToOutput(requestOffset, requestLength, + output); + } + } + + // write data to output + status_t error = _WritePages(pages, requestOffset - lineOffset, + requestLength, output); + _CachePages(pages, 0, linePageCount); + return error; +} + + +/*! Frees all pages in given range of the \a pages array. + \c NULL entries in the range are OK. All non \c NULL entries must refer + to pages with \c PAGE_STATE_UNUSED. The pages may belong to \c fCache or + may not have a cache. + \c fCache must not be locked. +*/ +void +CachedDataReader::_DiscardPages(vm_page** pages, size_t firstPage, + size_t pageCount) +{ + PRINT("%p->CachedDataReader::_DiscardPages(%" B_PRIuSIZE ", %" B_PRIuSIZE + ")\n", this, firstPage, pageCount); + + AutoLocker<VMCache> cacheLocker(fCache); + + for (size_t i = firstPage; i < firstPage + pageCount; i++) { + vm_page* page = pages[i]; + if (page == NULL) + continue; + + DEBUG_PAGE_ACCESS_START(page); + + ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED, + "page: %p @! page -m %p", page, page); + + if (page->Cache() != NULL) + fCache->RemovePage(page); + + vm_page_free(NULL, page); + } +} + + +/*! Marks all pages in the given range of the \a pages array cached. + There must not be any \c NULL entries in the given array range. All pages + must belong to \c cache and have state \c PAGE_STATE_UNUSED. + \c fCache must not be locked. +*/ +void +CachedDataReader::_CachePages(vm_page** pages, size_t firstPage, + size_t pageCount) +{ + PRINT("%p->CachedDataReader::_CachePages(%" B_PRIuSIZE ", %" B_PRIuSIZE + ")\n", this, firstPage, pageCount); + + AutoLocker<VMCache> cacheLocker(fCache); + + for (size_t i = firstPage; i < firstPage + pageCount; i++) { + vm_page* page = pages[i]; + ASSERT(page != NULL); + ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED + && page->Cache() == fCache, + "page: %p @! page -m %p", page, page); + + DEBUG_PAGE_ACCESS_START(page); + vm_page_set_state(page, PAGE_STATE_CACHED); + DEBUG_PAGE_ACCESS_END(page); + } +} + + +/*! Writes the contents of pages in \c pages to \a output. + \param pages The pages array. + \param pagesRelativeOffset The offset relative to \a pages[0] where to + start writing from. + \param requestLength The number of bytes to write. + \param output The output to which the data shall be written. + \return \c B_OK, if writing went fine, another error code otherwise. +*/ +status_t +CachedDataReader::_WritePages(vm_page** pages, size_t pagesRelativeOffset, + size_t requestLength, BDataOutput* output) +{ + PRINT("%p->CachedDataReader::_WritePages(%" B_PRIuSIZE ", %" B_PRIuSIZE + ", %p)\n", this, pagesRelativeOffset, requestLength, output); + + size_t firstPage = pagesRelativeOffset / B_PAGE_SIZE; + size_t endPage = (pagesRelativeOffset + requestLength + B_PAGE_SIZE - 1) + / B_PAGE_SIZE; + + // fallback to copying individual pages + size_t inPageOffset = pagesRelativeOffset % B_PAGE_SIZE; + for (size_t i = firstPage; i < endPage; i++) { + // map the page + void* handle; + addr_t address; + status_t error = vm_get_physical_page( + pages[i]->physical_page_number * B_PAGE_SIZE, &address, + &handle); + if (error != B_OK) + return error; + + // write the page's data + size_t toCopy = std::min(B_PAGE_SIZE - inPageOffset, requestLength); + error = output->WriteData((uint8*)(address + inPageOffset), toCopy); + + // unmap the page + vm_put_physical_page(address, handle); + + if (error != B_OK) + return error; + + inPageOffset = 0; + requestLength -= toCopy; + } + + return B_OK; +} + + +status_t +CachedDataReader::_ReadIntoPages(vm_page** pages, size_t firstPage, + size_t pageCount) +{ + PagesDataOutput output(pages + firstPage, pageCount); + + off_t firstPageOffset = (off_t)pages[firstPage]->cache_offset + * B_PAGE_SIZE; + generic_size_t requestLength = std::min( + firstPageOffset + (off_t)pageCount * B_PAGE_SIZE, + fCache->virtual_end) + - firstPageOffset; + + return fReader->ReadDataToOutput(firstPageOffset, requestLength, &output); +} + + +void +CachedDataReader::_LockCacheLine(CacheLineLocker* lineLocker) +{ + MutexLocker locker(fLock); + + CacheLineLocker* otherLineLocker + = fCacheLineLockers.Lookup(lineLocker->Offset()); + if (otherLineLocker == NULL) { + fCacheLineLockers.Insert(lineLocker); + return; + } + + // queue and wait + otherLineLocker->Queue().Add(lineLocker); + lineLocker->Wait(fLock); +} + + +void +CachedDataReader::_UnlockCacheLine(CacheLineLocker* lineLocker) +{ + MutexLocker locker(fLock); + + fCacheLineLockers.Remove(lineLocker); + + if (CacheLineLocker* nextLineLocker = lineLocker->Queue().RemoveHead()) { + nextLineLocker->Queue().MoveFrom(&lineLocker->Queue()); + fCacheLineLockers.Insert(nextLineLocker); + nextLineLocker->WakeUp(); + } +} diff --git a/src/add-ons/kernel/file_systems/packagefs/package/CachedDataReader.h b/src/add-ons/kernel/file_systems/packagefs/package/CachedDataReader.h new file mode 100644 index 0000000..7cba87c --- /dev/null +++ b/src/add-ons/kernel/file_systems/packagefs/package/CachedDataReader.h @@ -0,0 +1,152 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@xxxxxx. + * Distributed under the terms of the MIT License. + */ +#ifndef HEAP_CACHE_H +#define HEAP_CACHE_H + + +#include <package/hpkg/DataReader.h> + +#include <condition_variable.h> +#include <util/DoublyLinkedList.h> +#include <util/OpenHashTable.h> +#include <vm/vm_types.h> + + +using BPackageKit::BHPKG::BAbstractBufferedDataReader; +using BPackageKit::BHPKG::BDataOutput; +using BPackageKit::BHPKG::BDataReader; + + +class CachedDataReader : public BAbstractBufferedDataReader { +public: + CachedDataReader(); + virtual ~CachedDataReader(); + + status_t Init(BAbstractBufferedDataReader* reader, + off_t size); + + virtual status_t ReadData(off_t offset, void* buffer, + size_t size); + virtual status_t ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output); + +private: + class CacheLineLocker + : public DoublyLinkedListLinkImpl<CacheLineLocker> { + public: + CacheLineLocker(CachedDataReader* reader, off_t cacheLineOffset) + : + fReader(reader), + fOffset(cacheLineOffset) + { + fReader->_LockCacheLine(this); + } + + ~CacheLineLocker() + { + fReader->_UnlockCacheLine(this); + } + + off_t Offset() const + { + return fOffset; + } + + CacheLineLocker*& HashNext() + { + return fHashNext; + } + + DoublyLinkedList<CacheLineLocker>& Queue() + { + return fQueue; + } + + void Wait(mutex& lock) + { + fWaitCondition.Init(this, "cached reader line locker"); + ConditionVariableEntry waitEntry; + fWaitCondition.Add(&waitEntry); + mutex_unlock(&lock); + waitEntry.Wait(); + mutex_lock(&lock); + } + + void WakeUp() + { + fWaitCondition.NotifyOne(); + } + + private: + CachedDataReader* fReader; + off_t fOffset; + CacheLineLocker* fHashNext; + DoublyLinkedList<CacheLineLocker> fQueue; + ConditionVariable fWaitCondition; + }; + + friend class CacheLineLocker; + + struct LockerHashDefinition { + typedef off_t KeyType; + typedef CacheLineLocker ValueType; + + size_t HashKey(off_t key) const + { + return size_t(key / kCacheLineSize); + } + + size_t Hash(const CacheLineLocker* value) const + { + return HashKey(value->Offset()); + } + + bool Compare(off_t key, const CacheLineLocker* value) const + { + return value->Offset() == key; + } + + CacheLineLocker*& GetLink(CacheLineLocker* value) const + { + return value->HashNext(); + } + }; + + typedef BOpenHashTable<LockerHashDefinition> LockerTable; + + struct PagesDataOutput; + +private: + status_t _ReadCacheLine(off_t lineOffset, + size_t lineSize, off_t requestOffset, + size_t requestLength, BDataOutput* output); + + void _DiscardPages(vm_page** pages, size_t firstPage, + size_t pageCount); + void _CachePages(vm_page** pages, size_t firstPage, + size_t pageCount); + status_t _WritePages(vm_page** pages, + size_t pagesRelativeOffset, + size_t requestLength, BDataOutput* output); + status_t _ReadIntoPages(vm_page** pages, + size_t firstPage, size_t pageCount); + + void _LockCacheLine(CacheLineLocker* lineLocker); + void _UnlockCacheLine(CacheLineLocker* lineLocker); + +private: + static const size_t kCacheLineSize = 64 * 1024; + static const size_t kPagesPerCacheLine + = kCacheLineSize / B_PAGE_SIZE; + +private: + mutex fLock; + BAbstractBufferedDataReader* fReader; + VMCache* fCache; + LockerTable fCacheLineLockers; +}; + + +#endif // HEAP_CACHE_H diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp index 13d5be7..541ec3f 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp @@ -25,6 +25,7 @@ #include <package/hpkg/v1/PackageReaderImpl.h> #include <util/AutoLock.h> +#include "CachedDataReader.h" #include "DebugSupport.h" #include "GlobalFactory.h" #include "PackageDirectory.h" @@ -645,8 +646,8 @@ private: // #pragma mark - HeapReaderV2 -struct Package::HeapReaderV2 : public HeapReader, - private BAbstractBufferedDataReader, private BErrorOutput { +struct Package::HeapReaderV2 : public HeapReader, public CachedDataReader, + private BErrorOutput { public: HeapReaderV2() : @@ -668,6 +669,11 @@ public: fHeapReader->SetErrorOutput(this); fHeapReader->SetFD(fd); + status_t error = CachedDataReader::Init(fHeapReader, + fHeapReader->UncompressedHeapSize()); + if (error != B_OK) + return error; + return B_OK; } @@ -684,29 +690,61 @@ public: } private: - // BAbstractBufferedDataReader + // BErrorOutput - virtual status_t ReadData(off_t offset, void* buffer, size_t size) + virtual void PrintErrorVarArgs(const char* format, va_list args) { - return fHeapReader->ReadData(offset, buffer, size); + ERRORV(format, args); } - virtual status_t ReadDataToOutput(off_t offset, size_t size, - BDataOutput* output) +private: + PackageFileHeapReader* fHeapReader; +}; + + +// #pragma mark - Package + + +struct Package::CachingPackageReader : public PackageReaderImpl { + CachingPackageReader(BErrorOutput* errorOutput) + : + PackageReaderImpl(errorOutput), + fCachedHeapReader(NULL) { - return fHeapReader->ReadDataToOutput(offset, size, output); } -private: - // BErrorOutput + ~CachingPackageReader() + { + delete fCachedHeapReader; + } - virtual void PrintErrorVarArgs(const char* format, va_list args) + virtual status_t CreateCachedHeapReader( + PackageFileHeapReader* rawHeapReader, + BAbstractBufferedDataReader*& _cachedReader) { - ERRORV(format, args); + fCachedHeapReader = new(std::nothrow) HeapReaderV2; + if (fCachedHeapReader == NULL) + RETURN_ERROR(B_NO_MEMORY); + + status_t error = fCachedHeapReader->Init(rawHeapReader, FD()); + if (error != B_OK) + RETURN_ERROR(error); + + _cachedReader = fCachedHeapReader; + return B_OK; + } + + HeapReaderV2* DetachCachedHeapReader() + { + DetachHeapReader(); + + HeapReaderV2* cachedHeapReader = fCachedHeapReader; + fCachedHeapReader = NULL; + return cachedHeapReader; } private: - PackageFileHeapReader* fHeapReader; + HeapReaderV2* fCachedHeapReader; }; @@ -775,7 +813,7 @@ Package::Load() // try current package file format version { - PackageReaderImpl packageReader(&errorOutput); + CachingPackageReader packageReader(&errorOutput); status_t error = packageReader.Init(fd, false, BHPKG::B_HPKG_READER_DONT_PRINT_VERSION_MISMATCH_MESSAGE); if (error == B_OK) { @@ -789,17 +827,8 @@ Package::Load() if (error != B_OK) RETURN_ERROR(error); - // create a heap reader - HeapReaderV2* heapReader = new(std::nothrow) HeapReaderV2; - if (heapReader == NULL) - RETURN_ERROR(B_NO_MEMORY); - - error = heapReader->Init(packageReader.HeapReader(), fd); - if (error != B_OK) { - RETURN_ERROR(error); - } - - fHeapReader = heapReader; + // get the heap reader + fHeapReader = packageReader.DetachCachedHeapReader(); return B_OK; } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.h b/src/add-ons/kernel/file_systems/packagefs/package/Package.h index b75996b..caf2fb4 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.h @@ -100,6 +100,7 @@ private: struct HeapReader; struct HeapReaderV1; struct HeapReaderV2; + struct CachingPackageReader; private: mutex fLock; diff --git a/src/kits/package/hpkg/PackageReaderImpl.cpp b/src/kits/package/hpkg/PackageReaderImpl.cpp index acee5c1..83ff096 100644 --- a/src/kits/package/hpkg/PackageReaderImpl.cpp +++ b/src/kits/package/hpkg/PackageReaderImpl.cpp @@ -26,7 +26,6 @@ #include <package/hpkg/PackageData.h> #include <package/hpkg/PackageEntry.h> #include <package/hpkg/PackageEntryAttribute.h> -#include <package/hpkg/PackageFileHeapReader.h> #include <package/hpkg/ZlibDecompressor.h> @@ -340,7 +339,7 @@ PackageReaderImpl::Init(int fd, bool keepFD, uint32 flags) fd, keepFD, header, flags); if (error != B_OK) return error; - fHeapSize = HeapReader()->UncompressedHeapSize(); + fHeapSize = UncompressedHeapSize(); // init package attributes section error = InitSection(fPackageAttributesSection, fHeapSize, diff --git a/src/kits/package/hpkg/PackageWriterImpl.cpp b/src/kits/package/hpkg/PackageWriterImpl.cpp index 7c519c3..b7fc37a 100644 --- a/src/kits/package/hpkg/PackageWriterImpl.cpp +++ b/src/kits/package/hpkg/PackageWriterImpl.cpp @@ -636,7 +636,7 @@ PackageWriterImpl::_Init(const char* fileName, if (result != B_OK) return result; - fHeapWriter->Reinit(packageReader.HeapReader()); + fHeapWriter->Reinit(packageReader.RawHeapReader()); } return B_OK; diff --git a/src/kits/package/hpkg/ReaderImplBase.cpp b/src/kits/package/hpkg/ReaderImplBase.cpp index ad83419..ba4b0b5 100644 --- a/src/kits/package/hpkg/ReaderImplBase.cpp +++ b/src/kits/package/hpkg/ReaderImplBase.cpp @@ -543,6 +543,7 @@ ReaderImplBase::ReaderImplBase(const char* fileType, BErrorOutput* errorOutput) fErrorOutput(errorOutput), fFD(-1), fOwnsFD(false), + fRawHeapReader(NULL), fHeapReader(NULL), fCurrentSection(NULL), fScratchBuffer(NULL), @@ -554,6 +555,8 @@ ReaderImplBase::ReaderImplBase(const char* fileType, BErrorOutput* errorOutput) ReaderImplBase::~ReaderImplBase() { delete fHeapReader; + if (fRawHeapReader != fHeapReader) + delete fRawHeapReader; if (fOwnsFD && fFD >= 0) close(fFD); @@ -562,21 +565,24 @@ ReaderImplBase::~ReaderImplBase() } -status_t -ReaderImplBase::_Init(int fd, bool keepFD) +uint64 +ReaderImplBase::UncompressedHeapSize() const { - fFD = fd; - fOwnsFD = keepFD; + return fRawHeapReader->UncompressedHeapSize(); +} - // allocate a scratch buffer - fScratchBuffer = new(std::nothrow) uint8[kScratchBufferSize]; - if (fScratchBuffer == NULL) { - fErrorOutput->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - fScratchBufferSize = kScratchBufferSize; - return B_OK; +BAbstractBufferedDataReader* +ReaderImplBase::DetachHeapReader(PackageFileHeapReader** _rawHeapReader) +{ + BAbstractBufferedDataReader* heapReader = fHeapReader; + fHeapReader = NULL; + + if (_rawHeapReader != NULL) + *_rawHeapReader = fRawHeapReader; + fRawHeapReader = NULL; + + return heapReader; } @@ -589,9 +595,32 @@ ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize, return B_BAD_DATA; } - fHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, fFD, + fRawHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, fFD, offset, compressedSize, uncompressedSize); - return fHeapReader->Init(); + if (fRawHeapReader == NULL) + return B_NO_MEMORY; + + status_t error = fRawHeapReader->Init(); + if (error != B_OK) + return error; + + error = CreateCachedHeapReader(fRawHeapReader, fHeapReader); + if (error != B_OK) { + if (error != B_NOT_SUPPORTED) + return error; + + fHeapReader = fRawHeapReader; + } + + return B_OK; +} + + +status_t +ReaderImplBase::CreateCachedHeapReader(PackageFileHeapReader* heapReader, + BAbstractBufferedDataReader*& _cachedReader) +{ + return B_NOT_SUPPORTED; } @@ -798,6 +827,24 @@ ReaderImplBase::ParseAttributeTree(AttributeHandlerContext* context, status_t +ReaderImplBase::_Init(int fd, bool keepFD) +{ + fFD = fd; + fOwnsFD = keepFD; + + // allocate a scratch buffer + fScratchBuffer = new(std::nothrow) uint8[kScratchBufferSize]; + if (fScratchBuffer == NULL) { + fErrorOutput->PrintError("Error: Out of memory!\n"); + return B_NO_MEMORY; + } + fScratchBufferSize = kScratchBufferSize; + + return B_OK; +} + + +status_t ReaderImplBase::_ParseAttributeTree(AttributeHandlerContext* context) { int level = 0; diff --git a/src/kits/package/hpkg/RepositoryReaderImpl.cpp b/src/kits/package/hpkg/RepositoryReaderImpl.cpp index f59cb8f..98867ab 100644 --- a/src/kits/package/hpkg/RepositoryReaderImpl.cpp +++ b/src/kits/package/hpkg/RepositoryReaderImpl.cpp @@ -19,7 +19,6 @@ #include <Message.h> #include <package/hpkg/HPKGDefsPrivate.h> -#include <package/hpkg/PackageFileHeapReader.h> #include <package/hpkg/RepositoryContentHandler.h> @@ -80,7 +79,7 @@ RepositoryReaderImpl::Init(int fd, bool keepFD) // init package attributes section error = InitSection(fPackageAttributesSection, - HeapReader()->UncompressedHeapSize(), + UncompressedHeapSize(), B_BENDIAN_TO_HOST_INT64(header.packages_length), kMaxPackageAttributesSize, B_BENDIAN_TO_HOST_INT64(header.packages_strings_length), diff --git a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp index 6257d53..3ca86a4 100644 --- a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp +++ b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp @@ -769,7 +769,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory) return B_NO_MEMORY; BReference<PackageVolume> volumeReference(volume, true); - error = volume->Init(fd, packageReader.HeapReader()); + error = volume->Init(fd, packageReader.RawHeapReader()); if (error != B_OK) RETURN_ERROR(error);