Author: bonefish Date: 2010-01-27 13:45:53 +0100 (Wed, 27 Jan 2010) New Revision: 35316 Changeset: http://dev.haiku-os.org/changeset/35316/haiku Modified: haiku/trunk/headers/private/kernel/heap.h haiku/trunk/headers/private/kernel/slab/Slab.h haiku/trunk/headers/private/kernel/vfs.h haiku/trunk/headers/private/kernel/vm/VMAddressSpace.h haiku/trunk/headers/private/kernel/vm/VMArea.h haiku/trunk/headers/private/kernel/vm/VMCache.h haiku/trunk/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp haiku/trunk/src/system/kernel/arch/x86/x86_physical_page_mapper_large_memory.cpp haiku/trunk/src/system/kernel/cache/vnode_store.cpp haiku/trunk/src/system/kernel/cache/vnode_store.h haiku/trunk/src/system/kernel/device_manager/IORequest.cpp haiku/trunk/src/system/kernel/device_manager/IORequest.h haiku/trunk/src/system/kernel/device_manager/device_manager.cpp haiku/trunk/src/system/kernel/fs/vfs_request_io.cpp haiku/trunk/src/system/kernel/heap.cpp haiku/trunk/src/system/kernel/slab/allocator.cpp haiku/trunk/src/system/kernel/vm/VMAddressSpace.cpp haiku/trunk/src/system/kernel/vm/VMAnonymousCache.cpp haiku/trunk/src/system/kernel/vm/VMAnonymousCache.h haiku/trunk/src/system/kernel/vm/VMAnonymousNoSwapCache.cpp haiku/trunk/src/system/kernel/vm/VMAnonymousNoSwapCache.h haiku/trunk/src/system/kernel/vm/VMArea.cpp haiku/trunk/src/system/kernel/vm/VMCache.cpp haiku/trunk/src/system/kernel/vm/VMDeviceCache.cpp haiku/trunk/src/system/kernel/vm/VMDeviceCache.h haiku/trunk/src/system/kernel/vm/VMKernelAddressSpace.cpp haiku/trunk/src/system/kernel/vm/VMKernelAddressSpace.h haiku/trunk/src/system/kernel/vm/VMKernelArea.cpp haiku/trunk/src/system/kernel/vm/VMKernelArea.h haiku/trunk/src/system/kernel/vm/VMNullCache.cpp haiku/trunk/src/system/kernel/vm/VMNullCache.h haiku/trunk/src/system/kernel/vm/VMUserAddressSpace.cpp haiku/trunk/src/system/kernel/vm/VMUserAddressSpace.h haiku/trunk/src/system/kernel/vm/VMUserArea.cpp haiku/trunk/src/system/kernel/vm/VMUserArea.h haiku/trunk/src/system/kernel/vm/vm.cpp haiku/trunk/src/system/kernel/vm/vm_page.cpp Log: * Introduced {malloc,memalign,free}_etc() which take an additional "flags" argument. They replace the previous special-purpose allocation functions (malloc_nogrow(), vip_io_request_malloc()). * Moved the I/O VIP heap to heap.cpp accordingly. * Added quite a bit of passing around of allocation flags in the VM, particularly in the VM*AddressSpace classes. * Fixed IOBuffer::GetNextVirtualVec(): It was ignoring the VIP flag and always allocated on the normal heap. Modified: haiku/trunk/headers/private/kernel/heap.h =================================================================== --- haiku/trunk/headers/private/kernel/heap.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/headers/private/kernel/heap.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -23,6 +23,12 @@ #define HEAP_AREA_USE_THRESHOLD 1 * 1024 * 1024 +// allocation/deallocation flags for {malloc,free}_etc() +#define HEAP_DONT_WAIT_FOR_MEMORY 0x01 +#define HEAP_DONT_LOCK_KERNEL_SPACE 0x02 +#define HEAP_PRIORITY_VIP 0x04 + + typedef struct heap_class_s { const char *name; uint32 initial_percentage; @@ -41,11 +47,10 @@ extern "C" { #endif -// malloc- and memalign_nogrow disallow waiting for a grow to happen - only to -// be used by vm functions that may deadlock on a triggered area creation. -void* memalign_nogrow(size_t alignment, size_t size); -void* malloc_nogrow(size_t size); +void* memalign_etc(size_t alignment, size_t size, uint32 flags); +void free_etc(void* address, uint32 flags); + void* memalign(size_t alignment, size_t size); void deferred_free(void* block); @@ -74,6 +79,13 @@ #endif +static inline void* +malloc_etc(size_t size, uint32 flags) +{ + return memalign_etc(0, size, flags); +} + + #ifdef __cplusplus #include <new> @@ -81,21 +93,34 @@ #include <util/SinglyLinkedList.h> -static const struct nogrow_t { -} nogrow = {}; +struct malloc_flags { + uint32 flags; + malloc_flags(uint32 flags) + : + flags(flags) + { + } + malloc_flags(const malloc_flags& other) + : + flags(other.flags) + { + } +}; + + inline void* -operator new(size_t size, const nogrow_t& nogrow) throw() +operator new(size_t size, const malloc_flags& flags) throw() { - return malloc_nogrow(size); + return malloc_etc(size, flags.flags); } inline void* -operator new[](size_t size, const nogrow_t& nogrow) throw() +operator new[](size_t size, const malloc_flags& flags) throw() { - return malloc_nogrow(size); + return malloc_etc(size, flags.flags); } Modified: haiku/trunk/headers/private/kernel/slab/Slab.h =================================================================== --- haiku/trunk/headers/private/kernel/slab/Slab.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/headers/private/kernel/slab/Slab.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -8,24 +8,26 @@ #define _SLAB_SLAB_H_ -#include <KernelExport.h> -#include <OS.h> +#include <heap.h> enum { + /* object_cache_{alloc,free}() flags */ + CACHE_DONT_WAIT_FOR_MEMORY = HEAP_DONT_WAIT_FOR_MEMORY, + CACHE_DONT_LOCK_KERNEL_SPACE = HEAP_DONT_LOCK_KERNEL_SPACE, + CACHE_PRIORITY_VIP = HEAP_PRIORITY_VIP, + CACHE_ALLOC_FLAGS = CACHE_DONT_WAIT_FOR_MEMORY + | CACHE_DONT_LOCK_KERNEL_SPACE + | CACHE_PRIORITY_VIP, + /* create_object_cache_etc flags */ - CACHE_NO_DEPOT = 1 << 0, - CACHE_UNLOCKED_PAGES = 1 << 1, // unsupported - CACHE_LARGE_SLAB = 1 << 2, + CACHE_NO_DEPOT = 0x08000000, + CACHE_UNLOCKED_PAGES = 0x10000000, // unsupported + CACHE_LARGE_SLAB = 0x20000000, - /* object_cache_{alloc,free}() flags */ - CACHE_DONT_WAIT_FOR_MEMORY = 1 << 8, - CACHE_DONT_LOCK_KERNEL_SPACE = 1 << 9, - CACHE_PRIORITY_VIP = 1 << 10, - /* internal */ - CACHE_ALIGN_ON_SIZE = 1 << 30, - CACHE_DURING_BOOT = 1 << 31 + CACHE_ALIGN_ON_SIZE = 0x40000000, + CACHE_DURING_BOOT = 0x80000000 }; struct ObjectCache; Modified: haiku/trunk/headers/private/kernel/vfs.h =================================================================== --- haiku/trunk/headers/private/kernel/vfs.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/headers/private/kernel/vfs.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -289,8 +289,6 @@ bool partialTransfer, size_t bytesTransferred) = 0; - void operator delete(void* address, size_t size); - static status_t IORequestCallback(void* data, io_request* request, status_t status, bool partialTransfer, Modified: haiku/trunk/headers/private/kernel/vm/VMAddressSpace.h =================================================================== --- haiku/trunk/headers/private/kernel/vm/VMAddressSpace.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/headers/private/kernel/vm/VMAddressSpace.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -70,25 +70,31 @@ virtual VMArea* LookupArea(addr_t address) const = 0; virtual VMArea* CreateArea(const char* name, uint32 wiring, - uint32 protection) = 0; - virtual void DeleteArea(VMArea* area) = 0; + uint32 protection, + uint32 allocationFlags) = 0; + virtual void DeleteArea(VMArea* area, + uint32 allocationFlags) = 0; virtual status_t InsertArea(void** _address, uint32 addressSpec, - size_t size, VMArea* area) = 0; - virtual void RemoveArea(VMArea* area) = 0; + size_t size, VMArea* area, + uint32 allocationFlags) = 0; + virtual void RemoveArea(VMArea* area, + uint32 allocationFlags) = 0; virtual bool CanResizeArea(VMArea* area, size_t newSize) = 0; - virtual status_t ResizeArea(VMArea* area, size_t newSize) = 0; - virtual status_t ShrinkAreaHead(VMArea* area, size_t newSize) - = 0; - virtual status_t ShrinkAreaTail(VMArea* area, size_t newSize) - = 0; + virtual status_t ResizeArea(VMArea* area, size_t newSize, + uint32 allocationFlags) = 0; + virtual status_t ShrinkAreaHead(VMArea* area, size_t newSize, + uint32 allocationFlags) = 0; + virtual status_t ShrinkAreaTail(VMArea* area, size_t newSize, + uint32 allocationFlags) = 0; virtual status_t ReserveAddressRange(void** _address, uint32 addressSpec, size_t size, - uint32 flags) = 0; + uint32 flags, uint32 allocationFlags) = 0; virtual status_t UnreserveAddressRange(addr_t address, - size_t size) = 0; - virtual void UnreserveAllAddressRanges() = 0; + size_t size, uint32 allocationFlags) = 0; + virtual void UnreserveAllAddressRanges( + uint32 allocationFlags) = 0; virtual void Dump() const; Modified: haiku/trunk/headers/private/kernel/vm/VMArea.h =================================================================== --- haiku/trunk/headers/private/kernel/vm/VMArea.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/headers/private/kernel/vm/VMArea.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -53,7 +53,7 @@ uint32 wiring, uint32 protection); ~VMArea(); - status_t Init(const char* name); + status_t Init(const char* name, uint32 allocationFlags); protected: friend class VMAddressSpace; Modified: haiku/trunk/headers/private/kernel/vm/VMCache.h =================================================================== --- haiku/trunk/headers/private/kernel/vm/VMCache.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/headers/private/kernel/vm/VMCache.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -67,7 +67,7 @@ VMCache(); virtual ~VMCache(); - status_t Init(uint32 cacheType); + status_t Init(uint32 cacheType, uint32 allocationFlags); virtual void Delete(); Modified: haiku/trunk/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp =================================================================== --- haiku/trunk/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -1369,7 +1369,7 @@ area = vm_create_null_area(VMAddressSpace::KernelID(), "interrupt query pages", (void **)&queryPage, B_ANY_ADDRESS, - B_PAGE_SIZE, CREATE_AREA_PRIORITY_VIP); + B_PAGE_SIZE, 0); if (area < B_OK) return area; Modified: haiku/trunk/src/system/kernel/arch/x86/x86_physical_page_mapper_large_memory.cpp =================================================================== --- haiku/trunk/src/system/kernel/arch/x86/x86_physical_page_mapper_large_memory.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/arch/x86/x86_physical_page_mapper_large_memory.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -546,7 +546,7 @@ temp = (void*)fInitialPool.virtualBase; area = vm_create_null_area(VMAddressSpace::KernelID(), "physical page pool space", &temp, B_EXACT_ADDRESS, - 1024 * B_PAGE_SIZE, CREATE_AREA_PRIORITY_VIP); + 1024 * B_PAGE_SIZE, 0); if (area < B_OK) { panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to " "create area for physical page pool space."); Modified: haiku/trunk/src/system/kernel/cache/vnode_store.cpp =================================================================== --- haiku/trunk/src/system/kernel/cache/vnode_store.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/cache/vnode_store.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -17,9 +17,9 @@ status_t -VMVnodeCache::Init(struct vnode *vnode) +VMVnodeCache::Init(struct vnode *vnode, uint32 allocationFlags) { - status_t error = VMCache::Init(CACHE_TYPE_VNODE); + status_t error = VMCache::Init(CACHE_TYPE_VNODE, allocationFlags); if (error != B_OK) return error; Modified: haiku/trunk/src/system/kernel/cache/vnode_store.h =================================================================== --- haiku/trunk/src/system/kernel/cache/vnode_store.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/cache/vnode_store.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -15,7 +15,7 @@ class VMVnodeCache : public VMCache { public: - status_t Init(struct vnode *vnode); + status_t Init(struct vnode *vnode, uint32 allocationFlags); virtual bool HasPage(off_t offset); Modified: haiku/trunk/src/system/kernel/device_manager/IORequest.cpp =================================================================== --- haiku/trunk/src/system/kernel/device_manager/IORequest.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/device_manager/IORequest.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -28,8 +28,6 @@ #endif -#define VIP_HEAP_SIZE 1024 * 1024 - // partial I/O operation phases enum { PHASE_READ_BEGIN = 0, @@ -37,9 +35,7 @@ PHASE_DO_ALL = 2 }; -heap_allocator* sVIPHeap; - // #pragma mark - @@ -56,13 +52,6 @@ } -void -IORequestChunk::operator delete(void* address, size_t size) -{ - io_request_free(address); -} - - // #pragma mark - @@ -80,7 +69,7 @@ { size_t size = sizeof(IOBuffer) + sizeof(iovec) * (count - 1); IOBuffer* buffer - = (IOBuffer*)(vip ? vip_io_request_malloc(size) : malloc(size)); + = (IOBuffer*)(malloc_etc(size, vip ? HEAP_PRIORITY_VIP : 0)); if (buffer == NULL) return NULL; @@ -101,10 +90,7 @@ if (this == NULL) return; - if (fVIP) - vip_io_request_free(this); - else - free(this); + free_etc(this, fVIP ? HEAP_PRIORITY_VIP : 0); } @@ -130,7 +116,8 @@ { virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie; if (cookie == NULL) { - cookie = new(std::nothrow) virtual_vec_cookie; + cookie = new(malloc_flags(fVIP ? HEAP_PRIORITY_VIP : 0)) + virtual_vec_cookie; if (cookie == NULL) return B_NO_MEMORY; @@ -207,7 +194,7 @@ if (cookie->mapped_area >= 0) delete_area(cookie->mapped_area); - delete cookie; + free_etc(cookie, fVIP ? HEAP_PRIORITY_VIP : 0); } @@ -715,7 +702,9 @@ /* static */ IORequest* IORequest::Create(bool vip) { - return vip ? new(vip_io_alloc) IORequest : new(std::nothrow) IORequest; + return vip + ? new(malloc_flags(HEAP_PRIORITY_VIP)) IORequest + : new(std::nothrow) IORequest; } @@ -1309,106 +1298,3 @@ set_debug_variable("_buffer", (addr_t)fBuffer); set_debug_variable("_cvar", (addr_t)&fFinishedCondition); } - - -// #pragma mark - allocator - - -#if KERNEL_HEAP_LEAK_CHECK -static addr_t -get_caller() -{ - // Find the first return address outside of the allocator code. Note, that - // this makes certain assumptions about how the code for the functions - // ends up in the kernel object. - addr_t returnAddresses[5]; - int32 depth = arch_debug_get_stack_trace(returnAddresses, 5, 0, 1, - STACK_TRACE_KERNEL | STACK_TRACE_USER); - - // find the first return address inside the VIP allocator - int32 i = 0; - for (i = 0; i < depth; i++) { - if (returnAddresses[i] >= (addr_t)&get_caller - && returnAddresses[i] < (addr_t)&vip_io_request_allocator_init) { - break; - } - } - - // now continue until we have the first one outside - for (; i < depth; i++) { - if (returnAddresses[i] < (addr_t)&get_caller - || returnAddresses[i] > (addr_t)&vip_io_request_allocator_init) { - return returnAddresses[i]; - } - } - - return 0; -} -#endif - - -void* -vip_io_request_malloc(size_t size) -{ - void* address = heap_memalign(sVIPHeap, 0, size); -#if KDEBUG - if (address == NULL) - panic("vip_io_request_malloc(): VIP heap %p out of memory", sVIPHeap); -#endif - return address; -} - - -void -vip_io_request_free(void* address) -{ - heap_free(sVIPHeap, address); -} - - -void -io_request_free(void* address) -{ - if (heap_free(sVIPHeap, address) != B_OK) - free(address); -} - - -void -vip_io_request_allocator_init() -{ - static const heap_class heapClass = { - "VIP I/O", /* name */ - 100, /* initial percentage */ - B_PAGE_SIZE / 8, /* max allocation size */ - B_PAGE_SIZE, /* page size */ - 8, /* min bin size */ - 4, /* bin alignment */ - 8, /* min count per page */ - 16 /* max waste per page */ - }; - - void* address = NULL; - area_id area = create_area("VIP I/O heap", &address, B_ANY_KERNEL_ADDRESS, - VIP_HEAP_SIZE, B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < B_OK) { - panic("vip_io_request_allocator_init(): couldn't allocate VIP I/O " - "heap area"); - return; - } - - sVIPHeap = heap_create_allocator("VIP I/O heap", (addr_t)address, - VIP_HEAP_SIZE, &heapClass, false); - if (sVIPHeap == NULL) { - panic("vip_io_request_allocator_init(): failed to create VIP I/O " - "heap\n"); - return; - } - -#if KERNEL_HEAP_LEAK_CHECK - heap_set_get_caller(sVIPHeap, &get_caller); -#endif - - dprintf("vip_io_request_allocator_init(): created VIP I/O heap: %p\n", - sVIPHeap); -} Modified: haiku/trunk/src/system/kernel/device_manager/IORequest.h =================================================================== --- haiku/trunk/src/system/kernel/device_manager/IORequest.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/device_manager/IORequest.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -102,8 +102,6 @@ DoublyLinkedListLink<IORequestChunk>* ListLink() { return &fListLink; } - void operator delete(void* address, size_t size); - protected: void SetStatus(status_t status) { fStatus = status; } @@ -352,33 +350,4 @@ typedef DoublyLinkedList<IORequest> IORequestList; -// allocator for VIP I/O request memory -void* vip_io_request_malloc(size_t size); -void vip_io_request_free(void* address); - -void io_request_free(void* address); - // frees regardless of whether allocated with vip_io_request_malloc() or - // malloc() - -void vip_io_request_allocator_init(); - - -static const struct vip_io_alloc_t { -} vip_io_alloc = {}; - - -inline void* -operator new(size_t size, const vip_io_alloc_t& vip_io_alloc) throw () -{ - return vip_io_request_malloc(size); -} - - -inline void* -operator new[](size_t size, const vip_io_alloc_t& vip_io_alloc) throw () -{ - return vip_io_request_malloc(size); -} - - #endif // IO_REQUEST_H Modified: haiku/trunk/src/system/kernel/device_manager/device_manager.cpp =================================================================== --- haiku/trunk/src/system/kernel/device_manager/device_manager.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/device_manager/device_manager.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -2257,7 +2257,6 @@ { TRACE(("device manager init\n")); - vip_io_request_allocator_init(); IOSchedulerRoster::Init(); dm_init_id_generator(); Modified: haiku/trunk/src/system/kernel/fs/vfs_request_io.cpp =================================================================== --- haiku/trunk/src/system/kernel/fs/vfs_request_io.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/fs/vfs_request_io.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -1,5 +1,5 @@ /* - * Copyright 2008, Ingo Weinhold, ingo_weinhold@xxxxxxx + * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@xxxxxxx * Distributed under the terms of the MIT License. */ @@ -14,6 +14,9 @@ #endif +#include <heap.h> + + // #pragma mark - AsyncIOCallback @@ -22,13 +25,6 @@ } -void -AsyncIOCallback::operator delete(void* address, size_t size) -{ - io_request_free(address); -} - - /* static */ status_t AsyncIOCallback::IORequestCallback(void* data, io_request* request, status_t status, bool partialTransfer, size_t transferEndOffset) @@ -61,9 +57,6 @@ off_t request_offset; io_request_finished_callback next_finished_callback; void* next_finished_cookie; - - void operator delete(void* address, size_t size) - { io_request_free(address); } }; @@ -484,7 +477,7 @@ iterative_io_cookie* iterationCookie = (request->Flags() & B_VIP_IO_REQUEST) != 0 - ? new(vip_io_alloc) iterative_io_cookie + ? new(malloc_flags(HEAP_PRIORITY_VIP)) iterative_io_cookie : new(std::nothrow) iterative_io_cookie; if (iterationCookie == NULL) { // no memory -- fall back to synchronous I/O Modified: haiku/trunk/src/system/kernel/heap.cpp =================================================================== --- haiku/trunk/src/system/kernel/heap.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/heap.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -142,6 +142,10 @@ typedef SinglyLinkedList<DeferredDeletable> DeferredDeletableList; +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + +#define VIP_HEAP_SIZE 1024 * 1024 + // Heap class configuration #define HEAP_CLASS_COUNT 3 static heap_class sHeapClasses[HEAP_CLASS_COUNT] = { @@ -183,11 +187,15 @@ static uint32 *sLastGrowRequest[HEAP_CLASS_COUNT * B_MAX_CPU_COUNT]; static uint32 *sLastHandledGrowRequest[HEAP_CLASS_COUNT * B_MAX_CPU_COUNT]; +static heap_allocator *sVIPHeap; static heap_allocator *sGrowHeap = NULL; static thread_id sHeapGrowThread = -1; static sem_id sHeapGrowSem = -1; static sem_id sHeapGrownNotify = -1; static bool sAddGrowHeap = false; + +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + static DeferredFreeList sDeferredFreeList; static DeferredDeletableList sDeferredDeletableList; static spinlock sDeferredFreeListLock; @@ -271,6 +279,9 @@ // #pragma mark - Debug functions +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + + #if KERNEL_HEAP_LEAK_CHECK static addr_t get_caller() @@ -809,7 +820,9 @@ #endif // KERNEL_HEAP_LEAK_CHECK +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + #if PARANOID_HEAP_VALIDATION static void heap_validate_heap(heap_allocator *heap) @@ -1797,9 +1810,6 @@ } -#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC - - inline uint32 heap_index_for(size_t size, int32 cpu) { @@ -1818,35 +1828,47 @@ } -static void -deferred_deleter(void *arg, int iteration) +static void * +memalign_nogrow(size_t alignment, size_t size) { - // move entries and deletables to on-stack lists - InterruptsSpinLocker locker(sDeferredFreeListLock); - if (sDeferredFreeList.IsEmpty() && sDeferredDeletableList.IsEmpty()) - return; + // use dedicated memory in the grow thread by default + if (thread_get_current_thread_id() == sHeapGrowThread) { + void *result = heap_memalign(sGrowHeap, alignment, size); + if (!sAddGrowHeap && heap_should_grow(sGrowHeap)) { + // hopefully the heap grower will manage to create a new heap + // before running out of private memory... + dprintf("heap: requesting new grow heap\n"); + sAddGrowHeap = true; + release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE); + } - DeferredFreeList entries; - entries.MoveFrom(&sDeferredFreeList); + if (result != NULL) + return result; + } - DeferredDeletableList deletables; - deletables.MoveFrom(&sDeferredDeletableList); + // try public memory, there might be something available + void *result = NULL; + int32 cpuCount = MIN(smp_get_num_cpus(), + (int32)sHeapCount / HEAP_CLASS_COUNT); + int32 cpuNumber = smp_get_current_cpu(); + for (int32 i = 0; i < cpuCount; i++) { + uint32 heapIndex = heap_index_for(size, cpuNumber++ % cpuCount); + heap_allocator *heap = sHeaps[heapIndex]; + result = heap_memalign(heap, alignment, size); + if (result != NULL) + return result; + } - locker.Unlock(); + // no memory available + if (thread_get_current_thread_id() == sHeapGrowThread) + panic("heap: all heaps have run out of memory while growing\n"); + else + dprintf("heap: all heaps have run out of memory\n"); - // free the entries - while (DeferredFreeListEntry* entry = entries.RemoveHead()) - free(entry); - - // delete the deletables - while (DeferredDeletable* deletable = deletables.RemoveHead()) - delete deletable; + return NULL; } -// #pragma mark - - - static status_t heap_create_new_heap_area(heap_allocator *heap, const char *name, size_t size) { @@ -1907,6 +1929,41 @@ } +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + + +static void +deferred_deleter(void *arg, int iteration) +{ + // move entries and deletables to on-stack lists + InterruptsSpinLocker locker(sDeferredFreeListLock); + if (sDeferredFreeList.IsEmpty() && sDeferredDeletableList.IsEmpty()) + return; + + DeferredFreeList entries; + entries.MoveFrom(&sDeferredFreeList); + + DeferredDeletableList deletables; + deletables.MoveFrom(&sDeferredDeletableList); + + locker.Unlock(); + + // free the entries + while (DeferredFreeListEntry* entry = entries.RemoveHead()) + free(entry); + + // delete the deletables + while (DeferredDeletable* deletable = deletables.RemoveHead()) + delete deletable; +} + + +// #pragma mark - + + +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + + status_t heap_init(addr_t base, size_t size) { @@ -1988,9 +2045,13 @@ } +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + + status_t heap_init_post_thread() { +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC void *address = NULL; area_id growHeapArea = create_area("dedicated grow heap", &address, B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_DEDICATED_GROW_SIZE, B_FULL_LOCK, @@ -2039,11 +2100,43 @@ } } + // create the VIP heap + static const heap_class heapClass = { + "VIP I/O", /* name */ + 100, /* initial percentage */ + B_PAGE_SIZE / 8, /* max allocation size */ + B_PAGE_SIZE, /* page size */ + 8, /* min bin size */ + 4, /* bin alignment */ + 8, /* min count per page */ + 16 /* max waste per page */ + }; + + area_id vipHeapArea = create_area("VIP heap", &address, + B_ANY_KERNEL_ADDRESS, VIP_HEAP_SIZE, B_FULL_LOCK, + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (vipHeapArea < 0) { + panic("heap_init_post_thread(): couldn't allocate VIP heap area"); + return B_ERROR; + } + + sVIPHeap = heap_create_allocator("VIP heap", (addr_t)address, + VIP_HEAP_SIZE, &heapClass, false); + if (sVIPHeap == NULL) { + panic("heap_init_post_thread(): failed to create VIP heap\n"); + return B_ERROR; + } + + dprintf("heap_init_post_thread(): created VIP heap: %p\n", sVIPHeap); + + send_signal_etc(sHeapGrowThread, SIGCONT, B_DO_NOT_RESCHEDULE); + +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + // run the deferred deleter roughly once a second if (register_kernel_daemon(deferred_deleter, NULL, 10) != B_OK) panic("heap_init_post_thread(): failed to init deferred deleter"); - send_signal_etc(sHeapGrowThread, SIGCONT, B_DO_NOT_RESCHEDULE); return B_OK; } @@ -2148,50 +2241,27 @@ void * -memalign_nogrow(size_t alignment, size_t size) +memalign_etc(size_t alignment, size_t size, uint32 flags) { - // use dedicated memory in the grow thread by default - if (thread_get_current_thread_id() == sHeapGrowThread) { - void *result = heap_memalign(sGrowHeap, alignment, size); - if (!sAddGrowHeap && heap_should_grow(sGrowHeap)) { - // hopefully the heap grower will manage to create a new heap - // before running out of private memory... - dprintf("heap: requesting new grow heap\n"); - sAddGrowHeap = true; - release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE); - } + if ((flags & HEAP_PRIORITY_VIP) != 0) + return heap_memalign(sVIPHeap, alignment, size); - if (result != NULL) - return result; + if ((flags & (HEAP_DONT_WAIT_FOR_MEMORY | HEAP_DONT_LOCK_KERNEL_SPACE)) + != 0) { + return memalign_nogrow(alignment, size); } - // try public memory, there might be something available - void *result = NULL; - int32 cpuCount = MIN(smp_get_num_cpus(), - (int32)sHeapCount / HEAP_CLASS_COUNT); - int32 cpuNumber = smp_get_current_cpu(); - for (int32 i = 0; i < cpuCount; i++) { - uint32 heapIndex = heap_index_for(size, cpuNumber++ % cpuCount); - heap_allocator *heap = sHeaps[heapIndex]; - result = heap_memalign(heap, alignment, size); - if (result != NULL) - return result; - } - - // no memory available - if (thread_get_current_thread_id() == sHeapGrowThread) - panic("heap: all heaps have run out of memory while growing\n"); - else - dprintf("heap: all heaps have run out of memory\n"); - - return NULL; + return memalign(alignment, size); } -void * -malloc_nogrow(size_t size) +void +free_etc(void *address, uint32 flags) { - return memalign_nogrow(0, size); + if ((flags & HEAP_PRIORITY_VIP) != 0) + heap_free(sVIPHeap, address); + else + free(address); } @@ -2225,6 +2295,10 @@ if (heap_free(sGrowHeap, address) == B_OK) return; + // or maybe it was allocated from the VIP heap + if (heap_free(sVIPHeap, address) == B_OK) + return; + // or maybe it was a huge allocation using an area area_info areaInfo; area_id area = area_for(address); Modified: haiku/trunk/src/system/kernel/slab/allocator.cpp =================================================================== --- haiku/trunk/src/system/kernel/slab/allocator.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/slab/allocator.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -213,19 +213,17 @@ } -void* -memalign_nogrow(size_t alignment, size_t size) +void * +memalign_etc(size_t alignment, size_t size, uint32 flags) { - return block_alloc(size, alignment, - CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); + return block_alloc(size, alignment, flags & CACHE_ALLOC_FLAGS); } -void* -malloc_nogrow(size_t size) +void +free_etc(void *address, uint32 flags) { - return block_alloc(size, 0, - CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); + block_free(address, flags & CACHE_ALLOC_FLAGS); } Modified: haiku/trunk/src/system/kernel/vm/VMAddressSpace.cpp =================================================================== --- haiku/trunk/src/system/kernel/vm/VMAddressSpace.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/vm/VMAddressSpace.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -192,8 +192,10 @@ VMAddressSpace** _addressSpace) { VMAddressSpace* addressSpace = kernel - ? (VMAddressSpace*)new(nogrow) VMKernelAddressSpace(teamID, base, size) - : (VMAddressSpace*)new(nogrow) VMUserAddressSpace(teamID, base, size); + ? (VMAddressSpace*)new(std::nothrow) VMKernelAddressSpace(teamID, base, + size) + : (VMAddressSpace*)new(std::nothrow) VMUserAddressSpace(teamID, base, + size); if (addressSpace == NULL) return B_NO_MEMORY; Modified: haiku/trunk/src/system/kernel/vm/VMAnonymousCache.cpp =================================================================== --- haiku/trunk/src/system/kernel/vm/VMAnonymousCache.cpp 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/vm/VMAnonymousCache.cpp 2010-01-27 12:45:53 UTC (rev 35316) @@ -409,11 +409,6 @@ delete this; } - void operator delete(void* address, size_t size) - { - io_request_free(address); - } - private: VMAnonymousCache* fCache; page_num_t fPageIndex; @@ -447,13 +442,13 @@ status_t VMAnonymousCache::Init(bool canOvercommit, int32 numPrecommittedPages, - int32 numGuardPages) + int32 numGuardPages, uint32 allocationFlags) { TRACE("%p->VMAnonymousCache::Init(canOvercommit = %s, " "numPrecommittedPages = %ld, numGuardPages = %ld)\n", this, canOvercommit ? "yes" : "no", numPrecommittedPages, numGuardPages); - status_t error = VMCache::Init(CACHE_TYPE_RAM); + status_t error = VMCache::Init(CACHE_TYPE_RAM, allocationFlags); if (error != B_OK) return error; @@ -647,7 +642,7 @@ // create our callback WriteCallback* callback = (flags & B_VIP_IO_REQUEST) != 0 - ? new(vip_io_alloc) WriteCallback(this, _callback) + ? new(malloc_flags(HEAP_PRIORITY_VIP)) WriteCallback(this, _callback) : new(std::nothrow) WriteCallback(this, _callback); if (callback == NULL) { if (newSlot) { Modified: haiku/trunk/src/system/kernel/vm/VMAnonymousCache.h =================================================================== --- haiku/trunk/src/system/kernel/vm/VMAnonymousCache.h 2010-01-27 07:19:58 UTC (rev 35315) +++ haiku/trunk/src/system/kernel/vm/VMAnonymousCache.h 2010-01-27 12:45:53 UTC (rev 35316) @@ -35,7 +35,8 @@ status_t Init(bool canOvercommit, int32 numPrecommittedPages, [... truncated: 1302 lines follow ...]