[haiku-commits] r35200 - in haiku/trunk: headers/private/kernel/slab src/system/kernel/slab
- From: ingo_weinhold@xxxxxx
- To: haiku-commits@xxxxxxxxxxxxx
- Date: Wed, 20 Jan 2010 14:00:57 +0100 (CET)
Author: bonefish
Date: 2010-01-20 14:00:57 +0100 (Wed, 20 Jan 2010)
New Revision: 35200
Changeset: http://dev.haiku-os.org/changeset/35200/haiku
Modified:
haiku/trunk/headers/private/kernel/slab/ObjectDepot.h
haiku/trunk/src/system/kernel/slab/ObjectDepot.cpp
Log:
Replaced the locking strategy (formerly a recursive lock for the depot and
one for each per CPU store):
* The depot is now protected by a R/W lock combined with a spinlock. It is
required to either hold read lock + spinlock or just the write lock.
* When accessing the per CPU stores we only need to acquire the read lock
and disable interrupts. When switching magazines with the depot we
additionally get the spinlock.
* When allocating a new magazine we do completely unlock.
Modified: haiku/trunk/headers/private/kernel/slab/ObjectDepot.h
===================================================================
--- haiku/trunk/headers/private/kernel/slab/ObjectDepot.h 2010-01-20
12:12:54 UTC (rev 35199)
+++ haiku/trunk/headers/private/kernel/slab/ObjectDepot.h 2010-01-20
13:00:57 UTC (rev 35200)
@@ -13,7 +13,8 @@
struct DepotMagazine;
typedef struct object_depot {
- recursive_lock lock;
+ rw_lock outer_lock;
+ spinlock inner_lock;
DepotMagazine* full;
DepotMagazine* empty;
size_t full_count;
Modified: haiku/trunk/src/system/kernel/slab/ObjectDepot.cpp
===================================================================
--- haiku/trunk/src/system/kernel/slab/ObjectDepot.cpp 2010-01-20 12:12:54 UTC
(rev 35199)
+++ haiku/trunk/src/system/kernel/slab/ObjectDepot.cpp 2010-01-20 13:00:57 UTC
(rev 35200)
@@ -1,4 +1,5 @@
/*
+ * Copyright 2010, Ingo Weinhold, ingo_weinhold@xxxxxxx
* Copyright 2008, Axel Dörfler. All Rights Reserved.
* Copyright 2007, Hugo Santos. All Rights Reserved.
*
@@ -10,6 +11,8 @@
#include <algorithm>
+#include <int.h>
+#include <smp.h>
#include <util/AutoLock.h>
#include "slab_private.h"
@@ -35,7 +38,6 @@
struct depot_cpu_store {
- recursive_lock lock;
DepotMagazine* loaded;
DepotMagazine* previous;
};
@@ -107,7 +109,7 @@
static bool
exchange_with_full(object_depot* depot, DepotMagazine*& magazine)
{
- RecursiveLocker _(depot->lock);
+ SpinLocker _(depot->inner_lock);
if (depot->full == NULL)
return false;
@@ -124,16 +126,13 @@
static bool
exchange_with_empty(object_depot* depot, DepotMagazine*& magazine)
{
- RecursiveLocker _(depot->lock);
+ SpinLocker _(depot->inner_lock);
- if (depot->empty == NULL) {
- depot->empty = alloc_magazine();
- if (depot->empty == NULL)
- return false;
- } else {
- depot->empty_count--;
- }
+ if (depot->empty == NULL)
+ return false;
+ depot->empty_count--;
+
if (magazine) {
_push(depot->full, magazine);
depot->full_count++;
@@ -144,63 +143,19 @@
}
-static inline depot_cpu_store*
-object_depot_cpu(object_depot* depot)
+static void
+push_empty_magazine(object_depot* depot, DepotMagazine* magazine)
{
- return &depot->stores[smp_get_current_cpu()];
-}
+ SpinLocker _(depot->inner_lock);
-
-static void*
-object_depot_obtain_from_store(object_depot* depot, depot_cpu_store* store)
-{
- RecursiveLocker _(store->lock);
-
- // To better understand both the Alloc() and Free() logic refer to
- // Bonwick's ``Magazines and Vmem'' [in 2001 USENIX proceedings]
-
- // In a nutshell, we try to get an object from the loaded magazine
- // if it's not empty, or from the previous magazine if it's full
- // and finally from the Slab if the magazine depot has no full
magazines.
-
- if (store->loaded == NULL)
- return NULL;
-
- while (true) {
- if (!store->loaded->IsEmpty())
- return store->loaded->Pop();
-
- if (store->previous
- && (store->previous->IsFull()
- || exchange_with_full(depot, store->previous)))
{
- std::swap(store->previous, store->loaded);
- } else
- return NULL;
- }
+ _push(depot->empty, magazine);
}
-static int
-object_depot_return_to_store(object_depot* depot, depot_cpu_store* store,
- void* object)
+static inline depot_cpu_store*
+object_depot_cpu(object_depot* depot)
{
- RecursiveLocker _(store->lock);
-
- // We try to add the object to the loaded magazine if we have one
- // and it's not full, or to the previous one if it is empty. If
- // the magazine depot doesn't provide us with a new empty magazine
- // we return the object directly to the slab.
-
- while (true) {
- if (store->loaded && store->loaded->Push(object))
- return 1;
-
- if ((store->previous && store->previous->IsEmpty())
- || exchange_with_empty(depot, store->previous))
- std::swap(store->loaded, store->previous);
- else
- return 0;
- }
+ return &depot->stores[smp_get_current_cpu()];
}
@@ -215,19 +170,20 @@
depot->empty = NULL;
depot->full_count = depot->empty_count = 0;
- recursive_lock_init(&depot->lock, "depot");
+ rw_lock_init(&depot->outer_lock, "object depot");
+ B_INITIALIZE_SPINLOCK(&depot->inner_lock);
int cpuCount = smp_get_num_cpus();
depot->stores = (depot_cpu_store*)slab_internal_alloc(
sizeof(depot_cpu_store) * cpuCount, flags);
if (depot->stores == NULL) {
- recursive_lock_destroy(&depot->lock);
+ rw_lock_destroy(&depot->outer_lock);
return B_NO_MEMORY;
}
for (int i = 0; i < cpuCount; i++) {
- recursive_lock_init(&depot->stores[i].lock, "cpu store");
- depot->stores[i].loaded = depot->stores[i].previous = NULL;
+ depot->stores[i].loaded = NULL;
+ depot->stores[i].previous = NULL;
}
depot->cookie = cookie;
@@ -242,52 +198,125 @@
{
object_depot_make_empty(depot);
- int cpuCount = smp_get_num_cpus();
- for (int i = 0; i < cpuCount; i++)
- recursive_lock_destroy(&depot->stores[i].lock);
-
slab_internal_free(depot->stores);
- recursive_lock_destroy(&depot->lock);
+ rw_lock_destroy(&depot->outer_lock);
}
void*
object_depot_obtain(object_depot* depot)
{
- return object_depot_obtain_from_store(depot, object_depot_cpu(depot));
+ ReadLocker readLocker(depot->outer_lock);
+ InterruptsLocker interruptsLocker;
+
+ depot_cpu_store* store = object_depot_cpu(depot);
+
+ // To better understand both the Alloc() and Free() logic refer to
+ // Bonwick's ``Magazines and Vmem'' [in 2001 USENIX proceedings]
+
+ // In a nutshell, we try to get an object from the loaded magazine
+ // if it's not empty, or from the previous magazine if it's full
+ // and finally from the Slab if the magazine depot has no full
magazines.
+
+ if (store->loaded == NULL)
+ return NULL;
+
+ while (true) {
+ if (!store->loaded->IsEmpty())
+ return store->loaded->Pop();
+
+ if (store->previous
+ && (store->previous->IsFull()
+ || exchange_with_full(depot, store->previous)))
{
+ std::swap(store->previous, store->loaded);
+ } else
+ return NULL;
+ }
}
int
object_depot_store(object_depot* depot, void* object)
{
- return object_depot_return_to_store(depot, object_depot_cpu(depot),
- object);
+ ReadLocker readLocker(depot->outer_lock);
+ InterruptsLocker interruptsLocker;
+
+ depot_cpu_store* store = object_depot_cpu(depot);
+
+ // We try to add the object to the loaded magazine if we have one
+ // and it's not full, or to the previous one if it is empty. If
+ // the magazine depot doesn't provide us with a new empty magazine
+ // we return the object directly to the slab.
+
+ while (true) {
+ if (store->loaded && store->loaded->Push(object))
+ return 1;
+
+ if ((store->previous && store->previous->IsEmpty())
+ || exchange_with_empty(depot, store->previous)) {
+ std::swap(store->loaded, store->previous);
+ } else {
+ // allocate a new empty magazine
+ interruptsLocker.Unlock();
+ readLocker.Unlock();
+
+ DepotMagazine* magazine = alloc_magazine();
+ if (magazine == NULL)
+ return 0;
+
+ readLocker.Lock();
+ interruptsLocker.Lock();
+
+ push_empty_magazine(depot, magazine);
+ store = object_depot_cpu(depot);
+ }
+ }
}
void
object_depot_make_empty(object_depot* depot)
{
+ WriteLocker writeLocker(depot->outer_lock);
+
+ // collect the store magazines
+
+ DepotMagazine* storeMagazines = NULL;
+
int cpuCount = smp_get_num_cpus();
for (int i = 0; i < cpuCount; i++) {
- depot_cpu_store* store = &depot->stores[i];
+ depot_cpu_store& store = depot->stores[i];
- RecursiveLocker _(store->lock);
+ if (store.loaded) {
+ _push(storeMagazines, store.loaded);
+ store.loaded = NULL;
+ }
- if (depot->stores[i].loaded)
- empty_magazine(depot, depot->stores[i].loaded);
- if (depot->stores[i].previous)
- empty_magazine(depot, depot->stores[i].previous);
- depot->stores[i].loaded = depot->stores[i].previous = NULL;
+ if (store.previous) {
+ _push(storeMagazines, store.previous);
+ store.previous = NULL;
+ }
}
- RecursiveLocker _(depot->lock);
+ // detach the depot's full and empty magazines
- while (depot->full)
- empty_magazine(depot, _pop(depot->full));
+ DepotMagazine* fullMagazines = depot->full;
+ depot->full = NULL;
- while (depot->empty)
- empty_magazine(depot, _pop(depot->empty));
+ DepotMagazine* emptyMagazines = depot->empty;
+ depot->empty = NULL;
+
+ writeLocker.Unlock();
+
+ // free all magazines
+
+ while (storeMagazines != NULL)
+ empty_magazine(depot, _pop(storeMagazines));
+
+ while (fullMagazines != NULL)
+ empty_magazine(depot, _pop(fullMagazines));
+
+ while (emptyMagazines)
+ free_magazine(_pop(emptyMagazines));
}
Other related posts:
- » [haiku-commits] r35200 - in haiku/trunk: headers/private/kernel/slab src/system/kernel/slab - ingo_weinhold
