[haiku-commits] BRANCH pdziepak-github.scheduler [5687811] src/system/kernel/scheduler
- From: pdziepak-github.scheduler <community@xxxxxxxxxxxx>
- To: haiku-commits@xxxxxxxxxxxxx
- Date: Mon, 23 Dec 2013 22:00:32 +0100 (CET)
added 8 changesets to branch 'refs/remotes/pdziepak-github/scheduler'
old head: 6b20e521d5144c08c4f1fdae61379ce89d5e5cf4
new head: 56878118841a0a07aaaa550f813461666bb81cdd
overview: https://github.com/pdziepak/Haiku/compare/6b20e52...5687811
----------------------------------------------------------------------------
b107892: x86: Make arch/smp.h a C++ only header
2f7a304: x86: Fix style
Thanks Axel.
c6730f4: scheduler: Encapsulate PackageEntry fields
Apart from the refactoring this commit takes the opportunity and removes
unnecessary read locks when choosing a package and a core from idle lists.
The data structures are accessed in a thread safe way and it does not really
matter whether the obtained data becomes outdated just when we release the
lock or during our search for the appropriate package/core.
8f7b4cd: scheduler: Remove RunQueue::PeekSecondMaximum()
703edc6: kernel/util: Make MinMaxHeap::Peek*() const
58cac8f: scheduler: Encapsulate CoreEntry fields
adb2f9f: scheduler: Encapsulate CPUEntry fields
5687811: scheduler: Encapsulate ThreadData fields
[ Pawel Dziepak <pdziepak@xxxxxxxxxxx> ]
----------------------------------------------------------------------------
14 files changed, 857 insertions(+), 427 deletions(-)
headers/private/kernel/arch/smp.h | 13 +-
headers/private/kernel/util/MinMaxHeap.h | 8 +-
src/system/kernel/arch/x86/apic.cpp | 3 +-
src/system/kernel/arch/x86/timers/x86_apic.cpp | 1 -
src/system/kernel/scheduler/RunQueue.h | 21 +-
src/system/kernel/scheduler/low_latency.cpp | 46 +-
src/system/kernel/scheduler/power_saving.cpp | 40 +-
src/system/kernel/scheduler/scheduler.cpp | 285 +++++-------
src/system/kernel/scheduler/scheduler_common.h | 2 +-
src/system/kernel/scheduler/scheduler_cpu.cpp | 431 +++++++++++++++----
src/system/kernel/scheduler/scheduler_cpu.h | 297 ++++++++++++-
src/system/kernel/scheduler/scheduler_modes.h | 2 +-
src/system/kernel/scheduler/scheduler_thread.cpp | 19 +-
src/system/kernel/scheduler/scheduler_thread.h | 116 ++---
############################################################################
Commit: b107892ec0db0baecd237a82f4cfb09de5182fa5
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Fri Dec 20 21:05:26 2013 UTC
x86: Make arch/smp.h a C++ only header
----------------------------------------------------------------------------
diff --git a/headers/private/kernel/arch/smp.h
b/headers/private/kernel/arch/smp.h
index 440617b..4b59e45 100644
--- a/headers/private/kernel/arch/smp.h
+++ b/headers/private/kernel/arch/smp.h
@@ -8,23 +8,18 @@
#include <kernel.h>
+
struct kernel_args;
class CPUSet;
-#ifdef __cplusplus
-extern "C" {
-#endif
+status_t arch_smp_init(kernel_args* args);
+status_t arch_smp_per_cpu_init(kernel_args* args, int32 cpu);
-status_t arch_smp_init(struct kernel_args *args);
-status_t arch_smp_per_cpu_init(struct kernel_args *args, int32 cpu);
void arch_smp_send_ici(int32 target_cpu);
-void arch_smp_send_broadcast_ici(void);
+void arch_smp_send_broadcast_ici();
void arch_smp_send_multicast_ici(CPUSet& cpuSet);
-#ifdef __cplusplus
-}
-#endif
#endif /* KERNEL_ARCH_SMP_H */
diff --git a/src/system/kernel/arch/x86/timers/x86_apic.cpp
b/src/system/kernel/arch/x86/timers/x86_apic.cpp
index 6ac3e59..568cd65 100644
--- a/src/system/kernel/arch/x86/timers/x86_apic.cpp
+++ b/src/system/kernel/arch/x86/timers/x86_apic.cpp
@@ -14,7 +14,6 @@
#include <arch/x86/apic.h>
#include <arch/cpu.h>
-#include <arch/smp.h>
#include "apic_timer.h"
############################################################################
Commit: 2f7a30474ae6fd8267c11b4d527eeedae5906868
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Sun Dec 22 23:09:03 2013 UTC
x86: Fix style
Thanks Axel.
----------------------------------------------------------------------------
diff --git a/src/system/kernel/arch/x86/apic.cpp
b/src/system/kernel/arch/x86/apic.cpp
index c6719cc..1761029 100644
--- a/src/system/kernel/arch/x86/apic.cpp
+++ b/src/system/kernel/arch/x86/apic.cpp
@@ -31,7 +31,8 @@ apic_available()
}
-bool x2apic_available()
+bool
+x2apic_available()
{
return sX2APIC;
}
############################################################################
Commit: c6730f4452a16f0009119fee9b829a704a55b628
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Sun Dec 22 23:56:44 2013 UTC
scheduler: Encapsulate PackageEntry fields
Apart from the refactoring this commit takes the opportunity and removes
unnecessary read locks when choosing a package and a core from idle lists.
The data structures are accessed in a thread safe way and it does not really
matter whether the obtained data becomes outdated just when we release the
lock or during our search for the appropriate package/core.
----------------------------------------------------------------------------
diff --git a/src/system/kernel/scheduler/low_latency.cpp
b/src/system/kernel/scheduler/low_latency.cpp
index 4096997..441a609 100644
--- a/src/system/kernel/scheduler/low_latency.cpp
+++ b/src/system/kernel/scheduler/low_latency.cpp
@@ -19,7 +19,7 @@ const bigtime_t kCacheExpire = 100000;
static void
-switch_to_mode(void)
+switch_to_mode()
{
}
@@ -35,7 +35,7 @@ has_cache_expired(const ThreadData* threadData)
{
ASSERT(!gSingleCore);
- CoreEntry* core = threadData->GetCore();
+ CoreEntry* core = threadData->Core();
bigtime_t activeTime;
uint32 count;
@@ -48,39 +48,19 @@ has_cache_expired(const ThreadData* threadData)
}
-static inline PackageEntry*
-get_most_idle_package(void)
-{
- PackageEntry* current = &gPackageEntries[0];
- for (int32 i = 1; i < gPackageCount; i++) {
- if (gPackageEntries[i].fIdleCoreCount > current->fIdleCoreCount)
- current = &gPackageEntries[i];
- }
-
- if (current->fIdleCoreCount == 0)
- return NULL;
-
- return current;
-}
-
-
static CoreEntry*
choose_core(const ThreadData* /* threadData */)
{
- ReadSpinLocker locker(gIdlePackageLock);
// wake new package
PackageEntry* package = gIdlePackageList.Last();
if (package == NULL) {
// wake new core
- package = get_most_idle_package();
+ package = PackageEntry::GetMostIdlePackage();
}
- locker.Unlock();
CoreEntry* core = NULL;
- if (package != NULL) {
- ReadSpinLocker _(package->fCoreLock);
- core = package->fIdleCores.Last();
- }
+ if (package != NULL)
+ core = package->GetIdleCore();
if (core == NULL) {
ReadSpinLocker coreLocker(gCoreHeapsLock);
@@ -98,7 +78,7 @@ choose_core(const ThreadData* /* threadData */)
static bool
should_rebalance(const ThreadData* threadData)
{
- int32 coreLoad = threadData->GetCore()->GetLoad();
+ int32 coreLoad = threadData->Core()->GetLoad();
// If the thread produces more than 50% of the load, leave it here. In
// such situation it is better to move other threads away.
diff --git a/src/system/kernel/scheduler/power_saving.cpp
b/src/system/kernel/scheduler/power_saving.cpp
index c1eeb61..2ca77db 100644
--- a/src/system/kernel/scheduler/power_saving.cpp
+++ b/src/system/kernel/scheduler/power_saving.cpp
@@ -22,7 +22,7 @@ static CoreEntry* sSmallTaskCore;
static void
-switch_to_mode(void)
+switch_to_mode()
{
sSmallTaskCore = NULL;
}
@@ -46,7 +46,7 @@ has_cache_expired(const ThreadData* threadData)
static CoreEntry*
-choose_small_task_core(void)
+choose_small_task_core()
{
ReadSpinLocker locker(gCoreHeapsLock);
CoreEntry* core = gCoreLoadHeap.PeekMaximum();
@@ -64,27 +64,15 @@ choose_small_task_core(void)
static CoreEntry*
-choose_idle_core(void)
+choose_idle_core()
{
- PackageEntry* package = NULL;
+ PackageEntry* package = PackageEntry::GetLeastIdlePackage();
- for (int32 i = 0; i < gPackageCount; i++) {
- PackageEntry* current = &gPackageEntries[i];
- if (current->fIdleCoreCount != 0 && (package == NULL
- || current->fIdleCoreCount <
package->fIdleCoreCount)) {
- package = current;
- }
- }
-
- if (package == NULL) {
- ReadSpinLocker _(gIdlePackageLock);
+ if (package == NULL)
package = gIdlePackageList.Last();
- }
- if (package != NULL) {
- ReadSpinLocker _(package->fCoreLock);
- return package->fIdleCores.Last();
- }
+ if (package != NULL)
+ return package->GetIdleCore();
return NULL;
}
@@ -125,7 +113,7 @@ should_rebalance(const ThreadData* threadData)
{
ASSERT(!gSingleCore);
- CoreEntry* core = threadData->GetCore();
+ CoreEntry* core = threadData->Core();
int32 coreLoad = core->GetLoad();
if (coreLoad > kHighLoad) {
@@ -161,7 +149,7 @@ should_rebalance(const ThreadData* threadData)
static inline void
-pack_irqs(void)
+pack_irqs()
{
CoreEntry* smallTaskCore = atomic_pointer_get(&sSmallTaskCore);
if (smallTaskCore == NULL)
diff --git a/src/system/kernel/scheduler/scheduler.cpp
b/src/system/kernel/scheduler/scheduler.cpp
index 8046bd5..0dd94b9 100644
--- a/src/system/kernel/scheduler/scheduler.cpp
+++ b/src/system/kernel/scheduler/scheduler.cpp
@@ -61,7 +61,7 @@ public:
}
};
-class BigSchedulerLocking {
+class InterruptsBigSchedulerLocking {
public:
bool Lock(int* lockable)
{
@@ -80,11 +80,11 @@ public:
};
class InterruptsBigSchedulerLocker :
- public AutoLocker<int, BigSchedulerLocking> {
+ public AutoLocker<int, InterruptsBigSchedulerLocking> {
public:
InterruptsBigSchedulerLocker()
:
- AutoLocker<int, BigSchedulerLocking>(&fState, false, true)
+ AutoLocker<int, InterruptsBigSchedulerLocking>(&fState, false,
true)
{
}
@@ -157,10 +157,10 @@ enqueue(Thread* thread, bool newOne)
targetCPU = &gCPUEntries[thread->previous_cpu->cpu_num];
} else if (gSingleCore)
targetCore = &gCoreEntries[0];
- else if (threadData->GetCore() != NULL
+ else if (threadData->Core() != NULL
&& (!newOne || !threadData->HasCacheExpired())
&& !threadData->ShouldRebalance()) {
- targetCore = threadData->GetCore();
+ targetCore = threadData->Core();
}
bool rescheduleNeeded = threadData->ChooseCoreAndCPU(targetCore,
targetCPU);
@@ -194,7 +194,12 @@ enqueue(Thread* thread, bool newOne)
void
scheduler_enqueue_in_run_queue(Thread *thread)
{
- InterruptsSchedulerModeLocker _;
+#if KDEBUG
+ if (are_interrupts_enabled())
+ panic("scheduler_enqueue_in_run_queue: called with interrupts
enabled");
+#endif
+
+ SchedulerModeLocker _;
TRACE("enqueueing new thread %ld with static priority %ld\n",
thread->id,
thread->priority);
@@ -213,6 +218,11 @@ scheduler_enqueue_in_run_queue(Thread *thread)
int32
scheduler_set_thread_priority(Thread *thread, int32 priority)
{
+#if KDEBUG
+ if (!are_interrupts_enabled())
+ panic("scheduler_set_thread_priority: called with interrupts
disabled");
+#endif
+
InterruptsSpinLocker _(thread->scheduler_lock);
SchedulerModeLocker modeLocker;
@@ -230,12 +240,12 @@ scheduler_set_thread_priority(Thread *thread, int32
priority)
if (thread->state != B_THREAD_READY) {
if (thread->state == B_THREAD_RUNNING) {
- ASSERT(threadData->GetCore() != NULL);
+ ASSERT(threadData->Core() != NULL);
ASSERT(thread->cpu != NULL);
CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num];
- SpinLocker coreLocker(threadData->GetCore()->fCPULock);
+ SpinLocker coreLocker(threadData->Core()->fCPULock);
cpu->UpdatePriority(priority);
}
@@ -497,7 +507,7 @@ reschedule(int32 nextState)
NotifySchedulerListeners(&SchedulerListener::ThreadScheduled,
oldThread, nextThread);
- ASSERT(nextThreadData->GetCore() == core);
+ ASSERT(nextThreadData->Core() == core);
nextThread->state = B_THREAD_RUNNING;
// update CPU heap
@@ -540,6 +550,11 @@ reschedule(int32 nextState)
void
scheduler_reschedule(int32 nextState)
{
+#if KDEBUG
+ if (are_interrupts_enabled())
+ panic("scheduler_reschedule: called with interrupts enabled");
+#endif
+
if (!sSchedulerEnabled) {
Thread* thread = thread_get_current_thread();
if (thread != NULL && nextState != B_THREAD_READY)
@@ -588,7 +603,7 @@ scheduler_on_thread_destroy(Thread* thread)
thread. Interrupts must be disabled and will be disabled when returning.
*/
void
-scheduler_start(void)
+scheduler_start()
{
InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock);
@@ -621,7 +636,7 @@ unassign_thread(Thread* thread, void* data)
{
CoreEntry* core = static_cast<CoreEntry*>(data);
- if (thread->scheduler_data->GetCore() == core
+ if (thread->scheduler_data->Core() == core
&& thread->pinned_to_cpu == 0) {
thread->scheduler_data->UnassignCore();
}
@@ -631,6 +646,11 @@ unassign_thread(Thread* thread, void* data)
void
scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
{
+#if KDEBUG
+ if (are_interrupts_enabled())
+ panic("scheduler_set_cpu_enabled: called with interrupts
enabled");
+#endif
+
dprintf("scheduler: %s CPU %" B_PRId32 "\n",
enabled ? "enabling" : "disabling", cpuID);
@@ -667,12 +687,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
gCoreLoadHeap.RemoveMinimum();
}
- package->fIdleCores.Remove(core);
- package->fIdleCoreCount--;
- package->fCoreCount--;
-
- if (package->fCoreCount == 0)
- gIdlePackageList.Remove(package);
+ package->RemoveIdleCore(core);
// get rid of threads
thread_map(unassign_thread, core);
@@ -689,7 +704,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
threadData->fWentSleepCount = -1;
}
- ASSERT(threadData->GetCore() == NULL);
+ ASSERT(threadData->Core() == NULL);
enqueue(threadData->GetThread(), false);
}
} else if (oldCPUCount == 0) {
@@ -701,12 +716,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
core->fHighLoad = false;
gCoreLoadHeap.Insert(core, 0);
- package->fCoreCount++;
- package->fIdleCoreCount++;
- package->fIdleCores.Add(core);
-
- if (package->fCoreCount == 1)
- gIdlePackageList.Add(package);
+ package->AddIdleCore(core);
}
if (enabled) {
@@ -846,11 +856,8 @@ init()
new(&gIdlePackageList) IdlePackageList;
- for (int32 i = 0; i < packageCount; i++) {
- gPackageEntries[i].fPackageID = i;
- gPackageEntries[i].fCoreCount = coreCount / packageCount;
- gIdlePackageList.Insert(&gPackageEntries[i]);
- }
+ for (int32 i = 0; i < packageCount; i++)
+ gPackageEntries[i].Init(i);
for (int32 i = 0; i < coreCount; i++) {
gCoreEntries[i].fCoreID = i;
@@ -869,10 +876,8 @@ init()
gCPUEntries[i].fCore = core;
core->fPackage = package;
- if (core->fCPUHeap.PeekMaximum() == NULL) {
- package->fIdleCoreCount++;
- package->fIdleCores.Insert(core);
- }
+ if (core->fCPUHeap.PeekMaximum() == NULL)
+ package->AddIdleCore(core);
result = core->fCPUHeap.Insert(&gCPUEntries[i],
B_IDLE_PRIORITY);
if (result != B_OK)
@@ -888,7 +893,7 @@ init()
void
-scheduler_init(void)
+scheduler_init()
{
int32 cpuCount = smp_get_num_cpus();
dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s and %"
B_PRId32
@@ -914,7 +919,7 @@ scheduler_init(void)
void
-scheduler_enable_scheduling(void)
+scheduler_enable_scheduling()
{
sSchedulerEnabled = true;
}
@@ -972,7 +977,7 @@ _user_estimate_max_scheduling_latency(thread_id id)
BReference<Thread> threadReference(thread, true);
ThreadData* threadData = thread->scheduler_data;
- CoreEntry* core = threadData->GetCore();
+ CoreEntry* core = threadData->Core();
if (core == NULL)
core = &gCoreEntries[get_random<int32>() % gCoreCount];
@@ -1003,7 +1008,7 @@ _user_set_scheduler_mode(int32 mode)
int32
-_user_get_scheduler_mode(void)
+_user_get_scheduler_mode()
{
return gCurrentModeID;
}
diff --git a/src/system/kernel/scheduler/scheduler_common.h
b/src/system/kernel/scheduler/scheduler_common.h
index 75490dd..02c9691 100644
--- a/src/system/kernel/scheduler/scheduler_common.h
+++ b/src/system/kernel/scheduler/scheduler_common.h
@@ -45,7 +45,7 @@ const int kLoadDifference = kMaxLoad * 20 / 100;
extern bool gSingleCore;
-void init_debug_commands(void);
+void init_debug_commands();
} // namespace Scheduler
diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp
b/src/system/kernel/scheduler/scheduler_cpu.cpp
index 8c9b048..f4d55eb 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.cpp
+++ b/src/system/kernel/scheduler/scheduler_cpu.cpp
@@ -16,6 +16,13 @@
using namespace Scheduler;
+class Scheduler::DebugDumper {
+public:
+ static void DumpIdleCoresInPackage(PackageEntry* package);
+
+};
+
+
static CPUPriorityHeap sDebugCPUHeap;
static CoreLoadHeap sDebugCoreHeap;
@@ -67,37 +74,10 @@ CPUEntry::UpdatePriority(int32 priority)
return;
PackageEntry* packageEntry = fCore->fPackage;
- if (maxPriority == B_IDLE_PRIORITY) {
- WriteSpinLocker _(packageEntry->fCoreLock);
-
- // core goes idle
- ASSERT(packageEntry->fIdleCoreCount >= 0);
- ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount);
-
- packageEntry->fIdleCoreCount++;
- packageEntry->fIdleCores.Add(fCore);
-
- if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) {
- // package goes idle
- WriteSpinLocker _(gIdlePackageLock);
- gIdlePackageList.Add(packageEntry);
- }
- } else if (corePriority == B_IDLE_PRIORITY) {
- WriteSpinLocker _(packageEntry->fCoreLock);
-
- // core wakes up
- ASSERT(packageEntry->fIdleCoreCount > 0);
- ASSERT(packageEntry->fIdleCoreCount <=
packageEntry->fCoreCount);
-
- packageEntry->fIdleCoreCount--;
- packageEntry->fIdleCores.Remove(fCore);
-
- if (packageEntry->fIdleCoreCount + 1 ==
packageEntry->fCoreCount) {
- // package wakes up
- WriteSpinLocker _(gIdlePackageLock);
- gIdlePackageList.Remove(packageEntry);
- }
- }
+ if (maxPriority == B_IDLE_PRIORITY)
+ packageEntry->CoreGoesIdle(fCore);
+ else if (corePriority == B_IDLE_PRIORITY)
+ packageEntry->CoreWakesUp(fCore);
}
@@ -371,6 +351,94 @@ PackageEntry::PackageEntry()
}
+void
+PackageEntry::Init(int32 id)
+{
+ fPackageID = id;
+}
+
+
+inline void
+PackageEntry::CoreGoesIdle(CoreEntry* core)
+{
+ WriteSpinLocker _(fCoreLock);
+
+ ASSERT(fIdleCoreCount >= 0);
+ ASSERT(fIdleCoreCount < fCoreCount);
+
+ fIdleCoreCount++;
+ fIdleCores.Add(core);
+
+ if (fIdleCoreCount == fCoreCount) {
+ // package goes idle
+ WriteSpinLocker _(gIdlePackageLock);
+ gIdlePackageList.Add(this);
+ }
+}
+
+
+inline void
+PackageEntry::CoreWakesUp(CoreEntry* core)
+{
+ WriteSpinLocker _(fCoreLock);
+
+ ASSERT(fIdleCoreCount > 0);
+ ASSERT(fIdleCoreCount <= fCoreCount);
+
+ fIdleCoreCount--;
+ fIdleCores.Remove(core);
+
+ if (fIdleCoreCount + 1 == fCoreCount) {
+ // package wakes up
+ WriteSpinLocker _(gIdlePackageLock);
+ gIdlePackageList.Remove(this);
+ }
+}
+
+
+void
+PackageEntry::AddIdleCore(CoreEntry* core)
+{
+ fCoreCount++;
+ fIdleCoreCount++;
+ fIdleCores.Add(core);
+
+ if (fCoreCount == 1)
+ gIdlePackageList.Add(this);
+}
+
+
+void
+PackageEntry::RemoveIdleCore(CoreEntry* core)
+{
+ fIdleCores.Remove(core);
+ fIdleCoreCount--;
+ fCoreCount--;
+
+ if (fCoreCount == 0)
+ gIdlePackageList.Remove(this);
+}
+
+
+/* static */ void
+DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
+{
+ kprintf("%-7" B_PRId32 " ", package->fPackageID);
+
+ DoublyLinkedList<CoreEntry>::ReverseIterator iterator
+ = package->fIdleCores.GetReverseIterator();
+ if (iterator.HasNext()) {
+ while (iterator.HasNext()) {
+ CoreEntry* coreEntry = iterator.Next();
+ kprintf("%" B_PRId32 "%s", coreEntry->fCoreID,
+ iterator.HasNext() ? ", " : "");
+ }
+ } else
+ kprintf("-");
+ kprintf("\n");
+}
+
+
static int
dump_run_queue(int argc, char **argv)
{
@@ -429,22 +497,8 @@ dump_idle_cores(int argc, char** argv)
if (idleIterator.HasNext()) {
kprintf("package cores\n");
- while (idleIterator.HasNext()) {
- PackageEntry* entry = idleIterator.Next();
- kprintf("%-7" B_PRId32 " ", entry->fPackageID);
-
- DoublyLinkedList<CoreEntry>::ReverseIterator iterator
- = entry->fIdleCores.GetReverseIterator();
- if (iterator.HasNext()) {
- while (iterator.HasNext()) {
- CoreEntry* coreEntry = iterator.Next();
- kprintf("%" B_PRId32 "%s",
coreEntry->fCoreID,
- iterator.HasNext() ? ", " : "");
- }
- } else
- kprintf("-");
- kprintf("\n");
- }
+ while (idleIterator.HasNext())
+
DebugDumper::DumpIdleCoresInPackage(idleIterator.Next());
} else
kprintf("No idle packages.\n");
@@ -452,7 +506,7 @@ dump_idle_cores(int argc, char** argv)
}
-void Scheduler::init_debug_commands(void)
+void Scheduler::init_debug_commands()
{
new(&sDebugCPUHeap) CPUPriorityHeap(smp_get_num_cpus());
new(&sDebugCoreHeap) CoreLoadHeap(smp_get_num_cpus());
diff --git a/src/system/kernel/scheduler/scheduler_cpu.h
b/src/system/kernel/scheduler/scheduler_cpu.h
index 4d02a64..be37213 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.h
+++ b/src/system/kernel/scheduler/scheduler_cpu.h
@@ -21,6 +21,8 @@
namespace Scheduler {
+class DebugDumper;
+
struct ThreadData;
struct CPUEntry;
@@ -122,15 +124,32 @@ public:
// packages can go to the deep state of sleep). The heap stores only packages
// with at least one core active and one core idle. The packages with all cores
// idle are stored in gPackageIdleList (in LIFO manner).
-struct PackageEntry : public DoublyLinkedListLinkImpl<PackageEntry> {
+class PackageEntry : public DoublyLinkedListLinkImpl<PackageEntry> {
+public:
PackageEntry();
+ void
Init(int32 id);
+
+ inline void
CoreGoesIdle(CoreEntry* core);
+ inline void
CoreWakesUp(CoreEntry* core);
+
+ inline CoreEntry*
GetIdleCore() const;
+
+ void
AddIdleCore(CoreEntry* core);
+ void
RemoveIdleCore(CoreEntry* core);
+
+ static inline PackageEntry* GetMostIdlePackage();
+ static inline PackageEntry* GetLeastIdlePackage();
+
+private:
int32
fPackageID;
DoublyLinkedList<CoreEntry>
fIdleCores;
int32
fIdleCoreCount;
int32
fCoreCount;
rw_spinlock
fCoreLock;
+
+ friend class DebugDumper;
} CACHE_LINE_ALIGN;
typedef DoublyLinkedList<PackageEntry> IdlePackageList;
@@ -163,6 +182,48 @@ CoreEntry::GetCore(int32 cpu)
}
+inline CoreEntry*
+PackageEntry::GetIdleCore() const
+{
+ return fIdleCores.Last();
+}
+
+
+/* static */ inline PackageEntry*
+PackageEntry::GetMostIdlePackage()
+{
+ PackageEntry* current = &gPackageEntries[0];
+ for (int32 i = 1; i < gPackageCount; i++) {
+ if (gPackageEntries[i].fIdleCoreCount > current->fIdleCoreCount)
+ current = &gPackageEntries[i];
+ }
+
+ if (current->fIdleCoreCount == 0)
+ return NULL;
+
+ return current;
+}
+
+
+/* static */ inline PackageEntry*
+PackageEntry::GetLeastIdlePackage()
+{
+ PackageEntry* package = NULL;
+
+ for (int32 i = 0; i < gPackageCount; i++) {
+ PackageEntry* current = &gPackageEntries[i];
+
+ int32 currentIdleCoreCount = current->fIdleCoreCount;
+ if (currentIdleCoreCount != 0 && (package == NULL
+ || currentIdleCoreCount <
package->fIdleCoreCount)) {
+ package = current;
+ }
+ }
+
+ return package;
+}
+
+
} // namespace Scheduler
diff --git a/src/system/kernel/scheduler/scheduler_modes.h
b/src/system/kernel/scheduler/scheduler_modes.h
index 378befc..09670dd 100644
--- a/src/system/kernel/scheduler/scheduler_modes.h
+++ b/src/system/kernel/scheduler/scheduler_modes.h
@@ -21,7 +21,7 @@ struct scheduler_mode_operations {
bigtime_t maximum_latency;
- void (*switch_to_mode)(void);
+ void (*switch_to_mode)();
void (*set_cpu_enabled)(int32 cpu,
bool enabled);
bool (*has_cache_expired)(
const
Scheduler::ThreadData* threadData);
diff --git a/src/system/kernel/scheduler/scheduler_thread.h
b/src/system/kernel/scheduler/scheduler_thread.h
index 5f4ab2f..7ebf591 100644
--- a/src/system/kernel/scheduler/scheduler_thread.h
+++ b/src/system/kernel/scheduler/scheduler_thread.h
@@ -53,7 +53,7 @@ public:
inline Thread* GetThread() const { return fThread; }
inline int32 GetLoad() const { return fLoad; }
- inline CoreEntry* GetCore() const { return fCore; }
+ inline CoreEntry* Core() const { return fCore; }
inline void UnassignCore() { fCore = NULL; }
bigtime_t fStolenTime;
############################################################################
Commit: 8f7b4cd82f5b40515b1ab5a5cbb5525642c83c89
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Mon Dec 23 02:51:02 2013 UTC
scheduler: Remove RunQueue::PeekSecondMaximum()
----------------------------------------------------------------------------
diff --git a/src/system/kernel/scheduler/RunQueue.h
b/src/system/kernel/scheduler/RunQueue.h
index 4ecda72..9d54a3a 100644
--- a/src/system/kernel/scheduler/RunQueue.h
+++ b/src/system/kernel/scheduler/RunQueue.h
@@ -83,8 +83,7 @@ public:
inline status_t GetInitStatus();
- inline Element* PeekMaximum();
- inline Element* PeekSecondMaximum();
+ inline Element* PeekMaximum() const;
inline void PushFront(Element* element, unsigned int
priority);
inline void PushBack(Element* elementt, unsigned int
priority);
@@ -251,7 +250,7 @@ RUN_QUEUE_CLASS_NAME::GetInitStatus()
RUN_QUEUE_TEMPLATE_LIST
Element*
-RUN_QUEUE_CLASS_NAME::PeekMaximum()
+RUN_QUEUE_CLASS_NAME::PeekMaximum() const
{
int priority = fBitmap.GetHighestSet();
if (priority < 0)
@@ -272,22 +271,6 @@ RUN_QUEUE_CLASS_NAME::PeekMaximum()
RUN_QUEUE_TEMPLATE_LIST
-Element*
-RUN_QUEUE_CLASS_NAME::PeekSecondMaximum()
-{
- int priority = fBitmap.GetHighestSet();
- if (priority < 0)
- return NULL;
-
- fBitmap.Clear(priority);
- Element* element = PeekMaximum();
- fBitmap.Set(priority);
-
- return element;
-}
-
-
-RUN_QUEUE_TEMPLATE_LIST
void
RUN_QUEUE_CLASS_NAME::PushFront(Element* element,
unsigned int priority)
############################################################################
Commit: 703edc6cc3a0e6164968c7db8fda080054f033e2
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Mon Dec 23 02:52:13 2013 UTC
kernel/util: Make MinMaxHeap::Peek*() const
----------------------------------------------------------------------------
diff --git a/headers/private/kernel/util/MinMaxHeap.h
b/headers/private/kernel/util/MinMaxHeap.h
index cc4ab0a..40fd1b1 100644
--- a/headers/private/kernel/util/MinMaxHeap.h
+++ b/headers/private/kernel/util/MinMaxHeap.h
@@ -73,8 +73,8 @@ public:
MinMaxHeap(int initialSize);
~MinMaxHeap();
- inline Element* PeekMinimum();
- inline Element* PeekMaximum();
+ inline Element* PeekMinimum() const;
+ inline Element* PeekMaximum() const;
static const Key& GetKey(Element* element);
@@ -190,7 +190,7 @@ MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap()
MIN_MAX_HEAP_TEMPLATE_LIST
Element*
-MIN_MAX_HEAP_CLASS_NAME::PeekMinimum()
+MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() const
{
if (fMinLastElement > 0)
return fMinElements[0];
@@ -205,7 +205,7 @@ MIN_MAX_HEAP_CLASS_NAME::PeekMinimum()
MIN_MAX_HEAP_TEMPLATE_LIST
Element*
-MIN_MAX_HEAP_CLASS_NAME::PeekMaximum()
+MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() const
{
if (fMaxLastElement > 0)
return fMaxElements[0];
############################################################################
Commit: 58cac8fe0dff9ba5314913bebc7da07c0e8f9524
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Mon Dec 23 17:15:31 2013 UTC
scheduler: Encapsulate CoreEntry fields
----------------------------------------------------------------------------
diff --git a/src/system/kernel/scheduler/low_latency.cpp
b/src/system/kernel/scheduler/low_latency.cpp
index 441a609..74cddd6 100644
--- a/src/system/kernel/scheduler/low_latency.cpp
+++ b/src/system/kernel/scheduler/low_latency.cpp
@@ -37,13 +37,7 @@ has_cache_expired(const ThreadData* threadData)
CoreEntry* core = threadData->Core();
- bigtime_t activeTime;
- uint32 count;
- do {
- count = acquire_read_seqlock(&core->fActiveTimeLock);
- activeTime = core->fActiveTime;
- } while (!release_read_seqlock(&core->fActiveTimeLock, count));
-
+ bigtime_t activeTime = core->GetActiveTime();
return activeTime - threadData->fWentSleepActive > kCacheExpire;
}
@@ -138,9 +132,7 @@ rebalance_irqs(bool idle)
other = gCoreHighLoadHeap.PeekMinimum();
coreLocker.Unlock();
- SpinLocker cpuLocker(other->fCPULock);
- int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber;
- cpuLocker.Unlock();
+ int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
ASSERT(other != NULL);
diff --git a/src/system/kernel/scheduler/power_saving.cpp
b/src/system/kernel/scheduler/power_saving.cpp
index 2ca77db..d6f3a7d 100644
--- a/src/system/kernel/scheduler/power_saving.cpp
+++ b/src/system/kernel/scheduler/power_saving.cpp
@@ -164,9 +164,7 @@ pack_irqs()
irq_assignment* irq =
(irq_assignment*)list_get_first_item(&cpu->irqs);
locker.Unlock();
- ReadSpinLocker coreLocker(gCoreHeapsLock);
- int32 newCPU =
smallTaskCore->fCPUHeap.PeekMinimum()->fCPUNumber;
- coreLocker.Unlock();
+ int32 newCPU =
smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber;
if (newCPU != cpu->cpu_num)
assign_io_interrupt_to_cpu(irq->irq, newCPU);
@@ -209,9 +207,7 @@ rebalance_irqs(bool idle)
coreLocker.Unlock();
if (other == NULL)
return;
- SpinLocker cpuLocker(other->fCPULock);
- int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber;
- cpuLocker.Unlock();
+ int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu());
if (other == core)
diff --git a/src/system/kernel/scheduler/scheduler.cpp
b/src/system/kernel/scheduler/scheduler.cpp
index 0dd94b9..97e2e52 100644
--- a/src/system/kernel/scheduler/scheduler.cpp
+++ b/src/system/kernel/scheduler/scheduler.cpp
@@ -92,6 +92,11 @@ private:
int fState;
};
+class ThreadEnqueuer : public ThreadProcessing {
+public:
+ void operator()(ThreadData* thread);
+};
+
scheduler_mode gCurrentModeID;
scheduler_mode_operations* gCurrentMode;
@@ -133,6 +138,16 @@ static int32* sCPUToCore;
static int32* sCPUToPackage;
+static void enqueue(Thread* thread, bool newOne);
+
+
+void
+ThreadEnqueuer::operator()(ThreadData* thread)
+{
+ enqueue(thread->GetThread(), false);
+}
+
+
void
scheduler_dump_thread_data(Thread* thread)
{
@@ -245,7 +260,7 @@ scheduler_set_thread_priority(Thread *thread, int32
priority)
ASSERT(thread->cpu != NULL);
CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num];
- SpinLocker coreLocker(threadData->Core()->fCPULock);
+ CoreCPUHeapLocker _(threadData->Core());
cpu->UpdatePriority(priority);
}
@@ -467,7 +482,7 @@ reschedule(int32 nextState)
ThreadData* nextThreadData;
if (gCPU[thisCPU].disabled) {
if (!thread_is_idle_thread(oldThread)) {
- SpinLocker runQueueLocker(core->fQueueLock);
+ CoreRunQueueLocker _(core);
nextThreadData =
cpu->fRunQueue.GetHead(B_IDLE_PRIORITY);
cpu->fRunQueue.Remove(nextThreadData);
@@ -483,7 +498,7 @@ reschedule(int32 nextState)
}
Thread* nextThread = nextThreadData->GetThread();
- SpinLocker cpuLocker(core->fCPULock);
+ CoreCPUHeapLocker cpuLocker(core);
cpu->UpdatePriority(nextThreadData->GetEffectivePriority());
cpuLocker.Unlock();
@@ -511,9 +526,9 @@ reschedule(int32 nextState)
nextThread->state = B_THREAD_RUNNING;
// update CPU heap
- SpinLocker coreLocker(core->fCPULock);
+ cpuLocker.Lock();
cpu->UpdatePriority(nextThreadData->GetEffectivePriority());
- coreLocker.Unlock();
+ cpuLocker.Unlock();
// track kernel time (user time is tracked in thread_at_kernel_entry())
update_thread_times(oldThread, nextThread);
@@ -631,18 +646,6 @@ scheduler_set_operation_mode(scheduler_mode mode)
}
-static void
-unassign_thread(Thread* thread, void* data)
-{
- CoreEntry* core = static_cast<CoreEntry*>(data);
-
- if (thread->scheduler_data->Core() == core
- && thread->pinned_to_cpu == 0) {
- thread->scheduler_data->UnassignCore();
- }
-}
-
-
void
scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
{
@@ -660,78 +663,21 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
CPUEntry* cpu = &gCPUEntries[cpuID];
CoreEntry* core = cpu->fCore;
- PackageEntry* package = core->fPackage;
- int32 oldCPUCount = core->fCPUCount;
+ int32 oldCPUCount = core->CPUCount();
ASSERT(oldCPUCount >= 0);
- if (enabled)
- core->fCPUCount++;
- else {
- cpu->UpdatePriority(B_IDLE_PRIORITY);
- core->fCPUCount--;
- }
-
- gCPU[cpuID].disabled = !enabled;
-
- if (core->fCPUCount == 0) {
- // core has been disabled
- ASSERT(!enabled);
-
- if (core->fHighLoad) {
- gCoreHighLoadHeap.ModifyKey(core, -1);
- ASSERT(gCoreHighLoadHeap.PeekMinimum() == core);
- gCoreHighLoadHeap.RemoveMinimum();
- } else {
- gCoreLoadHeap.ModifyKey(core, -1);
- ASSERT(gCoreLoadHeap.PeekMinimum() == core);
- gCoreLoadHeap.RemoveMinimum();
- }
-
- package->RemoveIdleCore(core);
-
- // get rid of threads
- thread_map(unassign_thread, core);
-
- core->fThreadCount = 0;
- while (core->fRunQueue.PeekMaximum() != NULL) {
- ThreadData* threadData = core->fRunQueue.PeekMaximum();
-
- core->fRunQueue.Remove(threadData);
- threadData->fEnqueued = false;
-
- if (threadData->fWentSleepCount == 0) {
- core->fThreadList.Remove(threadData);
- threadData->fWentSleepCount = -1;
- }
-
- ASSERT(threadData->Core() == NULL);
- enqueue(threadData->GetThread(), false);
- }
- } else if (oldCPUCount == 0) {
- // core has been reenabled
- ASSERT(enabled);
-
- cpu->fLoad = 0;
- core->fLoad = 0;
- core->fHighLoad = false;
- gCoreLoadHeap.Insert(core, 0);
-
- package->AddIdleCore(core);
- }
-
if (enabled) {
- core->fCPUHeap.Insert(cpu, B_IDLE_PRIORITY);
cpu->fLoad = 0;
+ core->AddCPU(cpu);
} else {
- core->fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1);
- ASSERT(core->fCPUHeap.PeekMaximum() == cpu);
- core->fCPUHeap.RemoveMaximum();
+ cpu->UpdatePriority(B_IDLE_PRIORITY);
- ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad);
- core->fLoad -= cpu->fLoad;
- ASSERT(core->fLoad >= 0);
+ ThreadEnqueuer enqueuer;
+ core->RemoveCPU(cpu, enqueuer);
}
+ gCPU[cpuID].disabled = !enabled;
+
if (!enabled) {
cpu_ent* entry = &gCPU[cpuID];
@@ -856,32 +802,17 @@ init()
new(&gIdlePackageList) IdlePackageList;
- for (int32 i = 0; i < packageCount; i++)
- gPackageEntries[i].Init(i);
-
- for (int32 i = 0; i < coreCount; i++) {
- gCoreEntries[i].fCoreID = i;
- gCoreEntries[i].fCPUCount = cpuCount / coreCount;
-
- result = gCoreLoadHeap.Insert(&gCoreEntries[i], 0);
- if (result != B_OK)
- return result;
- }
-
for (int32 i = 0; i < cpuCount; i++) {
CoreEntry* core = &gCoreEntries[sCPUToCore[i]];
PackageEntry* package = &gPackageEntries[sCPUToPackage[i]];
+ package->Init(sCPUToPackage[i]);
+ core->Init(sCPUToCore[i], package);
+
gCPUEntries[i].fCPUNumber = i;
gCPUEntries[i].fCore = core;
- core->fPackage = package;
-
- if (core->fCPUHeap.PeekMaximum() == NULL)
- package->AddIdleCore(core);
- result = core->fCPUHeap.Insert(&gCPUEntries[i],
B_IDLE_PRIORITY);
- if (result != B_OK)
- return result;
+ core->AddCPU(&gCPUEntries[i]);
}
packageEntriesDeleter.Detach();
@@ -981,9 +912,9 @@ _user_estimate_max_scheduling_latency(thread_id id)
if (core == NULL)
core = &gCoreEntries[get_random<int32>() % gCoreCount];
- int32 threadCount = core->fThreadCount;
- if (core->fCPUCount > 0)
- threadCount /= core->fCPUCount;
+ int32 threadCount = core->ThreadCount();
+ if (core->CPUCount() > 0)
+ threadCount /= core->CPUCount();
if (threadData->GetEffectivePriority() > 0) {
threadCount -= threadCount * THREAD_MAX_SET_PRIORITY
diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp
b/src/system/kernel/scheduler/scheduler_cpu.cpp
index f4d55eb..896447e 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.cpp
+++ b/src/system/kernel/scheduler/scheduler_cpu.cpp
@@ -18,6 +18,7 @@ using namespace Scheduler;
class Scheduler::DebugDumper {
public:
+ static void DumpCoreRunQueue(CoreEntry* core);
static void DumpIdleCoresInPackage(PackageEntry* package);
};
@@ -63,17 +64,18 @@ CPUEntry::UpdatePriority(int32 priority)
if (gCPU[fCPUNumber].disabled)
return;
- int32 corePriority =
CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum());
- fCore->fCPUHeap.ModifyKey(this, priority);
+ CPUPriorityHeap* cpuHeap = fCore->CPUHeap();
+ int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum());
+ cpuHeap->ModifyKey(this, priority);
if (gSingleCore)
return;
- int32 maxPriority =
CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum());
+ int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum());
if (corePriority == maxPriority)
return;
- PackageEntry* packageEntry = fCore->fPackage;
+ PackageEntry* packageEntry = fCore->Package();
if (maxPriority == B_IDLE_PRIORITY)
packageEntry->CoreGoesIdle(fCore);
else if (corePriority == B_IDLE_PRIORITY)
@@ -93,9 +95,7 @@ CPUEntry::ComputeLoad()
if (oldLoad != fLoad) {
int32 delta = fLoad - oldLoad;
- atomic_add(&fCore->fLoad, delta);
-
- fCore->UpdateLoad();
+ fCore->UpdateLoad(delta);
}
if (fLoad > kVeryHighLoad)
@@ -106,9 +106,9 @@ CPUEntry::ComputeLoad()
ThreadData*
CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack)
{
- SpinLocker runQueueLocker(fCore->fQueueLock);
+ CoreRunQueueLocker _(fCore);
- ThreadData* sharedThread = fCore->fRunQueue.PeekMaximum();
+ ThreadData* sharedThread = fCore->PeekThread();
ThreadData* pinnedThread = fRunQueue.PeekMaximum();
ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread !=
NULL);
@@ -132,16 +132,7 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool
putAtBack)
if (sharedPriority > pinnedPriority) {
sharedThread->fEnqueued = false;
- fCore->fRunQueue.Remove(sharedThread);
- if (thread_is_idle_thread(sharedThread->GetThread())
- || fCore->fThreadList.Head() == sharedThread) {
- atomic_add(&fCore->fStarvationCounter, 1);
- }
-
- if (sharedThread->fWentSleepCount == 0)
- fCore->fThreadList.Remove(sharedThread);
-
- atomic_add(&fCore->fThreadCount, -1);
+ fCore->Remove(sharedThread, sharedThread->fWentSleepCount == 0);
return sharedThread;
}
@@ -259,15 +250,62 @@ CoreEntry::CoreEntry()
void
-CoreEntry::UpdateLoad()
+CoreEntry::Init(int32 id, PackageEntry* package)
{
- ASSERT(!gSingleCore);
+ fCoreID = id;
+ fPackage = package;
+}
+
+void
+CoreEntry::PushFront(ThreadData* thread, int32 priority)
+{
+ fRunQueue.PushFront(thread, priority);
+ atomic_add(&fThreadCount, 1);
+}
+
+
+void
+CoreEntry::PushBack(ThreadData* thread, int32 priority)
+{
+ fRunQueue.PushBack(thread, priority);
+ fThreadList.Insert(thread);
+
+ atomic_add(&fThreadCount, 1);
+}
+
+
+void
+CoreEntry::Remove(ThreadData* thread, bool starving)
+{
+ if (thread_is_idle_thread(thread->GetThread())
+ || fThreadList.Head() == thread) {
+ atomic_add(&fStarvationCounter, 1);
+ }
+ if (starving)
+ fThreadList.Remove(thread);
+ fRunQueue.Remove(thread);
+ atomic_add(&fThreadCount, -1);
+}
+
+
+inline ThreadData*
+CoreEntry::PeekThread() const
+{
+ return fRunQueue.PeekMaximum();
+}
+
+
+void
+CoreEntry::UpdateLoad(int32 delta)
+{
if (fCPUCount == 0) {
fLoad = 0;
return;
}
+ atomic_add(&fLoad, delta);
+
WriteSpinLocker coreLocker(gCoreHeapsLock);
int32 newKey = GetLoad();
@@ -310,6 +348,81 @@ CoreEntry::UpdateLoad()
}
+void
+CoreEntry::AddCPU(CPUEntry* cpu)
+{
+ ASSERT(fCPUCount >= 0);
+ if (fCPUCount++ == 0) {
+ // core has been reenabled
+ fLoad = 0;
+ fHighLoad = false;
+ gCoreLoadHeap.Insert(this, 0);
+
+ fPackage->AddIdleCore(this);
+ }
+
+ fCPUHeap.Insert(cpu, B_IDLE_PRIORITY);
+}
+
+
+void
+CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
+{
+ ASSERT(fCPUCount > 0);
+ if (--fCPUCount == 0) {
+ // core has been disabled
+ if (fHighLoad) {
+ gCoreHighLoadHeap.ModifyKey(this, -1);
+ ASSERT(gCoreHighLoadHeap.PeekMinimum() == this);
+ gCoreHighLoadHeap.RemoveMinimum();
+ } else {
+ gCoreLoadHeap.ModifyKey(this, -1);
+ ASSERT(gCoreLoadHeap.PeekMinimum() == this);
+ gCoreLoadHeap.RemoveMinimum();
+ }
+
+ fPackage->RemoveIdleCore(this);
+
+ // get rid of threads
+ thread_map(CoreEntry::_UnassignThread, this);
+
+ fThreadCount = 0;
+ while (fRunQueue.PeekMaximum() != NULL) {
+ ThreadData* threadData = fRunQueue.PeekMaximum();
+
+ fRunQueue.Remove(threadData);
+ threadData->fEnqueued = false;
+
+ if (threadData->fWentSleepCount == 0)
+ fThreadList.Remove(threadData);
+ threadData->fWentSleepCount = -1;
+
+ ASSERT(threadData->Core() == NULL);
+ threadPostProcessing(threadData);
+ }
+ }
+
+ fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1);
+ ASSERT(fCPUHeap.PeekMaximum() == cpu);
+ fCPUHeap.RemoveMaximum();
+
+ ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad);
+ fLoad -= cpu->fLoad;
+ ASSERT(fLoad >= 0);
+}
+
+
+/* static */ void
+CoreEntry::_UnassignThread(Thread* thread, void* data)
+{
+ CoreEntry* core = static_cast<CoreEntry*>(data);
+ ThreadData* threadData = thread->scheduler_data;
+
+ if (threadData->Core() == core && thread->pinned_to_cpu == 0)
+ threadData->UnassignCore();
+}
+
+
CoreLoadHeap::CoreLoadHeap(int32 coreCount)
:
MinMaxHeap<CoreEntry, int32>(coreCount)
@@ -323,7 +436,7 @@ CoreLoadHeap::Dump()
CoreEntry* entry = PeekMinimum();
while (entry) {
int32 key = GetKey(entry);
- kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID,
+ kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(),
entry->GetLoad() / 10);
RemoveMinimum();
@@ -421,6 +534,13 @@ PackageEntry::RemoveIdleCore(CoreEntry* core)
/* static */ void
+DebugDumper::DumpCoreRunQueue(CoreEntry* core)
+{
+ core->fRunQueue.Dump();
+}
+
+
+/* static */ void
DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
{
kprintf("%-7" B_PRId32 " ", package->fPackageID);
@@ -430,7 +550,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
if (iterator.HasNext()) {
while (iterator.HasNext()) {
CoreEntry* coreEntry = iterator.Next();
- kprintf("%" B_PRId32 "%s", coreEntry->fCoreID,
+ kprintf("%" B_PRId32 "%s", coreEntry->ID(),
iterator.HasNext() ? ", " : "");
}
} else
@@ -445,10 +565,9 @@ dump_run_queue(int argc, char **argv)
int32 cpuCount = smp_get_num_cpus();
int32 coreCount = gCoreCount;
-
for (int32 i = 0; i < coreCount; i++) {
kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "",
i);
- gCoreEntries[i].fRunQueue.Dump();
+ DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]);
}
for (int32 i = 0; i < cpuCount; i++) {
@@ -476,11 +595,11 @@ dump_cpu_heap(int argc, char** argv)
gCoreHighLoadHeap.Dump();
for (int32 i = 0; i < gCoreCount; i++) {
- if (gCoreEntries[i].fCPUCount < 2)
+ if (gCoreEntries[i].CPUCount() < 2)
continue;
kprintf("\nCore %" B_PRId32 " heap:\n", i);
- gCoreEntries[i].fCPUHeap.Dump();
+ gCoreEntries[i].CPUHeap()->Dump();
}
return 0;
diff --git a/src/system/kernel/scheduler/scheduler_cpu.h
b/src/system/kernel/scheduler/scheduler_cpu.h
index be37213..6df157e 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.h
+++ b/src/system/kernel/scheduler/scheduler_cpu.h
@@ -9,6 +9,7 @@
#include <OS.h>
#include <thread.h>
+#include <util/AutoLock.h>
#include <util/MinMaxHeap.h>
#include <cpufreq.h>
@@ -24,6 +25,7 @@ namespace Scheduler {
class DebugDumper;
struct ThreadData;
+class ThreadProcessing;
struct CPUEntry;
struct CoreEntry;
@@ -77,15 +79,57 @@ public:
void Dump();
};
-struct CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
- DoublyLinkedListLinkImpl<CoreEntry> {
+class CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
+ public DoublyLinkedListLinkImpl<CoreEntry> {
+public:
CoreEntry();
+ void
Init(int32 id, PackageEntry* package);
+
+ inline int32 ID() const
{ return fCoreID; }
+ inline PackageEntry* Package() const {
return fPackage; }
+ inline int32 CPUCount() const
+
{ return fCPUCount; }
+
+ inline void LockCPUHeap();
+ inline void UnlockCPUHeap();
+
+ inline CPUPriorityHeap* CPUHeap();
+
+ inline int32 ThreadCount()
const
+
{ return fThreadCount; }
+
+ inline void LockRunQueue();
+ inline void
UnlockRunQueue();
+
+ void
PushFront(ThreadData* thread,
+
int32 priority);
+ void
PushBack(ThreadData* thread,
+
int32 priority);
+ void
Remove(ThreadData* thread,
+
bool starving);
+ inline ThreadData* PeekThread()
const;
+
+ inline bigtime_t GetActiveTime()
const;
+ inline void
IncreaseActiveTime(
+
bigtime_t activeTime);
+
inline int32 GetLoad() const;
- void
UpdateLoad();
+ void
UpdateLoad(int32 delta);
+
+ inline int32
StarvationCounter() const;
+
+ void
AddCPU(CPUEntry* cpu);
+ void
RemoveCPU(CPUEntry* cpu,
+
ThreadProcessing&
+
threadPostProcessing);
static inline CoreEntry* GetCore(int32 cpu);
+private:
+ static void
_UnassignThread(Thread* thread,
+
void* core);
+
int32 fCoreID;
PackageEntry* fPackage;
@@ -101,12 +145,46 @@ struct CoreEntry : public MinMaxHeapLinkImpl<CoreEntry,
int32>,
spinlock
fQueueLock;
bigtime_t
fActiveTime;
- seqlock
fActiveTimeLock;
+ mutable seqlock fActiveTimeLock;
int32 fLoad;
bool
fHighLoad;
+
+ friend class DebugDumper;
} CACHE_LINE_ALIGN;
+class CoreRunQueueLocking {
+public:
+ inline bool Lock(CoreEntry* core)
+ {
+ core->LockRunQueue();
+ return true;
+ }
+
+ inline void Unlock(CoreEntry* core)
+ {
+ core->UnlockRunQueue();
+ }
+};
+
+typedef AutoLocker<CoreEntry, CoreRunQueueLocking> CoreRunQueueLocker;
+
+class CoreCPUHeapLocking {
+public:
+ inline bool Lock(CoreEntry* core)
+ {
+ core->LockCPUHeap();
+ return true;
+ }
+
+ inline void Unlock(CoreEntry* core)
+ {
+ core->UnlockCPUHeap();
+ }
+};
+
+typedef AutoLocker<CoreEntry, CoreCPUHeapLocking> CoreCPUHeapLocker;
+
class CoreLoadHeap : public MinMaxHeap<CoreEntry, int32> {
public:
CoreLoadHeap() { }
@@ -167,6 +245,64 @@ extern rw_spinlock gIdlePackageLock;
extern int32 gPackageCount;
+inline void
+CoreEntry::LockCPUHeap()
+{
+ acquire_spinlock(&fCPULock);
+}
+
+
+inline void
+CoreEntry::UnlockCPUHeap()
+{
+ release_spinlock(&fCPULock);
+}
+
+
+inline CPUPriorityHeap*
+CoreEntry::CPUHeap()
+{
+ return &fCPUHeap;
+}
+
+
+inline void
+CoreEntry::LockRunQueue()
+{
+ acquire_spinlock(&fQueueLock);
+}
+
+
+inline void
+CoreEntry::UnlockRunQueue()
+{
+ release_spinlock(&fQueueLock);
+}
+
+
+inline void
+CoreEntry::IncreaseActiveTime(bigtime_t activeTime)
+{
+ WriteSequentialLocker _(fActiveTimeLock);
+ fActiveTime += activeTime;
+}
+
+
+inline bigtime_t
+CoreEntry::GetActiveTime() const
+{
+ bigtime_t activeTime;
+
+ uint32 count;
+ do {
+ count = acquire_read_seqlock(&fActiveTimeLock);
+ activeTime = fActiveTime;
+ } while (!release_read_seqlock(&fActiveTimeLock, count));
+
+ return activeTime;
+}
+
+
inline int32
CoreEntry::GetLoad() const
{
@@ -175,6 +311,13 @@ CoreEntry::GetLoad() const
}
+inline int32
+CoreEntry::StarvationCounter() const
+{
+ return fStarvationCounter;
+}
+
+
/* static */ inline CoreEntry*
CoreEntry::GetCore(int32 cpu)
{
diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp
b/src/system/kernel/scheduler/scheduler_thread.cpp
index e1f280e..5b933a5 100644
--- a/src/system/kernel/scheduler/scheduler_thread.cpp
+++ b/src/system/kernel/scheduler/scheduler_thread.cpp
@@ -65,7 +65,7 @@ ThreadData::Dump() const
kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive);
kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount);
kprintf("\tcore:\t\t\t%" B_PRId32 "\n",
- fCore != NULL ? fCore->fCoreID : -1);
+ fCore != NULL ? fCore->ID() : -1);
if (fCore != NULL && HasCacheExpired())
kprintf("\tcache affinity has expired\n");
}
@@ -108,7 +108,7 @@ ThreadData::ComputeQuantum()
quantum += fStolenTime;
fStolenTime = 0;
- int32 threadCount = (fCore->fThreadCount + 1) / fCore->fCPUCount;
+ int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount();
threadCount = max_c(threadCount, 1);
quantum = std::min(gCurrentMode->maximum_latency / threadCount,
quantum);
@@ -137,7 +137,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool&
rescheduleNeeded) const
if (fThread->previous_cpu != NULL) {
CPUEntry* previousCPU =
&gCPUEntries[fThread->previous_cpu->cpu_num];
if (previousCPU->fCore == core) {
- SpinLocker cpuLocker(core->fCPULock);
+ CoreCPUHeapLocker _(core);
if (CPUPriorityHeap::GetKey(previousCPU) <
threadPriority) {
previousCPU->UpdatePriority(threadPriority);
rescheduleNeeded = true;
@@ -146,8 +146,8 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool&
rescheduleNeeded) const
}
}
- SpinLocker cpuLocker(core->fCPULock);
- CPUEntry* cpu = core->fCPUHeap.PeekMinimum();
+ CoreCPUHeapLocker _(core);
+ CPUEntry* cpu = core->CPUHeap()->PeekMinimum();
ASSERT(cpu != NULL);
if (CPUPriorityHeap::GetKey(cpu) < threadPriority) {
@@ -195,3 +195,8 @@ ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t
minQuantum,
return maxQuantum - result;
}
+
+ThreadProcessing::~ThreadProcessing()
+{
+}
+
diff --git a/src/system/kernel/scheduler/scheduler_thread.h
b/src/system/kernel/scheduler/scheduler_thread.h
index 7ebf591..2a49b68 100644
--- a/src/system/kernel/scheduler/scheduler_thread.h
+++ b/src/system/kernel/scheduler/scheduler_thread.h
@@ -93,6 +93,13 @@ private:
CoreEntry* fCore;
};
+class ThreadProcessing {
+public:
+ virtual ~ThreadProcessing();
+
+ virtual void operator()(ThreadData* thread) = 0;
+};
+
inline bool
ThreadData::HasCacheExpired() const
@@ -166,7 +173,7 @@ ThreadData::ShouldCancelPenalty() const
if (fCore == NULL)
return false;
- return atomic_get(&fCore->fStarvationCounter) != fWentSleepCount
+ return fCore->StarvationCounter() != fWentSleepCount
&& system_time() - fWentSleep > gCurrentMode->base_quantum;
}
@@ -177,13 +184,8 @@ ThreadData::GoesAway()
fLastInterruptTime = 0;
fWentSleep = system_time();
- fWentSleepCount = atomic_get(&fCore->fStarvationCounter);
-
- uint32 count;
- do {
- count = acquire_read_seqlock(&fCore->fActiveTimeLock);
- fWentSleepActive = fCore->fActiveTime;
- } while (!release_read_seqlock(&fCore->fActiveTimeLock, count));
+ fWentSleepCount = fCore->StarvationCounter();
+ fWentSleepActive = fCore->GetActiveTime();
}
@@ -195,7 +197,7 @@ ThreadData::PutBack()
int32 priority = GetEffectivePriority();
- SpinLocker runQueueLocker(fCore->fQueueLock);
+ CoreRunQueueLocker _(fCore);
ASSERT(!fEnqueued);
fEnqueued = true;
if (fThread->pinned_to_cpu > 0) {
@@ -203,10 +205,9 @@ ThreadData::PutBack()
CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num];
cpu->fRunQueue.PushFront(this, priority);
- } else {
- fCore->fRunQueue.PushFront(this, priority);
- atomic_add(&fCore->fThreadCount, 1);
- }
+ } else
+ fCore->PushFront(this, priority);
+ fCore->UnlockRunQueue();
}
@@ -219,7 +220,7 @@ ThreadData::Enqueue()
int32 priority = GetEffectivePriority();
- SpinLocker runQueueLocker(fCore->fQueueLock);
+ CoreRunQueueLocker _(fCore);
ASSERT(!fEnqueued);
fEnqueued = true;
if (fThread->pinned_to_cpu > 0) {
@@ -227,19 +228,15 @@ ThreadData::Enqueue()
CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
cpu->fRunQueue.PushBack(this, priority);
- } else {
- fCore->fRunQueue.PushBack(this, priority);
- fCore->fThreadList.Insert(this);
-
- atomic_add(&fCore->fThreadCount, 1);
- }
+ } else
+ fCore->PushBack(this, priority);
}
inline bool
ThreadData::Dequeue()
{
- SpinLocker runQueueLocker(fCore->fQueueLock);
+ CoreRunQueueLocker _(fCore);
if (!fEnqueued)
return false;
@@ -250,12 +247,8 @@ ThreadData::Dequeue()
CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
cpu->fRunQueue.Remove(this);
} else {
- fCore->fRunQueue.Remove(this);
-
ASSERT(fWentSleepCount < 1);
- if (fWentSleepCount == 0)
- fCore->fThreadList.Remove(this);
- atomic_add(&fCore->fThreadCount, -1);
+ fCore->Remove(this, fWentSleepCount == 0);
}
return true;
@@ -268,9 +261,8 @@ ThreadData::UpdateActivity(bigtime_t active)
fMeasureActiveTime += active;
gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active;
- WriteSequentialLocker locker(fCore->fActiveTimeLock);
- fCore->fActiveTime += active;
- locker.Unlock();
+ fCore->IncreaseActiveTime(active);
+
}
############################################################################
Commit: adb2f9ff288f28c8dde36bd88b1d7d3ce39344f3
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Mon Dec 23 19:12:03 2013 UTC
scheduler: Encapsulate CPUEntry fields
----------------------------------------------------------------------------
diff --git a/src/system/kernel/scheduler/low_latency.cpp
b/src/system/kernel/scheduler/low_latency.cpp
index 74cddd6..dd6399c 100644
--- a/src/system/kernel/scheduler/low_latency.cpp
+++ b/src/system/kernel/scheduler/low_latency.cpp
@@ -132,7 +132,7 @@ rebalance_irqs(bool idle)
other = gCoreHighLoadHeap.PeekMinimum();
coreLocker.Unlock();
- int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
+ int32 newCPU = other->CPUHeap()->PeekMinimum()->ID();
ASSERT(other != NULL);
diff --git a/src/system/kernel/scheduler/power_saving.cpp
b/src/system/kernel/scheduler/power_saving.cpp
index d6f3a7d..33bd769 100644
--- a/src/system/kernel/scheduler/power_saving.cpp
+++ b/src/system/kernel/scheduler/power_saving.cpp
@@ -164,7 +164,7 @@ pack_irqs()
irq_assignment* irq =
(irq_assignment*)list_get_first_item(&cpu->irqs);
locker.Unlock();
- int32 newCPU =
smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber;
+ int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID();
if (newCPU != cpu->cpu_num)
assign_io_interrupt_to_cpu(irq->irq, newCPU);
@@ -207,7 +207,7 @@ rebalance_irqs(bool idle)
coreLocker.Unlock();
if (other == NULL)
return;
- int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
+ int32 newCPU = other->CPUHeap()->PeekMinimum()->ID();
CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu());
if (other == core)
diff --git a/src/system/kernel/scheduler/scheduler.cpp
b/src/system/kernel/scheduler/scheduler.cpp
index 97e2e52..a4797ef 100644
--- a/src/system/kernel/scheduler/scheduler.cpp
+++ b/src/system/kernel/scheduler/scheduler.cpp
@@ -39,26 +39,60 @@
namespace Scheduler {
-class SchedulerModeLocker : public ReadSpinLocker {
+class SchedulerModeLocking {
+public:
+ bool Lock(int* /* lockable */)
+ {
+ CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler();
+ return true;
+ }
+
+ void Unlock(int* /* lockable */)
+ {
+ CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler();
+ }
+};
+
+class SchedulerModeLocker :
+ public AutoLocker<int, SchedulerModeLocking> {
public:
SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked =
true)
:
-
ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock,
- alreadyLocked, lockIfNotLocked)
+ AutoLocker<int, SchedulerModeLocking>(NULL, alreadyLocked,
+ lockIfNotLocked)
+ {
+ }
+};
+
+class InterruptsSchedulerModeLocking {
+public:
+ bool Lock(int* lockable)
+ {
+ *lockable = disable_interrupts();
+ CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler();
+ return true;
+ }
+
+ void Unlock(int* lockable)
{
+ CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler();
+ restore_interrupts(*lockable);
}
};
-class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker {
+class InterruptsSchedulerModeLocker :
+ public AutoLocker<int, InterruptsSchedulerModeLocking> {
public:
InterruptsSchedulerModeLocker(bool alreadyLocked = false,
bool lockIfNotLocked = true)
:
- InterruptsReadSpinLocker(
- gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock,
- alreadyLocked, lockIfNotLocked)
+ AutoLocker<int, InterruptsSchedulerModeLocking>(&fState,
alreadyLocked,
+ lockIfNotLocked)
{
}
+
+private:
+ int fState;
};
class InterruptsBigSchedulerLocking {
@@ -67,14 +101,14 @@ public:
{
*lockable = disable_interrupts();
for (int32 i = 0; i < smp_get_num_cpus(); i++)
-
acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock);
+ CPUEntry::GetCPU(i)->LockScheduler();
return true;
}
void Unlock(int* lockable)
{
for (int32 i = 0; i < smp_get_num_cpus(); i++)
-
release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock);
+ CPUEntry::GetCPU(i)->UnlockScheduler();
restore_interrupts(*lockable);
}
};
@@ -193,10 +227,10 @@ enqueue(Thread* thread, bool newOne)
if (threadPriority > heapPriority
|| (threadPriority == heapPriority && rescheduleNeeded)) {
- if (targetCPU->fCPUNumber == smp_get_current_cpu())
- gCPU[targetCPU->fCPUNumber].invoke_scheduler = true;
+ if (targetCPU->ID() == smp_get_current_cpu())
+ gCPU[targetCPU->ID()].invoke_scheduler = true;
else {
- smp_send_ici(targetCPU->fCPUNumber, SMP_MSG_RESCHEDULE,
0, 0, 0,
+ smp_send_ici(targetCPU->ID(), SMP_MSG_RESCHEDULE, 0, 0,
0,
NULL, SMP_MSG_FLAG_ASYNC);
}
}
@@ -484,8 +518,8 @@ reschedule(int32 nextState)
if (!thread_is_idle_thread(oldThread)) {
CoreRunQueueLocker _(core);
- nextThreadData =
cpu->fRunQueue.GetHead(B_IDLE_PRIORITY);
- cpu->fRunQueue.Remove(nextThreadData);
+ nextThreadData = cpu->PeekIdleThread();
+ cpu->Remove(nextThreadData);
nextThreadData->fEnqueued = false;
putOldThreadAtBack = oldThread->pinned_to_cpu == 0;
@@ -662,14 +696,13 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
gCurrentMode->set_cpu_enabled(cpuID, enabled);
CPUEntry* cpu = &gCPUEntries[cpuID];
- CoreEntry* core = cpu->fCore;
+ CoreEntry* core = cpu->Core();
int32 oldCPUCount = core->CPUCount();
ASSERT(oldCPUCount >= 0);
- if (enabled) {
- cpu->fLoad = 0;
- core->AddCPU(cpu);
- } else {
+ if (enabled)
+ cpu->Start();
+ else {
cpu->UpdatePriority(B_IDLE_PRIORITY);
ThreadEnqueuer enqueuer;
@@ -679,21 +712,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
gCPU[cpuID].disabled = !enabled;
if (!enabled) {
- cpu_ent* entry = &gCPU[cpuID];
-
- // get rid of irqs
- SpinLocker locker(entry->irqs_lock);
- irq_assignment* irq
- = (irq_assignment*)list_get_first_item(&entry->irqs);
- while (irq != NULL) {
- locker.Unlock();
-
- assign_io_interrupt_to_cpu(irq->irq, -1);
-
- locker.Lock();
- irq =
(irq_assignment*)list_get_first_item(&entry->irqs);
- }
- locker.Unlock();
+ cpu->Stop();
// don't wait until the thread quantum ends
if (smp_get_current_cpu() != cpuID) {
@@ -808,9 +827,7 @@ init()
package->Init(sCPUToPackage[i]);
core->Init(sCPUToCore[i], package);
-
- gCPUEntries[i].fCPUNumber = i;
- gCPUEntries[i].fCore = core;
+ gCPUEntries[i].Init(i, core);
core->AddCPU(&gCPUEntries[i]);
}
diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp
b/src/system/kernel/scheduler/scheduler_cpu.cpp
index 896447e..0899a9b 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.cpp
+++ b/src/system/kernel/scheduler/scheduler_cpu.cpp
@@ -18,6 +18,7 @@ using namespace Scheduler;
class Scheduler::DebugDumper {
public:
+ static void DumpCPURunQueue(CPUEntry* cpu);
static void DumpCoreRunQueue(CoreEntry* core);
static void DumpIdleCoresInPackage(PackageEntry* package);
@@ -59,6 +60,78 @@ CPUEntry::CPUEntry()
void
+CPUEntry::Init(int32 id, CoreEntry* core)
+{
+ fCPUNumber = id;
+ fCore = core;
+}
+
+
+void
+CPUEntry::Start()
+{
+ fLoad = 0;
+ fCore->AddCPU(this);
+}
+
+
+void
+CPUEntry::Stop()
+{
+ cpu_ent* entry = &gCPU[fCPUNumber];
+
+ // get rid of irqs
+ SpinLocker locker(entry->irqs_lock);
+ irq_assignment* irq
+ = (irq_assignment*)list_get_first_item(&entry->irqs);
+ while (irq != NULL) {
+ locker.Unlock();
+
+ assign_io_interrupt_to_cpu(irq->irq, -1);
+
+ locker.Lock();
+ irq = (irq_assignment*)list_get_first_item(&entry->irqs);
+ }
+ locker.Unlock();
+}
+
+
+void
+CPUEntry::PushFront(ThreadData* thread, int32 priority)
+{
+ fRunQueue.PushFront(thread, priority);
+}
+
+
+void
+CPUEntry::PushBack(ThreadData* thread, int32 priority)
+{
+ fRunQueue.PushBack(thread, priority);
+}
+
+
+void
+CPUEntry::Remove(ThreadData* thread)
+{
+ fRunQueue.Remove(thread);
+}
+
+
+inline ThreadData*
+CPUEntry::PeekThread() const
+{
+ return fRunQueue.PeekMaximum();
+}
+
+
+ThreadData*
+CPUEntry::PeekIdleThread() const
+{
+ return fRunQueue.GetHead(B_IDLE_PRIORITY);
+}
+
+
+void
CPUEntry::UpdatePriority(int32 priority)
{
if (gCPU[fCPUNumber].disabled)
@@ -213,10 +286,10 @@ CPUPriorityHeap::Dump()
kprintf("cpu priority load\n");
CPUEntry* entry = PeekMinimum();
while (entry) {
- int32 cpu = entry->fCPUNumber;
+ int32 cpu = entry->ID();
int32 key = GetKey(entry);
kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n",
cpu, key,
- entry->fLoad / 10);
+ entry->GetLoad() / 10);
RemoveMinimum();
sDebugCPUHeap.Insert(entry, key);
@@ -406,8 +479,8 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing&
threadPostProcessing)
ASSERT(fCPUHeap.PeekMaximum() == cpu);
fCPUHeap.RemoveMaximum();
- ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad);
- fLoad -= cpu->fLoad;
+ ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad);
+ fLoad -= cpu->GetLoad();
ASSERT(fLoad >= 0);
}
@@ -534,6 +607,19 @@ PackageEntry::RemoveIdleCore(CoreEntry* core)
/* static */ void
+DebugDumper::DumpCPURunQueue(CPUEntry* cpu)
+{
+ ThreadRunQueue::ConstIterator iterator =
cpu->fRunQueue.GetConstIterator();
+
+ if (iterator.HasNext()
+ && !thread_is_idle_thread(iterator.Next()->GetThread())) {
+ kprintf("\nCPU %" B_PRId32 " run queue:\n", cpu->ID());
+ cpu->fRunQueue.Dump();
+ }
+}
+
+
+/* static */ void
DebugDumper::DumpCoreRunQueue(CoreEntry* core)
{
core->fRunQueue.Dump();
@@ -570,17 +656,8 @@ dump_run_queue(int argc, char **argv)
DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]);
}
- for (int32 i = 0; i < cpuCount; i++) {
- CPUEntry* cpu = &gCPUEntries[i];
- ThreadRunQueue::ConstIterator iterator
- = cpu->fRunQueue.GetConstIterator();
-
- if (iterator.HasNext()
- &&
!thread_is_idle_thread(iterator.Next()->GetThread())) {
- kprintf("\nCPU %" B_PRId32 " run queue:\n", i);
- cpu->fRunQueue.Dump();
- }
- }
+ for (int32 i = 0; i < cpuCount; i++)
+ DebugDumper::DumpCPURunQueue(&gCPUEntries[i]);
return 0;
}
diff --git a/src/system/kernel/scheduler/scheduler_cpu.h
b/src/system/kernel/scheduler/scheduler_cpu.h
index 6df157e..9480619 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.h
+++ b/src/system/kernel/scheduler/scheduler_cpu.h
@@ -40,11 +40,38 @@ public:
void Dump()
const;
};
-struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
+class CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
+public:
CPUEntry();
+ void
Init(int32 id, CoreEntry* core);
+
+ inline int32 ID() const
{ return fCPUNumber; }
+ inline CoreEntry* Core() const
{ return fCore; }
+
+ void Start();
+ void Stop();
+
+ inline void
EnterScheduler();
+ inline void ExitScheduler();
+
+ inline void LockScheduler();
+ inline void
UnlockScheduler();
+
+ void
PushFront(ThreadData* thread,
+
int32 priority);
+ void
PushBack(ThreadData* thread,
+
int32 priority);
+ void
Remove(ThreadData* thread);
+ inline ThreadData* PeekThread()
const;
+ ThreadData*
PeekIdleThread() const;
+
void
UpdatePriority(int32 priority);
+ inline void
IncreaseActiveTime(
+
bigtime_t activeTime);
+
+ inline int32 GetLoad() const
{ return fLoad; }
void
ComputeLoad();
ThreadData*
ChooseNextThread(ThreadData* oldThread,
@@ -53,6 +80,12 @@ struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32>
{
void
TrackActivity(ThreadData* oldThreadData,
ThreadData* nextThreadData);
+ static inline CPUEntry* GetCPU(int32 cpu);
+
+private:
+ inline void
_RequestPerformanceLevel(
+
ThreadData* threadData);
+
int32
fCPUNumber;
CoreEntry* fCore;
@@ -65,10 +98,7 @@ struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32>
{
bigtime_t
fMeasureActiveTime;
bigtime_t
fMeasureTime;
-private:
- inline void
_RequestPerformanceLevel(
-
ThreadData* threadData);
-
+ friend class DebugDumper;
} CACHE_LINE_ALIGN;
class CPUPriorityHeap : public MinMaxHeap<CPUEntry, int32> {
@@ -246,6 +276,48 @@ extern int32 gPackageCount;
inline void
+CPUEntry::EnterScheduler()
+{
+ acquire_read_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::ExitScheduler()
+{
+ release_read_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::LockScheduler()
+{
+ acquire_write_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::UnlockScheduler()
+{
+ release_write_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::IncreaseActiveTime(bigtime_t activeTime)
+{
+ fMeasureActiveTime += activeTime;
+}
+
+
+/* static */ inline CPUEntry*
+CPUEntry::GetCPU(int32 cpu)
+{
+ return &gCPUEntries[cpu];
+}
+
+
+inline void
CoreEntry::LockCPUHeap()
{
acquire_spinlock(&fCPULock);
@@ -321,7 +393,7 @@ CoreEntry::StarvationCounter() const
/* static */ inline CoreEntry*
CoreEntry::GetCore(int32 cpu)
{
- return gCPUEntries[cpu].fCore;
+ return gCPUEntries[cpu].Core();
}
diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp
b/src/system/kernel/scheduler/scheduler_thread.cpp
index 5b933a5..3bcb4d7 100644
--- a/src/system/kernel/scheduler/scheduler_thread.cpp
+++ b/src/system/kernel/scheduler/scheduler_thread.cpp
@@ -77,7 +77,7 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore,
CPUEntry*& targetCPU)
bool rescheduleNeeded = false;
if (targetCore == NULL && targetCPU != NULL)
- targetCore = targetCPU->fCore;
+ targetCore = targetCPU->Core();
else if (targetCore != NULL && targetCPU == NULL)
targetCPU = _ChooseCPU(targetCore, rescheduleNeeded);
else if (targetCore == NULL && targetCPU == NULL) {
@@ -136,7 +136,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool&
rescheduleNeeded) const
if (fThread->previous_cpu != NULL) {
CPUEntry* previousCPU =
&gCPUEntries[fThread->previous_cpu->cpu_num];
- if (previousCPU->fCore == core) {
+ if (previousCPU->Core() == core) {
CoreCPUHeapLocker _(core);
if (CPUPriorityHeap::GetKey(previousCPU) <
threadPriority) {
previousCPU->UpdatePriority(threadPriority);
diff --git a/src/system/kernel/scheduler/scheduler_thread.h
b/src/system/kernel/scheduler/scheduler_thread.h
index 2a49b68..f0d0690 100644
--- a/src/system/kernel/scheduler/scheduler_thread.h
+++ b/src/system/kernel/scheduler/scheduler_thread.h
@@ -203,8 +203,8 @@ ThreadData::PutBack()
if (fThread->pinned_to_cpu > 0) {
ASSERT(fThread->cpu != NULL);
- CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num];
- cpu->fRunQueue.PushFront(this, priority);
+ CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num);
+ cpu->PushFront(this, priority);
} else
fCore->PushFront(this, priority);
fCore->UnlockRunQueue();
@@ -226,8 +226,8 @@ ThreadData::Enqueue()
if (fThread->pinned_to_cpu > 0) {
ASSERT(fThread->previous_cpu != NULL);
- CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
- cpu->fRunQueue.PushBack(this, priority);
+ CPUEntry* cpu =
CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
+ cpu->PushBack(this, priority);
} else
fCore->PushBack(this, priority);
}
@@ -244,8 +244,8 @@ ThreadData::Dequeue()
if (fThread->pinned_to_cpu > 0) {
ASSERT(fThread->previous_cpu != NULL);
- CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
- cpu->fRunQueue.Remove(this);
+ CPUEntry* cpu =
CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
+ cpu->Remove(this);
} else {
ASSERT(fWentSleepCount < 1);
fCore->Remove(this, fWentSleepCount == 0);
@@ -259,10 +259,8 @@ inline void
ThreadData::UpdateActivity(bigtime_t active)
{
fMeasureActiveTime += active;
- gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active;
-
+ CPUEntry::GetCPU(smp_get_current_cpu())->IncreaseActiveTime(active);
fCore->IncreaseActiveTime(active);
-
}
############################################################################
Commit: 56878118841a0a07aaaa550f813461666bb81cdd
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Mon Dec 23 20:32:21 2013 UTC
scheduler: Encapsulate ThreadData fields
----------------------------------------------------------------------------
diff --git a/src/system/kernel/scheduler/low_latency.cpp
b/src/system/kernel/scheduler/low_latency.cpp
index dd6399c..d4090b7 100644
--- a/src/system/kernel/scheduler/low_latency.cpp
+++ b/src/system/kernel/scheduler/low_latency.cpp
@@ -38,7 +38,7 @@ has_cache_expired(const ThreadData* threadData)
CoreEntry* core = threadData->Core();
bigtime_t activeTime = core->GetActiveTime();
- return activeTime - threadData->fWentSleepActive > kCacheExpire;
+ return activeTime - threadData->WentSleepActive() > kCacheExpire;
}
diff --git a/src/system/kernel/scheduler/power_saving.cpp
b/src/system/kernel/scheduler/power_saving.cpp
index 33bd769..8248c6b 100644
--- a/src/system/kernel/scheduler/power_saving.cpp
+++ b/src/system/kernel/scheduler/power_saving.cpp
@@ -41,7 +41,7 @@ has_cache_expired(const ThreadData* threadData)
{
ASSERT(!gSingleCore);
- return system_time() - threadData->fWentSleep > kCacheExpire;
+ return system_time() - threadData->WentSleep() > kCacheExpire;
}
diff --git a/src/system/kernel/scheduler/scheduler.cpp
b/src/system/kernel/scheduler/scheduler.cpp
index a4797ef..87bfc78 100644
--- a/src/system/kernel/scheduler/scheduler.cpp
+++ b/src/system/kernel/scheduler/scheduler.cpp
@@ -476,8 +476,8 @@ reschedule(int32 nextState)
ThreadData* oldThreadData = oldThread->scheduler_data;
// return time spent in interrupts
- oldThreadData->fStolenTime
- += gCPU[thisCPU].interrupt_time -
oldThreadData->fLastInterruptTime;
+ oldThreadData->IncreaseStolenTime(
+ gCPU[thisCPU].interrupt_time -
oldThreadData->LastInterruptTime());
bool enqueueOldThread = false;
bool putOldThreadAtBack = false;
@@ -520,7 +520,6 @@ reschedule(int32 nextState)
nextThreadData = cpu->PeekIdleThread();
cpu->Remove(nextThreadData);
- nextThreadData->fEnqueued = false;
putOldThreadAtBack = oldThread->pinned_to_cpu == 0;
} else
@@ -581,8 +580,7 @@ reschedule(int32 nextState)
add_timer(quantumTimer, &reschedule_event, quantum,
B_ONE_SHOT_RELATIVE_TIMER);
} else {
- nextThreadData->fQuantumStart = system_time();
-
+ nextThreadData->StartQuantum();
gCurrentMode->rebalance_irqs(true);
}
diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp
b/src/system/kernel/scheduler/scheduler_cpu.cpp
index 0899a9b..fe4800b 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.cpp
+++ b/src/system/kernel/scheduler/scheduler_cpu.cpp
@@ -113,6 +113,8 @@ CPUEntry::PushBack(ThreadData* thread, int32 priority)
void
CPUEntry::Remove(ThreadData* thread)
{
+ ASSERT(thread->IsEnqueued());
+ thread->SetDequeued();
fRunQueue.Remove(thread);
}
@@ -203,14 +205,11 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool
putAtBack)
return oldThread;
if (sharedPriority > pinnedPriority) {
- sharedThread->fEnqueued = false;
-
- fCore->Remove(sharedThread, sharedThread->fWentSleepCount == 0);
+ fCore->Remove(sharedThread);
return sharedThread;
}
- pinnedThread->fEnqueued = false;
- fRunQueue.Remove(pinnedThread);
+ Remove(pinnedThread);
return pinnedThread;
}
@@ -243,7 +242,7 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData,
ThreadData* nextThreadData)
cpuEntry->last_kernel_time = nextThread->kernel_time;
cpuEntry->last_user_time = nextThread->user_time;
- nextThreadData->fLastInterruptTime = cpuEntry->interrupt_time;
+ nextThreadData->SetLastInterruptTime(cpuEntry->interrupt_time);
_RequestPerformanceLevel(nextThreadData);
}
@@ -349,13 +348,15 @@ CoreEntry::PushBack(ThreadData* thread, int32 priority)
void
-CoreEntry::Remove(ThreadData* thread, bool starving)
+CoreEntry::Remove(ThreadData* thread)
{
+ ASSERT(thread->IsEnqueued());
+ thread->SetDequeued();
if (thread_is_idle_thread(thread->GetThread())
|| fThreadList.Head() == thread) {
atomic_add(&fStarvationCounter, 1);
}
- if (starving)
+ if (thread->WentSleepCount() == 0)
fThreadList.Remove(thread);
fRunQueue.Remove(thread);
atomic_add(&fThreadCount, -1);
@@ -459,20 +460,16 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing&
threadPostProcessing)
// get rid of threads
thread_map(CoreEntry::_UnassignThread, this);
- fThreadCount = 0;
while (fRunQueue.PeekMaximum() != NULL) {
ThreadData* threadData = fRunQueue.PeekMaximum();
- fRunQueue.Remove(threadData);
- threadData->fEnqueued = false;
-
- if (threadData->fWentSleepCount == 0)
- fThreadList.Remove(threadData);
- threadData->fWentSleepCount = -1;
+ Remove(threadData);
ASSERT(threadData->Core() == NULL);
threadPostProcessing(threadData);
}
+
+ fThreadCount = 0;
}
fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1);
diff --git a/src/system/kernel/scheduler/scheduler_cpu.h
b/src/system/kernel/scheduler/scheduler_cpu.h
index 9480619..f3a9d35 100644
--- a/src/system/kernel/scheduler/scheduler_cpu.h
+++ b/src/system/kernel/scheduler/scheduler_cpu.h
@@ -136,8 +136,7 @@ public:
int32 priority);
void
PushBack(ThreadData* thread,
int32 priority);
- void
Remove(ThreadData* thread,
-
bool starving);
+ void
Remove(ThreadData* thread);
inline ThreadData* PeekThread()
const;
inline bigtime_t GetActiveTime()
const;
diff --git a/src/system/kernel/scheduler/scheduler_thread.h
b/src/system/kernel/scheduler/scheduler_thread.h
index f0d0690..7a8b616 100644
--- a/src/system/kernel/scheduler/scheduler_thread.h
+++ b/src/system/kernel/scheduler/scheduler_thread.h
[ *** diff truncated: 111 lines dropped *** ]
Other related posts:
- » [haiku-commits] BRANCH pdziepak-github.scheduler [5687811] src/system/kernel/scheduler - pdziepak-github . scheduler
