added 5 changesets to branch 'refs/remotes/pdziepak-github/scheduler' old head: cec16c2dcfb0bddb0d9dc11fb63793c4ca9a53e0 new head: 0e94a12f8e0e5fe5ff5b2e3f83384f3586396c92 overview: https://github.com/pdziepak/Haiku/compare/cec16c2...0e94a12 ---------------------------------------------------------------------------- f95b6fd: scheudler: Do not include irq load in thread load 13a8983: scheduler: Fix power saving mode and other minor improvements 1e8ed55: cpufreq: Rank modules and choose the best one 2697078: scheduler: Clean scheduler_common.h scheduler_common.h is now meant for types, variables and functions used by both core scheduler code and implementations of scheduler modes. Functions like switch_thread() and update_thread_times() do not belong there anymore. 0e94a12: kernel: Make CACHE_LINE_ALIGN visible in the whole kernel [ Pawel Dziepak <pdziepak@xxxxxxxxxxx> ] ---------------------------------------------------------------------------- 12 files changed, 150 insertions(+), 113 deletions(-) headers/os/drivers/cpufreq.h | 2 + headers/private/kernel/arch/cpu.h | 2 + headers/private/kernel/arch/x86/arch_cpu.h | 2 + headers/private/kernel/cpu.h | 2 +- .../cpufreq/intel_pstates/intel_pstates.cpp | 2 + src/system/kernel/cpu.cpp | 13 +- src/system/kernel/int.cpp | 2 +- src/system/kernel/scheduler/low_latency.cpp | 7 ++ src/system/kernel/scheduler/power_saving.cpp | 23 ++-- src/system/kernel/scheduler/scheduler.cpp | 118 +++++++++++++++++-- src/system/kernel/scheduler/scheduler_common.h | 89 -------------- src/system/kernel/scheduler/scheduler_modes.h | 1 + ############################################################################ Commit: f95b6fdfc84784fd24255fff8a492b877a26cbfd Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx> Date: Sun Nov 24 22:18:58 2013 UTC scheudler: Do not include irq load in thread load ---------------------------------------------------------------------------- diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index bc4fd94..00b1d57 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -617,6 +617,12 @@ compute_cpu_load(int32 cpu) static inline void compute_thread_load(Thread* thread) { + if (thread->scheduler_data->last_interrupt_time > 0) { + bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; + interruptTime -= thread->scheduler_data->last_interrupt_time; + thread->scheduler_data->measure_active_time -= interruptTime; + } + compute_load(thread->scheduler_data->measure_time, thread->scheduler_data->measure_active_time, thread->scheduler_data->load); @@ -631,6 +637,8 @@ thread_goes_away(Thread* thread) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + schedulerThreadData->last_interrupt_time = 0; + schedulerThreadData->went_sleep = system_time(); schedulerThreadData->went_sleep_active = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); ############################################################################ Commit: 13a89839fc09fea7d328568ddb76d8369ce2839d Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx> Date: Sun Nov 24 22:20:42 2013 UTC scheduler: Fix power saving mode and other minor improvements ---------------------------------------------------------------------------- diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 5a0bebf..9fd6d2e 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -685,7 +685,7 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) if (newCPU == -1) newCPU = assign_cpu(); - dprintf_no_syslog("IRQ %ld CPU %" B_PRId32 " -> CPU %" B_PRId32 "\n", vector, oldCPU, newCPU); + if (newCPU == oldCPU) return; diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 5f1224c..832b7ab 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -19,6 +19,12 @@ switch_to_mode(void) } +static void +set_cpu_enabled(int32 /* cpu */, bool /* enabled */) +{ +} + + static bool has_cache_expired(Thread* thread) { @@ -180,6 +186,7 @@ scheduler_mode_operations gSchedulerLowLatencyMode = { true, switch_to_mode, + set_cpu_enabled, has_cache_expired, choose_core, should_rebalance, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 51485ee..63ad0e3 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -16,6 +16,21 @@ using namespace Scheduler; static int32 sSmallTaskCore; +static void +switch_to_mode(void) +{ + sSmallTaskCore = -1; +} + + +static void +set_cpu_enabled(int32 cpu, bool enabled) +{ + if (!enabled) + sSmallTaskCore = -1; +} + + static bool has_cache_expired(Thread* thread) { @@ -31,13 +46,6 @@ has_cache_expired(Thread* thread) } -static void -switch_to_mode(void) -{ - sSmallTaskCore = -1; -} - - static bool try_small_task_packing(Thread* thread) { @@ -244,6 +252,7 @@ scheduler_mode_operations gSchedulerPowerSavingMode = { false, switch_to_mode, + set_cpu_enabled, has_cache_expired, choose_core, should_rebalance, diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 00b1d57..e1d7c45 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -583,9 +583,6 @@ should_rebalance(Thread* thread) { ASSERT(!gSingleCore); - if (thread_is_idle_thread(thread)) - return false; - return sCurrentMode->should_rebalance(thread); } @@ -600,9 +597,6 @@ compute_cpu_load(int32 cpu) if (oldLoad < 0) return; - if (gCPUEntries[cpu].fLoad > kVeryHighLoad) - sCurrentMode->rebalance_irqs(false); - if (oldLoad != gCPUEntries[cpu].fLoad) { int32 core = gCPUToCore[cpu]; @@ -611,6 +605,9 @@ compute_cpu_load(int32 cpu) update_load_heaps(core); } + + if (gCPUEntries[cpu].fLoad > kVeryHighLoad) + sCurrentMode->rebalance_irqs(false); } @@ -1069,8 +1066,7 @@ static inline void update_cpu_performance(Thread* thread, int32 thisCore) { int32 load = max_c(thread->scheduler_data->load, - gCoreEntries[thisCore].fLoad); - load /= gCoreEntries[thisCore].fCPUCount; + get_core_load(&gCoreEntries[thisCore])); load = min_c(max_c(load, 0), kMaxLoad); if (load < kTargetLoad) { @@ -1369,6 +1365,8 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) gCPU[cpu].disabled = !enabled; + sCurrentMode->set_cpu_enabled(cpu, enabled); + CoreEntry* core = &gCoreEntries[gCPUToCore[cpu]]; PackageEntry* package = &gPackageEntries[gCPUToPackage[cpu]]; diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 3b13a9c..b413a6b 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -16,6 +16,7 @@ struct scheduler_mode_operations { bool avoid_boost; void (*switch_to_mode)(void); + void (*set_cpu_enabled)(int32 cpu, bool enabled); bool (*has_cache_expired)(Thread* thread); int32 (*choose_core)(Thread* thread); bool (*should_rebalance)(Thread* thread); ############################################################################ Commit: 1e8ed5558dd69d1fbbd1578a2c5b2bfa321a5e35 Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx> Date: Sun Nov 24 23:08:13 2013 UTC cpufreq: Rank modules and choose the best one ---------------------------------------------------------------------------- diff --git a/headers/os/drivers/cpufreq.h b/headers/os/drivers/cpufreq.h index 7a4301f..ad36f99 100644 --- a/headers/os/drivers/cpufreq.h +++ b/headers/os/drivers/cpufreq.h @@ -17,6 +17,8 @@ const int kCPUPerformanceScaleMax = 1000; typedef struct cpufreq_module_info { module_info info; + float rank; + status_t (*increase_performance)(int delta, bool allowBoost); status_t (*decrease_performance)(int delta); } cpufreq_module_info; diff --git a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp index 7af294b..f7f7fe1 100644 --- a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp +++ b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp @@ -222,6 +222,8 @@ static cpufreq_module_info sIntelPStates = { std_ops, }, + 1.0f, + increase_performance, decrease_performance, }; diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 5ab8d26..3b7a5ad 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -63,13 +63,22 @@ load_cpufreq_module() while (true) { char name[B_FILE_NAME_LENGTH]; size_t nameLength = sizeof(name); + cpufreq_module_info* current = NULL; if (read_next_module_name(cookie, name, &nameLength) != B_OK) break; - if (get_module(name, (module_info**)&sCPUPerformanceModule) == B_OK) { + if (get_module(name, (module_info**)¤t) == B_OK) { dprintf("found cpufreq module: %s\n", name); - break; + + if (sCPUPerformanceModule != NULL) { + if (sCPUPerformanceModule->rank < current->rank) { + put_module(sCPUPerformanceModule->info.name); + sCPUPerformanceModule = current; + } else + put_module(name); + } else + sCPUPerformanceModule = current; } } ############################################################################ Commit: 26970784cd0050d72a1bb96eccd027dfc2769ffb Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx> Date: Sun Nov 24 23:17:42 2013 UTC scheduler: Clean scheduler_common.h scheduler_common.h is now meant for types, variables and functions used by both core scheduler code and implementations of scheduler modes. Functions like switch_thread() and update_thread_times() do not belong there anymore. ---------------------------------------------------------------------------- diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index e1d7c45..648794c 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1089,8 +1089,94 @@ update_cpu_performance(Thread* thread, int32 thisCore) } +/*! Switches the currently running thread. + This is a service function for scheduler implementations. + + \param fromThread The currently running thread. + \param toThread The thread to switch to. Must be different from + \a fromThread. +*/ +static inline void +switch_thread(Thread* fromThread, Thread* toThread) +{ + // notify the user debugger code + if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_unscheduled(fromThread); + + // stop CPU time based user timers + acquire_spinlock(&fromThread->team->time_lock); + acquire_spinlock(&fromThread->time_lock); + if (fromThread->HasActiveCPUTimeUserTimers() + || fromThread->team->HasActiveCPUTimeUserTimers()) { + user_timer_stop_cpu_timers(fromThread, toThread); + } + release_spinlock(&fromThread->time_lock); + release_spinlock(&fromThread->team->time_lock); + + // update CPU and Thread structures and perform the context switch + cpu_ent* cpu = fromThread->cpu; + toThread->previous_cpu = toThread->cpu = cpu; + fromThread->cpu = NULL; + cpu->running_thread = toThread; + cpu->previous_thread = fromThread; + + arch_thread_set_current_thread(toThread); + arch_thread_context_switch(fromThread, toThread); + + release_spinlock(&fromThread->cpu->previous_thread->scheduler_lock); + + // The use of fromThread below looks weird, but is correct. fromThread had + // been unscheduled earlier, but is back now. For a thread scheduled the + // first time the same is done in thread.cpp:common_thread_entry(). + + // continue CPU time based user timers + acquire_spinlock(&fromThread->team->time_lock); + acquire_spinlock(&fromThread->time_lock); + if (fromThread->HasActiveCPUTimeUserTimers() + || fromThread->team->HasActiveCPUTimeUserTimers()) { + user_timer_continue_cpu_timers(fromThread, cpu->previous_thread); + } + release_spinlock(&fromThread->time_lock); + release_spinlock(&fromThread->team->time_lock); + + // notify the user debugger code + if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_scheduled(fromThread); +} + + +static inline void +update_thread_times(Thread* oldThread, Thread* nextThread) +{ + bigtime_t now = system_time(); + if (oldThread == nextThread) { + acquire_spinlock(&oldThread->time_lock); + oldThread->kernel_time += now - oldThread->last_time; + oldThread->last_time = now; + release_spinlock(&oldThread->time_lock); + } else { + acquire_spinlock(&oldThread->time_lock); + oldThread->kernel_time += now - oldThread->last_time; + oldThread->last_time = 0; + release_spinlock(&oldThread->time_lock); + + acquire_spinlock(&nextThread->time_lock); + nextThread->last_time = now; + release_spinlock(&nextThread->time_lock); + } + + // If the old thread's team has user time timers, check them now. + Team* team = oldThread->team; + + acquire_spinlock(&team->time_lock); + if (team->HasActiveUserTimeUserTimers()) + user_timer_check_team_user_timers(team); + release_spinlock(&team->time_lock); +} + + static void -_scheduler_reschedule(void) +reschedule(void) { ASSERT(!are_interrupts_enabled()); @@ -1206,7 +1292,7 @@ _scheduler_reschedule(void) compute_thread_load(nextThread); // track kernel time (user time is tracked in thread_at_kernel_entry()) - scheduler_update_thread_times(oldThread, nextThread); + update_thread_times(oldThread, nextThread); // track CPU activity track_cpu_activity(oldThread, nextThread, thisCore); @@ -1235,7 +1321,7 @@ _scheduler_reschedule(void) modeLocker.Unlock(); if (nextThread != oldThread) - scheduler_switch_thread(oldThread, nextThread); + switch_thread(oldThread, nextThread); } } @@ -1253,7 +1339,7 @@ scheduler_reschedule(void) return; } - _scheduler_reschedule(); + reschedule(); } @@ -1299,7 +1385,7 @@ scheduler_start(void) { InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); - _scheduler_reschedule(); + reschedule(); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 190e968..2f24441 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -184,90 +184,4 @@ get_core_load(struct Scheduler::CoreEntry* core) } -/*! Switches the currently running thread. - This is a service function for scheduler implementations. - - \param fromThread The currently running thread. - \param toThread The thread to switch to. Must be different from - \a fromThread. -*/ -static inline void -scheduler_switch_thread(Thread* fromThread, Thread* toThread) -{ - // notify the user debugger code - if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) - user_debug_thread_unscheduled(fromThread); - - // stop CPU time based user timers - acquire_spinlock(&fromThread->team->time_lock); - acquire_spinlock(&fromThread->time_lock); - if (fromThread->HasActiveCPUTimeUserTimers() - || fromThread->team->HasActiveCPUTimeUserTimers()) { - user_timer_stop_cpu_timers(fromThread, toThread); - } - release_spinlock(&fromThread->time_lock); - release_spinlock(&fromThread->team->time_lock); - - // update CPU and Thread structures and perform the context switch - cpu_ent* cpu = fromThread->cpu; - toThread->previous_cpu = toThread->cpu = cpu; - fromThread->cpu = NULL; - cpu->running_thread = toThread; - cpu->previous_thread = fromThread; - - arch_thread_set_current_thread(toThread); - arch_thread_context_switch(fromThread, toThread); - - release_spinlock(&fromThread->cpu->previous_thread->scheduler_lock); - - // The use of fromThread below looks weird, but is correct. fromThread had - // been unscheduled earlier, but is back now. For a thread scheduled the - // first time the same is done in thread.cpp:common_thread_entry(). - - // continue CPU time based user timers - acquire_spinlock(&fromThread->team->time_lock); - acquire_spinlock(&fromThread->time_lock); - if (fromThread->HasActiveCPUTimeUserTimers() - || fromThread->team->HasActiveCPUTimeUserTimers()) { - user_timer_continue_cpu_timers(fromThread, cpu->previous_thread); - } - release_spinlock(&fromThread->time_lock); - release_spinlock(&fromThread->team->time_lock); - - // notify the user debugger code - if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) - user_debug_thread_scheduled(fromThread); -} - - -static inline void -scheduler_update_thread_times(Thread* oldThread, Thread* nextThread) -{ - bigtime_t now = system_time(); - if (oldThread == nextThread) { - acquire_spinlock(&oldThread->time_lock); - oldThread->kernel_time += now - oldThread->last_time; - oldThread->last_time = now; - release_spinlock(&oldThread->time_lock); - } else { - acquire_spinlock(&oldThread->time_lock); - oldThread->kernel_time += now - oldThread->last_time; - oldThread->last_time = 0; - release_spinlock(&oldThread->time_lock); - - acquire_spinlock(&nextThread->time_lock); - nextThread->last_time = now; - release_spinlock(&nextThread->time_lock); - } - - // If the old thread's team has user time timers, check them now. - Team* team = oldThread->team; - - acquire_spinlock(&team->time_lock); - if (team->HasActiveUserTimeUserTimers()) - user_timer_check_team_user_timers(team); - release_spinlock(&team->time_lock); -} - - #endif // KERNEL_SCHEDULER_COMMON_H ############################################################################ Commit: 0e94a12f8e0e5fe5ff5b2e3f83384f3586396c92 Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx> Date: Sun Nov 24 23:35:15 2013 UTC kernel: Make CACHE_LINE_ALIGN visible in the whole kernel ---------------------------------------------------------------------------- diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index e1bf1fd..4d5754e 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -55,4 +55,6 @@ void arch_cpu_memory_read_write_barrier(void); #include <arch_cpu.h> +#define CACHE_LINE_ALIGN __attribute__((aligned(CACHE_LINE_SIZE))) + #endif /* _KERNEL_ARCH_CPU_H */ diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 33cf17e..eb2e24a 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -26,6 +26,8 @@ #define CPU_MAX_CACHE_LEVEL 8 +#define CACHE_LINE_SIZE 64 + // MSR registers (possibly Intel specific) #define IA32_MSR_TSC 0x10 diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 4a70428..27fb6ff 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -86,7 +86,7 @@ typedef struct cpu_ent { // arch-specific stuff arch_cpu_info arch; -} cpu_ent __attribute__((aligned(64))); +} cpu_ent CACHE_LINE_ALIGN; //extern cpu_ent gCPU[MAX_BOOT_CPUS]; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 2f24441..dc07cf5 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -18,9 +18,6 @@ #include "RunQueue.h" -#define CACHE_LINE_ALIGN __attribute__((aligned(64))) - - //#define TRACE_SCHEDULER #ifdef TRACE_SCHEDULER # define TRACE(...) dprintf_no_syslog(__VA_ARGS__)