[haiku-commits] BRANCH pdziepak-github.scheduler [7db89e8] in src: add-ons/kernel/power/cpuidle/intel_cstates add-ons/kernel/drivers/power/x86_cpuidle system/kernel add-ons/kernel/generic/cpuidle system/kernel/arch/x86
- From: pdziepak-github.scheduler <community@xxxxxxxxxxxx>
- To: haiku-commits@xxxxxxxxxxxxx
- Date: Tue, 26 Nov 2013 00:00:34 +0100 (CET)
added 1 changeset to branch 'refs/remotes/pdziepak-github/scheduler'
old head: 0e94a12f8e0e5fe5ff5b2e3f83384f3586396c92
new head: 7db89e8dc395db73368479fd9817b2b67899f3f6
overview: https://github.com/pdziepak/Haiku/compare/0e94a12...7db89e8
----------------------------------------------------------------------------
7db89e8: kernel: Rework cpuidle module
* Create new interface for cpuidle modules (similar to the cpufreq
interface)
* Generic cpuidle module is no longer needed
* Fix and update Intel C-State module
[ Pawel Dziepak <pdziepak@xxxxxxxxxxx> ]
----------------------------------------------------------------------------
Commit: 7db89e8dc395db73368479fd9817b2b67899f3f6
Author: Pawel Dziepak <pdziepak@xxxxxxxxxxx>
Date: Mon Nov 25 22:50:27 2013 UTC
----------------------------------------------------------------------------
22 files changed, 356 insertions(+), 339 deletions(-)
build/jam/packages/Haiku | 1 +
build/jam/packages/HaikuBootstrap | 1 +
headers/os/drivers/cpuidle.h | 46 +----
headers/private/kernel/arch/cpu.h | 1 -
headers/private/kernel/arch/x86/arch_cpu.h | 29 ++-
headers/private/kernel/cpu.h | 17 +-
.../drivers/power/x86_cpuidle/intel_cpuidle.cpp | 121 ------------
src/add-ons/kernel/generic/Jamfile | 1 -
src/add-ons/kernel/generic/cpuidle/Jamfile | 7 -
src/add-ons/kernel/generic/cpuidle/cpuidle.cpp | 105 ----------
src/add-ons/kernel/power/Jamfile | 1 +
src/add-ons/kernel/power/cpuidle/Jamfile | 4 +
.../kernel/power/cpuidle/intel_cstates/Jamfile | 8 +
.../cpuidle/intel_cstates/intel_cstates.cpp | 195 +++++++++++++++++++
src/system/kernel/UserTimer.cpp | 2 +-
src/system/kernel/arch/x86/arch_cpu.cpp | 37 ++--
.../kernel/arch/x86/arch_debug_console.cpp | 4 +-
src/system/kernel/cpu.cpp | 65 ++++++-
src/system/kernel/debug/debug.cpp | 4 +-
src/system/kernel/main.cpp | 2 +-
src/system/kernel/smp.cpp | 34 ++--
src/system/kernel/timer.cpp | 10 +-
----------------------------------------------------------------------------
diff --git a/build/jam/packages/Haiku b/build/jam/packages/Haiku
index 46df98a..5511bc4 100644
--- a/build/jam/packages/Haiku
+++ b/build/jam/packages/Haiku
@@ -53,6 +53,7 @@ AddFilesToPackage add-ons kernel partitioning_systems
: amiga_rdb apple efi_gpt intel session ;
AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ;
AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ;
+AddFilesToPackage add-ons kernel power cpuidle : intel_cstates@x86,x86_64 ;
if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 {
AddFilesToPackage add-ons kernel cpu : generic_x86 ;
diff --git a/build/jam/packages/HaikuBootstrap
b/build/jam/packages/HaikuBootstrap
index 1fafbb2..24d6ce6 100644
--- a/build/jam/packages/HaikuBootstrap
+++ b/build/jam/packages/HaikuBootstrap
@@ -52,6 +52,7 @@ AddFilesToPackage add-ons kernel partitioning_systems
: amiga_rdb apple efi_gpt intel session ;
AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ;
AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ;
+AddFilesToPackage add-ons kernel power cpuidle : intel_cstates@x86,x86_64 ;
if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 {
AddFilesToPackage add-ons kernel cpu : generic_x86 ;
diff --git a/headers/os/drivers/cpuidle.h b/headers/os/drivers/cpuidle.h
index 7649cda..c22b3e6 100644
--- a/headers/os/drivers/cpuidle.h
+++ b/headers/os/drivers/cpuidle.h
@@ -8,50 +8,18 @@
#include <module.h>
-#ifdef __cplusplus
-extern "C" {
-#endif
-#define CPUIDLE_CSTATE_MAX 8
-#define CSTATE_NAME_LENGTH 32
-#define B_CPUIDLE_MODULE_NAME "generic/cpuidle/v1"
+#define CPUIDLE_MODULES_PREFIX "power/cpuidle"
-struct CpuidleStat {
- uint64 usageCount;
- bigtime_t usageTime;
-};
+typedef struct cpuidle_module_info {
+ module_info info;
+ float rank;
-struct CpuidleInfo {
- int32 cstateSleep;
- CpuidleStat stats[CPUIDLE_CSTATE_MAX];
-};
+ void (*idle)(void);
+ void (*wait)(int32* variable, int32 test);
+} cpuidle_module_info;
-struct CpuidleDevice;
-
-struct CpuidleCstate {
- char name[CSTATE_NAME_LENGTH];
- int32 latency;
- int32 (*EnterIdle)(int32 state, CpuidleDevice *device);
- void *pData;
-};
-
-
-struct CpuidleDevice {
- CpuidleCstate cStates[CPUIDLE_CSTATE_MAX];
- int32 cStateCount;
-};
-
-
-struct CpuidleModuleInfo {
- module_info info;
- status_t (*AddDevice)(CpuidleDevice *device);
-};
-
-
-#ifdef __cplusplus
-}
-#endif
#endif // _CPUIDLE_MODULE_H
diff --git a/headers/private/kernel/arch/cpu.h
b/headers/private/kernel/arch/cpu.h
index 4d5754e..d5ea2de 100644
--- a/headers/private/kernel/arch/cpu.h
+++ b/headers/private/kernel/arch/cpu.h
@@ -41,7 +41,6 @@ ssize_t arch_cpu_user_strlcpy(char *to, const char *from,
size_t size,
status_t arch_cpu_user_memset(void *s, char c, size_t count,
addr_t *faultHandler);
-void arch_cpu_idle(void);
void arch_cpu_sync_icache(void *address, size_t length);
void arch_cpu_memory_read_barrier(void);
diff --git a/headers/private/kernel/arch/x86/arch_cpu.h
b/headers/private/kernel/arch/x86/arch_cpu.h
index eb2e24a..f4ccaf4 100644
--- a/headers/private/kernel/arch/x86/arch_cpu.h
+++ b/headers/private/kernel/arch/x86/arch_cpu.h
@@ -185,6 +185,10 @@
| IA32_FEATURE_AMD_EXT_RDTSCP \
| IA32_FEATURE_AMD_EXT_LONG)
+// x86 defined features from cpuid eax 5, ecx register
+#define IA32_FEATURE_POWER_MWAIT (1 << 0)
+#define IA32_FEATURE_INTERRUPT_MWAIT (1 << 1)
+
// x86 defined features from cpuid eax 6, eax register
// reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table
5-11)
#define IA32_FEATURE_DTS (1 << 0) //Digital Thermal Sensor
@@ -199,6 +203,9 @@
#define IA32_FEATURE_APERFMPERF (1 << 0) //IA32_APERF, IA32_MPERF
#define IA32_FEATURE_EPB (1 << 3) //IA32_ENERGY_PERF_BIAS
+// x86 defined features from cpuid eax 0x80000007, edx register
+#define IA32_FEATURE_INVARIANT_TSC (1 << 8)
+
// cr4 flags
#define IA32_CR4_PAE (1UL << 5)
#define IA32_CR4_GLOBAL_PAGES (1UL << 7)
@@ -282,8 +289,10 @@ enum x86_feature_type {
FEATURE_EXT, // cpuid eax=1, edx register
FEATURE_EXT_AMD_ECX, // cpuid eax=0x80000001, ecx register (AMD)
FEATURE_EXT_AMD, // cpuid eax=0x80000001, edx register (AMD)
+ FEATURE_5_ECX, // cpuid eax=5, ecx register
FEATURE_6_EAX, // cpuid eax=6, eax registers
FEATURE_6_ECX, // cpuid eax=6, ecx registers
+ FEATURE_EXT_7_EDX, // cpuid eax=0x80000007, edx register
FEATURE_NUM
};
@@ -330,9 +339,6 @@ typedef struct arch_cpu_info {
} arch_cpu_info;
-#undef PAUSE
-#define PAUSE() asm volatile ("pause;")
-
#define nop() __asm__ ("nop"::)
#define x86_read_cr0() ({ \
@@ -426,6 +432,9 @@ typedef struct arch_cpu_info {
})
+extern void (*gCpuIdleFunc)(void);
+
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -478,6 +487,20 @@ void x86_fnsave_swap(void* oldFpuState, const void*
newFpuState);
#endif
+static inline void
+arch_cpu_idle(void)
+{
+ gCpuIdleFunc();
+}
+
+
+static inline void
+arch_cpu_pause(void)
+{
+ asm volatile("pause");
+}
+
+
#ifdef __cplusplus
} // extern "C" {
#endif
diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h
index 27fb6ff..3337d38 100644
--- a/headers/private/kernel/cpu.h
+++ b/headers/private/kernel/cpu.h
@@ -17,12 +17,6 @@
#include <arch/cpu.h>
-// define PAUSE, if not done in arch/cpu.h
-#ifndef PAUSE
-# define PAUSE()
-#endif
-
-
struct kernel_args;
namespace BKernel {
@@ -114,6 +108,17 @@ cpu_topology_node* get_cpu_topology(void);
status_t increase_cpu_performance(int delta, bool allowBoost);
status_t decrease_cpu_performance(int delta);
+void cpu_idle(void);
+void cpu_wait(int32* variable, int32 test);
+
+
+static inline void
+cpu_pause(void)
+{
+ arch_cpu_pause();
+}
+
+
void _user_clear_caches(void *address, size_t length, uint32 flags);
bool _user_cpu_enabled(int32 cpu);
status_t _user_set_cpu_enabled(int32 cpu, bool enabled);
diff --git a/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp
b/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp
deleted file mode 100644
index cbbd1bc..0000000
--- a/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2012, Haiku, Inc. All Rights Reserved.
- * Distributed under the terms of the MIT License.
- *
- * Authors:
- * Yongcong Du <ycdu.vmcore@xxxxxxxxx>
- */
-
-#include <KernelExport.h>
-#include <cpu.h>
-#include <arch_system_info.h>
-
-#include <stdio.h>
-
-#include <cpuidle.h>
-
-#include "x86_cpuidle.h"
-
-static CpuidleDevice sIntelDevice;
-
-static void *kMwaitEax[] = {
- // C0, we never use it
- (void *)0x00,
- // MWAIT C1
- (void *)0x00,
- // MWAIT C2
- (void *)0x10,
- // MWAIT C3
- (void *)0x20,
- // MWAIT C4
- (void *)0x30,
- // MWAIT C5
- (void *)0x40,
- // MWAIT C6, 0x2 is used to fully shrink L2 cache
- (void *)0x52
-};
-
-
-static int32
-IntelCstateIdleEnter(int32 state, CpuidleDevice *device)
-{
- cpu_ent *cpu = get_cpu_struct();
- if (cpu->invoke_scheduler)
- return 0;
-
- CpuidleCstate *cState = &device->cStates[state];
- x86_monitor((void *)&cpu->invoke_scheduler, 0, 0);
- if (!cpu->invoke_scheduler)
- x86_mwait((unsigned long)cState->pData, 1);
-
- return state;
-}
-
-
-static CpuidleCstate sSnbcStates[CPUIDLE_CSTATE_MAX] = {
- {},
- {
- "C1-SNB",
- 1,
- IntelCstateIdleEnter,
- },
- {
- "C3-SNB",
- 80,
- IntelCstateIdleEnter,
- },
- {
- "C6-SNB",
- 104,
- IntelCstateIdleEnter,
- },
- {
- "C7-SNB",
- 109,
- IntelCstateIdleEnter,
- },
-};
-
-
-status_t
-intel_cpuidle_init(void)
-{
- dprintf("intel idle init\n");
- cpu_ent *cpu = get_cpu_struct();
- if (cpu->arch.vendor != VENDOR_INTEL || cpu->arch.family != 6)
- return B_ERROR;
-
- // Calculated Model Value: M = (Extended Model << 4) + Model
- uint32 model = (cpu->arch.extended_model << 4) + cpu->arch.model;
- if (model != 0x2a && model != 0x2d)
- return B_ERROR;
-
- cpuid_info cpuid;
- get_current_cpuid(&cpuid, 5, 0);
- /* ecx[0] monitor/mwait extension supported
- * ecx[1] support for treating interrupts as break-events for mwait
- * edx number of sub-states
- */
- if ((cpuid.regs.ecx & 0x1) == 0 ||
- (cpuid.regs.ecx & 0x2) == 0 ||
- cpuid.regs.edx == 0) {
- return B_ERROR;
- }
-
- sIntelDevice.cStateCount = 1;
- for (int32 i = 1; i < CPUIDLE_CSTATE_MAX; i++) {
- int32 subStates = (cpuid.regs.edx >> ((i) * 4)) & 0xf;
- // no sub-states means the state is not available
- if (!subStates)
- continue;
- sIntelDevice.cStates[sIntelDevice.cStateCount] =
- sSnbcStates[i];
- sIntelDevice.cStates[sIntelDevice.cStateCount].pData =
- kMwaitEax[i];
- sIntelDevice.cStateCount++;
- }
- status_t status = gIdle->AddDevice(&sIntelDevice);
- if (status == B_OK)
- dprintf("using intel idle\n");
- return status;
-}
diff --git a/src/add-ons/kernel/generic/Jamfile
b/src/add-ons/kernel/generic/Jamfile
index 9a1677d..1ac20ba 100644
--- a/src/add-ons/kernel/generic/Jamfile
+++ b/src/add-ons/kernel/generic/Jamfile
@@ -3,7 +3,6 @@ SubDir HAIKU_TOP src add-ons kernel generic ;
SubInclude HAIKU_TOP src add-ons kernel generic ata_adapter ;
SubInclude HAIKU_TOP src add-ons kernel generic atomizer ;
SubInclude HAIKU_TOP src add-ons kernel generic bios ;
-SubInclude HAIKU_TOP src add-ons kernel generic cpuidle ;
SubInclude HAIKU_TOP src add-ons kernel generic dpc ;
SubInclude HAIKU_TOP src add-ons kernel generic ide_adapter ;
SubInclude HAIKU_TOP src add-ons kernel generic locked_pool ;
diff --git a/src/add-ons/kernel/generic/cpuidle/Jamfile
b/src/add-ons/kernel/generic/cpuidle/Jamfile
deleted file mode 100644
index bb7cdd2..0000000
--- a/src/add-ons/kernel/generic/cpuidle/Jamfile
+++ /dev/null
@@ -1,7 +0,0 @@
-SubDir HAIKU_TOP src add-ons kernel generic cpuidle ;
-
-UsePrivateKernelHeaders ;
-
-KernelAddon cpuidle :
- cpuidle.cpp
- ;
diff --git a/src/add-ons/kernel/generic/cpuidle/cpuidle.cpp
b/src/add-ons/kernel/generic/cpuidle/cpuidle.cpp
deleted file mode 100644
index cc17be8..0000000
--- a/src/add-ons/kernel/generic/cpuidle/cpuidle.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2012, Haiku, Inc. All Rights Reserved.
- * Distributed under the terms of the MIT License.
- *
- * Authors:
- * Yongcong Du <ycdu.vmcore@xxxxxxxxx>
- */
-
-#include <KernelExport.h>
-#include <stdio.h>
-#include <string.h>
-
-#include <smp.h>
-#include <cpuidle.h>
-
-
-static CpuidleInfo sPerCPU[B_MAX_CPU_COUNT];
-static CpuidleDevice *sDevice;
-extern void (*gCpuIdleFunc)(void);
-
-
-/*
- * next cstate selection algorithm is based on NetBSD's
- * it's simple, stupid
- */
-static int32
-SelectCstate(CpuidleInfo *info)
-{
- static const int32 csFactor = 3;
-
- for (int32 i = sDevice->cStateCount - 1; i > 0; i--) {
- CpuidleCstate *cState = &sDevice->cStates[i];
- if (info->cstateSleep > cState->latency * csFactor)
- return i;
- }
-
- /* Choose C1 if there's no state found */
- return 1;
-}
-
-
-static inline void
-EnterCstate(int32 state, CpuidleInfo *info)
-{
- CpuidleCstate *cstate = &sDevice->cStates[state];
- bigtime_t now = system_time();
- int32 finalState = cstate->EnterIdle(state, sDevice);
- if (finalState > 0) {
- bigtime_t diff = system_time() - now;
- info->cstateSleep = diff;
- info->stats[finalState].usageCount++;
- info->stats[finalState].usageTime += diff;
- } else {
- info->cstateSleep = 0;
- }
-}
-
-
-static void
-CpuCstateIdle(void)
-{
- CpuidleInfo *info = &sPerCPU[smp_get_current_cpu()];
- int32 state = SelectCstate(info);
- EnterCstate(state, info);
-}
-
-
-static status_t
-std_ops(int32 op, ...)
-{
- switch (op) {
- case B_MODULE_INIT:
- return B_OK;
- case B_MODULE_UNINIT:
- return B_OK;
- }
-
- return B_ERROR;
-}
-
-
-status_t AddDevice(CpuidleDevice *device)
-{
- sDevice = device;
- memory_write_barrier();
- gCpuIdleFunc = CpuCstateIdle;
- return B_OK;
-}
-
-
-static CpuidleModuleInfo sCpuidleModule = {
- {
- B_CPUIDLE_MODULE_NAME,
- 0,
- std_ops
- },
-
- AddDevice,
-};
-
-
-module_info *modules[] = {
- (module_info *)&sCpuidleModule,
- NULL
-};
diff --git a/src/add-ons/kernel/power/Jamfile b/src/add-ons/kernel/power/Jamfile
index 84032da..289cb1f2 100644
--- a/src/add-ons/kernel/power/Jamfile
+++ b/src/add-ons/kernel/power/Jamfile
@@ -1,4 +1,5 @@
SubDir HAIKU_TOP src add-ons kernel power ;
SubInclude HAIKU_TOP src add-ons kernel power cpufreq ;
+SubInclude HAIKU_TOP src add-ons kernel power cpuidle ;
diff --git a/src/add-ons/kernel/power/cpuidle/Jamfile
b/src/add-ons/kernel/power/cpuidle/Jamfile
new file mode 100644
index 0000000..702f538
--- /dev/null
+++ b/src/add-ons/kernel/power/cpuidle/Jamfile
@@ -0,0 +1,4 @@
+SubDir HAIKU_TOP src add-ons kernel power cpuidle ;
+
+SubInclude HAIKU_TOP src add-ons kernel power cpuidle intel_cstates ;
+
diff --git a/src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile
b/src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile
new file mode 100644
index 0000000..4c8ee48
--- /dev/null
+++ b/src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile
@@ -0,0 +1,8 @@
+SubDir HAIKU_TOP src add-ons kernel power cpuidle intel_cstates ;
+
+UsePrivateKernelHeaders ;
+
+KernelAddon intel_cstates :
+ intel_cstates.cpp
+ ;
+
diff --git a/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp
b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp
new file mode 100644
index 0000000..d401b45
--- /dev/null
+++ b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp
@@ -0,0 +1,195 @@
+/*
+ * Copyright 2012-2013, Haiku, Inc. All Rights Reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ * Yongcong Du <ycdu.vmcore@xxxxxxxxx>
+ * Paweł Dziepak, <pdziepak@xxxxxxxxxxx>
+ */
+
+
+#include <arch_system_info.h>
+#include <cpu.h>
+#include <debug.h>
+#include <smp.h>
+#include <thread.h>
+#include <util/AutoLock.h>
+
+#include <cpuidle.h>
+#include <KernelExport.h>
+
+#include <stdio.h>
+
+
+#define CPUIDLE_CSTATE_MAX 8
+
+#define MWAIT_INTERRUPTS_BREAK (1 << 0)
+
+#define INTEL_CSTATES_MODULE_NAME CPUIDLE_MODULES_PREFIX
"/intel_cstates/v1"
+
+#define TIME_STEP 250
+
+struct CState {
+ uint32 fCode;
+ int fSubStatesCount;
+ int fSubStatesStep;
+};
+
+static CState sCStates[CPUIDLE_CSTATE_MAX];
+static int sCStateCount;
+
+static bigtime_t* sIdleTime;
+
+
+static inline void
+x86_monitor(void* address, uint32 ecx, uint32 edx)
+{
+ asm volatile("monitor" : : "a" (address), "c" (ecx), "d"(edx));
+}
+
+
+static inline void
+x86_mwait(uint32 eax, uint32 ecx)
+{
+ asm volatile("mwait" : : "a" (eax), "c" (ecx));
+}
+
+
+static void
+idle(void)
+{
+ ASSERT(thread_get_current_thread()->pinned_to_cpu > 0);
+ int32 cpu = smp_get_current_cpu();
+
+ bigtime_t idleTime = sIdleTime[cpu];
+ int state = min_c(idleTime / TIME_STEP, sCStateCount - 1);
+
+ ASSERT(state >= 0 && state < sCStateCount);
+
+ int subState = idleTime % TIME_STEP;
+ subState /= sCStates[state].fSubStatesStep;
+
+ ASSERT(subState >= 0 && subState < sCStates[state].fSubStatesCount);
+
+ InterruptsLocker locker;
+ int dummy;
+ bigtime_t start = system_time();
+ x86_monitor(&dummy, 0, 0);
+ x86_mwait(sCStates[state].fCode | subState, MWAIT_INTERRUPTS_BREAK);
+ bigtime_t delta = system_time() - start;
+ locker.Unlock();
+
+ sIdleTime[cpu] = (idleTime + delta) / 2;
+}
+
+
+static void
+wait(int32* variable, int32 test)
+{
+ InterruptsLocker _;
+ x86_monitor(variable, 0, 0);
+ if (*variable != test)
+ x86_mwait(sCStates[0].fCode, MWAIT_INTERRUPTS_BREAK);
+}
+
+
+static status_t
+init_cstates()
+{
+ if (!x86_check_feature(IA32_FEATURE_EXT_MONITOR, FEATURE_EXT))
+ return B_ERROR;
+ if (!x86_check_feature(IA32_FEATURE_POWER_MWAIT, FEATURE_5_ECX))
+ return B_ERROR;
+ if (!x86_check_feature(IA32_FEATURE_INTERRUPT_MWAIT, FEATURE_5_ECX))
+ return B_ERROR;
+
+ // we need invariant TSC
+ if (!x86_check_feature(IA32_FEATURE_INVARIANT_TSC, FEATURE_EXT_7_EDX))
+ return B_ERROR;
+
+ // get C-state data
+ cpuid_info cpuid;
+ get_current_cpuid(&cpuid, 0, 0);
+ uint32 maxBasicLeaf = cpuid.eax_0.max_eax;
+ if (maxBasicLeaf < 5)
+ return B_ERROR;
+
+ get_current_cpuid(&cpuid, 5, 0);
+ if ((cpuid.regs.eax & 0xffff) < sizeof(int32))
+ return B_ERROR;
+
+ char cStates[64];
+ unsigned int offset = 0;
+ for (int32 i = 1; i < CPUIDLE_CSTATE_MAX; i++) {
+ int32 subStates = (cpuid.regs.edx >> (i * 4)) & 0xf;
+ // no sub-states means the state is not available
+ if (subStates == 0)
+ continue;
+
+ if (offset < sizeof(cStates)) {
+ offset += snprintf(cStates + offset, sizeof(cStates) -
offset,
+ ", C%" B_PRId32, i);
+ }
+
+ sCStates[sCStateCount].fCode = sCStateCount * 0x10;
+ sCStates[sCStateCount].fSubStatesCount = subStates;
+ sCStates[sCStateCount].fSubStatesStep = TIME_STEP / subStates;
+ sCStateCount++;
+ }
+
+ if (sCStateCount == 0)
+ return B_ERROR;
+
+ sIdleTime = new(std::nothrow) bigtime_t[smp_get_num_cpus()];
+ if (sIdleTime == NULL)
+ return B_NO_MEMORY;
+ memset(sIdleTime, 0, sizeof(bigtime_t) * smp_get_num_cpus());
+
+ dprintf("using Intel C-States: C0%s\n", cStates);
+ return B_OK;
+}
+
+
+static status_t
+uninit_cstates()
+{
+ delete[] sIdleTime;
+ return B_OK;
+}
+
+
+static status_t
+std_ops(int32 op, ...)
+{
+ switch (op) {
+ case B_MODULE_INIT:
+ return init_cstates();
+
+ case B_MODULE_UNINIT:
+ uninit_cstates();
+ return B_OK;
+ }
+
+ return B_ERROR;
+}
+
+
+static cpuidle_module_info sIntelCStates = {
+ {
+ INTEL_CSTATES_MODULE_NAME,
+ 0,
+ std_ops,
+ },
+
+ 0.8f,
+
+ idle,
+ wait
+};
+
+
+module_info* modules[] = {
+ (module_info*)&sIntelCStates,
+ NULL
+};
+
diff --git a/src/system/kernel/UserTimer.cpp b/src/system/kernel/UserTimer.cpp
index b3da871..db816a4 100644
--- a/src/system/kernel/UserTimer.cpp
+++ b/src/system/kernel/UserTimer.cpp
@@ -196,7 +196,7 @@ UserTimer::HandleTimerHook(struct timer* timer)
while (!locked && atomic_get(&userTimer->fSkip) == 0) {
locked = try_acquire_write_seqlock(&sUserTimerLock);
if (!locked)
- PAUSE();
+ cpu_pause();
}
if (locked) {
diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp
b/src/system/kernel/arch/x86/arch_cpu.cpp
index be1afda..16d60bf 100644
--- a/src/system/kernel/arch/x86/arch_cpu.cpp
+++ b/src/system/kernel/arch/x86/arch_cpu.cpp
@@ -1,5 +1,6 @@
/*
* Copyright 2002-2010, Axel Dörfler, axeld@xxxxxxxxxxxxxxxx.
+ * Copyright 2013, Paweł Dziepak, pdziepak@xxxxxxxxxxx.
* Copyright 2012, Alex Smith, alex@xxxxxxxxxxxxxxxx.
* Distributed under the terms of the MIT License.
*
@@ -69,16 +70,6 @@ static const struct cpu_vendor_info vendor_info[VENDOR_NUM]
= {
#define K8_CMPHALT (K8_SMIONCMPHALT |
K8_C1EONCMPHALT)
-/*
- * 0 favors highest performance while 15 corresponds to the maximum energy
- * savings. 7 means balance between performance and energy savings.
- * Refer to Section 14.3.4 in <Intel 64 and IA-32 Architectures Software
- * Developer's Manual Volume 3> for details
- */
-#define ENERGY_PERF_BIAS_PERFORMANCE 0
-#define ENERGY_PERF_BIAS_BALANCE 7
-#define ENERGY_PERF_BIAS_POWERSAVE 15
-
struct set_mtrr_parameter {
int32 index;
uint64 base;
@@ -929,12 +920,22 @@ detect_cpu(int currentCPU)
cpu->arch.feature[FEATURE_EXT_AMD] &=
IA32_FEATURES_INTEL_EXT;
}
+ if (maxBasicLeaf >= 5) {
+ get_current_cpuid(&cpuid, 5, 0);
+ cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx;
+ }
+
if (maxBasicLeaf >= 6) {
get_current_cpuid(&cpuid, 6, 0);
cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax;
cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx;
}
+ if (maxExtendedLeaf >= 0x80000007) {
+ get_current_cpuid(&cpuid, 0x80000007, 0);
+ cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx;
+ }
+
detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf);
#if DUMP_FEATURE_STRING
@@ -1060,15 +1061,6 @@ arch_cpu_init_percpu(kernel_args* args, int cpu)
gCpuIdleFunc = halt_idle;
}
- if (x86_check_feature(IA32_FEATURE_EPB, FEATURE_6_ECX)) {
- uint64 msr = x86_read_msr(IA32_MSR_ENERGY_PERF_BIAS);
- if ((msr & 0xf) == ENERGY_PERF_BIAS_PERFORMANCE) {
- msr &= ~0xf;
- msr |= ENERGY_PERF_BIAS_BALANCE;
- x86_write_msr(IA32_MSR_ENERGY_PERF_BIAS, msr);
- }
- }
-
return B_OK;
}
@@ -1288,13 +1280,6 @@ arch_cpu_shutdown(bool rebootSystem)
void
-arch_cpu_idle(void)
-{
- gCpuIdleFunc();
-}
-
-
-void
arch_cpu_sync_icache(void* address, size_t length)
{
// instruction cache is always consistent on x86
diff --git a/src/system/kernel/arch/x86/arch_debug_console.cpp
b/src/system/kernel/arch/x86/arch_debug_console.cpp
index 8309a99..1f226db 100644
--- a/src/system/kernel/arch/x86/arch_debug_console.cpp
+++ b/src/system/kernel/arch/x86/arch_debug_console.cpp
@@ -316,7 +316,7 @@ arch_debug_blue_screen_getchar(void)
if (c >= 0)
return (char)c;
- PAUSE();
+ arch_cpu_pause();
}
}
@@ -352,7 +352,7 @@ arch_debug_serial_getchar(void)
if ((lineStatus & 0x1) != 0)
break;
- PAUSE();
+ arch_cpu_pause();
}
return in8(sSerialBasePort + SERIAL_RECEIVE_BUFFER);
diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp
index 3b7a5ad..2fdf6fd 100644
--- a/src/system/kernel/cpu.cpp
+++ b/src/system/kernel/cpu.cpp
@@ -16,6 +16,7 @@
#include <string.h>
#include <cpufreq.h>
+#include <cpuidle.h>
#include <boot/kernel_args.h>
#include <kscheduler.h>
@@ -30,6 +31,7 @@ uint32 gCPUCacheLevelCount;
static cpu_topology_node sCPUTopology;
static cpufreq_module_info* sCPUPerformanceModule;
+static cpuidle_module_info* sCPUIdleModule;
static spinlock sSetCpuLock;
@@ -89,6 +91,40 @@ load_cpufreq_module()
}
+static void
+load_cpuidle_module()
+{
+ void* cookie = open_module_list(CPUIDLE_MODULES_PREFIX);
+
+ while (true) {
+ char name[B_FILE_NAME_LENGTH];
+ size_t nameLength = sizeof(name);
+ cpuidle_module_info* current = NULL;
+
+ if (read_next_module_name(cookie, name, &nameLength) != B_OK)
+ break;
+
+ if (get_module(name, (module_info**)¤t) == B_OK) {
+ dprintf("found cpuidle module: %s\n", name);
+
+ if (sCPUIdleModule != NULL) {
+ if (sCPUIdleModule->rank < current->rank) {
+ put_module(sCPUIdleModule->info.name);
+ sCPUIdleModule = current;
+ } else
+ put_module(name);
+ } else
+ sCPUIdleModule = current;
+ }
+ }
+
+ close_module_list(cookie);
+
+ if (sCPUIdleModule == NULL)
+ dprintf("no valid cpuidle module found\n");
+}
+
+
status_t
cpu_init_post_modules(kernel_args *args)
{
@@ -97,6 +133,7 @@ cpu_init_post_modules(kernel_args *args)
return result;
load_cpufreq_module();
+ load_cpuidle_module();
return B_OK;
}
@@ -232,6 +269,8 @@ cpu_build_topology_tree(void)
int32 lastID[CPU_TOPOLOGY_LEVELS];
memset(&lastID, 0, sizeof(lastID));
cpu_rebuild_topology_tree(&sCPUTopology, lastID);
+
+ return B_OK;
}
@@ -260,6 +299,31 @@ decrease_cpu_performance(int delta)
}
+void
+cpu_idle(void)
+{
+#if KDEBUG
+ if (!are_interrupts_enabled())
+ panic("cpu_idle() called with interrupts disabled.");
+#endif
+
+ if (sCPUIdleModule != NULL)
+ sCPUIdleModule->idle();
+ else
+ arch_cpu_idle();
+}
+
+
+void
+cpu_wait(int32* variable, int32 test)
+{
+ if (sCPUIdleModule != NULL)
+ sCPUIdleModule->wait(variable, test);
+ else
+ arch_cpu_pause();
+}
+
+
// #pragma mark -
@@ -283,7 +347,6 @@ _user_cpu_enabled(int32 cpu)
status_t
_user_set_cpu_enabled(int32 cpu, bool enabled)
{
- cpu_status state;
int32 i, count;
if (cpu < 0 || cpu >= smp_get_num_cpus())
diff --git a/src/system/kernel/debug/debug.cpp
b/src/system/kernel/debug/debug.cpp
index e75acd6..b56145b 100644
--- a/src/system/kernel/debug/debug.cpp
+++ b/src/system/kernel/debug/debug.cpp
@@ -703,7 +703,7 @@ kgetc(void)
}
}
- PAUSE();
+ cpu_pause();
}
}
@@ -1008,7 +1008,7 @@ hand_over_kernel_debugger()
// initiated by the user, it is harmless, though.
sHandOverKDL = true;
while (atomic_get(&sHandOverKDLToCPU) >= 0)
- PAUSE();
+ cpu_wait(&sHandOverKDLToCPU, -1);
}
diff --git a/src/system/kernel/main.cpp b/src/system/kernel/main.cpp
index 9dedf47..f135265 100644
--- a/src/system/kernel/main.cpp
+++ b/src/system/kernel/main.cpp
@@ -262,7 +262,7 @@ _start(kernel_args *bootKernelArgs, int currentCPU)
#endif
for (;;)
- arch_cpu_idle();
+ cpu_idle();
return 0;
}
diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp
index a07df3f..0477b85 100644
--- a/src/system/kernel/smp.cpp
+++ b/src/system/kernel/smp.cpp
@@ -358,7 +358,7 @@ acquire_spinlock(spinlock* lock)
}
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_wait(lock, 0);
}
if (atomic_get_and_set((int32*)lock, 1) == 0)
break;
@@ -411,7 +411,7 @@ acquire_spinlock_nocheck(spinlock *lock)
count = 0;
}
- PAUSE();
+ cpu_wait(lock, 0);
}
if (atomic_get_and_set((int32*)lock, 1) == 0)
@@ -455,7 +455,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock)
}
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_wait(lock, 0);
}
if (atomic_get_and_set((int32*)lock, 1) == 0)
break;
@@ -570,7 +570,7 @@ acquire_write_spinlock(rw_spinlock* lock)
}
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_wait(&lock->lock, 0);
}
}
}
@@ -583,7 +583,7 @@ release_write_spinlock(rw_spinlock* lock)
uint32 previous = atomic_get_and_set(&lock->lock, 0);
if ((previous & 1u << 31) == 0) {
panic("release_write_spinlock: lock %p was already released
(value: "
- "%x)\n", lock, previous);
+ "%#" B_PRIx32 ")\n", lock, previous);
}
#else
atomic_set(&lock->lock, 0);
@@ -639,7 +639,7 @@ acquire_read_spinlock(rw_spinlock* lock)
}
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_wait(&lock->lock, 0);
}
}
}
@@ -652,7 +652,7 @@ release_read_spinlock(rw_spinlock* lock)
uint32 previous = atomic_add(&lock->lock, -1);
if ((previous & 1 << 31) != 0) {
panic("release_read_spinlock: lock %p was already released
(value:"
- " %x)\n", lock, previous);
+ " %#" B_PRIx32 ")\n", lock, previous);
}
#else
atomic_add(&lock->lock, -1);
@@ -695,7 +695,7 @@ release_read_seqlock(seqlock* lock, uint32 count) {
uint32 current = atomic_get((int32*)&lock->count);
if (count % 2 == 1 || current != count) {
- PAUSE();
+ cpu_pause();
return false;
}
@@ -719,7 +719,7 @@ retry:
state = disable_interrupts();
process_all_pending_ici(smp_get_current_cpu());
restore_interrupts(state);
- PAUSE();
+ cpu_pause();
}
state = disable_interrupts();
acquire_spinlock(&sFreeMessageSpinlock);
@@ -757,7 +757,7 @@ find_free_message_interrupts_disabled(int32 currentCPU,
while (sFreeMessageCount <= 0) {
release_spinlock(&sFreeMessageSpinlock);
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_pause();
acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock);
}
@@ -1014,7 +1014,7 @@ call_all_cpus_early(void (*function)(void*, int), void*
cookie)
// wait for all CPUs to finish
while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0)
- PAUSE();
+ cpu_pause();
}
function(cookie, 0);
@@ -1085,7 +1085,7 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data,
addr_t data2,
// if the message is sync after it has removed it from
the mailbox
while (msg->done == false) {
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_pause();
}
// for SYNC messages, it's our responsibility to put it
// back into the free list
@@ -1148,7 +1148,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message,
addr_t data,
// if the message is sync after it has removed it from the
mailbox
while (msg->done == false) {
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_pause();
}
// for SYNC messages, it's our responsibility to put it
@@ -1210,7 +1210,7 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t
data2, addr_t data3,
while (msg->done == false) {
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_pause();
}
TRACE(("smp_send_broadcast_ici: returning message to
free list\n"));
@@ -1274,7 +1274,7 @@ smp_send_broadcast_ici_interrupts_disabled(int32
currentCPU, int32 message,
while (msg->done == false) {
process_all_pending_ici(currentCPU);
- PAUSE();
+ cpu_pause();
}
TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld:
returning "
@@ -1310,7 +1310,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous)
if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0)
process_early_cpu_call(cpu);
- PAUSE();
+ cpu_pause();
}
return false;
@@ -1346,7 +1346,7 @@ smp_cpu_rendezvous(uint32* var, int current_cpu)
uint32 allReady = ((uint32)1 << sNumCPUs) - 1;
while ((uint32)atomic_get((int32*)var) != allReady)
- PAUSE();
+ cpu_wait((int32*)var, allReady);
}
diff --git a/src/system/kernel/timer.cpp b/src/system/kernel/timer.cpp
index d10e3c3..5a3ea20 100644
--- a/src/system/kernel/timer.cpp
+++ b/src/system/kernel/timer.cpp
@@ -447,9 +447,8 @@ cancel_timer(timer* event)
if (cpu != smp_get_current_cpu()) {
spinLocker.Unlock();
- while (atomic_get(&cpuData.current_event_in_progress) == 1) {
- PAUSE();
- }
+ while (atomic_get(&cpuData.current_event_in_progress) == 1)
+ cpu_wait(&cpuData.current_event_in_progress, 0);
}
return true;
@@ -461,7 +460,6 @@ spin(bigtime_t microseconds)
{
bigtime_t time = system_time();
- while ((system_time() - time) < microseconds) {
- PAUSE();
- }
+ while ((system_time() - time) < microseconds)
+ cpu_pause();
}
Other related posts:
- » [haiku-commits] BRANCH pdziepak-github.scheduler [7db89e8] in src: add-ons/kernel/power/cpuidle/intel_cstates add-ons/kernel/drivers/power/x86_cpuidle system/kernel add-ons/kernel/generic/cpuidle system/kernel/arch/x86 - pdziepak-github . scheduler
