arm64: errata: add detection for AMEVCNTR01 incrementing incorrectly

commit e89d120c4b720e232cc6a94f0fcbd59c15d41489 upstream.

The AMU counter AMEVCNTR01 (constant counter) should increment at the same
rate as the system counter. On affected Cortex-A510 cores, AMEVCNTR01
increments incorrectly giving a significantly higher output value. This
results in inaccurate task scheduler utilization tracking and incorrect
feedback on CPU frequency.

Work around this problem by returning 0 when reading the affected counter
in key locations that results in disabling all users of this counter from
using it either for frequency invariance or as FFH reference counter. This
effect is the same to firmware disabling affected counters.

Details on how the two features are affected by this erratum:

 - AMU counters will not be used for frequency invariance for affected
   CPUs and CPUs in the same cpufreq policy. AMUs can still be used for
   frequency invariance for unaffected CPUs in the system. Although
   unlikely, if no alternative method can be found to support frequency
   invariance for affected CPUs (cpufreq based or solution based on
   platform counters) frequency invariance will be disabled. Please check
   the chapter on frequency invariance at
   Documentation/scheduler/sched-capacity.rst for details of its effect.

 - Given that FFH can be used to fetch either the core or constant counter
   values, restrictions are lifted regarding any of these counters
   returning a valid (!0) value. Therefore FFH is considered supported
   if there is a least one CPU that support AMUs, independent of any
   counters being disabled or affected by this erratum. Clarifying
   comments are now added to the cpc_ffh_supported(), cpu_read_constcnt()
   and cpu_read_corecnt() functions.

The above is achieved through adding a new erratum: ARM64_ERRATUM_2457168.

Signed-off-by: Ionela Voinescu <ionela.voinescu@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: James Morse <james.morse@arm.com>
Link: https://lore.kernel.org/r/20220819103050.24211-1-ionela.voinescu@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst
index 46644736e58353ea957aaa28a8c5ff1e7461624b..663001f697733d4721726234db5823c0b4b847aa 100644 (file)
--- a/Documentation/arm64/silicon-errata.rst
+++ b/Documentation/arm64/silicon-errata.rst
@@ -94,6 +94,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A510     | #2441009        | ARM64_ERRATUM_2441009       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-A510     | #2457168        | ARM64_ERRATUM_2457168       |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1188873,1418040| ARM64_ERRATUM_1418040       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1349291        | N/A                         |

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 42f135e2176b4e10436e5f4d46353650c628b18a..24cce3b9ff1a700598a436a73f02176f3ce8b9e0 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -683,6 +683,23 @@ config ARM64_ERRATUM_2441009
 
          If unsure, say Y.
 
+config ARM64_ERRATUM_2457168
+       bool "Cortex-A510: 2457168: workaround for AMEVCNTR01 incrementing incorrectly"
+       depends on ARM64_AMU_EXTN
+       default y
+       help
+         This option adds the workaround for ARM Cortex-A510 erratum 2457168.
+
+         The AMU counter AMEVCNTR01 (constant counter) should increment at the same rate
+         as the system counter. On affected Cortex-A510 cores AMEVCNTR01 increments
+         incorrectly giving a significantly higher output value.
+
+         Work around this problem by returning 0 when reading the affected counter in
+         key locations that results in disabling all users of this counter. This effect
+         is the same to firmware disabling affected counters.
+
+         If unsure, say Y.
+
 config CAVIUM_ERRATUM_22375
        bool "Cavium erratum 22375, 24313"
        default y

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 23c57e0a7fd1403e9b31229dc62b425550066930..25c495f58f67ab23ccde1a1c2905558098fcd8e3 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -550,6 +550,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP,
                ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
        },
+#endif
+#ifdef CONFIG_ARM64_ERRATUM_2457168
+       {
+               .desc = "ARM erratum 2457168",
+               .capability = ARM64_WORKAROUND_2457168,
+               .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
+               /* Cortex-A510 r0p0-r1p1 */
+               CAP_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1)
+       },
 #endif
        {
        }

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 474aa55c2f68064c70a83276070dd7e27c4b8c29..3e52a9e8b50be798924cabfe3d62e71d6529c44a 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1736,7 +1736,10 @@ static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
                pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n",
                        smp_processor_id());
                cpumask_set_cpu(smp_processor_id(), &amu_cpus);
-               update_freq_counters_refs();
+
+               /* 0 reference values signal broken/disabled counters */
+               if (!this_cpu_has_cap(ARM64_WORKAROUND_2457168))
+                       update_freq_counters_refs();
        }
 }
 

diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 4dd14a6620c176e969962f6854d63538ae265394..acf67ef4c505d007c3ed41cff845ff0f22054909 100644 (file)
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -308,12 +308,25 @@ core_initcall(init_amu_fie);
 
 static void cpu_read_corecnt(void *val)
 {
+       /*
+        * A value of 0 can be returned if the current CPU does not support AMUs
+        * or if the counter is disabled for this CPU. A return value of 0 at
+        * counter read is properly handled as an error case by the users of the
+        * counter.
+        */
        *(u64 *)val = read_corecnt();
 }
 
 static void cpu_read_constcnt(void *val)
 {
-       *(u64 *)val = read_constcnt();
+       /*
+        * Return 0 if the current CPU is affected by erratum 2457168. A value
+        * of 0 is also returned if the current CPU does not support AMUs or if
+        * the counter is disabled. A return value of 0 at counter read is
+        * properly handled as an error case by the users of the counter.
+        */
+       *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ?
+                     0UL : read_constcnt();
 }
 
 static inline
@@ -340,7 +353,22 @@ int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val)
  */
 bool cpc_ffh_supported(void)
 {
-       return freq_counters_valid(get_cpu_with_amu_feat());
+       int cpu = get_cpu_with_amu_feat();
+
+       /*
+        * FFH is considered supported if there is at least one present CPU that
+        * supports AMUs. Using FFH to read core and reference counters for CPUs
+        * that do not support AMUs, have counters disabled or that are affected
+        * by errata, will result in a return value of 0.
+        *
+        * This is done to allow any enabled and valid counters to be read
+        * through FFH, knowing that potentially returning 0 as counter value is
+        * properly handled by the users of these counters.
+        */
+       if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask))
+               return false;
+
+       return true;
 }
 
 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)

diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index b71c6cbb230952a801a27c7cae8d6735e537c99c..cfaffd3c828907912c9f233edf43d997da32323a 100644 (file)
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -54,6 +54,7 @@ WORKAROUND_1418040
 WORKAROUND_1463225
 WORKAROUND_1508412
 WORKAROUND_1542419
+WORKAROUND_2457168
 WORKAROUND_CAVIUM_23154
 WORKAROUND_CAVIUM_27456
 WORKAROUND_CAVIUM_30115