From 8a6fe8f21ec4f049a7b1fe120ad50a5065a9c7a8 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 19 Jun 2024 16:01:11 +0200 Subject: [PATCH 01/84] s390/cpum_sf: Use refcount_t instead of atomic_t Replace atomic_t by refcount_t for reference counting of events. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 736c1d9632dd..041e22bee839 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -585,7 +585,7 @@ static void extend_sampling_buffer(struct sf_buffer *sfb, } /* Number of perf events counting hardware events */ -static atomic_t num_events; +static refcount_t num_events; /* Used to avoid races in calling reserve/release_cpumf_hardware */ static DEFINE_MUTEX(pmc_reserve_mutex); @@ -644,10 +644,8 @@ static int reserve_pmc_hardware(void) static void hw_perf_event_destroy(struct perf_event *event) { /* Release PMC if this is the last perf event */ - if (!atomic_add_unless(&num_events, -1, 1)) { - mutex_lock(&pmc_reserve_mutex); - if (atomic_dec_return(&num_events) == 0) - release_pmc_hardware(); + if (refcount_dec_and_mutex_lock(&num_events, &pmc_reserve_mutex)) { + release_pmc_hardware(); mutex_unlock(&pmc_reserve_mutex); } } @@ -810,22 +808,19 @@ static int __hw_perf_event_init(struct perf_event *event) struct hws_qsi_info_block si; struct perf_event_attr *attr = &event->attr; struct hw_perf_event *hwc = &event->hw; - int cpu, err; + int cpu, err = 0; /* Reserve CPU-measurement sampling facility */ - err = 0; - if (!atomic_inc_not_zero(&num_events)) { - mutex_lock(&pmc_reserve_mutex); - if (atomic_read(&num_events) == 0 && reserve_pmc_hardware()) - err = -EBUSY; - else - atomic_inc(&num_events); - mutex_unlock(&pmc_reserve_mutex); + mutex_lock(&pmc_reserve_mutex); + if (!refcount_inc_not_zero(&num_events)) { + err = reserve_pmc_hardware(); + if (!err) + refcount_set(&num_events, 1); } - event->destroy = hw_perf_event_destroy; - + mutex_unlock(&pmc_reserve_mutex); if (err) goto out; + event->destroy = hw_perf_event_destroy; /* Access per-CPU sampling information (query sampling info) */ /* @@ -2143,7 +2138,7 @@ static int cpusf_pmu_setup(unsigned int cpu, int flags) /* Ignore the notification if no events are scheduled on the PMU. * This might be racy... */ - if (!atomic_read(&num_events)) + if (!refcount_read(&num_events)) return 0; local_irq_disable(); From ba38df7a9b2c3d38a53bb26443add88ca24c970e Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 25 Jun 2024 11:46:04 +0200 Subject: [PATCH 02/84] s390/cpum_sf: Remove unused define PERF_CPUM_SF_MODE_MASK Remove unreferenced marco PERF_CPUM_SF_MODE_MASK. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/perf_event.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 9917e2717b2b..4879f3a5c474 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -58,8 +58,6 @@ struct perf_sf_sde_regs { #define PERF_EVENT_CPUM_CF_DIAG 0xBC000UL /* Event: Counter sets */ #define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */ #define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */ -#define PERF_CPUM_SF_MODE_MASK (PERF_CPUM_SF_BASIC_MODE| \ - PERF_CPUM_SF_DIAG_MODE) #define PERF_CPUM_SF_FREQ_MODE 0x0008 /* Sampling with frequency */ #define REG_NONE 0 From ea95be4bda694d9000677c04efd15a09cd3ee1d2 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 27 Jun 2024 09:02:06 +0200 Subject: [PATCH 03/84] s390/cpum_sf: Remove unused defines REG_NONE and REG_OVERFLOW Member hw_perf_event::reg.reg is set but never used, so remove it. Defines REG_NONE and REG_OVERFLOW are not referenced anymore. The initialization to zero takes place in function perf_event_alloc() where ... event = kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL | __GFP_ZERO, node); ... makes sure memory allocated for the event is zero'ed. This is done in the kernel's common code in kernel/events/core.c The struct perf_event contains member hw_perf_event as in struct perf_event { .... struct hw_perf_event hw; .... }; This contained sub-structure is also initialized to zero. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/perf_event.h | 2 -- arch/s390/kernel/perf_cpum_sf.c | 4 ---- 2 files changed, 6 deletions(-) diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 4879f3a5c474..05be93b5905d 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -60,8 +60,6 @@ struct perf_sf_sde_regs { #define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */ #define PERF_CPUM_SF_FREQ_MODE 0x0008 /* Sampling with frequency */ -#define REG_NONE 0 -#define REG_OVERFLOW 1 #define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config) #define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc) #define TEAR_REG(hwc) ((hwc)->last_tag) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 041e22bee839..1019fee3e734 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -876,10 +876,6 @@ static int __hw_perf_event_init(struct perf_event *event) if (err) goto out; - /* Initialize sample data overflow accounting */ - hwc->extra_reg.reg = REG_OVERFLOW; - OVERFLOW_REG(hwc) = 0; - /* Use AUX buffer. No need to allocate it by ourself */ if (attr->config == PERF_EVENT_CPUM_SF_DIAG) return 0; From 501cab2b1d12d7ce34f6b0c36c1ff45aec7500b2 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 25 Jun 2024 12:36:21 +0200 Subject: [PATCH 04/84] s390/cpum_sf: Rename macro to consistent prefix Rename macro SAMPLE_FREQ_MODE to SAMPL_FREQ_MODE to make its prefix consistent with all other macro starting with prefix SAMPL_XXX. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/perf_event.h | 2 +- arch/s390/kernel/perf_cpum_sf.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 05be93b5905d..547e26a33f67 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -66,7 +66,7 @@ struct perf_sf_sde_regs { #define SAMPL_RATE(hwc) ((hwc)->event_base) #define SAMPL_FLAGS(hwc) ((hwc)->config_base) #define SAMPL_DIAG_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_DIAG_MODE) -#define SAMPLE_FREQ_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_FREQ_MODE) +#define SAMPL_FREQ_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_FREQ_MODE) #define perf_arch_fetch_caller_regs(regs, __ip) do { \ (regs)->psw.addr = (__ip); \ diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 1019fee3e734..ca04a56e1520 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -798,7 +798,7 @@ static int __hw_perf_event_init_rate(struct perf_event *event, hw_init_period(hwc, SAMPL_RATE(hwc)); debug_sprintf_event(sfdbg, 4, "%s: cpu %d period %#llx freq %d,%#lx\n", __func__, event->cpu, event->attr.sample_period, - event->attr.freq, SAMPLE_FREQ_MODE(hwc)); + event->attr.freq, SAMPL_FREQ_MODE(hwc)); return 0; } @@ -1865,7 +1865,7 @@ static int cpumsf_pmu_check_period(struct perf_event *event, u64 value) si = cpuhw->qsi; } - do_freq = !!SAMPLE_FREQ_MODE(&event->hw); + do_freq = !!SAMPL_FREQ_MODE(&event->hw); rate = getrate(do_freq, value, &si); if (!rate) return -EINVAL; From 52d6ef92a4711964d403e82a00ae2ef509b53eae Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 27 Jun 2024 09:45:48 +0200 Subject: [PATCH 05/84] s390/cpum_sf: Move defines from header file to source file Some defines in common header file arch/s390/include/asm/perf_event.h are only used in one source file arch/s390/kernel/perf_cpum_sf.c. Move these defines from header to source file. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/perf_event.h | 16 ---------------- arch/s390/kernel/perf_cpum_sf.c | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 547e26a33f67..0fed84fe80d7 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -50,23 +50,7 @@ struct perf_sf_sde_regs { /* Perf PMU definitions for the counter facility */ #define PERF_CPUM_CF_MAX_CTR 0xffffUL /* Max ctr for ECCTR */ - -/* Perf PMU definitions for the sampling facility */ -#define PERF_CPUM_SF_MAX_CTR 2 -#define PERF_EVENT_CPUM_SF 0xB0000UL /* Event: Basic-sampling */ -#define PERF_EVENT_CPUM_SF_DIAG 0xBD000UL /* Event: Combined-sampling */ #define PERF_EVENT_CPUM_CF_DIAG 0xBC000UL /* Event: Counter sets */ -#define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */ -#define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */ -#define PERF_CPUM_SF_FREQ_MODE 0x0008 /* Sampling with frequency */ - -#define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config) -#define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc) -#define TEAR_REG(hwc) ((hwc)->last_tag) -#define SAMPL_RATE(hwc) ((hwc)->event_base) -#define SAMPL_FLAGS(hwc) ((hwc)->config_base) -#define SAMPL_DIAG_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_DIAG_MODE) -#define SAMPL_FREQ_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_FREQ_MODE) #define perf_arch_fetch_caller_regs(regs, __ip) do { \ (regs)->psw.addr = (__ip); \ diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index ca04a56e1520..a481f53c3338 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -24,6 +24,22 @@ #include #include +/* Perf PMU definitions for the sampling facility */ +#define PERF_CPUM_SF_MAX_CTR 2 +#define PERF_EVENT_CPUM_SF 0xB0000UL /* Event: Basic-sampling */ +#define PERF_EVENT_CPUM_SF_DIAG 0xBD000UL /* Event: Combined-sampling */ +#define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */ +#define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */ +#define PERF_CPUM_SF_FREQ_MODE 0x0008 /* Sampling with frequency */ + +#define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config) +#define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc) +#define TEAR_REG(hwc) ((hwc)->last_tag) +#define SAMPL_RATE(hwc) ((hwc)->event_base) +#define SAMPL_FLAGS(hwc) ((hwc)->config_base) +#define SAMPL_DIAG_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_DIAG_MODE) +#define SAMPL_FREQ_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_FREQ_MODE) + /* Minimum number of sample-data-block-tables: * At least one table is required for the sampling buffer structure. * A single table contains up to 511 pointers to sample-data-blocks. From d4559eabc1a61c1f9e99ed34489f060a2097598d Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 1 Jul 2024 10:59:09 +0200 Subject: [PATCH 06/84] s390/cpum_cf: Move defines from header file to source file The macros PERF_CPUM_CF_MAX_CTR and PERF_EVENT_CPUM_CF_DIAG are used in only one source file arch/s390/kernel/perf_cpum_cf.c. Move these defines from the header file arch/s390/include/asm/perf_event.h to the only user. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/perf_event.h | 4 ---- arch/s390/kernel/perf_cpum_cf.c | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 0fed84fe80d7..66200d4a2134 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -48,10 +48,6 @@ struct perf_sf_sde_regs { unsigned long reserved:63; /* reserved */ }; -/* Perf PMU definitions for the counter facility */ -#define PERF_CPUM_CF_MAX_CTR 0xffffUL /* Max ctr for ECCTR */ -#define PERF_EVENT_CPUM_CF_DIAG 0xBC000UL /* Event: Counter sets */ - #define perf_arch_fetch_caller_regs(regs, __ip) do { \ (regs)->psw.addr = (__ip); \ (regs)->gprs[15] = (unsigned long)__builtin_frame_address(0) - \ diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 6968be98af11..18b0d025f3a2 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -22,6 +22,10 @@ #include #include +/* Perf PMU definitions for the counter facility */ +#define PERF_CPUM_CF_MAX_CTR 0xffffUL /* Max ctr for ECCTR */ +#define PERF_EVENT_CPUM_CF_DIAG 0xBC000UL /* Event: Counter sets */ + enum cpumf_ctr_set { CPUMF_CTR_SET_BASIC = 0, /* Basic Counter Set */ CPUMF_CTR_SET_USER = 1, /* Problem-State Counter Set */ From b20182829001f945ac854e22f305eb92917d5a1a Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 25 Jul 2024 13:20:32 +0200 Subject: [PATCH 07/84] s390/cpum_sf: Use hwc as variable consistently In hw_perf_event_update() and cpumsf_pmu_enable() use variable hwc consistently to access event's hardware related data. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index a481f53c3338..1f9239d27da5 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1014,7 +1014,7 @@ static void cpumsf_pmu_enable(struct pmu *pmu) extend_sampling_buffer(&cpuhw->sfb, hwc); } /* Rate may be adjusted with ioctl() */ - cpuhw->lsctl.interval = SAMPL_RATE(&cpuhw->event->hw); + cpuhw->lsctl.interval = SAMPL_RATE(hwc); } /* (Re)enable the PMU and sampling facility */ @@ -1291,7 +1291,7 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) * AUX buffer is used when in diagnostic sampling mode. * No perf events/samples are created. */ - if (SAMPL_DIAG_MODE(&event->hw)) + if (SAMPL_DIAG_MODE(hwc)) return; sdbt = (unsigned long *)TEAR_REG(hwc); From 6bc565a99e77c8ced71c1184cebe4dda3bb4fa73 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 9 Jul 2024 10:55:04 +0200 Subject: [PATCH 08/84] s390/cpum_sf: Define and initialize variable Define and initialize a variable in one place. Remove space between cast and variable. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 1f9239d27da5..4d668b2ce64d 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1347,7 +1347,7 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) sdbt = get_next_sdbt(sdbt); /* Update event hardware registers */ - TEAR_REG(hwc) = (unsigned long) sdbt; + TEAR_REG(hwc) = (unsigned long)sdbt; /* Stop processing sample-data if all samples of the current * sample-data-block were flushed even if it was not full. @@ -1943,7 +1943,7 @@ static int cpumsf_pmu_add(struct perf_event *event, int flags) { struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); struct aux_buffer *aux; - int err; + int err = 0; if (cpuhw->flags & PMU_F_IN_USE) return -EAGAIN; @@ -1951,7 +1951,6 @@ static int cpumsf_pmu_add(struct perf_event *event, int flags) if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt) return -EINVAL; - err = 0; perf_pmu_disable(event->pmu); event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; From e09e58f425ad42726c008673b490aef53da10f1f Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 25 Jul 2024 14:13:06 +0200 Subject: [PATCH 09/84] s390/cpum_sf: Use variable name cpuhw consistently All functions but setup_pmc_cpu() use a local variable named cpuhw to refer to struct cpu_hw_sf. In setup_pmc_cpu() rename variable cpusf to cpuhw. This makes the naming scheme consistent with all other functions. No functional change. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 4d668b2ce64d..d87d58f2ec1b 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -610,23 +610,23 @@ static DEFINE_MUTEX(pmc_reserve_mutex); #define PMC_FAILURE 2 static void setup_pmc_cpu(void *flags) { - struct cpu_hw_sf *cpusf = this_cpu_ptr(&cpu_hw_sf); + struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); int err = 0; switch (*((int *)flags)) { case PMC_INIT: - memset(cpusf, 0, sizeof(*cpusf)); - err = qsi(&cpusf->qsi); + memset(cpuhw, 0, sizeof(*cpuhw)); + err = qsi(&cpuhw->qsi); if (err) break; - cpusf->flags |= PMU_F_RESERVED; + cpuhw->flags |= PMU_F_RESERVED; err = sf_disable(); break; case PMC_RELEASE: - cpusf->flags &= ~PMU_F_RESERVED; + cpuhw->flags &= ~PMU_F_RESERVED; err = sf_disable(); if (!err) - deallocate_buffers(cpusf); + deallocate_buffers(cpuhw); break; } if (err) { From d0e7915d2ad3c628488f76cb9156b203bd917d54 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 29 Jul 2024 17:09:40 +0200 Subject: [PATCH 10/84] s390/mm/ptdump: Generate address marker array dynamically Generate the address marker array dynamically instead of modifying a large static array at kernel startup. Each marker is added twice to the array: with and without a "start" indicator. This way the code and logic stays similar to other architectures. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/dump_pagetables.c | 191 +++++++++++++-------------------- 1 file changed, 77 insertions(+), 114 deletions(-) diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 0a67fcee4414..9e2dc42143b3 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -18,89 +18,12 @@ static unsigned long max_addr; struct addr_marker { int is_start; unsigned long start_address; + unsigned long size; const char *name; }; -enum address_markers_idx { - KVA_NR = 0, - LOWCORE_START_NR, - LOWCORE_END_NR, - AMODE31_START_NR, - AMODE31_END_NR, - KERNEL_START_NR, - KERNEL_END_NR, -#ifdef CONFIG_KFENCE - KFENCE_START_NR, - KFENCE_END_NR, -#endif - IDENTITY_START_NR, - IDENTITY_END_NR, - VMEMMAP_NR, - VMEMMAP_END_NR, - VMALLOC_NR, - VMALLOC_END_NR, -#ifdef CONFIG_KMSAN - KMSAN_VMALLOC_SHADOW_START_NR, - KMSAN_VMALLOC_SHADOW_END_NR, - KMSAN_VMALLOC_ORIGIN_START_NR, - KMSAN_VMALLOC_ORIGIN_END_NR, - KMSAN_MODULES_SHADOW_START_NR, - KMSAN_MODULES_SHADOW_END_NR, - KMSAN_MODULES_ORIGIN_START_NR, - KMSAN_MODULES_ORIGIN_END_NR, -#endif - MODULES_NR, - MODULES_END_NR, - ABS_LOWCORE_NR, - ABS_LOWCORE_END_NR, - MEMCPY_REAL_NR, - MEMCPY_REAL_END_NR, -#ifdef CONFIG_KASAN - KASAN_SHADOW_START_NR, - KASAN_SHADOW_END_NR, -#endif -}; - -static struct addr_marker address_markers[] = { - [KVA_NR] = {0, 0, "Kernel Virtual Address Space"}, - [LOWCORE_START_NR] = {1, 0, "Lowcore Start"}, - [LOWCORE_END_NR] = {0, 0, "Lowcore End"}, - [IDENTITY_START_NR] = {1, 0, "Identity Mapping Start"}, - [IDENTITY_END_NR] = {0, 0, "Identity Mapping End"}, - [AMODE31_START_NR] = {1, 0, "Amode31 Area Start"}, - [AMODE31_END_NR] = {0, 0, "Amode31 Area End"}, - [KERNEL_START_NR] = {1, (unsigned long)_stext, "Kernel Image Start"}, - [KERNEL_END_NR] = {0, (unsigned long)_end, "Kernel Image End"}, -#ifdef CONFIG_KFENCE - [KFENCE_START_NR] = {1, 0, "KFence Pool Start"}, - [KFENCE_END_NR] = {0, 0, "KFence Pool End"}, -#endif - [VMEMMAP_NR] = {1, 0, "vmemmap Area Start"}, - [VMEMMAP_END_NR] = {0, 0, "vmemmap Area End"}, - [VMALLOC_NR] = {1, 0, "vmalloc Area Start"}, - [VMALLOC_END_NR] = {0, 0, "vmalloc Area End"}, -#ifdef CONFIG_KMSAN - [KMSAN_VMALLOC_SHADOW_START_NR] = {1, 0, "Kmsan vmalloc Shadow Start"}, - [KMSAN_VMALLOC_SHADOW_END_NR] = {0, 0, "Kmsan vmalloc Shadow End"}, - [KMSAN_VMALLOC_ORIGIN_START_NR] = {1, 0, "Kmsan vmalloc Origins Start"}, - [KMSAN_VMALLOC_ORIGIN_END_NR] = {0, 0, "Kmsan vmalloc Origins End"}, - [KMSAN_MODULES_SHADOW_START_NR] = {1, 0, "Kmsan Modules Shadow Start"}, - [KMSAN_MODULES_SHADOW_END_NR] = {0, 0, "Kmsan Modules Shadow End"}, - [KMSAN_MODULES_ORIGIN_START_NR] = {1, 0, "Kmsan Modules Origins Start"}, - [KMSAN_MODULES_ORIGIN_END_NR] = {0, 0, "Kmsan Modules Origins End"}, -#endif - [MODULES_NR] = {1, 0, "Modules Area Start"}, - [MODULES_END_NR] = {0, 0, "Modules Area End"}, - [ABS_LOWCORE_NR] = {1, 0, "Lowcore Area Start"}, - [ABS_LOWCORE_END_NR] = {0, 0, "Lowcore Area End"}, - [MEMCPY_REAL_NR] = {1, 0, "Real Memory Copy Area Start"}, - [MEMCPY_REAL_END_NR] = {0, 0, "Real Memory Copy Area End"}, -#ifdef CONFIG_KASAN - [KASAN_SHADOW_START_NR] = {1, KASAN_SHADOW_START, "Kasan Shadow Start"}, - [KASAN_SHADOW_END_NR] = {0, KASAN_SHADOW_END, "Kasan Shadow End"}, -#endif - {1, -1UL, NULL} -}; +static struct addr_marker *markers; +static unsigned int markers_cnt; struct pg_state { struct ptdump_state ptdump; @@ -173,7 +96,8 @@ static void note_page_update_state(struct pg_state *st, unsigned long addr, unsi while (addr >= st->marker[1].start_address) { st->marker++; - pt_dump_seq_printf(m, "---[ %s ]---\n", st->marker->name); + pt_dump_seq_printf(m, "---[ %s %s ]---\n", st->marker->name, + st->marker->is_start ? "Start" : "End"); } st->start_address = addr; st->current_prot = prot; @@ -202,7 +126,7 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, if (level == -1) addr = max_addr; if (st->level == -1) { - pt_dump_seq_printf(m, "---[ %s ]---\n", st->marker->name); + pt_dump_seq_puts(m, "---[ Kernel Virtual Address Space ]---\n"); note_page_update_state(st, addr, prot, level); } else if (prot != st->current_prot || level != st->level || addr >= st->marker[1].start_address) { @@ -276,7 +200,7 @@ static int ptdump_show(struct seq_file *m, void *v) .check_wx = false, .wx_pages = 0, .start_address = 0, - .marker = address_markers, + .marker = markers, }; get_online_mems(); @@ -299,10 +223,23 @@ static int ptdump_cmp(const void *a, const void *b) if (ama->start_address < amb->start_address) return -1; /* - * If the start addresses of two markers are identical consider the - * marker which defines the start of an area higher than the one which - * defines the end of an area. This keeps pairs of markers sorted. + * If the start addresses of two markers are identical sort markers in an + * order that considers areas contained within other areas correctly. */ + if (ama->is_start && amb->is_start) { + if (ama->size > amb->size) + return -1; + if (ama->size < amb->size) + return 1; + return 0; + } + if (!ama->is_start && !amb->is_start) { + if (ama->size > amb->size) + return 1; + if (ama->size < amb->size) + return -1; + return 0; + } if (ama->is_start) return 1; if (amb->is_start) @@ -310,12 +247,41 @@ static int ptdump_cmp(const void *a, const void *b) return 0; } +static int add_marker(unsigned long start, unsigned long end, const char *name) +{ + size_t oldsize, newsize; + + oldsize = markers_cnt * sizeof(*markers); + newsize = oldsize + 2 * sizeof(*markers); + if (!oldsize) + markers = kvmalloc(newsize, GFP_KERNEL); + else + markers = kvrealloc(markers, oldsize, newsize, GFP_KERNEL); + if (!markers) + goto error; + markers[markers_cnt].is_start = 1; + markers[markers_cnt].start_address = start; + markers[markers_cnt].size = end - start; + markers[markers_cnt].name = name; + markers_cnt++; + markers[markers_cnt].is_start = 0; + markers[markers_cnt].start_address = end; + markers[markers_cnt].size = end - start; + markers[markers_cnt].name = name; + markers_cnt++; + return 0; +error: + markers_cnt = 0; + return -ENOMEM; +} + static int pt_dump_init(void) { #ifdef CONFIG_KFENCE unsigned long kfence_start = (unsigned long)__kfence_pool; #endif unsigned long lowcore = (unsigned long)get_lowcore(); + int rc; /* * Figure out the maximum virtual address being accessible with the @@ -324,41 +290,38 @@ static int pt_dump_init(void) */ max_addr = (get_lowcore()->kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); - address_markers[LOWCORE_START_NR].start_address = lowcore; - address_markers[LOWCORE_END_NR].start_address = lowcore + sizeof(struct lowcore); - address_markers[IDENTITY_START_NR].start_address = __identity_base; - address_markers[IDENTITY_END_NR].start_address = __identity_base + ident_map_size; - address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31; - address_markers[AMODE31_END_NR].start_address = (unsigned long)__eamode31; - address_markers[MODULES_NR].start_address = MODULES_VADDR; - address_markers[MODULES_END_NR].start_address = MODULES_END; - address_markers[ABS_LOWCORE_NR].start_address = __abs_lowcore; - address_markers[ABS_LOWCORE_END_NR].start_address = __abs_lowcore + ABS_LOWCORE_MAP_SIZE; - address_markers[MEMCPY_REAL_NR].start_address = __memcpy_real_area; - address_markers[MEMCPY_REAL_END_NR].start_address = __memcpy_real_area + MEMCPY_REAL_SIZE; - address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; - address_markers[VMEMMAP_END_NR].start_address = (unsigned long)vmemmap + vmemmap_size; - address_markers[VMALLOC_NR].start_address = VMALLOC_START; - address_markers[VMALLOC_END_NR].start_address = VMALLOC_END; + /* start + end markers - must be added first */ + rc = add_marker(0, -1UL, NULL); + rc |= add_marker((unsigned long)_stext, (unsigned long)_end, "Kernel Image"); + rc |= add_marker(lowcore, lowcore + sizeof(struct lowcore), "Lowcore"); + rc |= add_marker(__identity_base, __identity_base + ident_map_size, "Identity Mapping"); + rc |= add_marker((unsigned long)__samode31, (unsigned long)__eamode31, "Amode31 Area"); + rc |= add_marker(MODULES_VADDR, MODULES_END, "Modules Area"); + rc |= add_marker(__abs_lowcore, __abs_lowcore + ABS_LOWCORE_MAP_SIZE, "Lowcore Area"); + rc |= add_marker(__memcpy_real_area, __memcpy_real_area + MEMCPY_REAL_SIZE, "Real Memory Copy Area"); + rc |= add_marker((unsigned long)vmemmap, (unsigned long)vmemmap + vmemmap_size, "vmemmap Area"); + rc |= add_marker(VMALLOC_START, VMALLOC_END, "vmalloc Area"); #ifdef CONFIG_KFENCE - address_markers[KFENCE_START_NR].start_address = kfence_start; - address_markers[KFENCE_END_NR].start_address = kfence_start + KFENCE_POOL_SIZE; + rc |= add_marker(kfence_start, kfence_start + KFENCE_POOL_SIZE, "KFence Pool"); #endif #ifdef CONFIG_KMSAN - address_markers[KMSAN_VMALLOC_SHADOW_START_NR].start_address = KMSAN_VMALLOC_SHADOW_START; - address_markers[KMSAN_VMALLOC_SHADOW_END_NR].start_address = KMSAN_VMALLOC_SHADOW_END; - address_markers[KMSAN_VMALLOC_ORIGIN_START_NR].start_address = KMSAN_VMALLOC_ORIGIN_START; - address_markers[KMSAN_VMALLOC_ORIGIN_END_NR].start_address = KMSAN_VMALLOC_ORIGIN_END; - address_markers[KMSAN_MODULES_SHADOW_START_NR].start_address = KMSAN_MODULES_SHADOW_START; - address_markers[KMSAN_MODULES_SHADOW_END_NR].start_address = KMSAN_MODULES_SHADOW_END; - address_markers[KMSAN_MODULES_ORIGIN_START_NR].start_address = KMSAN_MODULES_ORIGIN_START; - address_markers[KMSAN_MODULES_ORIGIN_END_NR].start_address = KMSAN_MODULES_ORIGIN_END; + rc |= add_marker(KMSAN_VMALLOC_SHADOW_START, KMSAN_VMALLOC_SHADOW_END, "Kmsan vmalloc Shadow"); + rc |= add_marker(KMSAN_VMALLOC_ORIGIN_START, KMSAN_VMALLOC_ORIGIN_END, "Kmsan vmalloc Origins"); + rc |= add_marker(KMSAN_MODULES_SHADOW_START, KMSAN_MODULES_SHADOW_END, "Kmsan Modules Shadow"); + rc |= add_marker(KMSAN_MODULES_ORIGIN_START, KMSAN_MODULES_ORIGIN_END, "Kmsan Modules Origins"); #endif - sort(address_markers, ARRAY_SIZE(address_markers) - 1, - sizeof(address_markers[0]), ptdump_cmp, NULL); +#ifdef CONFIG_KASAN + rc |= add_marker(KASAN_SHADOW_START, KASAN_SHADOW_END, "Kasan Shadow"); +#endif + if (rc) + goto error; + sort(&markers[1], markers_cnt - 1, sizeof(*markers), ptdump_cmp, NULL); #ifdef CONFIG_PTDUMP_DEBUGFS debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, &ptdump_fops); #endif /* CONFIG_PTDUMP_DEBUGFS */ return 0; +error: + kvfree(markers); + return -ENOMEM; } device_initcall(pt_dump_init); From f2bb5b97b51ce5425e8f59f899643ce4eadba667 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 31 Jul 2024 15:25:59 +0200 Subject: [PATCH 11/84] s390/entry: Move early program check handler to entry.S Have all program check handlers in one file to make future changes easy. Reviewed-by: Alexander Gordeev Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/earlypgm.S | 23 ----------------------- arch/s390/kernel/entry.S | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 24 deletions(-) delete mode 100644 arch/s390/kernel/earlypgm.S diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index e47a4be54ff8..a70f25e9c17d 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -36,7 +36,7 @@ CFLAGS_stacktrace.o += -fno-optimize-sibling-calls CFLAGS_dumpstack.o += -fno-optimize-sibling-calls CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls -obj-y := head64.o traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o +obj-y := head64.o traps.o time.o process.o early.o setup.o idle.o vtime.o obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o cpufeature.o obj-y += sysinfo.o lgr.o os_info.o ctlreg.o diff --git a/arch/s390/kernel/earlypgm.S b/arch/s390/kernel/earlypgm.S deleted file mode 100644 index c634871f0d90..000000000000 --- a/arch/s390/kernel/earlypgm.S +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright IBM Corp. 2006, 2007 - * Author(s): Michael Holzheu - */ - -#include -#include - -SYM_CODE_START(early_pgm_check_handler) - stmg %r8,%r15,__LC_SAVE_AREA_SYNC - aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) - la %r11,STACK_FRAME_OVERHEAD(%r15) - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC - lgr %r2,%r11 - brasl %r14,__do_early_pgm_check - mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) - lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) - lpswe __LC_RETURN_PSW -SYM_CODE_END(early_pgm_check_handler) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 749410cfdbc0..b01f4ac43f72 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -599,6 +599,21 @@ SYM_CODE_START(restart_int_handler) 3: j 3b SYM_CODE_END(restart_int_handler) +SYM_CODE_START(early_pgm_check_handler) + stmg %r8,%r15,__LC_SAVE_AREA_SYNC + aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) + la %r11,STACK_FRAME_OVERHEAD(%r15) + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC + lgr %r2,%r11 + brasl %r14,__do_early_pgm_check + mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) + lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) + lpswe __LC_RETURN_PSW +SYM_CODE_END(early_pgm_check_handler) + .section .kprobes.text, "ax" #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK) From f101b305a7b9513a8042a2cf09018de4ff371af2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 31 Jul 2024 15:26:00 +0200 Subject: [PATCH 12/84] s390/entry: Make early program check handler relocated lowcore aware Add the missing pieces so the early program check handler also works with a relocated lowcore. Right now the result of an early program check in case of a relocated lowcore would be a program check loop. Fixes: 8f1e70adb1a3 ("s390/boot: Add cmdline option to relocate lowcore") Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 7 +++++-- arch/s390/kernel/entry.S | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 14d324865e33..ee051ad81c71 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -183,12 +183,15 @@ void __do_early_pgm_check(struct pt_regs *regs) static noinline __init void setup_lowcore_early(void) { + struct lowcore *lc = get_lowcore(); psw_t psw; psw.addr = (unsigned long)early_pgm_check_handler; psw.mask = PSW_KERNEL_BITS; - get_lowcore()->program_new_psw = psw; - get_lowcore()->preempt_count = INIT_PREEMPT_COUNT; + lc->program_new_psw = psw; + lc->preempt_count = INIT_PREEMPT_COUNT; + lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW); + lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW); } static __init void detect_diag9c(void) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index b01f4ac43f72..6539ec4800cd 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -600,18 +600,19 @@ SYM_CODE_START(restart_int_handler) SYM_CODE_END(restart_int_handler) SYM_CODE_START(early_pgm_check_handler) - stmg %r8,%r15,__LC_SAVE_AREA_SYNC + STMG_LC %r8,%r15,__LC_SAVE_AREA_SYNC + GET_LC %r13 aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) la %r11,STACK_FRAME_OVERHEAD(%r15) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC + mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW(%r13) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC(%r13) lgr %r2,%r11 brasl %r14,__do_early_pgm_check - mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) + mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) - lpswe __LC_RETURN_PSW + LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE SYM_CODE_END(early_pgm_check_handler) .section .kprobes.text, "ax" From 3c4d0ae0671827f4b536cc2d26f8b9c54584ccc5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 31 Jul 2024 15:26:01 +0200 Subject: [PATCH 13/84] s390/traps: Handle early warnings gracefully Add missing warning handling to the early program check handler. This way a warning is printed to the console as soon as the early console is setup, and the kernel continues to boot. Before this change a disabled wait psw was loaded instead and the machine was silently stopped without giving an idea about what happened. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index ee051ad81c71..0242fa78c918 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -177,8 +177,21 @@ static __init void setup_topology(void) void __do_early_pgm_check(struct pt_regs *regs) { - if (!fixup_exception(regs)) - disabled_wait(); + struct lowcore *lc = get_lowcore(); + unsigned long ip; + + regs->int_code = lc->pgm_int_code; + regs->int_parm_long = lc->trans_exc_code; + ip = __rewind_psw(regs->psw, regs->int_code >> 16); + + /* Monitor Event? Might be a warning */ + if ((regs->int_code & PGM_INT_CODE_MASK) == 0x40) { + if (report_bug(ip, regs) == BUG_TRAP_TYPE_WARN) + return; + } + if (fixup_exception(regs)) + return; + disabled_wait(); } static noinline __init void setup_lowcore_early(void) From 85878ff1b31f376a37b885824851fa8c1c3ae048 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Aug 2024 11:48:26 +0200 Subject: [PATCH 14/84] s390/entry: Move early_pgm_check_handler() to init text section Save some bytes and move early_pgm_check_handler() to init text section. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/entry.S | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 6539ec4800cd..36424e53d63b 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -599,6 +599,7 @@ SYM_CODE_START(restart_int_handler) 3: j 3b SYM_CODE_END(restart_int_handler) + __INIT SYM_CODE_START(early_pgm_check_handler) STMG_LC %r8,%r15,__LC_SAVE_AREA_SYNC GET_LC %r13 @@ -614,6 +615,7 @@ SYM_CODE_START(early_pgm_check_handler) lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE SYM_CODE_END(early_pgm_check_handler) + __FINIT .section .kprobes.text, "ax" From 391b8a6ce12891815491fd51a00619cab2589fe4 Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Tue, 16 Jul 2024 14:16:35 +0200 Subject: [PATCH 15/84] s390/ap_bus: Cleanup debug code The dynamic debugging provides function names on request. So remove all explicit function strings. Reviewed-by: Harald Freudenberger Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.c | 42 +++++++++++++++++------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 0998b17ecb37..ebc3fe2a7f40 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -733,7 +733,7 @@ static void ap_check_bindings_complete(void) if (!completion_done(&ap_apqn_bindings_complete)) { complete_all(&ap_apqn_bindings_complete); ap_send_bindings_complete_uevent(); - pr_debug("%s all apqn bindings complete\n", __func__); + pr_debug("all apqn bindings complete\n"); } } } @@ -768,7 +768,7 @@ int ap_wait_apqn_bindings_complete(unsigned long timeout) else if (l == 0 && timeout) rc = -ETIME; - pr_debug("%s rc=%d\n", __func__, rc); + pr_debug("rc=%d\n", rc); return rc; } EXPORT_SYMBOL(ap_wait_apqn_bindings_complete); @@ -795,8 +795,7 @@ static int __ap_revise_reserved(struct device *dev, void *dummy) drvres = to_ap_drv(dev->driver)->flags & AP_DRIVER_FLAG_DEFAULT; if (!!devres != !!drvres) { - pr_debug("%s reprobing queue=%02x.%04x\n", - __func__, card, queue); + pr_debug("reprobing queue=%02x.%04x\n", card, queue); rc = device_reprobe(dev); if (rc) AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", @@ -995,7 +994,7 @@ bool ap_bus_force_rescan(void) unsigned long scan_counter = atomic64_read(&ap_scan_bus_count); bool rc = false; - pr_debug(">%s scan counter=%lu\n", __func__, scan_counter); + pr_debug("> scan counter=%lu\n", scan_counter); /* Only trigger AP bus scans after the initial scan is done */ if (scan_counter <= 0) @@ -1024,7 +1023,7 @@ bool ap_bus_force_rescan(void) mutex_unlock(&ap_scan_bus_mutex); out: - pr_debug("%s rc=%d\n", __func__, rc); + pr_debug("rc=%d\n", rc); return rc; } EXPORT_SYMBOL(ap_bus_force_rescan); @@ -1038,7 +1037,7 @@ static int ap_bus_cfg_chg(struct notifier_block *nb, if (action != CHSC_NOTIFY_AP_CFG) return NOTIFY_DONE; - pr_debug("%s config change, forcing bus rescan\n", __func__); + pr_debug("config change, forcing bus rescan\n"); ap_bus_force_rescan(); @@ -1895,8 +1894,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED; } spin_unlock_bh(&aq->lock); - pr_debug("%s(%d,%d) queue dev checkstop on\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev checkstop on\n", + ac->id, dom); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); goto put_dev_and_continue; @@ -1906,8 +1905,8 @@ static inline void ap_scan_domains(struct ap_card *ac) if (aq->dev_state > AP_DEV_STATE_UNINITIATED) _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - pr_debug("%s(%d,%d) queue dev checkstop off\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev checkstop off\n", + ac->id, dom); goto put_dev_and_continue; } /* config state change */ @@ -1919,8 +1918,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_DECONFIGURED; } spin_unlock_bh(&aq->lock); - pr_debug("%s(%d,%d) queue dev config off\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev config off\n", + ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); @@ -1931,8 +1930,8 @@ static inline void ap_scan_domains(struct ap_card *ac) if (aq->dev_state > AP_DEV_STATE_UNINITIATED) _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - pr_debug("%s(%d,%d) queue dev config on\n", - __func__, ac->id, dom); + pr_debug("(%d,%d) queue dev config on\n", + ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); goto put_dev_and_continue; } @@ -2004,8 +2003,8 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - pr_debug("%s(%d) no type info (no APQN found), ignored\n", - __func__, ap); + pr_debug("(%d) no type info (no APQN found), ignored\n", + ap); } return; } @@ -2017,8 +2016,7 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - pr_debug("%s(%d) no valid type (0) info, ignored\n", - __func__, ap); + pr_debug("(%d) no valid type (0) info, ignored\n", ap); } return; } @@ -2197,7 +2195,7 @@ static bool ap_scan_bus(void) bool config_changed; int ap; - pr_debug(">%s\n", __func__); + pr_debug(">\n"); /* (re-)fetch configuration via QCI */ config_changed = ap_get_configuration(); @@ -2238,7 +2236,7 @@ static bool ap_scan_bus(void) } if (atomic64_inc_return(&ap_scan_bus_count) == 1) { - pr_debug("%s init scan complete\n", __func__); + pr_debug("init scan complete\n"); ap_send_init_scan_done_uevent(); } @@ -2246,7 +2244,7 @@ static bool ap_scan_bus(void) mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); - pr_debug("<%s config_changed=%d\n", __func__, config_changed); + pr_debug("< config_changed=%d\n", config_changed); return config_changed; } From ea31f0f6e251db7408cbe7500e97bc9d1694910c Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Tue, 16 Jul 2024 14:16:36 +0200 Subject: [PATCH 16/84] s390/ap_queue: Cleanup debug code The dynamic debugging provides function names on request. So remove all explicit function strings. Reviewed-by: Harald Freudenberger Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_queue.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 1f647ffd6f4d..8c878c5aa31f 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -171,8 +171,8 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) aq->queue_count = 0; list_splice_init(&aq->pendingq, &aq->requestq); aq->requestq_count += aq->pendingq_count; - pr_debug("%s queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n", - __func__, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), + pr_debug("queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n", + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), aq->pendingq_count, aq->requestq_count); aq->pendingq_count = 0; break; @@ -453,8 +453,8 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) case AP_BS_Q_USABLE: /* association is through */ aq->sm_state = AP_SM_STATE_IDLE; - pr_debug("%s queue 0x%02x.%04x associated with %u\n", - __func__, AP_QID_CARD(aq->qid), + pr_debug("queue 0x%02x.%04x associated with %u\n", + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), aq->assoc_idx); return AP_SM_WAIT_NONE; case AP_BS_Q_USABLE_NO_SECURE_KEY: @@ -697,8 +697,8 @@ static ssize_t ap_functions_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, + pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n", + status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } @@ -853,8 +853,8 @@ static ssize_t se_bind_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, + pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n", + status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } @@ -981,8 +981,8 @@ static ssize_t se_associate_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, + pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n", + status.response_code, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } From 1849850e8177b60ae67be4ae1a0bebaaaabcb4d7 Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Tue, 16 Jul 2024 14:16:37 +0200 Subject: [PATCH 17/84] s390/zcrypt_api: Cleanup debug code The dynamic debugging provides function names on request. So remove all explicit function strings. Reviewed-by: Harald Freudenberger Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_api.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 74036886ca87..f9a47b54c51a 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -715,7 +715,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - pr_debug("%s no matching queue found => ENODEV\n", __func__); + pr_debug("no matching queue found => ENODEV\n"); rc = -ENODEV; goto out; } @@ -819,7 +819,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - pr_debug("%s no matching queue found => ENODEV\n", __func__); + pr_debug("no matching queue found => ENODEV\n"); rc = -ENODEV; goto out; } @@ -940,8 +940,8 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - pr_debug("%s no match for address %02x.%04x => ENODEV\n", - __func__, xcrb->user_defined, *domain); + pr_debug("no match for address %02x.%04x => ENODEV\n", + xcrb->user_defined, *domain); rc = -ENODEV; goto out; } @@ -991,7 +991,7 @@ long zcrypt_send_cprb(struct ica_xcRB *xcrb) if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - pr_debug("%s rc=%d\n", __func__, rc); + pr_debug("rc=%d\n", rc); return rc; } @@ -1138,15 +1138,13 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, if (!pref_zq) { if (targets && target_num == 1) { - pr_debug("%s no match for address %02x.%04x => ENODEV\n", - __func__, (int)targets->ap_id, - (int)targets->dom_id); + pr_debug("no match for address %02x.%04x => ENODEV\n", + (int)targets->ap_id, (int)targets->dom_id); } else if (targets) { - pr_debug("%s no match for %d target addrs => ENODEV\n", - __func__, (int)target_num); + pr_debug("no match for %d target addrs => ENODEV\n", + (int)target_num); } else { - pr_debug("%s no match for address ff.ffff => ENODEV\n", - __func__); + pr_debug("no match for address ff.ffff => ENODEV\n"); } rc = -ENODEV; goto out_free; @@ -1195,7 +1193,7 @@ long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb) if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - pr_debug("%s rc=%d\n", __func__, rc); + pr_debug("rc=%d\n", rc); return rc; } @@ -1247,7 +1245,7 @@ static long zcrypt_rng(char *buffer) spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - pr_debug("%s no matching queue found => ENODEV\n", __func__); + pr_debug("no matching queue found => ENODEV\n"); rc = -ENODEV; goto out; } @@ -2037,8 +2035,7 @@ int zcrypt_wait_api_operational(void) break; default: /* other failure */ - pr_debug("%s ap_wait_init_apqn_bindings_complete()=%d\n", - __func__, rc); + pr_debug("ap_wait_init_apqn_bindings_complete()=%d\n", rc); break; } break; From a7a88eeae310f2acb564bb3e747cd3739daccc10 Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Tue, 16 Jul 2024 14:16:38 +0200 Subject: [PATCH 18/84] s390/zcrypt_msgtype50: Cleanup debug code The dynamic debugging provides function names on request. So remove all explicit function strings. Reviewed-by: Harald Freudenberger Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_msgtype50.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index 3b39cb8f926d..adc65eddaa1e 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -427,7 +427,7 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, len = t80h->len; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - pr_debug("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -487,8 +487,8 @@ static long zcrypt_msgtype50_modexpo(struct zcrypt_queue *zq, out: ap_msg->private = NULL; if (rc) - pr_debug("%s send me cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), + pr_debug("send me cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -537,8 +537,8 @@ static long zcrypt_msgtype50_modexpo_crt(struct zcrypt_queue *zq, out: ap_msg->private = NULL; if (rc) - pr_debug("%s send crt cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), + pr_debug("send crt cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), rc); return rc; } From 073ef6b2041075e08c780db34f67e6daf884c9f8 Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Tue, 16 Jul 2024 14:16:39 +0200 Subject: [PATCH 19/84] s390/zcrypt_msgtype6: Cleanup debug code The dynamic debugging provides function names on request. So remove all explicit function strings. Reviewed-by: Harald Freudenberger [dengler: fix indent] Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_msgtype6.c | 37 +++++++++++---------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 215f257d2360..b64c9d9fc613 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -437,9 +437,8 @@ static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg, ap_msg->flags |= AP_MSG_FLAG_ADMIN; break; default: - pr_debug("%s unknown CPRB minor version '%c%c'\n", - __func__, msg->cprbx.func_id[0], - msg->cprbx.func_id[1]); + pr_debug("unknown CPRB minor version '%c%c'\n", + msg->cprbx.func_id[0], msg->cprbx.func_id[1]); } /* copy data block */ @@ -629,9 +628,8 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy CPRB to user */ if (xcrb->reply_control_blk_length < msg->fmt2.count1) { - pr_debug("%s reply_control_blk_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_control_blk_length, - msg->fmt2.count1); + pr_debug("reply_control_blk_length %u < required %u => EMSGSIZE\n", + xcrb->reply_control_blk_length, msg->fmt2.count1); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_control_blk_addr, @@ -642,9 +640,8 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy data buffer to user */ if (msg->fmt2.count2) { if (xcrb->reply_data_length < msg->fmt2.count2) { - pr_debug("%s reply_data_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_data_length, - msg->fmt2.count2); + pr_debug("reply_data_length %u < required %u => EMSGSIZE\n", + xcrb->reply_data_length, msg->fmt2.count2); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_data_addr, @@ -673,9 +670,8 @@ static int convert_type86_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, char *data = reply->msg; if (xcrb->resp_len < msg->fmt2.count1) { - pr_debug("%s resp_len %u < required %u => EMSGSIZE\n", - __func__, (unsigned int)xcrb->resp_len, - msg->fmt2.count1); + pr_debug("resp_len %u < required %u => EMSGSIZE\n", + (unsigned int)xcrb->resp_len, msg->fmt2.count1); return -EMSGSIZE; } @@ -875,8 +871,7 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = sizeof(struct type86x_reply) + t86r->length; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - pr_debug("%s len mismatch => EMSGSIZE\n", - __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -890,8 +885,7 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - pr_debug("%s len mismatch => EMSGSIZE\n", - __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -941,8 +935,7 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - pr_debug("%s len mismatch => EMSGSIZE\n", - __func__); + pr_debug("len mismatch => EMSGSIZE\n"); msg->rc = -EMSGSIZE; goto out; } @@ -1154,8 +1147,8 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, out: if (rc) - pr_debug("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), + pr_debug("send cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -1277,8 +1270,8 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue * out: if (rc) - pr_debug("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), + pr_debug("send cprb at dev=%02x.%04x rc=%d\n", + AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), rc); return rc; } From 742a7557164bbde9f3263aeebc581fb985fe7dbc Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 8 Aug 2024 10:58:56 +0200 Subject: [PATCH 20/84] s390/cpum_sf: Ignore lsctl() return code in sf_disable() sf_disable() returns the condition code of instruction lsctl (load sampling controls). However the parameter to lsctl() in sf_disable() is a sample control block containing all zeroes. This invocation of lsctl() does not fail and returns always zero even when there is no authorization for sampling on the machine. In short, sampling can be always turned off. Ignore the return code of sf_disable() and change the function return to void. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index d87d58f2ec1b..13ab77e5ef21 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -170,12 +170,12 @@ static inline unsigned long *get_next_sdbt(unsigned long *s) /* * sf_disable() - Switch off sampling facility */ -static int sf_disable(void) +static void sf_disable(void) { struct hws_lsctl_request_block sreq; memset(&sreq, 0, sizeof(sreq)); - return lsctl(&sreq); + lsctl(&sreq); } /* @@ -620,13 +620,12 @@ static void setup_pmc_cpu(void *flags) if (err) break; cpuhw->flags |= PMU_F_RESERVED; - err = sf_disable(); + sf_disable(); break; case PMC_RELEASE: cpuhw->flags &= ~PMU_F_RESERVED; - err = sf_disable(); - if (!err) - deallocate_buffers(cpuhw); + sf_disable(); + deallocate_buffers(cpuhw); break; } if (err) { From 6d9a732d8a4ab4a56eb7b4b9922f85fedc5827dd Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 8 Aug 2024 10:58:57 +0200 Subject: [PATCH 21/84] s390/cpum_sf: Ignore qsi() return code qsi() executes the instruction qsi (query sample information) and stores the result of the query in a sample information block pointed to by the function argument. The instruction does not change the condition code register. The return code is always zero. No need to check for errors. Remove now unreferenced macros PMC_FAILURE and RS_INIT_FAILURE_QSI. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 79 ++++++++++++--------------------- 1 file changed, 28 insertions(+), 51 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 13ab77e5ef21..8ac28bc41b5b 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -393,8 +393,8 @@ static void sfb_set_limits(unsigned long min, unsigned long max) CPUM_SF_MAX_SDB = max; memset(&si, 0, sizeof(si)); - if (!qsi(&si)) - CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes); + qsi(&si); + CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes); } static unsigned long sfb_max_limit(struct hw_perf_event *hwc) @@ -607,18 +607,14 @@ static DEFINE_MUTEX(pmc_reserve_mutex); #define PMC_INIT 0 #define PMC_RELEASE 1 -#define PMC_FAILURE 2 static void setup_pmc_cpu(void *flags) { struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); - int err = 0; switch (*((int *)flags)) { case PMC_INIT: memset(cpuhw, 0, sizeof(*cpuhw)); - err = qsi(&cpuhw->qsi); - if (err) - break; + qsi(&cpuhw->qsi); cpuhw->flags |= PMU_F_RESERVED; sf_disable(); break; @@ -628,10 +624,6 @@ static void setup_pmc_cpu(void *flags) deallocate_buffers(cpuhw); break; } - if (err) { - *((int *)flags) |= PMC_FAILURE; - pr_err("Switching off the sampling facility failed with rc %i\n", err); - } } static void release_pmc_hardware(void) @@ -642,18 +634,12 @@ static void release_pmc_hardware(void) on_each_cpu(setup_pmc_cpu, &flags, 1); } -static int reserve_pmc_hardware(void) +static void reserve_pmc_hardware(void) { int flags = PMC_INIT; on_each_cpu(setup_pmc_cpu, &flags, 1); - if (flags & PMC_FAILURE) { - release_pmc_hardware(); - return -ENODEV; - } irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); - - return 0; } static void hw_perf_event_destroy(struct perf_event *event) @@ -828,13 +814,10 @@ static int __hw_perf_event_init(struct perf_event *event) /* Reserve CPU-measurement sampling facility */ mutex_lock(&pmc_reserve_mutex); if (!refcount_inc_not_zero(&num_events)) { - err = reserve_pmc_hardware(); - if (!err) - refcount_set(&num_events, 1); + reserve_pmc_hardware(); + refcount_set(&num_events, 1); } mutex_unlock(&pmc_reserve_mutex); - if (err) - goto out; event->destroy = hw_perf_event_destroy; /* Access per-CPU sampling information (query sampling info) */ @@ -848,9 +831,9 @@ static int __hw_perf_event_init(struct perf_event *event) */ memset(&si, 0, sizeof(si)); cpuhw = NULL; - if (event->cpu == -1) + if (event->cpu == -1) { qsi(&si); - else { + } else { /* Event is pinned to a particular CPU, retrieve the per-CPU * sampling structure for accessing the CPU-specific QSI. */ @@ -1061,21 +1044,18 @@ static void cpumsf_pmu_disable(struct pmu *pmu) return; } - /* Save state of TEAR and DEAR register contents */ - err = qsi(&si); - if (!err) { - /* TEAR/DEAR values are valid only if the sampling facility is - * enabled. Note that cpumsf_pmu_disable() might be called even - * for a disabled sampling facility because cpumsf_pmu_enable() - * controls the enable/disable state. - */ - if (si.es) { - cpuhw->lsctl.tear = si.tear; - cpuhw->lsctl.dear = si.dear; - } - } else - debug_sprintf_event(sfdbg, 3, "%s: qsi() failed with err %i\n", - __func__, err); + /* + * Save state of TEAR and DEAR register contents. + * TEAR/DEAR values are valid only if the sampling facility is + * enabled. Note that cpumsf_pmu_disable() might be called even + * for a disabled sampling facility because cpumsf_pmu_enable() + * controls the enable/disable state. + */ + qsi(&si); + if (si.es) { + cpuhw->lsctl.tear = si.tear; + cpuhw->lsctl.dear = si.dear; + } cpuhw->flags &= ~PMU_F_ENABLED; } @@ -1869,8 +1849,7 @@ static int cpumsf_pmu_check_period(struct perf_event *event, u64 value) memset(&si, 0, sizeof(si)); if (event->cpu == -1) { - if (qsi(&si)) - return -ENODEV; + qsi(&si); } else { /* Event is pinned to a particular CPU, retrieve the per-CPU * sampling structure for accessing the CPU-specific QSI. @@ -2210,10 +2189,12 @@ static const struct kernel_param_ops param_ops_sfb_size = { .get = param_get_sfb_size, }; -#define RS_INIT_FAILURE_QSI 0x0001 -#define RS_INIT_FAILURE_BSDES 0x0002 -#define RS_INIT_FAILURE_ALRT 0x0003 -#define RS_INIT_FAILURE_PERF 0x0004 +enum { + RS_INIT_FAILURE_BSDES = 2, /* Bad basic sampling size */ + RS_INIT_FAILURE_ALRT = 3, /* IRQ registration failure */ + RS_INIT_FAILURE_PERF = 4 /* PMU registration failure */ +}; + static void __init pr_cpumsf_err(unsigned int reason) { pr_err("Sampling facility support for perf is not available: " @@ -2229,11 +2210,7 @@ static int __init init_cpum_sampling_pmu(void) return -ENODEV; memset(&si, 0, sizeof(si)); - if (qsi(&si)) { - pr_cpumsf_err(RS_INIT_FAILURE_QSI); - return -ENODEV; - } - + qsi(&si); if (!si.as && !si.ad) return -ENODEV; From 14a34130e030b5f597e1d688498a4b9679521b38 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 25 Jul 2024 15:32:26 +0200 Subject: [PATCH 22/84] s390/cpum_sf: Rework debug_sprintf_event() messages Rework debug messages: - Remove most of the debug_sprintf_event() invocations. - Do not split string format statements - Remove colon after function name. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 147 +++----------------------------- 1 file changed, 10 insertions(+), 137 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 8ac28bc41b5b..8d49f24f1bf6 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -129,17 +129,6 @@ static inline unsigned long sample_rate_to_freq(struct hws_qsi_info_block *qsi, return USEC_PER_SEC * qsi->cpu_speed / rate; } -/* Return TOD timestamp contained in an trailer entry */ -static inline unsigned long long trailer_timestamp(struct hws_trailer_entry *te) -{ - /* TOD in STCKE format */ - if (te->header.t) - return *((unsigned long long *)&te->timestamp[1]); - - /* TOD in STCK format */ - return *((unsigned long long *)&te->timestamp[0]); -} - /* Return pointer to trailer entry of an sample data block */ static inline struct hws_trailer_entry *trailer_entry_ptr(unsigned long v) { @@ -224,8 +213,6 @@ static void free_sampling_buffer(struct sf_buffer *sfb) } } - debug_sprintf_event(sfdbg, 5, "%s: freed sdbt %#lx\n", __func__, - (unsigned long)sfb->sdbt); memset(sfb, 0, sizeof(*sfb)); } @@ -281,10 +268,8 @@ static int realloc_sampling_buffer(struct sf_buffer *sfb, * the sampling buffer origin. */ if (sfb->sdbt != get_next_sdbt(tail)) { - debug_sprintf_event(sfdbg, 3, "%s: " - "sampling buffer is not linked: origin %#lx" - " tail %#lx\n", __func__, - (unsigned long)sfb->sdbt, + debug_sprintf_event(sfdbg, 3, "%s buffer not linked origin %#lx tail %#lx\n", + __func__, (unsigned long)sfb->sdbt, (unsigned long)tail); return -EINVAL; } @@ -334,9 +319,6 @@ static int realloc_sampling_buffer(struct sf_buffer *sfb, *tail = virt_to_phys(sfb->sdbt) + 1; sfb->tail = tail; - debug_sprintf_event(sfdbg, 4, "%s: new buffer" - " settings: sdbt %lu sdb %lu\n", __func__, - sfb->num_sdbt, sfb->num_sdb); return rc; } @@ -373,15 +355,8 @@ static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb) /* Allocate requested number of sample-data-blocks */ rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL); - if (rc) { + if (rc) free_sampling_buffer(sfb); - debug_sprintf_event(sfdbg, 4, "%s: " - "realloc_sampling_buffer failed with rc %i\n", - __func__, rc); - } else - debug_sprintf_event(sfdbg, 4, - "%s: tear %#lx dear %#lx\n", __func__, - (unsigned long)sfb->sdbt, (unsigned long)*sfb->sdbt); return rc; } @@ -413,12 +388,6 @@ static unsigned long sfb_pending_allocs(struct sf_buffer *sfb, return 0; } -static int sfb_has_pending_allocs(struct sf_buffer *sfb, - struct hw_perf_event *hwc) -{ - return sfb_pending_allocs(sfb, hwc) > 0; -} - static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc) { /* Limit the number of SDBs to not exceed the maximum */ @@ -442,7 +411,6 @@ static void deallocate_buffers(struct cpu_hw_sf *cpuhw) static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc) { unsigned long n_sdb, freq; - size_t sample_size; /* Calculate sampling buffers using 4K pages * @@ -473,7 +441,6 @@ static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc) * ensure a minimum of CPUM_SF_MIN_SDBT (one table can manage up * to 511 SDBs). */ - sample_size = sizeof(struct hws_basic_entry); freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); n_sdb = CPUM_SF_MIN_SDB + DIV_ROUND_UP(freq, 10000); @@ -489,12 +456,6 @@ static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc) if (sf_buffer_available(cpuhw)) return 0; - debug_sprintf_event(sfdbg, 3, - "%s: rate %lu f %lu sdb %lu/%lu" - " sample_size %lu cpuhw %p\n", __func__, - SAMPL_RATE(hwc), freq, n_sdb, sfb_max_limit(hwc), - sample_size, cpuhw); - return alloc_sampling_buffer(&cpuhw->sfb, sfb_pending_allocs(&cpuhw->sfb, hwc)); } @@ -551,8 +512,6 @@ static void sfb_account_overflows(struct cpu_hw_sf *cpuhw, if (num) sfb_account_allocs(num, hwc); - debug_sprintf_event(sfdbg, 5, "%s: overflow %llu ratio %lu num %lu\n", - __func__, OVERFLOW_REG(hwc), ratio, num); OVERFLOW_REG(hwc) = 0; } @@ -570,13 +529,11 @@ static void sfb_account_overflows(struct cpu_hw_sf *cpuhw, static void extend_sampling_buffer(struct sf_buffer *sfb, struct hw_perf_event *hwc) { - unsigned long num, num_old; - int rc; + unsigned long num; num = sfb_pending_allocs(sfb, hwc); if (!num) return; - num_old = sfb->num_sdb; /* Disable the sampling facility to reset any states and also * clear pending measurement alerts. @@ -588,16 +545,7 @@ static void extend_sampling_buffer(struct sf_buffer *sfb, * called by perf. Because this is a reallocation, it is fine if the * new SDB-request cannot be satisfied immediately. */ - rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC); - if (rc) - debug_sprintf_event(sfdbg, 5, "%s: realloc failed with rc %i\n", - __func__, rc); - - if (sfb_has_pending_allocs(sfb, hwc)) - debug_sprintf_event(sfdbg, 5, "%s: " - "req %lu alloc %lu remaining %lu\n", - __func__, num, sfb->num_sdb - num_old, - sfb_pending_allocs(sfb, hwc)); + realloc_sampling_buffer(sfb, num, GFP_ATOMIC); } /* Number of perf events counting hardware events */ @@ -750,9 +698,6 @@ static unsigned long getrate(bool freq, unsigned long sample, */ if (sample_rate_to_freq(si, rate) > sysctl_perf_event_sample_rate) { - debug_sprintf_event(sfdbg, 1, "%s: " - "Sampling rate exceeds maximum " - "perf sample rate\n", __func__); rate = 0; } } @@ -797,9 +742,6 @@ static int __hw_perf_event_init_rate(struct perf_event *event, attr->sample_period = rate; SAMPL_RATE(hwc) = rate; hw_init_period(hwc, SAMPL_RATE(hwc)); - debug_sprintf_event(sfdbg, 4, "%s: cpu %d period %#llx freq %d,%#lx\n", - __func__, event->cpu, event->attr.sample_period, - event->attr.freq, SAMPL_FREQ_MODE(hwc)); return 0; } @@ -1012,12 +954,6 @@ static void cpumsf_pmu_enable(struct pmu *pmu) /* Load current program parameter */ lpp(&get_lowcore()->lpp); - - debug_sprintf_event(sfdbg, 6, "%s: es %i cs %i ed %i cd %i " - "interval %#lx tear %#lx dear %#lx\n", __func__, - cpuhw->lsctl.es, cpuhw->lsctl.cs, cpuhw->lsctl.ed, - cpuhw->lsctl.cd, cpuhw->lsctl.interval, - cpuhw->lsctl.tear, cpuhw->lsctl.dear); } static void cpumsf_pmu_disable(struct pmu *pmu) @@ -1221,11 +1157,6 @@ static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, /* Count discarded samples */ *overflow += 1; } else { - debug_sprintf_event(sfdbg, 4, - "%s: Found unknown" - " sampling data entry: te->f %i" - " basic.def %#4x (%p)\n", __func__, - te->header.f, sample->def, sample); /* Sample slot is not yet written or other record. * * This condition can occur if the buffer was reused @@ -1295,13 +1226,6 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) */ sampl_overflow += te->header.overflow; - /* Timestamps are valid for full sample-data-blocks only */ - debug_sprintf_event(sfdbg, 6, "%s: sdbt %#lx/%#lx " - "overflow %llu timestamp %#llx\n", - __func__, sdb, (unsigned long)sdbt, - te->header.overflow, - (te->header.f) ? trailer_timestamp(te) : 0ULL); - /* Collect all samples from a single sample-data-block and * flag if an (perf) event overflow happened. If so, the PMU * is stopped and remaining samples will be discarded. @@ -1348,19 +1272,8 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) * are dropped. * Slightly increase the interval to avoid hitting this limit. */ - if (event_overflow) { + if (event_overflow) SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10); - debug_sprintf_event(sfdbg, 1, "%s: rate adjustment %ld\n", - __func__, - DIV_ROUND_UP(SAMPL_RATE(hwc), 10)); - } - - if (sampl_overflow || event_overflow) - debug_sprintf_event(sfdbg, 4, "%s: " - "overflows: sample %llu event %llu" - " total %llu num_sdb %llu\n", - __func__, sampl_overflow, event_overflow, - OVERFLOW_REG(hwc), num_sdb); } static inline unsigned long aux_sdb_index(struct aux_buffer *aux, @@ -1428,9 +1341,6 @@ static void aux_output_end(struct perf_output_handle *handle) /* Remove alert indicators in the buffer */ te = aux_sdb_trailer(aux, aux->alert_mark); te->header.a = 0; - - debug_sprintf_event(sfdbg, 6, "%s: SDBs %ld range %ld head %ld\n", - __func__, i, range_scan, aux->head); } /* @@ -1461,10 +1371,6 @@ static int aux_output_begin(struct perf_output_handle *handle, * SDBs between aux->head and aux->empty_mark are already ready * for new data. range_scan is num of SDBs not within them. */ - debug_sprintf_event(sfdbg, 6, - "%s: range %ld head %ld alert %ld empty %ld\n", - __func__, range, aux->head, aux->alert_mark, - aux->empty_mark); if (range > aux_sdb_num_empty(aux)) { range_scan = range - aux_sdb_num_empty(aux); idx = aux->empty_mark + 1; @@ -1490,12 +1396,6 @@ static int aux_output_begin(struct perf_output_handle *handle, cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long); cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]); - debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld empty %ld " - "index %ld tear %#lx dear %#lx\n", __func__, - aux->head, aux->alert_mark, aux->empty_mark, - head / CPUM_SF_SDB_PER_TABLE, - cpuhw->lsctl.tear, cpuhw->lsctl.dear); - return 0; } @@ -1557,14 +1457,11 @@ static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index, static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, unsigned long long *overflow) { - unsigned long i, range_scan, idx, idx_old; union hws_trailer_header old, prev, new; + unsigned long i, range_scan, idx; unsigned long long orig_overflow; struct hws_trailer_entry *te; - debug_sprintf_event(sfdbg, 6, "%s: range %ld head %ld alert %ld " - "empty %ld\n", __func__, range, aux->head, - aux->alert_mark, aux->empty_mark); if (range <= aux_sdb_num_empty(aux)) /* * No need to scan. All SDBs in range are marked as empty. @@ -1587,7 +1484,7 @@ static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, * indicator fall into this range, set it. */ range_scan = range - aux_sdb_num_empty(aux); - idx_old = idx = aux->empty_mark + 1; + idx = aux->empty_mark + 1; for (i = 0; i < range_scan; i++, idx++) { te = aux_sdb_trailer(aux, idx); prev.val = READ_ONCE_ALIGNED_128(te->header.val); @@ -1609,9 +1506,6 @@ static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, /* Update empty_mark to new position */ aux->empty_mark = aux->head + range - 1; - debug_sprintf_event(sfdbg, 6, "%s: range_scan %ld idx %ld..%ld " - "empty %ld\n", __func__, range_scan, idx_old, - idx - 1, aux->empty_mark); return true; } @@ -1633,7 +1527,7 @@ static void hw_collect_aux(struct cpu_hw_sf *cpuhw) /* Inform user space new data arrived */ size = aux_sdb_num_alert(aux) << PAGE_SHIFT; - debug_sprintf_event(sfdbg, 6, "%s: #alert %ld\n", __func__, + debug_sprintf_event(sfdbg, 6, "%s #alert %ld\n", __func__, size >> PAGE_SHIFT); perf_aux_output_end(handle, size); @@ -1667,23 +1561,11 @@ static void hw_collect_aux(struct cpu_hw_sf *cpuhw) perf_aux_output_end(&cpuhw->handle, size); pr_err("Sample data caused the AUX buffer with %lu " "pages to overflow\n", aux->sfb.num_sdb); - debug_sprintf_event(sfdbg, 1, "%s: head %ld range %ld " - "overflow %lld\n", __func__, - aux->head, range, overflow); } else { size = aux_sdb_num_alert(aux) << PAGE_SHIFT; perf_aux_output_end(&cpuhw->handle, size); - debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld " - "already full, try another\n", - __func__, - aux->head, aux->alert_mark); } } - - if (done) - debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld " - "empty %ld\n", __func__, aux->head, - aux->alert_mark, aux->empty_mark); } /* @@ -1705,8 +1587,6 @@ static void aux_buffer_free(void *data) kfree(aux->sdbt_index); kfree(aux->sdb_index); kfree(aux); - - debug_sprintf_event(sfdbg, 4, "%s: SDBTs %lu\n", __func__, num_sdbt); } static void aux_sdb_init(unsigned long sdb) @@ -1814,9 +1694,6 @@ static void *aux_buffer_setup(struct perf_event *event, void **pages, */ aux->empty_mark = sfb->num_sdb - 1; - debug_sprintf_event(sfdbg, 4, "%s: SDBTs %lu SDBs %lu\n", __func__, - sfb->num_sdbt, sfb->num_sdb); - return aux; no_sdbt: @@ -1867,10 +1744,6 @@ static int cpumsf_pmu_check_period(struct perf_event *event, u64 value) event->attr.sample_period = rate; SAMPL_RATE(&event->hw) = rate; hw_init_period(&event->hw, SAMPL_RATE(&event->hw)); - debug_sprintf_event(sfdbg, 4, "%s:" - " cpu %d value %#llx period %#llx freq %d\n", - __func__, event->cpu, value, - event->attr.sample_period, do_freq); return 0; } @@ -2099,7 +1972,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code, /* Report measurement alerts only for non-PRA codes */ if (alert != CPU_MF_INT_SF_PRA) - debug_sprintf_event(sfdbg, 6, "%s: alert %#x\n", __func__, + debug_sprintf_event(sfdbg, 6, "%s alert %#x\n", __func__, alert); /* Sampling authorization change request */ From b495e710157606889f2d8bdc62aebf2aa02f67a7 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 10 Jul 2024 12:23:47 +0200 Subject: [PATCH 23/84] s390/cpum_sf: Remove WARN_ON_ONCE statements Remove WARN_ON_ONCE statements. These have not triggered in the past. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_cpum_sf.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 8d49f24f1bf6..5b765e3ccf0c 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1359,7 +1359,7 @@ static int aux_output_begin(struct perf_output_handle *handle, unsigned long range, i, range_scan, idx, head, base, offset; struct hws_trailer_entry *te; - if (WARN_ON_ONCE(handle->head & ~PAGE_MASK)) + if (handle->head & ~PAGE_MASK) return -EINVAL; aux->head = handle->head >> PAGE_SHIFT; @@ -1522,7 +1522,7 @@ static void hw_collect_aux(struct cpu_hw_sf *cpuhw) unsigned long num_sdb; aux = perf_get_aux(handle); - if (WARN_ON_ONCE(!aux)) + if (!aux) return; /* Inform user space new data arrived */ @@ -1541,7 +1541,7 @@ static void hw_collect_aux(struct cpu_hw_sf *cpuhw) num_sdb); break; } - if (WARN_ON_ONCE(!aux)) + if (!aux) return; /* Update head and alert_mark to new position */ @@ -1754,12 +1754,8 @@ static void cpumsf_pmu_start(struct perf_event *event, int flags) { struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf); - if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) + if (!(event->hw.state & PERF_HES_STOPPED)) return; - - if (flags & PERF_EF_RELOAD) - WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); - perf_pmu_disable(event->pmu); event->hw.state = 0; cpuhw->lsctl.cs = 1; From 0bc6a69f5fda17c907dfdf5850a293d13010e9e5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Aug 2024 17:16:11 +0200 Subject: [PATCH 24/84] s390/early: Add __init to __do_early_pgm_check() __do_early_pgm_check() is a function which is only needed during early setup code. Mark it __init in order to save a few bytes. Signed-off-by: Heiko Carstens Acked-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 0242fa78c918..25ce02be77a3 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -175,7 +175,7 @@ static __init void setup_topology(void) topology_max_mnest = max_mnest; } -void __do_early_pgm_check(struct pt_regs *regs) +void __init __do_early_pgm_check(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); unsigned long ip; From 6708948e361fc1eea858f8ce333c1aab7411c114 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Aug 2024 17:16:12 +0200 Subject: [PATCH 25/84] s390/early: Dump register contents and call trace for early crashes If the early program check handler cannot resolve a program check dump register contents and a call trace to the console before loading a disabled wait psw. This makes debugging much easier. Emit an extra message with early_printk() for cases where regular printk() via the early console is not yet working so that at least some information is available. Signed-off-by: Heiko Carstens Acked-by: Vasily Gorbik Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/setup.h | 2 ++ arch/s390/kernel/early.c | 10 ++++++++++ arch/s390/kernel/early_printk.c | 16 ++++++++++++---- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 8505737712ee..50b943f30155 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -115,6 +115,8 @@ extern unsigned int console_irq; #define SET_CONSOLE_VT220 do { console_mode = 4; } while (0) #define SET_CONSOLE_HVC do { console_mode = 5; } while (0) +void register_early_console(void); + #ifdef CONFIG_VMCP void vmcp_cma_reserve(void); #else diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 25ce02be77a3..cc4264d23019 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -7,6 +7,7 @@ #define KMSG_COMPONENT "setup" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include #include #include #include @@ -191,6 +192,15 @@ void __init __do_early_pgm_check(struct pt_regs *regs) } if (fixup_exception(regs)) return; + /* + * Unhandled exception - system cannot continue but try to get some + * helpful messages to the console. Use early_printk() to print + * some basic information in case it is too early for printk(). + */ + register_early_console(); + early_printk("PANIC: early exception %04x PSW: %016lx %016lx\n", + regs->int_code & 0xffff, regs->psw.mask, regs->psw.addr); + show_regs(regs); disabled_wait(); } diff --git a/arch/s390/kernel/early_printk.c b/arch/s390/kernel/early_printk.c index d9d53f44008a..cefe020a3be3 100644 --- a/arch/s390/kernel/early_printk.c +++ b/arch/s390/kernel/early_printk.c @@ -6,6 +6,7 @@ #include #include #include +#include #include static void sclp_early_write(struct console *con, const char *s, unsigned int len) @@ -20,6 +21,16 @@ static struct console sclp_early_console = { .index = -1, }; +void __init register_early_console(void) +{ + if (early_console) + return; + if (!sclp.has_linemode && !sclp.has_vt220) + return; + early_console = &sclp_early_console; + register_console(early_console); +} + static int __init setup_early_printk(char *buf) { if (early_console) @@ -27,10 +38,7 @@ static int __init setup_early_printk(char *buf) /* Accept only "earlyprintk" and "earlyprintk=sclp" */ if (buf && !str_has_prefix(buf, "sclp")) return 0; - if (!sclp.has_linemode && !sclp.has_vt220) - return 0; - early_console = &sclp_early_console; - register_console(early_console); + register_early_console(); return 0; } early_param("earlyprintk", setup_early_printk); From 017f1f0d397604f289d9075d8481e42a9b85194a Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 21 Aug 2024 20:06:13 +0200 Subject: [PATCH 26/84] s390/ftrace: Remove unused ftrace_plt_template* Unused since commit b860b9346e2d ("s390/ftrace: remove dead code"). Reviewed-by: Ilya Leoshkevich Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ftrace.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/s390/kernel/ftrace.h b/arch/s390/kernel/ftrace.h index 7f75a9616406..23337065f402 100644 --- a/arch/s390/kernel/ftrace.h +++ b/arch/s390/kernel/ftrace.h @@ -18,7 +18,5 @@ extern const char ftrace_shared_hotpatch_trampoline_br[]; extern const char ftrace_shared_hotpatch_trampoline_br_end[]; extern const char ftrace_shared_hotpatch_trampoline_exrl[]; extern const char ftrace_shared_hotpatch_trampoline_exrl_end[]; -extern const char ftrace_plt_template[]; -extern const char ftrace_plt_template_end[]; #endif /* _FTRACE_H */ From d759be28232f8a4ebdfc1a2c20989e54a6965dbf Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 21 Aug 2024 20:06:16 +0200 Subject: [PATCH 27/84] s390/ftrace: Use kernel ftrace trampoline for modules Now that both the kernel modules area and the kernel image itself are located within 4 GB, there is no longer a need to maintain a separate ftrace_plt trampoline. Use the existing trampoline in the kernel. Reviewed-by: Ilya Leoshkevich Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ftrace.c | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 0bd6adc40a34..9552b1e7121f 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -50,10 +50,6 @@ struct ftrace_insn { s32 disp; } __packed; -#ifdef CONFIG_MODULES -static char *ftrace_plt; -#endif /* CONFIG_MODULES */ - static const char *ftrace_shared_hotpatch_trampoline(const char **end) { const char *tstart, *tend; @@ -99,7 +95,6 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) if (mod) { next_trampoline = &mod->arch.next_trampoline; trampolines_end = mod->arch.trampolines_end; - shared = ftrace_plt; } #endif @@ -215,25 +210,6 @@ void ftrace_arch_code_modify_post_process(void) text_poke_sync_lock(); } -#ifdef CONFIG_MODULES - -static int __init ftrace_plt_init(void) -{ - const char *start, *end; - - ftrace_plt = execmem_alloc(EXECMEM_FTRACE, PAGE_SIZE); - if (!ftrace_plt) - panic("cannot allocate ftrace plt\n"); - - start = ftrace_shared_hotpatch_trampoline(&end); - memcpy(ftrace_plt, start, end - start); - set_memory_rox((unsigned long)ftrace_plt, 1); - return 0; -} -device_initcall(ftrace_plt_init); - -#endif /* CONFIG_MODULES */ - #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* * Hook the return address and push it in the stack of return addresses From 57216cc985b34c4cd6107a15fb04ca881465176d Mon Sep 17 00:00:00 2001 From: Jens Remus Date: Thu, 22 Aug 2024 17:30:21 +0200 Subject: [PATCH 28/84] s390/build: Avoid relocation information in final vmlinux Since commit 778666df60f0 ("s390: compile relocatable kernel without -fPIE") the kernel vmlinux ELF file is linked with --emit-relocs to preserve all relocations, so that all absolute relocations can be extracted using the 'relocs' tool to adjust them during boot. Port and adapt Petr Pavlu's x86 commit 9d9173e9ceb6 ("x86/build: Avoid relocation information in final vmlinux") to s390 to strip all relocations from the final vmlinux ELF file to optimize its size. Following is his original commit message with minor adaptions for s390: The Linux build process on s390 roughly consists of compiling all input files, statically linking them into a vmlinux ELF file, and then taking and turning this file into an actual bzImage bootable file. vmlinux has in this process two main purposes: 1) It is an intermediate build target on the way to produce the final bootable image. 2) It is a file that is expected to be used by debuggers and standard ELF tooling to work with the built kernel. For the second purpose, a vmlinux file is typically collected by various package build recipes, such as distribution spec files, including the kernel's own tar-pkg target. When building the kernel vmlinux contains also relocation information produced by using the --emit-relocs linker option. This is utilized by subsequent build steps to create relocs.S and produce a relocatable image. However, the information is not needed by debuggers and other standard ELF tooling. The issue is then that the collected vmlinux file and hence distribution packages end up unnecessarily large because of this extra data. The following is a size comparison of vmlinux v6.10 with and without the relocation information: | Configuration | With relocs | Stripped relocs | | defconfig | 696 MB | 320 MB | | -CONFIG_DEBUG_INFO | 48 MB | 32 MB | Optimize a resulting vmlinux by adding a postlink step that splits the relocation information into relocs.S and then strips it from the vmlinux binary. Reviewed-by: Vasily Gorbik Signed-off-by: Jens Remus Signed-off-by: Vasily Gorbik --- arch/s390/Makefile.postlink | 38 +++++++++++++++++++++++++++++++++++++ arch/s390/boot/Makefile | 8 +++----- 2 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 arch/s390/Makefile.postlink diff --git a/arch/s390/Makefile.postlink b/arch/s390/Makefile.postlink new file mode 100644 index 000000000000..df82f5410769 --- /dev/null +++ b/arch/s390/Makefile.postlink @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0 +# =========================================================================== +# Post-link s390 pass +# =========================================================================== +# +# 1. Separate relocations from vmlinux into relocs.S. +# 2. Strip relocations from vmlinux. + +PHONY := __archpost +__archpost: + +-include include/config/auto.conf +include $(srctree)/scripts/Kbuild.include + +CMD_RELOCS=arch/s390/tools/relocs +OUT_RELOCS = arch/s390/boot +quiet_cmd_relocs = RELOCS $(OUT_RELOCS)/relocs.S + cmd_relocs = \ + mkdir -p $(OUT_RELOCS); \ + $(CMD_RELOCS) $@ > $(OUT_RELOCS)/relocs.S + +quiet_cmd_strip_relocs = RSTRIP $@ + cmd_strip_relocs = \ + $(OBJCOPY) --remove-section='.rel.*' --remove-section='.rel__*' \ + --remove-section='.rela.*' --remove-section='.rela__*' $@ + +vmlinux: FORCE + $(call cmd,relocs) + $(call cmd,strip_relocs) + +clean: + @rm -f $(OUT_RELOCS)/relocs.S + +PHONY += FORCE clean + +FORCE: + +.PHONY: $(PHONY) diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 4f476884d340..b2252bb15445 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -109,11 +109,9 @@ OBJCOPYFLAGS_vmlinux.bin := -O binary --remove-section=.comment --remove-section $(obj)/vmlinux.bin: vmlinux FORCE $(call if_changed,objcopy) -CMD_RELOCS=arch/s390/tools/relocs -quiet_cmd_relocs = RELOCS $@ - cmd_relocs = $(CMD_RELOCS) $< > $@ -$(obj)/relocs.S: vmlinux FORCE - $(call if_changed,relocs) +# relocs.S is created by the vmlinux postlink step. +$(obj)/relocs.S: vmlinux + @true suffix-$(CONFIG_KERNEL_GZIP) := .gz suffix-$(CONFIG_KERNEL_BZIP2) := .bz2 From a84dd0d8ae24bdc6da341187fc4c1a0adfce2ccc Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Sat, 24 Aug 2024 02:14:04 +0200 Subject: [PATCH 29/84] s390/ftrace: Avoid calling unwinder in ftrace_return_address() ftrace_return_address() is called extremely often from performance-critical code paths when debugging features like CONFIG_TRACE_IRQFLAGS are enabled. For example, with debug_defconfig, ftrace selftests on my LPAR currently execute ftrace_return_address() as follows: ftrace_return_address(0) - 0 times (common code uses __builtin_return_address(0) instead) ftrace_return_address(1) - 2,986,805,401 times (with this patch applied) ftrace_return_address(2) - 140 times ftrace_return_address(>2) - 0 times The use of __builtin_return_address(n) was replaced by return_address() with an unwinder call by commit cae74ba8c295 ("s390/ftrace: Use unwinder instead of __builtin_return_address()") because __builtin_return_address(n) simply walks the stack backchain and doesn't check for reaching the stack top. For shallow stacks with fewer than "n" frames, this results in reads at low addresses and random memory accesses. While calling the fully functional unwinder "works", it is very slow for this purpose. Moreover, potentially following stack switches and walking past IRQ context is simply wrong thing to do for ftrace_return_address(). Reimplement return_address() to essentially be __builtin_return_address(n) with checks for reaching the stack top. Since the ftrace_return_address(n) argument is always a constant, keep the implementation in the header, allowing both GCC and Clang to unroll the loop and optimize it to the bare minimum. Fixes: cae74ba8c295 ("s390/ftrace: Use unwinder instead of __builtin_return_address()") Cc: stable@vger.kernel.org Reported-by: Sumanth Korikkar Reviewed-by: Heiko Carstens Acked-by: Sumanth Korikkar Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ftrace.h | 17 ++++++++++++++++- arch/s390/kernel/stacktrace.c | 19 ------------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index fbadca645af7..406746666eb7 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -6,8 +6,23 @@ #define MCOUNT_INSN_SIZE 6 #ifndef __ASSEMBLY__ +#include -unsigned long return_address(unsigned int n); +static __always_inline unsigned long return_address(unsigned int n) +{ + struct stack_frame *sf; + + if (!n) + return (unsigned long)__builtin_return_address(0); + + sf = (struct stack_frame *)current_frame_address(); + do { + sf = (struct stack_frame *)sf->back_chain; + if (!sf) + return 0; + } while (--n); + return sf->gprs[8]; +} #define ftrace_return_address(n) return_address(n) void ftrace_caller(void); diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c index 640363b2a105..9f59837d159e 100644 --- a/arch/s390/kernel/stacktrace.c +++ b/arch/s390/kernel/stacktrace.c @@ -162,22 +162,3 @@ void arch_stack_walk_user(stack_trace_consume_fn consume_entry, void *cookie, { arch_stack_walk_user_common(consume_entry, cookie, NULL, regs, false); } - -unsigned long return_address(unsigned int n) -{ - struct unwind_state state; - unsigned long addr; - - /* Increment to skip current stack entry */ - n++; - - unwind_for_each_frame(&state, NULL, NULL, 0) { - addr = unwind_get_return_address(&state); - if (!addr) - break; - if (!n--) - return addr; - } - return 0; -} -EXPORT_SYMBOL_GPL(return_address); From 73c81973b44b10c460268567fac4eead17284867 Mon Sep 17 00:00:00 2001 From: Jens Remus Date: Fri, 23 Aug 2024 12:05:14 +0200 Subject: [PATCH 30/84] s390/disassembler: Use proper format specifiers for operand values Treat register numbers as unsigned. Treat signed operand values as signed. This resolves multiple instances of the Cppcheck warning: warning: %i in format string (no. 1) requires 'int' but the argument type is 'unsigned int'. [invalidPrintfArgType_sint] Acked-by: Vasily Gorbik Signed-off-by: Jens Remus Signed-off-by: Vasily Gorbik --- arch/s390/kernel/dis.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 89dc826a8d2e..117fb75913d7 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -455,21 +455,21 @@ static int print_insn(char *buffer, unsigned char *code, unsigned long addr) if (separator) ptr += sprintf(ptr, "%c", separator); if (operand->flags & OPERAND_GPR) - ptr += sprintf(ptr, "%%r%i", value); + ptr += sprintf(ptr, "%%r%u", value); else if (operand->flags & OPERAND_FPR) - ptr += sprintf(ptr, "%%f%i", value); + ptr += sprintf(ptr, "%%f%u", value); else if (operand->flags & OPERAND_AR) - ptr += sprintf(ptr, "%%a%i", value); + ptr += sprintf(ptr, "%%a%u", value); else if (operand->flags & OPERAND_CR) - ptr += sprintf(ptr, "%%c%i", value); + ptr += sprintf(ptr, "%%c%u", value); else if (operand->flags & OPERAND_VR) - ptr += sprintf(ptr, "%%v%i", value); + ptr += sprintf(ptr, "%%v%u", value); else if (operand->flags & OPERAND_PCREL) { void *pcrel = (void *)((int)value + addr); ptr += sprintf(ptr, "%px", pcrel); } else if (operand->flags & OPERAND_SIGNED) - ptr += sprintf(ptr, "%i", value); + ptr += sprintf(ptr, "%i", (int)value); else ptr += sprintf(ptr, "%u", value); if (operand->flags & OPERAND_DISP) From 7f4f1f47a3f69e2ca8315ed7202bb1a4c4a444b9 Mon Sep 17 00:00:00 2001 From: Jens Remus Date: Fri, 23 Aug 2024 12:05:15 +0200 Subject: [PATCH 31/84] s390/disassembler: Update instruction mnemonics to latest spec Over the course of CPU generations a few instructions got extended, changing their base mnemonic, while keeping the former as an extended mnemonic. Update the instruction mnemonics in the disassembler to their latest base mnemonic as documented in the latest IBM z/Architecture Principles of Operation specification [1]. With the IBM z14 the base mnemonics of the following vector instructions have been changed: - Vector FP Load Lengthened (VFLL) - Vector FP Load Rounded (VFLR) With Message-Security-Assist Extension 5 Perform Pseudorandom Number Operation (PPNO) has been renamed to Perform Random Number Operation (PRNO). With Vector Enhancements Facility 2 the base mnemonics of the following vector instructions have been changed: - Vector FP Convert from Fixed (VCFPS) - Vector FP Convert from Logical (VCFPL) - Vector FP Convert to Fixed (VCSFP) - Vector FP Convert to Logical (VCLFP) [1] IBM z/Architecture Principles of Operation, SA22-7832-13, IBM z16, https://publibfp.dhe.ibm.com/epubs/pdf/a227832d.pdf Acked-by: Vasily Gorbik Signed-off-by: Jens Remus Signed-off-by: Vasily Gorbik --- arch/s390/tools/opcodes.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index 5f008e794898..aada7aa9ada2 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -527,7 +527,7 @@ b938 sortl RRE_RR b939 dfltcc RRF_R0RR2 b93a kdsa RRE_RR b93b nnpa RRE_00 -b93c ppno RRE_RR +b93c prno RRE_RR b93e kimd RRE_RR b93f klmd RRE_RR b941 cfdtr RRF_UURF @@ -1017,12 +1017,12 @@ e7bb vac VRR_VVVU0V e7bc vgfma VRR_VVVU0V e7bd vsbcbi VRR_VVVU0V e7bf vsbi VRR_VVVU0V -e7c0 vclgd VRR_VV0UUU -e7c1 vcdlg VRR_VV0UUU -e7c2 vcgd VRR_VV0UUU -e7c3 vcdg VRR_VV0UUU -e7c4 vlde VRR_VV0UU2 -e7c5 vled VRR_VV0UUU +e7c0 vclfp VRR_VV0UUU +e7c1 vcfpl VRR_VV0UUU +e7c2 vcsfp VRR_VV0UUU +e7c3 vcfps VRR_VV0UUU +e7c4 vfll VRR_VV0UU2 +e7c5 vflr VRR_VV0UUU e7c7 vfi VRR_VV0UUU e7ca wfk VRR_VV0UU2 e7cb wfc VRR_VV0UU2 From 4eac37ffaf007fba766e61a5d7e384fcdc033cd6 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 26 Aug 2024 23:16:06 +0200 Subject: [PATCH 32/84] s390: Always enable EXPOLINE_EXTERN if supported Since commit ba05b39d54ee ("s390/expoline: Make modules use kernel expolines"), there is no longer any reason not to use CONFIG_EXPOLINE_EXTERN when supported by the compiler. On the positive side: - there is only a single set of expolines generated and used by both the kernel code and modules, - it eliminates expolines "comdat" sections, which can confuse tools like kpatch. Always enable EXPOLINE_EXTERN if supported by the compiler. Suggested-by: Heiko Carstens Reviewed-by: Sumanth Korikkar Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/Kconfig | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a822f952f64a..abc7e1cc0e7c 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -557,17 +557,13 @@ config EXPOLINE If unsure, say N. config EXPOLINE_EXTERN - def_bool y if EXPOLINE - depends on EXPOLINE - depends on CC_IS_GCC && GCC_VERSION >= 110200 - depends on $(success,$(srctree)/arch/s390/tools/gcc-thunk-extern.sh $(CC)) - prompt "Generate expolines as extern functions." + def_bool EXPOLINE && CC_IS_GCC && GCC_VERSION >= 110200 && \ + $(success,$(srctree)/arch/s390/tools/gcc-thunk-extern.sh $(CC)) help - This option is required for some tooling like kpatch. The kernel is - compiled with -mindirect-branch=thunk-extern and requires a newer - compiler. - - If unsure, say N. + Generate expolines as external functions if the compiler supports it. + This option is required for some tooling like kpatch, if expolines + are enabled. The kernel is compiled with + -mindirect-branch=thunk-extern, which requires a newer compiler. choice prompt "Expoline default" From acb684d3b049e37b987b1be56265319842b9273a Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 26 Aug 2024 22:13:41 +0200 Subject: [PATCH 33/84] s390/disassembler: Add instructions Add more instructions to the kernel disassembler. Reviewed-by: Jens Remus Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/dis.c | 7 ++++++- arch/s390/tools/opcodes.txt | 30 +++++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 117fb75913d7..cc7e905d0ff4 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -122,6 +122,7 @@ enum { U8_32, /* 8 bit unsigned value starting at 32 */ U12_16, /* 12 bit unsigned value starting at 16 */ U16_16, /* 16 bit unsigned value starting at 16 */ + U16_20, /* 16 bit unsigned value starting at 20 */ U16_32, /* 16 bit unsigned value starting at 32 */ U32_16, /* 32 bit unsigned value starting at 16 */ VX_12, /* Vector index register starting at position 12 */ @@ -184,6 +185,7 @@ static const struct s390_operand operands[] = { [U8_32] = { 8, 32, 0 }, [U12_16] = { 12, 16, 0 }, [U16_16] = { 16, 16, 0 }, + [U16_20] = { 16, 20, 0 }, [U16_32] = { 16, 32, 0 }, [U32_16] = { 32, 16, 0 }, [VX_12] = { 4, 12, OPERAND_INDEX | OPERAND_VR }, @@ -300,14 +302,17 @@ static const unsigned char formats[][6] = { [INSTR_VRI_V0UU2] = { V_8, U16_16, U4_32, 0, 0, 0 }, [INSTR_VRI_V0UUU] = { V_8, U8_16, U8_24, U4_32, 0, 0 }, [INSTR_VRI_VR0UU] = { V_8, R_12, U8_28, U4_24, 0, 0 }, + [INSTR_VRI_VV0UU] = { V_8, V_12, U8_28, U4_24, 0, 0 }, [INSTR_VRI_VVUU] = { V_8, V_12, U16_16, U4_32, 0, 0 }, [INSTR_VRI_VVUUU] = { V_8, V_12, U12_16, U4_32, U4_28, 0 }, [INSTR_VRI_VVUUU2] = { V_8, V_12, U8_28, U8_16, U4_24, 0 }, [INSTR_VRI_VVV0U] = { V_8, V_12, V_16, U8_24, 0, 0 }, [INSTR_VRI_VVV0UU] = { V_8, V_12, V_16, U8_24, U4_32, 0 }, [INSTR_VRI_VVV0UU2] = { V_8, V_12, V_16, U8_28, U4_24, 0 }, - [INSTR_VRR_0V] = { V_12, 0, 0, 0, 0, 0 }, + [INSTR_VRI_VVV0UV] = { V_8, V_12, V_16, V_32, U8_24, 0 }, + [INSTR_VRR_0V0U] = { V_12, U16_20, 0, 0, 0, 0 }, [INSTR_VRR_0VV0U] = { V_12, V_16, U4_24, 0, 0, 0 }, + [INSTR_VRR_0VVU] = { V_12, V_16, U16_20, 0, 0, 0 }, [INSTR_VRR_RV0UU] = { R_8, V_12, U4_24, U4_28, 0, 0 }, [INSTR_VRR_VRR] = { V_8, R_12, R_16, 0, 0, 0 }, [INSTR_VRR_VV] = { V_8, V_12, 0, 0, 0, 0 }, diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index aada7aa9ada2..b781e1f05c9a 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -528,8 +528,8 @@ b939 dfltcc RRF_R0RR2 b93a kdsa RRE_RR b93b nnpa RRE_00 b93c prno RRE_RR -b93e kimd RRE_RR -b93f klmd RRE_RR +b93e kimd RRF_U0RR +b93f klmd RRF_U0RR b941 cfdtr RRF_UURF b942 clgdtr RRF_UURF b943 clfdtr RRF_UURF @@ -549,6 +549,10 @@ b964 nngrk RRF_R0RR2 b965 ocgrk RRF_R0RR2 b966 nogrk RRF_R0RR2 b967 nxgrk RRF_R0RR2 +b968 clzg RRE_RR +b969 ctzg RRE_RR +b96c bextg RRF_R0RR2 +b96d bdepg RRF_R0RR2 b972 crt RRF_U0RR b973 clrt RRF_U0RR b974 nnrk RRF_R0RR2 @@ -796,6 +800,16 @@ e35b sy RXY_RRRD e35c mfy RXY_RRRD e35e aly RXY_RRRD e35f sly RXY_RRRD +e360 lxab RXY_RRRD +e361 llxab RXY_RRRD +e362 lxah RXY_RRRD +e363 llxah RXY_RRRD +e364 lxaf RXY_RRRD +e365 llxaf RXY_RRRD +e366 lxag RXY_RRRD +e367 llxag RXY_RRRD +e368 lxaq RXY_RRRD +e369 llxaq RXY_RRRD e370 sthy RXY_RRRD e371 lay RXY_RRRD e372 stcy RXY_RRRD @@ -880,6 +894,8 @@ e63c vupkz VSI_URDV e63d vstrl VSI_URDV e63f vstrlr VRS_RRDV e649 vlip VRI_V0UU2 +e64a vcvdq VRI_VV0UU +e64e vcvbq VRR_VV0U2 e650 vcvb VRR_RV0UU e651 vclzdp VRR_VV0U2 e652 vcvbg VRR_RV0UU @@ -893,7 +909,7 @@ e65b vpsop VRI_VVUUU2 e65c vupkzl VRR_VV0U2 e65d vcfn VRR_VV0UU2 e65e vclfnl VRR_VV0UU2 -e65f vtp VRR_0V +e65f vtp VRR_0V0U e670 vpkzr VRI_VVV0UU2 e671 vap VRI_VVV0UU2 e672 vsrpr VRI_VVV0UU2 @@ -908,6 +924,7 @@ e67b vrp VRI_VVV0UU2 e67c vscshp VRR_VVV e67d vcsph VRR_VVV0U0 e67e vsdp VRI_VVV0UU2 +e67f vtz VRR_0VVU e700 vleb VRX_VRRDU e701 vleh VRX_VRRDU e702 vleg VRX_VRRDU @@ -948,6 +965,7 @@ e74d vrep VRI_VVUU e750 vpopct VRR_VV0U e752 vctz VRR_VV0U e753 vclz VRR_VV0U +e754 vgem VRR_VV0U e756 vlr VRX_VV e75c vistr VRR_VV0U0U e75f vseg VRR_VV0U @@ -985,6 +1003,8 @@ e784 vpdi VRR_VVV0U e785 vbperm VRR_VVV e786 vsld VRI_VVV0U e787 vsrd VRI_VVV0U +e788 veval VRI_VVV0UV +e789 vblend VRR_VVVU0V e78a vstrc VRR_VVVUU0V e78b vstrs VRR_VVVUU0V e78c vperm VRR_VVV0V @@ -1010,6 +1030,10 @@ e7ac vmale VRR_VVVU0V e7ad vmalo VRR_VVVU0V e7ae vmae VRR_VVVU0V e7af vmao VRR_VVVU0V +e7b0 vdl VRR_VVV0UU +e7b1 vrl VRR_VVV0UU +e7b2 vd VRR_VVV0UU +e7b3 vr VRR_VVV0UU e7b4 vgfm VRR_VVV0U e7b8 vmsl VRR_VVVUU0V e7b9 vaccc VRR_VVVU0V From 4f7a31a7ef19e59cc89d1013cfd909e05d89b6ff Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 26 Aug 2024 14:22:48 +0200 Subject: [PATCH 34/84] s390/hypfs_diag: Remove unused dentry variable Remove leftover dentry variable after hypfs refactoring. Before 2fcb3686e160, hypfs_diag.c and other hypfs files were using debugfs_create_file() explicitly for creating debugfs files and were storing the returned pointer. After the refactor, common debugfs file operations and also the related dentry pointers have been moved into hypfs_dbfs.c and redefined as new common mechanisms. Therefore the dentry variable and the debugfs_remove() function calls in hypfs_diag.c are now redundant. Current code is not effected since the dentry pointer in hypfs_diag is implicitly assigned to NULL and debugfs_remove() returns without an error if the passed pointer is NULL. Acked-by: Alexander Gordeev Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/hypfs/hypfs_diag.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/s390/hypfs/hypfs_diag.c b/arch/s390/hypfs/hypfs_diag.c index 26a009f9c49e..c8af67d20994 100644 --- a/arch/s390/hypfs/hypfs_diag.c +++ b/arch/s390/hypfs/hypfs_diag.c @@ -29,8 +29,6 @@ static enum diag204_format diag204_info_type; /* used diag 204 data format */ static void *diag204_buf; /* 4K aligned buffer for diag204 data */ static int diag204_buf_pages; /* number of pages for diag204 data */ -static struct dentry *dbfs_d204_file; - enum diag204_format diag204_get_info_type(void) { return diag204_info_type; @@ -214,16 +212,13 @@ __init int hypfs_diag_init(void) hypfs_dbfs_create_file(&dbfs_file_d204); rc = hypfs_diag_fs_init(); - if (rc) { + if (rc) pr_err("The hardware system does not provide all functions required by hypfs\n"); - debugfs_remove(dbfs_d204_file); - } return rc; } void hypfs_diag_exit(void) { - debugfs_remove(dbfs_d204_file); hypfs_diag_fs_exit(); diag204_free_buffer(); hypfs_dbfs_remove_file(&dbfs_file_d204); From 80625b670312e74512d65b19e9470184386ab265 Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Wed, 7 Aug 2024 18:06:28 +0200 Subject: [PATCH 35/84] s390/crypto: Add hardware acceleration for full AES-XTS mode Add new cipher exploiting the full AES-XTS hardware acceleration introduced with message-security assist extension 10. The full AES-XTS cipher is registered as preferred cipher in addition to the discrete AES-XTS variant. Reviewed-by: Harald Freudenberger Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- arch/s390/crypto/aes_s390.c | 120 +++++++++++++++++++++++++++++++++- arch/s390/include/asm/cpacf.h | 2 + 2 files changed, 119 insertions(+), 3 deletions(-) diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index c6fe5405de4a..8cc02d6e0d0f 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -51,8 +51,13 @@ struct s390_aes_ctx { }; struct s390_xts_ctx { - u8 key[32]; - u8 pcc_key[32]; + union { + u8 keys[64]; + struct { + u8 key[32]; + u8 pcc_key[32]; + }; + }; int key_len; unsigned long fc; struct crypto_skcipher *fallback; @@ -526,6 +531,108 @@ static struct skcipher_alg xts_aes_alg = { .decrypt = xts_aes_decrypt, }; +static int fullxts_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm); + unsigned long fc; + int err; + + err = xts_fallback_setkey(tfm, in_key, key_len); + if (err) + return err; + + /* Pick the correct function code based on the key length */ + fc = (key_len == 32) ? CPACF_KM_XTS_128_FULL : + (key_len == 64) ? CPACF_KM_XTS_256_FULL : 0; + + /* Check if the function code is available */ + xts_ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; + if (!xts_ctx->fc) + return 0; + + /* Store double-key */ + memcpy(xts_ctx->keys, in_key, key_len); + xts_ctx->key_len = key_len; + return 0; +} + +static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifier) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm); + unsigned int offset, nbytes, n; + struct skcipher_walk walk; + int ret; + struct { + __u8 key[64]; + __u8 tweak[16]; + __u8 nap[16]; + } fxts_param = { + .nap = {0}, + }; + + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + if (unlikely(!xts_ctx->fc || (req->cryptlen % AES_BLOCK_SIZE) != 0)) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, xts_ctx->fallback); + return (modifier & CPACF_DECRYPT) ? + crypto_skcipher_decrypt(subreq) : + crypto_skcipher_encrypt(subreq); + } + + ret = skcipher_walk_virt(&walk, req, false); + if (ret) + return ret; + + offset = xts_ctx->key_len & 0x20; + memcpy(fxts_param.key + offset, xts_ctx->keys, xts_ctx->key_len); + memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE); + fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */ + + while ((nbytes = walk.nbytes) != 0) { + /* only use complete blocks */ + n = nbytes & ~(AES_BLOCK_SIZE - 1); + cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset, + walk.dst.virt.addr, walk.src.virt.addr, n); + ret = skcipher_walk_done(&walk, nbytes - n); + } + memzero_explicit(&fxts_param, sizeof(fxts_param)); + return ret; +} + +static int fullxts_aes_encrypt(struct skcipher_request *req) +{ + return fullxts_aes_crypt(req, 0); +} + +static int fullxts_aes_decrypt(struct skcipher_request *req) +{ + return fullxts_aes_crypt(req, CPACF_DECRYPT); +} + +static struct skcipher_alg fullxts_aes_alg = { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "full-xts-aes-s390", + .base.cra_priority = 403, /* aes-xts-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct s390_xts_ctx), + .base.cra_module = THIS_MODULE, + .init = xts_fallback_init, + .exit = xts_fallback_exit, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = fullxts_aes_set_key, + .encrypt = fullxts_aes_encrypt, + .decrypt = fullxts_aes_decrypt, +}; + static int ctr_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -955,7 +1062,7 @@ static struct aead_alg gcm_aes_aead = { }; static struct crypto_alg *aes_s390_alg; -static struct skcipher_alg *aes_s390_skcipher_algs[4]; +static struct skcipher_alg *aes_s390_skcipher_algs[5]; static int aes_s390_skciphers_num; static struct aead_alg *aes_s390_aead_alg; @@ -1012,6 +1119,13 @@ static int __init aes_s390_init(void) goto out_err; } + if (cpacf_test_func(&km_functions, CPACF_KM_XTS_128_FULL) || + cpacf_test_func(&km_functions, CPACF_KM_XTS_256_FULL)) { + ret = aes_s390_register_skcipher(&fullxts_aes_alg); + if (ret) + goto out_err; + } + if (cpacf_test_func(&km_functions, CPACF_KM_XTS_128) || cpacf_test_func(&km_functions, CPACF_KM_XTS_256)) { ret = aes_s390_register_skcipher(&xts_aes_alg); diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index dae8843b164f..b2f355751bf6 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -54,6 +54,8 @@ #define CPACF_KM_XTS_256 0x34 #define CPACF_KM_PXTS_128 0x3a #define CPACF_KM_PXTS_256 0x3c +#define CPACF_KM_XTS_128_FULL 0x52 +#define CPACF_KM_XTS_256_FULL 0x54 /* * Function codes for the KMC (CIPHER MESSAGE WITH CHAINING) From c3dcb058b110d07e56cc8129273e1342905b611c Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Wed, 7 Aug 2024 18:06:29 +0200 Subject: [PATCH 36/84] s390/crypto: Add hardware acceleration for HMAC modes Add new shash exploiting the HMAC hardware accelerations for SHA224, SHA256, SHA384 and SHA512 introduced with message-security assist extension 11. Reviewed-by: Harald Freudenberger Signed-off-by: Holger Dengler Signed-off-by: Vasily Gorbik --- arch/s390/configs/debug_defconfig | 1 + arch/s390/configs/defconfig | 1 + arch/s390/crypto/Kconfig | 10 + arch/s390/crypto/Makefile | 1 + arch/s390/crypto/hmac_s390.c | 359 ++++++++++++++++++++++++++++++ arch/s390/include/asm/cpacf.h | 55 +++-- 6 files changed, 410 insertions(+), 17 deletions(-) create mode 100644 arch/s390/crypto/hmac_s390.c diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index ea63a7342f5f..6c57f024acae 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -794,6 +794,7 @@ CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_CHACHA_S390=m +CONFIG_CRYPTO_HMAC_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m CONFIG_CRYPTO_PAES_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index d8b28ff8ff45..c75e375570fa 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -781,6 +781,7 @@ CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_CHACHA_S390=m +CONFIG_CRYPTO_HMAC_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m CONFIG_CRYPTO_PAES_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 06ee706b0d78..d3eb3a233693 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -132,4 +132,14 @@ config CRYPTO_CHACHA_S390 It is available as of z13. +config CRYPTO_HMAC_S390 + tristate "Keyed-hash message authentication code: HMAC" + depends on S390 + select CRYPTO_HASH + help + s390 specific HMAC hardware support for SHA224, SHA256, SHA384 and + SHA512. + + Architecture: s390 + endmenu diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 1b1cc478fa94..a0cb96937c3d 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o obj-$(CONFIG_S390_PRNG) += prng.o obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o +obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o obj-y += arch_random.o crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c new file mode 100644 index 000000000000..bba9a818dfdc --- /dev/null +++ b/arch/s390/crypto/hmac_s390.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright IBM Corp. 2024 + * + * s390 specific HMAC support. + */ + +#define KMSG_COMPONENT "hmac_s390" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include +#include +#include + +/* + * KMAC param block layout for sha2 function codes: + * The layout of the param block for the KMAC instruction depends on the + * blocksize of the used hashing sha2-algorithm function codes. The param block + * contains the hash chaining value (cv), the input message bit-length (imbl) + * and the hmac-secret (key). To prevent code duplication, the sizes of all + * these are calculated based on the blocksize. + * + * param-block: + * +-------+ + * | cv | + * +-------+ + * | imbl | + * +-------+ + * | key | + * +-------+ + * + * sizes: + * part | sh2-alg | calculation | size | type + * -----+---------+-------------+------+-------- + * cv | 224/256 | blocksize/2 | 32 | u64[8] + * | 384/512 | | 64 | u128[8] + * imbl | 224/256 | blocksize/8 | 8 | u64 + * | 384/512 | | 16 | u128 + * key | 224/256 | blocksize | 64 | u8[64] + * | 384/512 | | 128 | u8[128] + */ + +#define MAX_DIGEST_SIZE SHA512_DIGEST_SIZE +#define MAX_IMBL_SIZE sizeof(u128) +#define MAX_BLOCK_SIZE SHA512_BLOCK_SIZE + +#define SHA2_CV_SIZE(bs) ((bs) >> 1) +#define SHA2_IMBL_SIZE(bs) ((bs) >> 3) + +#define SHA2_IMBL_OFFSET(bs) (SHA2_CV_SIZE(bs)) +#define SHA2_KEY_OFFSET(bs) (SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs)) + +struct s390_hmac_ctx { + u8 key[MAX_BLOCK_SIZE]; +}; + +union s390_kmac_gr0 { + unsigned long reg; + struct { + unsigned long : 48; + unsigned long ikp : 1; + unsigned long iimp : 1; + unsigned long ccup : 1; + unsigned long : 6; + unsigned long fc : 7; + }; +}; + +struct s390_kmac_sha2_ctx { + u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + MAX_BLOCK_SIZE]; + union s390_kmac_gr0 gr0; + u8 buf[MAX_BLOCK_SIZE]; + unsigned int buflen; +}; + +/* + * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize + */ +static inline void kmac_sha2_set_imbl(u8 *param, unsigned int buflen, + unsigned int blocksize) +{ + u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize); + + switch (blocksize) { + case SHA256_BLOCK_SIZE: + *(u64 *)imbl = (u64)buflen * BITS_PER_BYTE; + break; + case SHA512_BLOCK_SIZE: + *(u128 *)imbl = (u128)buflen * BITS_PER_BYTE; + break; + default: + break; + } +} + +static int hash_key(const u8 *in, unsigned int inlen, + u8 *digest, unsigned int digestsize) +{ + unsigned long func; + union { + struct sha256_paramblock { + u32 h[8]; + u64 mbl; + } sha256; + struct sha512_paramblock { + u64 h[8]; + u128 mbl; + } sha512; + } __packed param; + +#define PARAM_INIT(x, y, z) \ + param.sha##x.h[0] = SHA##y ## _H0; \ + param.sha##x.h[1] = SHA##y ## _H1; \ + param.sha##x.h[2] = SHA##y ## _H2; \ + param.sha##x.h[3] = SHA##y ## _H3; \ + param.sha##x.h[4] = SHA##y ## _H4; \ + param.sha##x.h[5] = SHA##y ## _H5; \ + param.sha##x.h[6] = SHA##y ## _H6; \ + param.sha##x.h[7] = SHA##y ## _H7; \ + param.sha##x.mbl = (z) + + switch (digestsize) { + case SHA224_DIGEST_SIZE: + func = CPACF_KLMD_SHA_256; + PARAM_INIT(256, 224, inlen * 8); + break; + case SHA256_DIGEST_SIZE: + func = CPACF_KLMD_SHA_256; + PARAM_INIT(256, 256, inlen * 8); + break; + case SHA384_DIGEST_SIZE: + func = CPACF_KLMD_SHA_512; + PARAM_INIT(512, 384, inlen * 8); + break; + case SHA512_DIGEST_SIZE: + func = CPACF_KLMD_SHA_512; + PARAM_INIT(512, 512, inlen * 8); + break; + default: + return -EINVAL; + } + +#undef PARAM_INIT + + cpacf_klmd(func, ¶m, in, inlen); + + memcpy(digest, ¶m, digestsize); + + return 0; +} + +static int s390_hmac_sha2_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct s390_hmac_ctx *tfm_ctx = crypto_shash_ctx(tfm); + unsigned int ds = crypto_shash_digestsize(tfm); + unsigned int bs = crypto_shash_blocksize(tfm); + + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + if (keylen > bs) + return hash_key(key, keylen, tfm_ctx->key, ds); + + memcpy(tfm_ctx->key, key, keylen); + return 0; +} + +static int s390_hmac_sha2_init(struct shash_desc *desc) +{ + struct s390_hmac_ctx *tfm_ctx = crypto_shash_ctx(desc->tfm); + struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + + memcpy(ctx->param + SHA2_KEY_OFFSET(bs), + tfm_ctx->key, bs); + + ctx->buflen = 0; + ctx->gr0.reg = 0; + switch (crypto_shash_digestsize(desc->tfm)) { + case SHA224_DIGEST_SIZE: + ctx->gr0.fc = CPACF_KMAC_HMAC_SHA_224; + break; + case SHA256_DIGEST_SIZE: + ctx->gr0.fc = CPACF_KMAC_HMAC_SHA_256; + break; + case SHA384_DIGEST_SIZE: + ctx->gr0.fc = CPACF_KMAC_HMAC_SHA_384; + break; + case SHA512_DIGEST_SIZE: + ctx->gr0.fc = CPACF_KMAC_HMAC_SHA_512; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int s390_hmac_sha2_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int offset, n; + + /* check current buffer */ + offset = ctx->buflen % bs; + ctx->buflen += len; + if (offset + len < bs) + goto store; + + /* process one stored block */ + if (offset) { + n = bs - offset; + memcpy(ctx->buf + offset, data, n); + ctx->gr0.iimp = 1; + _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs); + data += n; + len -= n; + offset = 0; + } + /* process as many blocks as possible */ + if (len >= bs) { + n = (len / bs) * bs; + ctx->gr0.iimp = 1; + _cpacf_kmac(&ctx->gr0.reg, ctx->param, data, n); + data += n; + len -= n; + } +store: + /* store incomplete block in buffer */ + if (len) + memcpy(ctx->buf + offset, data, len); + + return 0; +} + +static int s390_hmac_sha2_final(struct shash_desc *desc, u8 *out) +{ + struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); + unsigned int bs = crypto_shash_blocksize(desc->tfm); + + ctx->gr0.iimp = 0; + kmac_sha2_set_imbl(ctx->param, ctx->buflen, bs); + _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, ctx->buflen % bs); + memcpy(out, ctx->param, crypto_shash_digestsize(desc->tfm)); + + return 0; +} + +static int s390_hmac_sha2_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc); + unsigned int ds = crypto_shash_digestsize(desc->tfm); + int rc; + + rc = s390_hmac_sha2_init(desc); + if (rc) + return rc; + + ctx->gr0.iimp = 0; + kmac_sha2_set_imbl(ctx->param, len, + crypto_shash_blocksize(desc->tfm)); + _cpacf_kmac(&ctx->gr0.reg, ctx->param, data, len); + memcpy(out, ctx->param, ds); + + return 0; +} + +#define S390_HMAC_SHA2_ALG(x) { \ + .fc = CPACF_KMAC_HMAC_SHA_##x, \ + .alg = { \ + .init = s390_hmac_sha2_init, \ + .update = s390_hmac_sha2_update, \ + .final = s390_hmac_sha2_final, \ + .digest = s390_hmac_sha2_digest, \ + .setkey = s390_hmac_sha2_setkey, \ + .descsize = sizeof(struct s390_kmac_sha2_ctx), \ + .halg = { \ + .digestsize = SHA##x##_DIGEST_SIZE, \ + .base = { \ + .cra_name = "hmac(sha" #x ")", \ + .cra_driver_name = "hmac_s390_sha" #x, \ + .cra_blocksize = SHA##x##_BLOCK_SIZE, \ + .cra_priority = 400, \ + .cra_ctxsize = sizeof(struct s390_hmac_ctx), \ + .cra_module = THIS_MODULE, \ + }, \ + }, \ + }, \ +} + +static struct s390_hmac_alg { + bool registered; + unsigned int fc; + struct shash_alg alg; +} s390_hmac_algs[] = { + S390_HMAC_SHA2_ALG(224), + S390_HMAC_SHA2_ALG(256), + S390_HMAC_SHA2_ALG(384), + S390_HMAC_SHA2_ALG(512), +}; + +static __always_inline void _s390_hmac_algs_unregister(void) +{ + struct s390_hmac_alg *hmac; + int i; + + for (i = ARRAY_SIZE(s390_hmac_algs) - 1; i >= 0; i--) { + hmac = &s390_hmac_algs[i]; + if (!hmac->registered) + continue; + crypto_unregister_shash(&hmac->alg); + } +} + +static int __init hmac_s390_init(void) +{ + struct s390_hmac_alg *hmac; + int i, rc = -ENODEV; + + if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_256)) + return -ENODEV; + if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_512)) + return -ENODEV; + + for (i = 0; i < ARRAY_SIZE(s390_hmac_algs); i++) { + hmac = &s390_hmac_algs[i]; + if (!cpacf_query_func(CPACF_KMAC, hmac->fc)) + continue; + + rc = crypto_register_shash(&hmac->alg); + if (rc) { + pr_err("unable to register %s\n", + hmac->alg.halg.base.cra_name); + goto out; + } + hmac->registered = true; + pr_debug("registered %s\n", hmac->alg.halg.base.cra_name); + } + return rc; +out: + _s390_hmac_algs_unregister(); + return rc; +} + +static void __exit hmac_s390_exit(void) +{ + _s390_hmac_algs_unregister(); +} + +module_cpu_feature_match(S390_CPU_FEATURE_MSA, hmac_s390_init); +module_exit(hmac_s390_exit); + +MODULE_DESCRIPTION("S390 HMAC driver"); +MODULE_LICENSE("GPL"); diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index b2f355751bf6..2dbe2e2088bb 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -123,6 +123,10 @@ #define CPACF_KMAC_DEA 0x01 #define CPACF_KMAC_TDEA_128 0x02 #define CPACF_KMAC_TDEA_192 0x03 +#define CPACF_KMAC_HMAC_SHA_224 0x70 +#define CPACF_KMAC_HMAC_SHA_256 0x71 +#define CPACF_KMAC_HMAC_SHA_384 0x72 +#define CPACF_KMAC_HMAC_SHA_512 0x73 /* * Function codes for the PCKMO (PERFORM CRYPTOGRAPHIC KEY MANAGEMENT) @@ -426,10 +430,41 @@ static inline void cpacf_klmd(unsigned long func, void *param, : "cc", "memory", "0", "1"); } +/** + * _cpacf_kmac() - executes the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) + * instruction and updates flags in gr0 + * @gr0: pointer to gr0 (fc and flags) passed to KMAC; see CPACF_KMAC_xxx defines + * @param: address of parameter block; see POP for details on each func + * @src: address of source memory area + * @src_len: length of src operand in bytes + * + * Returns 0 for the query func, number of processed bytes for digest funcs + */ +static inline int _cpacf_kmac(unsigned long *gr0, void *param, + const u8 *src, long src_len) +{ + union register_pair s; + + s.even = (unsigned long)src; + s.odd = (unsigned long)src_len; + asm volatile( + " lgr 0,%[r0]\n" + " lgr 1,%[pba]\n" + "0: .insn rre,%[opc] << 16,0,%[src]\n" + " brc 1,0b\n" /* handle partial completion */ + " lgr %[r0],0\n" + : [r0] "+d" (*gr0), [src] "+&d" (s.pair) + : [pba] "d" ((unsigned long)param), + [opc] "i" (CPACF_KMAC) + : "cc", "memory", "0", "1"); + + return src_len - s.odd; +} + /** * cpacf_kmac() - executes the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) - * instruction - * @func: the function code passed to KM; see CPACF_KMAC_xxx defines + * instruction + * @func: function code passed to KMAC; see CPACF_KMAC_xxx defines * @param: address of parameter block; see POP for details on each func * @src: address of source memory area * @src_len: length of src operand in bytes @@ -439,21 +474,7 @@ static inline void cpacf_klmd(unsigned long func, void *param, static inline int cpacf_kmac(unsigned long func, void *param, const u8 *src, long src_len) { - union register_pair s; - - s.even = (unsigned long)src; - s.odd = (unsigned long)src_len; - asm volatile( - " lgr 0,%[fc]\n" - " lgr 1,%[pba]\n" - "0: .insn rre,%[opc] << 16,0,%[src]\n" - " brc 1,0b\n" /* handle partial completion */ - : [src] "+&d" (s.pair) - : [fc] "d" (func), [pba] "d" ((unsigned long)param), - [opc] "i" (CPACF_KMAC) - : "cc", "memory", "0", "1"); - - return src_len - s.odd; + return _cpacf_kmac(&func, param, src, src_len); } /** From 7344eea1b30253d4fade26b255b578ec8eaf0853 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 22 Aug 2024 11:32:16 +0200 Subject: [PATCH 37/84] s390/pkey: Split pkey_unlocked_ioctl function Split the very huge ioctl handling function pkey_unlocked_ioctl() into individual functions per each IOCTL command. There is no change in functional code coming with this patch. The work is a simple copy-and-paste with the goal to have the functionality absolutely untouched. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/pkey_api.c | 932 ++++++++++++++++++--------------- 1 file changed, 517 insertions(+), 415 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index ffc0b5db55c2..f3dca56f0808 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -1344,444 +1344,546 @@ static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns) return memdup_user(uapqns, nr_apqns * sizeof(struct pkey_apqn)); } +static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) +{ + struct pkey_genseck kgs; + int rc; + + if (copy_from_user(&kgs, ugs, sizeof(kgs))) + return -EFAULT; + rc = cca_genseckey(kgs.cardnr, kgs.domain, + kgs.keytype, kgs.seckey.seckey); + pr_debug("%s cca_genseckey()=%d\n", __func__, rc); + if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs))) + rc = -EFAULT; + memzero_explicit(&kgs, sizeof(kgs)); + + return rc; +} + +static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs) +{ + struct pkey_clr2seck kcs; + int rc; + + if (copy_from_user(&kcs, ucs, sizeof(kcs))) + return -EFAULT; + rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, + kcs.clrkey.clrkey, kcs.seckey.seckey); + pr_debug("%s cca_clr2seckey()=%d\n", __func__, rc); + if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs))) + rc = -EFAULT; + memzero_explicit(&kcs, sizeof(kcs)); + + return rc; +} + +static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp) +{ + struct pkey_sec2protk ksp; + int rc; + + if (copy_from_user(&ksp, usp, sizeof(ksp))) + return -EFAULT; + ksp.protkey.len = sizeof(ksp.protkey.protkey); + rc = cca_sec2protkey(ksp.cardnr, ksp.domain, + ksp.seckey.seckey, ksp.protkey.protkey, + &ksp.protkey.len, &ksp.protkey.type); + pr_debug("%s cca_sec2protkey()=%d\n", __func__, rc); + if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) + rc = -EFAULT; + memzero_explicit(&ksp, sizeof(ksp)); + + return rc; +} + +static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp) +{ + struct pkey_clr2protk kcp; + int rc; + + if (copy_from_user(&kcp, ucp, sizeof(kcp))) + return -EFAULT; + kcp.protkey.len = sizeof(kcp.protkey.protkey); + rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, + kcp.protkey.protkey, + &kcp.protkey.len, &kcp.protkey.type); + pr_debug("%s pkey_clr2protkey()=%d\n", __func__, rc); + if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp))) + rc = -EFAULT; + memzero_explicit(&kcp, sizeof(kcp)); + + return rc; +} + +static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc) +{ + struct pkey_findcard kfc; + int rc; + + if (copy_from_user(&kfc, ufc, sizeof(kfc))) + return -EFAULT; + rc = cca_findcard(kfc.seckey.seckey, + &kfc.cardnr, &kfc.domain, 1); + pr_debug("%s cca_findcard()=%d\n", __func__, rc); + if (rc < 0) + return rc; + if (copy_to_user(ufc, &kfc, sizeof(kfc))) + return -EFAULT; + + return 0; +} + +static int pkey_ioctl_skey2pkey(struct pkey_skey2pkey __user *usp) +{ + struct pkey_skey2pkey ksp; + int rc; + + if (copy_from_user(&ksp, usp, sizeof(ksp))) + return -EFAULT; + ksp.protkey.len = sizeof(ksp.protkey.protkey); + rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, + &ksp.protkey.len, &ksp.protkey.type); + pr_debug("%s pkey_skey2pkey()=%d\n", __func__, rc); + if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) + rc = -EFAULT; + memzero_explicit(&ksp, sizeof(ksp)); + + return rc; +} + +static int pkey_ioctl_verifykey(struct pkey_verifykey __user *uvk) +{ + struct pkey_verifykey kvk; + int rc; + + if (copy_from_user(&kvk, uvk, sizeof(kvk))) + return -EFAULT; + rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, + &kvk.keysize, &kvk.attributes); + pr_debug("%s pkey_verifykey()=%d\n", __func__, rc); + if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk))) + rc = -EFAULT; + memzero_explicit(&kvk, sizeof(kvk)); + + return rc; +} + +static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp) +{ + struct pkey_genprotk kgp; + int rc; + + if (copy_from_user(&kgp, ugp, sizeof(kgp))) + return -EFAULT; + kgp.protkey.len = sizeof(kgp.protkey.protkey); + rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, + &kgp.protkey.len, &kgp.protkey.type); + pr_debug("%s pkey_genprotkey()=%d\n", __func__, rc); + if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp))) + rc = -EFAULT; + memzero_explicit(&kgp, sizeof(kgp)); + + return rc; +} + +static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) +{ + struct pkey_verifyprotk kvp; + int rc; + + if (copy_from_user(&kvp, uvp, sizeof(kvp))) + return -EFAULT; + rc = pkey_verifyprotkey(kvp.protkey.protkey, + kvp.protkey.len, kvp.protkey.type); + pr_debug("%s pkey_verifyprotkey()=%d\n", __func__, rc); + memzero_explicit(&kvp, sizeof(kvp)); + + return rc; +} + +static int pkey_ioctl_kblob2protk(struct pkey_kblob2pkey __user *utp) +{ + struct pkey_kblob2pkey ktp; + u8 *kkey; + int rc; + + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) + return PTR_ERR(kkey); + ktp.protkey.len = sizeof(ktp.protkey.protkey); + rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, + &ktp.protkey.len, &ktp.protkey.type); + pr_debug("%s pkey_keyblob2pkey()=%d\n", __func__, rc); + kfree_sensitive(kkey); + if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) + rc = -EFAULT; + memzero_explicit(&ktp, sizeof(ktp)); + + return rc; +} + +static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) +{ + size_t klen = KEYBLOBBUFSIZE; + struct pkey_genseck2 kgs; + struct pkey_apqn *apqns; + u8 *kkey; + int rc; + + if (copy_from_user(&kgs, ugs, sizeof(kgs))) + return -EFAULT; + apqns = _copy_apqns_from_user(kgs.apqns, kgs.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = kzalloc(klen, GFP_KERNEL); + if (!kkey) { + kfree(apqns); + return -ENOMEM; + } + rc = pkey_genseckey2(apqns, kgs.apqn_entries, + kgs.type, kgs.size, kgs.keygenflags, + kkey, &klen); + pr_debug("%s pkey_genseckey2()=%d\n", __func__, rc); + kfree(apqns); + if (rc) { + kfree_sensitive(kkey); + return rc; + } + if (kgs.key) { + if (kgs.keylen < klen) { + kfree_sensitive(kkey); + return -EINVAL; + } + if (copy_to_user(kgs.key, kkey, klen)) { + kfree_sensitive(kkey); + return -EFAULT; + } + } + kgs.keylen = klen; + if (copy_to_user(ugs, &kgs, sizeof(kgs))) + rc = -EFAULT; + kfree_sensitive(kkey); + + return rc; +} + +static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs) +{ + size_t klen = KEYBLOBBUFSIZE; + struct pkey_clr2seck2 kcs; + struct pkey_apqn *apqns; + u8 *kkey; + int rc; + + if (copy_from_user(&kcs, ucs, sizeof(kcs))) + return -EFAULT; + apqns = _copy_apqns_from_user(kcs.apqns, kcs.apqn_entries); + if (IS_ERR(apqns)) { + memzero_explicit(&kcs, sizeof(kcs)); + return PTR_ERR(apqns); + } + kkey = kzalloc(klen, GFP_KERNEL); + if (!kkey) { + kfree(apqns); + memzero_explicit(&kcs, sizeof(kcs)); + return -ENOMEM; + } + rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, + kcs.type, kcs.size, kcs.keygenflags, + kcs.clrkey.clrkey, kkey, &klen); + pr_debug("%s pkey_clr2seckey2()=%d\n", __func__, rc); + kfree(apqns); + if (rc) { + kfree_sensitive(kkey); + memzero_explicit(&kcs, sizeof(kcs)); + return rc; + } + if (kcs.key) { + if (kcs.keylen < klen) { + kfree_sensitive(kkey); + memzero_explicit(&kcs, sizeof(kcs)); + return -EINVAL; + } + if (copy_to_user(kcs.key, kkey, klen)) { + kfree_sensitive(kkey); + memzero_explicit(&kcs, sizeof(kcs)); + return -EFAULT; + } + } + kcs.keylen = klen; + if (copy_to_user(ucs, &kcs, sizeof(kcs))) + rc = -EFAULT; + memzero_explicit(&kcs, sizeof(kcs)); + kfree_sensitive(kkey); + + return rc; +} + +static int pkey_ioctl_verifykey2(struct pkey_verifykey2 __user *uvk) +{ + struct pkey_verifykey2 kvk; + u8 *kkey; + int rc; + + if (copy_from_user(&kvk, uvk, sizeof(kvk))) + return -EFAULT; + kkey = _copy_key_from_user(kvk.key, kvk.keylen); + if (IS_ERR(kkey)) + return PTR_ERR(kkey); + rc = pkey_verifykey2(kkey, kvk.keylen, + &kvk.cardnr, &kvk.domain, + &kvk.type, &kvk.size, &kvk.flags); + pr_debug("%s pkey_verifykey2()=%d\n", __func__, rc); + kfree_sensitive(kkey); + if (rc) + return rc; + if (copy_to_user(uvk, &kvk, sizeof(kvk))) + return -EFAULT; + + return 0; +} + +static int pkey_ioctl_kblob2protk2(struct pkey_kblob2pkey2 __user *utp) +{ + struct pkey_apqn *apqns = NULL; + struct pkey_kblob2pkey2 ktp; + u8 *kkey; + int rc; + + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); + } + ktp.protkey.len = sizeof(ktp.protkey.protkey); + rc = pkey_keyblob2pkey2(apqns, ktp.apqn_entries, + kkey, ktp.keylen, + ktp.protkey.protkey, &ktp.protkey.len, + &ktp.protkey.type); + pr_debug("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); + kfree(apqns); + kfree_sensitive(kkey); + if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) + rc = -EFAULT; + memzero_explicit(&ktp, sizeof(ktp)); + + return rc; +} + +static int pkey_ioctl_apqns4k(struct pkey_apqns4key __user *uak) +{ + struct pkey_apqn *apqns = NULL; + struct pkey_apqns4key kak; + size_t nr_apqns, len; + u8 *kkey; + int rc; + + if (copy_from_user(&kak, uak, sizeof(kak))) + return -EFAULT; + nr_apqns = kak.apqn_entries; + if (nr_apqns) { + apqns = kmalloc_array(nr_apqns, + sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!apqns) + return -ENOMEM; + } + kkey = _copy_key_from_user(kak.key, kak.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); + } + rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, + apqns, &nr_apqns); + pr_debug("%s pkey_apqns4key()=%d\n", __func__, rc); + kfree_sensitive(kkey); + if (rc && rc != -ENOSPC) { + kfree(apqns); + return rc; + } + if (!rc && kak.apqns) { + if (nr_apqns > kak.apqn_entries) { + kfree(apqns); + return -EINVAL; + } + len = nr_apqns * sizeof(struct pkey_apqn); + if (len) { + if (copy_to_user(kak.apqns, apqns, len)) { + kfree(apqns); + return -EFAULT; + } + } + } + kak.apqn_entries = nr_apqns; + if (copy_to_user(uak, &kak, sizeof(kak))) + rc = -EFAULT; + kfree(apqns); + + return rc; +} + +static int pkey_ioctl_apqns4kt(struct pkey_apqns4keytype __user *uat) +{ + struct pkey_apqn *apqns = NULL; + struct pkey_apqns4keytype kat; + size_t nr_apqns, len; + int rc; + + if (copy_from_user(&kat, uat, sizeof(kat))) + return -EFAULT; + nr_apqns = kat.apqn_entries; + if (nr_apqns) { + apqns = kmalloc_array(nr_apqns, + sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!apqns) + return -ENOMEM; + } + rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, + kat.flags, apqns, &nr_apqns); + pr_debug("%s pkey_apqns4keytype()=%d\n", __func__, rc); + if (rc && rc != -ENOSPC) { + kfree(apqns); + return rc; + } + if (!rc && kat.apqns) { + if (nr_apqns > kat.apqn_entries) { + kfree(apqns); + return -EINVAL; + } + len = nr_apqns * sizeof(struct pkey_apqn); + if (len) { + if (copy_to_user(kat.apqns, apqns, len)) { + kfree(apqns); + return -EFAULT; + } + } + } + kat.apqn_entries = nr_apqns; + if (copy_to_user(uat, &kat, sizeof(kat))) + rc = -EFAULT; + kfree(apqns); + + return rc; +} + +static int pkey_ioctl_kblob2protk3(struct pkey_kblob2pkey3 __user *utp) +{ + u32 protkeylen = PROTKEYBLOBBUFSIZE; + struct pkey_apqn *apqns = NULL; + struct pkey_kblob2pkey3 ktp; + u8 *kkey, *protkey; + int rc; + + if (copy_from_user(&ktp, utp, sizeof(ktp))) + return -EFAULT; + apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); + if (IS_ERR(apqns)) + return PTR_ERR(apqns); + kkey = _copy_key_from_user(ktp.key, ktp.keylen); + if (IS_ERR(kkey)) { + kfree(apqns); + return PTR_ERR(kkey); + } + protkey = kmalloc(protkeylen, GFP_KERNEL); + if (!protkey) { + kfree(apqns); + kfree_sensitive(kkey); + return -ENOMEM; + } + rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, + kkey, ktp.keylen, + protkey, &protkeylen, &ktp.pkeytype); + pr_debug("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); + kfree(apqns); + kfree_sensitive(kkey); + if (rc) { + kfree_sensitive(protkey); + return rc; + } + if (ktp.pkey && ktp.pkeylen) { + if (protkeylen > ktp.pkeylen) { + kfree_sensitive(protkey); + return -EINVAL; + } + if (copy_to_user(ktp.pkey, protkey, protkeylen)) { + kfree_sensitive(protkey); + return -EFAULT; + } + } + kfree_sensitive(protkey); + ktp.pkeylen = protkeylen; + if (copy_to_user(utp, &ktp, sizeof(ktp))) + return -EFAULT; + + return 0; +} + static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { int rc; switch (cmd) { - case PKEY_GENSECK: { - struct pkey_genseck __user *ugs = (void __user *)arg; - struct pkey_genseck kgs; - - if (copy_from_user(&kgs, ugs, sizeof(kgs))) - return -EFAULT; - rc = cca_genseckey(kgs.cardnr, kgs.domain, - kgs.keytype, kgs.seckey.seckey); - pr_debug("%s cca_genseckey()=%d\n", __func__, rc); - if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs))) - rc = -EFAULT; - memzero_explicit(&kgs, sizeof(kgs)); + case PKEY_GENSECK: + rc = pkey_ioctl_genseck((struct pkey_genseck __user *)arg); break; - } - case PKEY_CLR2SECK: { - struct pkey_clr2seck __user *ucs = (void __user *)arg; - struct pkey_clr2seck kcs; - - if (copy_from_user(&kcs, ucs, sizeof(kcs))) - return -EFAULT; - rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, - kcs.clrkey.clrkey, kcs.seckey.seckey); - pr_debug("%s cca_clr2seckey()=%d\n", __func__, rc); - if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs))) - rc = -EFAULT; - memzero_explicit(&kcs, sizeof(kcs)); + case PKEY_CLR2SECK: + rc = pkey_ioctl_clr2seck((struct pkey_clr2seck __user *)arg); break; - } - case PKEY_SEC2PROTK: { - struct pkey_sec2protk __user *usp = (void __user *)arg; - struct pkey_sec2protk ksp; - - if (copy_from_user(&ksp, usp, sizeof(ksp))) - return -EFAULT; - ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = cca_sec2protkey(ksp.cardnr, ksp.domain, - ksp.seckey.seckey, ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - pr_debug("%s cca_sec2protkey()=%d\n", __func__, rc); - if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) - rc = -EFAULT; - memzero_explicit(&ksp, sizeof(ksp)); + case PKEY_SEC2PROTK: + rc = pkey_ioctl_sec2protk((struct pkey_sec2protk __user *)arg); break; - } - case PKEY_CLR2PROTK: { - struct pkey_clr2protk __user *ucp = (void __user *)arg; - struct pkey_clr2protk kcp; - - if (copy_from_user(&kcp, ucp, sizeof(kcp))) - return -EFAULT; - kcp.protkey.len = sizeof(kcp.protkey.protkey); - rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, - kcp.protkey.protkey, - &kcp.protkey.len, &kcp.protkey.type); - pr_debug("%s pkey_clr2protkey()=%d\n", __func__, rc); - if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp))) - rc = -EFAULT; - memzero_explicit(&kcp, sizeof(kcp)); + case PKEY_CLR2PROTK: + rc = pkey_ioctl_clr2protk((struct pkey_clr2protk __user *)arg); break; - } - case PKEY_FINDCARD: { - struct pkey_findcard __user *ufc = (void __user *)arg; - struct pkey_findcard kfc; - - if (copy_from_user(&kfc, ufc, sizeof(kfc))) - return -EFAULT; - rc = cca_findcard(kfc.seckey.seckey, - &kfc.cardnr, &kfc.domain, 1); - pr_debug("%s cca_findcard()=%d\n", __func__, rc); - if (rc < 0) - break; - if (copy_to_user(ufc, &kfc, sizeof(kfc))) - return -EFAULT; + case PKEY_FINDCARD: + rc = pkey_ioctl_findcard((struct pkey_findcard __user *)arg); break; - } - case PKEY_SKEY2PKEY: { - struct pkey_skey2pkey __user *usp = (void __user *)arg; - struct pkey_skey2pkey ksp; - - if (copy_from_user(&ksp, usp, sizeof(ksp))) - return -EFAULT; - ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - pr_debug("%s pkey_skey2pkey()=%d\n", __func__, rc); - if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) - rc = -EFAULT; - memzero_explicit(&ksp, sizeof(ksp)); + case PKEY_SKEY2PKEY: + rc = pkey_ioctl_skey2pkey((struct pkey_skey2pkey __user *)arg); break; - } - case PKEY_VERIFYKEY: { - struct pkey_verifykey __user *uvk = (void __user *)arg; - struct pkey_verifykey kvk; - - if (copy_from_user(&kvk, uvk, sizeof(kvk))) - return -EFAULT; - rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, - &kvk.keysize, &kvk.attributes); - pr_debug("%s pkey_verifykey()=%d\n", __func__, rc); - if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk))) - rc = -EFAULT; - memzero_explicit(&kvk, sizeof(kvk)); + case PKEY_VERIFYKEY: + rc = pkey_ioctl_verifykey((struct pkey_verifykey __user *)arg); break; - } - case PKEY_GENPROTK: { - struct pkey_genprotk __user *ugp = (void __user *)arg; - struct pkey_genprotk kgp; - - if (copy_from_user(&kgp, ugp, sizeof(kgp))) - return -EFAULT; - kgp.protkey.len = sizeof(kgp.protkey.protkey); - rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, - &kgp.protkey.len, &kgp.protkey.type); - pr_debug("%s pkey_genprotkey()=%d\n", __func__, rc); - if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp))) - rc = -EFAULT; - memzero_explicit(&kgp, sizeof(kgp)); + case PKEY_GENPROTK: + rc = pkey_ioctl_genprotk((struct pkey_genprotk __user *)arg); break; - } - case PKEY_VERIFYPROTK: { - struct pkey_verifyprotk __user *uvp = (void __user *)arg; - struct pkey_verifyprotk kvp; - - if (copy_from_user(&kvp, uvp, sizeof(kvp))) - return -EFAULT; - rc = pkey_verifyprotkey(kvp.protkey.protkey, - kvp.protkey.len, kvp.protkey.type); - pr_debug("%s pkey_verifyprotkey()=%d\n", __func__, rc); - memzero_explicit(&kvp, sizeof(kvp)); + case PKEY_VERIFYPROTK: + rc = pkey_ioctl_verifyprotk((struct pkey_verifyprotk __user *)arg); break; - } - case PKEY_KBLOB2PROTK: { - struct pkey_kblob2pkey __user *utp = (void __user *)arg; - struct pkey_kblob2pkey ktp; - u8 *kkey; - - if (copy_from_user(&ktp, utp, sizeof(ktp))) - return -EFAULT; - kkey = _copy_key_from_user(ktp.key, ktp.keylen); - if (IS_ERR(kkey)) - return PTR_ERR(kkey); - ktp.protkey.len = sizeof(ktp.protkey.protkey); - rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, - &ktp.protkey.len, &ktp.protkey.type); - pr_debug("%s pkey_keyblob2pkey()=%d\n", __func__, rc); - kfree_sensitive(kkey); - if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) - rc = -EFAULT; - memzero_explicit(&ktp, sizeof(ktp)); + case PKEY_KBLOB2PROTK: + rc = pkey_ioctl_kblob2protk((struct pkey_kblob2pkey __user *)arg); break; - } - case PKEY_GENSECK2: { - struct pkey_genseck2 __user *ugs = (void __user *)arg; - size_t klen = KEYBLOBBUFSIZE; - struct pkey_genseck2 kgs; - struct pkey_apqn *apqns; - u8 *kkey; - - if (copy_from_user(&kgs, ugs, sizeof(kgs))) - return -EFAULT; - apqns = _copy_apqns_from_user(kgs.apqns, kgs.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = kzalloc(klen, GFP_KERNEL); - if (!kkey) { - kfree(apqns); - return -ENOMEM; - } - rc = pkey_genseckey2(apqns, kgs.apqn_entries, - kgs.type, kgs.size, kgs.keygenflags, - kkey, &klen); - pr_debug("%s pkey_genseckey2()=%d\n", __func__, rc); - kfree(apqns); - if (rc) { - kfree_sensitive(kkey); - break; - } - if (kgs.key) { - if (kgs.keylen < klen) { - kfree_sensitive(kkey); - return -EINVAL; - } - if (copy_to_user(kgs.key, kkey, klen)) { - kfree_sensitive(kkey); - return -EFAULT; - } - } - kgs.keylen = klen; - if (copy_to_user(ugs, &kgs, sizeof(kgs))) - rc = -EFAULT; - kfree_sensitive(kkey); + case PKEY_GENSECK2: + rc = pkey_ioctl_genseck2((struct pkey_genseck2 __user *)arg); break; - } - case PKEY_CLR2SECK2: { - struct pkey_clr2seck2 __user *ucs = (void __user *)arg; - size_t klen = KEYBLOBBUFSIZE; - struct pkey_clr2seck2 kcs; - struct pkey_apqn *apqns; - u8 *kkey; - - if (copy_from_user(&kcs, ucs, sizeof(kcs))) - return -EFAULT; - apqns = _copy_apqns_from_user(kcs.apqns, kcs.apqn_entries); - if (IS_ERR(apqns)) { - memzero_explicit(&kcs, sizeof(kcs)); - return PTR_ERR(apqns); - } - kkey = kzalloc(klen, GFP_KERNEL); - if (!kkey) { - kfree(apqns); - memzero_explicit(&kcs, sizeof(kcs)); - return -ENOMEM; - } - rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, - kcs.type, kcs.size, kcs.keygenflags, - kcs.clrkey.clrkey, kkey, &klen); - pr_debug("%s pkey_clr2seckey2()=%d\n", __func__, rc); - kfree(apqns); - if (rc) { - kfree_sensitive(kkey); - memzero_explicit(&kcs, sizeof(kcs)); - break; - } - if (kcs.key) { - if (kcs.keylen < klen) { - kfree_sensitive(kkey); - memzero_explicit(&kcs, sizeof(kcs)); - return -EINVAL; - } - if (copy_to_user(kcs.key, kkey, klen)) { - kfree_sensitive(kkey); - memzero_explicit(&kcs, sizeof(kcs)); - return -EFAULT; - } - } - kcs.keylen = klen; - if (copy_to_user(ucs, &kcs, sizeof(kcs))) - rc = -EFAULT; - memzero_explicit(&kcs, sizeof(kcs)); - kfree_sensitive(kkey); + case PKEY_CLR2SECK2: + rc = pkey_ioctl_clr2seck2((struct pkey_clr2seck2 __user *)arg); break; - } - case PKEY_VERIFYKEY2: { - struct pkey_verifykey2 __user *uvk = (void __user *)arg; - struct pkey_verifykey2 kvk; - u8 *kkey; - - if (copy_from_user(&kvk, uvk, sizeof(kvk))) - return -EFAULT; - kkey = _copy_key_from_user(kvk.key, kvk.keylen); - if (IS_ERR(kkey)) - return PTR_ERR(kkey); - rc = pkey_verifykey2(kkey, kvk.keylen, - &kvk.cardnr, &kvk.domain, - &kvk.type, &kvk.size, &kvk.flags); - pr_debug("%s pkey_verifykey2()=%d\n", __func__, rc); - kfree_sensitive(kkey); - if (rc) - break; - if (copy_to_user(uvk, &kvk, sizeof(kvk))) - return -EFAULT; + case PKEY_VERIFYKEY2: + rc = pkey_ioctl_verifykey2((struct pkey_verifykey2 __user *)arg); break; - } - case PKEY_KBLOB2PROTK2: { - struct pkey_kblob2pkey2 __user *utp = (void __user *)arg; - struct pkey_apqn *apqns = NULL; - struct pkey_kblob2pkey2 ktp; - u8 *kkey; - - if (copy_from_user(&ktp, utp, sizeof(ktp))) - return -EFAULT; - apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = _copy_key_from_user(ktp.key, ktp.keylen); - if (IS_ERR(kkey)) { - kfree(apqns); - return PTR_ERR(kkey); - } - ktp.protkey.len = sizeof(ktp.protkey.protkey); - rc = pkey_keyblob2pkey2(apqns, ktp.apqn_entries, - kkey, ktp.keylen, - ktp.protkey.protkey, &ktp.protkey.len, - &ktp.protkey.type); - pr_debug("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); - kfree(apqns); - kfree_sensitive(kkey); - if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) - rc = -EFAULT; - memzero_explicit(&ktp, sizeof(ktp)); + case PKEY_KBLOB2PROTK2: + rc = pkey_ioctl_kblob2protk2((struct pkey_kblob2pkey2 __user *)arg); break; - } - case PKEY_APQNS4K: { - struct pkey_apqns4key __user *uak = (void __user *)arg; - struct pkey_apqn *apqns = NULL; - struct pkey_apqns4key kak; - size_t nr_apqns, len; - u8 *kkey; - - if (copy_from_user(&kak, uak, sizeof(kak))) - return -EFAULT; - nr_apqns = kak.apqn_entries; - if (nr_apqns) { - apqns = kmalloc_array(nr_apqns, - sizeof(struct pkey_apqn), - GFP_KERNEL); - if (!apqns) - return -ENOMEM; - } - kkey = _copy_key_from_user(kak.key, kak.keylen); - if (IS_ERR(kkey)) { - kfree(apqns); - return PTR_ERR(kkey); - } - rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, - apqns, &nr_apqns); - pr_debug("%s pkey_apqns4key()=%d\n", __func__, rc); - kfree_sensitive(kkey); - if (rc && rc != -ENOSPC) { - kfree(apqns); - break; - } - if (!rc && kak.apqns) { - if (nr_apqns > kak.apqn_entries) { - kfree(apqns); - return -EINVAL; - } - len = nr_apqns * sizeof(struct pkey_apqn); - if (len) { - if (copy_to_user(kak.apqns, apqns, len)) { - kfree(apqns); - return -EFAULT; - } - } - } - kak.apqn_entries = nr_apqns; - if (copy_to_user(uak, &kak, sizeof(kak))) - rc = -EFAULT; - kfree(apqns); + case PKEY_APQNS4K: + rc = pkey_ioctl_apqns4k((struct pkey_apqns4key __user *)arg); break; - } - case PKEY_APQNS4KT: { - struct pkey_apqns4keytype __user *uat = (void __user *)arg; - struct pkey_apqn *apqns = NULL; - struct pkey_apqns4keytype kat; - size_t nr_apqns, len; - - if (copy_from_user(&kat, uat, sizeof(kat))) - return -EFAULT; - nr_apqns = kat.apqn_entries; - if (nr_apqns) { - apqns = kmalloc_array(nr_apqns, - sizeof(struct pkey_apqn), - GFP_KERNEL); - if (!apqns) - return -ENOMEM; - } - rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, - kat.flags, apqns, &nr_apqns); - pr_debug("%s pkey_apqns4keytype()=%d\n", __func__, rc); - if (rc && rc != -ENOSPC) { - kfree(apqns); - break; - } - if (!rc && kat.apqns) { - if (nr_apqns > kat.apqn_entries) { - kfree(apqns); - return -EINVAL; - } - len = nr_apqns * sizeof(struct pkey_apqn); - if (len) { - if (copy_to_user(kat.apqns, apqns, len)) { - kfree(apqns); - return -EFAULT; - } - } - } - kat.apqn_entries = nr_apqns; - if (copy_to_user(uat, &kat, sizeof(kat))) - rc = -EFAULT; - kfree(apqns); + case PKEY_APQNS4KT: + rc = pkey_ioctl_apqns4kt((struct pkey_apqns4keytype __user *)arg); break; - } - case PKEY_KBLOB2PROTK3: { - struct pkey_kblob2pkey3 __user *utp = (void __user *)arg; - u32 protkeylen = PROTKEYBLOBBUFSIZE; - struct pkey_apqn *apqns = NULL; - struct pkey_kblob2pkey3 ktp; - u8 *kkey, *protkey; - - if (copy_from_user(&ktp, utp, sizeof(ktp))) - return -EFAULT; - apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries); - if (IS_ERR(apqns)) - return PTR_ERR(apqns); - kkey = _copy_key_from_user(ktp.key, ktp.keylen); - if (IS_ERR(kkey)) { - kfree(apqns); - return PTR_ERR(kkey); - } - protkey = kmalloc(protkeylen, GFP_KERNEL); - if (!protkey) { - kfree(apqns); - kfree_sensitive(kkey); - return -ENOMEM; - } - rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, - kkey, ktp.keylen, - protkey, &protkeylen, &ktp.pkeytype); - pr_debug("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); - kfree(apqns); - kfree_sensitive(kkey); - if (rc) { - kfree_sensitive(protkey); - break; - } - if (ktp.pkey && ktp.pkeylen) { - if (protkeylen > ktp.pkeylen) { - kfree_sensitive(protkey); - return -EINVAL; - } - if (copy_to_user(ktp.pkey, protkey, protkeylen)) { - kfree_sensitive(protkey); - return -EFAULT; - } - } - kfree_sensitive(protkey); - ktp.pkeylen = protkeylen; - if (copy_to_user(utp, &ktp, sizeof(ktp))) - return -EFAULT; + case PKEY_KBLOB2PROTK3: + rc = pkey_ioctl_kblob2protk3((struct pkey_kblob2pkey3 __user *)arg); break; - } default: /* unknown/unsupported ioctl cmd */ return -ENOTTY; From 86fbf5e2a0ca58f10261a264ee25bf2a936ee5d2 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 22 Aug 2024 11:32:17 +0200 Subject: [PATCH 38/84] s390/pkey: Rework and split PKEY kernel module code This is a huge rework of all the pkey kernel module code. The goal is to split the code into individual parts with a dedicated calling interface: - move all the sysfs related code into pkey_sysfs.c - all the CCA related code goes to pkey_cca.c - the EP11 stuff has been moved to pkey_ep11.c - the PCKMO related code is now in pkey_pckmo.c The CCA, EP11 and PCKMO code may be seen as "handlers" with a similar calling interface. The new header file pkey_base.h declares this calling interface. The remaining code in pkey_api.c handles the ioctl, the pkey module things and the "handler" independent code on top of the calling interface invoking the handlers. This regrouping of the code will be the base for a real pkey kernel module split into a pkey base module which acts as a dispatcher and handler modules providing their service. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 4 +- arch/s390/include/asm/pkey.h | 4 +- drivers/s390/crypto/Makefile | 2 +- drivers/s390/crypto/pkey_api.c | 2174 +++++-------------------- drivers/s390/crypto/pkey_base.h | 162 ++ drivers/s390/crypto/pkey_cca.c | 453 ++++++ drivers/s390/crypto/pkey_ep11.c | 406 +++++ drivers/s390/crypto/pkey_pckmo.c | 346 ++++ drivers/s390/crypto/pkey_sysfs.c | 506 ++++++ drivers/s390/crypto/zcrypt_ccamisc.c | 8 +- drivers/s390/crypto/zcrypt_ccamisc.h | 6 +- drivers/s390/crypto/zcrypt_ep11misc.c | 28 +- drivers/s390/crypto/zcrypt_ep11misc.h | 14 +- 13 files changed, 2337 insertions(+), 1776 deletions(-) create mode 100644 drivers/s390/crypto/pkey_base.h create mode 100644 drivers/s390/crypto/pkey_cca.c create mode 100644 drivers/s390/crypto/pkey_ep11.c create mode 100644 drivers/s390/crypto/pkey_pckmo.c create mode 100644 drivers/s390/crypto/pkey_sysfs.c diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 99ea3f12c5d2..d68647d64eb4 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -133,8 +133,8 @@ static inline int __paes_keyblob2pkey(struct key_blob *kb, if (msleep_interruptible(1000)) return -EINTR; } - ret = pkey_keyblob2pkey(kb->key, kb->keylen, - pk->protkey, &pk->len, &pk->type); + ret = pkey_key2protkey(kb->key, kb->keylen, + pk->protkey, &pk->len, &pk->type); } return ret; diff --git a/arch/s390/include/asm/pkey.h b/arch/s390/include/asm/pkey.h index 47d80a7451a6..5dca1a46a9f6 100644 --- a/arch/s390/include/asm/pkey.h +++ b/arch/s390/include/asm/pkey.h @@ -22,7 +22,7 @@ * @param protkey pointer to buffer receiving the protected key * @return 0 on success, negative errno value on failure */ -int pkey_keyblob2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); #endif /* _KAPI_PKEY_H */ diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile index bd94811fd9f1..863d6fbd2e79 100644 --- a/drivers/s390/crypto/Makefile +++ b/drivers/s390/crypto/Makefile @@ -14,7 +14,7 @@ obj-$(CONFIG_ZCRYPT) += zcrypt.o obj-$(CONFIG_ZCRYPT) += zcrypt_cex4.o # pkey kernel module -pkey-objs := pkey_api.o +pkey-objs := pkey_api.o pkey_cca.o pkey_ep11.o pkey_pckmo.o pkey_sysfs.o obj-$(CONFIG_PKEY) += pkey.o # adjunct processor matrix diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index f3dca56f0808..732437bf3823 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -17,36 +17,26 @@ #include #include #include -#include #include #include #include #include -#include #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" #include "zcrypt_ep11misc.h" +#include "pkey_base.h" + MODULE_LICENSE("GPL"); MODULE_AUTHOR("IBM Corporation"); MODULE_DESCRIPTION("s390 protected key interface"); -#define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ -#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header)) -#define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ -#define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ -#define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ - /* - * debug feature data and functions + * Debug feature data and functions */ -static debug_info_t *pkey_dbf_info; - -#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__) -#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__) -#define PKEY_DBF_ERR(...) debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__) +debug_info_t *pkey_dbf_info; static void __init pkey_debug_init(void) { @@ -61,408 +51,232 @@ static void __exit pkey_debug_exit(void) debug_unregister(pkey_dbf_info); } -/* inside view of a protected key token (only type 0x00 version 0x01) */ -struct protaeskeytoken { - u8 type; /* 0x00 for PAES specific key tokens */ - u8 res0[3]; - u8 version; /* should be 0x01 for protected AES key token */ - u8 res1[3]; - u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ - u32 len; /* bytes actually stored in protkey[] */ - u8 protkey[MAXPROTKEYSIZE]; /* the protected key blob */ -} __packed; +/* + * Helper functions + */ -/* inside view of a clear key token (type 0x00 version 0x02) */ -struct clearkeytoken { - u8 type; /* 0x00 for PAES specific key tokens */ - u8 res0[3]; - u8 version; /* 0x02 for clear key token */ - u8 res1[3]; - u32 keytype; /* key type, one of the PKEY_KEYTYPE_* values */ - u32 len; /* bytes actually stored in clearkey[] */ - u8 clearkey[]; /* clear key value */ -} __packed; - -/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */ -static inline u32 pkey_keytype_aes_to_size(u32 keytype) +static int apqns4key(const u8 *key, size_t keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) { - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - return 16; - case PKEY_KEYTYPE_AES_192: - return 24; - case PKEY_KEYTYPE_AES_256: - return 32; - default: - return 0; + if (pkey_is_cca_key(key, keylen)) { + return pkey_cca_apqns4key(key, keylen, flags, + apqns, nr_apqns); + } else if (pkey_is_ep11_key(key, keylen)) { + return pkey_ep11_apqns4key(key, keylen, flags, + apqns, nr_apqns); + } else { + struct keytoken_header *hdr = (struct keytoken_header *)key; + + PKEY_DBF_ERR("%s unknown/unsupported key type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; } } -/* - * Create a protected key from a clear key value via PCKMO instruction. - */ -static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int apqns4keytype(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) { - /* mask of available pckmo subfunctions */ - static cpacf_mask_t pckmo_functions; + if (pkey_is_cca_keytype(ktype)) { + return pkey_cca_apqns4type(ktype, cur_mkvp, alt_mkvp, flags, + apqns, nr_apqns); + } else if (pkey_is_ep11_keytype(ktype)) { + return pkey_ep11_apqns4type(ktype, cur_mkvp, alt_mkvp, flags, + apqns, nr_apqns); + } else { + PKEY_DBF_ERR("%s unknown/unsupported key type %d\n", + __func__, ktype); + return -EINVAL; + } +} - u8 paramblock[112]; - u32 pkeytype; - int keysize; - long fc; +static int genseck2(const struct pkey_apqn *apqns, size_t nr_apqns, + enum pkey_key_type keytype, enum pkey_key_size keybitsize, + u32 flags, u8 *keybuf, u32 *keybuflen) +{ + int i, rc; + u32 u; - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - /* 16 byte key, 32 byte aes wkvp, total 48 bytes */ - keysize = 16; - pkeytype = keytype; - fc = CPACF_PCKMO_ENC_AES_128_KEY; - break; - case PKEY_KEYTYPE_AES_192: - /* 24 byte key, 32 byte aes wkvp, total 56 bytes */ - keysize = 24; - pkeytype = keytype; - fc = CPACF_PCKMO_ENC_AES_192_KEY; - break; - case PKEY_KEYTYPE_AES_256: - /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ - keysize = 32; - pkeytype = keytype; - fc = CPACF_PCKMO_ENC_AES_256_KEY; - break; - case PKEY_KEYTYPE_ECC_P256: - /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ - keysize = 32; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_P256_KEY; - break; - case PKEY_KEYTYPE_ECC_P384: - /* 48 byte key, 32 byte aes wkvp, total 80 bytes */ - keysize = 48; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_P384_KEY; - break; - case PKEY_KEYTYPE_ECC_P521: - /* 80 byte key, 32 byte aes wkvp, total 112 bytes */ - keysize = 80; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_P521_KEY; - break; - case PKEY_KEYTYPE_ECC_ED25519: - /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ - keysize = 32; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY; - break; - case PKEY_KEYTYPE_ECC_ED448: - /* 64 byte key, 32 byte aes wkvp, total 96 bytes */ - keysize = 64; - pkeytype = PKEY_KEYTYPE_ECC; - fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + if (pkey_is_cca_keytype(keytype)) { + /* As of now only CCA AES key generation is supported */ + u = pkey_aes_bitsize_to_keytype(keybitsize); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = pkey_cca_gen_key(apqns[i].card, + apqns[i].domain, + u, keytype, keybitsize, flags, + keybuf, keybuflen); + } + } else if (pkey_is_ep11_keytype(keytype)) { + /* As of now only EP11 AES key generation is supported */ + u = pkey_aes_bitsize_to_keytype(keybitsize); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = pkey_ep11_gen_key(apqns[i].card, + apqns[i].domain, + u, keytype, keybitsize, flags, + keybuf, keybuflen); + } + } else { + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, keytype); return -EINVAL; } - if (*protkeylen < keysize + AES_WK_VP_SIZE) { - PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n", - __func__, *protkeylen, keysize + AES_WK_VP_SIZE); - return -EINVAL; - } - - /* Did we already check for PCKMO ? */ - if (!pckmo_functions.bytes[0]) { - /* no, so check now */ - if (!cpacf_query(CPACF_PCKMO, &pckmo_functions)) - return -ENODEV; - } - /* check for the pckmo subfunction we need now */ - if (!cpacf_test_func(&pckmo_functions, fc)) { - PKEY_DBF_ERR("%s pckmo functions not available\n", __func__); - return -ENODEV; - } - - /* prepare param block */ - memset(paramblock, 0, sizeof(paramblock)); - memcpy(paramblock, clrkey, keysize); - - /* call the pckmo instruction */ - cpacf_pckmo(fc, paramblock); - - /* copy created protected key to key buffer including the wkvp block */ - *protkeylen = keysize + AES_WK_VP_SIZE; - memcpy(protkey, paramblock, *protkeylen); - *protkeytype = pkeytype; - - return 0; + return rc; } -/* - * Find card and transform secure key into protected key. - */ -static int pkey_skey2pkey(const u8 *key, u8 *protkey, - u32 *protkeylen, u32 *protkeytype) +static int clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, + enum pkey_key_type keytype, enum pkey_key_size kbitsize, + u32 flags, const u8 *clrkey, u8 *keybuf, u32 *keybuflen) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - u16 cardnr, domain; - int rc, verify; + int i, rc; + u32 u; - zcrypt_wait_api_operational(); - - /* - * The cca_xxx2protkey call may fail when a card has been - * addressed where the master key was changed after last fetch - * of the mkvp into the cache. Try 3 times: First without verify - * then with verify and last round with verify and old master - * key verification pattern match not ignored. - */ - for (verify = 0; verify < 3; verify++) { - rc = cca_findcard(key, &cardnr, &domain, verify); - if (rc < 0) - continue; - if (rc > 0 && verify < 2) - continue; - switch (hdr->version) { - case TOKVER_CCA_AES: - rc = cca_sec2protkey(cardnr, domain, key, - protkey, protkeylen, protkeytype); - break; - case TOKVER_CCA_VLSC: - rc = cca_cipher2protkey(cardnr, domain, key, - protkey, protkeylen, - protkeytype); - break; - default: + if (pkey_is_cca_keytype(keytype)) { + /* As of now only CCA AES key generation is supported */ + u = pkey_aes_bitsize_to_keytype(kbitsize); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, kbitsize); return -EINVAL; } - if (rc == 0) - break; + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = pkey_cca_clr2key(apqns[i].card, + apqns[i].domain, + u, keytype, kbitsize, flags, + clrkey, kbitsize / 8, + keybuf, keybuflen); + } + } else if (pkey_is_ep11_keytype(keytype)) { + /* As of now only EP11 AES key generation is supported */ + u = pkey_aes_bitsize_to_keytype(kbitsize); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, kbitsize); + return -EINVAL; + } + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = pkey_ep11_clr2key(apqns[i].card, + apqns[i].domain, + u, keytype, kbitsize, flags, + clrkey, kbitsize / 8, + keybuf, keybuflen); + } + } else { + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; } - if (rc) - pr_debug("%s failed rc=%d\n", __func__, rc); - return rc; } -/* - * Construct EP11 key with given clear key value. - */ -static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, - u8 *keybuf, size_t *keybuflen) +static int ccakey2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - u32 nr_apqns, *apqns = NULL; - u16 card, dom; - int i, rc; + struct pkey_apqn *local_apqns = NULL; + int i, j, rc; - zcrypt_wait_api_operational(); - - /* build a list of apqns suitable for ep11 keys with cpacf support */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - NULL); - if (rc) - goto out; - - /* go through the list of apqns and try to bild an ep11 key */ - for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_clr2keyblob(card, dom, clrkeylen * 8, - 0, clrkey, keybuf, keybuflen, - PKEY_TYPE_EP11); - if (rc == 0) - break; + /* alloc space for list of apqns if no list given */ + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + apqns = local_apqns; } -out: - kfree(apqns); - if (rc) - pr_debug("%s failed rc=%d\n", __func__, rc); - return rc; -} - -/* - * Find card and transform EP11 secure key into protected key. - */ -static int pkey_ep11key2pkey(const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - u32 nr_apqns, *apqns = NULL; - int i, j, rc = -ENODEV; - u16 card, dom; - - zcrypt_wait_api_operational(); - /* try two times in case of failure */ - for (i = 0; i < 2 && rc; i++) { - - /* build a list of apqns suitable for this key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - ep11_kb_wkvp(key, keylen)); - if (rc) - continue; /* retry findcard on failure */ - - /* go through the list of apqns and try to derive an pkey */ - for (rc = -ENODEV, j = 0; j < nr_apqns && rc; j++) { - card = apqns[j] >> 16; - dom = apqns[j] & 0xFFFF; - rc = ep11_kblob2protkey(card, dom, key, keylen, - protkey, protkeylen, protkeytype); + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + if (local_apqns) { + /* gather list of apqns able to deal with this key */ + nr_apqns = MAXAPQNSINLIST; + rc = pkey_cca_apqns4key(key, keylen, 0, + local_apqns, &nr_apqns); + if (rc) + continue; + } + /* go through the list of apqns until success or end */ + for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { + rc = pkey_cca_key2protkey(apqns[j].card, + apqns[j].domain, + key, keylen, + protkey, protkeylen, + protkeytype); } - - kfree(apqns); } - if (rc) - pr_debug("%s failed rc=%d\n", __func__, rc); + kfree(local_apqns); return rc; } -/* - * Verify key and give back some info about the key. - */ -static int pkey_verifykey(const struct pkey_seckey *seckey, - u16 *pcardnr, u16 *pdomain, - u16 *pkeysize, u32 *pattributes) +static int ep11key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - struct secaeskeytoken *t = (struct secaeskeytoken *)seckey; - u16 cardnr, domain; - int rc; + struct pkey_apqn *local_apqns = NULL; + int i, j, rc; - /* check the secure key for valid AES secure key */ - rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, (u8 *)seckey, 0); - if (rc) - goto out; - if (pattributes) - *pattributes = PKEY_VERIFY_ATTR_AES; - if (pkeysize) - *pkeysize = t->bitsize; - - /* try to find a card which can handle this key */ - rc = cca_findcard(seckey->seckey, &cardnr, &domain, 1); - if (rc < 0) - goto out; - - if (rc > 0) { - /* key mkvp matches to old master key mkvp */ - pr_debug("%s secure key has old mkvp\n", __func__); - if (pattributes) - *pattributes |= PKEY_VERIFY_ATTR_OLD_MKVP; - rc = 0; + /* alloc space for list of apqns if no list given */ + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + apqns = local_apqns; } - if (pcardnr) - *pcardnr = cardnr; - if (pdomain) - *pdomain = domain; + /* try two times in case of failure */ + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + if (local_apqns) { + /* gather list of apqns able to deal with this key */ + nr_apqns = MAXAPQNSINLIST; + rc = pkey_ep11_apqns4key(key, keylen, 0, + local_apqns, &nr_apqns); + if (rc) + continue; + } + /* go through the list of apqns until success or end */ + for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { + rc = pkey_ep11_key2protkey(apqns[j].card, + apqns[j].domain, + key, keylen, + protkey, protkeylen, + protkeytype); + } + } + + kfree(local_apqns); -out: - pr_debug("%s rc=%d\n", __func__, rc); return rc; } -/* - * Generate a random protected key - */ -static int pkey_genprotkey(u32 keytype, u8 *protkey, - u32 *protkeylen, u32 *protkeytype) -{ - u8 clrkey[32]; - int keysize; - int rc; - - keysize = pkey_keytype_aes_to_size(keytype); - if (!keysize) { - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, - keytype); - return -EINVAL; - } - - /* generate a dummy random clear key */ - get_random_bytes(clrkey, keysize); - - /* convert it to a dummy protected key */ - rc = pkey_clr2protkey(keytype, clrkey, - protkey, protkeylen, protkeytype); - if (rc) - return rc; - - /* replace the key part of the protected key with random bytes */ - get_random_bytes(protkey, keysize); - - return 0; -} - -/* - * Verify if a protected key is still valid - */ -static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, - u32 protkeytype) -{ - struct { - u8 iv[AES_BLOCK_SIZE]; - u8 key[MAXPROTKEYSIZE]; - } param; - u8 null_msg[AES_BLOCK_SIZE]; - u8 dest_buf[AES_BLOCK_SIZE]; - unsigned int k, pkeylen; - unsigned long fc; - - switch (protkeytype) { - case PKEY_KEYTYPE_AES_128: - pkeylen = 16 + AES_WK_VP_SIZE; - fc = CPACF_KMC_PAES_128; - break; - case PKEY_KEYTYPE_AES_192: - pkeylen = 24 + AES_WK_VP_SIZE; - fc = CPACF_KMC_PAES_192; - break; - case PKEY_KEYTYPE_AES_256: - pkeylen = 32 + AES_WK_VP_SIZE; - fc = CPACF_KMC_PAES_256; - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__, - protkeytype); - return -EINVAL; - } - if (protkeylen != pkeylen) { - PKEY_DBF_ERR("%s invalid protected key size %u for keytype %u\n", - __func__, protkeylen, protkeytype); - return -EINVAL; - } - - memset(null_msg, 0, sizeof(null_msg)); - - memset(param.iv, 0, sizeof(param.iv)); - memcpy(param.key, protkey, protkeylen); - - k = cpacf_kmc(fc | CPACF_ENCRYPT, ¶m, null_msg, dest_buf, - sizeof(null_msg)); - if (k != sizeof(null_msg)) { - PKEY_DBF_ERR("%s protected key is not valid\n", __func__); - return -EKEYREJECTED; - } - - return 0; -} - -/* Helper for pkey_nonccatok2pkey, handles aes clear key token */ -static int nonccatokaes2pkey(const struct clearkeytoken *t, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pckmokey2protkey_fallback(const struct clearkeytoken *t, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { size_t tmpbuflen = max_t(size_t, SECKEYBLOBSIZE, MAXEP11AESKEYBLOBSIZE); + struct pkey_apqn *apqns = NULL; + u32 keysize, tmplen; u8 *tmpbuf = NULL; - u32 keysize; - int rc; + size_t nr_apqns; + int i, j, rc; + + /* As of now only for AES keys a fallback is available */ keysize = pkey_keytype_aes_to_size(t->keytype); if (!keysize) { @@ -471,861 +285,148 @@ static int nonccatokaes2pkey(const struct clearkeytoken *t, return -EINVAL; } if (t->len != keysize) { - PKEY_DBF_ERR("%s non clear key aes token: invalid key len %u\n", + PKEY_DBF_ERR("%s clear key AES token: invalid key len %u\n", __func__, t->len); return -EINVAL; } - /* try direct way with the PCKMO instruction */ - rc = pkey_clr2protkey(t->keytype, t->clearkey, - protkey, protkeylen, protkeytype); - if (!rc) - goto out; - - /* PCKMO failed, so try the CCA secure key way */ + /* alloc tmp buffer and space for apqns */ tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); if (!tmpbuf) return -ENOMEM; - zcrypt_wait_api_operational(); - rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, t->clearkey, tmpbuf); - if (rc) - goto try_via_ep11; - rc = pkey_skey2pkey(tmpbuf, - protkey, protkeylen, protkeytype); - if (!rc) - goto out; + nr_apqns = MAXAPQNSINLIST; + apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); + if (!apqns) { + kfree(tmpbuf); + return -ENOMEM; + } + + /* try two times in case of failure */ + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + + /* CCA secure key way */ + nr_apqns = MAXAPQNSINLIST; + rc = pkey_cca_apqns4type(PKEY_TYPE_CCA_DATA, + NULL, NULL, 0, apqns, &nr_apqns); + if (rc) + goto try_via_ep11; + for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { + tmplen = tmpbuflen; + rc = pkey_cca_clr2key(apqns[j].card, apqns[j].domain, + t->keytype, PKEY_TYPE_CCA_DATA, + 8 * keysize, 0, + t->clearkey, t->len, + tmpbuf, &tmplen); + } + if (rc) + goto try_via_ep11; + for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { + rc = pkey_cca_key2protkey(apqns[j].card, + apqns[j].domain, + tmpbuf, tmplen, + protkey, protkeylen, + protkeytype); + } + if (!rc) + break; try_via_ep11: - /* if the CCA way also failed, let's try via EP11 */ - rc = pkey_clr2ep11key(t->clearkey, t->len, - tmpbuf, &tmpbuflen); - if (rc) - goto failure; - rc = pkey_ep11key2pkey(tmpbuf, tmpbuflen, - protkey, protkeylen, protkeytype); - if (!rc) - goto out; - -failure: - PKEY_DBF_ERR("%s unable to build protected key from clear", __func__); - -out: - kfree(tmpbuf); - return rc; -} - -/* Helper for pkey_nonccatok2pkey, handles ecc clear key token */ -static int nonccatokecc2pkey(const struct clearkeytoken *t, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - u32 keylen; - int rc; - - switch (t->keytype) { - case PKEY_KEYTYPE_ECC_P256: - keylen = 32; - break; - case PKEY_KEYTYPE_ECC_P384: - keylen = 48; - break; - case PKEY_KEYTYPE_ECC_P521: - keylen = 80; - break; - case PKEY_KEYTYPE_ECC_ED25519: - keylen = 32; - break; - case PKEY_KEYTYPE_ECC_ED448: - keylen = 64; - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); - return -EINVAL; - } - - if (t->len != keylen) { - PKEY_DBF_ERR("%s non clear key ecc token: invalid key len %u\n", - __func__, t->len); - return -EINVAL; - } - - /* only one path possible: via PCKMO instruction */ - rc = pkey_clr2protkey(t->keytype, t->clearkey, - protkey, protkeylen, protkeytype); - if (rc) { - PKEY_DBF_ERR("%s unable to build protected key from clear", - __func__); - } - - return rc; -} - -/* - * Transform a non-CCA key token into a protected key - */ -static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int rc = -EINVAL; - - switch (hdr->version) { - case TOKVER_PROTECTED_KEY: { - struct protaeskeytoken *t; - - if (keylen != sizeof(struct protaeskeytoken)) - goto out; - t = (struct protaeskeytoken *)key; - rc = pkey_verifyprotkey(t->protkey, t->len, t->keytype); + /* the CCA way failed, try via EP11 */ + nr_apqns = MAXAPQNSINLIST; + rc = pkey_ep11_apqns4type(PKEY_TYPE_EP11_AES, + NULL, NULL, 0, apqns, &nr_apqns); if (rc) - goto out; - memcpy(protkey, t->protkey, t->len); - *protkeylen = t->len; - *protkeytype = t->keytype; - break; - } - case TOKVER_CLEAR_KEY: { - struct clearkeytoken *t = (struct clearkeytoken *)key; - - if (keylen < sizeof(struct clearkeytoken) || - keylen != sizeof(*t) + t->len) - goto out; - switch (t->keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - rc = nonccatokaes2pkey(t, protkey, - protkeylen, protkeytype); - break; - case PKEY_KEYTYPE_ECC_P256: - case PKEY_KEYTYPE_ECC_P384: - case PKEY_KEYTYPE_ECC_P521: - case PKEY_KEYTYPE_ECC_ED25519: - case PKEY_KEYTYPE_ECC_ED448: - rc = nonccatokecc2pkey(t, protkey, - protkeylen, protkeytype); - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported non cca clear key type %u\n", - __func__, t->keytype); - return -EINVAL; - } - break; - } - case TOKVER_EP11_AES: { - /* check ep11 key for exportable as protected key */ - rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); - if (rc) - goto out; - rc = pkey_ep11key2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - } - case TOKVER_EP11_AES_WITH_HEADER: - /* check ep11 key with header for exportable as protected key */ - rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1); - if (rc) - goto out; - rc = pkey_ep11key2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported non-CCA token version %d\n", - __func__, hdr->version); - } - -out: - return rc; -} - -/* - * Transform a CCA internal key token into a protected key - */ -static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - - switch (hdr->version) { - case TOKVER_CCA_AES: - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - break; - case TOKVER_CCA_VLSC: - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported CCA internal token version %d\n", - __func__, hdr->version); - return -EINVAL; - } - - return pkey_skey2pkey(key, protkey, protkeylen, protkeytype); -} - -/* - * Transform a key blob (of any type) into a protected key - */ -int pkey_keyblob2pkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int rc; - - if (keylen < sizeof(struct keytoken_header)) { - PKEY_DBF_ERR("%s invalid keylen %d\n", __func__, keylen); - return -EINVAL; - } - - switch (hdr->type) { - case TOKTYPE_NON_CCA: - rc = pkey_nonccatok2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - case TOKTYPE_CCA_INTERNAL: - rc = pkey_ccainttok2pkey(key, keylen, - protkey, protkeylen, protkeytype); - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); - return -EINVAL; - } - - pr_debug("%s rc=%d\n", __func__, rc); - return rc; -} -EXPORT_SYMBOL(pkey_keyblob2pkey); - -static int pkey_genseckey2(const struct pkey_apqn *apqns, size_t nr_apqns, - enum pkey_key_type ktype, enum pkey_key_size ksize, - u32 kflags, u8 *keybuf, size_t *keybufsize) -{ - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - /* check key type and size */ - switch (ktype) { - case PKEY_TYPE_CCA_DATA: - case PKEY_TYPE_CCA_CIPHER: - if (*keybufsize < SECKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11: - if (*keybufsize < MINEP11AESKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11_AES: - if (*keybufsize < (sizeof(struct ep11kblob_header) + - MINEP11AESKEYBLOBSIZE)) - return -EINVAL; - break; - default: - return -EINVAL; - } - switch (ksize) { - case PKEY_SIZE_AES_128: - case PKEY_SIZE_AES_192: - case PKEY_SIZE_AES_256: - break; - default: - return -EINVAL; - } - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (ktype == PKEY_TYPE_EP11 || - ktype == PKEY_TYPE_EP11_AES) { - rc = ep11_genaeskey(card, dom, ksize, kflags, - keybuf, keybufsize, ktype); - } else if (ktype == PKEY_TYPE_CCA_DATA) { - rc = cca_genseckey(card, dom, ksize, keybuf); - *keybufsize = (rc ? 0 : SECKEYBLOBSIZE); - } else { - /* TOKVER_CCA_VLSC */ - rc = cca_gencipherkey(card, dom, ksize, kflags, - keybuf, keybufsize); - } - if (rc == 0) - break; - } - - return rc; -} - -static int pkey_clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, - enum pkey_key_type ktype, enum pkey_key_size ksize, - u32 kflags, const u8 *clrkey, - u8 *keybuf, size_t *keybufsize) -{ - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - /* check key type and size */ - switch (ktype) { - case PKEY_TYPE_CCA_DATA: - case PKEY_TYPE_CCA_CIPHER: - if (*keybufsize < SECKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11: - if (*keybufsize < MINEP11AESKEYBLOBSIZE) - return -EINVAL; - break; - case PKEY_TYPE_EP11_AES: - if (*keybufsize < (sizeof(struct ep11kblob_header) + - MINEP11AESKEYBLOBSIZE)) - return -EINVAL; - break; - default: - return -EINVAL; - } - switch (ksize) { - case PKEY_SIZE_AES_128: - case PKEY_SIZE_AES_192: - case PKEY_SIZE_AES_256: - break; - default: - return -EINVAL; - } - - zcrypt_wait_api_operational(); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (ktype == PKEY_TYPE_EP11 || - ktype == PKEY_TYPE_EP11_AES) { - rc = ep11_clr2keyblob(card, dom, ksize, kflags, - clrkey, keybuf, keybufsize, - ktype); - } else if (ktype == PKEY_TYPE_CCA_DATA) { - rc = cca_clr2seckey(card, dom, ksize, - clrkey, keybuf); - *keybufsize = (rc ? 0 : SECKEYBLOBSIZE); - } else { - /* TOKVER_CCA_VLSC */ - rc = cca_clr2cipherkey(card, dom, ksize, kflags, - clrkey, keybuf, keybufsize); - } - if (rc == 0) - break; - } - - return rc; -} - -static int pkey_verifykey2(const u8 *key, size_t keylen, - u16 *cardnr, u16 *domain, - enum pkey_key_type *ktype, - enum pkey_key_size *ksize, u32 *flags) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - u32 _nr_apqns, *_apqns = NULL; - int rc; - - if (keylen < sizeof(struct keytoken_header)) - return -EINVAL; - - if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) { - struct secaeskeytoken *t = (struct secaeskeytoken *)key; - - rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_CCA_DATA; - if (ksize) - *ksize = (enum pkey_key_size)t->bitsize; - - rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX3C, AES_MK_SET, t->mkvp, 0, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - if (rc == -ENODEV) { - rc = cca_findcard2(&_apqns, &_nr_apqns, - *cardnr, *domain, - ZCRYPT_CEX3C, AES_MK_SET, - 0, t->mkvp, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + continue; + for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { + tmplen = tmpbuflen; + rc = pkey_ep11_clr2key(apqns[j].card, apqns[j].domain, + t->keytype, PKEY_TYPE_EP11_AES, + 8 * keysize, 0, + t->clearkey, t->len, + tmpbuf, &tmplen); } if (rc) - goto out; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - - } else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) { - struct cipherkeytoken *t = (struct cipherkeytoken *)key; - - rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_CCA_CIPHER; - if (ksize) { - *ksize = PKEY_SIZE_UNKNOWN; - if (!t->plfver && t->wpllen == 512) - *ksize = PKEY_SIZE_AES_128; - else if (!t->plfver && t->wpllen == 576) - *ksize = PKEY_SIZE_AES_192; - else if (!t->plfver && t->wpllen == 640) - *ksize = PKEY_SIZE_AES_256; - } - - rc = cca_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX6, AES_MK_SET, t->mkvp0, 0, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - if (rc == -ENODEV) { - rc = cca_findcard2(&_apqns, &_nr_apqns, - *cardnr, *domain, - ZCRYPT_CEX6, AES_MK_SET, - 0, t->mkvp0, 1); - if (rc == 0 && flags) - *flags = PKEY_FLAGS_MATCH_ALT_MKVP; - } - if (rc) - goto out; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES) { - struct ep11keyblob *kb = (struct ep11keyblob *)key; - int api; - - rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_EP11; - if (ksize) - *ksize = kb->head.bitlen; - - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX7, api, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; - - if (flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - struct ep11kblob_header *kh = (struct ep11kblob_header *)key; - int api; - - rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1); - if (rc) - goto out; - if (ktype) - *ktype = PKEY_TYPE_EP11_AES; - if (ksize) - *ksize = kh->bitlen; - - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, - ZCRYPT_CEX7, api, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; - - if (flags) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - - *cardnr = ((struct pkey_apqn *)_apqns)->card; - *domain = ((struct pkey_apqn *)_apqns)->domain; - } else { - rc = -EINVAL; - } - -out: - kfree(_apqns); - return rc; -} - -static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - if (keylen < sizeof(struct keytoken_header)) - return -EINVAL; - - if (hdr->type == TOKTYPE_CCA_INTERNAL) { - if (hdr->version == TOKVER_CCA_AES) { - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) - return -EINVAL; - } else if (hdr->version == TOKVER_CCA_VLSC) { - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - if (cca_check_secaescipherkey(pkey_dbf_info, - 3, key, 0, 1)) - return -EINVAL; - } else { - PKEY_DBF_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); - return -EINVAL; - } - } else if (hdr->type == TOKTYPE_NON_CCA) { - if (hdr->version == TOKVER_EP11_AES) { - if (ep11_check_aes_key(pkey_dbf_info, - 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - if (ep11_check_aes_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1)) - return -EINVAL; - } else { - return pkey_nonccatok2pkey(key, keylen, + continue; + for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { + rc = pkey_ep11_key2protkey(apqns[j].card, + apqns[j].domain, + tmpbuf, tmplen, protkey, protkeylen, protkeytype); } - } else { - PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); - return -EINVAL; } - zcrypt_wait_api_operational(); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) { - rc = cca_sec2protkey(card, dom, key, - protkey, protkeylen, protkeytype); - } else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) { - rc = cca_cipher2protkey(card, dom, key, - protkey, protkeylen, - protkeytype); - } else { - rc = ep11_kblob2protkey(card, dom, key, keylen, - protkey, protkeylen, - protkeytype); - } - if (rc == 0) - break; - } + kfree(tmpbuf); + kfree(apqns); return rc; } -static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int pckmokey2protkey(const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - u32 _nr_apqns, *_apqns = NULL; int rc; - if (keylen < sizeof(struct keytoken_header) || flags == 0) - return -EINVAL; + rc = pkey_pckmo_key2protkey(key, keylen, + protkey, protkeylen, + protkeytype); + if (rc == -ENODEV) { + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct clearkeytoken *t = (struct clearkeytoken *)key; - zcrypt_wait_api_operational(); - - if (hdr->type == TOKTYPE_NON_CCA && - (hdr->version == TOKVER_EP11_AES_WITH_HEADER || - hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - struct ep11keyblob *kb = (struct ep11keyblob *) - (key + sizeof(struct ep11kblob_header)); - int minhwtype = 0, api = 0; - - if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) - return -EINVAL; - if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { - minhwtype = ZCRYPT_CEX7; - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - } - rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, api, kb->wkvp); - if (rc) - goto out; - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) { - struct ep11keyblob *kb = (struct ep11keyblob *)key; - int minhwtype = 0, api = 0; - - if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) - return -EINVAL; - if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { - minhwtype = ZCRYPT_CEX7; - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - } - rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, api, kb->wkvp); - if (rc) - goto out; - } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { - u64 cur_mkvp = 0, old_mkvp = 0; - int minhwtype = ZCRYPT_CEX3C; - - if (hdr->version == TOKVER_CCA_AES) { - struct secaeskeytoken *t = (struct secaeskeytoken *)key; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = t->mkvp; - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = t->mkvp; - } else if (hdr->version == TOKVER_CCA_VLSC) { - struct cipherkeytoken *t = (struct cipherkeytoken *)key; - - minhwtype = ZCRYPT_CEX6; - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = t->mkvp0; - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = t->mkvp0; - } else { - /* unknown cca internal token type */ - return -EINVAL; - } - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, AES_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { - struct eccprivkeytoken *t = (struct eccprivkeytoken *)key; - u64 cur_mkvp = 0, old_mkvp = 0; - - if (t->secid == 0x20) { - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = t->mkvp; - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = t->mkvp; - } else { - /* unknown cca internal 2 token type */ - return -EINVAL; - } - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, APKA_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - } else { - return -EINVAL; - } - - if (apqns) { - if (*nr_apqns < _nr_apqns) - rc = -ENOSPC; - else - memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); - } - *nr_apqns = _nr_apqns; - -out: - kfree(_apqns); - return rc; -} - -static int pkey_apqns4keytype(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) -{ - u32 _nr_apqns, *_apqns = NULL; - int rc; - - zcrypt_wait_api_operational(); - - if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) { - u64 cur_mkvp = 0, old_mkvp = 0; - int minhwtype = ZCRYPT_CEX3C; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = *((u64 *)cur_mkvp); - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = *((u64 *)alt_mkvp); - if (ktype == PKEY_TYPE_CCA_CIPHER) - minhwtype = ZCRYPT_CEX6; - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - minhwtype, AES_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - } else if (ktype == PKEY_TYPE_CCA_ECC) { - u64 cur_mkvp = 0, old_mkvp = 0; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - cur_mkvp = *((u64 *)cur_mkvp); - if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) - old_mkvp = *((u64 *)alt_mkvp); - rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, APKA_MK_SET, - cur_mkvp, old_mkvp, 1); - if (rc) - goto out; - - } else if (ktype == PKEY_TYPE_EP11 || - ktype == PKEY_TYPE_EP11_AES || - ktype == PKEY_TYPE_EP11_ECC) { - u8 *wkvp = NULL; - int api; - - if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) - wkvp = cur_mkvp; - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, api, wkvp); - if (rc) - goto out; - - } else { - return -EINVAL; - } - - if (apqns) { - if (*nr_apqns < _nr_apqns) - rc = -ENOSPC; - else - memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); - } - *nr_apqns = _nr_apqns; - -out: - kfree(_apqns); - return rc; -} - -static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int i, card, dom, rc; - - /* check for at least one apqn given */ - if (!apqns || !nr_apqns) - return -EINVAL; - - if (keylen < sizeof(struct keytoken_header)) - return -EINVAL; - - if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES_WITH_HEADER && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - /* EP11 AES key blob with header */ - if (ep11_check_aes_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_ECC_WITH_HEADER && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - /* EP11 ECC key blob with header */ - if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) { - /* EP11 AES key blob with header in session field */ - if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { - if (hdr->version == TOKVER_CCA_AES) { - /* CCA AES data key */ - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) - return -EINVAL; - } else if (hdr->version == TOKVER_CCA_VLSC) { - /* CCA AES cipher key */ - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - if (cca_check_secaescipherkey(pkey_dbf_info, - 3, key, 0, 1)) - return -EINVAL; - } else { - PKEY_DBF_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); - return -EINVAL; - } - } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { - /* CCA ECC (private) key */ - if (keylen < sizeof(struct eccprivkeytoken)) - return -EINVAL; - if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) - return -EINVAL; - } else if (hdr->type == TOKTYPE_NON_CCA) { - return pkey_nonccatok2pkey(key, keylen, - protkey, protkeylen, protkeytype); - } else { - PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); - return -EINVAL; - } - - /* simple try all apqns from the list */ - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; + /* maybe a fallback is possible */ if (hdr->type == TOKTYPE_NON_CCA && - (hdr->version == TOKVER_EP11_AES_WITH_HEADER || - hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); - else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); - else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) - rc = cca_sec2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) - rc = cca_cipher2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) - rc = cca_ecc2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - else - return -EINVAL; + hdr->version == TOKVER_CLEAR_KEY) { + rc = pckmokey2protkey_fallback(t, protkey, + protkeylen, + protkeytype); + if (rc) + rc = -ENODEV; + } } + if (rc) + PKEY_DBF_ERR("%s unable to build protected key from clear, rc=%d", + __func__, rc); + return rc; } +static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + if (pkey_is_cca_key(key, keylen)) { + return ccakey2protkey(apqns, nr_apqns, key, keylen, + protkey, protkeylen, protkeytype); + } else if (pkey_is_ep11_key(key, keylen)) { + return ep11key2protkey(apqns, nr_apqns, key, keylen, + protkey, protkeylen, protkeytype); + } else if (pkey_is_pckmo_key(key, keylen)) { + return pckmokey2protkey(key, keylen, + protkey, protkeylen, protkeytype); + } else { + struct keytoken_header *hdr = (struct keytoken_header *)key; + + PKEY_DBF_ERR("%s unknown/unsupported key type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } +} + /* - * File io functions + * In-Kernel function: Transform a key blob (of any type) into a protected key + */ +int pkey_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + return key2protkey(NULL, 0, key, keylen, + protkey, protkeylen, protkeytype); +} +EXPORT_SYMBOL(pkey_key2protkey); + +/* + * Ioctl functions */ static void *_copy_key_from_user(void __user *ukey, size_t keylen) @@ -1347,13 +448,16 @@ static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns) static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) { struct pkey_genseck kgs; + u32 keybuflen; int rc; if (copy_from_user(&kgs, ugs, sizeof(kgs))) return -EFAULT; - rc = cca_genseckey(kgs.cardnr, kgs.domain, - kgs.keytype, kgs.seckey.seckey); - pr_debug("%s cca_genseckey()=%d\n", __func__, rc); + keybuflen = sizeof(kgs.seckey.seckey); + rc = pkey_cca_gen_key(kgs.cardnr, kgs.domain, + kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, + kgs.seckey.seckey, &keybuflen); + pr_debug("pkey_cca_gen_key()=%d\n", rc); if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs))) rc = -EFAULT; memzero_explicit(&kgs, sizeof(kgs)); @@ -1364,13 +468,18 @@ static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs) { struct pkey_clr2seck kcs; + u32 keybuflen; int rc; if (copy_from_user(&kcs, ucs, sizeof(kcs))) return -EFAULT; - rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, - kcs.clrkey.clrkey, kcs.seckey.seckey); - pr_debug("%s cca_clr2seckey()=%d\n", __func__, rc); + keybuflen = sizeof(kcs.seckey.seckey); + rc = pkey_cca_clr2key(kcs.cardnr, kcs.domain, + kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, + kcs.clrkey.clrkey, + pkey_keytype_aes_to_size(kcs.keytype), + kcs.seckey.seckey, &keybuflen); + pr_debug("pkey_cca_clr2key()=%d\n", rc); if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs))) rc = -EFAULT; memzero_explicit(&kcs, sizeof(kcs)); @@ -1386,10 +495,11 @@ static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp) if (copy_from_user(&ksp, usp, sizeof(ksp))) return -EFAULT; ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = cca_sec2protkey(ksp.cardnr, ksp.domain, - ksp.seckey.seckey, ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - pr_debug("%s cca_sec2protkey()=%d\n", __func__, rc); + rc = pkey_cca_key2protkey(ksp.cardnr, ksp.domain, + ksp.seckey.seckey, sizeof(ksp.seckey.seckey), + ksp.protkey.protkey, + &ksp.protkey.len, &ksp.protkey.type); + pr_debug("pkey_cca_key2protkey()=%d\n", rc); if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) rc = -EFAULT; memzero_explicit(&ksp, sizeof(ksp)); @@ -1405,10 +515,10 @@ static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp) if (copy_from_user(&kcp, ucp, sizeof(kcp))) return -EFAULT; kcp.protkey.len = sizeof(kcp.protkey.protkey); - rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, - kcp.protkey.protkey, - &kcp.protkey.len, &kcp.protkey.type); - pr_debug("%s pkey_clr2protkey()=%d\n", __func__, rc); + rc = pkey_pckmo_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, + kcp.protkey.protkey, + &kcp.protkey.len, &kcp.protkey.type); + pr_debug("pkey_pckmo_clr2protkey()=%d\n", rc); if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp))) rc = -EFAULT; memzero_explicit(&kcp, sizeof(kcp)); @@ -1419,15 +529,37 @@ static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp) static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc) { struct pkey_findcard kfc; + struct pkey_apqn *apqns; + size_t nr_apqns; int rc; if (copy_from_user(&kfc, ufc, sizeof(kfc))) return -EFAULT; - rc = cca_findcard(kfc.seckey.seckey, - &kfc.cardnr, &kfc.domain, 1); - pr_debug("%s cca_findcard()=%d\n", __func__, rc); - if (rc < 0) + + if (!pkey_is_cca_key(kfc.seckey.seckey, sizeof(kfc.seckey.seckey))) + return -EINVAL; + + nr_apqns = MAXAPQNSINLIST; + apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); + if (!apqns) + return -ENOMEM; + rc = pkey_cca_apqns4key(kfc.seckey.seckey, + sizeof(kfc.seckey.seckey), + PKEY_FLAGS_MATCH_CUR_MKVP, + apqns, &nr_apqns); + if (rc == -ENODEV) + rc = pkey_cca_apqns4key(kfc.seckey.seckey, + sizeof(kfc.seckey.seckey), + PKEY_FLAGS_MATCH_ALT_MKVP, + apqns, &nr_apqns); + pr_debug("pkey_cca_apqns4key()=%d\n", rc); + if (rc) { + kfree(apqns); return rc; + } + kfc.cardnr = apqns[0].card; + kfc.domain = apqns[0].domain; + kfree(apqns); if (copy_to_user(ufc, &kfc, sizeof(kfc))) return -EFAULT; @@ -1437,14 +569,38 @@ static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc) static int pkey_ioctl_skey2pkey(struct pkey_skey2pkey __user *usp) { struct pkey_skey2pkey ksp; - int rc; + struct pkey_apqn *apqns; + size_t nr_apqns; + int i, rc; if (copy_from_user(&ksp, usp, sizeof(ksp))) return -EFAULT; + + if (!pkey_is_cca_key(ksp.seckey.seckey, sizeof(ksp.seckey.seckey))) + return -EINVAL; + + nr_apqns = MAXAPQNSINLIST; + apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); + if (!apqns) + return -ENOMEM; + rc = pkey_cca_apqns4key(ksp.seckey.seckey, sizeof(ksp.seckey.seckey), + 0, apqns, &nr_apqns); + pr_debug("pkey_cca_apqns4key()=%d\n", rc); + if (rc) { + kfree(apqns); + return rc; + } ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - pr_debug("%s pkey_skey2pkey()=%d\n", __func__, rc); + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = pkey_cca_key2protkey(apqns[i].card, apqns[i].domain, + ksp.seckey.seckey, + sizeof(ksp.seckey.seckey), + ksp.protkey.protkey, + &ksp.protkey.len, + &ksp.protkey.type); + pr_debug("pkey_cca_key2protkey()=%d\n", rc); + } + kfree(apqns); if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) rc = -EFAULT; memzero_explicit(&ksp, sizeof(ksp)); @@ -1454,14 +610,24 @@ static int pkey_ioctl_skey2pkey(struct pkey_skey2pkey __user *usp) static int pkey_ioctl_verifykey(struct pkey_verifykey __user *uvk) { + u32 keytype, keybitsize, flags; struct pkey_verifykey kvk; int rc; if (copy_from_user(&kvk, uvk, sizeof(kvk))) return -EFAULT; - rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, - &kvk.keysize, &kvk.attributes); - pr_debug("%s pkey_verifykey()=%d\n", __func__, rc); + kvk.cardnr = 0xFFFF; + kvk.domain = 0xFFFF; + rc = pkey_cca_verifykey(kvk.seckey.seckey, sizeof(kvk.seckey.seckey), + &kvk.cardnr, &kvk.domain, + &keytype, &keybitsize, &flags); + pr_debug("pkey_cca_verifykey()=%d\n", rc); + if (!rc && keytype != PKEY_TYPE_CCA_DATA) + rc = -EINVAL; + kvk.attributes = PKEY_VERIFY_ATTR_AES; + kvk.keysize = (u16)keybitsize; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + kvk.attributes |= PKEY_VERIFY_ATTR_OLD_MKVP; if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk))) rc = -EFAULT; memzero_explicit(&kvk, sizeof(kvk)); @@ -1477,9 +643,9 @@ static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp) if (copy_from_user(&kgp, ugp, sizeof(kgp))) return -EFAULT; kgp.protkey.len = sizeof(kgp.protkey.protkey); - rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, - &kgp.protkey.len, &kgp.protkey.type); - pr_debug("%s pkey_genprotkey()=%d\n", __func__, rc); + rc = pkey_pckmo_gen_protkey(kgp.keytype, kgp.protkey.protkey, + &kgp.protkey.len, &kgp.protkey.type); + pr_debug("pkey_gen_protkey()=%d\n", rc); if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp))) rc = -EFAULT; memzero_explicit(&kgp, sizeof(kgp)); @@ -1494,9 +660,9 @@ static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) if (copy_from_user(&kvp, uvp, sizeof(kvp))) return -EFAULT; - rc = pkey_verifyprotkey(kvp.protkey.protkey, - kvp.protkey.len, kvp.protkey.type); - pr_debug("%s pkey_verifyprotkey()=%d\n", __func__, rc); + rc = pkey_pckmo_verify_protkey(kvp.protkey.protkey, + kvp.protkey.len, kvp.protkey.type); + pr_debug("pkey_verify_protkey()=%d\n", rc); memzero_explicit(&kvp, sizeof(kvp)); return rc; @@ -1514,9 +680,10 @@ static int pkey_ioctl_kblob2protk(struct pkey_kblob2pkey __user *utp) if (IS_ERR(kkey)) return PTR_ERR(kkey); ktp.protkey.len = sizeof(ktp.protkey.protkey); - rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, - &ktp.protkey.len, &ktp.protkey.type); - pr_debug("%s pkey_keyblob2pkey()=%d\n", __func__, rc); + rc = key2protkey(NULL, 0, kkey, ktp.keylen, + ktp.protkey.protkey, &ktp.protkey.len, + &ktp.protkey.type); + pr_debug("key2protkey()=%d\n", rc); kfree_sensitive(kkey); if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) rc = -EFAULT; @@ -1527,7 +694,7 @@ static int pkey_ioctl_kblob2protk(struct pkey_kblob2pkey __user *utp) static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) { - size_t klen = KEYBLOBBUFSIZE; + u32 klen = KEYBLOBBUFSIZE; struct pkey_genseck2 kgs; struct pkey_apqn *apqns; u8 *kkey; @@ -1543,10 +710,10 @@ static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) kfree(apqns); return -ENOMEM; } - rc = pkey_genseckey2(apqns, kgs.apqn_entries, - kgs.type, kgs.size, kgs.keygenflags, - kkey, &klen); - pr_debug("%s pkey_genseckey2()=%d\n", __func__, rc); + rc = genseck2(apqns, kgs.apqn_entries, + kgs.type, kgs.size, kgs.keygenflags, + kkey, &klen); + pr_debug("genseckey2()=%d\n", rc); kfree(apqns); if (rc) { kfree_sensitive(kkey); @@ -1572,7 +739,7 @@ static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs) { - size_t klen = KEYBLOBBUFSIZE; + u32 klen = KEYBLOBBUFSIZE; struct pkey_clr2seck2 kcs; struct pkey_apqn *apqns; u8 *kkey; @@ -1591,10 +758,10 @@ static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs) memzero_explicit(&kcs, sizeof(kcs)); return -ENOMEM; } - rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, - kcs.type, kcs.size, kcs.keygenflags, - kcs.clrkey.clrkey, kkey, &klen); - pr_debug("%s pkey_clr2seckey2()=%d\n", __func__, rc); + rc = clr2seckey2(apqns, kcs.apqn_entries, + kcs.type, kcs.size, kcs.keygenflags, + kcs.clrkey.clrkey, kkey, &klen); + pr_debug("clr2seckey2()=%d\n", rc); kfree(apqns); if (rc) { kfree_sensitive(kkey); @@ -1633,10 +800,19 @@ static int pkey_ioctl_verifykey2(struct pkey_verifykey2 __user *uvk) kkey = _copy_key_from_user(kvk.key, kvk.keylen); if (IS_ERR(kkey)) return PTR_ERR(kkey); - rc = pkey_verifykey2(kkey, kvk.keylen, - &kvk.cardnr, &kvk.domain, - &kvk.type, &kvk.size, &kvk.flags); - pr_debug("%s pkey_verifykey2()=%d\n", __func__, rc); + if (pkey_is_cca_key(kkey, kvk.keylen)) { + rc = pkey_cca_verifykey(kkey, kvk.keylen, + &kvk.cardnr, &kvk.domain, + &kvk.type, &kvk.size, &kvk.flags); + pr_debug("pkey_cca_verifykey()=%d\n", rc); + } else if (pkey_is_ep11_key(kkey, kvk.keylen)) { + rc = pkey_ep11_verifykey(kkey, kvk.keylen, + &kvk.cardnr, &kvk.domain, + &kvk.type, &kvk.size, &kvk.flags); + pr_debug("pkey_ep11_verifykey()=%d\n", rc); + } else { + rc = -EINVAL; + } kfree_sensitive(kkey); if (rc) return rc; @@ -1664,11 +840,10 @@ static int pkey_ioctl_kblob2protk2(struct pkey_kblob2pkey2 __user *utp) return PTR_ERR(kkey); } ktp.protkey.len = sizeof(ktp.protkey.protkey); - rc = pkey_keyblob2pkey2(apqns, ktp.apqn_entries, - kkey, ktp.keylen, - ktp.protkey.protkey, &ktp.protkey.len, - &ktp.protkey.type); - pr_debug("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); + rc = key2protkey(apqns, ktp.apqn_entries, kkey, ktp.keylen, + ktp.protkey.protkey, &ktp.protkey.len, + &ktp.protkey.type); + pr_debug("key2protkey()=%d\n", rc); kfree(apqns); kfree_sensitive(kkey); if (!rc && copy_to_user(utp, &ktp, sizeof(ktp))) @@ -1701,9 +876,9 @@ static int pkey_ioctl_apqns4k(struct pkey_apqns4key __user *uak) kfree(apqns); return PTR_ERR(kkey); } - rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, - apqns, &nr_apqns); - pr_debug("%s pkey_apqns4key()=%d\n", __func__, rc); + rc = apqns4key(kkey, kak.keylen, kak.flags, + apqns, &nr_apqns); + pr_debug("apqns4key()=%d\n", rc); kfree_sensitive(kkey); if (rc && rc != -ENOSPC) { kfree(apqns); @@ -1747,9 +922,9 @@ static int pkey_ioctl_apqns4kt(struct pkey_apqns4keytype __user *uat) if (!apqns) return -ENOMEM; } - rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, - kat.flags, apqns, &nr_apqns); - pr_debug("%s pkey_apqns4keytype()=%d\n", __func__, rc); + rc = apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, + kat.flags, apqns, &nr_apqns); + pr_debug("apqns4keytype()=%d\n", rc); if (rc && rc != -ENOSPC) { kfree(apqns); return rc; @@ -1799,10 +974,9 @@ static int pkey_ioctl_kblob2protk3(struct pkey_kblob2pkey3 __user *utp) kfree_sensitive(kkey); return -ENOMEM; } - rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, - kkey, ktp.keylen, - protkey, &protkeylen, &ktp.pkeytype); - pr_debug("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); + rc = key2protkey(apqns, ktp.apqn_entries, kkey, ktp.keylen, + protkey, &protkeylen, &ktp.pkeytype); + pr_debug("key2protkey()=%d\n", rc); kfree(apqns); kfree_sensitive(kkey); if (rc) { @@ -1893,495 +1067,9 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } /* - * Sysfs and file io operations + * File io operations */ -/* - * Sysfs attribute read function for all protected key binary attributes. - * The implementation can not deal with partial reads, because a new random - * protected key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - */ -static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, - loff_t off, size_t count) -{ - struct protaeskeytoken protkeytoken; - struct pkey_protkey protkey; - int rc; - - if (off != 0 || count < sizeof(protkeytoken)) - return -EINVAL; - if (is_xts) - if (count < 2 * sizeof(protkeytoken)) - return -EINVAL; - - memset(&protkeytoken, 0, sizeof(protkeytoken)); - protkeytoken.type = TOKTYPE_NON_CCA; - protkeytoken.version = TOKVER_PROTECTED_KEY; - protkeytoken.keytype = keytype; - - protkey.len = sizeof(protkey.protkey); - rc = pkey_genprotkey(protkeytoken.keytype, - protkey.protkey, &protkey.len, &protkey.type); - if (rc) - return rc; - - protkeytoken.len = protkey.len; - memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); - - memcpy(buf, &protkeytoken, sizeof(protkeytoken)); - - if (is_xts) { - /* xts needs a second protected key, reuse protkey struct */ - protkey.len = sizeof(protkey.protkey); - rc = pkey_genprotkey(protkeytoken.keytype, - protkey.protkey, &protkey.len, &protkey.type); - if (rc) - return rc; - - protkeytoken.len = protkey.len; - memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); - - memcpy(buf + sizeof(protkeytoken), &protkeytoken, - sizeof(protkeytoken)); - - return 2 * sizeof(protkeytoken); - } - - return sizeof(protkeytoken); -} - -static ssize_t protkey_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, - off, count); -} - -static ssize_t protkey_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, - off, count); -} - -static ssize_t protkey_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, - off, count); -} - -static ssize_t protkey_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, - off, count); -} - -static ssize_t protkey_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(protkey_aes_128, sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_192, sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_256, sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_128_xts, 2 * sizeof(struct protaeskeytoken)); -static BIN_ATTR_RO(protkey_aes_256_xts, 2 * sizeof(struct protaeskeytoken)); - -static struct bin_attribute *protkey_attrs[] = { - &bin_attr_protkey_aes_128, - &bin_attr_protkey_aes_192, - &bin_attr_protkey_aes_256, - &bin_attr_protkey_aes_128_xts, - &bin_attr_protkey_aes_256_xts, - NULL -}; - -static struct attribute_group protkey_attr_group = { - .name = "protkey", - .bin_attrs = protkey_attrs, -}; - -/* - * Sysfs attribute read function for all secure key ccadata binary attributes. - * The implementation can not deal with partial reads, because a new random - * protected key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - */ -static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, - loff_t off, size_t count) -{ - struct pkey_seckey *seckey = (struct pkey_seckey *)buf; - int rc; - - if (off != 0 || count < sizeof(struct secaeskeytoken)) - return -EINVAL; - if (is_xts) - if (count < 2 * sizeof(struct secaeskeytoken)) - return -EINVAL; - - rc = cca_genseckey(-1, -1, keytype, seckey->seckey); - if (rc) - return rc; - - if (is_xts) { - seckey++; - rc = cca_genseckey(-1, -1, keytype, seckey->seckey); - if (rc) - return rc; - - return 2 * sizeof(struct secaeskeytoken); - } - - return sizeof(struct secaeskeytoken); -} - -static ssize_t ccadata_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, - off, count); -} - -static ssize_t ccadata_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, - off, count); -} - -static ssize_t ccadata_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, - off, count); -} - -static ssize_t ccadata_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, - off, count); -} - -static ssize_t ccadata_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(ccadata_aes_128, sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_192, sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_256, sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_128_xts, 2 * sizeof(struct secaeskeytoken)); -static BIN_ATTR_RO(ccadata_aes_256_xts, 2 * sizeof(struct secaeskeytoken)); - -static struct bin_attribute *ccadata_attrs[] = { - &bin_attr_ccadata_aes_128, - &bin_attr_ccadata_aes_192, - &bin_attr_ccadata_aes_256, - &bin_attr_ccadata_aes_128_xts, - &bin_attr_ccadata_aes_256_xts, - NULL -}; - -static struct attribute_group ccadata_attr_group = { - .name = "ccadata", - .bin_attrs = ccadata_attrs, -}; - -#define CCACIPHERTOKENSIZE (sizeof(struct cipherkeytoken) + 80) - -/* - * Sysfs attribute read function for all secure key ccacipher binary attributes. - * The implementation can not deal with partial reads, because a new random - * secure key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - */ -static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, - bool is_xts, char *buf, loff_t off, - size_t count) -{ - size_t keysize = CCACIPHERTOKENSIZE; - u32 nr_apqns, *apqns = NULL; - int i, rc, card, dom; - - if (off != 0 || count < CCACIPHERTOKENSIZE) - return -EINVAL; - if (is_xts) - if (count < 2 * CCACIPHERTOKENSIZE) - return -EINVAL; - - /* build a list of apqns able to generate an cipher key */ - rc = cca_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX6, 0, 0, 0, 0); - if (rc) - return rc; - - memset(buf, 0, is_xts ? 2 * keysize : keysize); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); - if (rc == 0) - break; - } - if (rc) - return rc; - - if (is_xts) { - keysize = CCACIPHERTOKENSIZE; - buf += CCACIPHERTOKENSIZE; - rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); - if (rc == 0) - return 2 * CCACIPHERTOKENSIZE; - } - - return CCACIPHERTOKENSIZE; -} - -static ssize_t ccacipher_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, false, buf, - off, count); -} - -static ssize_t ccacipher_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_192, false, buf, - off, count); -} - -static ssize_t ccacipher_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, false, buf, - off, count); -} - -static ssize_t ccacipher_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, true, buf, - off, count); -} - -static ssize_t ccacipher_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(ccacipher_aes_128, CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_192, CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_256, CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_128_xts, 2 * CCACIPHERTOKENSIZE); -static BIN_ATTR_RO(ccacipher_aes_256_xts, 2 * CCACIPHERTOKENSIZE); - -static struct bin_attribute *ccacipher_attrs[] = { - &bin_attr_ccacipher_aes_128, - &bin_attr_ccacipher_aes_192, - &bin_attr_ccacipher_aes_256, - &bin_attr_ccacipher_aes_128_xts, - &bin_attr_ccacipher_aes_256_xts, - NULL -}; - -static struct attribute_group ccacipher_attr_group = { - .name = "ccacipher", - .bin_attrs = ccacipher_attrs, -}; - -/* - * Sysfs attribute read function for all ep11 aes key binary attributes. - * The implementation can not deal with partial reads, because a new random - * secure key blob is generated with each read. In case of partial reads - * (i.e. off != 0 or count < key blob size) -EINVAL is returned. - * This function and the sysfs attributes using it provide EP11 key blobs - * padded to the upper limit of MAXEP11AESKEYBLOBSIZE which is currently - * 336 bytes. - */ -static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, - bool is_xts, char *buf, loff_t off, - size_t count) -{ - size_t keysize = MAXEP11AESKEYBLOBSIZE; - u32 nr_apqns, *apqns = NULL; - int i, rc, card, dom; - - if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) - return -EINVAL; - if (is_xts) - if (count < 2 * MAXEP11AESKEYBLOBSIZE) - return -EINVAL; - - /* build a list of apqns able to generate an cipher key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - NULL); - if (rc) - return rc; - - memset(buf, 0, is_xts ? 2 * keysize : keysize); - - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, - PKEY_TYPE_EP11_AES); - if (rc == 0) - break; - } - if (rc) - return rc; - - if (is_xts) { - keysize = MAXEP11AESKEYBLOBSIZE; - buf += MAXEP11AESKEYBLOBSIZE; - rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, - PKEY_TYPE_EP11_AES); - if (rc == 0) - return 2 * MAXEP11AESKEYBLOBSIZE; - } - - return MAXEP11AESKEYBLOBSIZE; -} - -static ssize_t ep11_aes_128_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, false, buf, - off, count); -} - -static ssize_t ep11_aes_192_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_192, false, buf, - off, count); -} - -static ssize_t ep11_aes_256_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, false, buf, - off, count); -} - -static ssize_t ep11_aes_128_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, true, buf, - off, count); -} - -static ssize_t ep11_aes_256_xts_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *attr, - char *buf, loff_t off, - size_t count) -{ - return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, true, buf, - off, count); -} - -static BIN_ATTR_RO(ep11_aes_128, MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_192, MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_256, MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_128_xts, 2 * MAXEP11AESKEYBLOBSIZE); -static BIN_ATTR_RO(ep11_aes_256_xts, 2 * MAXEP11AESKEYBLOBSIZE); - -static struct bin_attribute *ep11_attrs[] = { - &bin_attr_ep11_aes_128, - &bin_attr_ep11_aes_192, - &bin_attr_ep11_aes_256, - &bin_attr_ep11_aes_128_xts, - &bin_attr_ep11_aes_256_xts, - NULL -}; - -static struct attribute_group ep11_attr_group = { - .name = "ep11", - .bin_attrs = ep11_attrs, -}; - -static const struct attribute_group *pkey_attr_groups[] = { - &protkey_attr_group, - &ccadata_attr_group, - &ccacipher_attr_group, - &ep11_attr_group, - NULL, -}; - static const struct file_operations pkey_fops = { .owner = THIS_MODULE, .open = nonseekable_open, diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h new file mode 100644 index 000000000000..f714d42969b6 --- /dev/null +++ b/drivers/s390/crypto/pkey_base.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright IBM Corp. 2024 + * + * Pkey base: debug feature, defines and structs + * common to all pkey code. + */ + +#ifndef _PKEY_BASE_H_ +#define _PKEY_BASE_H_ + +#include +#include +#include + +/* + * pkey debug feature + */ + +extern debug_info_t *pkey_dbf_info; + +#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__) +#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__) +#define PKEY_DBF_ERR(...) debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__) + +/* + * common defines and common structs + */ + +#define KEYBLOBBUFSIZE 8192 /* key buffer size used for internal processing */ +#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header)) +#define PROTKEYBLOBBUFSIZE 256 /* protected key buffer size used internal */ +#define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ +#define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ + +/* inside view of a protected key token (only type 0x00 version 0x01) */ +struct protaeskeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* should be 0x01 for protected AES key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ + u32 len; /* bytes actually stored in protkey[] */ + u8 protkey[MAXPROTKEYSIZE]; /* the protected key blob */ +} __packed; + +/* inside view of a clear key token (type 0x00 version 0x02) */ +struct clearkeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* 0x02 for clear key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE_* values */ + u32 len; /* bytes actually stored in clearkey[] */ + u8 clearkey[]; /* clear key value */ +} __packed; + +/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */ +static inline u32 pkey_keytype_aes_to_size(u32 keytype) +{ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + return 16; + case PKEY_KEYTYPE_AES_192: + return 24; + case PKEY_KEYTYPE_AES_256: + return 32; + default: + return 0; + } +} + +/* helper function which translates AES key bit size into PKEY_KEYTYPE_AES_* */ +static inline u32 pkey_aes_bitsize_to_keytype(u32 keybitsize) +{ + switch (keybitsize) { + case 128: + return PKEY_KEYTYPE_AES_128; + case 192: + return PKEY_KEYTYPE_AES_192; + case 256: + return PKEY_KEYTYPE_AES_256; + default: + return 0; + } +} + +/* + * pkey_cca.c: + */ + +bool pkey_is_cca_key(const u8 *key, u32 keylen); +bool pkey_is_cca_keytype(enum pkey_key_type); +int pkey_cca_key2protkey(u16 card, u16 dom, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_cca_gen_key(u16 card, u16 dom, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen); +int pkey_cca_clr2key(u16 card, u16 dom, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen); +int pkey_cca_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags); +int pkey_cca_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); +int pkey_cca_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); + +/* + * pkey_ep11.c: + */ + +bool pkey_is_ep11_key(const u8 *key, u32 keylen); +bool pkey_is_ep11_keytype(enum pkey_key_type); +int pkey_ep11_key2protkey(u16 card, u16 dom, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_ep11_gen_key(u16 card, u16 dom, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen); +int pkey_ep11_clr2key(u16 card, u16 dom, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen); +int pkey_ep11_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags); +int pkey_ep11_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); +int pkey_ep11_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); + +/* + * pkey_pckmo.c: + */ + +bool pkey_is_pckmo_key(const u8 *key, u32 keylen); +int pkey_pckmo_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_pckmo_gen_protkey(u32 keytype, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_pckmo_clr2protkey(u32 keytype, const u8 *clrkey, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, + u32 protkeytype); + +/* + * pkey_sysfs.c: + */ + +extern const struct attribute_group *pkey_attr_groups[]; + +#endif /* _PKEY_BASE_H_ */ diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c new file mode 100644 index 000000000000..65e520d7a864 --- /dev/null +++ b/drivers/s390/crypto/pkey_cca.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey cca specific code + * + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "pkey" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include "zcrypt_api.h" +#include "zcrypt_ccamisc.h" + +#include "pkey_base.h" + +/* + * Check key blob for known and supported CCA key. + */ +bool pkey_is_cca_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + + if (keylen < sizeof(*hdr)) + return false; + + switch (hdr->type) { + case TOKTYPE_CCA_INTERNAL: + switch (hdr->version) { + case TOKVER_CCA_AES: + case TOKVER_CCA_VLSC: + return true; + default: + return false; + } + case TOKTYPE_CCA_INTERNAL_PKA: + return true; + default: + return false; + } +} + +bool pkey_is_cca_keytype(enum pkey_key_type key_type) +{ + switch (key_type) { + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + case PKEY_TYPE_CCA_ECC: + return true; + default: + return false; + } +} + +int pkey_cca_key2protkey(u16 card, u16 dom, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + /* CCA AES data key */ + if (keylen != sizeof(struct secaeskeytoken)) + return -EINVAL; + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) + return -EINVAL; + rc = cca_sec2protkey(card, dom, key, protkey, + protkeylen, protkeytype); + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + /* CCA AES cipher key */ + if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) + return -EINVAL; + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) + return -EINVAL; + rc = cca_cipher2protkey(card, dom, key, protkey, + protkeylen, protkeytype); + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + /* CCA ECC (private) key */ + if (keylen < sizeof(struct eccprivkeytoken)) + return -EINVAL; + if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) + return -EINVAL; + rc = cca_ecc2protkey(card, dom, key, protkey, + protkeylen, protkeytype); + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + rc = -EINVAL; + } + + pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); + return rc; +} + +/* + * Generate CCA secure key. + * As of now only CCA AES Data or Cipher secure keys are + * supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +int pkey_cca_gen_key(u16 card, u16 dom, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen) +{ + int len, rc; + + /* check keytype, subtype, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + switch (subtype) { + case 0: + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (subtype == PKEY_TYPE_CCA_CIPHER) { + rc = cca_gencipherkey(card, dom, keybitsize, flags, + keybuf, keybuflen); + } else { + /* 0 or PKEY_TYPE_CCA_DATA */ + rc = cca_genseckey(card, dom, keybitsize, keybuf); + *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); + } + + pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); + return rc; +} + +/* + * Generate CCA secure key with given clear key value. + * As of now only CCA AES Data or Cipher secure keys are + * supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +int pkey_cca_clr2key(u16 card, u16 dom, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen) +{ + int len, rc; + + /* check keytype, subtype, clrkeylen, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + if (clrkeylen != len) { + PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", + __func__, clrkeylen, len); + return -EINVAL; + } + switch (subtype) { + case 0: + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (subtype == PKEY_TYPE_CCA_CIPHER) { + rc = cca_clr2cipherkey(card, dom, keybitsize, + flags, clrkey, keybuf, keybuflen); + } else { + /* 0 or PKEY_TYPE_CCA_DATA */ + rc = cca_clr2seckey(card, dom, keybitsize, + clrkey, keybuf); + *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); + } + + pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); + return rc; +} + +int pkey_cca_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 nr_apqns, *apqns = NULL; + int rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + struct secaeskeytoken *t = (struct secaeskeytoken *)key; + + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); + if (rc) + goto out; + *keytype = PKEY_TYPE_CCA_DATA; + *keybitsize = t->bitsize; + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX3C, AES_MK_SET, + t->mkvp, 0, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + if (rc == -ENODEV) { + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX3C, AES_MK_SET, + 0, t->mkvp, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + } + if (rc) + goto out; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + struct cipherkeytoken *t = (struct cipherkeytoken *)key; + + rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_CCA_CIPHER; + *keybitsize = PKEY_SIZE_UNKNOWN; + if (!t->plfver && t->wpllen == 512) + *keybitsize = PKEY_SIZE_AES_128; + else if (!t->plfver && t->wpllen == 576) + *keybitsize = PKEY_SIZE_AES_192; + else if (!t->plfver && t->wpllen == 640) + *keybitsize = PKEY_SIZE_AES_256; + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX6, AES_MK_SET, + t->mkvp0, 0, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + if (rc == -ENODEV) { + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX6, AES_MK_SET, + 0, t->mkvp0, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + } + if (rc) + goto out; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else { + /* unknown/unsupported key blob */ + rc = -EINVAL; + } + +out: + kfree(apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +int pkey_cca_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 _nr_apqns, *_apqns = NULL; + int rc; + + if (!flags) + flags = PKEY_FLAGS_MATCH_CUR_MKVP | PKEY_FLAGS_MATCH_ALT_MKVP; + + if (keylen < sizeof(struct keytoken_header)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_CCA_INTERNAL) { + u64 cur_mkvp = 0, old_mkvp = 0; + int minhwtype = ZCRYPT_CEX3C; + + if (hdr->version == TOKVER_CCA_AES) { + struct secaeskeytoken *t = (struct secaeskeytoken *)key; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp; + } else if (hdr->version == TOKVER_CCA_VLSC) { + struct cipherkeytoken *t = (struct cipherkeytoken *)key; + + minhwtype = ZCRYPT_CEX6; + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp0; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp0; + } else { + /* unknown CCA internal token type */ + return -EINVAL; + } + rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, AES_MK_SET, + cur_mkvp, old_mkvp, 1); + if (rc) + goto out; + + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + struct eccprivkeytoken *t = (struct eccprivkeytoken *)key; + u64 cur_mkvp = 0, old_mkvp = 0; + + if (t->secid == 0x20) { + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = t->mkvp; + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = t->mkvp; + } else { + /* unknown CCA internal 2 token type */ + return -EINVAL; + } + rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, APKA_MK_SET, + cur_mkvp, old_mkvp, 1); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + kfree(_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +int pkey_cca_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) +{ + u32 _nr_apqns, *_apqns = NULL; + int rc; + + zcrypt_wait_api_operational(); + + if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) { + u64 cur_mkvp = 0, old_mkvp = 0; + int minhwtype = ZCRYPT_CEX3C; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = *((u64 *)cur_mkvp); + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = *((u64 *)alt_mkvp); + if (ktype == PKEY_TYPE_CCA_CIPHER) + minhwtype = ZCRYPT_CEX6; + rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, AES_MK_SET, + cur_mkvp, old_mkvp, 1); + if (rc) + goto out; + + } else if (ktype == PKEY_TYPE_CCA_ECC) { + u64 cur_mkvp = 0, old_mkvp = 0; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + cur_mkvp = *((u64 *)cur_mkvp); + if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) + old_mkvp = *((u64 *)alt_mkvp); + rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, APKA_MK_SET, + cur_mkvp, old_mkvp, 1); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported key type %d", + __func__, (int)ktype); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + kfree(_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c new file mode 100644 index 000000000000..fdc9de43a6c1 --- /dev/null +++ b/drivers/s390/crypto/pkey_ep11.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey ep11 specific code + * + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "pkey" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include "zcrypt_api.h" +#include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" + +#include "pkey_base.h" + +/* + * Check key blob for known and supported EP11 key. + */ +bool pkey_is_ep11_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + + if (keylen < sizeof(*hdr)) + return false; + + switch (hdr->type) { + case TOKTYPE_NON_CCA: + switch (hdr->version) { + case TOKVER_EP11_AES: + case TOKVER_EP11_AES_WITH_HEADER: + case TOKVER_EP11_ECC_WITH_HEADER: + return true; + default: + return false; + } + default: + return false; + } +} + +bool pkey_is_ep11_keytype(enum pkey_key_type key_type) +{ + switch (key_type) { + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + case PKEY_TYPE_EP11_ECC: + return true; + default: + return false; + } +} + +int pkey_ep11_key2protkey(u16 card, u16 dom, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 AES key blob with header */ + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) + return -EINVAL; + rc = ep11_kblob2protkey(card, dom, key, hdr->len, + protkey, protkeylen, + protkeytype); + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_ECC_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 ECC key blob with header */ + if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) + return -EINVAL; + rc = ep11_kblob2protkey(card, dom, key, hdr->len, + protkey, protkeylen, + protkeytype); + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + /* EP11 AES key blob with header in session field */ + if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) + return -EINVAL; + rc = ep11_kblob2protkey(card, dom, key, hdr->len, + protkey, protkeylen, + protkeytype); + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); + return rc; +} + +/* + * Generate EP11 secure key. + * As of now only EP11 AES secure keys are supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES + * or 0 (results in subtype PKEY_TYPE_EP11_AES), + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +int pkey_ep11_gen_key(u16 card, u16 dom, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen) +{ + int len, rc; + + /* check keytype, subtype, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + switch (subtype) { + case 0: + subtype = PKEY_TYPE_EP11_AES; + break; + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + rc = ep11_genaeskey(card, dom, keybitsize, flags, + keybuf, keybuflen, subtype); + + pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); + return rc; +} + +/* + * Generate EP11 secure key with given clear key value. + * As of now only EP11 AES secure keys are supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES + * or 0 (assumes PKEY_TYPE_EP11_AES then). + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +int pkey_ep11_clr2key(u16 card, u16 dom, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen) +{ + int len, rc; + + /* check keytype, subtype, clrkeylen, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + if (clrkeylen != len) { + PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", + __func__, clrkeylen, len); + return -EINVAL; + } + switch (subtype) { + case 0: + subtype = PKEY_TYPE_EP11_AES; + break; + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + rc = ep11_clr2keyblob(card, dom, keybitsize, flags, + clrkey, keybuf, keybuflen, subtype); + + pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); + return rc; +} + +int pkey_ep11_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 nr_apqns, *apqns = NULL; + int rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES) { + struct ep11keyblob *kb = (struct ep11keyblob *)key; + int api; + + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_EP11; + *keybitsize = kb->head.bitlen; + + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX7, api, + ep11_kb_wkvp(key, keylen)); + if (rc) + goto out; + + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER) { + struct ep11kblob_header *kh = (struct ep11kblob_header *)key; + int api; + + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_EP11_AES; + *keybitsize = kh->bitlen; + + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX7, api, + ep11_kb_wkvp(key, keylen)); + if (rc) + goto out; + + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else { + /* unknown/unsupported key blob */ + rc = -EINVAL; + } + +out: + kfree(apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +int pkey_ep11_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 _nr_apqns, *_apqns = NULL; + int rc; + + if (!flags) + flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + if (keylen < sizeof(struct keytoken_header) || flags == 0) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_NON_CCA && + (hdr->version == TOKVER_EP11_AES_WITH_HEADER || + hdr->version == TOKVER_EP11_ECC_WITH_HEADER) && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + struct ep11keyblob *kb = (struct ep11keyblob *) + (key + sizeof(struct ep11kblob_header)); + int minhwtype = 0, api = 0; + + if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) + return -EINVAL; + if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { + minhwtype = ZCRYPT_CEX7; + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + } + rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, api, kb->wkvp); + if (rc) + goto out; + + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + struct ep11keyblob *kb = (struct ep11keyblob *)key; + int minhwtype = 0, api = 0; + + if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) + return -EINVAL; + if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { + minhwtype = ZCRYPT_CEX7; + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + } + rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, api, kb->wkvp); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + kfree(_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +int pkey_ep11_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) +{ + u32 _nr_apqns, *_apqns = NULL; + int rc; + + zcrypt_wait_api_operational(); + + if (ktype == PKEY_TYPE_EP11 || + ktype == PKEY_TYPE_EP11_AES || + ktype == PKEY_TYPE_EP11_ECC) { + u8 *wkvp = NULL; + int api; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + wkvp = cur_mkvp; + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, api, wkvp); + if (rc) + goto out; + + } else { + PKEY_DBF_ERR("%s unknown/unsupported key type %d\n", + __func__, (int)ktype); + return -EINVAL; + } + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); + } + *nr_apqns = _nr_apqns; + +out: + kfree(_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c new file mode 100644 index 000000000000..30a9d2f64853 --- /dev/null +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey pckmo specific code + * + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "pkey" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include + +#include "zcrypt_api.h" +#include "zcrypt_ccamisc.h" + +#include "pkey_base.h" + +/* + * Check key blob for known and supported here. + */ +bool pkey_is_pckmo_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct clearkeytoken *t = (struct clearkeytoken *)key; + + if (keylen < sizeof(*hdr)) + return false; + + switch (hdr->type) { + case TOKTYPE_NON_CCA: + switch (hdr->version) { + case TOKVER_CLEAR_KEY: + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + case PKEY_KEYTYPE_ECC_P256: + case PKEY_KEYTYPE_ECC_P384: + case PKEY_KEYTYPE_ECC_P521: + case PKEY_KEYTYPE_ECC_ED25519: + case PKEY_KEYTYPE_ECC_ED448: + return true; + default: + return false; + } + case TOKVER_PROTECTED_KEY: + return true; + default: + return false; + } + default: + return false; + } +} + +int pkey_pckmo_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc = -EINVAL; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + if (hdr->type != TOKTYPE_NON_CCA) + return -EINVAL; + + switch (hdr->version) { + case TOKVER_PROTECTED_KEY: { + struct protaeskeytoken *t; + + if (keylen != sizeof(struct protaeskeytoken)) + goto out; + t = (struct protaeskeytoken *)key; + rc = pkey_pckmo_verify_protkey(t->protkey, t->len, + t->keytype); + if (rc) + goto out; + memcpy(protkey, t->protkey, t->len); + *protkeylen = t->len; + *protkeytype = t->keytype; + break; + } + case TOKVER_CLEAR_KEY: { + struct clearkeytoken *t = (struct clearkeytoken *)key; + u32 keysize = 0; + + if (keylen < sizeof(struct clearkeytoken) || + keylen != sizeof(*t) + t->len) + goto out; + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + keysize = pkey_keytype_aes_to_size(t->keytype); + break; + case PKEY_KEYTYPE_ECC_P256: + keysize = 32; + break; + case PKEY_KEYTYPE_ECC_P384: + keysize = 48; + break; + case PKEY_KEYTYPE_ECC_P521: + keysize = 80; + break; + case PKEY_KEYTYPE_ECC_ED25519: + keysize = 32; + break; + case PKEY_KEYTYPE_ECC_ED448: + keysize = 64; + break; + default: + break; + } + if (!keysize) { + PKEY_DBF_ERR("%s clear key token: unknown keytype %u\n", + __func__, t->keytype); + goto out; + } + if (t->len != keysize) { + PKEY_DBF_ERR("%s clear key token: invalid key len %u\n", + __func__, t->len); + goto out; + } + rc = pkey_pckmo_clr2protkey(t->keytype, t->clearkey, + protkey, protkeylen, protkeytype); + break; + } + default: + PKEY_DBF_ERR("%s unknown non-CCA token version %d\n", + __func__, hdr->version); + break; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate a random protected key. + * Currently only the generation of AES protected keys + * is supported. + */ +int pkey_pckmo_gen_protkey(u32 keytype, u8 *protkey, + u32 *protkeylen, u32 *protkeytype) +{ + u8 clrkey[32]; + int keysize; + int rc; + + keysize = pkey_keytype_aes_to_size(keytype); + if (!keysize) { + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, + keytype); + return -EINVAL; + } + + /* generate a dummy random clear key */ + get_random_bytes(clrkey, keysize); + + /* convert it to a dummy protected key */ + rc = pkey_pckmo_clr2protkey(keytype, clrkey, + protkey, protkeylen, protkeytype); + if (rc) + goto out; + + /* replace the key part of the protected key with random bytes */ + get_random_bytes(protkey, keysize); + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Create a protected key from a clear key value via PCKMO instruction. + */ +int pkey_pckmo_clr2protkey(u32 keytype, const u8 *clrkey, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + /* mask of available pckmo subfunctions */ + static cpacf_mask_t pckmo_functions; + + int keysize, rc = -EINVAL; + u8 paramblock[112]; + u32 pkeytype; + long fc; + + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + /* 16 byte key, 32 byte aes wkvp, total 48 bytes */ + keysize = 16; + pkeytype = keytype; + fc = CPACF_PCKMO_ENC_AES_128_KEY; + break; + case PKEY_KEYTYPE_AES_192: + /* 24 byte key, 32 byte aes wkvp, total 56 bytes */ + keysize = 24; + pkeytype = keytype; + fc = CPACF_PCKMO_ENC_AES_192_KEY; + break; + case PKEY_KEYTYPE_AES_256: + /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ + keysize = 32; + pkeytype = keytype; + fc = CPACF_PCKMO_ENC_AES_256_KEY; + break; + case PKEY_KEYTYPE_ECC_P256: + /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ + keysize = 32; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P256_KEY; + break; + case PKEY_KEYTYPE_ECC_P384: + /* 48 byte key, 32 byte aes wkvp, total 80 bytes */ + keysize = 48; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P384_KEY; + break; + case PKEY_KEYTYPE_ECC_P521: + /* 80 byte key, 32 byte aes wkvp, total 112 bytes */ + keysize = 80; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_P521_KEY; + break; + case PKEY_KEYTYPE_ECC_ED25519: + /* 32 byte key, 32 byte aes wkvp, total 64 bytes */ + keysize = 32; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY; + break; + case PKEY_KEYTYPE_ECC_ED448: + /* 64 byte key, 32 byte aes wkvp, total 96 bytes */ + keysize = 64; + pkeytype = PKEY_KEYTYPE_ECC; + fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, keytype); + goto out; + } + + if (*protkeylen < keysize + AES_WK_VP_SIZE) { + PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n", + __func__, *protkeylen, keysize + AES_WK_VP_SIZE); + goto out; + } + + /* Did we already check for PCKMO ? */ + if (!pckmo_functions.bytes[0]) { + /* no, so check now */ + if (!cpacf_query(CPACF_PCKMO, &pckmo_functions)) { + PKEY_DBF_ERR("%s cpacf_query() failed\n", __func__); + rc = -ENODEV; + goto out; + } + } + /* check for the pckmo subfunction we need now */ + if (!cpacf_test_func(&pckmo_functions, fc)) { + PKEY_DBF_ERR("%s pckmo functions not available\n", __func__); + rc = -ENODEV; + goto out; + } + + /* prepare param block */ + memset(paramblock, 0, sizeof(paramblock)); + memcpy(paramblock, clrkey, keysize); + + /* call the pckmo instruction */ + cpacf_pckmo(fc, paramblock); + + /* copy created protected key to key buffer including the wkvp block */ + *protkeylen = keysize + AES_WK_VP_SIZE; + memcpy(protkey, paramblock, *protkeylen); + *protkeytype = pkeytype; + + rc = 0; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Verify a protected key blob. + * Currently only AES protected keys are supported. + */ +int pkey_pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, + u32 protkeytype) +{ + struct { + u8 iv[AES_BLOCK_SIZE]; + u8 key[MAXPROTKEYSIZE]; + } param; + u8 null_msg[AES_BLOCK_SIZE]; + u8 dest_buf[AES_BLOCK_SIZE]; + unsigned int k, pkeylen; + unsigned long fc; + int rc = -EINVAL; + + switch (protkeytype) { + case PKEY_KEYTYPE_AES_128: + pkeylen = 16 + AES_WK_VP_SIZE; + fc = CPACF_KMC_PAES_128; + break; + case PKEY_KEYTYPE_AES_192: + pkeylen = 24 + AES_WK_VP_SIZE; + fc = CPACF_KMC_PAES_192; + break; + case PKEY_KEYTYPE_AES_256: + pkeylen = 32 + AES_WK_VP_SIZE; + fc = CPACF_KMC_PAES_256; + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__, + protkeytype); + goto out; + } + if (protkeylen != pkeylen) { + PKEY_DBF_ERR("%s invalid protected key size %u for keytype %u\n", + __func__, protkeylen, protkeytype); + goto out; + } + + memset(null_msg, 0, sizeof(null_msg)); + + memset(param.iv, 0, sizeof(param.iv)); + memcpy(param.key, protkey, protkeylen); + + k = cpacf_kmc(fc | CPACF_ENCRYPT, ¶m, null_msg, dest_buf, + sizeof(null_msg)); + if (k != sizeof(null_msg)) { + PKEY_DBF_ERR("%s protected key is not valid\n", __func__); + rc = -EKEYREJECTED; + goto out; + } + + rc = 0; + +out: + pr_debug("rc=%d\n", rc); + return rc; +} diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c new file mode 100644 index 000000000000..727293fbf331 --- /dev/null +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey module sysfs related functions + * + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "pkey" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include + +#include "zcrypt_api.h" +#include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" + +#include "pkey_base.h" + +/* + * Sysfs attribute read function for all protected key binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, + loff_t off, size_t count) +{ + struct protaeskeytoken protkeytoken; + struct pkey_protkey protkey; + int rc; + + if (off != 0 || count < sizeof(protkeytoken)) + return -EINVAL; + if (is_xts) + if (count < 2 * sizeof(protkeytoken)) + return -EINVAL; + + memset(&protkeytoken, 0, sizeof(protkeytoken)); + protkeytoken.type = TOKTYPE_NON_CCA; + protkeytoken.version = TOKVER_PROTECTED_KEY; + protkeytoken.keytype = keytype; + + protkey.len = sizeof(protkey.protkey); + rc = pkey_pckmo_gen_protkey(protkeytoken.keytype, + protkey.protkey, &protkey.len, + &protkey.type); + if (rc) + return rc; + + protkeytoken.len = protkey.len; + memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); + + memcpy(buf, &protkeytoken, sizeof(protkeytoken)); + + if (is_xts) { + /* xts needs a second protected key, reuse protkey struct */ + protkey.len = sizeof(protkey.protkey); + rc = pkey_pckmo_gen_protkey(protkeytoken.keytype, + protkey.protkey, &protkey.len, + &protkey.type); + if (rc) + return rc; + + protkeytoken.len = protkey.len; + memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len); + + memcpy(buf + sizeof(protkeytoken), &protkeytoken, + sizeof(protkeytoken)); + + return 2 * sizeof(protkeytoken); + } + + return sizeof(protkeytoken); +} + +static ssize_t protkey_aes_128_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, + off, count); +} + +static ssize_t protkey_aes_192_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, + off, count); +} + +static ssize_t protkey_aes_256_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, + off, count); +} + +static ssize_t protkey_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, + off, count); +} + +static ssize_t protkey_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, + off, count); +} + +static BIN_ATTR_RO(protkey_aes_128, sizeof(struct protaeskeytoken)); +static BIN_ATTR_RO(protkey_aes_192, sizeof(struct protaeskeytoken)); +static BIN_ATTR_RO(protkey_aes_256, sizeof(struct protaeskeytoken)); +static BIN_ATTR_RO(protkey_aes_128_xts, 2 * sizeof(struct protaeskeytoken)); +static BIN_ATTR_RO(protkey_aes_256_xts, 2 * sizeof(struct protaeskeytoken)); + +static struct bin_attribute *protkey_attrs[] = { + &bin_attr_protkey_aes_128, + &bin_attr_protkey_aes_192, + &bin_attr_protkey_aes_256, + &bin_attr_protkey_aes_128_xts, + &bin_attr_protkey_aes_256_xts, + NULL +}; + +static struct attribute_group protkey_attr_group = { + .name = "protkey", + .bin_attrs = protkey_attrs, +}; + +/* + * Sysfs attribute read function for all secure key ccadata binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, + loff_t off, size_t count) +{ + struct pkey_seckey *seckey = (struct pkey_seckey *)buf; + int rc; + + if (off != 0 || count < sizeof(struct secaeskeytoken)) + return -EINVAL; + if (is_xts) + if (count < 2 * sizeof(struct secaeskeytoken)) + return -EINVAL; + + rc = cca_genseckey(-1, -1, keytype, seckey->seckey); + if (rc) + return rc; + + if (is_xts) { + seckey++; + rc = cca_genseckey(-1, -1, keytype, seckey->seckey); + if (rc) + return rc; + + return 2 * sizeof(struct secaeskeytoken); + } + + return sizeof(struct secaeskeytoken); +} + +static ssize_t ccadata_aes_128_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf, + off, count); +} + +static ssize_t ccadata_aes_192_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf, + off, count); +} + +static ssize_t ccadata_aes_256_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf, + off, count); +} + +static ssize_t ccadata_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf, + off, count); +} + +static ssize_t ccadata_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf, + off, count); +} + +static BIN_ATTR_RO(ccadata_aes_128, sizeof(struct secaeskeytoken)); +static BIN_ATTR_RO(ccadata_aes_192, sizeof(struct secaeskeytoken)); +static BIN_ATTR_RO(ccadata_aes_256, sizeof(struct secaeskeytoken)); +static BIN_ATTR_RO(ccadata_aes_128_xts, 2 * sizeof(struct secaeskeytoken)); +static BIN_ATTR_RO(ccadata_aes_256_xts, 2 * sizeof(struct secaeskeytoken)); + +static struct bin_attribute *ccadata_attrs[] = { + &bin_attr_ccadata_aes_128, + &bin_attr_ccadata_aes_192, + &bin_attr_ccadata_aes_256, + &bin_attr_ccadata_aes_128_xts, + &bin_attr_ccadata_aes_256_xts, + NULL +}; + +static struct attribute_group ccadata_attr_group = { + .name = "ccadata", + .bin_attrs = ccadata_attrs, +}; + +#define CCACIPHERTOKENSIZE (sizeof(struct cipherkeytoken) + 80) + +/* + * Sysfs attribute read function for all secure key ccacipher binary attributes. + * The implementation can not deal with partial reads, because a new random + * secure key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, + bool is_xts, char *buf, loff_t off, + size_t count) +{ + u32 keysize = CCACIPHERTOKENSIZE; + u32 nr_apqns, *apqns = NULL; + int i, rc, card, dom; + + if (off != 0 || count < CCACIPHERTOKENSIZE) + return -EINVAL; + if (is_xts) + if (count < 2 * CCACIPHERTOKENSIZE) + return -EINVAL; + + /* build a list of apqns able to generate an cipher key */ + rc = cca_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX6, 0, 0, 0, 0); + if (rc) + return rc; + + memset(buf, 0, is_xts ? 2 * keysize : keysize); + + /* simple try all apqns from the list */ + for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { + card = apqns[i] >> 16; + dom = apqns[i] & 0xFFFF; + rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); + if (rc == 0) + break; + } + if (rc) + return rc; + + if (is_xts) { + keysize = CCACIPHERTOKENSIZE; + buf += CCACIPHERTOKENSIZE; + rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); + if (rc == 0) + return 2 * CCACIPHERTOKENSIZE; + } + + return CCACIPHERTOKENSIZE; +} + +static ssize_t ccacipher_aes_128_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, false, buf, + off, count); +} + +static ssize_t ccacipher_aes_192_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_192, false, buf, + off, count); +} + +static ssize_t ccacipher_aes_256_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, false, buf, + off, count); +} + +static ssize_t ccacipher_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, true, buf, + off, count); +} + +static ssize_t ccacipher_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, true, buf, + off, count); +} + +static BIN_ATTR_RO(ccacipher_aes_128, CCACIPHERTOKENSIZE); +static BIN_ATTR_RO(ccacipher_aes_192, CCACIPHERTOKENSIZE); +static BIN_ATTR_RO(ccacipher_aes_256, CCACIPHERTOKENSIZE); +static BIN_ATTR_RO(ccacipher_aes_128_xts, 2 * CCACIPHERTOKENSIZE); +static BIN_ATTR_RO(ccacipher_aes_256_xts, 2 * CCACIPHERTOKENSIZE); + +static struct bin_attribute *ccacipher_attrs[] = { + &bin_attr_ccacipher_aes_128, + &bin_attr_ccacipher_aes_192, + &bin_attr_ccacipher_aes_256, + &bin_attr_ccacipher_aes_128_xts, + &bin_attr_ccacipher_aes_256_xts, + NULL +}; + +static struct attribute_group ccacipher_attr_group = { + .name = "ccacipher", + .bin_attrs = ccacipher_attrs, +}; + +/* + * Sysfs attribute read function for all ep11 aes key binary attributes. + * The implementation can not deal with partial reads, because a new random + * secure key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + * This function and the sysfs attributes using it provide EP11 key blobs + * padded to the upper limit of MAXEP11AESKEYBLOBSIZE which is currently + * 336 bytes. + */ +static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, + bool is_xts, char *buf, loff_t off, + size_t count) +{ + u32 keysize = MAXEP11AESKEYBLOBSIZE; + u32 nr_apqns, *apqns = NULL; + int i, rc, card, dom; + + if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) + return -EINVAL; + if (is_xts) + if (count < 2 * MAXEP11AESKEYBLOBSIZE) + return -EINVAL; + + /* build a list of apqns able to generate an cipher key */ + rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, + ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, + NULL); + if (rc) + return rc; + + memset(buf, 0, is_xts ? 2 * keysize : keysize); + + /* simple try all apqns from the list */ + for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { + card = apqns[i] >> 16; + dom = apqns[i] & 0xFFFF; + rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, + PKEY_TYPE_EP11_AES); + if (rc == 0) + break; + } + if (rc) + return rc; + + if (is_xts) { + keysize = MAXEP11AESKEYBLOBSIZE; + buf += MAXEP11AESKEYBLOBSIZE; + rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, + PKEY_TYPE_EP11_AES); + if (rc == 0) + return 2 * MAXEP11AESKEYBLOBSIZE; + } + + return MAXEP11AESKEYBLOBSIZE; +} + +static ssize_t ep11_aes_128_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, false, buf, + off, count); +} + +static ssize_t ep11_aes_192_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_192, false, buf, + off, count); +} + +static ssize_t ep11_aes_256_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, false, buf, + off, count); +} + +static ssize_t ep11_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, true, buf, + off, count); +} + +static ssize_t ep11_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, true, buf, + off, count); +} + +static BIN_ATTR_RO(ep11_aes_128, MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_192, MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_256, MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_128_xts, 2 * MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_256_xts, 2 * MAXEP11AESKEYBLOBSIZE); + +static struct bin_attribute *ep11_attrs[] = { + &bin_attr_ep11_aes_128, + &bin_attr_ep11_aes_192, + &bin_attr_ep11_aes_256, + &bin_attr_ep11_aes_128_xts, + &bin_attr_ep11_aes_256_xts, + NULL +}; + +static struct attribute_group ep11_attr_group = { + .name = "ep11", + .bin_attrs = ep11_attrs, +}; + +const struct attribute_group *pkey_attr_groups[] = { + &protkey_attr_group, + &ccadata_attr_group, + &ccacipher_attr_group, + &ep11_attr_group, + NULL, +}; diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 7bef2cc4e461..43a27cb3db84 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -172,7 +172,7 @@ EXPORT_SYMBOL(cca_check_secaescipherkey); * key token. Returns 0 on success or errno value on failure. */ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, - const u8 *token, size_t keysize, + const u8 *token, u32 keysize, int checkcpacfexport) { struct eccprivkeytoken *t = (struct eccprivkeytoken *)token; @@ -187,7 +187,7 @@ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, } if (t->len > keysize) { if (dbg) - DBF("%s token check failed, len %d > keysize %zu\n", + DBF("%s token check failed, len %d > keysize %u\n", __func__, (int)t->len, keysize); return -EINVAL; } @@ -737,7 +737,7 @@ static const u8 aes_cipher_key_skeleton[] = { * Generate (random) CCA AES CIPHER secure key. */ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize) + u8 *keybuf, u32 *keybufsize) { int rc; u8 *mem, *ptr; @@ -1085,7 +1085,7 @@ out: * Build CCA AES CIPHER secure key with a given clear key value. */ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize) + const u8 *clrkey, u8 *keybuf, u32 *keybufsize) { int rc; u8 *token; diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h index 5ddf02f965f9..aed7e8384542 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.h +++ b/drivers/s390/crypto/zcrypt_ccamisc.h @@ -153,7 +153,7 @@ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, * key token. Returns 0 on success or errno value on failure. */ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, - const u8 *token, size_t keysize, + const u8 *token, u32 keysize, int checkcpacfexport); /* @@ -178,7 +178,7 @@ int cca_sec2protkey(u16 cardnr, u16 domain, * Generate (random) CCA AES CIPHER secure key. */ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize); + u8 *keybuf, u32 *keybufsize); /* * Derive proteced key from CCA AES cipher secure key. @@ -190,7 +190,7 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, * Build CCA AES CIPHER secure key with a given clear key value. */ int cca_clr2cipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize); + const u8 *clrkey, u8 *keybuf, u32 *keybufsize); /* * Derive proteced key from CCA ECC secure private key. diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index b43db17a4e0e..cb7e6da43602 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -203,7 +203,7 @@ out: * For valid ep11 keyblobs, returns a reference to the wrappingkey verification * pattern. Otherwise NULL. */ -const u8 *ep11_kb_wkvp(const u8 *keyblob, size_t keybloblen) +const u8 *ep11_kb_wkvp(const u8 *keyblob, u32 keybloblen) { struct ep11keyblob *kb; @@ -217,7 +217,7 @@ EXPORT_SYMBOL(ep11_kb_wkvp); * Simple check if the key blob is a valid EP11 AES key blob with header. */ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp) + const u8 *key, u32 keylen, int checkcpacfexp) { struct ep11kblob_header *hdr = (struct ep11kblob_header *)key; struct ep11keyblob *kb = (struct ep11keyblob *)(key + sizeof(*hdr)); @@ -225,7 +225,7 @@ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) if (keylen < sizeof(*hdr) + sizeof(*kb)) { - DBF("%s key check failed, keylen %zu < %zu\n", + DBF("%s key check failed, keylen %u < %zu\n", __func__, keylen, sizeof(*hdr) + sizeof(*kb)); return -EINVAL; } @@ -250,7 +250,7 @@ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, } if (hdr->len > keylen) { if (dbg) - DBF("%s key check failed, header len %d keylen %zu mismatch\n", + DBF("%s key check failed, header len %d keylen %u mismatch\n", __func__, (int)hdr->len, keylen); return -EINVAL; } @@ -284,7 +284,7 @@ EXPORT_SYMBOL(ep11_check_aes_key_with_hdr); * Simple check if the key blob is a valid EP11 ECC key blob with header. */ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp) + const u8 *key, u32 keylen, int checkcpacfexp) { struct ep11kblob_header *hdr = (struct ep11kblob_header *)key; struct ep11keyblob *kb = (struct ep11keyblob *)(key + sizeof(*hdr)); @@ -292,7 +292,7 @@ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) if (keylen < sizeof(*hdr) + sizeof(*kb)) { - DBF("%s key check failed, keylen %zu < %zu\n", + DBF("%s key check failed, keylen %u < %zu\n", __func__, keylen, sizeof(*hdr) + sizeof(*kb)); return -EINVAL; } @@ -317,7 +317,7 @@ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, } if (hdr->len > keylen) { if (dbg) - DBF("%s key check failed, header len %d keylen %zu mismatch\n", + DBF("%s key check failed, header len %d keylen %u mismatch\n", __func__, (int)hdr->len, keylen); return -EINVAL; } @@ -352,14 +352,14 @@ EXPORT_SYMBOL(ep11_check_ecc_key_with_hdr); * the header in the session field (old style EP11 AES key). */ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp) + const u8 *key, u32 keylen, int checkcpacfexp) { struct ep11keyblob *kb = (struct ep11keyblob *)key; #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) if (keylen < sizeof(*kb)) { - DBF("%s key check failed, keylen %zu < %zu\n", + DBF("%s key check failed, keylen %u < %zu\n", __func__, keylen, sizeof(*kb)); return -EINVAL; } @@ -378,7 +378,7 @@ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, } if (kb->head.len > keylen) { if (dbg) - DBF("%s key check failed, header len %d keylen %zu mismatch\n", + DBF("%s key check failed, header len %d keylen %u mismatch\n", __func__, (int)kb->head.len, keylen); return -EINVAL; } @@ -932,7 +932,7 @@ out: } int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize, u32 keybufver) + u8 *keybuf, u32 *keybufsize, u32 keybufver) { struct ep11kblob_header *hdr; size_t hdr_size, pl_size; @@ -1256,7 +1256,7 @@ static int ep11_unwrapkey(u16 card, u16 domain, const u8 *enckey, size_t enckeysize, u32 mech, const u8 *iv, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize, + u8 *keybuf, u32 *keybufsize, u8 keybufver) { struct ep11kblob_header *hdr; @@ -1412,7 +1412,7 @@ out: } int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize, + const u8 *clrkey, u8 *keybuf, u32 *keybufsize, u32 keytype) { int rc; @@ -1471,7 +1471,7 @@ out: EXPORT_SYMBOL(ep11_clr2keyblob); int ep11_kblob2protkey(u16 card, u16 dom, - const u8 *keyblob, size_t keybloblen, + const u8 *keyblob, u32 keybloblen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct ep11kblob_header *hdr; diff --git a/drivers/s390/crypto/zcrypt_ep11misc.h b/drivers/s390/crypto/zcrypt_ep11misc.h index 9d17fd5228a7..9f1bdffdec68 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.h +++ b/drivers/s390/crypto/zcrypt_ep11misc.h @@ -54,7 +54,7 @@ static inline bool is_ep11_keyblob(const u8 *key) * For valid ep11 keyblobs, returns a reference to the wrappingkey verification * pattern. Otherwise NULL. */ -const u8 *ep11_kb_wkvp(const u8 *kblob, size_t kbloblen); +const u8 *ep11_kb_wkvp(const u8 *kblob, u32 kbloblen); /* * Simple check if the key blob is a valid EP11 AES key blob with header. @@ -63,7 +63,7 @@ const u8 *ep11_kb_wkvp(const u8 *kblob, size_t kbloblen); * Returns 0 on success or errno value on failure. */ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp); + const u8 *key, u32 keylen, int checkcpacfexp); /* * Simple check if the key blob is a valid EP11 ECC key blob with header. @@ -72,7 +72,7 @@ int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl, * Returns 0 on success or errno value on failure. */ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp); + const u8 *key, u32 keylen, int checkcpacfexp); /* * Simple check if the key blob is a valid EP11 AES key blob with @@ -82,7 +82,7 @@ int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl, * Returns 0 on success or errno value on failure. */ int ep11_check_aes_key(debug_info_t *dbg, int dbflvl, - const u8 *key, size_t keylen, int checkcpacfexp); + const u8 *key, u32 keylen, int checkcpacfexp); /* EP11 card info struct */ struct ep11_card_info { @@ -115,13 +115,13 @@ int ep11_get_domain_info(u16 card, u16 domain, struct ep11_domain_info *info); * Generate (random) EP11 AES secure key. */ int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, - u8 *keybuf, size_t *keybufsize, u32 keybufver); + u8 *keybuf, u32 *keybufsize, u32 keybufver); /* * Generate EP11 AES secure key with given clear key value. */ int ep11_clr2keyblob(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, - const u8 *clrkey, u8 *keybuf, size_t *keybufsize, + const u8 *clrkey, u8 *keybuf, u32 *keybufsize, u32 keytype); /* @@ -149,7 +149,7 @@ int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, /* * Derive proteced key from EP11 key blob (AES and ECC keys). */ -int ep11_kblob2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, +int ep11_kblob2protkey(u16 card, u16 dom, const u8 *key, u32 keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype); void zcrypt_ep11misc_exit(void); From ea88e1710a9f19345c94c195f9cd7365e50343b0 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 22 Aug 2024 11:32:18 +0200 Subject: [PATCH 39/84] s390/pkey: Unify pkey cca, ep11 and pckmo functions signatures As a preparation step for introducing a common function API between the pkey API module and the handlers (that is the cca, ep11 and pckmo code) this patch unifies the functions signatures exposed by the handlers and reworks all the invocation code of these functions. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/pkey_api.c | 49 ++++++------ drivers/s390/crypto/pkey_base.h | 29 ++++--- drivers/s390/crypto/pkey_cca.c | 4 +- drivers/s390/crypto/pkey_ep11.c | 13 ++-- drivers/s390/crypto/pkey_pckmo.c | 91 ++++++++++++++++++---- drivers/s390/crypto/pkey_sysfs.c | 125 +++++++++++++++++++++---------- 6 files changed, 213 insertions(+), 98 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 732437bf3823..31382c23ec14 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -109,7 +109,7 @@ static int genseck2(const struct pkey_apqn *apqns, size_t nr_apqns, rc = pkey_cca_gen_key(apqns[i].card, apqns[i].domain, u, keytype, keybitsize, flags, - keybuf, keybuflen); + keybuf, keybuflen, NULL); } } else if (pkey_is_ep11_keytype(keytype)) { /* As of now only EP11 AES key generation is supported */ @@ -123,7 +123,7 @@ static int genseck2(const struct pkey_apqn *apqns, size_t nr_apqns, rc = pkey_ep11_gen_key(apqns[i].card, apqns[i].domain, u, keytype, keybitsize, flags, - keybuf, keybuflen); + keybuf, keybuflen, NULL); } } else { PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", @@ -154,7 +154,7 @@ static int clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, apqns[i].domain, u, keytype, kbitsize, flags, clrkey, kbitsize / 8, - keybuf, keybuflen); + keybuf, keybuflen, NULL); } } else if (pkey_is_ep11_keytype(keytype)) { /* As of now only EP11 AES key generation is supported */ @@ -169,7 +169,7 @@ static int clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, apqns[i].domain, u, keytype, kbitsize, flags, clrkey, kbitsize / 8, - keybuf, keybuflen); + keybuf, keybuflen, NULL); } } else { PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", @@ -308,6 +308,7 @@ static int pckmokey2protkey_fallback(const struct clearkeytoken *t, nr_apqns = MAXAPQNSINLIST; rc = pkey_cca_apqns4type(PKEY_TYPE_CCA_DATA, NULL, NULL, 0, apqns, &nr_apqns); + pr_debug("pkey_cca_apqns4type(CCA_DATA)=%d\n", rc); if (rc) goto try_via_ep11; for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { @@ -316,7 +317,8 @@ static int pckmokey2protkey_fallback(const struct clearkeytoken *t, t->keytype, PKEY_TYPE_CCA_DATA, 8 * keysize, 0, t->clearkey, t->len, - tmpbuf, &tmplen); + tmpbuf, &tmplen, NULL); + pr_debug("pkey_cca_clr2key()=%d\n", rc); } if (rc) goto try_via_ep11; @@ -326,6 +328,7 @@ static int pckmokey2protkey_fallback(const struct clearkeytoken *t, tmpbuf, tmplen, protkey, protkeylen, protkeytype); + pr_debug("pkey_cca_key2protkey()=%d\n", rc); } if (!rc) break; @@ -335,6 +338,7 @@ try_via_ep11: nr_apqns = MAXAPQNSINLIST; rc = pkey_ep11_apqns4type(PKEY_TYPE_EP11_AES, NULL, NULL, 0, apqns, &nr_apqns); + pr_debug("pkey_ep11_apqns4type(EP11_AES)=%d\n", rc); if (rc) continue; for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { @@ -343,7 +347,8 @@ try_via_ep11: t->keytype, PKEY_TYPE_EP11_AES, 8 * keysize, 0, t->clearkey, t->len, - tmpbuf, &tmplen); + tmpbuf, &tmplen, NULL); + pr_debug("pkey_ep11_clr2key()=%d\n", rc); } if (rc) continue; @@ -353,6 +358,7 @@ try_via_ep11: tmpbuf, tmplen, protkey, protkeylen, protkeytype); + pr_debug("pkey_ep11_key2protkey()=%d\n", rc); } } @@ -367,9 +373,8 @@ static int pckmokey2protkey(const u8 *key, size_t keylen, { int rc; - rc = pkey_pckmo_key2protkey(key, keylen, - protkey, protkeylen, - protkeytype); + rc = pkey_pckmo_key2protkey(0, 0, key, keylen, + protkey, protkeylen, protkeytype); if (rc == -ENODEV) { struct keytoken_header *hdr = (struct keytoken_header *)key; struct clearkeytoken *t = (struct clearkeytoken *)key; @@ -456,7 +461,7 @@ static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) keybuflen = sizeof(kgs.seckey.seckey); rc = pkey_cca_gen_key(kgs.cardnr, kgs.domain, kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, - kgs.seckey.seckey, &keybuflen); + kgs.seckey.seckey, &keybuflen, NULL); pr_debug("pkey_cca_gen_key()=%d\n", rc); if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs))) rc = -EFAULT; @@ -478,7 +483,7 @@ static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs) kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, kcs.clrkey.clrkey, pkey_keytype_aes_to_size(kcs.keytype), - kcs.seckey.seckey, &keybuflen); + kcs.seckey.seckey, &keybuflen, NULL); pr_debug("pkey_cca_clr2key()=%d\n", rc); if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs))) rc = -EFAULT; @@ -515,10 +520,11 @@ static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp) if (copy_from_user(&kcp, ucp, sizeof(kcp))) return -EFAULT; kcp.protkey.len = sizeof(kcp.protkey.protkey); - rc = pkey_pckmo_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, - kcp.protkey.protkey, - &kcp.protkey.len, &kcp.protkey.type); - pr_debug("pkey_pckmo_clr2protkey()=%d\n", rc); + rc = pkey_pckmo_clr2key(0, 0, kcp.keytype, 0, 0, 0, + kcp.clrkey.clrkey, 0, + kcp.protkey.protkey, + &kcp.protkey.len, &kcp.protkey.type); + pr_debug("pkey_pckmo_clr2key()=%d\n", rc); if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp))) rc = -EFAULT; memzero_explicit(&kcp, sizeof(kcp)); @@ -643,9 +649,10 @@ static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp) if (copy_from_user(&kgp, ugp, sizeof(kgp))) return -EFAULT; kgp.protkey.len = sizeof(kgp.protkey.protkey); - rc = pkey_pckmo_gen_protkey(kgp.keytype, kgp.protkey.protkey, - &kgp.protkey.len, &kgp.protkey.type); - pr_debug("pkey_gen_protkey()=%d\n", rc); + rc = pkey_pckmo_gen_key(0, 0, kgp.keytype, 0, 0, 0, + kgp.protkey.protkey, + &kgp.protkey.len, &kgp.protkey.type); + pr_debug("pkey_pckmo_gen_key()=%d\n", rc); if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp))) rc = -EFAULT; memzero_explicit(&kgp, sizeof(kgp)); @@ -660,9 +667,9 @@ static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) if (copy_from_user(&kvp, uvp, sizeof(kvp))) return -EFAULT; - rc = pkey_pckmo_verify_protkey(kvp.protkey.protkey, - kvp.protkey.len, kvp.protkey.type); - pr_debug("pkey_verify_protkey()=%d\n", rc); + rc = pkey_pckmo_verifykey(kvp.protkey.protkey, kvp.protkey.len, + 0, 0, &kvp.protkey.type, 0, 0); + pr_debug("pkey_pckmo_verifykey()=%d\n", rc); memzero_explicit(&kvp, sizeof(kvp)); return rc; diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h index f714d42969b6..560106cbd450 100644 --- a/drivers/s390/crypto/pkey_base.h +++ b/drivers/s390/crypto/pkey_base.h @@ -97,12 +97,12 @@ int pkey_cca_key2protkey(u16 card, u16 dom, int pkey_cca_gen_key(u16 card, u16 dom, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen); + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); int pkey_cca_clr2key(u16 card, u16 dom, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen); + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); int pkey_cca_verifykey(const u8 *key, u32 keylen, u16 *card, u16 *dom, u32 *keytype, u32 *keybitsize, u32 *flags); @@ -124,12 +124,12 @@ int pkey_ep11_key2protkey(u16 card, u16 dom, int pkey_ep11_gen_key(u16 card, u16 dom, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen); + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); int pkey_ep11_clr2key(u16 card, u16 dom, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen); + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); int pkey_ep11_verifykey(const u8 *key, u32 keylen, u16 *card, u16 *dom, u32 *keytype, u32 *keybitsize, u32 *flags); @@ -144,14 +144,21 @@ int pkey_ep11_apqns4type(enum pkey_key_type ktype, */ bool pkey_is_pckmo_key(const u8 *key, u32 keylen); -int pkey_pckmo_key2protkey(const u8 *key, u32 keylen, +int pkey_pckmo_key2protkey(u16 _card, u16 _dom, + const u8 *key, u32 keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype); -int pkey_pckmo_gen_protkey(u32 keytype, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -int pkey_pckmo_clr2protkey(u32 keytype, const u8 *clrkey, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -int pkey_pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, - u32 protkeytype); +int pkey_pckmo_gen_key(u16 _card, u16 _dom, + u32 keytype, u32 _keysubtype, + u32 _keybitsize, u32 _flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo); +int pkey_pckmo_clr2key(u16 _card, u16 _dom, + u32 keytype, u32 _keysubtype, + u32 _keybitsize, u32 _flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo); +int pkey_pckmo_verifykey(const u8 *key, u32 keylen, + u16 *_card, u16 *_dom, + u32 *keytype, u32 *_keybitsize, u32 *_flags); /* * pkey_sysfs.c: diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c index 65e520d7a864..1bf9019ec561 100644 --- a/drivers/s390/crypto/pkey_cca.c +++ b/drivers/s390/crypto/pkey_cca.c @@ -112,7 +112,7 @@ int pkey_cca_key2protkey(u16 card, u16 dom, int pkey_cca_gen_key(u16 card, u16 dom, u32 keytype, u32 subtype, u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen) + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) { int len, rc; @@ -173,7 +173,7 @@ int pkey_cca_clr2key(u16 card, u16 dom, u32 keytype, u32 subtype, u32 keybitsize, u32 flags, const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen) + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) { int len, rc; diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c index fdc9de43a6c1..4c49e07ece74 100644 --- a/drivers/s390/crypto/pkey_ep11.c +++ b/drivers/s390/crypto/pkey_ep11.c @@ -71,8 +71,7 @@ int pkey_ep11_key2protkey(u16 card, u16 dom, 3, key, keylen, 1)) return -EINVAL; rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); + protkey, protkeylen, protkeytype); } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_ECC_WITH_HEADER && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { @@ -81,8 +80,7 @@ int pkey_ep11_key2protkey(u16 card, u16 dom, 3, key, keylen, 1)) return -EINVAL; rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); + protkey, protkeylen, protkeytype); } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_AES && is_ep11_keyblob(key)) { @@ -90,8 +88,7 @@ int pkey_ep11_key2protkey(u16 card, u16 dom, if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) return -EINVAL; rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, - protkeytype); + protkey, protkeylen, protkeytype); } else { PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", __func__, hdr->type, hdr->version); @@ -114,7 +111,7 @@ int pkey_ep11_key2protkey(u16 card, u16 dom, int pkey_ep11_gen_key(u16 card, u16 dom, u32 keytype, u32 subtype, u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen) + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) { int len, rc; @@ -171,7 +168,7 @@ int pkey_ep11_clr2key(u16 card, u16 dom, u32 keytype, u32 subtype, u32 keybitsize, u32 flags, const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen) + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) { int len, rc; diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c index 30a9d2f64853..d2c2c61f449b 100644 --- a/drivers/s390/crypto/pkey_pckmo.c +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -17,10 +17,67 @@ #include "pkey_base.h" +/* + * Prototypes + */ + +static bool is_pckmo_key(const u8 *key, u32 keylen); +static int pckmo_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +static int pckmo_gen_protkey(u32 keytype, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +static int pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, + u32 protkeytype); + +/* + * Wrapper functions + */ + +bool pkey_is_pckmo_key(const u8 *key, u32 keylen) +{ + return is_pckmo_key(key, keylen); +} + +int pkey_pckmo_key2protkey(u16 _card, u16 _dom, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *keyinfo) +{ + return pckmo_key2protkey(key, keylen, + protkey, protkeylen, keyinfo); +} + +int pkey_pckmo_gen_key(u16 _card, u16 _dom, + u32 keytype, u32 _keysubtype, + u32 _keybitsize, u32 _flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + return pckmo_gen_protkey(keytype, + keybuf, keybuflen, keyinfo); +} + +int pkey_pckmo_clr2key(u16 _card, u16 _dom, + u32 keytype, u32 _keysubtype, + u32 _keybitsize, u32 _flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + return pckmo_clr2protkey(keytype, clrkey, clrkeylen, + keybuf, keybuflen, keyinfo); +} + +int pkey_pckmo_verifykey(const u8 *key, u32 keylen, + u16 *_card, u16 *_dom, + u32 *keytype, u32 *_keybitsize, u32 *_flags) +{ + return pckmo_verify_protkey(key, keylen, *keytype); +} + /* * Check key blob for known and supported here. */ -bool pkey_is_pckmo_key(const u8 *key, u32 keylen) +static bool is_pckmo_key(const u8 *key, u32 keylen) { struct keytoken_header *hdr = (struct keytoken_header *)key; struct clearkeytoken *t = (struct clearkeytoken *)key; @@ -55,8 +112,8 @@ bool pkey_is_pckmo_key(const u8 *key, u32 keylen) } } -int pkey_pckmo_key2protkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pckmo_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { struct keytoken_header *hdr = (struct keytoken_header *)key; int rc = -EINVAL; @@ -73,8 +130,7 @@ int pkey_pckmo_key2protkey(const u8 *key, u32 keylen, if (keylen != sizeof(struct protaeskeytoken)) goto out; t = (struct protaeskeytoken *)key; - rc = pkey_pckmo_verify_protkey(t->protkey, t->len, - t->keytype); + rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype); if (rc) goto out; memcpy(protkey, t->protkey, t->len); @@ -123,8 +179,8 @@ int pkey_pckmo_key2protkey(const u8 *key, u32 keylen, __func__, t->len); goto out; } - rc = pkey_pckmo_clr2protkey(t->keytype, t->clearkey, - protkey, protkeylen, protkeytype); + rc = pckmo_clr2protkey(t->keytype, t->clearkey, t->len, + protkey, protkeylen, protkeytype); break; } default: @@ -143,8 +199,8 @@ out: * Currently only the generation of AES protected keys * is supported. */ -int pkey_pckmo_gen_protkey(u32 keytype, u8 *protkey, - u32 *protkeylen, u32 *protkeytype) +static int pckmo_gen_protkey(u32 keytype, u8 *protkey, + u32 *protkeylen, u32 *protkeytype) { u8 clrkey[32]; int keysize; @@ -161,8 +217,8 @@ int pkey_pckmo_gen_protkey(u32 keytype, u8 *protkey, get_random_bytes(clrkey, keysize); /* convert it to a dummy protected key */ - rc = pkey_pckmo_clr2protkey(keytype, clrkey, - protkey, protkeylen, protkeytype); + rc = pckmo_clr2protkey(keytype, clrkey, keysize, + protkey, protkeylen, protkeytype); if (rc) goto out; @@ -177,8 +233,8 @@ out: /* * Create a protected key from a clear key value via PCKMO instruction. */ -int pkey_pckmo_clr2protkey(u32 keytype, const u8 *clrkey, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { /* mask of available pckmo subfunctions */ static cpacf_mask_t pckmo_functions; @@ -243,6 +299,11 @@ int pkey_pckmo_clr2protkey(u32 keytype, const u8 *clrkey, goto out; } + if (clrkeylen && clrkeylen < keysize) { + PKEY_DBF_ERR("%s clear key size too small: %u < %d\n", + __func__, clrkeylen, keysize); + goto out; + } if (*protkeylen < keysize + AES_WK_VP_SIZE) { PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n", __func__, *protkeylen, keysize + AES_WK_VP_SIZE); @@ -288,8 +349,8 @@ out: * Verify a protected key blob. * Currently only AES protected keys are supported. */ -int pkey_pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, - u32 protkeytype) +static int pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, + u32 protkeytype) { struct { u8 iv[AES_BLOCK_SIZE]; diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c index 727293fbf331..684f87d6e9f1 100644 --- a/drivers/s390/crypto/pkey_sysfs.c +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -42,9 +42,10 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, protkeytoken.keytype = keytype; protkey.len = sizeof(protkey.protkey); - rc = pkey_pckmo_gen_protkey(protkeytoken.keytype, - protkey.protkey, &protkey.len, - &protkey.type); + rc = pkey_pckmo_gen_key(0, 0, + protkeytoken.keytype, 0, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); if (rc) return rc; @@ -56,9 +57,10 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, if (is_xts) { /* xts needs a second protected key, reuse protkey struct */ protkey.len = sizeof(protkey.protkey); - rc = pkey_pckmo_gen_protkey(protkeytoken.keytype, - protkey.protkey, &protkey.len, - &protkey.type); + rc = pkey_pckmo_gen_key(0, 0, + protkeytoken.keytype, 0, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); if (rc) return rc; @@ -154,6 +156,7 @@ static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, loff_t off, size_t count) { struct pkey_seckey *seckey = (struct pkey_seckey *)buf; + u32 buflen; int rc; if (off != 0 || count < sizeof(struct secaeskeytoken)) @@ -162,13 +165,19 @@ static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, if (count < 2 * sizeof(struct secaeskeytoken)) return -EINVAL; - rc = cca_genseckey(-1, -1, keytype, seckey->seckey); + buflen = sizeof(seckey->seckey); + rc = pkey_cca_gen_key(-1, -1, keytype, + PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); if (rc) return rc; if (is_xts) { seckey++; - rc = cca_genseckey(-1, -1, keytype, seckey->seckey); + buflen = sizeof(seckey->seckey); + rc = pkey_cca_gen_key(-1, -1, keytype, + PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); if (rc) return rc; @@ -261,8 +270,9 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, size_t count) { u32 keysize = CCACIPHERTOKENSIZE; - u32 nr_apqns, *apqns = NULL; + struct pkey_apqn *apqns = NULL; int i, rc, card, dom; + size_t nr_apqns; if (off != 0 || count < CCACIPHERTOKENSIZE) return -EINVAL; @@ -270,33 +280,51 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, if (count < 2 * CCACIPHERTOKENSIZE) return -EINVAL; + nr_apqns = MAXAPQNSINLIST; + apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); + if (!apqns) + return -ENOMEM; + /* build a list of apqns able to generate an cipher key */ - rc = cca_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX6, 0, 0, 0, 0); - if (rc) + rc = pkey_cca_apqns4type(PKEY_TYPE_CCA_CIPHER, + NULL, NULL, 0, + apqns, &nr_apqns); + if (rc) { + kfree(apqns); return rc; + } memset(buf, 0, is_xts ? 2 * keysize : keysize); /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); - if (rc == 0) - break; + for (i = 0, rc = -ENODEV; rc && i < nr_apqns; i++) { + card = apqns[i].card; + dom = apqns[i].domain; + rc = pkey_cca_gen_key(card, dom, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); } - if (rc) + if (rc) { + kfree(apqns); return rc; + } if (is_xts) { keysize = CCACIPHERTOKENSIZE; buf += CCACIPHERTOKENSIZE; - rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); - if (rc == 0) - return 2 * CCACIPHERTOKENSIZE; + rc = pkey_cca_gen_key(card, dom, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); + kfree(apqns); + if (rc) + return rc; + return 2 * CCACIPHERTOKENSIZE; } + kfree(apqns); + return CCACIPHERTOKENSIZE; } @@ -384,8 +412,9 @@ static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, size_t count) { u32 keysize = MAXEP11AESKEYBLOBSIZE; - u32 nr_apqns, *apqns = NULL; + struct pkey_apqn *apqns = NULL; int i, rc, card, dom; + size_t nr_apqns; if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) return -EINVAL; @@ -393,37 +422,51 @@ static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, if (count < 2 * MAXEP11AESKEYBLOBSIZE) return -EINVAL; - /* build a list of apqns able to generate an cipher key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - NULL); - if (rc) + nr_apqns = MAXAPQNSINLIST; + apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); + if (!apqns) + return -ENOMEM; + + /* build a list of apqns able to generate an EP11 AES key */ + rc = pkey_ep11_apqns4type(PKEY_TYPE_EP11_AES, + NULL, NULL, 0, + apqns, &nr_apqns); + if (rc) { + kfree(apqns); return rc; + } memset(buf, 0, is_xts ? 2 * keysize : keysize); /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, - PKEY_TYPE_EP11_AES); - if (rc == 0) - break; + for (i = 0, rc = -ENODEV; rc && i < nr_apqns; i++) { + card = apqns[i].card; + dom = apqns[i].domain; + rc = pkey_ep11_gen_key(card, dom, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); } - if (rc) + if (rc) { + kfree(apqns); return rc; + } if (is_xts) { keysize = MAXEP11AESKEYBLOBSIZE; buf += MAXEP11AESKEYBLOBSIZE; - rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize, - PKEY_TYPE_EP11_AES); - if (rc == 0) - return 2 * MAXEP11AESKEYBLOBSIZE; + rc = pkey_ep11_gen_key(card, dom, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); + kfree(apqns); + if (rc) + return rc; + return 2 * MAXEP11AESKEYBLOBSIZE; } + kfree(apqns); + return MAXEP11AESKEYBLOBSIZE; } From 8fcc231ce3bea12b78bb94b280cdc03cff342435 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 22 Aug 2024 11:32:19 +0200 Subject: [PATCH 40/84] s390/pkey: Introduce pkey base with handler registry and handler modules Introduce pkey base kernel code with a simple pkey handler registry. Regroup the pkey code into these kernel modules: - pkey is the pkey api supporting the ioctls, sysfs and in-kernel api. Also the pkey base code which offers the handler registry and handler wrapping invocation functions is integrated there. This module is automatically loaded in via CPU feature if the MSA feature is available. - pkey-cca is the CCA related handler code kernel module a offering CCA specific implementation for pkey. This module is loaded in via MODULE_DEVICE_TABLE when a CEX[4-8] card becomes available. - pkey-ep11 is the EP11 related handler code kernel module offering an EP11 specific implementation for pkey. This module is loaded in via MODULE_DEVICE_TABLE when a CEX[4-8] card becomes available. - pkey-pckmo is the PCKMO related handler code kernel module. This module is loaded in via CPU feature if the MSA feature is available, but on init a check for availability of the pckmo instruction is performed. The handler modules register via a pkey_handler struct at the pkey base code and the pkey customer (that is currently the pkey api code fetches a handler via pkey handler registry functions and calls the unified handler functions via the pkey base handler functions. As a result the pkey-cca, pkey-ep11 and pkey-pckmo modules get independent from each other and it becomes possible to write new handlers which offer another kind of implementation without implicit dependencies to other handler implementations and/or kernel device drivers. For each of these 4 kernel modules there is an individual Kconfig entry: CONFIG_PKEY for the base and api, CONFIG_PKEY_CCA for the PKEY CCA support handler, CONFIG_PKEY_EP11 for the EP11 support handler and CONFIG_PKEY_PCKMO for the pckmo support. The both CEX related handler modules (PKEY CCA and PKEY EP11) have a dependency to the zcrypt api of the zcrypt device driver. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- arch/s390/configs/debug_defconfig | 3 + arch/s390/configs/defconfig | 3 + arch/s390/include/uapi/asm/pkey.h | 1 + drivers/crypto/Kconfig | 77 +++- drivers/s390/crypto/Makefile | 16 +- drivers/s390/crypto/pkey_api.c | 688 +++++++++--------------------- drivers/s390/crypto/pkey_base.c | 293 +++++++++++++ drivers/s390/crypto/pkey_base.h | 144 +++---- drivers/s390/crypto/pkey_cca.c | 659 ++++++++++++++++------------ drivers/s390/crypto/pkey_ep11.c | 612 +++++++++++++++----------- drivers/s390/crypto/pkey_pckmo.c | 412 ++++++++++-------- drivers/s390/crypto/pkey_sysfs.c | 121 ++---- 12 files changed, 1697 insertions(+), 1332 deletions(-) create mode 100644 drivers/s390/crypto/pkey_base.c diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 6c57f024acae..7ec1b8cd0de9 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -797,6 +797,9 @@ CONFIG_CRYPTO_CHACHA_S390=m CONFIG_CRYPTO_HMAC_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m +CONFIG_PKEY_CCA=m +CONFIG_PKEY_EP11=m +CONFIG_PKEY_PCKMO=m CONFIG_CRYPTO_PAES_S390=m CONFIG_CRYPTO_DEV_VIRTIO=m CONFIG_SYSTEM_BLACKLIST_KEYRING=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index c75e375570fa..df4addd1834a 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -784,6 +784,9 @@ CONFIG_CRYPTO_CHACHA_S390=m CONFIG_CRYPTO_HMAC_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m +CONFIG_PKEY_CCA=m +CONFIG_PKEY_EP11=m +CONFIG_PKEY_PCKMO=m CONFIG_CRYPTO_PAES_S390=m CONFIG_CRYPTO_DEV_VIRTIO=m CONFIG_SYSTEM_BLACKLIST_KEYRING=y diff --git a/arch/s390/include/uapi/asm/pkey.h b/arch/s390/include/uapi/asm/pkey.h index 5ad76471e73f..04183080cdbb 100644 --- a/arch/s390/include/uapi/asm/pkey.h +++ b/arch/s390/include/uapi/asm/pkey.h @@ -50,6 +50,7 @@ enum pkey_key_type { PKEY_TYPE_CCA_ECC = (__u32) 0x1f, PKEY_TYPE_EP11_AES = (__u32) 6, PKEY_TYPE_EP11_ECC = (__u32) 7, + PKEY_TYPE_PROTKEY = (__u32) 8, }; /* the newer ioctls use a pkey_key_size enum for key size information */ diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 94f23c6fc93b..08b1238bcd7b 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -21,7 +21,7 @@ config CRYPTO_DEV_PADLOCK (so called VIA PadLock ACE, Advanced Cryptography Engine) that provides instructions for very fast cryptographic operations with supported algorithms. - + The instructions are used only when the CPU supports them. Otherwise software encryption is used. @@ -78,18 +78,79 @@ config ZCRYPT config PKEY tristate "Kernel API for protected key handling" depends on S390 - depends on ZCRYPT help - With this option enabled the pkey kernel module provides an API + With this option enabled the pkey kernel modules provide an API for creation and handling of protected keys. Other parts of the kernel or userspace applications may use these functions. - Select this option if you want to enable the kernel and userspace - API for proteced key handling. + The protected key support is distributed into: + - A pkey base and API kernel module (pkey.ko) which offers the + infrastructure for the pkey handler kernel modules, the ioctl + and the sysfs API and the in-kernel API to the crypto cipher + implementations using protected key. + - A pkey pckmo kernel module (pkey-pckmo.ko) which is automatically + loaded when pckmo support (that is generation of protected keys + from clear key values) is available. + - A pkey CCA kernel module (pkey-cca.ko) which is automatically + loaded when a CEX crypto card is available. + - A pkey EP11 kernel module (pkey-ep11.ko) which is automatically + loaded when a CEX crypto card is available. - Please note that creation of protected keys from secure keys - requires to have at least one CEX card in coprocessor mode - available at runtime. + Select this option if you want to enable the kernel and userspace + API for protected key handling. + +config PKEY_CCA + tristate "PKEY CCA support handler" + depends on PKEY + depends on ZCRYPT + help + This is the CCA support handler for deriving protected keys + from CCA (secure) keys. Also this handler provides an alternate + way to make protected keys from clear key values. + + The PKEY CCA support handler needs a Crypto Express card (CEX) + in CCA mode. + + If you have selected the PKEY option then you should also enable + this option unless you are sure you never need to derive protected + keys from CCA key material. + +config PKEY_EP11 + tristate "PKEY EP11 support handler" + depends on PKEY + depends on ZCRYPT + help + This is the EP11 support handler for deriving protected keys + from EP11 (secure) keys. Also this handler provides an alternate + way to make protected keys from clear key values. + + The PKEY EP11 support handler needs a Crypto Express card (CEX) + in EP11 mode. + + If you have selected the PKEY option then you should also enable + this option unless you are sure you never need to derive protected + keys from EP11 key material. + +config PKEY_PCKMO + tristate "PKEY PCKMO support handler" + depends on PKEY + help + This is the PCKMO support handler for deriving protected keys + from clear key values via invoking the PCKMO instruction. + + The PCKMO instruction can be enabled and disabled in the crypto + settings at the LPAR profile. This handler checks for availability + during initialization and if build as a kernel module unloads + itself if PCKMO is disabled. + + The PCKMO way of deriving protected keys from clear key material + is especially used during self test of protected key ciphers like + PAES but the CCA and EP11 handler provide alternate ways to + generate protected keys from clear key values. + + If you have selected the PKEY option then you should also enable + this option unless you are sure you never need to derive protected + keys from clear key values directly via PCKMO. config CRYPTO_PAES_S390 tristate "PAES cipher algorithms" diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile index 863d6fbd2e79..c88b6e071847 100644 --- a/drivers/s390/crypto/Makefile +++ b/drivers/s390/crypto/Makefile @@ -13,10 +13,22 @@ obj-$(CONFIG_ZCRYPT) += zcrypt.o # adapter drivers depend on ap.o and zcrypt.o obj-$(CONFIG_ZCRYPT) += zcrypt_cex4.o -# pkey kernel module -pkey-objs := pkey_api.o pkey_cca.o pkey_ep11.o pkey_pckmo.o pkey_sysfs.o +# pkey base and api module +pkey-objs := pkey_base.o pkey_api.o pkey_sysfs.o obj-$(CONFIG_PKEY) += pkey.o +# pkey cca handler module +pkey-cca-objs := pkey_cca.o +obj-$(CONFIG_PKEY_CCA) += pkey-cca.o + +# pkey ep11 handler module +pkey-ep11-objs := pkey_ep11.o +obj-$(CONFIG_PKEY_EP11) += pkey-ep11.o + +# pkey pckmo handler module +pkey-pckmo-objs := pkey_pckmo.o +obj-$(CONFIG_PKEY_PCKMO) += pkey-pckmo.o + # adjunct processor matrix vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o obj-$(CONFIG_VFIO_AP) += vfio_ap.o diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 31382c23ec14..c59051ab1cfb 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -10,271 +10,26 @@ #define KMSG_COMPONENT "pkey" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include #include #include -#include #include -#include -#include -#include -#include -#include -#include #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" -#include "zcrypt_ep11misc.h" #include "pkey_base.h" -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("IBM Corporation"); -MODULE_DESCRIPTION("s390 protected key interface"); - -/* - * Debug feature data and functions - */ - -debug_info_t *pkey_dbf_info; - -static void __init pkey_debug_init(void) -{ - /* 5 arguments per dbf entry (including the format string ptr) */ - pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); - debug_register_view(pkey_dbf_info, &debug_sprintf_view); - debug_set_level(pkey_dbf_info, 3); -} - -static void __exit pkey_debug_exit(void) -{ - debug_unregister(pkey_dbf_info); -} - /* * Helper functions */ -static int apqns4key(const u8 *key, size_t keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) -{ - if (pkey_is_cca_key(key, keylen)) { - return pkey_cca_apqns4key(key, keylen, flags, - apqns, nr_apqns); - } else if (pkey_is_ep11_key(key, keylen)) { - return pkey_ep11_apqns4key(key, keylen, flags, - apqns, nr_apqns); - } else { - struct keytoken_header *hdr = (struct keytoken_header *)key; - - PKEY_DBF_ERR("%s unknown/unsupported key type %d version %d\n", - __func__, hdr->type, hdr->version); - return -EINVAL; - } -} - -static int apqns4keytype(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) -{ - if (pkey_is_cca_keytype(ktype)) { - return pkey_cca_apqns4type(ktype, cur_mkvp, alt_mkvp, flags, - apqns, nr_apqns); - } else if (pkey_is_ep11_keytype(ktype)) { - return pkey_ep11_apqns4type(ktype, cur_mkvp, alt_mkvp, flags, - apqns, nr_apqns); - } else { - PKEY_DBF_ERR("%s unknown/unsupported key type %d\n", - __func__, ktype); - return -EINVAL; - } -} - -static int genseck2(const struct pkey_apqn *apqns, size_t nr_apqns, - enum pkey_key_type keytype, enum pkey_key_size keybitsize, - u32 flags, u8 *keybuf, u32 *keybuflen) -{ - int i, rc; - u32 u; - - if (pkey_is_cca_keytype(keytype)) { - /* As of now only CCA AES key generation is supported */ - u = pkey_aes_bitsize_to_keytype(keybitsize); - if (!u) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); - return -EINVAL; - } - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - rc = pkey_cca_gen_key(apqns[i].card, - apqns[i].domain, - u, keytype, keybitsize, flags, - keybuf, keybuflen, NULL); - } - } else if (pkey_is_ep11_keytype(keytype)) { - /* As of now only EP11 AES key generation is supported */ - u = pkey_aes_bitsize_to_keytype(keybitsize); - if (!u) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); - return -EINVAL; - } - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - rc = pkey_ep11_gen_key(apqns[i].card, - apqns[i].domain, - u, keytype, keybitsize, flags, - keybuf, keybuflen, NULL); - } - } else { - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", - __func__, keytype); - return -EINVAL; - } - - return rc; -} - -static int clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, - enum pkey_key_type keytype, enum pkey_key_size kbitsize, - u32 flags, const u8 *clrkey, u8 *keybuf, u32 *keybuflen) -{ - int i, rc; - u32 u; - - if (pkey_is_cca_keytype(keytype)) { - /* As of now only CCA AES key generation is supported */ - u = pkey_aes_bitsize_to_keytype(kbitsize); - if (!u) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, kbitsize); - return -EINVAL; - } - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - rc = pkey_cca_clr2key(apqns[i].card, - apqns[i].domain, - u, keytype, kbitsize, flags, - clrkey, kbitsize / 8, - keybuf, keybuflen, NULL); - } - } else if (pkey_is_ep11_keytype(keytype)) { - /* As of now only EP11 AES key generation is supported */ - u = pkey_aes_bitsize_to_keytype(kbitsize); - if (!u) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, kbitsize); - return -EINVAL; - } - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - rc = pkey_ep11_clr2key(apqns[i].card, - apqns[i].domain, - u, keytype, kbitsize, flags, - clrkey, kbitsize / 8, - keybuf, keybuflen, NULL); - } - } else { - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", - __func__, keytype); - return -EINVAL; - } - - return rc; -} - -static int ccakey2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct pkey_apqn *local_apqns = NULL; - int i, j, rc; - - /* alloc space for list of apqns if no list given */ - if (!apqns || (nr_apqns == 1 && - apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { - nr_apqns = MAXAPQNSINLIST; - local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), - GFP_KERNEL); - if (!local_apqns) - return -ENOMEM; - apqns = local_apqns; - } - - /* try two times in case of failure */ - for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { - if (local_apqns) { - /* gather list of apqns able to deal with this key */ - nr_apqns = MAXAPQNSINLIST; - rc = pkey_cca_apqns4key(key, keylen, 0, - local_apqns, &nr_apqns); - if (rc) - continue; - } - /* go through the list of apqns until success or end */ - for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { - rc = pkey_cca_key2protkey(apqns[j].card, - apqns[j].domain, - key, keylen, - protkey, protkeylen, - protkeytype); - } - } - - kfree(local_apqns); - - return rc; -} - -static int ep11key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, - const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct pkey_apqn *local_apqns = NULL; - int i, j, rc; - - /* alloc space for list of apqns if no list given */ - if (!apqns || (nr_apqns == 1 && - apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { - nr_apqns = MAXAPQNSINLIST; - local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), - GFP_KERNEL); - if (!local_apqns) - return -ENOMEM; - apqns = local_apqns; - } - - /* try two times in case of failure */ - for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { - if (local_apqns) { - /* gather list of apqns able to deal with this key */ - nr_apqns = MAXAPQNSINLIST; - rc = pkey_ep11_apqns4key(key, keylen, 0, - local_apqns, &nr_apqns); - if (rc) - continue; - } - /* go through the list of apqns until success or end */ - for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { - rc = pkey_ep11_key2protkey(apqns[j].card, - apqns[j].domain, - key, keylen, - protkey, protkeylen, - protkeytype); - } - } - - kfree(local_apqns); - - return rc; -} - -static int pckmokey2protkey_fallback(const struct clearkeytoken *t, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static int key2protkey_fallback(const struct clearkeytoken *t, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) { size_t tmpbuflen = max_t(size_t, SECKEYBLOBSIZE, MAXEP11AESKEYBLOBSIZE); - struct pkey_apqn *apqns = NULL; - u32 keysize, tmplen; + u32 keysize, keybitsize, tmplen; u8 *tmpbuf = NULL; - size_t nr_apqns; - int i, j, rc; + int i, rc; /* As of now only for AES keys a fallback is available */ @@ -289,110 +44,53 @@ static int pckmokey2protkey_fallback(const struct clearkeytoken *t, __func__, t->len); return -EINVAL; } + keybitsize = 8 * keysize; - /* alloc tmp buffer and space for apqns */ + /* alloc tmp key buffer */ tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); if (!tmpbuf) return -ENOMEM; - nr_apqns = MAXAPQNSINLIST; - apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); - if (!apqns) { - kfree(tmpbuf); - return -ENOMEM; - } /* try two times in case of failure */ for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { /* CCA secure key way */ - nr_apqns = MAXAPQNSINLIST; - rc = pkey_cca_apqns4type(PKEY_TYPE_CCA_DATA, - NULL, NULL, 0, apqns, &nr_apqns); - pr_debug("pkey_cca_apqns4type(CCA_DATA)=%d\n", rc); + tmplen = tmpbuflen; + rc = pkey_handler_clr_to_key(NULL, 0, + t->keytype, PKEY_TYPE_CCA_DATA, + keybitsize, 0, + t->clearkey, t->len, + tmpbuf, &tmplen, NULL); + pr_debug("clr_to_key()=%d\n", rc); if (rc) goto try_via_ep11; - for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { - tmplen = tmpbuflen; - rc = pkey_cca_clr2key(apqns[j].card, apqns[j].domain, - t->keytype, PKEY_TYPE_CCA_DATA, - 8 * keysize, 0, - t->clearkey, t->len, - tmpbuf, &tmplen, NULL); - pr_debug("pkey_cca_clr2key()=%d\n", rc); - } - if (rc) - goto try_via_ep11; - for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { - rc = pkey_cca_key2protkey(apqns[j].card, - apqns[j].domain, - tmpbuf, tmplen, - protkey, protkeylen, - protkeytype); - pr_debug("pkey_cca_key2protkey()=%d\n", rc); - } + rc = pkey_handler_key_to_protkey(NULL, 0, + tmpbuf, tmplen, + protkey, protkeylen, + protkeytype); + pr_debug("key_to_protkey()=%d\n", rc); if (!rc) break; try_via_ep11: /* the CCA way failed, try via EP11 */ - nr_apqns = MAXAPQNSINLIST; - rc = pkey_ep11_apqns4type(PKEY_TYPE_EP11_AES, - NULL, NULL, 0, apqns, &nr_apqns); - pr_debug("pkey_ep11_apqns4type(EP11_AES)=%d\n", rc); + tmplen = tmpbuflen; + rc = pkey_handler_clr_to_key(NULL, 0, + t->keytype, PKEY_TYPE_EP11_AES, + keybitsize, 0, + t->clearkey, t->len, + tmpbuf, &tmplen, NULL); + pr_debug("clr_to_key()=%d\n", rc); if (rc) continue; - for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { - tmplen = tmpbuflen; - rc = pkey_ep11_clr2key(apqns[j].card, apqns[j].domain, - t->keytype, PKEY_TYPE_EP11_AES, - 8 * keysize, 0, - t->clearkey, t->len, - tmpbuf, &tmplen, NULL); - pr_debug("pkey_ep11_clr2key()=%d\n", rc); - } - if (rc) - continue; - for (j = 0, rc = -ENODEV; j < nr_apqns && rc; j++) { - rc = pkey_ep11_key2protkey(apqns[j].card, - apqns[j].domain, - tmpbuf, tmplen, - protkey, protkeylen, - protkeytype); - pr_debug("pkey_ep11_key2protkey()=%d\n", rc); - } + rc = pkey_handler_key_to_protkey(NULL, 0, + tmpbuf, tmplen, + protkey, protkeylen, + protkeytype); + pr_debug("key_to_protkey()=%d\n", rc); } kfree(tmpbuf); - kfree(apqns); - - return rc; -} - -static int pckmokey2protkey(const u8 *key, size_t keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - int rc; - - rc = pkey_pckmo_key2protkey(0, 0, key, keylen, - protkey, protkeylen, protkeytype); - if (rc == -ENODEV) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - struct clearkeytoken *t = (struct clearkeytoken *)key; - - /* maybe a fallback is possible */ - if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_CLEAR_KEY) { - rc = pckmokey2protkey_fallback(t, protkey, - protkeylen, - protkeytype); - if (rc) - rc = -ENODEV; - } - } - - if (rc) - PKEY_DBF_ERR("%s unable to build protected key from clear, rc=%d", - __func__, rc); return rc; } @@ -401,22 +99,26 @@ static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, size_t keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - if (pkey_is_cca_key(key, keylen)) { - return ccakey2protkey(apqns, nr_apqns, key, keylen, - protkey, protkeylen, protkeytype); - } else if (pkey_is_ep11_key(key, keylen)) { - return ep11key2protkey(apqns, nr_apqns, key, keylen, - protkey, protkeylen, protkeytype); - } else if (pkey_is_pckmo_key(key, keylen)) { - return pckmokey2protkey(key, keylen, - protkey, protkeylen, protkeytype); - } else { - struct keytoken_header *hdr = (struct keytoken_header *)key; + struct keytoken_header *hdr = (struct keytoken_header *)key; + int i, rc; - PKEY_DBF_ERR("%s unknown/unsupported key type %d version %d\n", - __func__, hdr->type, hdr->version); - return -EINVAL; + /* retry two times */ + for (rc = -ENODEV, i = 0; rc && i < 2; i++) { + /* First try the direct way */ + rc = pkey_handler_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype); + /* For some clear key tokens there exists a fallback way */ + if (rc && + hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_CLEAR_KEY) + rc = key2protkey_fallback((struct clearkeytoken *)key, + protkey, protkeylen, + protkeytype); } + + return rc; } /* @@ -453,16 +155,20 @@ static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns) static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) { struct pkey_genseck kgs; + struct pkey_apqn apqn; u32 keybuflen; int rc; if (copy_from_user(&kgs, ugs, sizeof(kgs))) return -EFAULT; + + apqn.card = kgs.cardnr; + apqn.domain = kgs.domain; keybuflen = sizeof(kgs.seckey.seckey); - rc = pkey_cca_gen_key(kgs.cardnr, kgs.domain, - kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, - kgs.seckey.seckey, &keybuflen, NULL); - pr_debug("pkey_cca_gen_key()=%d\n", rc); + rc = pkey_handler_gen_key(&apqn, 1, + kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, + kgs.seckey.seckey, &keybuflen, NULL); + pr_debug("gen_key()=%d\n", rc); if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs))) rc = -EFAULT; memzero_explicit(&kgs, sizeof(kgs)); @@ -473,18 +179,22 @@ static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs) static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs) { struct pkey_clr2seck kcs; + struct pkey_apqn apqn; u32 keybuflen; int rc; if (copy_from_user(&kcs, ucs, sizeof(kcs))) return -EFAULT; + + apqn.card = kcs.cardnr; + apqn.domain = kcs.domain; keybuflen = sizeof(kcs.seckey.seckey); - rc = pkey_cca_clr2key(kcs.cardnr, kcs.domain, - kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, - kcs.clrkey.clrkey, - pkey_keytype_aes_to_size(kcs.keytype), - kcs.seckey.seckey, &keybuflen, NULL); - pr_debug("pkey_cca_clr2key()=%d\n", rc); + rc = pkey_handler_clr_to_key(&apqn, 1, + kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0, + kcs.clrkey.clrkey, + pkey_keytype_aes_to_size(kcs.keytype), + kcs.seckey.seckey, &keybuflen, NULL); + pr_debug("clr_to_key()=%d\n", rc); if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs))) rc = -EFAULT; memzero_explicit(&kcs, sizeof(kcs)); @@ -495,16 +205,21 @@ static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs) static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp) { struct pkey_sec2protk ksp; + struct pkey_apqn apqn; int rc; if (copy_from_user(&ksp, usp, sizeof(ksp))) return -EFAULT; + + apqn.card = ksp.cardnr; + apqn.domain = ksp.domain; ksp.protkey.len = sizeof(ksp.protkey.protkey); - rc = pkey_cca_key2protkey(ksp.cardnr, ksp.domain, - ksp.seckey.seckey, sizeof(ksp.seckey.seckey), - ksp.protkey.protkey, - &ksp.protkey.len, &ksp.protkey.type); - pr_debug("pkey_cca_key2protkey()=%d\n", rc); + rc = pkey_handler_key_to_protkey(&apqn, 1, + ksp.seckey.seckey, + sizeof(ksp.seckey.seckey), + ksp.protkey.protkey, + &ksp.protkey.len, &ksp.protkey.type); + pr_debug("key_to_protkey()=%d\n", rc); if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) rc = -EFAULT; memzero_explicit(&ksp, sizeof(ksp)); @@ -515,16 +230,43 @@ static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp) static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp) { struct pkey_clr2protk kcp; + struct clearkeytoken *t; + u32 keylen; + u8 *tmpbuf; int rc; if (copy_from_user(&kcp, ucp, sizeof(kcp))) return -EFAULT; + + /* build a 'clear key token' from the clear key value */ + keylen = pkey_keytype_aes_to_size(kcp.keytype); + if (!keylen) { + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, kcp.keytype); + memzero_explicit(&kcp, sizeof(kcp)); + return -EINVAL; + } + tmpbuf = kzalloc(sizeof(*t) + keylen, GFP_KERNEL); + if (!tmpbuf) { + memzero_explicit(&kcp, sizeof(kcp)); + return -ENOMEM; + } + t = (struct clearkeytoken *)tmpbuf; + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_CLEAR_KEY; + t->keytype = (keylen - 8) >> 3; + t->len = keylen; + memcpy(t->clearkey, kcp.clrkey.clrkey, keylen); kcp.protkey.len = sizeof(kcp.protkey.protkey); - rc = pkey_pckmo_clr2key(0, 0, kcp.keytype, 0, 0, 0, - kcp.clrkey.clrkey, 0, - kcp.protkey.protkey, - &kcp.protkey.len, &kcp.protkey.type); - pr_debug("pkey_pckmo_clr2key()=%d\n", rc); + + rc = key2protkey(NULL, 0, + tmpbuf, sizeof(*t) + keylen, + kcp.protkey.protkey, + &kcp.protkey.len, &kcp.protkey.type); + pr_debug("key2protkey()=%d\n", rc); + + kfree_sensitive(tmpbuf); + if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp))) rc = -EFAULT; memzero_explicit(&kcp, sizeof(kcp)); @@ -542,23 +284,21 @@ static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc) if (copy_from_user(&kfc, ufc, sizeof(kfc))) return -EFAULT; - if (!pkey_is_cca_key(kfc.seckey.seckey, sizeof(kfc.seckey.seckey))) - return -EINVAL; - nr_apqns = MAXAPQNSINLIST; apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); if (!apqns) return -ENOMEM; - rc = pkey_cca_apqns4key(kfc.seckey.seckey, - sizeof(kfc.seckey.seckey), - PKEY_FLAGS_MATCH_CUR_MKVP, - apqns, &nr_apqns); - if (rc == -ENODEV) - rc = pkey_cca_apqns4key(kfc.seckey.seckey, + + rc = pkey_handler_apqns_for_key(kfc.seckey.seckey, sizeof(kfc.seckey.seckey), - PKEY_FLAGS_MATCH_ALT_MKVP, + PKEY_FLAGS_MATCH_CUR_MKVP, apqns, &nr_apqns); - pr_debug("pkey_cca_apqns4key()=%d\n", rc); + if (rc == -ENODEV) + rc = pkey_handler_apqns_for_key(kfc.seckey.seckey, + sizeof(kfc.seckey.seckey), + PKEY_FLAGS_MATCH_ALT_MKVP, + apqns, &nr_apqns); + pr_debug("apqns_for_key()=%d\n", rc); if (rc) { kfree(apqns); return rc; @@ -575,38 +315,19 @@ static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc) static int pkey_ioctl_skey2pkey(struct pkey_skey2pkey __user *usp) { struct pkey_skey2pkey ksp; - struct pkey_apqn *apqns; - size_t nr_apqns; - int i, rc; + int rc; if (copy_from_user(&ksp, usp, sizeof(ksp))) return -EFAULT; - if (!pkey_is_cca_key(ksp.seckey.seckey, sizeof(ksp.seckey.seckey))) - return -EINVAL; - - nr_apqns = MAXAPQNSINLIST; - apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); - if (!apqns) - return -ENOMEM; - rc = pkey_cca_apqns4key(ksp.seckey.seckey, sizeof(ksp.seckey.seckey), - 0, apqns, &nr_apqns); - pr_debug("pkey_cca_apqns4key()=%d\n", rc); - if (rc) { - kfree(apqns); - return rc; - } ksp.protkey.len = sizeof(ksp.protkey.protkey); - for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { - rc = pkey_cca_key2protkey(apqns[i].card, apqns[i].domain, - ksp.seckey.seckey, - sizeof(ksp.seckey.seckey), - ksp.protkey.protkey, - &ksp.protkey.len, - &ksp.protkey.type); - pr_debug("pkey_cca_key2protkey()=%d\n", rc); - } - kfree(apqns); + rc = pkey_handler_key_to_protkey(NULL, 0, + ksp.seckey.seckey, + sizeof(ksp.seckey.seckey), + ksp.protkey.protkey, + &ksp.protkey.len, + &ksp.protkey.type); + pr_debug("key_to_protkey()=%d\n", rc); if (!rc && copy_to_user(usp, &ksp, sizeof(ksp))) rc = -EFAULT; memzero_explicit(&ksp, sizeof(ksp)); @@ -622,12 +343,14 @@ static int pkey_ioctl_verifykey(struct pkey_verifykey __user *uvk) if (copy_from_user(&kvk, uvk, sizeof(kvk))) return -EFAULT; + kvk.cardnr = 0xFFFF; kvk.domain = 0xFFFF; - rc = pkey_cca_verifykey(kvk.seckey.seckey, sizeof(kvk.seckey.seckey), - &kvk.cardnr, &kvk.domain, - &keytype, &keybitsize, &flags); - pr_debug("pkey_cca_verifykey()=%d\n", rc); + rc = pkey_handler_verify_key(kvk.seckey.seckey, + sizeof(kvk.seckey.seckey), + &kvk.cardnr, &kvk.domain, + &keytype, &keybitsize, &flags); + pr_debug("verify_key()=%d\n", rc); if (!rc && keytype != PKEY_TYPE_CCA_DATA) rc = -EINVAL; kvk.attributes = PKEY_VERIFY_ATTR_AES; @@ -648,11 +371,13 @@ static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp) if (copy_from_user(&kgp, ugp, sizeof(kgp))) return -EFAULT; + kgp.protkey.len = sizeof(kgp.protkey.protkey); - rc = pkey_pckmo_gen_key(0, 0, kgp.keytype, 0, 0, 0, - kgp.protkey.protkey, - &kgp.protkey.len, &kgp.protkey.type); - pr_debug("pkey_pckmo_gen_key()=%d\n", rc); + rc = pkey_handler_gen_key(NULL, 0, kgp.keytype, + PKEY_TYPE_PROTKEY, 0, 0, + kgp.protkey.protkey, &kgp.protkey.len, + &kgp.protkey.type); + pr_debug("gen_key()=%d\n", rc); if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp))) rc = -EFAULT; memzero_explicit(&kgp, sizeof(kgp)); @@ -663,13 +388,40 @@ static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp) static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp) { struct pkey_verifyprotk kvp; + struct protaeskeytoken *t; + u32 keytype; + u8 *tmpbuf; int rc; if (copy_from_user(&kvp, uvp, sizeof(kvp))) return -EFAULT; - rc = pkey_pckmo_verifykey(kvp.protkey.protkey, kvp.protkey.len, - 0, 0, &kvp.protkey.type, 0, 0); - pr_debug("pkey_pckmo_verifykey()=%d\n", rc); + + keytype = pkey_aes_bitsize_to_keytype(8 * kvp.protkey.len); + if (!keytype) { + PKEY_DBF_ERR("%s unknown/unsupported protkey length %u\n", + __func__, kvp.protkey.len); + memzero_explicit(&kvp, sizeof(kvp)); + return -EINVAL; + } + + /* build a 'protected key token' from the raw protected key */ + tmpbuf = kzalloc(sizeof(*t), GFP_KERNEL); + if (!tmpbuf) { + memzero_explicit(&kvp, sizeof(kvp)); + return -ENOMEM; + } + t = (struct protaeskeytoken *)tmpbuf; + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_PROTECTED_KEY; + t->keytype = keytype; + t->len = kvp.protkey.len; + memcpy(t->protkey, kvp.protkey.protkey, kvp.protkey.len); + + rc = pkey_handler_verify_key(tmpbuf, sizeof(*t), + NULL, NULL, NULL, NULL, NULL); + pr_debug("verify_key()=%d\n", rc); + + kfree_sensitive(tmpbuf); memzero_explicit(&kvp, sizeof(kvp)); return rc; @@ -706,9 +458,16 @@ static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) struct pkey_apqn *apqns; u8 *kkey; int rc; + u32 u; if (copy_from_user(&kgs, ugs, sizeof(kgs))) return -EFAULT; + u = pkey_aes_bitsize_to_keytype(kgs.size); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, kgs.size); + return -EINVAL; + } apqns = _copy_apqns_from_user(kgs.apqns, kgs.apqn_entries); if (IS_ERR(apqns)) return PTR_ERR(apqns); @@ -717,10 +476,10 @@ static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs) kfree(apqns); return -ENOMEM; } - rc = genseck2(apqns, kgs.apqn_entries, - kgs.type, kgs.size, kgs.keygenflags, - kkey, &klen); - pr_debug("genseckey2()=%d\n", rc); + rc = pkey_handler_gen_key(apqns, kgs.apqn_entries, + u, kgs.type, kgs.size, kgs.keygenflags, + kkey, &klen, NULL); + pr_debug("gen_key()=%d\n", rc); kfree(apqns); if (rc) { kfree_sensitive(kkey); @@ -751,9 +510,17 @@ static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs) struct pkey_apqn *apqns; u8 *kkey; int rc; + u32 u; if (copy_from_user(&kcs, ucs, sizeof(kcs))) return -EFAULT; + u = pkey_aes_bitsize_to_keytype(kcs.size); + if (!u) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, kcs.size); + memzero_explicit(&kcs, sizeof(kcs)); + return -EINVAL; + } apqns = _copy_apqns_from_user(kcs.apqns, kcs.apqn_entries); if (IS_ERR(apqns)) { memzero_explicit(&kcs, sizeof(kcs)); @@ -765,10 +532,11 @@ static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs) memzero_explicit(&kcs, sizeof(kcs)); return -ENOMEM; } - rc = clr2seckey2(apqns, kcs.apqn_entries, - kcs.type, kcs.size, kcs.keygenflags, - kcs.clrkey.clrkey, kkey, &klen); - pr_debug("clr2seckey2()=%d\n", rc); + rc = pkey_handler_clr_to_key(apqns, kcs.apqn_entries, + u, kcs.type, kcs.size, kcs.keygenflags, + kcs.clrkey.clrkey, kcs.size / 8, + kkey, &klen, NULL); + pr_debug("clr_to_key()=%d\n", rc); kfree(apqns); if (rc) { kfree_sensitive(kkey); @@ -807,26 +575,17 @@ static int pkey_ioctl_verifykey2(struct pkey_verifykey2 __user *uvk) kkey = _copy_key_from_user(kvk.key, kvk.keylen); if (IS_ERR(kkey)) return PTR_ERR(kkey); - if (pkey_is_cca_key(kkey, kvk.keylen)) { - rc = pkey_cca_verifykey(kkey, kvk.keylen, - &kvk.cardnr, &kvk.domain, - &kvk.type, &kvk.size, &kvk.flags); - pr_debug("pkey_cca_verifykey()=%d\n", rc); - } else if (pkey_is_ep11_key(kkey, kvk.keylen)) { - rc = pkey_ep11_verifykey(kkey, kvk.keylen, - &kvk.cardnr, &kvk.domain, - &kvk.type, &kvk.size, &kvk.flags); - pr_debug("pkey_ep11_verifykey()=%d\n", rc); - } else { - rc = -EINVAL; - } + + rc = pkey_handler_verify_key(kkey, kvk.keylen, + &kvk.cardnr, &kvk.domain, + &kvk.type, &kvk.size, &kvk.flags); + pr_debug("verify_key()=%d\n", rc); + kfree_sensitive(kkey); - if (rc) - return rc; - if (copy_to_user(uvk, &kvk, sizeof(kvk))) + if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk))) return -EFAULT; - return 0; + return rc; } static int pkey_ioctl_kblob2protk2(struct pkey_kblob2pkey2 __user *utp) @@ -883,9 +642,9 @@ static int pkey_ioctl_apqns4k(struct pkey_apqns4key __user *uak) kfree(apqns); return PTR_ERR(kkey); } - rc = apqns4key(kkey, kak.keylen, kak.flags, - apqns, &nr_apqns); - pr_debug("apqns4key()=%d\n", rc); + rc = pkey_handler_apqns_for_key(kkey, kak.keylen, kak.flags, + apqns, &nr_apqns); + pr_debug("apqns_for_key()=%d\n", rc); kfree_sensitive(kkey); if (rc && rc != -ENOSPC) { kfree(apqns); @@ -929,9 +688,10 @@ static int pkey_ioctl_apqns4kt(struct pkey_apqns4keytype __user *uat) if (!apqns) return -ENOMEM; } - rc = apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, - kat.flags, apqns, &nr_apqns); - pr_debug("apqns4keytype()=%d\n", rc); + rc = pkey_handler_apqns_for_keytype(kat.type, + kat.cur_mkvp, kat.alt_mkvp, + kat.flags, apqns, &nr_apqns); + pr_debug("apqns_for_keytype()=%d\n", rc); if (rc && rc != -ENOSPC) { kfree(apqns); return rc; @@ -1092,43 +852,13 @@ static struct miscdevice pkey_dev = { .groups = pkey_attr_groups, }; -/* - * Module init - */ -static int __init pkey_init(void) +int __init pkey_api_init(void) { - cpacf_mask_t func_mask; - - /* - * The pckmo instruction should be available - even if we don't - * actually invoke it. This instruction comes with MSA 3 which - * is also the minimum level for the kmc instructions which - * are able to work with protected keys. - */ - if (!cpacf_query(CPACF_PCKMO, &func_mask)) - return -ENODEV; - - /* check for kmc instructions available */ - if (!cpacf_query(CPACF_KMC, &func_mask)) - return -ENODEV; - if (!cpacf_test_func(&func_mask, CPACF_KMC_PAES_128) || - !cpacf_test_func(&func_mask, CPACF_KMC_PAES_192) || - !cpacf_test_func(&func_mask, CPACF_KMC_PAES_256)) - return -ENODEV; - - pkey_debug_init(); - + /* register as a misc device */ return misc_register(&pkey_dev); } -/* - * Module exit - */ -static void __exit pkey_exit(void) +void __exit pkey_api_exit(void) { misc_deregister(&pkey_dev); - pkey_debug_exit(); } - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_init); -module_exit(pkey_exit); diff --git a/drivers/s390/crypto/pkey_base.c b/drivers/s390/crypto/pkey_base.c new file mode 100644 index 000000000000..e7abc32ca5f9 --- /dev/null +++ b/drivers/s390/crypto/pkey_base.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pkey base: debug feature, pkey handler registry + * + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "pkey" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include +#include +#include + +#include "pkey_base.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key base and api"); + +/* + * pkey debug feature + */ +debug_info_t *pkey_dbf_info; +EXPORT_SYMBOL(pkey_dbf_info); + +/* + * pkey handler registry + */ + +static DEFINE_SPINLOCK(handler_list_write_lock); +static LIST_HEAD(handler_list); + +int pkey_handler_register(struct pkey_handler *handler) +{ + const struct pkey_handler *h; + + if (!handler || + !handler->is_supported_key || + !handler->is_supported_keytype) + return -EINVAL; + + if (!try_module_get(handler->module)) + return -ENXIO; + + spin_lock(&handler_list_write_lock); + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (h == handler) { + rcu_read_unlock(); + spin_unlock(&handler_list_write_lock); + module_put(handler->module); + return -EEXIST; + } + } + rcu_read_unlock(); + + list_add_rcu(&handler->list, &handler_list); + spin_unlock(&handler_list_write_lock); + synchronize_rcu(); + + module_put(handler->module); + + PKEY_DBF_INFO("%s pkey handler '%s' registered\n", __func__, + handler->name ?: ""); + + return 0; +} +EXPORT_SYMBOL(pkey_handler_register); + +int pkey_handler_unregister(struct pkey_handler *handler) +{ + spin_lock(&handler_list_write_lock); + list_del_rcu(&handler->list); + INIT_LIST_HEAD_RCU(&handler->list); + spin_unlock(&handler_list_write_lock); + synchronize_rcu(); + + PKEY_DBF_INFO("%s pkey handler '%s' unregistered\n", __func__, + handler->name ?: ""); + + return 0; +} +EXPORT_SYMBOL(pkey_handler_unregister); + +/* + * Handler invocation functions. + */ + +const struct pkey_handler *pkey_handler_get_keybased(const u8 *key, u32 keylen) +{ + const struct pkey_handler *h; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (!try_module_get(h->module)) + continue; + if (h->is_supported_key(key, keylen)) { + rcu_read_unlock(); + return h; + } + module_put(h->module); + } + rcu_read_unlock(); + + return NULL; +} +EXPORT_SYMBOL(pkey_handler_get_keybased); + +const struct pkey_handler *pkey_handler_get_keytypebased(enum pkey_key_type kt) +{ + const struct pkey_handler *h; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (!try_module_get(h->module)) + continue; + if (h->is_supported_keytype(kt)) { + rcu_read_unlock(); + return h; + } + module_put(h->module); + } + rcu_read_unlock(); + + return NULL; +} +EXPORT_SYMBOL(pkey_handler_get_keytypebased); + +void pkey_handler_put(const struct pkey_handler *handler) +{ + const struct pkey_handler *h; + + if (!handler) + return; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (h == handler) { + module_put(h->module); + break; + } + } + rcu_read_unlock(); +} +EXPORT_SYMBOL(pkey_handler_put); + +int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keybased(key, keylen); + if (h && h->key_to_protkey) { + rc = h->key_to_protkey(apqns, nr_apqns, key, keylen, + protkey, protkeylen, + protkeytype); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_key_to_protkey); + +int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keytypebased(keysubtype); + if (h && h->gen_key) { + rc = h->gen_key(apqns, nr_apqns, keytype, keysubtype, + keybitsize, flags, + keybuf, keybuflen, keyinfo); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_gen_key); + +int pkey_handler_clr_to_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keytypebased(keysubtype); + if (h && h->clr_to_key) { + rc = h->clr_to_key(apqns, nr_apqns, keytype, keysubtype, + keybitsize, flags, clrkey, clrkeylen, + keybuf, keybuflen, keyinfo); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_clr_to_key); + +int pkey_handler_verify_key(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keybased(key, keylen); + if (h && h->verify_key) { + rc = h->verify_key(key, keylen, card, dom, + keytype, keybitsize, flags); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_verify_key); + +int pkey_handler_apqns_for_key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keybased(key, keylen); + if (h && h->apqns_for_key) + rc = h->apqns_for_key(key, keylen, flags, apqns, nr_apqns); + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_apqns_for_key); + +int pkey_handler_apqns_for_keytype(enum pkey_key_type keysubtype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) +{ + const struct pkey_handler *h; + int rc = -ENODEV; + + h = pkey_handler_get_keytypebased(keysubtype); + if (h && h->apqns_for_keytype) { + rc = h->apqns_for_keytype(keysubtype, + cur_mkvp, alt_mkvp, flags, + apqns, nr_apqns); + } + pkey_handler_put(h); + + return rc; +} +EXPORT_SYMBOL(pkey_handler_apqns_for_keytype); + +/* + * Module init + */ +static int __init pkey_init(void) +{ + int rc; + + /* init debug feature */ + pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); + debug_register_view(pkey_dbf_info, &debug_sprintf_view); + debug_set_level(pkey_dbf_info, 4); + + /* the handler registry does not need any init */ + + rc = pkey_api_init(); + if (rc) + debug_unregister(pkey_dbf_info); + + return rc; +} + +/* + * Module exit + */ +static void __exit pkey_exit(void) +{ + pkey_api_exit(); +} + +module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_init); +module_exit(pkey_exit); diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h index 560106cbd450..7f97c6e598da 100644 --- a/drivers/s390/crypto/pkey_base.h +++ b/drivers/s390/crypto/pkey_base.h @@ -86,79 +86,10 @@ static inline u32 pkey_aes_bitsize_to_keytype(u32 keybitsize) } /* - * pkey_cca.c: + * pkey_api.c: */ - -bool pkey_is_cca_key(const u8 *key, u32 keylen); -bool pkey_is_cca_keytype(enum pkey_key_type); -int pkey_cca_key2protkey(u16 card, u16 dom, - const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -int pkey_cca_gen_key(u16 card, u16 dom, - u32 keytype, u32 keysubtype, - u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); -int pkey_cca_clr2key(u16 card, u16 dom, - u32 keytype, u32 keysubtype, - u32 keybitsize, u32 flags, - const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); -int pkey_cca_verifykey(const u8 *key, u32 keylen, - u16 *card, u16 *dom, - u32 *keytype, u32 *keybitsize, u32 *flags); -int pkey_cca_apqns4key(const u8 *key, u32 keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns); -int pkey_cca_apqns4type(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns); - -/* - * pkey_ep11.c: - */ - -bool pkey_is_ep11_key(const u8 *key, u32 keylen); -bool pkey_is_ep11_keytype(enum pkey_key_type); -int pkey_ep11_key2protkey(u16 card, u16 dom, - const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -int pkey_ep11_gen_key(u16 card, u16 dom, - u32 keytype, u32 keysubtype, - u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); -int pkey_ep11_clr2key(u16 card, u16 dom, - u32 keytype, u32 keysubtype, - u32 keybitsize, u32 flags, - const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo); -int pkey_ep11_verifykey(const u8 *key, u32 keylen, - u16 *card, u16 *dom, - u32 *keytype, u32 *keybitsize, u32 *flags); -int pkey_ep11_apqns4key(const u8 *key, u32 keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns); -int pkey_ep11_apqns4type(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns); - -/* - * pkey_pckmo.c: - */ - -bool pkey_is_pckmo_key(const u8 *key, u32 keylen); -int pkey_pckmo_key2protkey(u16 _card, u16 _dom, - const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -int pkey_pckmo_gen_key(u16 _card, u16 _dom, - u32 keytype, u32 _keysubtype, - u32 _keybitsize, u32 _flags, - u8 *keybuf, u32 *keybuflen, u32 *keyinfo); -int pkey_pckmo_clr2key(u16 _card, u16 _dom, - u32 keytype, u32 _keysubtype, - u32 _keybitsize, u32 _flags, - const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen, u32 *keyinfo); -int pkey_pckmo_verifykey(const u8 *key, u32 keylen, - u16 *_card, u16 *_dom, - u32 *keytype, u32 *_keybitsize, u32 *_flags); +int __init pkey_api_init(void); +void __exit pkey_api_exit(void); /* * pkey_sysfs.c: @@ -166,4 +97,73 @@ int pkey_pckmo_verifykey(const u8 *key, u32 keylen, extern const struct attribute_group *pkey_attr_groups[]; +/* + * pkey handler registry + */ + +struct pkey_handler { + struct module *module; + const char *name; + /* + * is_supported_key() and is_supported_keytype() are called + * within an rcu_read_lock() scope and thus must not sleep! + */ + bool (*is_supported_key)(const u8 *key, u32 keylen); + bool (*is_supported_keytype)(enum pkey_key_type); + int (*key_to_protkey)(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); + int (*gen_key)(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo); + int (*clr_to_key)(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo); + int (*verify_key)(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags); + int (*apqns_for_key)(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); + int (*apqns_for_keytype)(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); + /* used internal by pkey base */ + struct list_head list; +}; + +int pkey_handler_register(struct pkey_handler *handler); +int pkey_handler_unregister(struct pkey_handler *handler); + +/* + * invocation function for the registered pkey handlers + */ + +const struct pkey_handler *pkey_handler_get_keybased(const u8 *key, u32 keylen); +const struct pkey_handler *pkey_handler_get_keytypebased(enum pkey_key_type kt); +void pkey_handler_put(const struct pkey_handler *handler); + +int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo); +int pkey_handler_clr_to_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo); +int pkey_handler_verify_key(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags); +int pkey_handler_apqns_for_key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); +int pkey_handler_apqns_for_keytype(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns); + #endif /* _PKEY_BASE_H_ */ diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c index 1bf9019ec561..ba2ae253b2ba 100644 --- a/drivers/s390/crypto/pkey_cca.c +++ b/drivers/s390/crypto/pkey_cca.c @@ -8,15 +8,34 @@ #define KMSG_COMPONENT "pkey" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include +#include +#include + #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" - #include "pkey_base.h" +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key CCA handler"); + +#if IS_MODULE(CONFIG_PKEY_CCA) +static struct ap_device_id pkey_cca_card_ids[] = { + { .dev_type = AP_DEVICE_TYPE_CEX4 }, + { .dev_type = AP_DEVICE_TYPE_CEX5 }, + { .dev_type = AP_DEVICE_TYPE_CEX6 }, + { .dev_type = AP_DEVICE_TYPE_CEX7 }, + { .dev_type = AP_DEVICE_TYPE_CEX8 }, + { /* end of list */ }, +}; +MODULE_DEVICE_TABLE(ap, pkey_cca_card_ids); +#endif + /* * Check key blob for known and supported CCA key. */ -bool pkey_is_cca_key(const u8 *key, u32 keylen) +static bool is_cca_key(const u8 *key, u32 keylen) { struct keytoken_header *hdr = (struct keytoken_header *)key; @@ -39,7 +58,7 @@ bool pkey_is_cca_key(const u8 *key, u32 keylen) } } -bool pkey_is_cca_keytype(enum pkey_key_type key_type) +static bool is_cca_keytype(enum pkey_key_type key_type) { switch (key_type) { case PKEY_TYPE_CCA_DATA: @@ -51,268 +70,8 @@ bool pkey_is_cca_keytype(enum pkey_key_type key_type) } } -int pkey_cca_key2protkey(u16 card, u16 dom, - const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int rc; - - if (keylen < sizeof(*hdr)) - return -EINVAL; - - zcrypt_wait_api_operational(); - - if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) { - /* CCA AES data key */ - if (keylen != sizeof(struct secaeskeytoken)) - return -EINVAL; - if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) - return -EINVAL; - rc = cca_sec2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - } else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) { - /* CCA AES cipher key */ - if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) - return -EINVAL; - if (cca_check_secaescipherkey(pkey_dbf_info, - 3, key, 0, 1)) - return -EINVAL; - rc = cca_cipher2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { - /* CCA ECC (private) key */ - if (keylen < sizeof(struct eccprivkeytoken)) - return -EINVAL; - if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) - return -EINVAL; - rc = cca_ecc2protkey(card, dom, key, protkey, - protkeylen, protkeytype); - } else { - PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", - __func__, hdr->type, hdr->version); - rc = -EINVAL; - } - - pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); - return rc; -} - -/* - * Generate CCA secure key. - * As of now only CCA AES Data or Cipher secure keys are - * supported. - * keytype is one of the PKEY_KEYTYPE_* constants, - * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, - * keybitsize is the bit size of the key (may be 0 for - * keytype PKEY_KEYTYPE_AES_*). - */ -int pkey_cca_gen_key(u16 card, u16 dom, - u32 keytype, u32 subtype, - u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) -{ - int len, rc; - - /* check keytype, subtype, keybitsize */ - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - len = pkey_keytype_aes_to_size(keytype); - if (keybitsize && keybitsize != 8 * len) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); - return -EINVAL; - } - keybitsize = 8 * len; - switch (subtype) { - case 0: - case PKEY_TYPE_CCA_DATA: - case PKEY_TYPE_CCA_CIPHER: - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", - __func__, subtype); - return -EINVAL; - } - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", - __func__, keytype); - return -EINVAL; - } - - zcrypt_wait_api_operational(); - - if (subtype == PKEY_TYPE_CCA_CIPHER) { - rc = cca_gencipherkey(card, dom, keybitsize, flags, - keybuf, keybuflen); - } else { - /* 0 or PKEY_TYPE_CCA_DATA */ - rc = cca_genseckey(card, dom, keybitsize, keybuf); - *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); - } - - pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); - return rc; -} - -/* - * Generate CCA secure key with given clear key value. - * As of now only CCA AES Data or Cipher secure keys are - * supported. - * keytype is one of the PKEY_KEYTYPE_* constants, - * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, - * keybitsize is the bit size of the key (may be 0 for - * keytype PKEY_KEYTYPE_AES_*). - */ -int pkey_cca_clr2key(u16 card, u16 dom, - u32 keytype, u32 subtype, - u32 keybitsize, u32 flags, - const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) -{ - int len, rc; - - /* check keytype, subtype, clrkeylen, keybitsize */ - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - len = pkey_keytype_aes_to_size(keytype); - if (keybitsize && keybitsize != 8 * len) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); - return -EINVAL; - } - keybitsize = 8 * len; - if (clrkeylen != len) { - PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", - __func__, clrkeylen, len); - return -EINVAL; - } - switch (subtype) { - case 0: - case PKEY_TYPE_CCA_DATA: - case PKEY_TYPE_CCA_CIPHER: - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", - __func__, subtype); - return -EINVAL; - } - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", - __func__, keytype); - return -EINVAL; - } - - zcrypt_wait_api_operational(); - - if (subtype == PKEY_TYPE_CCA_CIPHER) { - rc = cca_clr2cipherkey(card, dom, keybitsize, - flags, clrkey, keybuf, keybuflen); - } else { - /* 0 or PKEY_TYPE_CCA_DATA */ - rc = cca_clr2seckey(card, dom, keybitsize, - clrkey, keybuf); - *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); - } - - pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); - return rc; -} - -int pkey_cca_verifykey(const u8 *key, u32 keylen, - u16 *card, u16 *dom, - u32 *keytype, u32 *keybitsize, u32 *flags) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - u32 nr_apqns, *apqns = NULL; - int rc; - - if (keylen < sizeof(*hdr)) - return -EINVAL; - - zcrypt_wait_api_operational(); - - if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_AES) { - struct secaeskeytoken *t = (struct secaeskeytoken *)key; - - rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); - if (rc) - goto out; - *keytype = PKEY_TYPE_CCA_DATA; - *keybitsize = t->bitsize; - rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, - ZCRYPT_CEX3C, AES_MK_SET, - t->mkvp, 0, 1); - if (!rc) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - if (rc == -ENODEV) { - rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, - ZCRYPT_CEX3C, AES_MK_SET, - 0, t->mkvp, 1); - if (!rc) - *flags = PKEY_FLAGS_MATCH_ALT_MKVP; - } - if (rc) - goto out; - - *card = ((struct pkey_apqn *)apqns)->card; - *dom = ((struct pkey_apqn *)apqns)->domain; - - } else if (hdr->type == TOKTYPE_CCA_INTERNAL && - hdr->version == TOKVER_CCA_VLSC) { - struct cipherkeytoken *t = (struct cipherkeytoken *)key; - - rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); - if (rc) - goto out; - *keytype = PKEY_TYPE_CCA_CIPHER; - *keybitsize = PKEY_SIZE_UNKNOWN; - if (!t->plfver && t->wpllen == 512) - *keybitsize = PKEY_SIZE_AES_128; - else if (!t->plfver && t->wpllen == 576) - *keybitsize = PKEY_SIZE_AES_192; - else if (!t->plfver && t->wpllen == 640) - *keybitsize = PKEY_SIZE_AES_256; - rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, - ZCRYPT_CEX6, AES_MK_SET, - t->mkvp0, 0, 1); - if (!rc) - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - if (rc == -ENODEV) { - rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, - ZCRYPT_CEX6, AES_MK_SET, - 0, t->mkvp0, 1); - if (!rc) - *flags = PKEY_FLAGS_MATCH_ALT_MKVP; - } - if (rc) - goto out; - - *card = ((struct pkey_apqn *)apqns)->card; - *dom = ((struct pkey_apqn *)apqns)->domain; - - } else { - /* unknown/unsupported key blob */ - rc = -EINVAL; - } - -out: - kfree(apqns); - pr_debug("rc=%d\n", rc); - return rc; -} - -int pkey_cca_apqns4key(const u8 *key, u32 keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int cca_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) { struct keytoken_header *hdr = (struct keytoken_header *)key; u32 _nr_apqns, *_apqns = NULL; @@ -394,9 +153,9 @@ out: return rc; } -int pkey_cca_apqns4type(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int cca_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) { u32 _nr_apqns, *_apqns = NULL; int rc; @@ -451,3 +210,367 @@ out: pr_debug("rc=%d\n", rc); return rc; } + +static int cca_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct pkey_apqn *local_apqns = NULL; + int i, rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + /* CCA AES data key */ + if (keylen != sizeof(struct secaeskeytoken)) + return -EINVAL; + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + /* CCA AES cipher key */ + if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) + return -EINVAL; + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + /* CCA ECC (private) key */ + if (keylen < sizeof(struct eccprivkeytoken)) + return -EINVAL; + if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) + return -EINVAL; + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + rc = cca_apqns4key(key, keylen, 0, local_apqns, &nr_apqns); + if (rc) + goto out; + apqns = local_apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + rc = cca_sec2protkey(apqns[i].card, apqns[i].domain, + key, protkey, + protkeylen, protkeytype); + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + rc = cca_cipher2protkey(apqns[i].card, apqns[i].domain, + key, protkey, + protkeylen, protkeytype); + } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { + rc = cca_ecc2protkey(apqns[i].card, apqns[i].domain, + key, protkey, + protkeylen, protkeytype); + } else { + rc = -EINVAL; + break; + } + } + +out: + kfree(local_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate CCA secure key. + * As of now only CCA AES Data or Cipher secure keys are + * supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int cca_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) +{ + struct pkey_apqn *local_apqns = NULL; + int i, len, rc; + + /* check keytype, subtype, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + switch (subtype) { + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + rc = cca_apqns4type(subtype, NULL, NULL, 0, + local_apqns, &nr_apqns); + if (rc) + goto out; + apqns = local_apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (subtype == PKEY_TYPE_CCA_CIPHER) { + rc = cca_gencipherkey(apqns[i].card, apqns[i].domain, + keybitsize, flags, + keybuf, keybuflen); + } else { + /* PKEY_TYPE_CCA_DATA */ + rc = cca_genseckey(apqns[i].card, apqns[i].domain, + keybitsize, keybuf); + *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); + } + } + +out: + kfree(local_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate CCA secure key with given clear key value. + * As of now only CCA AES Data or Cipher secure keys are + * supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER, + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int cca_clr2key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) +{ + struct pkey_apqn *local_apqns = NULL; + int i, len, rc; + + /* check keytype, subtype, clrkeylen, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + if (clrkeylen != len) { + PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", + __func__, clrkeylen, len); + return -EINVAL; + } + switch (subtype) { + case PKEY_TYPE_CCA_DATA: + case PKEY_TYPE_CCA_CIPHER: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + rc = cca_apqns4type(subtype, NULL, NULL, 0, + local_apqns, &nr_apqns); + if (rc) + goto out; + apqns = local_apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (subtype == PKEY_TYPE_CCA_CIPHER) { + rc = cca_clr2cipherkey(apqns[i].card, apqns[i].domain, + keybitsize, flags, clrkey, + keybuf, keybuflen); + } else { + /* PKEY_TYPE_CCA_DATA */ + rc = cca_clr2seckey(apqns[i].card, apqns[i].domain, + keybitsize, clrkey, keybuf); + *keybuflen = (rc ? 0 : SECKEYBLOBSIZE); + } + } + +out: + kfree(local_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +static int cca_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 nr_apqns, *apqns = NULL; + int rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_AES) { + struct secaeskeytoken *t = (struct secaeskeytoken *)key; + + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); + if (rc) + goto out; + *keytype = PKEY_TYPE_CCA_DATA; + *keybitsize = t->bitsize; + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX3C, AES_MK_SET, + t->mkvp, 0, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + if (rc == -ENODEV) { + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX3C, AES_MK_SET, + 0, t->mkvp, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + } + if (rc) + goto out; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else if (hdr->type == TOKTYPE_CCA_INTERNAL && + hdr->version == TOKVER_CCA_VLSC) { + struct cipherkeytoken *t = (struct cipherkeytoken *)key; + + rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_CCA_CIPHER; + *keybitsize = PKEY_SIZE_UNKNOWN; + if (!t->plfver && t->wpllen == 512) + *keybitsize = PKEY_SIZE_AES_128; + else if (!t->plfver && t->wpllen == 576) + *keybitsize = PKEY_SIZE_AES_192; + else if (!t->plfver && t->wpllen == 640) + *keybitsize = PKEY_SIZE_AES_256; + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX6, AES_MK_SET, + t->mkvp0, 0, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + if (rc == -ENODEV) { + rc = cca_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX6, AES_MK_SET, + 0, t->mkvp0, 1); + if (!rc) + *flags = PKEY_FLAGS_MATCH_ALT_MKVP; + } + if (rc) + goto out; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else { + /* unknown/unsupported key blob */ + rc = -EINVAL; + } + +out: + kfree(apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +static struct pkey_handler cca_handler = { + .module = THIS_MODULE, + .name = "PKEY CCA handler", + .is_supported_key = is_cca_key, + .is_supported_keytype = is_cca_keytype, + .key_to_protkey = cca_key2protkey, + .gen_key = cca_gen_key, + .clr_to_key = cca_clr2key, + .verify_key = cca_verifykey, + .apqns_for_key = cca_apqns4key, + .apqns_for_keytype = cca_apqns4type, +}; + +/* + * Module init + */ +static int __init pkey_cca_init(void) +{ + /* register this module as pkey handler for all the cca stuff */ + return pkey_handler_register(&cca_handler); +} + +/* + * Module exit + */ +static void __exit pkey_cca_exit(void) +{ + /* unregister this module as pkey handler */ + pkey_handler_unregister(&cca_handler); +} + +module_init(pkey_cca_init); +module_exit(pkey_cca_exit); diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c index 4c49e07ece74..624e55195d93 100644 --- a/drivers/s390/crypto/pkey_ep11.c +++ b/drivers/s390/crypto/pkey_ep11.c @@ -8,16 +8,35 @@ #define KMSG_COMPONENT "pkey" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include +#include +#include + #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" #include "zcrypt_ep11misc.h" - #include "pkey_base.h" +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key EP11 handler"); + +#if IS_MODULE(CONFIG_PKEY_EP11) +static struct ap_device_id pkey_ep11_card_ids[] = { + { .dev_type = AP_DEVICE_TYPE_CEX4 }, + { .dev_type = AP_DEVICE_TYPE_CEX5 }, + { .dev_type = AP_DEVICE_TYPE_CEX6 }, + { .dev_type = AP_DEVICE_TYPE_CEX7 }, + { .dev_type = AP_DEVICE_TYPE_CEX8 }, + { /* end of list */ }, +}; +MODULE_DEVICE_TABLE(ap, pkey_ep11_card_ids); +#endif + /* * Check key blob for known and supported EP11 key. */ -bool pkey_is_ep11_key(const u8 *key, u32 keylen) +static bool is_ep11_key(const u8 *key, u32 keylen) { struct keytoken_header *hdr = (struct keytoken_header *)key; @@ -39,7 +58,7 @@ bool pkey_is_ep11_key(const u8 *key, u32 keylen) } } -bool pkey_is_ep11_keytype(enum pkey_key_type key_type) +static bool is_ep11_keytype(enum pkey_key_type key_type) { switch (key_type) { case PKEY_TYPE_EP11: @@ -51,245 +70,8 @@ bool pkey_is_ep11_keytype(enum pkey_key_type key_type) } } -int pkey_ep11_key2protkey(u16 card, u16 dom, - const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - int rc; - - if (keylen < sizeof(*hdr)) - return -EINVAL; - - zcrypt_wait_api_operational(); - - if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES_WITH_HEADER && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - /* EP11 AES key blob with header */ - if (ep11_check_aes_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1)) - return -EINVAL; - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, protkeytype); - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_ECC_WITH_HEADER && - is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { - /* EP11 ECC key blob with header */ - if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1)) - return -EINVAL; - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, protkeytype); - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES && - is_ep11_keyblob(key)) { - /* EP11 AES key blob with header in session field */ - if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) - return -EINVAL; - rc = ep11_kblob2protkey(card, dom, key, hdr->len, - protkey, protkeylen, protkeytype); - } else { - PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", - __func__, hdr->type, hdr->version); - return -EINVAL; - } - - pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); - return rc; -} - -/* - * Generate EP11 secure key. - * As of now only EP11 AES secure keys are supported. - * keytype is one of the PKEY_KEYTYPE_* constants, - * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES - * or 0 (results in subtype PKEY_TYPE_EP11_AES), - * keybitsize is the bit size of the key (may be 0 for - * keytype PKEY_KEYTYPE_AES_*). - */ -int pkey_ep11_gen_key(u16 card, u16 dom, - u32 keytype, u32 subtype, - u32 keybitsize, u32 flags, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) -{ - int len, rc; - - /* check keytype, subtype, keybitsize */ - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - len = pkey_keytype_aes_to_size(keytype); - if (keybitsize && keybitsize != 8 * len) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); - return -EINVAL; - } - keybitsize = 8 * len; - switch (subtype) { - case 0: - subtype = PKEY_TYPE_EP11_AES; - break; - case PKEY_TYPE_EP11: - case PKEY_TYPE_EP11_AES: - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", - __func__, subtype); - return -EINVAL; - } - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", - __func__, keytype); - return -EINVAL; - } - - zcrypt_wait_api_operational(); - - rc = ep11_genaeskey(card, dom, keybitsize, flags, - keybuf, keybuflen, subtype); - - pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); - return rc; -} - -/* - * Generate EP11 secure key with given clear key value. - * As of now only EP11 AES secure keys are supported. - * keytype is one of the PKEY_KEYTYPE_* constants, - * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES - * or 0 (assumes PKEY_TYPE_EP11_AES then). - * keybitsize is the bit size of the key (may be 0 for - * keytype PKEY_KEYTYPE_AES_*). - */ -int pkey_ep11_clr2key(u16 card, u16 dom, - u32 keytype, u32 subtype, - u32 keybitsize, u32 flags, - const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) -{ - int len, rc; - - /* check keytype, subtype, clrkeylen, keybitsize */ - switch (keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - len = pkey_keytype_aes_to_size(keytype); - if (keybitsize && keybitsize != 8 * len) { - PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); - return -EINVAL; - } - keybitsize = 8 * len; - if (clrkeylen != len) { - PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", - __func__, clrkeylen, len); - return -EINVAL; - } - switch (subtype) { - case 0: - subtype = PKEY_TYPE_EP11_AES; - break; - case PKEY_TYPE_EP11: - case PKEY_TYPE_EP11_AES: - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", - __func__, subtype); - return -EINVAL; - } - break; - default: - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", - __func__, keytype); - return -EINVAL; - } - - zcrypt_wait_api_operational(); - - rc = ep11_clr2keyblob(card, dom, keybitsize, flags, - clrkey, keybuf, keybuflen, subtype); - - pr_debug("card=%d dom=%d rc=%d\n", card, dom, rc); - return rc; -} - -int pkey_ep11_verifykey(const u8 *key, u32 keylen, - u16 *card, u16 *dom, - u32 *keytype, u32 *keybitsize, u32 *flags) -{ - struct keytoken_header *hdr = (struct keytoken_header *)key; - u32 nr_apqns, *apqns = NULL; - int rc; - - if (keylen < sizeof(*hdr)) - return -EINVAL; - - zcrypt_wait_api_operational(); - - if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES) { - struct ep11keyblob *kb = (struct ep11keyblob *)key; - int api; - - rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); - if (rc) - goto out; - *keytype = PKEY_TYPE_EP11; - *keybitsize = kb->head.bitlen; - - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&apqns, &nr_apqns, *card, *dom, - ZCRYPT_CEX7, api, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; - - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - - *card = ((struct pkey_apqn *)apqns)->card; - *dom = ((struct pkey_apqn *)apqns)->domain; - - } else if (hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - struct ep11kblob_header *kh = (struct ep11kblob_header *)key; - int api; - - rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, - 3, key, keylen, 1); - if (rc) - goto out; - *keytype = PKEY_TYPE_EP11_AES; - *keybitsize = kh->bitlen; - - api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; - rc = ep11_findcard2(&apqns, &nr_apqns, *card, *dom, - ZCRYPT_CEX7, api, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; - - *flags = PKEY_FLAGS_MATCH_CUR_MKVP; - - *card = ((struct pkey_apqn *)apqns)->card; - *dom = ((struct pkey_apqn *)apqns)->domain; - - } else { - /* unknown/unsupported key blob */ - rc = -EINVAL; - } - -out: - kfree(apqns); - pr_debug("rc=%d\n", rc); - return rc; -} - -int pkey_ep11_apqns4key(const u8 *key, u32 keylen, u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int ep11_apqns4key(const u8 *key, u32 keylen, u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) { struct keytoken_header *hdr = (struct keytoken_header *)key; u32 _nr_apqns, *_apqns = NULL; @@ -359,9 +141,9 @@ out: return rc; } -int pkey_ep11_apqns4type(enum pkey_key_type ktype, - u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, - struct pkey_apqn *apqns, size_t *nr_apqns) +static int ep11_apqns4type(enum pkey_key_type ktype, + u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, + struct pkey_apqn *apqns, size_t *nr_apqns) { u32 _nr_apqns, *_apqns = NULL; int rc; @@ -401,3 +183,343 @@ out: pr_debug("rc=%d\n", rc); return rc; } + +static int ep11_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + struct pkey_apqn *local_apqns = NULL; + int i, rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 AES key blob with header */ + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_ECC_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + /* EP11 ECC key blob with header */ + if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) + return -EINVAL; + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + /* EP11 AES key blob with header in session field */ + if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) + return -EINVAL; + } else { + PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n", + __func__, hdr->type, hdr->version); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + rc = ep11_apqns4key(key, keylen, 0, local_apqns, &nr_apqns); + if (rc) + goto out; + apqns = local_apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain, + key, hdr->len, protkey, + protkeylen, protkeytype); + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_ECC_WITH_HEADER && + is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { + rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain, + key, hdr->len, protkey, + protkeylen, protkeytype); + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES && + is_ep11_keyblob(key)) { + rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain, + key, hdr->len, protkey, + protkeylen, protkeytype); + } else { + rc = -EINVAL; + break; + } + } + +out: + kfree(local_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate EP11 secure key. + * As of now only EP11 AES secure keys are supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES + * or 0 (results in subtype PKEY_TYPE_EP11_AES), + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int ep11_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) +{ + struct pkey_apqn *local_apqns = NULL; + int i, len, rc; + + /* check keytype, subtype, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + switch (subtype) { + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + rc = ep11_apqns4type(subtype, NULL, NULL, 0, + local_apqns, &nr_apqns); + if (rc) + goto out; + apqns = local_apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = ep11_genaeskey(apqns[i].card, apqns[i].domain, + keybitsize, flags, + keybuf, keybuflen, subtype); + } + +out: + kfree(local_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate EP11 secure key with given clear key value. + * As of now only EP11 AES secure keys are supported. + * keytype is one of the PKEY_KEYTYPE_* constants, + * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES + * or 0 (assumes PKEY_TYPE_EP11_AES then). + * keybitsize is the bit size of the key (may be 0 for + * keytype PKEY_KEYTYPE_AES_*). + */ +static int ep11_clr2key(const struct pkey_apqn *apqns, size_t nr_apqns, + u32 keytype, u32 subtype, + u32 keybitsize, u32 flags, + const u8 *clrkey, u32 clrkeylen, + u8 *keybuf, u32 *keybuflen, u32 *_keyinfo) +{ + struct pkey_apqn *local_apqns = NULL; + int i, len, rc; + + /* check keytype, subtype, clrkeylen, keybitsize */ + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + len = pkey_keytype_aes_to_size(keytype); + if (keybitsize && keybitsize != 8 * len) { + PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + keybitsize = 8 * len; + if (clrkeylen != len) { + PKEY_DBF_ERR("%s invalid clear key len %d != %d\n", + __func__, clrkeylen, len); + return -EINVAL; + } + switch (subtype) { + case PKEY_TYPE_EP11: + case PKEY_TYPE_EP11_AES: + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); + return -EINVAL; + } + + zcrypt_wait_api_operational(); + + if (!apqns || (nr_apqns == 1 && + apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) { + nr_apqns = MAXAPQNSINLIST; + local_apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), + GFP_KERNEL); + if (!local_apqns) + return -ENOMEM; + rc = ep11_apqns4type(subtype, NULL, NULL, 0, + local_apqns, &nr_apqns); + if (rc) + goto out; + apqns = local_apqns; + } + + for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) { + rc = ep11_clr2keyblob(apqns[i].card, apqns[i].domain, + keybitsize, flags, clrkey, + keybuf, keybuflen, subtype); + } + +out: + kfree(local_apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +static int ep11_verifykey(const u8 *key, u32 keylen, + u16 *card, u16 *dom, + u32 *keytype, u32 *keybitsize, u32 *flags) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + u32 nr_apqns, *apqns = NULL; + int rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + zcrypt_wait_api_operational(); + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES) { + struct ep11keyblob *kb = (struct ep11keyblob *)key; + int api; + + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_EP11; + *keybitsize = kb->head.bitlen; + + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX7, api, + ep11_kb_wkvp(key, keylen)); + if (rc) + goto out; + + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_EP11_AES_WITH_HEADER) { + struct ep11kblob_header *kh = (struct ep11kblob_header *)key; + int api; + + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); + if (rc) + goto out; + *keytype = PKEY_TYPE_EP11_AES; + *keybitsize = kh->bitlen; + + api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4; + rc = ep11_findcard2(&apqns, &nr_apqns, *card, *dom, + ZCRYPT_CEX7, api, + ep11_kb_wkvp(key, keylen)); + if (rc) + goto out; + + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *card = ((struct pkey_apqn *)apqns)->card; + *dom = ((struct pkey_apqn *)apqns)->domain; + + } else { + /* unknown/unsupported key blob */ + rc = -EINVAL; + } + +out: + kfree(apqns); + pr_debug("rc=%d\n", rc); + return rc; +} + +static struct pkey_handler ep11_handler = { + .module = THIS_MODULE, + .name = "PKEY EP11 handler", + .is_supported_key = is_ep11_key, + .is_supported_keytype = is_ep11_keytype, + .key_to_protkey = ep11_key2protkey, + .gen_key = ep11_gen_key, + .clr_to_key = ep11_clr2key, + .verify_key = ep11_verifykey, + .apqns_for_key = ep11_apqns4key, + .apqns_for_keytype = ep11_apqns4type, +}; + +/* + * Module init + */ +static int __init pkey_ep11_init(void) +{ + /* register this module as pkey handler for all the ep11 stuff */ + return pkey_handler_register(&ep11_handler); +} + +/* + * Module exit + */ +static void __exit pkey_ep11_exit(void) +{ + /* unregister this module as pkey handler */ + pkey_handler_unregister(&ep11_handler); +} + +module_init(pkey_ep11_init); +module_exit(pkey_ep11_exit); diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c index d2c2c61f449b..0667e5510671 100644 --- a/drivers/s390/crypto/pkey_pckmo.c +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -8,71 +8,20 @@ #define KMSG_COMPONENT "pkey" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include +#include +#include #include #include #include #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" - #include "pkey_base.h" -/* - * Prototypes - */ - -static bool is_pckmo_key(const u8 *key, u32 keylen); -static int pckmo_key2protkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -static int pckmo_gen_protkey(u32 keytype, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype); -static int pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, - u32 protkeytype); - -/* - * Wrapper functions - */ - -bool pkey_is_pckmo_key(const u8 *key, u32 keylen) -{ - return is_pckmo_key(key, keylen); -} - -int pkey_pckmo_key2protkey(u16 _card, u16 _dom, - const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *keyinfo) -{ - return pckmo_key2protkey(key, keylen, - protkey, protkeylen, keyinfo); -} - -int pkey_pckmo_gen_key(u16 _card, u16 _dom, - u32 keytype, u32 _keysubtype, - u32 _keybitsize, u32 _flags, - u8 *keybuf, u32 *keybuflen, u32 *keyinfo) -{ - return pckmo_gen_protkey(keytype, - keybuf, keybuflen, keyinfo); -} - -int pkey_pckmo_clr2key(u16 _card, u16 _dom, - u32 keytype, u32 _keysubtype, - u32 _keybitsize, u32 _flags, - const u8 *clrkey, u32 clrkeylen, - u8 *keybuf, u32 *keybuflen, u32 *keyinfo) -{ - return pckmo_clr2protkey(keytype, clrkey, clrkeylen, - keybuf, keybuflen, keyinfo); -} - -int pkey_pckmo_verifykey(const u8 *key, u32 keylen, - u16 *_card, u16 *_dom, - u32 *keytype, u32 *_keybitsize, u32 *_flags) -{ - return pckmo_verify_protkey(key, keylen, *keytype); -} +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("s390 protected key PCKMO handler"); /* * Check key blob for known and supported here. @@ -112,122 +61,14 @@ static bool is_pckmo_key(const u8 *key, u32 keylen) } } -static int pckmo_key2protkey(const u8 *key, u32 keylen, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) +static bool is_pckmo_keytype(enum pkey_key_type keytype) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - int rc = -EINVAL; - - if (keylen < sizeof(*hdr)) - return -EINVAL; - if (hdr->type != TOKTYPE_NON_CCA) - return -EINVAL; - - switch (hdr->version) { - case TOKVER_PROTECTED_KEY: { - struct protaeskeytoken *t; - - if (keylen != sizeof(struct protaeskeytoken)) - goto out; - t = (struct protaeskeytoken *)key; - rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype); - if (rc) - goto out; - memcpy(protkey, t->protkey, t->len); - *protkeylen = t->len; - *protkeytype = t->keytype; - break; - } - case TOKVER_CLEAR_KEY: { - struct clearkeytoken *t = (struct clearkeytoken *)key; - u32 keysize = 0; - - if (keylen < sizeof(struct clearkeytoken) || - keylen != sizeof(*t) + t->len) - goto out; - switch (t->keytype) { - case PKEY_KEYTYPE_AES_128: - case PKEY_KEYTYPE_AES_192: - case PKEY_KEYTYPE_AES_256: - keysize = pkey_keytype_aes_to_size(t->keytype); - break; - case PKEY_KEYTYPE_ECC_P256: - keysize = 32; - break; - case PKEY_KEYTYPE_ECC_P384: - keysize = 48; - break; - case PKEY_KEYTYPE_ECC_P521: - keysize = 80; - break; - case PKEY_KEYTYPE_ECC_ED25519: - keysize = 32; - break; - case PKEY_KEYTYPE_ECC_ED448: - keysize = 64; - break; - default: - break; - } - if (!keysize) { - PKEY_DBF_ERR("%s clear key token: unknown keytype %u\n", - __func__, t->keytype); - goto out; - } - if (t->len != keysize) { - PKEY_DBF_ERR("%s clear key token: invalid key len %u\n", - __func__, t->len); - goto out; - } - rc = pckmo_clr2protkey(t->keytype, t->clearkey, t->len, - protkey, protkeylen, protkeytype); - break; - } + switch (keytype) { + case PKEY_TYPE_PROTKEY: + return true; default: - PKEY_DBF_ERR("%s unknown non-CCA token version %d\n", - __func__, hdr->version); - break; + return false; } - -out: - pr_debug("rc=%d\n", rc); - return rc; -} - -/* - * Generate a random protected key. - * Currently only the generation of AES protected keys - * is supported. - */ -static int pckmo_gen_protkey(u32 keytype, u8 *protkey, - u32 *protkeylen, u32 *protkeytype) -{ - u8 clrkey[32]; - int keysize; - int rc; - - keysize = pkey_keytype_aes_to_size(keytype); - if (!keysize) { - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, - keytype); - return -EINVAL; - } - - /* generate a dummy random clear key */ - get_random_bytes(clrkey, keysize); - - /* convert it to a dummy protected key */ - rc = pckmo_clr2protkey(keytype, clrkey, keysize, - protkey, protkeylen, protkeytype); - if (rc) - goto out; - - /* replace the key part of the protected key with random bytes */ - get_random_bytes(protkey, keysize); - -out: - pr_debug("rc=%d\n", rc); - return rc; } /* @@ -346,7 +187,7 @@ out: } /* - * Verify a protected key blob. + * Verify a raw protected key blob. * Currently only AES protected keys are supported. */ static int pckmo_verify_protkey(const u8 *protkey, u32 protkeylen, @@ -405,3 +246,232 @@ out: pr_debug("rc=%d\n", rc); return rc; } + +static int pckmo_key2protkey(const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc = -EINVAL; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + if (hdr->type != TOKTYPE_NON_CCA) + return -EINVAL; + + switch (hdr->version) { + case TOKVER_PROTECTED_KEY: { + struct protaeskeytoken *t; + + if (keylen != sizeof(struct protaeskeytoken)) + goto out; + t = (struct protaeskeytoken *)key; + rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype); + if (rc) + goto out; + memcpy(protkey, t->protkey, t->len); + *protkeylen = t->len; + *protkeytype = t->keytype; + break; + } + case TOKVER_CLEAR_KEY: { + struct clearkeytoken *t = (struct clearkeytoken *)key; + u32 keysize = 0; + + if (keylen < sizeof(struct clearkeytoken) || + keylen != sizeof(*t) + t->len) + goto out; + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + keysize = pkey_keytype_aes_to_size(t->keytype); + break; + case PKEY_KEYTYPE_ECC_P256: + keysize = 32; + break; + case PKEY_KEYTYPE_ECC_P384: + keysize = 48; + break; + case PKEY_KEYTYPE_ECC_P521: + keysize = 80; + break; + case PKEY_KEYTYPE_ECC_ED25519: + keysize = 32; + break; + case PKEY_KEYTYPE_ECC_ED448: + keysize = 64; + break; + default: + break; + } + if (!keysize) { + PKEY_DBF_ERR("%s clear key token: unknown keytype %u\n", + __func__, t->keytype); + goto out; + } + if (t->len != keysize) { + PKEY_DBF_ERR("%s clear key token: invalid key len %u\n", + __func__, t->len); + goto out; + } + rc = pckmo_clr2protkey(t->keytype, t->clearkey, t->len, + protkey, protkeylen, protkeytype); + break; + } + default: + PKEY_DBF_ERR("%s unknown non-CCA token version %d\n", + __func__, hdr->version); + break; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Generate a random protected key. + * Currently only the generation of AES protected keys + * is supported. + */ +static int pckmo_gen_protkey(u32 keytype, u32 subtype, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + u8 clrkey[32]; + int keysize; + int rc; + + keysize = pkey_keytype_aes_to_size(keytype); + if (!keysize) { + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, + keytype); + return -EINVAL; + } + if (subtype != PKEY_TYPE_PROTKEY) { + PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n", + __func__, subtype); + return -EINVAL; + } + + /* generate a dummy random clear key */ + get_random_bytes(clrkey, keysize); + + /* convert it to a dummy protected key */ + rc = pckmo_clr2protkey(keytype, clrkey, keysize, + protkey, protkeylen, protkeytype); + if (rc) + goto out; + + /* replace the key part of the protected key with random bytes */ + get_random_bytes(protkey, keysize); + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Verify a protected key token blob. + * Currently only AES protected keys are supported. + */ +static int pckmo_verify_key(const u8 *key, u32 keylen) +{ + struct keytoken_header *hdr = (struct keytoken_header *)key; + int rc = -EINVAL; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + if (hdr->type != TOKTYPE_NON_CCA) + return -EINVAL; + + switch (hdr->version) { + case TOKVER_PROTECTED_KEY: { + struct protaeskeytoken *t; + + if (keylen != sizeof(struct protaeskeytoken)) + goto out; + t = (struct protaeskeytoken *)key; + rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype); + break; + } + default: + PKEY_DBF_ERR("%s unknown non-CCA token version %d\n", + __func__, hdr->version); + break; + } + +out: + pr_debug("rc=%d\n", rc); + return rc; +} + +/* + * Wrapper functions used for the pkey handler struct + */ + +static int pkey_pckmo_key2protkey(const struct pkey_apqn *_apqns, + size_t _nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, u32 *keyinfo) +{ + return pckmo_key2protkey(key, keylen, + protkey, protkeylen, keyinfo); +} + +static int pkey_pckmo_gen_key(const struct pkey_apqn *_apqns, size_t _nr_apqns, + u32 keytype, u32 keysubtype, + u32 _keybitsize, u32 _flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + return pckmo_gen_protkey(keytype, keysubtype, + keybuf, keybuflen, keyinfo); +} + +static int pkey_pckmo_verifykey(const u8 *key, u32 keylen, + u16 *_card, u16 *_dom, + u32 *_keytype, u32 *_keybitsize, u32 *_flags) +{ + return pckmo_verify_key(key, keylen); +} + +static struct pkey_handler pckmo_handler = { + .module = THIS_MODULE, + .name = "PKEY PCKMO handler", + .is_supported_key = is_pckmo_key, + .is_supported_keytype = is_pckmo_keytype, + .key_to_protkey = pkey_pckmo_key2protkey, + .gen_key = pkey_pckmo_gen_key, + .verify_key = pkey_pckmo_verifykey, +}; + +/* + * Module init + */ +static int __init pkey_pckmo_init(void) +{ + cpacf_mask_t func_mask; + + /* + * The pckmo instruction should be available - even if we don't + * actually invoke it. This instruction comes with MSA 3 which + * is also the minimum level for the kmc instructions which + * are able to work with protected keys. + */ + if (!cpacf_query(CPACF_PCKMO, &func_mask)) + return -ENODEV; + + /* register this module as pkey handler for all the pckmo stuff */ + return pkey_handler_register(&pckmo_handler); +} + +/* + * Module exit + */ +static void __exit pkey_pckmo_exit(void) +{ + /* unregister this module as pkey handler */ + pkey_handler_unregister(&pckmo_handler); +} + +module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_pckmo_init); +module_exit(pkey_pckmo_exit); diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c index 684f87d6e9f1..242eb6b1a158 100644 --- a/drivers/s390/crypto/pkey_sysfs.c +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -8,7 +8,6 @@ #define KMSG_COMPONENT "pkey" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt -#include #include #include "zcrypt_api.h" @@ -42,10 +41,10 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, protkeytoken.keytype = keytype; protkey.len = sizeof(protkey.protkey); - rc = pkey_pckmo_gen_key(0, 0, - protkeytoken.keytype, 0, 0, 0, - protkey.protkey, &protkey.len, - &protkey.type); + rc = pkey_handler_gen_key(NULL, 0, keytype, + PKEY_TYPE_PROTKEY, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); if (rc) return rc; @@ -57,10 +56,10 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, if (is_xts) { /* xts needs a second protected key, reuse protkey struct */ protkey.len = sizeof(protkey.protkey); - rc = pkey_pckmo_gen_key(0, 0, - protkeytoken.keytype, 0, 0, 0, - protkey.protkey, &protkey.len, - &protkey.type); + rc = pkey_handler_gen_key(NULL, 0, keytype, + PKEY_TYPE_PROTKEY, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); if (rc) return rc; @@ -166,18 +165,18 @@ static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, return -EINVAL; buflen = sizeof(seckey->seckey); - rc = pkey_cca_gen_key(-1, -1, keytype, - PKEY_TYPE_CCA_DATA, 0, 0, - seckey->seckey, &buflen, NULL); + rc = pkey_handler_gen_key(NULL, 0, keytype, + PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); if (rc) return rc; if (is_xts) { seckey++; buflen = sizeof(seckey->seckey); - rc = pkey_cca_gen_key(-1, -1, keytype, - PKEY_TYPE_CCA_DATA, 0, 0, - seckey->seckey, &buflen, NULL); + rc = pkey_handler_gen_key(NULL, 0, keytype, + PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); if (rc) return rc; @@ -270,9 +269,7 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, size_t count) { u32 keysize = CCACIPHERTOKENSIZE; - struct pkey_apqn *apqns = NULL; - int i, rc, card, dom; - size_t nr_apqns; + int rc; if (off != 0 || count < CCACIPHERTOKENSIZE) return -EINVAL; @@ -280,51 +277,27 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, if (count < 2 * CCACIPHERTOKENSIZE) return -EINVAL; - nr_apqns = MAXAPQNSINLIST; - apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); - if (!apqns) - return -ENOMEM; - - /* build a list of apqns able to generate an cipher key */ - rc = pkey_cca_apqns4type(PKEY_TYPE_CCA_CIPHER, - NULL, NULL, 0, - apqns, &nr_apqns); - if (rc) { - kfree(apqns); - return rc; - } - memset(buf, 0, is_xts ? 2 * keysize : keysize); - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; rc && i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - rc = pkey_cca_gen_key(card, dom, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_CCA_CIPHER, keybits, 0, - buf, &keysize, NULL); - } - if (rc) { - kfree(apqns); + rc = pkey_handler_gen_key(NULL, 0, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); + if (rc) return rc; - } if (is_xts) { keysize = CCACIPHERTOKENSIZE; buf += CCACIPHERTOKENSIZE; - rc = pkey_cca_gen_key(card, dom, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_CCA_CIPHER, keybits, 0, - buf, &keysize, NULL); - kfree(apqns); + rc = pkey_handler_gen_key(NULL, 0, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); if (rc) return rc; return 2 * CCACIPHERTOKENSIZE; } - kfree(apqns); - return CCACIPHERTOKENSIZE; } @@ -412,9 +385,7 @@ static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, size_t count) { u32 keysize = MAXEP11AESKEYBLOBSIZE; - struct pkey_apqn *apqns = NULL; - int i, rc, card, dom; - size_t nr_apqns; + int rc; if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) return -EINVAL; @@ -422,51 +393,27 @@ static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, if (count < 2 * MAXEP11AESKEYBLOBSIZE) return -EINVAL; - nr_apqns = MAXAPQNSINLIST; - apqns = kmalloc_array(nr_apqns, sizeof(struct pkey_apqn), GFP_KERNEL); - if (!apqns) - return -ENOMEM; - - /* build a list of apqns able to generate an EP11 AES key */ - rc = pkey_ep11_apqns4type(PKEY_TYPE_EP11_AES, - NULL, NULL, 0, - apqns, &nr_apqns); - if (rc) { - kfree(apqns); - return rc; - } - memset(buf, 0, is_xts ? 2 * keysize : keysize); - /* simple try all apqns from the list */ - for (i = 0, rc = -ENODEV; rc && i < nr_apqns; i++) { - card = apqns[i].card; - dom = apqns[i].domain; - rc = pkey_ep11_gen_key(card, dom, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_EP11_AES, keybits, 0, - buf, &keysize, NULL); - } - if (rc) { - kfree(apqns); + rc = pkey_handler_gen_key(NULL, 0, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); + if (rc) return rc; - } if (is_xts) { keysize = MAXEP11AESKEYBLOBSIZE; buf += MAXEP11AESKEYBLOBSIZE; - rc = pkey_ep11_gen_key(card, dom, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_EP11_AES, keybits, 0, - buf, &keysize, NULL); - kfree(apqns); + rc = pkey_handler_gen_key(NULL, 0, + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); if (rc) return rc; return 2 * MAXEP11AESKEYBLOBSIZE; } - kfree(apqns); - return MAXEP11AESKEYBLOBSIZE; } From 2fc401b94434ae97d1c9c1283fcf816c1ad9457c Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 22 Aug 2024 11:32:20 +0200 Subject: [PATCH 41/84] s390/pkey: Add slowpath function to CCA and EP11 handler For some keys there exists an alternative but usually slower path to convert the key material into a protected key. This patch introduces a new handler function slowpath_key_to_protkey() which provides this alternate path for the CCA and EP11 handler code. With that even the knowledge about how and when this can be used within the pkey API code can be removed. So now the pkey API just tries the primary way and if that fails simple tries the alternative way. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/pkey_api.c | 105 +++++--------------------------- drivers/s390/crypto/pkey_base.c | 40 ++++++++++++ drivers/s390/crypto/pkey_base.h | 10 +++ drivers/s390/crypto/pkey_cca.c | 73 +++++++++++++++++++--- drivers/s390/crypto/pkey_ep11.c | 73 +++++++++++++++++++--- 5 files changed, 192 insertions(+), 109 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index c59051ab1cfb..d6c5e5ae915b 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -22,102 +22,29 @@ /* * Helper functions */ - -static int key2protkey_fallback(const struct clearkeytoken *t, - u8 *protkey, u32 *protkeylen, u32 *protkeytype) -{ - size_t tmpbuflen = max_t(size_t, SECKEYBLOBSIZE, MAXEP11AESKEYBLOBSIZE); - u32 keysize, keybitsize, tmplen; - u8 *tmpbuf = NULL; - int i, rc; - - /* As of now only for AES keys a fallback is available */ - - keysize = pkey_keytype_aes_to_size(t->keytype); - if (!keysize) { - PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); - return -EINVAL; - } - if (t->len != keysize) { - PKEY_DBF_ERR("%s clear key AES token: invalid key len %u\n", - __func__, t->len); - return -EINVAL; - } - keybitsize = 8 * keysize; - - /* alloc tmp key buffer */ - tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); - if (!tmpbuf) - return -ENOMEM; - - /* try two times in case of failure */ - for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { - - /* CCA secure key way */ - tmplen = tmpbuflen; - rc = pkey_handler_clr_to_key(NULL, 0, - t->keytype, PKEY_TYPE_CCA_DATA, - keybitsize, 0, - t->clearkey, t->len, - tmpbuf, &tmplen, NULL); - pr_debug("clr_to_key()=%d\n", rc); - if (rc) - goto try_via_ep11; - rc = pkey_handler_key_to_protkey(NULL, 0, - tmpbuf, tmplen, - protkey, protkeylen, - protkeytype); - pr_debug("key_to_protkey()=%d\n", rc); - if (!rc) - break; - -try_via_ep11: - /* the CCA way failed, try via EP11 */ - tmplen = tmpbuflen; - rc = pkey_handler_clr_to_key(NULL, 0, - t->keytype, PKEY_TYPE_EP11_AES, - keybitsize, 0, - t->clearkey, t->len, - tmpbuf, &tmplen, NULL); - pr_debug("clr_to_key()=%d\n", rc); - if (rc) - continue; - rc = pkey_handler_key_to_protkey(NULL, 0, - tmpbuf, tmplen, - protkey, protkeylen, - protkeytype); - pr_debug("key_to_protkey()=%d\n", rc); - } - - kfree(tmpbuf); - - return rc; -} - static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, size_t keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - struct keytoken_header *hdr = (struct keytoken_header *)key; - int i, rc; + int rc; - /* retry two times */ - for (rc = -ENODEV, i = 0; rc && i < 2; i++) { - /* First try the direct way */ - rc = pkey_handler_key_to_protkey(apqns, nr_apqns, - key, keylen, - protkey, protkeylen, - protkeytype); - /* For some clear key tokens there exists a fallback way */ - if (rc && - hdr->type == TOKTYPE_NON_CCA && - hdr->version == TOKVER_CLEAR_KEY) - rc = key2protkey_fallback((struct clearkeytoken *)key, - protkey, protkeylen, - protkeytype); + /* try the direct way */ + rc = pkey_handler_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype); + + /* if this did not work, try the slowpath way */ + if (rc == -ENODEV) { + rc = pkey_handler_slowpath_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype); + if (rc) + rc = -ENODEV; } + pr_debug("rc=%d\n", rc); return rc; } diff --git a/drivers/s390/crypto/pkey_base.c b/drivers/s390/crypto/pkey_base.c index e7abc32ca5f9..976b9ddfbe15 100644 --- a/drivers/s390/crypto/pkey_base.c +++ b/drivers/s390/crypto/pkey_base.c @@ -167,6 +167,46 @@ int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns, } EXPORT_SYMBOL(pkey_handler_key_to_protkey); +/* + * This handler invocation is special as there may be more than + * one handler providing support for the very same key (type). + * And the handler may not respond true on is_supported_key(), + * so simple try and check return value here. + */ +int pkey_handler_slowpath_key_to_protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype) +{ + const struct pkey_handler *h, *htmp[10]; + int i, n = 0, rc = -ENODEV; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (!try_module_get(h->module)) + continue; + if (h->slowpath_key_to_protkey && n < ARRAY_SIZE(htmp)) + htmp[n++] = h; + else + module_put(h->module); + } + rcu_read_unlock(); + + for (i = 0; i < n; i++) { + h = htmp[i]; + if (rc) + rc = h->slowpath_key_to_protkey(apqns, nr_apqns, + key, keylen, + protkey, protkeylen, + protkeytype); + module_put(h->module); + } + + return rc; +} +EXPORT_SYMBOL(pkey_handler_slowpath_key_to_protkey); + int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h index 7f97c6e598da..41fd69e7c66e 100644 --- a/drivers/s390/crypto/pkey_base.h +++ b/drivers/s390/crypto/pkey_base.h @@ -113,6 +113,11 @@ struct pkey_handler { int (*key_to_protkey)(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, u32 keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype); + int (*slowpath_key_to_protkey)(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype); int (*gen_key)(const struct pkey_apqn *apqns, size_t nr_apqns, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, @@ -148,6 +153,11 @@ void pkey_handler_put(const struct pkey_handler *handler); int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns, const u8 *key, u32 keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype); +int pkey_handler_slowpath_key_to_protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype); int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns, u32 keytype, u32 keysubtype, u32 keybitsize, u32 flags, diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c index ba2ae253b2ba..937051381720 100644 --- a/drivers/s390/crypto/pkey_cca.c +++ b/drivers/s390/crypto/pkey_cca.c @@ -541,17 +541,70 @@ out: return rc; } +/* + * This function provides an alternate but usually slow way + * to convert a 'clear key token' with AES key material into + * a protected key. This is done via an intermediate step + * which creates a CCA AES DATA secure key first and then + * derives the protected key from this secure key. + */ +static int cca_slowpath_key2protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype) +{ + const struct keytoken_header *hdr = (const struct keytoken_header *)key; + const struct clearkeytoken *t = (const struct clearkeytoken *)key; + u32 tmplen, keysize = 0; + u8 *tmpbuf; + int i, rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_CLEAR_KEY) + keysize = pkey_keytype_aes_to_size(t->keytype); + if (!keysize || t->len != keysize) + return -EINVAL; + + /* alloc tmp key buffer */ + tmpbuf = kmalloc(SECKEYBLOBSIZE, GFP_ATOMIC); + if (!tmpbuf) + return -ENOMEM; + + /* try two times in case of failure */ + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + tmplen = SECKEYBLOBSIZE; + rc = cca_clr2key(NULL, 0, t->keytype, PKEY_TYPE_CCA_DATA, + 8 * keysize, 0, t->clearkey, t->len, + tmpbuf, &tmplen, NULL); + pr_debug("cca_clr2key()=%d\n", rc); + if (rc) + continue; + rc = cca_key2protkey(NULL, 0, tmpbuf, tmplen, + protkey, protkeylen, protkeytype); + pr_debug("cca_key2protkey()=%d\n", rc); + } + + kfree(tmpbuf); + pr_debug("rc=%d\n", rc); + return rc; +} + static struct pkey_handler cca_handler = { - .module = THIS_MODULE, - .name = "PKEY CCA handler", - .is_supported_key = is_cca_key, - .is_supported_keytype = is_cca_keytype, - .key_to_protkey = cca_key2protkey, - .gen_key = cca_gen_key, - .clr_to_key = cca_clr2key, - .verify_key = cca_verifykey, - .apqns_for_key = cca_apqns4key, - .apqns_for_keytype = cca_apqns4type, + .module = THIS_MODULE, + .name = "PKEY CCA handler", + .is_supported_key = is_cca_key, + .is_supported_keytype = is_cca_keytype, + .key_to_protkey = cca_key2protkey, + .slowpath_key_to_protkey = cca_slowpath_key2protkey, + .gen_key = cca_gen_key, + .clr_to_key = cca_clr2key, + .verify_key = cca_verifykey, + .apqns_for_key = cca_apqns4key, + .apqns_for_keytype = cca_apqns4type, }; /* diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c index 624e55195d93..f42d397a9cb6 100644 --- a/drivers/s390/crypto/pkey_ep11.c +++ b/drivers/s390/crypto/pkey_ep11.c @@ -490,17 +490,70 @@ out: return rc; } +/* + * This function provides an alternate but usually slow way + * to convert a 'clear key token' with AES key material into + * a protected key. That is done via an intermediate step + * which creates an EP11 AES secure key first and then derives + * the protected key from this secure key. + */ +static int ep11_slowpath_key2protkey(const struct pkey_apqn *apqns, + size_t nr_apqns, + const u8 *key, u32 keylen, + u8 *protkey, u32 *protkeylen, + u32 *protkeytype) +{ + const struct keytoken_header *hdr = (const struct keytoken_header *)key; + const struct clearkeytoken *t = (const struct clearkeytoken *)key; + u32 tmplen, keysize = 0; + u8 *tmpbuf; + int i, rc; + + if (keylen < sizeof(*hdr)) + return -EINVAL; + + if (hdr->type == TOKTYPE_NON_CCA && + hdr->version == TOKVER_CLEAR_KEY) + keysize = pkey_keytype_aes_to_size(t->keytype); + if (!keysize || t->len != keysize) + return -EINVAL; + + /* alloc tmp key buffer */ + tmpbuf = kmalloc(MAXEP11AESKEYBLOBSIZE, GFP_ATOMIC); + if (!tmpbuf) + return -ENOMEM; + + /* try two times in case of failure */ + for (i = 0, rc = -ENODEV; i < 2 && rc; i++) { + tmplen = MAXEP11AESKEYBLOBSIZE; + rc = ep11_clr2key(NULL, 0, t->keytype, PKEY_TYPE_EP11, + 8 * keysize, 0, t->clearkey, t->len, + tmpbuf, &tmplen, NULL); + pr_debug("ep11_clr2key()=%d\n", rc); + if (rc) + continue; + rc = ep11_key2protkey(NULL, 0, tmpbuf, tmplen, + protkey, protkeylen, protkeytype); + pr_debug("ep11_key2protkey()=%d\n", rc); + } + + kfree(tmpbuf); + pr_debug("rc=%d\n", rc); + return rc; +} + static struct pkey_handler ep11_handler = { - .module = THIS_MODULE, - .name = "PKEY EP11 handler", - .is_supported_key = is_ep11_key, - .is_supported_keytype = is_ep11_keytype, - .key_to_protkey = ep11_key2protkey, - .gen_key = ep11_gen_key, - .clr_to_key = ep11_clr2key, - .verify_key = ep11_verifykey, - .apqns_for_key = ep11_apqns4key, - .apqns_for_keytype = ep11_apqns4type, + .module = THIS_MODULE, + .name = "PKEY EP11 handler", + .is_supported_key = is_ep11_key, + .is_supported_keytype = is_ep11_keytype, + .key_to_protkey = ep11_key2protkey, + .slowpath_key_to_protkey = ep11_slowpath_key2protkey, + .gen_key = ep11_gen_key, + .clr_to_key = ep11_clr2key, + .verify_key = ep11_verifykey, + .apqns_for_key = ep11_apqns4key, + .apqns_for_keytype = ep11_apqns4type, }; /* From 177b621bf0685e8733fa6a7c796865f4200a6e2f Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 22 Aug 2024 11:32:21 +0200 Subject: [PATCH 42/84] s390/pkey: Add function to enforce pkey handler modules load There is a use case during early boot with an secure key encrypted root file system where the paes cipher may try to derive a protected key from secure key while the AP bus is still in the process of scanning the bus and building up the zcrypt device drivers. As the detection of CEX cards also triggers the modprobe of the pkey handler modules, these modules may come into existence too late. Yet another use case happening during early boot is for use of an protected key encrypted swap file(system). There is an ephemeral protected key read via sysfs to set up the swap file. But this only works when the pkey_pckmo module is already in - which may happen at a later time as the load is triggered via CPU feature. This patch introduces a new function pkey_handler_request_modules() and invokes it which unconditional tries to load in the pkey handler modules. This function is called for the in-kernel API to derive a protected key from whatever and in the sysfs API when the first attempt to simple invoke the handler function failed. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/pkey_api.c | 13 +++++- drivers/s390/crypto/pkey_base.c | 29 ++++++++++++ drivers/s390/crypto/pkey_base.h | 5 ++ drivers/s390/crypto/pkey_sysfs.c | 80 ++++++++++++++++++++------------ 4 files changed, 95 insertions(+), 32 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index d6c5e5ae915b..c20251e00cf9 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -54,8 +54,17 @@ static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, int pkey_key2protkey(const u8 *key, u32 keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - return key2protkey(NULL, 0, key, keylen, - protkey, protkeylen, protkeytype); + int rc; + + rc = key2protkey(NULL, 0, key, keylen, + protkey, protkeylen, protkeytype); + if (rc == -ENODEV) { + pkey_handler_request_modules(); + rc = key2protkey(NULL, 0, key, keylen, + protkey, protkeylen, protkeytype); + } + + return rc; } EXPORT_SYMBOL(pkey_key2protkey); diff --git a/drivers/s390/crypto/pkey_base.c b/drivers/s390/crypto/pkey_base.c index 976b9ddfbe15..fea243322838 100644 --- a/drivers/s390/crypto/pkey_base.c +++ b/drivers/s390/crypto/pkey_base.c @@ -300,6 +300,35 @@ int pkey_handler_apqns_for_keytype(enum pkey_key_type keysubtype, } EXPORT_SYMBOL(pkey_handler_apqns_for_keytype); +void pkey_handler_request_modules(void) +{ +#ifdef CONFIG_MODULES + static const char * const pkey_handler_modules[] = { + "pkey_cca", "pkey_ep11", "pkey_pckmo" }; + int i; + + for (i = 0; i < ARRAY_SIZE(pkey_handler_modules); i++) { + const struct pkey_handler *h; + bool found = false; + + rcu_read_lock(); + list_for_each_entry_rcu(h, &handler_list, list) { + if (h->module && + !strcmp(h->module->name, pkey_handler_modules[i])) { + found = true; + break; + } + } + rcu_read_unlock(); + if (!found) { + pr_debug("request_module(%s)\n", pkey_handler_modules[i]); + request_module(pkey_handler_modules[i]); + } + } +#endif +} +EXPORT_SYMBOL(pkey_handler_request_modules); + /* * Module init */ diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h index 41fd69e7c66e..fd151903fd06 100644 --- a/drivers/s390/crypto/pkey_base.h +++ b/drivers/s390/crypto/pkey_base.h @@ -176,4 +176,9 @@ int pkey_handler_apqns_for_keytype(enum pkey_key_type ktype, u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags, struct pkey_apqn *apqns, size_t *nr_apqns); +/* + * Unconditional try to load all handler modules + */ +void pkey_handler_request_modules(void); + #endif /* _PKEY_BASE_H_ */ diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c index 242eb6b1a158..56205f53f77b 100644 --- a/drivers/s390/crypto/pkey_sysfs.c +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -16,6 +16,32 @@ #include "pkey_base.h" +/* + * Wrapper around pkey_handler_gen_key() which deals with the + * ENODEV return code and then tries to enforce a pkey handler + * module load. + */ +static int sys_pkey_handler_gen_key(u32 keytype, u32 keysubtype, + u32 keybitsize, u32 flags, + u8 *keybuf, u32 *keybuflen, u32 *keyinfo) +{ + int rc; + + rc = pkey_handler_gen_key(NULL, 0, + keytype, keysubtype, + keybitsize, flags, + keybuf, keybuflen, keyinfo); + if (rc == -ENODEV) { + pkey_handler_request_modules(); + rc = pkey_handler_gen_key(NULL, 0, + keytype, keysubtype, + keybitsize, flags, + keybuf, keybuflen, keyinfo); + } + + return rc; +} + /* * Sysfs attribute read function for all protected key binary attributes. * The implementation can not deal with partial reads, because a new random @@ -41,10 +67,9 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, protkeytoken.keytype = keytype; protkey.len = sizeof(protkey.protkey); - rc = pkey_handler_gen_key(NULL, 0, keytype, - PKEY_TYPE_PROTKEY, 0, 0, - protkey.protkey, &protkey.len, - &protkey.type); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); if (rc) return rc; @@ -56,10 +81,9 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, if (is_xts) { /* xts needs a second protected key, reuse protkey struct */ protkey.len = sizeof(protkey.protkey); - rc = pkey_handler_gen_key(NULL, 0, keytype, - PKEY_TYPE_PROTKEY, 0, 0, - protkey.protkey, &protkey.len, - &protkey.type); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + protkey.protkey, &protkey.len, + &protkey.type); if (rc) return rc; @@ -165,18 +189,16 @@ static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf, return -EINVAL; buflen = sizeof(seckey->seckey); - rc = pkey_handler_gen_key(NULL, 0, keytype, - PKEY_TYPE_CCA_DATA, 0, 0, - seckey->seckey, &buflen, NULL); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); if (rc) return rc; if (is_xts) { seckey++; buflen = sizeof(seckey->seckey); - rc = pkey_handler_gen_key(NULL, 0, keytype, - PKEY_TYPE_CCA_DATA, 0, 0, - seckey->seckey, &buflen, NULL); + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_CCA_DATA, 0, 0, + seckey->seckey, &buflen, NULL); if (rc) return rc; @@ -279,20 +301,19 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, memset(buf, 0, is_xts ? 2 * keysize : keysize); - rc = pkey_handler_gen_key(NULL, 0, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_CCA_CIPHER, keybits, 0, - buf, &keysize, NULL); + rc = sys_pkey_handler_gen_key(pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); if (rc) return rc; if (is_xts) { keysize = CCACIPHERTOKENSIZE; buf += CCACIPHERTOKENSIZE; - rc = pkey_handler_gen_key(NULL, 0, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_CCA_CIPHER, keybits, 0, - buf, &keysize, NULL); + rc = sys_pkey_handler_gen_key( + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_CCA_CIPHER, keybits, 0, + buf, &keysize, NULL); if (rc) return rc; return 2 * CCACIPHERTOKENSIZE; @@ -395,20 +416,19 @@ static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, memset(buf, 0, is_xts ? 2 * keysize : keysize); - rc = pkey_handler_gen_key(NULL, 0, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_EP11_AES, keybits, 0, - buf, &keysize, NULL); + rc = sys_pkey_handler_gen_key(pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); if (rc) return rc; if (is_xts) { keysize = MAXEP11AESKEYBLOBSIZE; buf += MAXEP11AESKEYBLOBSIZE; - rc = pkey_handler_gen_key(NULL, 0, - pkey_aes_bitsize_to_keytype(keybits), - PKEY_TYPE_EP11_AES, keybits, 0, - buf, &keysize, NULL); + rc = sys_pkey_handler_gen_key( + pkey_aes_bitsize_to_keytype(keybits), + PKEY_TYPE_EP11_AES, keybits, 0, + buf, &keysize, NULL); if (rc) return rc; return 2 * MAXEP11AESKEYBLOBSIZE; From 88c02b3f79a61e659749773865998e0c33247e86 Mon Sep 17 00:00:00 2001 From: Joerg Schmidbauer Date: Wed, 28 Aug 2024 13:52:30 +0200 Subject: [PATCH 43/84] s390/sha3: Support sha3 performance enhancements On newer machines the SHA3 performance of CPACF instructions KIMD and KLMD can be enhanced by using additional modifier bits. This allows the application to omit initializing the ICV, but also affects the internal processing of the instructions. Performance is mostly gained when processing short messages. The new CPACF feature is backwards compatible with older machines, i.e. the new modifier bits are ignored on older machines. However, to save the ICV initialization, the application must detect the MSA level and omit the ICV initialization only if this feature is supported. Reviewed-by: Holger Dengler Signed-off-by: Joerg Schmidbauer Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/sha.h | 1 + arch/s390/crypto/sha3_256_s390.c | 8 ++++++-- arch/s390/crypto/sha3_512_s390.c | 8 ++++++-- arch/s390/crypto/sha_common.c | 19 +++++++++++++++---- arch/s390/include/asm/cpacf.h | 12 ++++++++++-- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index 65ea12fc87a1..2bb22db54c31 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -25,6 +25,7 @@ struct s390_sha_ctx { u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)]; u8 buf[SHA_MAX_BLOCK_SIZE]; int func; /* KIMD function to use */ + int first_message_part; }; struct shash_desc; diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index e1350e033a32..5bba972a2646 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -21,9 +21,11 @@ static int sha3_256_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - memset(sctx->state, 0, sizeof(sctx->state)); + if (!test_facility(86)) /* msa 12 */ + memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_256; + sctx->first_message_part = 1; return 0; } @@ -88,9 +90,11 @@ static int sha3_224_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - memset(sctx->state, 0, sizeof(sctx->state)); + if (!test_facility(86)) /* msa 12 */ + memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_224; + sctx->first_message_part = 1; return 0; } diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 06c142ed9bb1..3554cbbafe9c 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -20,9 +20,11 @@ static int sha3_512_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - memset(sctx->state, 0, sizeof(sctx->state)); + if (!test_facility(86)) /* msa 12 */ + memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_512; + sctx->first_message_part = 1; return 0; } @@ -97,9 +99,11 @@ static int sha3_384_init(struct shash_desc *desc) { struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - memset(sctx->state, 0, sizeof(sctx->state)); + if (!test_facility(86)) /* msa 12 */ + memset(sctx->state, 0, sizeof(sctx->state)); sctx->count = 0; sctx->func = CPACF_KIMD_SHA3_384; + sctx->first_message_part = 1; return 0; } diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 686fe7aa192f..f6c7fda21abc 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -18,6 +18,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) struct s390_sha_ctx *ctx = shash_desc_ctx(desc); unsigned int bsize = crypto_shash_blocksize(desc->tfm); unsigned int index, n; + int fc; /* how much is already in the buffer? */ index = ctx->count % bsize; @@ -26,10 +27,15 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) if ((index + len) < bsize) goto store; + fc = ctx->func; + if (ctx->first_message_part) + fc |= test_facility(86) ? CPACF_KIMD_NIP : 0; + /* process one stored block */ if (index) { memcpy(ctx->buf + index, data, bsize - index); - cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize); + cpacf_kimd(fc, ctx->state, ctx->buf, bsize); + ctx->first_message_part = 0; data += bsize - index; len -= bsize - index; index = 0; @@ -38,7 +44,8 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) /* process as many blocks as possible */ if (len >= bsize) { n = (len / bsize) * bsize; - cpacf_kimd(ctx->func, ctx->state, data, n); + cpacf_kimd(fc, ctx->state, data, n); + ctx->first_message_part = 0; data += n; len -= n; } @@ -75,7 +82,7 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) unsigned int bsize = crypto_shash_blocksize(desc->tfm); u64 bits; unsigned int n; - int mbl_offset; + int mbl_offset, fc; n = ctx->count % bsize; bits = ctx->count * 8; @@ -109,7 +116,11 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) return -EINVAL; } - cpacf_klmd(ctx->func, ctx->state, ctx->buf, n); + fc = ctx->func; + fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0; + if (ctx->first_message_part) + fc |= CPACF_KLMD_NIP; + cpacf_klmd(fc, ctx->state, ctx->buf, n); /* copy digest to out */ memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm)); diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index 2dbe2e2088bb..4b6a8c26e1ce 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -171,6 +171,14 @@ #define CPACF_KMA_LAAD 0x200 /* Last-AAD */ #define CPACF_KMA_HS 0x400 /* Hash-subkey Supplied */ +/* + * Flags for the KIMD/KLMD (COMPUTE INTERMEDIATE/LAST MESSAGE DIGEST) + * instructions + */ +#define CPACF_KIMD_NIP 0x8000 +#define CPACF_KLMD_DUFOP 0x4000 +#define CPACF_KLMD_NIP 0x8000 + typedef struct { unsigned char bytes[16]; } cpacf_mask_t; /* @@ -397,7 +405,7 @@ static inline void cpacf_kimd(unsigned long func, void *param, asm volatile( " lgr 0,%[fc]\n" " lgr 1,%[pba]\n" - "0: .insn rre,%[opc] << 16,0,%[src]\n" + "0: .insn rrf,%[opc] << 16,0,%[src],8,0\n" " brc 1,0b\n" /* handle partial completion */ : [src] "+&d" (s.pair) : [fc] "d" (func), [pba] "d" ((unsigned long)(param)), @@ -422,7 +430,7 @@ static inline void cpacf_klmd(unsigned long func, void *param, asm volatile( " lgr 0,%[fc]\n" " lgr 1,%[pba]\n" - "0: .insn rre,%[opc] << 16,0,%[src]\n" + "0: .insn rrf,%[opc] << 16,0,%[src],8,0\n" " brc 1,0b\n" /* handle partial completion */ : [src] "+&d" (s.pair) : [fc] "d" (func), [pba] "d" ((unsigned long)param), From 56199bb956c3ea82e39c72d2972ebf8c18c6a8c0 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Wed, 28 Aug 2024 14:25:08 +0200 Subject: [PATCH 44/84] s390/ap: Fix deadlock caused by recursive lock of the AP bus scan mutex There is a possibility to deadlock with an recursive lock of the AP bus scan mutex ap_scan_bus_mutex: ... kernel: ============================================ ... kernel: WARNING: possible recursive locking detected ... kernel: 5.14.0-496.el9.s390x #3 Not tainted ... kernel: -------------------------------------------- ... kernel: kworker/12:1/130 is trying to acquire lock: ... kernel: 0000000358bc1510 (ap_scan_bus_mutex){+.+.}-{3:3}, at: ap_bus_force_rescan+0x92/0x108 ... kernel: but task is already holding lock: ... kernel: 0000000358bc1510 (ap_scan_bus_mutex){+.+.}-{3:3}, at: ap_scan_bus_wq_callback+0x28/0x60 ... kernel: other info that might help us debug this: ... kernel: Possible unsafe locking scenario: ... kernel: CPU0 ... kernel: ---- ... kernel: lock(ap_scan_bus_mutex); ... kernel: lock(ap_scan_bus_mutex); ... kernel: *** DEADLOCK *** Here is how the callstack looks like: ... [<00000003576fe9ce>] process_one_work+0x2a6/0x748 ... [<0000000358150c00>] ap_scan_bus_wq_callback+0x40/0x60 <- mutex locked ... [<00000003581506e2>] ap_scan_bus+0x5a/0x3b0 ... [<000000035815037c>] ap_scan_adapter+0x5b4/0x8c0 ... [<000000035814fa34>] ap_scan_domains+0x2d4/0x668 ... [<0000000357d989b4>] device_add+0x4a4/0x6b8 ... [<0000000357d9bb54>] bus_probe_device+0xb4/0xc8 ... [<0000000357d9daa8>] __device_attach+0x120/0x1b0 ... [<0000000357d9a632>] bus_for_each_drv+0x8a/0xd0 ... [<0000000357d9d548>] __device_attach_driver+0xc0/0x140 ... [<0000000357d9d3d8>] driver_probe_device+0x40/0xf0 ... [<0000000357d9cec2>] really_probe+0xd2/0x460 ... [<000000035814d7b0>] ap_device_probe+0x150/0x208 ... [<000003ff802a5c46>] zcrypt_cex4_queue_probe+0xb6/0x1c0 [zcrypt_cex4] ... [<000003ff7fb2d36e>] zcrypt_queue_register+0xe6/0x1b0 [zcrypt] ... [<000003ff7fb2c8ac>] zcrypt_rng_device_add+0x94/0xd8 [zcrypt] ... [<0000000357d7bc52>] hwrng_register+0x212/0x228 ... [<0000000357d7b8c2>] add_early_randomness+0x102/0x110 ... [<000003ff7fb29c94>] zcrypt_rng_data_read+0x94/0xb8 [zcrypt] ... [<0000000358150aca>] ap_bus_force_rescan+0x92/0x108 ... [<0000000358177572>] mutex_lock_interruptible_nested+0x32/0x40 <- lock again Note this only happens when the very first random data providing crypto card appears via hot plug in the system AND is in disabled state ("deconfig"). Then the initial pull of random data fails and a re-scan of the AP bus is triggered while already in the middle of an AP bus scan caused by the appearing new hardware. The fix is relatively simple once the scenario us understood: The AP bus force rescan function will immediately return if there is currently an AP bus scan running with the very same thread id. Fixes: eacf5b3651c5 ("s390/ap: introduce mutex to lock the AP bus scan") Signed-off-by: Harald Freudenberger Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index ebc3fe2a7f40..f3253cb27a76 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -107,6 +107,7 @@ debug_info_t *ap_dbf_info; static bool ap_scan_bus(void); static bool ap_scan_bus_result; /* result of last ap_scan_bus() */ static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */ +static struct task_struct *ap_scan_bus_task; /* thread holding the scan mutex */ static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */ static int ap_scan_bus_time = AP_CONFIG_TIME; static struct timer_list ap_scan_bus_timer; @@ -1000,11 +1001,25 @@ bool ap_bus_force_rescan(void) if (scan_counter <= 0) goto out; + /* + * There is one unlikely but nevertheless valid scenario where the + * thread holding the mutex may try to send some crypto load but + * all cards are offline so a rescan is triggered which causes + * a recursive call of ap_bus_force_rescan(). A simple return if + * the mutex is already locked by this thread solves this. + */ + if (mutex_is_locked(&ap_scan_bus_mutex)) { + if (ap_scan_bus_task == current) + goto out; + } + /* Try to acquire the AP scan bus mutex */ if (mutex_trylock(&ap_scan_bus_mutex)) { /* mutex acquired, run the AP bus scan */ + ap_scan_bus_task = current; ap_scan_bus_result = ap_scan_bus(); rc = ap_scan_bus_result; + ap_scan_bus_task = NULL; mutex_unlock(&ap_scan_bus_mutex); goto out; } @@ -2277,7 +2292,9 @@ static void ap_scan_bus_wq_callback(struct work_struct *unused) * system_long_wq which invokes this function here again. */ if (mutex_trylock(&ap_scan_bus_mutex)) { + ap_scan_bus_task = current; ap_scan_bus_result = ap_scan_bus(); + ap_scan_bus_task = NULL; mutex_unlock(&ap_scan_bus_mutex); } } From ee3daf7c05e70a0b88723ceb18587aed75716729 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Wed, 28 Aug 2024 13:50:04 +0200 Subject: [PATCH 45/84] s390/entry: Unify save_area_sync and save_area_async In the past two save areas existed because interrupt handlers and system call / program check handlers where entered with interrupts enabled. To prevent a handler from overwriting the save areas from the previous handler, interrupts used the async save area, while system call and program check handler used the sync save area. Since the removal of critical section cleanup from entry.S, handlers are entered with interrupts disabled. When the interrupts are re-enabled, the save area is no longer need. Therefore merge both save areas into one. Reviewed-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/boot/head.S | 4 ++-- arch/s390/include/asm/lowcore.h | 4 ++-- arch/s390/kernel/asm-offsets.c | 3 +-- arch/s390/kernel/entry.S | 22 +++++++++++----------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/s390/boot/head.S b/arch/s390/boot/head.S index 637c29c3f6e3..0a47b16f6412 100644 --- a/arch/s390/boot/head.S +++ b/arch/s390/boot/head.S @@ -299,11 +299,11 @@ SYM_CODE_END(startup_normal) # the save area and does disabled wait with a faulty address. # SYM_CODE_START_LOCAL(startup_pgm_check_handler) - stmg %r8,%r15,__LC_SAVE_AREA_SYNC + stmg %r8,%r15,__LC_SAVE_AREA la %r8,4095 stctg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r8) stmg %r0,%r7,__LC_GPREGS_SAVE_AREA-4095(%r8) - mvc __LC_GPREGS_SAVE_AREA-4095+64(64,%r8),__LC_SAVE_AREA_SYNC + mvc __LC_GPREGS_SAVE_AREA-4095+64(64,%r8),__LC_SAVE_AREA mvc __LC_PSW_SAVE_AREA-4095(16,%r8),__LC_PGM_OLD_PSW mvc __LC_RETURN_PSW(16),__LC_PGM_OLD_PSW ni __LC_RETURN_PSW,0xfc # remove IO and EX bits diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 183ac29afaf8..48c64716d1f2 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -98,8 +98,8 @@ struct lowcore { psw_t io_new_psw; /* 0x01f0 */ /* Save areas. */ - __u64 save_area_sync[8]; /* 0x0200 */ - __u64 save_area_async[8]; /* 0x0240 */ + __u64 save_area[8]; /* 0x0200 */ + __u8 pad_0x0240[0x0280-0x0240]; /* 0x0240 */ __u64 save_area_restart[1]; /* 0x0280 */ __u64 pcpu; /* 0x0288 */ diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index ffa0dd2dbaac..5529248d84fb 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -112,8 +112,7 @@ int main(void) OFFSET(__LC_MCK_NEW_PSW, lowcore, mcck_new_psw); OFFSET(__LC_IO_NEW_PSW, lowcore, io_new_psw); /* software defined lowcore locations 0x200 - 0xdff*/ - OFFSET(__LC_SAVE_AREA_SYNC, lowcore, save_area_sync); - OFFSET(__LC_SAVE_AREA_ASYNC, lowcore, save_area_async); + OFFSET(__LC_SAVE_AREA, lowcore, save_area); OFFSET(__LC_SAVE_AREA_RESTART, lowcore, save_area_restart); OFFSET(__LC_PCPU, lowcore, pcpu); OFFSET(__LC_RETURN_PSW, lowcore, return_psw); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 36424e53d63b..42ac4145dfe0 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -264,7 +264,7 @@ EXPORT_SYMBOL(sie_exit) */ SYM_CODE_START(system_call) - STMG_LC %r8,%r15,__LC_SAVE_AREA_SYNC + STMG_LC %r8,%r15,__LC_SAVE_AREA GET_LC %r13 stpt __LC_SYS_ENTER_TIMER(%r13) BPOFF @@ -287,7 +287,7 @@ SYM_CODE_START(system_call) xgr %r10,%r10 xgr %r11,%r11 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs - mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC(%r13) + mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) MBEAR %r2,%r13 lgr %r3,%r14 brasl %r14,__do_syscall @@ -323,7 +323,7 @@ SYM_CODE_END(ret_from_fork) */ SYM_CODE_START(pgm_check_handler) - STMG_LC %r8,%r15,__LC_SAVE_AREA_SYNC + STMG_LC %r8,%r15,__LC_SAVE_AREA GET_LC %r13 stpt __LC_SYS_ENTER_TIMER(%r13) BPOFF @@ -338,16 +338,16 @@ SYM_CODE_START(pgm_check_handler) jnz 2f # -> enabled, can't be a double fault tm __LC_PGM_ILC+3(%r13),0x80 # check for per exception jnz .Lpgm_svcper # -> single stepped svc -2: CHECK_STACK __LC_SAVE_AREA_SYNC,%r13 +2: CHECK_STACK __LC_SAVE_AREA,%r13 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) # CHECK_VMAP_STACK branches to stack_overflow or 4f - CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,%r13,4f + CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4f 3: lg %r15,__LC_KERNEL_STACK(%r13) 4: la %r11,STACK_FRAME_OVERHEAD(%r15) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC(%r13) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r13) stctg %c1,%c1,__PT_CR1(%r11) #if IS_ENABLED(CONFIG_KVM) @@ -398,7 +398,7 @@ SYM_CODE_END(pgm_check_handler) */ .macro INT_HANDLER name,lc_old_psw,handler SYM_CODE_START(\name) - STMG_LC %r8,%r15,__LC_SAVE_AREA_ASYNC + STMG_LC %r8,%r15,__LC_SAVE_AREA GET_LC %r13 stckf __LC_INT_CLOCK(%r13) stpt __LC_SYS_ENTER_TIMER(%r13) @@ -414,7 +414,7 @@ SYM_CODE_START(\name) BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST SIEEXIT __SF_SIE_CONTROL(%r15),%r13 #endif -0: CHECK_STACK __LC_SAVE_AREA_ASYNC,%r13 +0: CHECK_STACK __LC_SAVE_AREA,%r13 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 2f 1: lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13) @@ -432,7 +432,7 @@ SYM_CODE_START(\name) xgr %r7,%r7 xgr %r10,%r10 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC(%r13) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) MBEAR %r11,%r13 stmg %r8,%r9,__PT_PSW(%r11) lgr %r2,%r11 # pass pointer to pt_regs @@ -601,14 +601,14 @@ SYM_CODE_END(restart_int_handler) __INIT SYM_CODE_START(early_pgm_check_handler) - STMG_LC %r8,%r15,__LC_SAVE_AREA_SYNC + STMG_LC %r8,%r15,__LC_SAVE_AREA GET_LC %r13 aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) la %r11,STACK_FRAME_OVERHEAD(%r15) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stmg %r0,%r7,__PT_R0(%r11) mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW(%r13) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC(%r13) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) lgr %r2,%r11 brasl %r14,__do_early_pgm_check mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) From bb91ed0ee3a76406846b99d6aca8121100a14ff5 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Aug 2024 19:06:49 +0200 Subject: [PATCH 46/84] s390/setup: Recognize sequential instruction fetching facility When sequential instruction fetching facility is present, certain guarantees are provided for code patching. In particular, atomic overwrites within 8 aligned bytes is safe from an instruction-fetching point of view. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/setup.h | 2 ++ arch/s390/kernel/early.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 50b943f30155..70b920b32827 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -34,6 +34,7 @@ #define MACHINE_FLAG_SCC BIT(17) #define MACHINE_FLAG_PCI_MIO BIT(18) #define MACHINE_FLAG_RDP BIT(19) +#define MACHINE_FLAG_SEQ_INSN BIT(20) #define LPP_MAGIC BIT(31) #define LPP_PID_MASK _AC(0xffffffff, UL) @@ -95,6 +96,7 @@ extern unsigned long mio_wb_bit_mask; #define MACHINE_HAS_SCC (get_lowcore()->machine_flags & MACHINE_FLAG_SCC) #define MACHINE_HAS_PCI_MIO (get_lowcore()->machine_flags & MACHINE_FLAG_PCI_MIO) #define MACHINE_HAS_RDP (get_lowcore()->machine_flags & MACHINE_FLAG_RDP) +#define MACHINE_HAS_SEQ_INSN (get_lowcore()->machine_flags & MACHINE_FLAG_SEQ_INSN) /* * Console mode. Override with conmode= diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index cc4264d23019..62f8f5a750a3 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -268,6 +268,8 @@ static __init void detect_machine_facilities(void) } if (test_facility(194)) get_lowcore()->machine_flags |= MACHINE_FLAG_RDP; + if (test_facility(85)) + get_lowcore()->machine_flags |= MACHINE_FLAG_SEQ_INSN; } static inline void save_vector_registers(void) From 30799152c375c4e1f40c7f2b98d766748d3b3333 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Aug 2024 19:06:52 +0200 Subject: [PATCH 47/84] s390/kprobes: Avoid stop machine if possible Avoid stop machine on kprobes arm/disarm when sequential instruction fetching is present. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/kprobes.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 05c83505e979..6295faf0987d 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -152,7 +153,12 @@ void arch_arm_kprobe(struct kprobe *p) { struct swap_insn_args args = {.p = p, .arm_kprobe = 1}; - stop_machine_cpuslocked(swap_instruction, &args, NULL); + if (MACHINE_HAS_SEQ_INSN) { + swap_instruction(&args); + text_poke_sync(); + } else { + stop_machine_cpuslocked(swap_instruction, &args, NULL); + } } NOKPROBE_SYMBOL(arch_arm_kprobe); @@ -160,7 +166,12 @@ void arch_disarm_kprobe(struct kprobe *p) { struct swap_insn_args args = {.p = p, .arm_kprobe = 0}; - stop_machine_cpuslocked(swap_instruction, &args, NULL); + if (MACHINE_HAS_SEQ_INSN) { + swap_instruction(&args); + text_poke_sync(); + } else { + stop_machine_cpuslocked(swap_instruction, &args, NULL); + } } NOKPROBE_SYMBOL(arch_disarm_kprobe); From efd9cd019e95d399e27be8dfbfc1df91517c0813 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Aug 2024 19:06:55 +0200 Subject: [PATCH 48/84] s390/ftrace: Avoid trampolines if possible When a sequential instruction fetching facility is present, it is safe to patch ftrace NOPs in function prologues. All of them are 8-byte aligned. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ftrace.c | 59 +++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 9552b1e7121f..03eadf7d098e 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -69,7 +69,7 @@ static const char *ftrace_shared_hotpatch_trampoline(const char **end) bool ftrace_need_init_nop(void) { - return true; + return !MACHINE_HAS_SEQ_INSN; } int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) @@ -139,8 +139,35 @@ static struct ftrace_hotpatch_trampoline *ftrace_get_trampoline(struct dyn_ftrac return trampoline; } -int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, - unsigned long addr) +static inline struct ftrace_insn +ftrace_generate_branch_insn(unsigned long ip, unsigned long target) +{ + /* brasl r0,target or brcl 0,0 */ + return (struct ftrace_insn){ .opc = target ? 0xc005 : 0xc004, + .disp = target ? (target - ip) / 2 : 0 }; +} + +static int ftrace_patch_branch_insn(unsigned long ip, unsigned long old_target, + unsigned long target) +{ + struct ftrace_insn orig = ftrace_generate_branch_insn(ip, old_target); + struct ftrace_insn new = ftrace_generate_branch_insn(ip, target); + struct ftrace_insn old; + + if (!IS_ALIGNED(ip, 8)) + return -EINVAL; + if (copy_from_kernel_nofault(&old, (void *)ip, sizeof(old))) + return -EFAULT; + /* Verify that the to be replaced code matches what we expect. */ + if (memcmp(&orig, &old, sizeof(old))) + return -EINVAL; + s390_kernel_write((void *)ip, &new, sizeof(new)); + return 0; +} + +static int ftrace_modify_trampoline_call(struct dyn_ftrace *rec, + unsigned long old_addr, + unsigned long addr) { struct ftrace_hotpatch_trampoline *trampoline; u64 old; @@ -156,6 +183,15 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, return 0; } +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, + unsigned long addr) +{ + if (MACHINE_HAS_SEQ_INSN) + return ftrace_patch_branch_insn(rec->ip, old_addr, addr); + else + return ftrace_modify_trampoline_call(rec, old_addr, addr); +} + static int ftrace_patch_branch_mask(void *addr, u16 expected, bool enable) { u16 old; @@ -174,11 +210,14 @@ static int ftrace_patch_branch_mask(void *addr, u16 expected, bool enable) int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - /* Expect brcl 0xf,... */ - return ftrace_patch_branch_mask((void *)rec->ip, 0xc0f4, false); + /* Expect brcl 0xf,... for the !MACHINE_HAS_SEQ_INSN case */ + if (MACHINE_HAS_SEQ_INSN) + return ftrace_patch_branch_insn(rec->ip, addr, 0); + else + return ftrace_patch_branch_mask((void *)rec->ip, 0xc0f4, false); } -int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +static int ftrace_make_trampoline_call(struct dyn_ftrace *rec, unsigned long addr) { struct ftrace_hotpatch_trampoline *trampoline; @@ -190,6 +229,14 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) return ftrace_patch_branch_mask((void *)rec->ip, 0xc004, true); } +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + if (MACHINE_HAS_SEQ_INSN) + return ftrace_patch_branch_insn(rec->ip, 0, addr); + else + return ftrace_make_trampoline_call(rec, addr); +} + int ftrace_update_ftrace_func(ftrace_func_t func) { ftrace_func = func; From 5200614080cd7a0fca7424b711cab503dfe6bdca Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Aug 2024 19:06:58 +0200 Subject: [PATCH 49/84] s390/ftrace: Use get/copy_from_kernel_nofault consistently Use get/copy_from_kernel_nofault to access the kernel text consistently. Replace memcmp() in ftrace_init_nop() to ensure that in case of inconsistencies in the 'mcount' table, the kernel reports a failure instead of potentially crashing. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ftrace.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 03eadf7d098e..5ae1bf6167bf 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -76,12 +76,13 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) { static struct ftrace_hotpatch_trampoline *next_vmlinux_trampoline = __ftrace_hotpatch_trampolines_start; - static const char orig[6] = { 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00 }; + static const struct ftrace_insn orig = { .opc = 0xc004, .disp = 0 }; static struct ftrace_hotpatch_trampoline *trampoline; struct ftrace_hotpatch_trampoline **next_trampoline; struct ftrace_hotpatch_trampoline *trampolines_end; struct ftrace_hotpatch_trampoline tmp; struct ftrace_insn *insn; + struct ftrace_insn old; const char *shared; s32 disp; @@ -102,8 +103,10 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) return -ENOMEM; trampoline = (*next_trampoline)++; + if (copy_from_kernel_nofault(&old, (void *)rec->ip, sizeof(old))) + return -EFAULT; /* Check for the compiler-generated fentry nop (brcl 0, .). */ - if (WARN_ON_ONCE(memcmp((const void *)rec->ip, &orig, sizeof(orig)))) + if (WARN_ON_ONCE(memcmp(&orig, &old, sizeof(old)))) return -EINVAL; /* Generate the trampoline. */ From 36dff49b9634f6054efbfda83682b2081a64b35a Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Aug 2024 19:07:00 +0200 Subject: [PATCH 50/84] s390/ftrace: Avoid extra serialization for graph caller patching The only context where ftrace_enable_ftrace_graph_caller() or ftrace_disable_ftrace_graph_caller() is called also calls ftrace_arch_code_modify_post_process(), which already performs text_poke_sync_lock(). ftrace_run_update_code() arch_ftrace_update_code() ftrace_modify_all_code() ftrace_enable_ftrace_graph_caller()/ftrace_disable_ftrace_graph_caller() ftrace_arch_code_modify_post_process() text_poke_sync_lock() Remove the redundant serialization. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ftrace.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 5ae1bf6167bf..0b6e62d1d8b8 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -290,26 +290,14 @@ NOKPROBE_SYMBOL(prepare_ftrace_return); */ int ftrace_enable_ftrace_graph_caller(void) { - int rc; - /* Expect brc 0xf,... */ - rc = ftrace_patch_branch_mask(ftrace_graph_caller, 0xa7f4, false); - if (rc) - return rc; - text_poke_sync_lock(); - return 0; + return ftrace_patch_branch_mask(ftrace_graph_caller, 0xa7f4, false); } int ftrace_disable_ftrace_graph_caller(void) { - int rc; - /* Expect brc 0x0,... */ - rc = ftrace_patch_branch_mask(ftrace_graph_caller, 0xa704, true); - if (rc) - return rc; - text_poke_sync_lock(); - return 0; + return ftrace_patch_branch_mask(ftrace_graph_caller, 0xa704, true); } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ From 324db0faf8e1cbfde7a033e77f674f2bb819b54e Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sat, 24 Aug 2024 20:07:49 +0800 Subject: [PATCH 51/84] s390/hypfs: Remove obsoleted declaration for hypfs_dbfs_exit The hypfs_dbfs_exit() have been removed since commit 3325b4d85799 ("s390/hypfs: factor out filesystem code"), and now it is useless, so remove it. Signed-off-by: Gaosheng Cui Acked-by: Vasily Gorbik Signed-off-by: Vasily Gorbik --- arch/s390/hypfs/hypfs.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/s390/hypfs/hypfs.h b/arch/s390/hypfs/hypfs.h index 65f4036fd541..83ebf54cca6b 100644 --- a/arch/s390/hypfs/hypfs.h +++ b/arch/s390/hypfs/hypfs.h @@ -78,7 +78,6 @@ struct hypfs_dbfs_file { struct dentry *dentry; }; -extern void hypfs_dbfs_exit(void); extern void hypfs_dbfs_create_file(struct hypfs_dbfs_file *df); extern void hypfs_dbfs_remove_file(struct hypfs_dbfs_file *df); From 2c6c9ccc76434d5609300aa578a4737e1686b320 Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Mon, 12 Aug 2024 13:39:27 +0200 Subject: [PATCH 52/84] s390/wti: Introduce infrastructure for warning track interrupt The warning-track interrupt (wti) provides a notification that the receiving CPU will be pre-empted from its physical CPU within a short time frame. This time frame is called grace period and depends on the machine type. Giving up the CPU on time may prevent a task to get stuck while holding a resource. Reviewed-by: Heiko Carstens Reviewed-by: Mete Durlu Signed-off-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 5 +++-- arch/s390/include/asm/diag.h | 9 +++++++++ arch/s390/include/asm/irq.h | 2 ++ arch/s390/include/asm/sclp.h | 1 + arch/s390/kernel/diag.c | 17 +++++++++++++++++ arch/s390/kernel/irq.c | 1 + drivers/s390/char/sclp_early.c | 1 + 7 files changed, 34 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 72a9556d04f3..e6527f51ad0b 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -202,8 +202,9 @@ union ctlreg0 { unsigned long : 3; unsigned long ccc : 1; /* Cryptography counter control */ unsigned long pec : 1; /* PAI extension control */ - unsigned long : 17; - unsigned long : 3; + unsigned long : 15; + unsigned long wti : 1; /* Warning-track */ + unsigned long : 4; unsigned long lap : 1; /* Low-address-protection control */ unsigned long : 4; unsigned long edat : 1; /* Enhanced-DAT-enablement control */ diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index c0d43512f4fc..e1316e181230 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -38,6 +38,7 @@ enum diag_stat_enum { DIAG_STAT_X308, DIAG_STAT_X318, DIAG_STAT_X320, + DIAG_STAT_X49C, DIAG_STAT_X500, NR_DIAG_STAT }; @@ -363,4 +364,12 @@ void _diag0c_amode31(unsigned long rx); void _diag308_reset_amode31(void); int _diag8c_amode31(struct diag8c *addr, struct ccw_dev_id *devno, size_t len); +/* diag 49c subcodes */ +enum diag49c_sc { + DIAG49C_SUBC_ACK = 0, + DIAG49C_SUBC_REG = 1 +}; + +int diag49c(unsigned long subcode); + #endif /* _ASM_S390_DIAG_H */ diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 54b42817f70a..d9e705f4a697 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -47,6 +47,7 @@ enum interruption_class { IRQEXT_CMS, IRQEXT_CMC, IRQEXT_FTP, + IRQEXT_WTI, IRQIO_CIO, IRQIO_DAS, IRQIO_C15, @@ -99,6 +100,7 @@ int unregister_external_irq(u16 code, ext_int_handler_t handler); enum irq_subclass { IRQ_SUBCLASS_MEASUREMENT_ALERT = 5, IRQ_SUBCLASS_SERVICE_SIGNAL = 9, + IRQ_SUBCLASS_WARNING_TRACK = 33, }; #define CR0_IRQ_SUBCLASS_MASK \ diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index da3dad18fe50..eb00fa1771da 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -72,6 +72,7 @@ struct sclp_info { unsigned char has_core_type : 1; unsigned char has_sprp : 1; unsigned char has_hvs : 1; + unsigned char has_wti : 1; unsigned char has_esca : 1; unsigned char has_sief2 : 1; unsigned char has_64bscao : 1; diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index ac7b8c8e3133..007e1795670e 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -52,6 +52,7 @@ static const struct diag_desc diag_map[NR_DIAG_STAT] = { [DIAG_STAT_X308] = { .code = 0x308, .name = "List-Directed IPL" }, [DIAG_STAT_X318] = { .code = 0x318, .name = "CP Name and Version Codes" }, [DIAG_STAT_X320] = { .code = 0x320, .name = "Certificate Store" }, + [DIAG_STAT_X49C] = { .code = 0x49c, .name = "Warning-Track Interruption" }, [DIAG_STAT_X500] = { .code = 0x500, .name = "Virtio Service" }, }; @@ -303,3 +304,19 @@ int diag26c(void *req, void *resp, enum diag26c_sc subcode) return diag_amode31_ops.diag26c(virt_to_phys(req), virt_to_phys(resp), subcode); } EXPORT_SYMBOL(diag26c); + +int diag49c(unsigned long subcode) +{ + int rc; + + diag_stat_inc(DIAG_STAT_X49C); + asm volatile( + " diag %[subcode],0,0x49c\n" + " ipm %[rc]\n" + " srl %[rc],28\n" + : [rc] "=d" (rc) + : [subcode] "d" (subcode) + : "cc"); + return rc; +} +EXPORT_SYMBOL(diag49c); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 1af5a08d72ab..2639a3d12736 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -76,6 +76,7 @@ static const struct irq_class irqclass_sub_desc[] = { {.irq = IRQEXT_CMS, .name = "CMS", .desc = "[EXT] CPU-Measurement: Sampling"}, {.irq = IRQEXT_CMC, .name = "CMC", .desc = "[EXT] CPU-Measurement: Counter"}, {.irq = IRQEXT_FTP, .name = "FTP", .desc = "[EXT] HMC FTP Service"}, + {.irq = IRQEXT_WTI, .name = "WTI", .desc = "[EXT] Warning Track"}, {.irq = IRQIO_CIO, .name = "CIO", .desc = "[I/O] Common I/O Layer Interrupt"}, {.irq = IRQIO_DAS, .name = "DAS", .desc = "[I/O] DASD"}, {.irq = IRQIO_C15, .name = "C15", .desc = "[I/O] 3215"}, diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 07df04af82f2..29156455970e 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -44,6 +44,7 @@ static void __init sclp_early_facilities_detect(void) sclp.has_ibs = !!(sccb->fac117 & 0x20); sclp.has_gisaf = !!(sccb->fac118 & 0x08); sclp.has_hvs = !!(sccb->fac119 & 0x80); + sclp.has_wti = !!(sccb->fac119 & 0x40); sclp.has_kss = !!(sccb->fac98 & 0x01); sclp.has_aisii = !!(sccb->fac118 & 0x40); sclp.has_aeni = !!(sccb->fac118 & 0x20); From cafeff5a030983bbf37b11ab766b68d7da3b8aab Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Mon, 12 Aug 2024 13:39:28 +0200 Subject: [PATCH 53/84] s390/wti: Prepare graceful CPU pre-emption on wti reception When a warning track interrupt is received, the kernel has only a very limited amount of time to make sure, that the CPU can be yielded as gracefully as possible before being pre-empted by the hypervisor. The interrupt handler for the wti therefore unparks a kernel thread which has being created on boot re-using the CPU hotplug kernel thread infrastructure. These threads exist per CPU and are assigned the highest possible real-time priority. This makes sure, that said threads will execute as soon as possible as the scheduler should pre-empt any other running user tasks to run the real-time thread. Furthermore, the interrupt handler disables all I/O interrupts to prevent additional interrupt processing on the soon-preempted CPU. Interrupt handlers are likely to take kernel locks, which in the worst case, will be kept while the interrupt handler is pre-empted from itself underlying physical CPU. In that case, all tasks or interrupt handlers on other CPUs would have to wait for the pre-empted CPU being dispatched again. By preventing further interrupt processing, this risk is minimized. Once the CPU gets dispatched again, the real-time kernel thread regains control, reenables interrupts and parks itself again. Acked-by: Heiko Carstens Reviewed-by: Mete Durlu Signed-off-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/wti.c | 141 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 arch/s390/kernel/wti.c diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index a70f25e9c17d..badeaa5ccd83 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -43,7 +43,7 @@ obj-y += sysinfo.o lgr.o os_info.o ctlreg.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o -obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o facility.o uv.o +obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o facility.o uv.o wti.o extra-y += vmlinux.lds diff --git a/arch/s390/kernel/wti.c b/arch/s390/kernel/wti.c new file mode 100644 index 000000000000..3abae3908e65 --- /dev/null +++ b/arch/s390/kernel/wti.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for warning track interruption + * + * Copyright IBM Corp. 2023 + */ + +#include +#include +#include +#include +#include + +struct wti_state { + /* + * Represents the real-time thread responsible to + * acknowledge the warning-track interrupt and trigger + * preliminary and postliminary precautions. + */ + struct task_struct *thread; + /* + * If pending is true, the real-time thread must be scheduled. + * If not, a wake up of that thread will remain a noop. + */ + bool pending; +}; + +static DEFINE_PER_CPU(struct wti_state, wti_state); + +/* + * During a warning-track grace period, interrupts are disabled + * to prevent delays of the warning-track acknowledgment. + * + * Once the CPU is physically dispatched again, interrupts are + * re-enabled. + */ + +static void wti_irq_disable(void) +{ + unsigned long flags; + struct ctlreg cr6; + + local_irq_save(flags); + local_ctl_store(6, &cr6); + /* disable all I/O interrupts */ + cr6.val &= ~0xff000000UL; + local_ctl_load(6, &cr6); + local_irq_restore(flags); +} + +static void wti_irq_enable(void) +{ + unsigned long flags; + struct ctlreg cr6; + + local_irq_save(flags); + local_ctl_store(6, &cr6); + /* enable all I/O interrupts */ + cr6.val |= 0xff000000UL; + local_ctl_load(6, &cr6); + local_irq_restore(flags); +} + +static void wti_interrupt(struct ext_code ext_code, + unsigned int param32, unsigned long param64) +{ + struct wti_state *st = this_cpu_ptr(&wti_state); + + inc_irq_stat(IRQEXT_WTI); + wti_irq_disable(); + st->pending = true; + wake_up_process(st->thread); +} + +static int wti_pending(unsigned int cpu) +{ + struct wti_state *st = per_cpu_ptr(&wti_state, cpu); + + return st->pending; +} + +static void wti_thread_fn(unsigned int cpu) +{ + struct wti_state *st = per_cpu_ptr(&wti_state, cpu); + + st->pending = false; + /* + * Yield CPU voluntarily to the hypervisor. Control + * resumes when hypervisor decides to dispatch CPU + * to this LPAR again. + */ + diag49c(DIAG49C_SUBC_ACK); + wti_irq_enable(); +} + +static struct smp_hotplug_thread wti_threads = { + .store = &wti_state.thread, + .thread_should_run = wti_pending, + .thread_fn = wti_thread_fn, + .thread_comm = "cpuwti/%u", + .selfparking = false, +}; + +static int __init wti_init(void) +{ + struct sched_param wti_sched_param = { .sched_priority = MAX_RT_PRIO - 1 }; + struct wti_state *st; + int cpu, rc; + + rc = -EOPNOTSUPP; + if (!sclp.has_wti) + goto out; + rc = smpboot_register_percpu_thread(&wti_threads); + if (WARN_ON(rc)) + goto out; + for_each_online_cpu(cpu) { + st = per_cpu_ptr(&wti_state, cpu); + sched_setscheduler(st->thread, SCHED_FIFO, &wti_sched_param); + } + rc = register_external_irq(EXT_IRQ_WARNING_TRACK, wti_interrupt); + if (rc) { + pr_warn("Couldn't request external interrupt 0x1007\n"); + goto out_thread; + } + irq_subclass_register(IRQ_SUBCLASS_WARNING_TRACK); + rc = diag49c(DIAG49C_SUBC_REG); + if (rc) { + pr_warn("Failed to register warning track interrupt through DIAG 49C\n"); + rc = -EOPNOTSUPP; + goto out_subclass; + } + goto out; +out_subclass: + irq_subclass_unregister(IRQ_SUBCLASS_WARNING_TRACK); + unregister_external_irq(EXT_IRQ_WARNING_TRACK, wti_interrupt); +out_thread: + smpboot_unregister_percpu_thread(&wti_threads); +out: + return rc; +} +late_initcall(wti_init); From 42419bcdfdcb287918e53500a04aeb532b41ed1f Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Mon, 12 Aug 2024 13:39:29 +0200 Subject: [PATCH 54/84] s390/wti: Add wti accounting for missed grace periods A virtual CPU that has received a warning-track interrupt may fail to acknowledge the interrupt within the warning-track grace period. While this is usually not a problem, it will become necessary to investigate if there is a large number of such missed warning-track interrupts. Therefore, it is necessary to track these events. The information is tracked through the s390 debug facility and can be found under /sys/kernel/debug/s390dbf/wti/. The hex_ascii output is formatted as: The values pid and current psw are collected when a warning track interrupt is received. Symbol is either the kernel symbol matching the collected psw or redacted to when running in user space. Each line represents the currently executing process when a warning track interrupt was received which was then not acknowledged within its grace period. Acked-by: Heiko Carstens Reviewed-by: Mete Durlu Signed-off-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/wti.c | 51 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/wti.c b/arch/s390/kernel/wti.c index 3abae3908e65..da8bbffbed9d 100644 --- a/arch/s390/kernel/wti.c +++ b/arch/s390/kernel/wti.c @@ -5,13 +5,25 @@ * Copyright IBM Corp. 2023 */ +#include #include #include #include +#include #include #include +#define WTI_DBF_LEN 64 + +struct wti_debug { + unsigned long missed; + unsigned long addr; + pid_t pid; +}; + struct wti_state { + /* debug data for s390dbf */ + struct wti_debug dbg; /* * Represents the real-time thread responsible to * acknowledge the warning-track interrupt and trigger @@ -27,6 +39,8 @@ struct wti_state { static DEFINE_PER_CPU(struct wti_state, wti_state); +static debug_info_t *wti_dbg; + /* * During a warning-track grace period, interrupts are disabled * to prevent delays of the warning-track acknowledgment. @@ -61,6 +75,16 @@ static void wti_irq_enable(void) local_irq_restore(flags); } +static void store_debug_data(struct wti_state *st) +{ + struct pt_regs *regs = get_irq_regs(); + + st->dbg.pid = current->pid; + st->dbg.addr = 0; + if (!user_mode(regs)) + st->dbg.addr = regs->psw.addr; +} + static void wti_interrupt(struct ext_code ext_code, unsigned int param32, unsigned long param64) { @@ -68,6 +92,7 @@ static void wti_interrupt(struct ext_code ext_code, inc_irq_stat(IRQEXT_WTI); wti_irq_disable(); + store_debug_data(st); st->pending = true; wake_up_process(st->thread); } @@ -79,6 +104,19 @@ static int wti_pending(unsigned int cpu) return st->pending; } +static void wti_dbf_grace_period(struct wti_state *st) +{ + struct wti_debug *wdi = &st->dbg; + char buf[WTI_DBF_LEN]; + + if (wdi->addr) + snprintf(buf, sizeof(buf), "%d %pS", wdi->pid, (void *)wdi->addr); + else + snprintf(buf, sizeof(buf), "%d ", wdi->pid); + debug_text_event(wti_dbg, 2, buf); + wdi->missed++; +} + static void wti_thread_fn(unsigned int cpu) { struct wti_state *st = per_cpu_ptr(&wti_state, cpu); @@ -89,7 +127,8 @@ static void wti_thread_fn(unsigned int cpu) * resumes when hypervisor decides to dispatch CPU * to this LPAR again. */ - diag49c(DIAG49C_SUBC_ACK); + if (diag49c(DIAG49C_SUBC_ACK)) + wti_dbf_grace_period(st); wti_irq_enable(); } @@ -129,7 +168,17 @@ static int __init wti_init(void) rc = -EOPNOTSUPP; goto out_subclass; } + wti_dbg = debug_register("wti", 1, 1, WTI_DBF_LEN); + if (!wti_dbg) { + rc = -ENOMEM; + goto out_debug_register; + } + rc = debug_register_view(wti_dbg, &debug_hex_ascii_view); + if (rc) + goto out_debug_register; goto out; +out_debug_register: + debug_unregister(wti_dbg); out_subclass: irq_subclass_unregister(IRQ_SUBCLASS_WARNING_TRACK); unregister_external_irq(EXT_IRQ_WARNING_TRACK, wti_interrupt); From 307b675cf0193f37fbc50ff9de5322dc1361aa6b Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Mon, 12 Aug 2024 13:39:30 +0200 Subject: [PATCH 55/84] s390/wti: Add debugfs file to display missed grace periods per cpu Introduce a new debug file which allows to determine how many warning track grace periods were missed on each CPU. The new file can be found as /sys/kernel/debug/s390/wti It is formatted as: CPU0 CPU1 [...] CPUx xyz xyz [...] xyz Acked-by: Heiko Carstens Reviewed-by: Mete Durlu Signed-off-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/wti.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/arch/s390/kernel/wti.c b/arch/s390/kernel/wti.c index da8bbffbed9d..949fdbf0e8b6 100644 --- a/arch/s390/kernel/wti.c +++ b/arch/s390/kernel/wti.c @@ -5,6 +5,8 @@ * Copyright IBM Corp. 2023 */ +#include +#include #include #include #include @@ -117,6 +119,26 @@ static void wti_dbf_grace_period(struct wti_state *st) wdi->missed++; } +static int wti_show(struct seq_file *seq, void *v) +{ + struct wti_state *st; + int cpu; + + cpus_read_lock(); + seq_puts(seq, " "); + for_each_online_cpu(cpu) + seq_printf(seq, "CPU%-8d", cpu); + seq_putc(seq, '\n'); + for_each_online_cpu(cpu) { + st = per_cpu_ptr(&wti_state, cpu); + seq_printf(seq, " %10lu", st->dbg.missed); + } + seq_putc(seq, '\n'); + cpus_read_unlock(); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(wti); + static void wti_thread_fn(unsigned int cpu) { struct wti_state *st = per_cpu_ptr(&wti_state, cpu); @@ -143,6 +165,7 @@ static struct smp_hotplug_thread wti_threads = { static int __init wti_init(void) { struct sched_param wti_sched_param = { .sched_priority = MAX_RT_PRIO - 1 }; + struct dentry *wti_dir; struct wti_state *st; int cpu, rc; @@ -168,6 +191,8 @@ static int __init wti_init(void) rc = -EOPNOTSUPP; goto out_subclass; } + wti_dir = debugfs_create_dir("wti", arch_debugfs_dir); + debugfs_create_file("stat", 0400, wti_dir, NULL, &wti_fops); wti_dbg = debug_register("wti", 1, 1, WTI_DBF_LEN); if (!wti_dbg) { rc = -ENOMEM; From 9dd333e7afc4a4855f821a7be83733b8ef48d5ba Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Mon, 12 Aug 2024 13:39:31 +0200 Subject: [PATCH 56/84] s390/topology: Add sysctl handler for polarization Provide an additional path to set the polarization of the system, such that a user no longer relies on the sysfs interface only and is able configure the polarization for every reboot via sysctl control files. The new sysctl can be set as follows: - s390.polarization=0 for horizontal polarization - s390.polarization=1 for vertical polarization Acked-by: Heiko Carstens Tested-by: Mete Durlu Signed-off-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/topology.c | 58 ++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 22029ecae1c5..4bd0ee8eb1ab 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -376,6 +376,25 @@ void topology_expect_change(void) static int cpu_management; +static int set_polarization(int polarization) +{ + int rc = 0; + + cpus_read_lock(); + mutex_lock(&smp_cpu_state_mutex); + if (cpu_management == polarization) + goto out; + rc = topology_set_cpu_management(polarization); + if (rc) + goto out; + cpu_management = polarization; + topology_expect_change(); +out: + mutex_unlock(&smp_cpu_state_mutex); + cpus_read_unlock(); + return rc; +} + static ssize_t dispatching_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -400,19 +419,7 @@ static ssize_t dispatching_store(struct device *dev, return -EINVAL; if (val != 0 && val != 1) return -EINVAL; - rc = 0; - cpus_read_lock(); - mutex_lock(&smp_cpu_state_mutex); - if (cpu_management == val) - goto out; - rc = topology_set_cpu_management(val); - if (rc) - goto out; - cpu_management = val; - topology_expect_change(); -out: - mutex_unlock(&smp_cpu_state_mutex); - cpus_read_unlock(); + rc = set_polarization(val); return rc ? rc : count; } static DEVICE_ATTR_RW(dispatching); @@ -624,12 +631,37 @@ static int topology_ctl_handler(const struct ctl_table *ctl, int write, return rc; } +static int polarization_ctl_handler(const struct ctl_table *ctl, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int polarization; + int rc; + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &polarization, + .maxlen = sizeof(int), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + + polarization = cpu_management; + rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; + return set_polarization(polarization); +} + static struct ctl_table topology_ctl_table[] = { { .procname = "topology", .mode = 0644, .proc_handler = topology_ctl_handler, }, + { + .procname = "polarization", + .mode = 0644, + .proc_handler = polarization_ctl_handler, + }, }; static int __init topology_init(void) From 7e627f819302b8c22098b0575c35a3349c4e306f Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Mon, 12 Aug 2024 13:39:32 +0200 Subject: [PATCH 57/84] s390/topology: Add config option to switch to vertical during boot By default, all systems on s390 start in horizontal cpu polarization. Selecting the new config option SCHED_TOPOLOGY_VERTICAL allows to build a kernel that switches to vertical polarization during boot. Acked-by: Heiko Carstens Tested-by: Mete Durlu Signed-off-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/Kconfig | 8 ++++++++ arch/s390/kernel/topology.c | 2 ++ 2 files changed, 10 insertions(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index abc7e1cc0e7c..2ee634bab222 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -513,6 +513,14 @@ config SCHED_TOPOLOGY making when dealing with machines that have multi-threading, multiple cores or multiple books. +config SCHED_TOPOLOGY_VERTICAL + def_bool y + bool "Use vertical CPU polarization by default" + depends on SCHED_TOPOLOGY + help + Use vertical CPU polarization by default if available. + The default CPU polarization is horizontal. + source "kernel/Kconfig.hz" config CERT_STORE diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 4bd0ee8eb1ab..58da6d1bae45 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -674,6 +674,8 @@ static int __init topology_init(void) set_topology_timer(); else topology_update_polarization_simple(); + if (IS_ENABLED(CONFIG_SCHED_TOPOLOGY_VERTICAL)) + set_polarization(1); register_sysctl("s390", topology_ctl_table); dev_root = bus_get_dev_root(&cpu_subsys); From 26ceef523d5442a8bc88e334c3e84cdbd9e3fb92 Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:33 +0200 Subject: [PATCH 58/84] s390/smp: Add cpu capacities Linux scheduler allows architectures to assign capacity values to individual CPUs. This hints scheduler the performance difference between CPUs and allows more efficient task distribution them. Implement helper methods to set and get CPU capacities for s390. This is particularly helpful in vertical polarization configurations of LPARs. On vertical polarization an LPARs CPUs can get different polarization values depending on the CEC configuration. CPUs with different polarization values can perform different from each other, using CPU capacities this can be reflected to linux scheduler. Acked-by: Vasily Gorbik Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/processor.h | 1 + arch/s390/include/asm/smp.h | 4 ++++ arch/s390/include/asm/topology.h | 3 +++ arch/s390/kernel/smp.c | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 5ecd442535b9..9a5236acc0a8 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -44,6 +44,7 @@ struct pcpu { unsigned long ec_mask; /* bit mask for ec_xxx functions */ unsigned long ec_clk; /* sigp timestamp for ec_xxx */ unsigned long flags; /* per CPU flags */ + unsigned long capacity; /* cpu capacity for scheduler */ signed char state; /* physical cpu state */ signed char polarization; /* physical polarization */ u16 address; /* physical cpu address */ diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index cd835f4fb11a..7feca96c48c6 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -12,6 +12,7 @@ #include #define raw_smp_processor_id() (get_lowcore()->cpu_nr) +#define arch_scale_cpu_capacity smp_cpu_get_capacity extern struct mutex smp_cpu_state_mutex; extern unsigned int smp_cpu_mt_shift; @@ -34,6 +35,9 @@ extern void smp_save_dump_secondary_cpus(void); extern void smp_yield_cpu(int cpu); extern void smp_cpu_set_polarization(int cpu, int val); extern int smp_cpu_get_polarization(int cpu); +extern void smp_cpu_set_capacity(int cpu, unsigned long val); +extern void smp_set_core_capacity(int cpu, unsigned long val); +extern unsigned long smp_cpu_get_capacity(int cpu); extern int smp_cpu_get_cpu_address(int cpu); extern void smp_fill_possible_mask(void); extern void smp_detect_cpus(void); diff --git a/arch/s390/include/asm/topology.h b/arch/s390/include/asm/topology.h index 3a0ac0c7a9a3..cef06bffad80 100644 --- a/arch/s390/include/asm/topology.h +++ b/arch/s390/include/asm/topology.h @@ -67,6 +67,9 @@ static inline void topology_expect_change(void) { } #define POLARIZATION_VM (2) #define POLARIZATION_VH (3) +#define CPU_CAPACITY_HIGH SCHED_CAPACITY_SCALE +#define CPU_CAPACITY_LOW (SCHED_CAPACITY_SCALE >> 3) + #define SD_BOOK_INIT SD_CPU_INIT #ifdef CONFIG_NUMA diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index fbba37ec53cf..4df56fdb2488 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -671,6 +671,25 @@ int smp_cpu_get_polarization(int cpu) return per_cpu(pcpu_devices, cpu).polarization; } +void smp_cpu_set_capacity(int cpu, unsigned long val) +{ + per_cpu(pcpu_devices, cpu).capacity = val; +} + +unsigned long smp_cpu_get_capacity(int cpu) +{ + return per_cpu(pcpu_devices, cpu).capacity; +} + +void smp_set_core_capacity(int cpu, unsigned long val) +{ + int i; + + cpu = smp_get_base_cpu(cpu); + for (i = cpu; (i <= cpu + smp_cpu_mtid) && (i < nr_cpu_ids); i++) + smp_cpu_set_capacity(i, val); +} + int smp_cpu_get_cpu_address(int cpu) { return per_cpu(pcpu_devices, cpu).address; @@ -719,6 +738,7 @@ static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail, else pcpu->state = CPU_STATE_STANDBY; smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); + smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH); set_cpu_present(cpu, true); if (!early && arch_register_cpu(cpu)) set_cpu_present(cpu, false); @@ -961,6 +981,7 @@ void __init smp_prepare_boot_cpu(void) ipl_pcpu->state = CPU_STATE_CONFIGURED; lc->pcpu = (unsigned long)ipl_pcpu; smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN); + smp_cpu_set_capacity(0, CPU_CAPACITY_HIGH); } void __init smp_setup_processor_id(void) From 6843d6d97c03b8fa506d188a483bc494a6ac4c89 Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:34 +0200 Subject: [PATCH 59/84] s390/hiperdispatch: Introduce hiperdispatch When LPAR is in vertical polarization, CPUs get different polarization values, namely vertical high, vertical medium and vertical low. These values represent the likelyhood of the CPU getting physical runtime. Vertical high CPUs will always get runtime and others get varying runtime depending on the load the CEC is under. Vertical high and vertical medium CPUs are considered the CPUs which the current LPAR has the entitlement to run on. The vertical lows are on the other hand are borrowed CPUs which would only be given to the LPAR by hipervisor when the other LPARs are not utilizing them. Using the CPU capacities, hint linux scheduler when it should prioritise vertical high and vertical medium CPUs over vertical low CPUs. By tracking various system statistics hiperdispatch determines when to adjust cpu capacities. After each adjustment, rebuilding of scheduler domains is necessary to notify the scheduler about capacity changes but since this operation is costly it should be done as sparsely as possible. Acked-by: Vasily Gorbik Co-developed-by: Tobias Huschle Signed-off-by: Tobias Huschle Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/hiperdispatch.h | 14 ++ arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/hiperdispatch.c | 199 ++++++++++++++++++++++++++ arch/s390/kernel/topology.c | 18 ++- 4 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 arch/s390/include/asm/hiperdispatch.h create mode 100644 arch/s390/kernel/hiperdispatch.c diff --git a/arch/s390/include/asm/hiperdispatch.h b/arch/s390/include/asm/hiperdispatch.h new file mode 100644 index 000000000000..27e23aa27a24 --- /dev/null +++ b/arch/s390/include/asm/hiperdispatch.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 2024 + */ + +#ifndef _ASM_HIPERDISPATCH_H +#define _ASM_HIPERDISPATCH_H + +void hd_reset_state(void); +void hd_add_core(int cpu); +void hd_disable_hiperdispatch(void); +int hd_enable_hiperdispatch(void); + +#endif /* _ASM_HIPERDISPATCH_H */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index badeaa5ccd83..5ceb08b338d3 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -51,7 +51,7 @@ obj-$(CONFIG_SYSFS) += nospec-sysfs.o CFLAGS_REMOVE_nospec-branch.o += $(CC_FLAGS_EXPOLINE) obj-$(CONFIG_MODULES) += module.o -obj-$(CONFIG_SCHED_TOPOLOGY) += topology.o +obj-$(CONFIG_SCHED_TOPOLOGY) += topology.o hiperdispatch.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_AUDIT) += audit.o compat-obj-$(CONFIG_AUDIT) += compat_audit.o diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c new file mode 100644 index 000000000000..233872d59b76 --- /dev/null +++ b/arch/s390/kernel/hiperdispatch.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "hd" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +/* + * Hiperdispatch: + * Dynamically calculates the optimum number of high capacity COREs + * by considering the state the system is in. When hiperdispatch decides + * that a capacity update is necessary, it schedules a topology update. + * During topology updates the CPU capacities are always re-adjusted. + * + * There is two places where CPU capacities are being accessed within + * hiperdispatch. + * -> hiperdispatch's reoccuring work function reads CPU capacities to + * determine high capacity CPU count. + * -> during a topology update hiperdispatch's adjustment function + * updates CPU capacities. + * These two can run on different CPUs in parallel which can cause + * hiperdispatch to make wrong decisions. This can potentially cause + * some overhead by leading to extra rebuild_sched_domains() calls + * for correction. Access to capacities within hiperdispatch has to be + * serialized to prevent the overhead. + * + * Hiperdispatch decision making revolves around steal time. + * HD_STEAL_THRESHOLD value is taken as reference. Whenever steal time + * crosses the threshold value hiperdispatch falls back to giving high + * capacities to entitled CPUs. When steal time drops below the + * threshold boundary, hiperdispatch utilizes all CPUs by giving all + * of them high capacity. + * + * The theory behind HD_STEAL_THRESHOLD is related to the SMP thread + * performance. Comparing the throughput of; + * - single CORE, with N threads, running N tasks + * - N separate COREs running N tasks, + * using individual COREs for individual tasks yield better + * performance. This performance difference is roughly ~30% (can change + * between machine generations) + * + * Hiperdispatch tries to hint scheduler to use individual COREs for + * each task, as long as steal time on those COREs are less than 30%, + * therefore delaying the throughput loss caused by using SMP threads. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define HD_DELAY_FACTOR (4) +#define HD_DELAY_INTERVAL (HZ / 4) +#define HD_STEAL_THRESHOLD 30 + +static cpumask_t hd_vl_coremask; /* Mask containing all vertical low COREs */ +static cpumask_t hd_vmvl_cpumask; /* Mask containing vertical medium and low CPUs */ +static int hd_high_capacity_cores; /* Current CORE count with high capacity */ +static int hd_entitled_cores; /* Total vertical high and medium CORE count */ +static int hd_online_cores; /* Current online CORE count */ + +static unsigned long hd_previous_steal; /* Previous iteration's CPU steal timer total */ + +static void hd_capacity_work_fn(struct work_struct *work); +static DECLARE_DELAYED_WORK(hd_capacity_work, hd_capacity_work_fn); + +void hd_reset_state(void) +{ + cpumask_clear(&hd_vl_coremask); + cpumask_clear(&hd_vmvl_cpumask); + hd_entitled_cores = 0; + hd_online_cores = 0; +} + +void hd_add_core(int cpu) +{ + const struct cpumask *siblings; + int polarization; + + hd_online_cores++; + polarization = smp_cpu_get_polarization(cpu); + siblings = topology_sibling_cpumask(cpu); + switch (polarization) { + case POLARIZATION_VH: + hd_entitled_cores++; + break; + case POLARIZATION_VM: + hd_entitled_cores++; + cpumask_or(&hd_vmvl_cpumask, &hd_vmvl_cpumask, siblings); + break; + case POLARIZATION_VL: + cpumask_set_cpu(cpu, &hd_vl_coremask); + cpumask_or(&hd_vmvl_cpumask, &hd_vmvl_cpumask, siblings); + break; + } +} + +static void hd_update_capacities(void) +{ + int cpu, upscaling_cores; + unsigned long capacity; + + upscaling_cores = hd_high_capacity_cores - hd_entitled_cores; + capacity = upscaling_cores > 0 ? CPU_CAPACITY_HIGH : CPU_CAPACITY_LOW; + hd_high_capacity_cores = hd_entitled_cores; + for_each_cpu(cpu, &hd_vl_coremask) { + smp_set_core_capacity(cpu, capacity); + if (capacity != CPU_CAPACITY_HIGH) + continue; + hd_high_capacity_cores++; + upscaling_cores--; + if (upscaling_cores == 0) + capacity = CPU_CAPACITY_LOW; + } +} + +void hd_disable_hiperdispatch(void) +{ + cancel_delayed_work_sync(&hd_capacity_work); + hd_high_capacity_cores = hd_online_cores; + hd_previous_steal = 0; +} + +int hd_enable_hiperdispatch(void) +{ + if (hd_entitled_cores == 0) + return 0; + if (hd_online_cores <= hd_entitled_cores) + return 0; + mod_delayed_work(system_wq, &hd_capacity_work, HD_DELAY_INTERVAL * HD_DELAY_FACTOR); + hd_update_capacities(); + return 1; +} + +static unsigned long hd_calculate_steal_percentage(void) +{ + unsigned long time_delta, steal_delta, steal, percentage; + static ktime_t prev; + int cpus, cpu; + ktime_t now; + + cpus = 0; + steal = 0; + percentage = 0; + for_each_cpu(cpu, &hd_vmvl_cpumask) { + steal += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; + cpus++; + } + /* + * If there is no vertical medium and low CPUs steal time + * is 0 as vertical high CPUs shouldn't experience steal time. + */ + if (cpus == 0) + return percentage; + now = ktime_get(); + time_delta = ktime_to_ns(ktime_sub(now, prev)); + if (steal > hd_previous_steal && hd_previous_steal != 0) { + steal_delta = (steal - hd_previous_steal) * 100 / time_delta; + percentage = steal_delta / cpus; + } + hd_previous_steal = steal; + prev = now; + return percentage; +} + +static void hd_capacity_work_fn(struct work_struct *work) +{ + unsigned long steal_percentage, new_cores; + + mutex_lock(&smp_cpu_state_mutex); + /* + * If online cores are less or equal to entitled cores hiperdispatch + * does not need to make any adjustments, call a topology update to + * disable hiperdispatch. + * Normally this check is handled on topology update, but during cpu + * unhotplug, topology and cpu mask updates are done in reverse + * order, causing hd_enable_hiperdispatch() to get stale data. + */ + if (hd_online_cores <= hd_entitled_cores) { + topology_schedule_update(); + mutex_unlock(&smp_cpu_state_mutex); + return; + } + steal_percentage = hd_calculate_steal_percentage(); + if (steal_percentage < HD_STEAL_THRESHOLD) + new_cores = hd_online_cores; + else + new_cores = hd_entitled_cores; + if (hd_high_capacity_cores != new_cores) { + hd_high_capacity_cores = new_cores; + topology_schedule_update(); + } + mutex_unlock(&smp_cpu_state_mutex); + schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL); +} diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 58da6d1bae45..813e5da9a973 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #define PTF_HORIZONTAL (0UL) @@ -47,6 +48,7 @@ static int topology_mode = TOPOLOGY_MODE_UNINITIALIZED; static void set_topology_timer(void); static void topology_work_fn(struct work_struct *work); static struct sysinfo_15_1_x *tl_info; +static int cpu_management; static DECLARE_WORK(topology_work, topology_work_fn); @@ -144,6 +146,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core, cpumask_set_cpu(cpu, &book->mask); cpumask_set_cpu(cpu, &socket->mask); smp_cpu_set_polarization(cpu, tl_core->pp); + smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH); } } } @@ -270,6 +273,7 @@ void update_cpu_masks(void) topo->drawer_id = id; } } + hd_reset_state(); for_each_online_cpu(cpu) { topo = &cpu_topology[cpu]; pkg_first = cpumask_first(&topo->core_mask); @@ -278,8 +282,10 @@ void update_cpu_masks(void) for_each_cpu(sibling, &topo->core_mask) { topo_sibling = &cpu_topology[sibling]; smt_first = cpumask_first(&topo_sibling->thread_mask); - if (sibling == smt_first) + if (sibling == smt_first) { topo_package->booted_cores++; + hd_add_core(sibling); + } } } else { topo->booted_cores = topo_package->booted_cores; @@ -303,8 +309,10 @@ static void __arch_update_dedicated_flag(void *arg) static int __arch_update_cpu_topology(void) { struct sysinfo_15_1_x *info = tl_info; - int rc = 0; + int rc, hd_status; + hd_status = 0; + rc = 0; mutex_lock(&smp_cpu_state_mutex); if (MACHINE_HAS_TOPOLOGY) { rc = 1; @@ -314,7 +322,11 @@ static int __arch_update_cpu_topology(void) update_cpu_masks(); if (!MACHINE_HAS_TOPOLOGY) topology_update_polarization_simple(); + if (cpu_management == 1) + hd_status = hd_enable_hiperdispatch(); mutex_unlock(&smp_cpu_state_mutex); + if (hd_status == 0) + hd_disable_hiperdispatch(); return rc; } @@ -374,8 +386,6 @@ void topology_expect_change(void) set_topology_timer(); } -static int cpu_management; - static int set_polarization(int polarization) { int rc = 0; From c0d4ba054f6a32c5d0a6740d90c97f91faf73fd7 Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:35 +0200 Subject: [PATCH 60/84] s390/hiperdispatch: Add steal time averaging The measurements done by hiperdispatch can have sudden spikes and dips during run time. To prevent these outliers effecting the decision making process and causing adjustment overhead, use weighted average of the steal time. Acked-by: Vasily Gorbik Co-developed-by: Tobias Huschle Signed-off-by: Tobias Huschle Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/kernel/hiperdispatch.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index 233872d59b76..ed5b2d7d11c4 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -56,6 +56,7 @@ #define HD_DELAY_FACTOR (4) #define HD_DELAY_INTERVAL (HZ / 4) #define HD_STEAL_THRESHOLD 30 +#define HD_STEAL_AVG_WEIGHT 16 static cpumask_t hd_vl_coremask; /* Mask containing all vertical low COREs */ static cpumask_t hd_vmvl_cpumask; /* Mask containing vertical medium and low CPUs */ @@ -136,6 +137,14 @@ int hd_enable_hiperdispatch(void) return 1; } +static unsigned long hd_steal_avg(unsigned long new) +{ + static unsigned long steal; + + steal = (steal * (HD_STEAL_AVG_WEIGHT - 1) + new) / HD_STEAL_AVG_WEIGHT; + return steal; +} + static unsigned long hd_calculate_steal_percentage(void) { unsigned long time_delta, steal_delta, steal, percentage; @@ -185,7 +194,7 @@ static void hd_capacity_work_fn(struct work_struct *work) mutex_unlock(&smp_cpu_state_mutex); return; } - steal_percentage = hd_calculate_steal_percentage(); + steal_percentage = hd_steal_avg(hd_calculate_steal_percentage()); if (steal_percentage < HD_STEAL_THRESHOLD) new_cores = hd_online_cores; else From 1e5aa12d470b1af613fca89a3069529fa9c92cfb Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:36 +0200 Subject: [PATCH 61/84] s390/hiperdispatch: Add trace events Add trace events to debug hiperdispatch behavior and track domain rebuilding. Two events provide information about the decision making of hiperdispatch and the adjustments made. Acked-by: Vasily Gorbik Co-developed-by: Tobias Huschle Signed-off-by: Tobias Huschle Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/trace/hiperdispatch.h | 58 +++++++++++++++++++++ arch/s390/kernel/hiperdispatch.c | 5 ++ 2 files changed, 63 insertions(+) create mode 100644 arch/s390/include/asm/trace/hiperdispatch.h diff --git a/arch/s390/include/asm/trace/hiperdispatch.h b/arch/s390/include/asm/trace/hiperdispatch.h new file mode 100644 index 000000000000..46462ee645b0 --- /dev/null +++ b/arch/s390/include/asm/trace/hiperdispatch.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Tracepoint header for hiperdispatch + * + * Copyright IBM Corp. 2024 + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM s390 + +#if !defined(_TRACE_S390_HIPERDISPATCH_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_S390_HIPERDISPATCH_H + +#include + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE + +#define TRACE_INCLUDE_PATH asm/trace +#define TRACE_INCLUDE_FILE hiperdispatch + +TRACE_EVENT(s390_hd_work_fn, + TP_PROTO(int steal_time_percentage, + int entitled_core_count, + int highcap_core_count), + TP_ARGS(steal_time_percentage, + entitled_core_count, + highcap_core_count), + TP_STRUCT__entry(__field(int, steal_time_percentage) + __field(int, entitled_core_count) + __field(int, highcap_core_count)), + TP_fast_assign(__entry->steal_time_percentage = steal_time_percentage; + __entry->entitled_core_count = entitled_core_count; + __entry->highcap_core_count = highcap_core_count;), + TP_printk("steal: %d entitled_core_count: %d highcap_core_count: %d", + __entry->steal_time_percentage, + __entry->entitled_core_count, + __entry->highcap_core_count) +); + +TRACE_EVENT(s390_hd_rebuild_domains, + TP_PROTO(int current_highcap_core_count, + int new_highcap_core_count), + TP_ARGS(current_highcap_core_count, + new_highcap_core_count), + TP_STRUCT__entry(__field(int, current_highcap_core_count) + __field(int, new_highcap_core_count)), + TP_fast_assign(__entry->current_highcap_core_count = current_highcap_core_count; + __entry->new_highcap_core_count = new_highcap_core_count), + TP_printk("change highcap_core_count: %u -> %u", + __entry->current_highcap_core_count, + __entry->new_highcap_core_count) +); + +#endif /* _TRACE_S390_HIPERDISPATCH_H */ + +/* This part must be outside protection */ +#include diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index ed5b2d7d11c4..3ba7a3492924 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -53,6 +53,9 @@ #include #include +#define CREATE_TRACE_POINTS +#include + #define HD_DELAY_FACTOR (4) #define HD_DELAY_INTERVAL (HZ / 4) #define HD_STEAL_THRESHOLD 30 @@ -200,9 +203,11 @@ static void hd_capacity_work_fn(struct work_struct *work) else new_cores = hd_entitled_cores; if (hd_high_capacity_cores != new_cores) { + trace_s390_hd_rebuild_domains(hd_high_capacity_cores, new_cores); hd_high_capacity_cores = new_cores; topology_schedule_update(); } + trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores); mutex_unlock(&smp_cpu_state_mutex); schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL); } From b9271a533433dd4049723d4668418709e1927e12 Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:37 +0200 Subject: [PATCH 62/84] s390/hiperdispatch: Add hiperdispatch sysctl interface Expose hiperdispatch controls via sysctl. The user can now toggle hiperdispatch via assigning 0 or 1 to s390.hiperdispatch attribute. When hiperdipatch is toggled on, it tries to adjust CPU capacities, while system is in vertical polarization to gain performance benefits from different CPU polarizations. Disabling hiperdispatch reverts the CPU capacities to their default (HIGH_CAPACITY) and stops the dynamic adjustments. Introduce a kconfig option HIPERDISPATCH_ON which allows users to use hiperdispatch by default on vertical polarization. Using the sysctl attribute s390.hiperdispatch would overwrite this behavior. Acked-by: Vasily Gorbik Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/Kconfig | 12 +++++++ arch/s390/kernel/hiperdispatch.c | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 2ee634bab222..bc2aab60c1c8 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -521,6 +521,18 @@ config SCHED_TOPOLOGY_VERTICAL Use vertical CPU polarization by default if available. The default CPU polarization is horizontal. +config HIPERDISPATCH_ON + def_bool y + bool "Use hiperdispatch on vertical polarization by default" + depends on SCHED_TOPOLOGY + depends on PROC_SYSCTL + help + Hiperdispatch aims to improve the CPU scheduler's decision + making when using vertical polarization by adjusting CPU + capacities dynamically. Set this option to use hiperdispatch + on vertical polarization by default. This can be overwritten + by sysctl's s390.hiperdispatch attribute later on. + source "kernel/Kconfig.hz" config CERT_STORE diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index 3ba7a3492924..8ff4adab7402 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -48,8 +48,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -69,9 +71,21 @@ static int hd_online_cores; /* Current online CORE count */ static unsigned long hd_previous_steal; /* Previous iteration's CPU steal timer total */ +static int hd_enabled; + static void hd_capacity_work_fn(struct work_struct *work); static DECLARE_DELAYED_WORK(hd_capacity_work, hd_capacity_work_fn); +static int hd_set_hiperdispatch_mode(int enable) +{ + if (!MACHINE_HAS_TOPOLOGY) + enable = 0; + if (hd_enabled == enable) + return 0; + hd_enabled = enable; + return 1; +} + void hd_reset_state(void) { cpumask_clear(&hd_vl_coremask); @@ -131,6 +145,8 @@ void hd_disable_hiperdispatch(void) int hd_enable_hiperdispatch(void) { + if (hd_enabled == 0) + return 0; if (hd_entitled_cores == 0) return 0; if (hd_online_cores <= hd_entitled_cores) @@ -211,3 +227,47 @@ static void hd_capacity_work_fn(struct work_struct *work) mutex_unlock(&smp_cpu_state_mutex); schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL); } + +static int hiperdispatch_ctl_handler(const struct ctl_table *ctl, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int hiperdispatch; + int rc; + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &hiperdispatch, + .maxlen = sizeof(int), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + + hiperdispatch = hd_enabled; + rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; + mutex_lock(&smp_cpu_state_mutex); + if (hd_set_hiperdispatch_mode(hiperdispatch)) + topology_schedule_update(); + mutex_unlock(&smp_cpu_state_mutex); + return 0; +} + +static struct ctl_table hiperdispatch_ctl_table[] = { + { + .procname = "hiperdispatch", + .mode = 0644, + .proc_handler = hiperdispatch_ctl_handler, + }, +}; + +static int __init hd_init(void) +{ + if (IS_ENABLED(CONFIG_HIPERDISPATCH_ON)) { + hd_set_hiperdispatch_mode(1); + topology_schedule_update(); + } + if (!register_sysctl("s390", hiperdispatch_ctl_table)) + pr_warn("Failed to register s390.hiperdispatch sysctl attribute\n"); + return 0; +} +late_initcall(hd_init); From 441cc6f5b66e814405766ddec4af5071d22e98da Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:38 +0200 Subject: [PATCH 63/84] s390/hiperdispatch: Add hiperdispatch debug attributes Add two attributes for debug purposes. They can be found under; /sys/devices/system/cpu/hiperdispatch/ * hd_stime_threshold : allows user to adjust steal time threshold * hd_delay_factor : allows user to adjust delay factor of hiperdispatch work (after topology updates, delayed work is always delayed extra by this factor) hd_stime_threshold can have values between 0-100 as it represents a percentage value. hd_delay_factor can have values greater than 1. It is multiplied with the default delay to achieve a longer interval, pushing back the next hiperdispatch adjustment after a topology update. Ex: if delay interval is 250ms and the delay factor is 4; delayed interval is now 1000ms(1sec). After each capacity adjustment or topology change, work has a delayed interval of 1 sec for one interval. Acked-by: Vasily Gorbik Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/kernel/hiperdispatch.c | 84 +++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index 8ff4adab7402..d38710f39d81 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -46,7 +46,9 @@ */ #include +#include #include +#include #include #include #include @@ -71,6 +73,8 @@ static int hd_online_cores; /* Current online CORE count */ static unsigned long hd_previous_steal; /* Previous iteration's CPU steal timer total */ +static unsigned int hd_steal_threshold = HD_STEAL_THRESHOLD; +static unsigned int hd_delay_factor = HD_DELAY_FACTOR; static int hd_enabled; static void hd_capacity_work_fn(struct work_struct *work); @@ -151,7 +155,7 @@ int hd_enable_hiperdispatch(void) return 0; if (hd_online_cores <= hd_entitled_cores) return 0; - mod_delayed_work(system_wq, &hd_capacity_work, HD_DELAY_INTERVAL * HD_DELAY_FACTOR); + mod_delayed_work(system_wq, &hd_capacity_work, HD_DELAY_INTERVAL * hd_delay_factor); hd_update_capacities(); return 1; } @@ -214,7 +218,7 @@ static void hd_capacity_work_fn(struct work_struct *work) return; } steal_percentage = hd_steal_avg(hd_calculate_steal_percentage()); - if (steal_percentage < HD_STEAL_THRESHOLD) + if (steal_percentage < hd_steal_threshold) new_cores = hd_online_cores; else new_cores = hd_entitled_cores; @@ -260,6 +264,81 @@ static struct ctl_table hiperdispatch_ctl_table[] = { }, }; +static ssize_t hd_steal_threshold_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%u\n", hd_steal_threshold); +} + +static ssize_t hd_steal_threshold_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + unsigned int val; + int rc; + + rc = kstrtouint(buf, 0, &val); + if (rc) + return rc; + if (val > 100) + return -ERANGE; + hd_steal_threshold = val; + return count; +} + +static DEVICE_ATTR_RW(hd_steal_threshold); + +static ssize_t hd_delay_factor_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%u\n", hd_delay_factor); +} + +static ssize_t hd_delay_factor_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + unsigned int val; + int rc; + + rc = kstrtouint(buf, 0, &val); + if (rc) + return rc; + if (!val) + return -ERANGE; + hd_delay_factor = val; + return count; +} + +static DEVICE_ATTR_RW(hd_delay_factor); + +static struct attribute *hd_attrs[] = { + &dev_attr_hd_steal_threshold.attr, + &dev_attr_hd_delay_factor.attr, + NULL, +}; + +static const struct attribute_group hd_attr_group = { + .name = "hiperdispatch", + .attrs = hd_attrs, +}; + +static void __init hd_create_attributes(void) +{ + struct device *dev; + + dev = bus_get_dev_root(&cpu_subsys); + if (!dev) + return; + if (sysfs_create_group(&dev->kobj, &hd_attr_group)) + pr_warn("Unable to create hiperdispatch attribute group\n"); + put_device(dev); +} + static int __init hd_init(void) { if (IS_ENABLED(CONFIG_HIPERDISPATCH_ON)) { @@ -268,6 +347,7 @@ static int __init hd_init(void) } if (!register_sysctl("s390", hiperdispatch_ctl_table)) pr_warn("Failed to register s390.hiperdispatch sysctl attribute\n"); + hd_create_attributes(); return 0; } late_initcall(hd_init); From ea31f1c6b470bba02df0b600ed3cf447d4851f0d Mon Sep 17 00:00:00 2001 From: Mete Durlu Date: Mon, 12 Aug 2024 13:39:39 +0200 Subject: [PATCH 64/84] s390/hiperdispatch: Add hiperdispatch debug counters Add three counters to follow and understand hiperdispatch behavior; * adjustment_count (amount of capacity adjustments triggered) * greedy_time_ms (time spent while all cpus are on high capacity) * conservative_time_ms (time spent while only entitled cpus are on high capacity) These counters can be found under /sys/kernel/debug/s390/hiperdispatch/ Time counters are in format and only cover the time spent when hiperdispatch is active. Acked-by: Vasily Gorbik Signed-off-by: Mete Durlu Signed-off-by: Vasily Gorbik --- arch/s390/kernel/hiperdispatch.c | 77 ++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index d38710f39d81..2a99a216ab62 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -46,11 +46,13 @@ */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -72,6 +74,9 @@ static int hd_entitled_cores; /* Total vertical high and medium CORE count */ static int hd_online_cores; /* Current online CORE count */ static unsigned long hd_previous_steal; /* Previous iteration's CPU steal timer total */ +static unsigned long hd_high_time; /* Total time spent while all cpus have high capacity */ +static unsigned long hd_low_time; /* Total time spent while vl cpus have low capacity */ +static atomic64_t hd_adjustments; /* Total occurrence count of hiperdispatch adjustments */ static unsigned int hd_steal_threshold = HD_STEAL_THRESHOLD; static unsigned int hd_delay_factor = HD_DELAY_FACTOR; @@ -121,6 +126,33 @@ void hd_add_core(int cpu) } } +/* Serialize update and read operations of debug counters. */ +static DEFINE_MUTEX(hd_counter_mutex); + +static void hd_update_times(void) +{ + static ktime_t prev; + ktime_t now; + + /* + * Check if hiperdispatch is active, if not set the prev to 0. + * This way it is possible to differentiate the first update iteration after + * enabling hiperdispatch. + */ + if (hd_entitled_cores == 0 || hd_enabled == 0) { + prev = ktime_set(0, 0); + return; + } + now = ktime_get(); + if (ktime_after(prev, 0)) { + if (hd_high_capacity_cores == hd_online_cores) + hd_high_time += ktime_ms_delta(now, prev); + else + hd_low_time += ktime_ms_delta(now, prev); + } + prev = now; +} + static void hd_update_capacities(void) { int cpu, upscaling_cores; @@ -149,6 +181,9 @@ void hd_disable_hiperdispatch(void) int hd_enable_hiperdispatch(void) { + mutex_lock(&hd_counter_mutex); + hd_update_times(); + mutex_unlock(&hd_counter_mutex); if (hd_enabled == 0) return 0; if (hd_entitled_cores == 0) @@ -225,6 +260,7 @@ static void hd_capacity_work_fn(struct work_struct *work) if (hd_high_capacity_cores != new_cores) { trace_s390_hd_rebuild_domains(hd_high_capacity_cores, new_cores); hd_high_capacity_cores = new_cores; + atomic64_inc(&hd_adjustments); topology_schedule_update(); } trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores); @@ -327,6 +363,46 @@ static const struct attribute_group hd_attr_group = { .attrs = hd_attrs, }; +static int hd_greedy_time_get(void *unused, u64 *val) +{ + mutex_lock(&hd_counter_mutex); + hd_update_times(); + *val = hd_high_time; + mutex_unlock(&hd_counter_mutex); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(hd_greedy_time_fops, hd_greedy_time_get, NULL, "%llu\n"); + +static int hd_conservative_time_get(void *unused, u64 *val) +{ + mutex_lock(&hd_counter_mutex); + hd_update_times(); + *val = hd_low_time; + mutex_unlock(&hd_counter_mutex); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(hd_conservative_time_fops, hd_conservative_time_get, NULL, "%llu\n"); + +static int hd_adjustment_count_get(void *unused, u64 *val) +{ + *val = atomic64_read(&hd_adjustments); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(hd_adjustments_fops, hd_adjustment_count_get, NULL, "%llu\n"); + +static void __init hd_create_debugfs_counters(void) +{ + struct dentry *dir; + + dir = debugfs_create_dir("hiperdispatch", arch_debugfs_dir); + debugfs_create_file("conservative_time_ms", 0400, dir, NULL, &hd_conservative_time_fops); + debugfs_create_file("greedy_time_ms", 0400, dir, NULL, &hd_greedy_time_fops); + debugfs_create_file("adjustment_count", 0400, dir, NULL, &hd_adjustments_fops); +} + static void __init hd_create_attributes(void) { struct device *dev; @@ -347,6 +423,7 @@ static int __init hd_init(void) } if (!register_sysctl("s390", hiperdispatch_ctl_table)) pr_warn("Failed to register s390.hiperdispatch sysctl attribute\n"); + hd_create_debugfs_counters(); hd_create_attributes(); return 0; } From 4ae48555d0edcec910f283868cf454720f0f0623 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Fri, 23 Aug 2024 10:36:49 +0200 Subject: [PATCH 65/84] s390/pai_crypto: Add support for MSA 10 and 11 pai counters Update the internal array of PAI crypto counter string table with new counters supported with Message Security Assist extension (MSA) 10 and MSA 11. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Acked-by: Finn Callies Tested-by: Finn Callies Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 2f5a20e300f6..fa7325454266 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -738,6 +738,22 @@ static const char * const paicrypt_ctrnames[] = { [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY", [155] = "IBM_RESERVED_155", [156] = "IBM_RESERVED_156", + [157] = "KM_FULL_XTS_AES_128", + [158] = "KM_FULL_XTS_AES_256", + [159] = "KM_FULL_XTS_ENCRYPTED_AES_128", + [160] = "KM_FULL_XTS_ENCRYPTED_AES_256", + [161] = "KMAC_HMAC_SHA_224", + [162] = "KMAC_HMAC_SHA_256", + [163] = "KMAC_HMAC_SHA_384", + [164] = "KMAC_HMAC_SHA_512", + [165] = "KMAC_HMAC_ENCRYPTED_SHA_224", + [166] = "KMAC_HMAC_ENCRYPTED_SHA_256", + [167] = "KMAC_HMAC_ENCRYPTED_SHA_384", + [168] = "KMAC_HMAC_ENCRYPTED_SHA_512", + [169] = "PCKMO_ENCRYPT_HMAC_512_KEY", + [170] = "PCKMO_ENCRYPT_HMAC_1024_KEY", + [171] = "PCKMO_ENCRYPT_AES_XTS_128", + [172] = "PCKMO_ENCRYPT_AES_XTS_256", }; static void __init attr_event_free(struct attribute **attrs, int num) From 0114009953c119021870c42c778b251440a93844 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Fri, 23 Aug 2024 10:36:50 +0200 Subject: [PATCH 66/84] s390/pai_ext: Update PAI extension 1 counters Update the internal array of PAI extension 1 NNPA counter string table to support specialized processor instrumentation assist instructions. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_ext.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 6295531b39a2..7f462bef1fc0 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -635,6 +635,15 @@ static const char * const paiext_ctrnames[] = { [25] = "NNPA_1MFRAME", [26] = "NNPA_2GFRAME", [27] = "NNPA_ACCESSEXCEPT", + [28] = "NNPA_TRANSFORM", + [29] = "NNPA_GELU", + [30] = "NNPA_MOMENTS", + [31] = "NNPA_LAYERNORM", + [32] = "NNPA_MATMUL_OP_BCAST1", + [33] = "NNPA_SQRT", + [34] = "NNPA_INVSQRT", + [35] = "NNPA_NORM", + [36] = "NNPA_REDUCE", }; static void __init attr_event_free(struct attribute **attrs, int num) From 131b8db78558120f58c5dc745ea9655f6b854162 Mon Sep 17 00:00:00 2001 From: Gerald Schaefer Date: Mon, 2 Sep 2024 14:02:19 +0200 Subject: [PATCH 67/84] s390/mm: Add cond_resched() to cmm_alloc/free_pages() Adding/removing large amount of pages at once to/from the CMM balloon can result in rcu_sched stalls or workqueue lockups, because of busy looping w/o cond_resched(). Prevent this by adding a cond_resched(). cmm_free_pages() holds a spin_lock while looping, so it cannot be added directly to the existing loop. Instead, introduce a wrapper function that operates on maximum 256 pages at once, and add it there. Signed-off-by: Gerald Schaefer Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/mm/cmm.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/s390/mm/cmm.c b/arch/s390/mm/cmm.c index 75d15bf41d97..d01724a715d0 100644 --- a/arch/s390/mm/cmm.c +++ b/arch/s390/mm/cmm.c @@ -95,11 +95,12 @@ static long cmm_alloc_pages(long nr, long *counter, (*counter)++; spin_unlock(&cmm_lock); nr--; + cond_resched(); } return nr; } -static long cmm_free_pages(long nr, long *counter, struct cmm_page_array **list) +static long __cmm_free_pages(long nr, long *counter, struct cmm_page_array **list) { struct cmm_page_array *pa; unsigned long addr; @@ -123,6 +124,21 @@ static long cmm_free_pages(long nr, long *counter, struct cmm_page_array **list) return nr; } +static long cmm_free_pages(long nr, long *counter, struct cmm_page_array **list) +{ + long inc = 0; + + while (nr) { + inc = min(256L, nr); + nr -= inc; + inc = __cmm_free_pages(inc, counter, list); + if (inc) + break; + cond_resched(); + } + return nr + inc; +} + static int cmm_oom_notify(struct notifier_block *self, unsigned long dummy, void *parm) { From 8fe32188f931a36ad0d444d653ee05b11bedc849 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 2 Sep 2024 15:19:43 +0200 Subject: [PATCH 68/84] s390/cpacf: Add MSA 10 and 11 new PCKMO functions Add the defines for the new PCKMO functions covering MSA 10 (AES XTS "double" keys) and MSA 11 (HMAC keys) support. Signed-off-by: Harald Freudenberger Reviewed-by: Ingo Franzki Signed-off-by: Heiko Carstens --- arch/s390/include/asm/cpacf.h | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index 4b6a8c26e1ce..b8a98e6ffb4d 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -132,18 +132,22 @@ * Function codes for the PCKMO (PERFORM CRYPTOGRAPHIC KEY MANAGEMENT) * instruction */ -#define CPACF_PCKMO_QUERY 0x00 -#define CPACF_PCKMO_ENC_DES_KEY 0x01 -#define CPACF_PCKMO_ENC_TDES_128_KEY 0x02 -#define CPACF_PCKMO_ENC_TDES_192_KEY 0x03 -#define CPACF_PCKMO_ENC_AES_128_KEY 0x12 -#define CPACF_PCKMO_ENC_AES_192_KEY 0x13 -#define CPACF_PCKMO_ENC_AES_256_KEY 0x14 -#define CPACF_PCKMO_ENC_ECC_P256_KEY 0x20 -#define CPACF_PCKMO_ENC_ECC_P384_KEY 0x21 -#define CPACF_PCKMO_ENC_ECC_P521_KEY 0x22 -#define CPACF_PCKMO_ENC_ECC_ED25519_KEY 0x28 -#define CPACF_PCKMO_ENC_ECC_ED448_KEY 0x29 +#define CPACF_PCKMO_QUERY 0x00 +#define CPACF_PCKMO_ENC_DES_KEY 0x01 +#define CPACF_PCKMO_ENC_TDES_128_KEY 0x02 +#define CPACF_PCKMO_ENC_TDES_192_KEY 0x03 +#define CPACF_PCKMO_ENC_AES_128_KEY 0x12 +#define CPACF_PCKMO_ENC_AES_192_KEY 0x13 +#define CPACF_PCKMO_ENC_AES_256_KEY 0x14 +#define CPACF_PCKMO_ENC_AES_XTS_128_DOUBLE_KEY 0x15 +#define CPACF_PCKMO_ENC_AES_XTS_256_DOUBLE_KEY 0x16 +#define CPACF_PCKMO_ENC_ECC_P256_KEY 0x20 +#define CPACF_PCKMO_ENC_ECC_P384_KEY 0x21 +#define CPACF_PCKMO_ENC_ECC_P521_KEY 0x22 +#define CPACF_PCKMO_ENC_ECC_ED25519_KEY 0x28 +#define CPACF_PCKMO_ENC_ECC_ED448_KEY 0x29 +#define CPACF_PCKMO_ENC_HMAC_512_KEY 0x76 +#define CPACF_PCKMO_ENC_HMAC_1024_KEY 0x7a /* * Function codes for the PRNO (PERFORM RANDOM NUMBER OPERATION) From fd197556eef50d9c1e27cefd5bfa6df1ca0cc854 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 2 Sep 2024 15:19:44 +0200 Subject: [PATCH 69/84] s390/pkey: Add AES xts and HMAC clear key token support Add support for deriving protected keys from clear key token for AES xts and HMAC keys via PCKMO instruction. Add support for protected key generation and unwrap of protected key tokens for these key types. Furthermore 4 new sysfs attributes are introduced: - /sys/devices/virtual/misc/pkey/protkey/protkey_aes_xts_128 - /sys/devices/virtual/misc/pkey/protkey/protkey_aes_xts_256 - /sys/devices/virtual/misc/pkey/protkey/protkey_hmac_512 - /sys/devices/virtual/misc/pkey/protkey/protkey_hmac_1024 Signed-off-by: Harald Freudenberger Reviewed-by: Ingo Franzki Signed-off-by: Heiko Carstens --- arch/s390/include/uapi/asm/pkey.h | 4 + drivers/s390/crypto/pkey_base.h | 15 +++- drivers/s390/crypto/pkey_pckmo.c | 102 ++++++++++++++++++++--- drivers/s390/crypto/pkey_sysfs.c | 132 ++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 13 deletions(-) diff --git a/arch/s390/include/uapi/asm/pkey.h b/arch/s390/include/uapi/asm/pkey.h index 04183080cdbb..60431d00e6bd 100644 --- a/arch/s390/include/uapi/asm/pkey.h +++ b/arch/s390/include/uapi/asm/pkey.h @@ -41,6 +41,10 @@ #define PKEY_KEYTYPE_ECC_P521 7 #define PKEY_KEYTYPE_ECC_ED25519 8 #define PKEY_KEYTYPE_ECC_ED448 9 +#define PKEY_KEYTYPE_AES_XTS_128 10 +#define PKEY_KEYTYPE_AES_XTS_256 11 +#define PKEY_KEYTYPE_HMAC_512 12 +#define PKEY_KEYTYPE_HMAC_1024 13 /* the newer ioctls use a pkey_key_type enum for type information */ enum pkey_key_type { diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h index fd151903fd06..7a1a5ce192d8 100644 --- a/drivers/s390/crypto/pkey_base.h +++ b/drivers/s390/crypto/pkey_base.h @@ -33,11 +33,22 @@ extern debug_info_t *pkey_dbf_info; #define MAXAPQNSINLIST 64 /* max 64 apqns within a apqn list */ #define AES_WK_VP_SIZE 32 /* Size of WK VP block appended to a prot key */ -/* inside view of a protected key token (only type 0x00 version 0x01) */ +/* inside view of a generic protected key token */ +struct protkeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* should be 0x01 for protected key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ + u32 len; /* bytes actually stored in protkey[] */ + u8 protkey[]; /* the protected key blob */ +} __packed; + +/* inside view of a protected AES key token */ struct protaeskeytoken { u8 type; /* 0x00 for PAES specific key tokens */ u8 res0[3]; - u8 version; /* should be 0x01 for protected AES key token */ + u8 version; /* should be 0x01 for protected key token */ u8 res1[3]; u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ u32 len; /* bytes actually stored in protkey[] */ diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c index 0667e5510671..98079b1ed6db 100644 --- a/drivers/s390/crypto/pkey_pckmo.c +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -47,6 +47,10 @@ static bool is_pckmo_key(const u8 *key, u32 keylen) case PKEY_KEYTYPE_ECC_P521: case PKEY_KEYTYPE_ECC_ED25519: case PKEY_KEYTYPE_ECC_ED448: + case PKEY_KEYTYPE_AES_XTS_128: + case PKEY_KEYTYPE_AES_XTS_256: + case PKEY_KEYTYPE_HMAC_512: + case PKEY_KEYTYPE_HMAC_1024: return true; default: return false; @@ -81,7 +85,7 @@ static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen, static cpacf_mask_t pckmo_functions; int keysize, rc = -EINVAL; - u8 paramblock[112]; + u8 paramblock[160]; u32 pkeytype; long fc; @@ -134,6 +138,30 @@ static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen, pkeytype = PKEY_KEYTYPE_ECC; fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; break; + case PKEY_KEYTYPE_AES_XTS_128: + /* 2x16 byte keys, 32 byte aes wkvp, total 64 bytes */ + keysize = 32; + pkeytype = PKEY_KEYTYPE_AES_XTS_128; + fc = CPACF_PCKMO_ENC_AES_XTS_128_DOUBLE_KEY; + break; + case PKEY_KEYTYPE_AES_XTS_256: + /* 2x32 byte keys, 32 byte aes wkvp, total 96 bytes */ + keysize = 64; + pkeytype = PKEY_KEYTYPE_AES_XTS_256; + fc = CPACF_PCKMO_ENC_AES_XTS_256_DOUBLE_KEY; + break; + case PKEY_KEYTYPE_HMAC_512: + /* 64 byte key, 32 byte aes wkvp, total 96 bytes */ + keysize = 64; + pkeytype = PKEY_KEYTYPE_HMAC_512; + fc = CPACF_PCKMO_ENC_HMAC_512_KEY; + break; + case PKEY_KEYTYPE_HMAC_1024: + /* 128 byte key, 32 byte aes wkvp, total 160 bytes */ + keysize = 128; + pkeytype = PKEY_KEYTYPE_HMAC_1024; + fc = CPACF_PCKMO_ENC_HMAC_1024_KEY; + break; default: PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__, keytype); @@ -260,14 +288,39 @@ static int pckmo_key2protkey(const u8 *key, u32 keylen, switch (hdr->version) { case TOKVER_PROTECTED_KEY: { - struct protaeskeytoken *t; + struct protkeytoken *t = (struct protkeytoken *)key; - if (keylen != sizeof(struct protaeskeytoken)) + if (keylen < sizeof(*t)) goto out; - t = (struct protaeskeytoken *)key; - rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype); - if (rc) + switch (t->keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + if (keylen != sizeof(struct protaeskeytoken)) + goto out; + rc = pckmo_verify_protkey(t->protkey, t->len, + t->keytype); + if (rc) + goto out; + break; + case PKEY_KEYTYPE_AES_XTS_128: + if (t->len != 64 || keylen != sizeof(*t) + t->len) + goto out; + break; + case PKEY_KEYTYPE_AES_XTS_256: + case PKEY_KEYTYPE_HMAC_512: + if (t->len != 96 || keylen != sizeof(*t) + t->len) + goto out; + break; + case PKEY_KEYTYPE_HMAC_1024: + if (t->len != 160 || keylen != sizeof(*t) + t->len) + goto out; + break; + default: + PKEY_DBF_ERR("%s protected key token: unknown keytype %u\n", + __func__, t->keytype); goto out; + } memcpy(protkey, t->protkey, t->len); *protkeylen = t->len; *protkeytype = t->keytype; @@ -301,6 +354,18 @@ static int pckmo_key2protkey(const u8 *key, u32 keylen, case PKEY_KEYTYPE_ECC_ED448: keysize = 64; break; + case PKEY_KEYTYPE_AES_XTS_128: + keysize = 32; + break; + case PKEY_KEYTYPE_AES_XTS_256: + keysize = 64; + break; + case PKEY_KEYTYPE_HMAC_512: + keysize = 64; + break; + case PKEY_KEYTYPE_HMAC_1024: + keysize = 128; + break; default: break; } @@ -337,14 +402,29 @@ out: static int pckmo_gen_protkey(u32 keytype, u32 subtype, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { - u8 clrkey[32]; + u8 clrkey[128]; int keysize; int rc; - keysize = pkey_keytype_aes_to_size(keytype); - if (!keysize) { - PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, - keytype); + switch (keytype) { + case PKEY_KEYTYPE_AES_128: + case PKEY_KEYTYPE_AES_192: + case PKEY_KEYTYPE_AES_256: + keysize = pkey_keytype_aes_to_size(keytype); + break; + case PKEY_KEYTYPE_AES_XTS_128: + keysize = 32; + break; + case PKEY_KEYTYPE_AES_XTS_256: + case PKEY_KEYTYPE_HMAC_512: + keysize = 64; + break; + case PKEY_KEYTYPE_HMAC_1024: + keysize = 128; + break; + default: + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", + __func__, keytype); return -EINVAL; } if (subtype != PKEY_TYPE_PROTKEY) { diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c index 56205f53f77b..cc0fc1e264bd 100644 --- a/drivers/s390/crypto/pkey_sysfs.c +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -99,6 +99,90 @@ static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf, return sizeof(protkeytoken); } +/* + * Sysfs attribute read function for the AES XTS prot key binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_protkey_aes_xts_attr_read(u32 keytype, char *buf, + loff_t off, size_t count) +{ + struct protkeytoken *t = (struct protkeytoken *)buf; + u32 protlen, prottype; + int rc; + + switch (keytype) { + case PKEY_KEYTYPE_AES_XTS_128: + protlen = 64; + break; + case PKEY_KEYTYPE_AES_XTS_256: + protlen = 96; + break; + default: + return -EINVAL; + } + + if (off != 0 || count < sizeof(*t) + protlen) + return -EINVAL; + + memset(t, 0, sizeof(*t) + protlen); + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_PROTECTED_KEY; + t->keytype = keytype; + + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + t->protkey, &protlen, &prottype); + if (rc) + return rc; + + t->len = protlen; + + return sizeof(*t) + protlen; +} + +/* + * Sysfs attribute read function for the HMAC prot key binary attributes. + * The implementation can not deal with partial reads, because a new random + * protected key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + */ +static ssize_t pkey_protkey_hmac_attr_read(u32 keytype, char *buf, + loff_t off, size_t count) +{ + struct protkeytoken *t = (struct protkeytoken *)buf; + u32 protlen, prottype; + int rc; + + switch (keytype) { + case PKEY_KEYTYPE_HMAC_512: + protlen = 96; + break; + case PKEY_KEYTYPE_HMAC_1024: + protlen = 160; + break; + default: + return -EINVAL; + } + + if (off != 0 || count < sizeof(*t) + protlen) + return -EINVAL; + + memset(t, 0, sizeof(*t) + protlen); + t->type = TOKTYPE_NON_CCA; + t->version = TOKVER_PROTECTED_KEY; + t->keytype = keytype; + + rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0, + t->protkey, &protlen, &prottype); + if (rc) + return rc; + + t->len = protlen; + + return sizeof(*t) + protlen; +} + static ssize_t protkey_aes_128_read(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, @@ -149,11 +233,55 @@ static ssize_t protkey_aes_256_xts_read(struct file *filp, off, count); } +static ssize_t protkey_aes_xts_128_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_xts_attr_read(PKEY_KEYTYPE_AES_XTS_128, + buf, off, count); +} + +static ssize_t protkey_aes_xts_256_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_aes_xts_attr_read(PKEY_KEYTYPE_AES_XTS_256, + buf, off, count); +} + +static ssize_t protkey_hmac_512_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_hmac_attr_read(PKEY_KEYTYPE_HMAC_512, + buf, off, count); +} + +static ssize_t protkey_hmac_1024_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_protkey_hmac_attr_read(PKEY_KEYTYPE_HMAC_1024, + buf, off, count); +} + static BIN_ATTR_RO(protkey_aes_128, sizeof(struct protaeskeytoken)); static BIN_ATTR_RO(protkey_aes_192, sizeof(struct protaeskeytoken)); static BIN_ATTR_RO(protkey_aes_256, sizeof(struct protaeskeytoken)); static BIN_ATTR_RO(protkey_aes_128_xts, 2 * sizeof(struct protaeskeytoken)); static BIN_ATTR_RO(protkey_aes_256_xts, 2 * sizeof(struct protaeskeytoken)); +static BIN_ATTR_RO(protkey_aes_xts_128, sizeof(struct protkeytoken) + 64); +static BIN_ATTR_RO(protkey_aes_xts_256, sizeof(struct protkeytoken) + 96); +static BIN_ATTR_RO(protkey_hmac_512, sizeof(struct protkeytoken) + 96); +static BIN_ATTR_RO(protkey_hmac_1024, sizeof(struct protkeytoken) + 160); static struct bin_attribute *protkey_attrs[] = { &bin_attr_protkey_aes_128, @@ -161,6 +289,10 @@ static struct bin_attribute *protkey_attrs[] = { &bin_attr_protkey_aes_256, &bin_attr_protkey_aes_128_xts, &bin_attr_protkey_aes_256_xts, + &bin_attr_protkey_aes_xts_128, + &bin_attr_protkey_aes_xts_256, + &bin_attr_protkey_hmac_512, + &bin_attr_protkey_hmac_1024, NULL }; From 992b7066800f3957e40b3a9d5ed4f041b998fec1 Mon Sep 17 00:00:00 2001 From: Ingo Franzki Date: Wed, 4 Sep 2024 14:44:16 +0200 Subject: [PATCH 70/84] s390/sha3: Fix SHA3 selftests failures Since commit "s390/sha3: Support sha3 performance enhancements" the selftests of the sha3_256_s390 and sha3_512_s390 kernel digests sometimes fail with: alg: shash: sha3-256-s390 test failed (wrong result) on test vector 3, cfg="import/export" alg: self-tests for sha3-256 using sha3-256-s390 failed (rc=-22) or with alg: ahash: sha3-256-s390 test failed (wrong result) on test vector 3, cfg="digest misaligned splits crossing pages" alg: self-tests for sha3-256 using sha3-256-s390 failed (rc=-22) The first failure is because the newly introduced context field 'first_message_part' is not copied during export and import operations. Because of that the value of 'first_message_part' is more or less random after an import into a newly allocated context and may or may not fit to the state of the imported SHA3 operation, causing an invalid hash when it does not fit. Save the 'first_message_part' field in the currently unused field 'partial' of struct sha3_state, even though the meaning of 'partial' is not exactly the same as 'first_message_part'. For the caller the returned state blob is opaque and it must only be ensured that the state can be imported later on by the module that exported it. The second failure is when on entry of s390_sha_update() the flag 'first_message_part' is on, and kimd is called in the first 'if (index)' block as well as in the second 'if (len >= bsize)' block. In this case, the 'first_message_part' is turned off after the first kimd, but the function code incorrectly retains the NIP flag. Reset the NIP flag after the first kimd unconditionally besides turning 'first_message_part' off. Reported-by: Marc Hartmayer Fixes: 88c02b3f79a6 ("s390/sha3: Support sha3 performance enhancements") Reviewed-by: Harald Freudenberger Reviewed-by: Holger Dengler Reviewed-by: Joerg Schmidbauer Signed-off-by: Ingo Franzki Signed-off-by: Heiko Carstens --- arch/s390/crypto/sha3_256_s390.c | 3 +++ arch/s390/crypto/sha3_512_s390.c | 3 +++ arch/s390/crypto/sha_common.c | 1 + 3 files changed, 7 insertions(+) diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index 5bba972a2646..a84ef692f572 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -38,6 +38,7 @@ static int sha3_256_export(struct shash_desc *desc, void *out) octx->rsiz = sctx->count; memcpy(octx->st, sctx->state, sizeof(octx->st)); memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); + octx->partial = sctx->first_message_part; return 0; } @@ -50,6 +51,7 @@ static int sha3_256_import(struct shash_desc *desc, const void *in) sctx->count = ictx->rsiz; memcpy(sctx->state, ictx->st, sizeof(ictx->st)); memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); + sctx->first_message_part = ictx->partial; sctx->func = CPACF_KIMD_SHA3_256; return 0; @@ -63,6 +65,7 @@ static int sha3_224_import(struct shash_desc *desc, const void *in) sctx->count = ictx->rsiz; memcpy(sctx->state, ictx->st, sizeof(ictx->st)); memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); + sctx->first_message_part = ictx->partial; sctx->func = CPACF_KIMD_SHA3_224; return 0; diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 3554cbbafe9c..07528fc98ff7 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -39,6 +39,7 @@ static int sha3_512_export(struct shash_desc *desc, void *out) memcpy(octx->st, sctx->state, sizeof(octx->st)); memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); + octx->partial = sctx->first_message_part; return 0; } @@ -54,6 +55,7 @@ static int sha3_512_import(struct shash_desc *desc, const void *in) memcpy(sctx->state, ictx->st, sizeof(ictx->st)); memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); + sctx->first_message_part = ictx->partial; sctx->func = CPACF_KIMD_SHA3_512; return 0; @@ -70,6 +72,7 @@ static int sha3_384_import(struct shash_desc *desc, const void *in) memcpy(sctx->state, ictx->st, sizeof(ictx->st)); memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); + sctx->first_message_part = ictx->partial; sctx->func = CPACF_KIMD_SHA3_384; return 0; diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index f6c7fda21abc..961d7d522af1 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -36,6 +36,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) memcpy(ctx->buf + index, data, bsize - index); cpacf_kimd(fc, ctx->state, ctx->buf, bsize); ctx->first_message_part = 0; + fc &= ~CPACF_KIMD_NIP; data += bsize - index; len -= bsize - index; index = 0; From 6fa7aea6a9fc8eb49d879ac66bf27460e70c83b3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:22 +0200 Subject: [PATCH 71/84] s390/als: Remove obsolete comment The bss section of the decompressor is part of the compressed kernel image since commit 980d5f9ab36b ("s390/boot: enable .bss section for compressed kernel"). Remove a now incorrect comment that states that the bss section must not be accessed. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/boot/als.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/arch/s390/boot/als.c b/arch/s390/boot/als.c index 47c48fbfb563..efb6caa89e03 100644 --- a/arch/s390/boot/als.c +++ b/arch/s390/boot/als.c @@ -9,14 +9,6 @@ #include #include "boot.h" -/* - * The code within this file will be called very early. It may _not_ - * access anything within the bss section, since that is not cleared - * yet and may contain data (e.g. initrd) that must be saved by other - * code. - * For temporary objects the stack (16k) should be used. - */ - static unsigned long als[] = { FACILITIES_ALS }; static void u16_to_hex(char *str, u16 val) From db545f5387472176cf1e7df76fe97a4f56b7158d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:23 +0200 Subject: [PATCH 72/84] s390/boot: Increase minimum architecture to z10 The decompressor code is partially compiled with march=z900 so it is possible to print an error message in case a kernel is booted on a machine which misses facilities to execute the kernel. Given that the decompressor code also includes header files from the core kernel this causes problems for inline assemblies and other code where the minimum assumed architecture level is set to z10 in the meantime. If such code is also used in the decompressor (e.g. inline functions) z900 support must be implemented again. In order to avoid this and to keep things simple just raise the minimum architecture level to z10 for the decompressor just like for the kernel. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/boot/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index b2252bb15445..df01e7be1560 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -18,11 +18,7 @@ KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) # Use minimum architecture for als.c to be able to print an error # message if the kernel is started on a machine which is too old # -ifndef CONFIG_CC_IS_CLANG -CC_FLAGS_MARCH_MINIMUM := -march=z900 -else CC_FLAGS_MARCH_MINIMUM := -march=z10 -endif ifneq ($(CC_FLAGS_MARCH),$(CC_FLAGS_MARCH_MINIMUM)) AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) From 0147addc4fb72a39448b8873d8acdf3a0f29aa65 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:24 +0200 Subject: [PATCH 73/84] s390/facility: Disable compile time optimization for decompressor code Disable compile time optimizations of test_facility() for the decompressor. The decompressor should not contain any optimized code depending on the architecture level set the kernel image is compiled for to avoid unexpected operation exceptions. Add a __DECOMPRESSOR check to test_facility() to enforce that facilities are always checked during runtime for the decompressor. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/facility.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h index b7d234838a36..65ebf86506cd 100644 --- a/arch/s390/include/asm/facility.h +++ b/arch/s390/include/asm/facility.h @@ -59,8 +59,10 @@ static inline int test_facility(unsigned long nr) unsigned long facilities_als[] = { FACILITIES_ALS }; if (__builtin_constant_p(nr) && nr < sizeof(facilities_als) * 8) { - if (__test_facility(nr, &facilities_als)) - return 1; + if (__test_facility(nr, &facilities_als)) { + if (!__is_defined(__DECOMPRESSOR)) + return 1; + } } return __test_facility(nr, &stfle_fac_list); } From 697b37371f4af8b237f47cd4aa4a2255a273b4ce Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:25 +0200 Subject: [PATCH 74/84] s390: Provide MARCH_HAS_*_FEATURES defines Provide MARCH_HAS_*_FEATURES defines which are supposed to be used everywhere instead of the CONFIG_HAVE_MARCH_*_FEATURES defines. Various header files contain code which depend on the CONFIG_HAVE_MARCH_*_FEATURES defines, allowing for compile time optimizations. If such code is used within the decompressor wrong code may be generated (the compiler may generate instructions which are not available for the minimum architecture level of the decompressor). Therefore provide a new header file with MARCH_HAS_*_FEATURES defines, which are only available if __DECOMPRESSOR is not defined. This way code generation for the kernel image is still optimized depending on CONFIG_HAVE_MARCH_*_FEATURES, while code generated for the decompressor is compiled for the minimum architecture level. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/march.h | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 arch/s390/include/asm/march.h diff --git a/arch/s390/include/asm/march.h b/arch/s390/include/asm/march.h new file mode 100644 index 000000000000..fd9eef3be44c --- /dev/null +++ b/arch/s390/include/asm/march.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_S390_MARCH_H +#define __ASM_S390_MARCH_H + +#include + +#define MARCH_HAS_Z10_FEATURES 1 + +#ifndef __DECOMPRESSOR + +#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES +#define MARCH_HAS_Z196_FEATURES 1 +#endif + +#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES +#define MARCH_HAS_ZEC12_FEATURES 1 +#endif + +#ifdef CONFIG_HAVE_MARCH_Z13_FEATURES +#define MARCH_HAS_Z13_FEATURES 1 +#endif + +#ifdef CONFIG_HAVE_MARCH_Z14_FEATURES +#define MARCH_HAS_Z14_FEATURES 1 +#endif + +#ifdef CONFIG_HAVE_MARCH_Z15_FEATURES +#define MARCH_HAS_Z15_FEATURES 1 +#endif + +#ifdef CONFIG_HAVE_MARCH_Z16_FEATURES +#define MARCH_HAS_Z16_FEATURES 1 +#endif + +#endif /* __DECOMPRESSOR */ + +#endif /* __ASM_S390_MARCH_H */ From ebcc369f18919c92aeca2003618fcfeade04121b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:26 +0200 Subject: [PATCH 75/84] s390: Use MARCH_HAS_*_FEATURES defines Replace CONFIG_HAVE_MARCH_*_FEATURES with MARCH_HAS_*_FEATURES everywhere so code gets compiled correctly depending on if the target is the kernel or the decompressor. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/arch_hweight.h | 15 ++++++++------- arch/s390/include/asm/atomic_ops.h | 7 ++++--- arch/s390/include/asm/barrier.h | 4 +++- arch/s390/include/asm/percpu.h | 7 ++++--- arch/s390/include/asm/preempt.h | 7 ++++--- arch/s390/kernel/mcount.S | 5 +++-- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/arch/s390/include/asm/arch_hweight.h b/arch/s390/include/asm/arch_hweight.h index 50e23ce854e5..aca08b0acbc1 100644 --- a/arch/s390/include/asm/arch_hweight.h +++ b/arch/s390/include/asm/arch_hweight.h @@ -4,6 +4,7 @@ #define _ASM_S390_ARCH_HWEIGHT_H #include +#include static __always_inline unsigned long popcnt_z196(unsigned long w) { @@ -29,9 +30,9 @@ static __always_inline unsigned long popcnt_z15(unsigned long w) static __always_inline unsigned long __arch_hweight64(__u64 w) { - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES)) + if (__is_defined(MARCH_HAS_Z15_FEATURES)) return popcnt_z15(w); - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) { + if (__is_defined(MARCH_HAS_Z196_FEATURES)) { w = popcnt_z196(w); w += w >> 32; w += w >> 16; @@ -43,9 +44,9 @@ static __always_inline unsigned long __arch_hweight64(__u64 w) static __always_inline unsigned int __arch_hweight32(unsigned int w) { - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES)) + if (__is_defined(MARCH_HAS_Z15_FEATURES)) return popcnt_z15(w); - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) { + if (__is_defined(MARCH_HAS_Z196_FEATURES)) { w = popcnt_z196(w); w += w >> 16; w += w >> 8; @@ -56,9 +57,9 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) static __always_inline unsigned int __arch_hweight16(unsigned int w) { - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES)) + if (__is_defined(MARCH_HAS_Z15_FEATURES)) return popcnt_z15((unsigned short)w); - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) { + if (__is_defined(MARCH_HAS_Z196_FEATURES)) { w = popcnt_z196(w); w += w >> 8; return w & 0xff; @@ -68,7 +69,7 @@ static __always_inline unsigned int __arch_hweight16(unsigned int w) static __always_inline unsigned int __arch_hweight8(unsigned int w) { - if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) + if (__is_defined(MARCH_HAS_Z196_FEATURES)) return popcnt_z196((unsigned char)w); return __sw_hweight8(w); } diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h index 742c7919cbcd..65380da9e75f 100644 --- a/arch/s390/include/asm/atomic_ops.h +++ b/arch/s390/include/asm/atomic_ops.h @@ -9,6 +9,7 @@ #define __ARCH_S390_ATOMIC_OPS__ #include +#include static __always_inline int __atomic_read(const atomic_t *v) { @@ -56,7 +57,7 @@ static __always_inline void __atomic64_set(atomic64_t *v, s64 i) } } -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES +#ifdef MARCH_HAS_Z196_FEATURES #define __ATOMIC_OP(op_name, op_type, op_string, op_barrier) \ static __always_inline op_type op_name(op_type val, op_type *ptr) \ @@ -107,7 +108,7 @@ __ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi") #undef __ATOMIC_CONST_OPS #undef __ATOMIC_CONST_OP -#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ +#else /* MARCH_HAS_Z196_FEATURES */ #define __ATOMIC_OP(op_name, op_string) \ static __always_inline int op_name(int val, int *ptr) \ @@ -166,7 +167,7 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr") #define __atomic64_add_const(val, ptr) __atomic64_add(val, ptr) #define __atomic64_add_const_barrier(val, ptr) __atomic64_add(val, ptr) -#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ +#endif /* MARCH_HAS_Z196_FEATURES */ static __always_inline int __atomic_cmpxchg(int *ptr, int old, int new) { diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index 82de2a7c4160..d82130d7f2b6 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -8,13 +8,15 @@ #ifndef __ASM_BARRIER_H #define __ASM_BARRIER_H +#include + /* * Force strict CPU ordering. * And yes, this is required on UP too when we're talking * to devices. */ -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES +#ifdef MARCH_HAS_Z196_FEATURES /* Fast-BCR without checkpoint synchronization */ #define __ASM_BCR_SERIALIZE "bcr 14,0\n" #else diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 89a28740b6ab..84f6b8357b45 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -4,6 +4,7 @@ #include #include +#include /* * s390 uses its own implementation for per cpu data, the offset of @@ -50,7 +51,7 @@ #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) -#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES +#ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -61,7 +62,7 @@ #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) -#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ +#else /* MARCH_HAS_Z196_FEATURES */ #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ { \ @@ -129,7 +130,7 @@ #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") -#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ +#endif /* MARCH_HAS_Z196_FEATURES */ #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ ({ \ diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 3ae5f31c665d..deca3f221836 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -5,8 +5,9 @@ #include #include #include +#include -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES +#ifdef MARCH_HAS_Z196_FEATURES /* We use the MSB mostly because its available */ #define PREEMPT_NEED_RESCHED 0x80000000 @@ -75,7 +76,7 @@ static __always_inline bool should_resched(int preempt_offset) preempt_offset); } -#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ +#else /* MARCH_HAS_Z196_FEATURES */ #define PREEMPT_ENABLED (0) @@ -123,7 +124,7 @@ static __always_inline bool should_resched(int preempt_offset) tif_need_resched()); } -#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ +#endif /* MARCH_HAS_Z196_FEATURES */ #define init_task_preempt_count(p) do { } while (0) /* Deferred to CPU bringup time */ diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index ae4d4fd9afcd..7e267ef63a7f 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -9,6 +9,7 @@ #include #include #include +#include #define STACK_FRAME_SIZE_PTREGS (STACK_FRAME_OVERHEAD + __PT_SIZE) #define STACK_PTREGS (STACK_FRAME_OVERHEAD) @@ -88,7 +89,7 @@ SYM_CODE_START(ftrace_caller) SYM_CODE_END(ftrace_caller) SYM_CODE_START(ftrace_common) -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES +#ifdef MARCH_HAS_Z196_FEATURES aghik %r2,%r0,-MCOUNT_INSN_SIZE lgrl %r4,function_trace_op lgrl %r1,ftrace_func @@ -115,7 +116,7 @@ SYM_INNER_LABEL(ftrace_graph_caller, SYM_L_GLOBAL) .Lftrace_graph_caller_end: #endif lg %r0,(STACK_FREGS_PTREGS_PSW+8)(%r15) -#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES +#ifdef MARCH_HAS_Z196_FEATURES ltg %r1,STACK_FREGS_PTREGS_ORIG_GPR2(%r15) locgrz %r1,%r0 #else From fccb175bc89a0d37e3ff513bb6bf1f73b3a48950 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:27 +0200 Subject: [PATCH 76/84] s390/boot: Compile all files with the same march flag Only a couple of files of the decompressor are compiled with the minimum architecture level. This is problematic for potential function calls between compile units, especially if a target function is within a compile until compiled for a higher architecture level, since that may lead to an unexpected operation exception. Therefore compile all files of the decompressor for the same (minimum) architecture level. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/boot/Makefile | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index df01e7be1560..1ee63054e085 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -11,25 +11,16 @@ KASAN_SANITIZE := n KCSAN_SANITIZE := n KMSAN_SANITIZE := n -KBUILD_AFLAGS := $(KBUILD_AFLAGS_DECOMPRESSOR) -KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) - # -# Use minimum architecture for als.c to be able to print an error +# Use minimum architecture level so it is possible to print an error # message if the kernel is started on a machine which is too old # CC_FLAGS_MARCH_MINIMUM := -march=z10 -ifneq ($(CC_FLAGS_MARCH),$(CC_FLAGS_MARCH_MINIMUM)) -AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) -AFLAGS_head.o += $(CC_FLAGS_MARCH_MINIMUM) -AFLAGS_REMOVE_mem.o += $(CC_FLAGS_MARCH) -AFLAGS_mem.o += $(CC_FLAGS_MARCH_MINIMUM) -CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) -CFLAGS_als.o += $(CC_FLAGS_MARCH_MINIMUM) -CFLAGS_REMOVE_sclp_early_core.o += $(CC_FLAGS_MARCH) -CFLAGS_sclp_early_core.o += $(CC_FLAGS_MARCH_MINIMUM) -endif +KBUILD_AFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_AFLAGS_DECOMPRESSOR)) +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_CFLAGS_DECOMPRESSOR)) +KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) +KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char From bfda6108144628581ee50a127bb616cf097b1c7c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:28 +0200 Subject: [PATCH 77/84] s390/boot: Rename decompressor_printk() to boot_printk() Rename decompressor_printk() to boot_printk() just to have a shorter function name, which also makes the code more readable. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/boot/boot.h | 2 +- arch/s390/boot/pgm_check_info.c | 50 +++++++++++++++------------------ arch/s390/boot/physmem_info.c | 24 ++++++++-------- arch/s390/boot/startup.c | 2 +- 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 83e2ce050b6c..7521a9d75fa2 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -70,7 +70,7 @@ void print_pgm_check_info(void); unsigned long randomize_within_range(unsigned long size, unsigned long align, unsigned long min, unsigned long max); void setup_vmem(unsigned long kernel_start, unsigned long kernel_end, unsigned long asce_limit); -void __printf(1, 2) decompressor_printk(const char *fmt, ...); +void __printf(1, 2) boot_printk(const char *fmt, ...); void print_stacktrace(unsigned long sp); void error(char *m); int get_random(unsigned long limit, unsigned long *value); diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index 5352b3d356da..94b4d5fd8159 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -80,7 +80,7 @@ static noinline char *strsym(void *ip) return buf; } -void decompressor_printk(const char *fmt, ...) +void boot_printk(const char *fmt, ...) { char buf[1024] = { 0 }; char *end = buf + sizeof(buf) - 1; /* make sure buf is 0 terminated */ @@ -129,13 +129,13 @@ void print_stacktrace(unsigned long sp) (unsigned long)_stack_end }; bool first = true; - decompressor_printk("Call Trace:\n"); + boot_printk("Call Trace:\n"); while (!(sp & 0x7) && on_stack(&boot_stack, sp, sizeof(struct stack_frame))) { struct stack_frame *sf = (struct stack_frame *)sp; - decompressor_printk(first ? "(sp:%016lx [<%016lx>] %pS)\n" : - " sp:%016lx [<%016lx>] %pS\n", - sp, sf->gprs[8], (void *)sf->gprs[8]); + boot_printk(first ? "(sp:%016lx [<%016lx>] %pS)\n" : + " sp:%016lx [<%016lx>] %pS\n", + sp, sf->gprs[8], (void *)sf->gprs[8]); if (sf->back_chain <= sp) break; sp = sf->back_chain; @@ -148,34 +148,30 @@ void print_pgm_check_info(void) unsigned long *gpregs = (unsigned long *)get_lowcore()->gpregs_save_area; struct psw_bits *psw = &psw_bits(get_lowcore()->psw_save_area); - decompressor_printk("Linux version %s\n", kernel_version); + boot_printk("Linux version %s\n", kernel_version); if (!is_prot_virt_guest() && early_command_line[0]) - decompressor_printk("Kernel command line: %s\n", early_command_line); - decompressor_printk("Kernel fault: interruption code %04x ilc:%x\n", - get_lowcore()->pgm_code, get_lowcore()->pgm_ilc >> 1); + boot_printk("Kernel command line: %s\n", early_command_line); + boot_printk("Kernel fault: interruption code %04x ilc:%x\n", + get_lowcore()->pgm_code, get_lowcore()->pgm_ilc >> 1); if (kaslr_enabled()) { - decompressor_printk("Kernel random base: %lx\n", __kaslr_offset); - decompressor_printk("Kernel random base phys: %lx\n", __kaslr_offset_phys); + boot_printk("Kernel random base: %lx\n", __kaslr_offset); + boot_printk("Kernel random base phys: %lx\n", __kaslr_offset_phys); } - decompressor_printk("PSW : %016lx %016lx (%pS)\n", - get_lowcore()->psw_save_area.mask, - get_lowcore()->psw_save_area.addr, - (void *)get_lowcore()->psw_save_area.addr); - decompressor_printk( + boot_printk("PSW : %016lx %016lx (%pS)\n", + get_lowcore()->psw_save_area.mask, + get_lowcore()->psw_save_area.addr, + (void *)get_lowcore()->psw_save_area.addr); + boot_printk( " R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x P:%x AS:%x CC:%x PM:%x RI:%x EA:%x\n", psw->per, psw->dat, psw->io, psw->ext, psw->key, psw->mcheck, psw->wait, psw->pstate, psw->as, psw->cc, psw->pm, psw->ri, psw->eaba); - decompressor_printk("GPRS: %016lx %016lx %016lx %016lx\n", - gpregs[0], gpregs[1], gpregs[2], gpregs[3]); - decompressor_printk(" %016lx %016lx %016lx %016lx\n", - gpregs[4], gpregs[5], gpregs[6], gpregs[7]); - decompressor_printk(" %016lx %016lx %016lx %016lx\n", - gpregs[8], gpregs[9], gpregs[10], gpregs[11]); - decompressor_printk(" %016lx %016lx %016lx %016lx\n", - gpregs[12], gpregs[13], gpregs[14], gpregs[15]); + boot_printk("GPRS: %016lx %016lx %016lx %016lx\n", gpregs[0], gpregs[1], gpregs[2], gpregs[3]); + boot_printk(" %016lx %016lx %016lx %016lx\n", gpregs[4], gpregs[5], gpregs[6], gpregs[7]); + boot_printk(" %016lx %016lx %016lx %016lx\n", gpregs[8], gpregs[9], gpregs[10], gpregs[11]); + boot_printk(" %016lx %016lx %016lx %016lx\n", gpregs[12], gpregs[13], gpregs[14], gpregs[15]); print_stacktrace(get_lowcore()->gpregs_save_area[15]); - decompressor_printk("Last Breaking-Event-Address:\n"); - decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)get_lowcore()->pgm_last_break, - (void *)get_lowcore()->pgm_last_break); + boot_printk("Last Breaking-Event-Address:\n"); + boot_printk(" [<%016lx>] %pS\n", (unsigned long)get_lowcore()->pgm_last_break, + (void *)get_lowcore()->pgm_last_break); } diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 4c9ad8258f7e..d87a95ff3418 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -190,25 +190,25 @@ static void die_oom(unsigned long size, unsigned long align, unsigned long min, enum reserved_range_type t; int i; - decompressor_printk("Linux version %s\n", kernel_version); + boot_printk("Linux version %s\n", kernel_version); if (!is_prot_virt_guest() && early_command_line[0]) - decompressor_printk("Kernel command line: %s\n", early_command_line); - decompressor_printk("Out of memory allocating %lx bytes %lx aligned in range %lx:%lx\n", - size, align, min, max); - decompressor_printk("Reserved memory ranges:\n"); + boot_printk("Kernel command line: %s\n", early_command_line); + boot_printk("Out of memory allocating %lx bytes %lx aligned in range %lx:%lx\n", + size, align, min, max); + boot_printk("Reserved memory ranges:\n"); for_each_physmem_reserved_range(t, range, &start, &end) { - decompressor_printk("%016lx %016lx %s\n", start, end, get_rr_type_name(t)); + boot_printk("%016lx %016lx %s\n", start, end, get_rr_type_name(t)); total_reserved_mem += end - start; } - decompressor_printk("Usable online memory ranges (info source: %s [%x]):\n", - get_physmem_info_source(), physmem_info.info_source); + boot_printk("Usable online memory ranges (info source: %s [%x]):\n", + get_physmem_info_source(), physmem_info.info_source); for_each_physmem_usable_range(i, &start, &end) { - decompressor_printk("%016lx %016lx\n", start, end); + boot_printk("%016lx %016lx\n", start, end); total_mem += end - start; } - decompressor_printk("Usable online memory total: %lx Reserved: %lx Free: %lx\n", - total_mem, total_reserved_mem, - total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0); + boot_printk("Usable online memory total: %lx Reserved: %lx Free: %lx\n", + total_mem, total_reserved_mem, + total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0); print_stacktrace(current_frame_address()); sclp_early_printk("\n\n -- System halted\n"); disabled_wait(); diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index ce232552bc1c..80fad3c08378 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -296,7 +296,7 @@ static unsigned long setup_kernel_memory_layout(unsigned long kernel_size) kernel_start = round_down(kernel_end - kernel_size, THREAD_SIZE); } else if (vmax < __NO_KASLR_END_KERNEL || vsize > __NO_KASLR_END_KERNEL) { kernel_start = round_down(vmax - kernel_size, THREAD_SIZE); - decompressor_printk("The kernel base address is forced to %lx\n", kernel_start); + boot_printk("The kernel base address is forced to %lx\n", kernel_start); } else { kernel_start = __NO_KASLR_START_KERNEL; } From dc7155550730dfab3b7cc59dfbf0d8c84ac67bf0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:29 +0200 Subject: [PATCH 78/84] s390/boot: Use boot_printk() instead of sclp_early_printk() Consistently use boot_printk() everywhere instead of sclp_early_printk() at some places. For some places it was required (e.g. als.c), in order to stay in code compiled for the same architecture level, for other places it is not obvious why sclp_early_printk() was used instead of decompressor_printk(). Given that the whole decompressor code is compiled for the same architecture level, there is no requirement left to use different printk functions. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/boot/als.c | 41 +++++++---------------------------- arch/s390/boot/ipl_parm.c | 2 +- arch/s390/boot/kaslr.c | 2 +- arch/s390/boot/physmem_info.c | 2 +- arch/s390/boot/startup.c | 5 +---- 5 files changed, 12 insertions(+), 40 deletions(-) diff --git a/arch/s390/boot/als.c b/arch/s390/boot/als.c index efb6caa89e03..11e0c3d5dbc8 100644 --- a/arch/s390/boot/als.c +++ b/arch/s390/boot/als.c @@ -11,32 +11,6 @@ static unsigned long als[] = { FACILITIES_ALS }; -static void u16_to_hex(char *str, u16 val) -{ - int i, num; - - for (i = 1; i <= 4; i++) { - num = (val >> (16 - 4 * i)) & 0xf; - if (num >= 10) - num += 7; - *str++ = '0' + num; - } - *str = '\0'; -} - -static void print_machine_type(void) -{ - static char mach_str[80] = "Detected machine-type number: "; - char type_str[5]; - struct cpuid id; - - get_cpu_id(&id); - u16_to_hex(type_str, id.machine); - strcat(mach_str, type_str); - strcat(mach_str, "\n"); - sclp_early_printk(mach_str); -} - static void u16_to_decimal(char *str, u16 val) { int div = 1; @@ -72,8 +46,7 @@ void print_missing_facilities(void) * z/VM adds a four character prefix. */ if (strlen(als_str) > 70) { - strcat(als_str, "\n"); - sclp_early_printk(als_str); + boot_printk("%s\n", als_str); *als_str = '\0'; } u16_to_decimal(val_str, i * BITS_PER_LONG + j); @@ -81,16 +54,18 @@ void print_missing_facilities(void) first = 0; } } - strcat(als_str, "\n"); - sclp_early_printk(als_str); + boot_printk("%s\n", als_str); } static void facility_mismatch(void) { - sclp_early_printk("The Linux kernel requires more recent processor hardware\n"); - print_machine_type(); + struct cpuid id; + + get_cpu_id(&id); + boot_printk("The Linux kernel requires more recent processor hardware\n"); + boot_printk("Detected machine-type number: %4x\n", id.machine); print_missing_facilities(); - sclp_early_printk("See Principles of Operations for facility bits\n"); + boot_printk("See Principles of Operations for facility bits\n"); disabled_wait(); } diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 1773b72a6a7b..557462e62cd7 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -215,7 +215,7 @@ static void check_cleared_facilities(void) for (i = 0; i < ARRAY_SIZE(als); i++) { if ((stfle_fac_list[i] & als[i]) != als[i]) { - sclp_early_printk("Warning: The Linux kernel requires facilities cleared via command line option\n"); + boot_printk("Warning: The Linux kernel requires facilities cleared via command line option\n"); print_missing_facilities(); break; } diff --git a/arch/s390/boot/kaslr.c b/arch/s390/boot/kaslr.c index bd3bf5ef472d..f864d2bff775 100644 --- a/arch/s390/boot/kaslr.c +++ b/arch/s390/boot/kaslr.c @@ -32,7 +32,7 @@ struct prng_parm { static int check_prng(void) { if (!cpacf_query_func(CPACF_KMC, CPACF_KMC_PRNG)) { - sclp_early_printk("KASLR disabled: CPU has no PRNG\n"); + boot_printk("KASLR disabled: CPU has no PRNG\n"); return 0; } if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index d87a95ff3418..1d131a81cb8b 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -210,7 +210,7 @@ static void die_oom(unsigned long size, unsigned long align, unsigned long min, total_mem, total_reserved_mem, total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0); print_stacktrace(current_frame_address()); - sclp_early_printk("\n\n -- System halted\n"); + boot_printk("\n\n -- System halted\n"); disabled_wait(); } diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 80fad3c08378..f8f9dbf88d9a 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -39,10 +39,7 @@ struct machine_info machine; void error(char *x) { - sclp_early_printk("\n\n"); - sclp_early_printk(x); - sclp_early_printk("\n\n -- System halted"); - + boot_printk("\n\n%s\n\n -- System halted", x); disabled_wait(); } From 5c9a274202ca144764e98d4032fed76693e8cfc6 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 4 Sep 2024 11:39:30 +0200 Subject: [PATCH 79/84] s390/boot: Move boot_printk() code to own file Keep the printk code separate from the program check code and move boot_printk() and helper functions to own printk.c file. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/boot/Makefile | 3 +- arch/s390/boot/pgm_check_info.c | 112 ----------------------------- arch/s390/boot/printk.c | 124 ++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 113 deletions(-) create mode 100644 arch/s390/boot/printk.c diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 1ee63054e085..8bc1308ac892 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -26,7 +26,8 @@ CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char obj-y := head.o als.o startup.o physmem_info.o ipl_parm.o ipl_report.o vmem.o obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o -obj-y += version.o pgm_check_info.o ctype.o ipl_data.o relocs.o alternative.o uv.o +obj-y += version.o pgm_check_info.o ctype.o ipl_data.o relocs.o alternative.o +obj-y += uv.o printk.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-y += $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) info.o obj-$(CONFIG_KERNEL_ZSTD) += clz_ctz.o diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index 94b4d5fd8159..5abe59fb3bc0 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -11,118 +11,6 @@ #include #include "boot.h" -const char hex_asc[] = "0123456789abcdef"; - -static char *as_hex(char *dst, unsigned long val, int pad) -{ - char *p, *end = p = dst + max(pad, (int)__fls(val | 1) / 4 + 1); - - for (*p-- = 0; p >= dst; val >>= 4) - *p-- = hex_asc[val & 0x0f]; - return end; -} - -static char *symstart(char *p) -{ - while (*p) - p--; - return p + 1; -} - -static noinline char *findsym(unsigned long ip, unsigned short *off, unsigned short *len) -{ - /* symbol entries are in a form "10000 c4 startup\0" */ - char *a = _decompressor_syms_start; - char *b = _decompressor_syms_end; - unsigned long start; - unsigned long size; - char *pivot; - char *endp; - - while (a < b) { - pivot = symstart(a + (b - a) / 2); - start = simple_strtoull(pivot, &endp, 16); - size = simple_strtoull(endp + 1, &endp, 16); - if (ip < start) { - b = pivot; - continue; - } - if (ip > start + size) { - a = pivot + strlen(pivot) + 1; - continue; - } - *off = ip - start; - *len = size; - return endp + 1; - } - return NULL; -} - -static noinline char *strsym(void *ip) -{ - static char buf[64]; - unsigned short off; - unsigned short len; - char *p; - - p = findsym((unsigned long)ip, &off, &len); - if (p) { - strncpy(buf, p, sizeof(buf)); - /* reserve 15 bytes for offset/len in symbol+0x1234/0x1234 */ - p = buf + strnlen(buf, sizeof(buf) - 15); - strcpy(p, "+0x"); - p = as_hex(p + 3, off, 0); - strcpy(p, "/0x"); - as_hex(p + 3, len, 0); - } else { - as_hex(buf, (unsigned long)ip, 16); - } - return buf; -} - -void boot_printk(const char *fmt, ...) -{ - char buf[1024] = { 0 }; - char *end = buf + sizeof(buf) - 1; /* make sure buf is 0 terminated */ - unsigned long pad; - char *p = buf; - va_list args; - - va_start(args, fmt); - for (; p < end && *fmt; fmt++) { - if (*fmt != '%') { - *p++ = *fmt; - continue; - } - pad = isdigit(*++fmt) ? simple_strtol(fmt, (char **)&fmt, 10) : 0; - switch (*fmt) { - case 's': - p = buf + strlcat(buf, va_arg(args, char *), sizeof(buf)); - break; - case 'p': - if (*++fmt != 'S') - goto out; - p = buf + strlcat(buf, strsym(va_arg(args, void *)), sizeof(buf)); - break; - case 'l': - if (*++fmt != 'x' || end - p <= max(sizeof(long) * 2, pad)) - goto out; - p = as_hex(p, va_arg(args, unsigned long), pad); - break; - case 'x': - if (end - p <= max(sizeof(int) * 2, pad)) - goto out; - p = as_hex(p, va_arg(args, unsigned int), pad); - break; - default: - goto out; - } - } -out: - va_end(args); - sclp_early_printk(buf); -} - void print_stacktrace(unsigned long sp) { struct stack_info boot_stack = { STACK_TYPE_TASK, (unsigned long)_stack_start, diff --git a/arch/s390/boot/printk.c b/arch/s390/boot/printk.c new file mode 100644 index 000000000000..35f18f2b936e --- /dev/null +++ b/arch/s390/boot/printk.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "boot.h" + +const char hex_asc[] = "0123456789abcdef"; + +static char *as_hex(char *dst, unsigned long val, int pad) +{ + char *p, *end = p = dst + max(pad, (int)__fls(val | 1) / 4 + 1); + + for (*p-- = 0; p >= dst; val >>= 4) + *p-- = hex_asc[val & 0x0f]; + return end; +} + +static char *symstart(char *p) +{ + while (*p) + p--; + return p + 1; +} + +static noinline char *findsym(unsigned long ip, unsigned short *off, unsigned short *len) +{ + /* symbol entries are in a form "10000 c4 startup\0" */ + char *a = _decompressor_syms_start; + char *b = _decompressor_syms_end; + unsigned long start; + unsigned long size; + char *pivot; + char *endp; + + while (a < b) { + pivot = symstart(a + (b - a) / 2); + start = simple_strtoull(pivot, &endp, 16); + size = simple_strtoull(endp + 1, &endp, 16); + if (ip < start) { + b = pivot; + continue; + } + if (ip > start + size) { + a = pivot + strlen(pivot) + 1; + continue; + } + *off = ip - start; + *len = size; + return endp + 1; + } + return NULL; +} + +static noinline char *strsym(void *ip) +{ + static char buf[64]; + unsigned short off; + unsigned short len; + char *p; + + p = findsym((unsigned long)ip, &off, &len); + if (p) { + strncpy(buf, p, sizeof(buf)); + /* reserve 15 bytes for offset/len in symbol+0x1234/0x1234 */ + p = buf + strnlen(buf, sizeof(buf) - 15); + strcpy(p, "+0x"); + p = as_hex(p + 3, off, 0); + strcpy(p, "/0x"); + as_hex(p + 3, len, 0); + } else { + as_hex(buf, (unsigned long)ip, 16); + } + return buf; +} + +void boot_printk(const char *fmt, ...) +{ + char buf[1024] = { 0 }; + char *end = buf + sizeof(buf) - 1; /* make sure buf is 0 terminated */ + unsigned long pad; + char *p = buf; + va_list args; + + va_start(args, fmt); + for (; p < end && *fmt; fmt++) { + if (*fmt != '%') { + *p++ = *fmt; + continue; + } + pad = isdigit(*++fmt) ? simple_strtol(fmt, (char **)&fmt, 10) : 0; + switch (*fmt) { + case 's': + p = buf + strlcat(buf, va_arg(args, char *), sizeof(buf)); + break; + case 'p': + if (*++fmt != 'S') + goto out; + p = buf + strlcat(buf, strsym(va_arg(args, void *)), sizeof(buf)); + break; + case 'l': + if (*++fmt != 'x' || end - p <= max(sizeof(long) * 2, pad)) + goto out; + p = as_hex(p, va_arg(args, unsigned long), pad); + break; + case 'x': + if (end - p <= max(sizeof(int) * 2, pad)) + goto out; + p = as_hex(p, va_arg(args, unsigned int), pad); + break; + default: + goto out; + } + } +out: + va_end(args); + sclp_early_printk(buf); +} From ab22f8d90878a882775fb0380572725a56104fb1 Mon Sep 17 00:00:00 2001 From: Jens Remus Date: Tue, 10 Sep 2024 15:41:17 +0200 Subject: [PATCH 80/84] s390/disassembler: Remove duplicate instruction format RSY_RDRU Instruction format RSY_RDRU is a duplicate of RSY_RURD2. Use the latter, as it follows the s390-specific conventions for instruction format naming used in binutils. Reviewed-by: Heiko Carstens Signed-off-by: Jens Remus Signed-off-by: Vasily Gorbik --- arch/s390/kernel/dis.c | 1 - arch/s390/tools/opcodes.txt | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index cc7e905d0ff4..94eb8168ea44 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -259,7 +259,6 @@ static const unsigned char formats[][6] = { [INSTR_RSL_R0RD] = { D_20, L4_8, B_16, 0, 0, 0 }, [INSTR_RSY_AARD] = { A_8, A_12, D20_20, B_16, 0, 0 }, [INSTR_RSY_CCRD] = { C_8, C_12, D20_20, B_16, 0, 0 }, - [INSTR_RSY_RDRU] = { R_8, D20_20, B_16, U4_12, 0, 0 }, [INSTR_RSY_RRRD] = { R_8, R_12, D20_20, B_16, 0, 0 }, [INSTR_RSY_RURD] = { R_8, U4_12, D20_20, B_16, 0, 0 }, [INSTR_RSY_RURD2] = { R_8, D20_20, B_16, U4_12, 0, 0 }, diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index b781e1f05c9a..def2659f6602 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -1118,9 +1118,9 @@ eb54 niy SIY_URD eb55 cliy SIY_URD eb56 oiy SIY_URD eb57 xiy SIY_URD -eb60 lric RSY_RDRU -eb61 stric RSY_RDRU -eb62 mric RSY_RDRU +eb60 lric RSY_RURD2 +eb61 stric RSY_RURD2 +eb62 mric RSY_RURD2 eb6a asi SIY_IRD eb6e alsi SIY_IRD eb71 lpswey SIY_RD @@ -1128,7 +1128,7 @@ eb7a agsi SIY_IRD eb7e algsi SIY_IRD eb80 icmh RSY_RURD eb81 icmy RSY_RURD -eb8a sqbs RSY_RDRU +eb8a sqbs RSY_RURD2 eb8e mvclu RSY_RRRD eb8f clclu RSY_RRRD eb90 stmy RSY_RRRD From d2dec49d76f741c746ba375abe009cdcc69fb3a9 Mon Sep 17 00:00:00 2001 From: Finn Callies Date: Wed, 11 Sep 2024 09:21:06 +0200 Subject: [PATCH 81/84] s390/crypto: Add KDSA CPACF Instruction Add the function code definitions for using the KDSA function to the CPACF header file. Suggested-by: Harald Freudenberger Reviewed-by: Harald Freudenberger Acked-by: Heiko Carstens Signed-off-by: Finn Callies Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/cpacf.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index b8a98e6ffb4d..748bd806d93b 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -183,6 +183,27 @@ #define CPACF_KLMD_DUFOP 0x4000 #define CPACF_KLMD_NIP 0x8000 +/* + * Function codes for KDSA (COMPUTE DIGITAL SIGNATURE AUTHENTICATION) + * instruction + */ +#define CPACF_KDSA_QUERY 0x00 +#define CPACF_KDSA_ECDSA_VERIFY_P256 0x01 +#define CPACF_KDSA_ECDSA_VERIFY_P384 0x02 +#define CPACF_KDSA_ECDSA_VERIFY_P521 0x03 +#define CPACF_KDSA_ECDSA_SIGN_P256 0x09 +#define CPACF_KDSA_ECDSA_SIGN_P384 0x0a +#define CPACF_KDSA_ECDSA_SIGN_P521 0x0b +#define CPACF_KDSA_ENC_ECDSA_SIGN_P256 0x11 +#define CPACF_KDSA_ENC_ECDSA_SIGN_P384 0x12 +#define CPACF_KDSA_ENC_ECDSA_SIGN_P521 0x13 +#define CPACF_KDSA_EDDSA_VERIFY_ED25519 0x20 +#define CPACF_KDSA_EDDSA_VERIFY_ED448 0x24 +#define CPACF_KDSA_EDDSA_SIGN_ED25519 0x28 +#define CPACF_KDSA_EDDSA_SIGN_ED448 0x2c +#define CPACF_KDSA_ENC_EDDSA_SIGN_ED25519 0x30 +#define CPACF_KDSA_ENC_EDDSA_SIGN_ED448 0x34 + typedef struct { unsigned char bytes[16]; } cpacf_mask_t; /* @@ -287,6 +308,8 @@ static __always_inline int __cpacf_check_opcode(unsigned int opcode) return test_facility(57); /* check for MSA5 */ case CPACF_KMA: return test_facility(146); /* check for MSA8 */ + case CPACF_KDSA: + return test_facility(155); /* check for MSA9 */ default: __cpacf_bad_opcode(); return 0; From 27aad7f7a4bbaae910bbac88247a05081bb8b21e Mon Sep 17 00:00:00 2001 From: Finn Callies Date: Wed, 11 Sep 2024 09:21:07 +0200 Subject: [PATCH 82/84] s390/crypto: Rework RRE and RRF CPACF inline functions Rework of the __cpacf_query_rre() and __cpacf_query_rrf() functions to support additional function codes. A function code is passed as a new parameter to specify which subfunction of the supplied Instruction is to be called. Suggested-by: Harald Freudenberger Reviewed-by: Harald Freudenberger Acked-by: Heiko Carstens Signed-off-by: Finn Callies Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/cpacf.h | 130 ++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 61 deletions(-) diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index 748bd806d93b..52652f0acfce 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -204,6 +204,8 @@ #define CPACF_KDSA_ENC_EDDSA_SIGN_ED25519 0x30 #define CPACF_KDSA_ENC_EDDSA_SIGN_ED448 0x34 +#define CPACF_FC_QUERY 0x00 + typedef struct { unsigned char bytes[16]; } cpacf_mask_t; /* @@ -214,78 +216,83 @@ typedef struct { unsigned char bytes[16]; } cpacf_mask_t; void __cpacf_bad_opcode(void); static __always_inline void __cpacf_query_rre(u32 opc, u8 r1, u8 r2, - cpacf_mask_t *mask) + u8 *pb, u8 fc) { asm volatile( - " la %%r1,%[mask]\n" - " xgr %%r0,%%r0\n" + " la %%r1,%[pb]\n" + " lghi %%r0,%[fc]\n" " .insn rre,%[opc] << 16,%[r1],%[r2]\n" - : [mask] "=R" (*mask) - : [opc] "i" (opc), + : [pb] "=R" (*pb) + : [opc] "i" (opc), [fc] "i" (fc), [r1] "i" (r1), [r2] "i" (r2) - : "cc", "r0", "r1"); + : "cc", "memory", "r0", "r1"); } -static __always_inline void __cpacf_query_rrf(u32 opc, - u8 r1, u8 r2, u8 r3, u8 m4, - cpacf_mask_t *mask) +static __always_inline void __cpacf_query_rrf(u32 opc, u8 r1, u8 r2, u8 r3, + u8 m4, u8 *pb, u8 fc) { asm volatile( - " la %%r1,%[mask]\n" - " xgr %%r0,%%r0\n" + " la %%r1,%[pb]\n" + " lghi %%r0,%[fc]\n" " .insn rrf,%[opc] << 16,%[r1],%[r2],%[r3],%[m4]\n" - : [mask] "=R" (*mask) - : [opc] "i" (opc), [r1] "i" (r1), [r2] "i" (r2), - [r3] "i" (r3), [m4] "i" (m4) - : "cc", "r0", "r1"); + : [pb] "=R" (*pb) + : [opc] "i" (opc), [fc] "i" (fc), [r1] "i" (r1), + [r2] "i" (r2), [r3] "i" (r3), [m4] "i" (m4) + : "cc", "memory", "r0", "r1"); +} + +static __always_inline void __cpacf_query_insn(unsigned int opcode, void *pb, + u8 fc) +{ + switch (opcode) { + case CPACF_KDSA: + __cpacf_query_rre(CPACF_KDSA, 0, 2, pb, fc); + break; + case CPACF_KIMD: + __cpacf_query_rre(CPACF_KIMD, 0, 2, pb, fc); + break; + case CPACF_KLMD: + __cpacf_query_rre(CPACF_KLMD, 0, 2, pb, fc); + break; + case CPACF_KM: + __cpacf_query_rre(CPACF_KM, 2, 4, pb, fc); + break; + case CPACF_KMA: + __cpacf_query_rrf(CPACF_KMA, 2, 4, 6, 0, pb, fc); + break; + case CPACF_KMAC: + __cpacf_query_rre(CPACF_KMAC, 0, 2, pb, fc); + break; + case CPACF_KMC: + __cpacf_query_rre(CPACF_KMC, 2, 4, pb, fc); + break; + case CPACF_KMCTR: + __cpacf_query_rrf(CPACF_KMCTR, 2, 4, 6, 0, pb, fc); + break; + case CPACF_KMF: + __cpacf_query_rre(CPACF_KMF, 2, 4, pb, fc); + break; + case CPACF_KMO: + __cpacf_query_rre(CPACF_KMO, 2, 4, pb, fc); + break; + case CPACF_PCC: + __cpacf_query_rre(CPACF_PCC, 0, 0, pb, fc); + break; + case CPACF_PCKMO: + __cpacf_query_rre(CPACF_PCKMO, 0, 0, pb, fc); + break; + case CPACF_PRNO: + __cpacf_query_rre(CPACF_PRNO, 2, 4, pb, fc); + break; + default: + __cpacf_bad_opcode(); + } } static __always_inline void __cpacf_query(unsigned int opcode, cpacf_mask_t *mask) { - switch (opcode) { - case CPACF_KDSA: - __cpacf_query_rre(CPACF_KDSA, 0, 2, mask); - break; - case CPACF_KIMD: - __cpacf_query_rre(CPACF_KIMD, 0, 2, mask); - break; - case CPACF_KLMD: - __cpacf_query_rre(CPACF_KLMD, 0, 2, mask); - break; - case CPACF_KM: - __cpacf_query_rre(CPACF_KM, 2, 4, mask); - break; - case CPACF_KMA: - __cpacf_query_rrf(CPACF_KMA, 2, 4, 6, 0, mask); - break; - case CPACF_KMAC: - __cpacf_query_rre(CPACF_KMAC, 0, 2, mask); - break; - case CPACF_KMC: - __cpacf_query_rre(CPACF_KMC, 2, 4, mask); - break; - case CPACF_KMCTR: - __cpacf_query_rrf(CPACF_KMCTR, 2, 4, 6, 0, mask); - break; - case CPACF_KMF: - __cpacf_query_rre(CPACF_KMF, 2, 4, mask); - break; - case CPACF_KMO: - __cpacf_query_rre(CPACF_KMO, 2, 4, mask); - break; - case CPACF_PCC: - __cpacf_query_rre(CPACF_PCC, 0, 0, mask); - break; - case CPACF_PCKMO: - __cpacf_query_rre(CPACF_PCKMO, 0, 0, mask); - break; - case CPACF_PRNO: - __cpacf_query_rre(CPACF_PRNO, 2, 4, mask); - break; - default: - __cpacf_bad_opcode(); - } + __cpacf_query_insn(opcode, mask, CPACF_FC_QUERY); } static __always_inline int __cpacf_check_opcode(unsigned int opcode) @@ -317,14 +324,15 @@ static __always_inline int __cpacf_check_opcode(unsigned int opcode) } /** - * cpacf_query() - check if a specific CPACF function is available + * cpacf_query() - Query the function code mask for this CPACF opcode * @opcode: the opcode of the crypto instruction - * @func: the function code to test for + * @mask: ptr to struct cpacf_mask_t * * Executes the query function for the given crypto instruction @opcode * and checks if @func is available * - * Returns 1 if @func is available for @opcode, 0 otherwise + * On success 1 is returned and the mask is filled with the function + * code mask for this CPACF opcode, otherwise 0 is returned. */ static __always_inline int cpacf_query(unsigned int opcode, cpacf_mask_t *mask) { From 9bbd1bfb865555df64fe4740c528f6d53529ad17 Mon Sep 17 00:00:00 2001 From: Finn Callies Date: Wed, 11 Sep 2024 09:21:08 +0200 Subject: [PATCH 83/84] s390/crypto: Add Support for Query Authentication Information Introduce functions __cpacf_qai() and wrapper cpacf_qai() to the respective existing functions __cpacf_query() and cpacf_query() are introduced to support the Query Authentication Information feature of MSA 13. Suggested-by: Harald Freudenberger Reviewed-by: Harald Freudenberger Acked-by: Heiko Carstens Signed-off-by: Finn Callies Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/cpacf.h | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index 52652f0acfce..1d3a4b0c650f 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -205,8 +205,10 @@ #define CPACF_KDSA_ENC_EDDSA_SIGN_ED448 0x34 #define CPACF_FC_QUERY 0x00 +#define CPACF_FC_QUERY_AUTH_INFO 0x7F typedef struct { unsigned char bytes[16]; } cpacf_mask_t; +typedef struct { unsigned char bytes[256]; } cpacf_qai_t; /* * Prototype for a not existing function to produce a link @@ -349,7 +351,8 @@ static inline int cpacf_test_func(cpacf_mask_t *mask, unsigned int func) return (mask->bytes[func >> 3] & (0x80 >> (func & 7))) != 0; } -static __always_inline int cpacf_query_func(unsigned int opcode, unsigned int func) +static __always_inline int cpacf_query_func(unsigned int opcode, + unsigned int func) { cpacf_mask_t mask; @@ -358,6 +361,32 @@ static __always_inline int cpacf_query_func(unsigned int opcode, unsigned int fu return 0; } +static __always_inline void __cpacf_qai(unsigned int opcode, cpacf_qai_t *qai) +{ + __cpacf_query_insn(opcode, qai, CPACF_FC_QUERY_AUTH_INFO); +} + +/** + * cpacf_qai() - Get the query authentication information for a CPACF opcode + * @opcode: the opcode of the crypto instruction + * @mask: ptr to struct cpacf_qai_t + * + * Executes the query authentication information function for the given crypto + * instruction @opcode and checks if @func is available + * + * On success 1 is returned and the mask is filled with the query authentication + * information for this CPACF opcode, otherwise 0 is returned. + */ +static __always_inline int cpacf_qai(unsigned int opcode, cpacf_qai_t *qai) +{ + if (cpacf_query_func(opcode, CPACF_FC_QUERY_AUTH_INFO)) { + __cpacf_qai(opcode, qai); + return 1; + } + memset(qai, 0, sizeof(*qai)); + return 0; +} + /** * cpacf_km() - executes the KM (CIPHER MESSAGE) instruction * @func: the function code passed to KM; see CPACF_KM_xxx defines From 9fed8d7c46f37151037334ef5e8b30b945baaceb Mon Sep 17 00:00:00 2001 From: Finn Callies Date: Wed, 11 Sep 2024 09:21:09 +0200 Subject: [PATCH 84/84] s390/crypto: Display Query and Query Authentication Information in sysfs Displays the query (fc=0) and query authentication information (fc=127) as binary in sysfs per CPACF instruction. Files are located in /sys/devices/system/cpu/cpacf/. These information can be fetched via asm already except for PCKMO because this instruction is privileged. To offer a unified interface all CPACF instructions will have this information displayed in sysfs in files _query_raw and _query_auth_info_raw. A new tool introduced into s390-tools called cpacfinfo will use this information to convert and display in human readable form. Suggested-by: Harald Freudenberger Reviewed-by: Harald Freudenberger Acked-by: Heiko Carstens Signed-off-by: Finn Callies Signed-off-by: Vasily Gorbik --- arch/s390/kernel/Makefile | 1 + arch/s390/kernel/cpacf.c | 119 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 arch/s390/kernel/cpacf.c diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 5ceb08b338d3..48caae8c7e10 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -50,6 +50,7 @@ extra-y += vmlinux.lds obj-$(CONFIG_SYSFS) += nospec-sysfs.o CFLAGS_REMOVE_nospec-branch.o += $(CC_FLAGS_EXPOLINE) +obj-$(CONFIG_SYSFS) += cpacf.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_SCHED_TOPOLOGY) += topology.o hiperdispatch.o obj-$(CONFIG_NUMA) += numa.o diff --git a/arch/s390/kernel/cpacf.c b/arch/s390/kernel/cpacf.c new file mode 100644 index 000000000000..c8575dbc890d --- /dev/null +++ b/arch/s390/kernel/cpacf.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 2024 + */ + +#define KMSG_COMPONENT "cpacf" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include +#include + +#define CPACF_QUERY(name, instruction) \ +static ssize_t name##_query_raw_read(struct file *fp, \ + struct kobject *kobj, \ + struct bin_attribute *attr, \ + char *buf, loff_t offs, \ + size_t count) \ +{ \ + cpacf_mask_t mask; \ + \ + if (!cpacf_query(CPACF_##instruction, &mask)) \ + return -EOPNOTSUPP; \ + return memory_read_from_buffer(buf, count, &offs, &mask, sizeof(mask)); \ +} \ +static BIN_ATTR_RO(name##_query_raw, sizeof(cpacf_mask_t)) + +CPACF_QUERY(km, KM); +CPACF_QUERY(kmc, KMC); +CPACF_QUERY(kimd, KIMD); +CPACF_QUERY(klmd, KLMD); +CPACF_QUERY(kmac, KMAC); +CPACF_QUERY(pckmo, PCKMO); +CPACF_QUERY(kmf, KMF); +CPACF_QUERY(kmctr, KMCTR); +CPACF_QUERY(kmo, KMO); +CPACF_QUERY(pcc, PCC); +CPACF_QUERY(prno, PRNO); +CPACF_QUERY(kma, KMA); +CPACF_QUERY(kdsa, KDSA); + +#define CPACF_QAI(name, instruction) \ +static ssize_t name##_query_auth_info_raw_read( \ + struct file *fp, struct kobject *kobj, \ + struct bin_attribute *attr, char *buf, loff_t offs, \ + size_t count) \ +{ \ + cpacf_qai_t qai; \ + \ + if (!cpacf_qai(CPACF_##instruction, &qai)) \ + return -EOPNOTSUPP; \ + return memory_read_from_buffer(buf, count, &offs, &qai, \ + sizeof(qai)); \ +} \ +static BIN_ATTR_RO(name##_query_auth_info_raw, sizeof(cpacf_qai_t)) + +CPACF_QAI(km, KM); +CPACF_QAI(kmc, KMC); +CPACF_QAI(kimd, KIMD); +CPACF_QAI(klmd, KLMD); +CPACF_QAI(kmac, KMAC); +CPACF_QAI(pckmo, PCKMO); +CPACF_QAI(kmf, KMF); +CPACF_QAI(kmctr, KMCTR); +CPACF_QAI(kmo, KMO); +CPACF_QAI(pcc, PCC); +CPACF_QAI(prno, PRNO); +CPACF_QAI(kma, KMA); +CPACF_QAI(kdsa, KDSA); + +static struct bin_attribute *cpacf_attrs[] = { + &bin_attr_km_query_raw, + &bin_attr_kmc_query_raw, + &bin_attr_kimd_query_raw, + &bin_attr_klmd_query_raw, + &bin_attr_kmac_query_raw, + &bin_attr_pckmo_query_raw, + &bin_attr_kmf_query_raw, + &bin_attr_kmctr_query_raw, + &bin_attr_kmo_query_raw, + &bin_attr_pcc_query_raw, + &bin_attr_prno_query_raw, + &bin_attr_kma_query_raw, + &bin_attr_kdsa_query_raw, + &bin_attr_km_query_auth_info_raw, + &bin_attr_kmc_query_auth_info_raw, + &bin_attr_kimd_query_auth_info_raw, + &bin_attr_klmd_query_auth_info_raw, + &bin_attr_kmac_query_auth_info_raw, + &bin_attr_pckmo_query_auth_info_raw, + &bin_attr_kmf_query_auth_info_raw, + &bin_attr_kmctr_query_auth_info_raw, + &bin_attr_kmo_query_auth_info_raw, + &bin_attr_pcc_query_auth_info_raw, + &bin_attr_prno_query_auth_info_raw, + &bin_attr_kma_query_auth_info_raw, + &bin_attr_kdsa_query_auth_info_raw, + NULL, +}; + +static const struct attribute_group cpacf_attr_grp = { + .name = "cpacf", + .bin_attrs = cpacf_attrs, +}; + +static int __init cpacf_init(void) +{ + struct device *cpu_root; + int rc = 0; + + cpu_root = bus_get_dev_root(&cpu_subsys); + if (cpu_root) { + rc = sysfs_create_group(&cpu_root->kobj, &cpacf_attr_grp); + put_device(cpu_root); + } + return rc; +} +device_initcall(cpacf_init);