From 184c00ec1d37569d5afd57eadba7e200ce8f6f59 Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Fri, 10 Jul 2026 16:34:37 +0800 Subject: [PATCH 01/21] selftests/riscv: ptrace: Fix memory leak of regset_data in vector tests The regset_data buffer allocated with calloc() in the parent process of several vector ptrace tests is never freed before returning, causing memory leaks in: - ptrace_v_not_enabled - ptrace_v_early_debug - ptrace_v_syscall_clobbering - v_csr_invalid/ptrace_v_invalid_values - v_csr_valid/ptrace_v_valid_values Add free(regset_data) before kill(pid, SIGKILL) to release the allocated buffer. Signed-off-by: Wang Yan Reviewed-by: Sergey Matyukevich Link: https://patch.msgid.link/20260710083437.489648-1-wangyan01@kylinos.cn Signed-off-by: Paul Walmsley --- .../testing/selftests/riscv/vector/validate_v_ptrace.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/riscv/vector/validate_v_ptrace.c b/tools/testing/selftests/riscv/vector/validate_v_ptrace.c index b038e2175c8050..a388b7963d4771 100644 --- a/tools/testing/selftests/riscv/vector/validate_v_ptrace.c +++ b/tools/testing/selftests/riscv/vector/validate_v_ptrace.c @@ -75,7 +75,7 @@ TEST(ptrace_v_not_enabled) ASSERT_EQ(-1, ret); /* cleanup */ - + free(regset_data); ASSERT_EQ(0, kill(pid, SIGKILL)); } } @@ -207,7 +207,7 @@ TEST(ptrace_v_early_debug) EXPECT_EQ(vl_csr, regset_data->vl); /* cleanup */ - + free(regset_data); ASSERT_EQ(0, kill(pid, SIGKILL)); } } @@ -331,7 +331,7 @@ TEST(ptrace_v_syscall_clobbering) EXPECT_EQ(0UL, regset_data->vl); /* cleanup */ - + free(regset_data); ASSERT_EQ(0, kill(pid, SIGKILL)); } } @@ -649,7 +649,7 @@ TEST_F(v_csr_invalid, ptrace_v_invalid_values) ASSERT_EQ(ret, -1); /* cleanup */ - + free(regset_data); ASSERT_EQ(0, kill(pid, SIGKILL)); } } @@ -911,7 +911,7 @@ TEST_F(v_csr_valid, ptrace_v_valid_values) EXPECT_EQ(regset_data->vlenb, vlenb); /* cleanup */ - + free(regset_data); ASSERT_EQ(0, kill(pid, SIGKILL)); } } From 2b6a7252dcaf1b5d127d03e972dba4eb64ac4e82 Mon Sep 17 00:00:00 2001 From: Linux RISC-V bot Date: Fri, 10 Jul 2026 16:34:29 +0000 Subject: [PATCH 02/21] Adding CI files --- .github/scripts/build_ubuntu_defconfig.sh | 29 + .github/scripts/ci/__init__.py | 2 + .github/scripts/ci/base.py | 120 + .github/scripts/ci/shelltest.py | 67 + .github/scripts/cleanup_pr.py | 238 + .github/scripts/config.json | 28 + .github/scripts/defconfig.sh | 18 + .github/scripts/kselftest.sh | 22 + .github/scripts/libhugetlbfs.sh | 31 + .github/scripts/libs/__init__.py | 6 + .github/scripts/libs/context.py | 95 + .github/scripts/libs/email.py | 66 + .github/scripts/libs/githubtool.py | 77 + .github/scripts/libs/patchwork.py | 181 + .github/scripts/libs/repotool.py | 92 + .github/scripts/libs/utils.py | 105 + .github/scripts/ltp.sh | 43 + .github/scripts/pw_ci.py | 343 + .../scripts/pw_tests/build_rv32_defconfig.sh | 26 + .../pw_tests/build_rv64_clang_allmodconfig.sh | 88 + .../pw_tests/build_rv64_gcc_allmodconfig.sh | 88 + .../build_rv64_nommu_k210_defconfig.sh | 26 + .../build_rv64_nommu_virt_defconfig.sh | 26 + .github/scripts/pw_tests/checkpatch.sh | 34 + .github/scripts/pw_tests/dtb_warn_rv64.sh | 61 + .github/scripts/pw_tests/header_inline.sh | 19 + .github/scripts/pw_tests/kdoc.sh | 50 + .github/scripts/pw_tests/module_param.sh | 21 + .github/scripts/pw_tests/verify_fixes.sh | 212 + .github/scripts/pw_tests/verify_signedoff.sh | 81 + .github/scripts/requirements.txt | 5 + .github/scripts/series.sh | 40 + .github/scripts/series/build_all.sh | 18 + .github/scripts/series/build_kernel.sh | 94 + .../scripts/series/build_only_defconfig.sh | 13 + .../scripts/series/build_only_kselftest.sh | 15 + .github/scripts/series/build_selftest.sh | 69 + .../scripts/series/generate_build_configs.sh | 36 + .github/scripts/series/generate_kconfigs.sh | 65 + .github/scripts/series/generate_metadata.py | 29 + .../series/generate_qemu_test_configs.sh | 24 + .github/scripts/series/generate_test_runs.sh | 47 + .../scripts/series/github_ci_squad_results.py | 78 + .../kconfigs/defconfig/early_boot_alternative | 2 + .../defconfig/early_boot_alternative_reloc | 3 + .../scripts/series/kconfigs/defconfig/flatmem | 3 + .../kconfigs/defconfig/hardened_usercopy_slub | 2 + .../scripts/series/kconfigs/defconfig/kasan | 2 + .../series/kconfigs/defconfig/kasan_inline | 2 + .../defconfig/kasan_sparsemem_novmemmmap | 5 + .../defconfig/kasan_sparsemem_vmemmmap | 5 + .../series/kconfigs/defconfig/kasan_vmalloc | 2 + .../scripts/series/kconfigs/defconfig/kfence | 3 + .../series/kconfigs/defconfig/legacy_sbi | 4 + .../scripts/series/kconfigs/defconfig/lockdep | 3 + .../scripts/series/kconfigs/defconfig/medany | 2 + .../scripts/series/kconfigs/defconfig/medlow | 2 + .github/scripts/series/kconfigs/defconfig/noc | 3 + .../scripts/series/kconfigs/defconfig/nosmp | 1 + .github/scripts/series/kconfigs/defconfig/pmu | 4 + .../scripts/series/kconfigs/defconfig/preempt | 2 + .../series/kconfigs/defconfig/preempt_rt | 1 + .../series/kconfigs/defconfig/qspinlock | 1 + .../series/kconfigs/defconfig/randomize_base | 2 + .../scripts/series/kconfigs/defconfig/rseq | 1 + .../series/kconfigs/defconfig/rseq_debug | 3 + .../scripts/series/kconfigs/defconfig/size | 1 + .../series/kconfigs/defconfig/sparsemem | 3 + .../series/kconfigs/defconfig/spinwait | 1 + .../series/kconfigs/defconfig/strict_rwx | 1 + .../scripts/series/kconfigs/defconfig/svnapot | 1 + .../series/kconfigs/defconfig/ticket_spinlock | 1 + .../series/kconfigs/defconfig/vmap_stack | 1 + .../scripts/series/kconfigs/ubuntu_defconfig | 5503 +++++++++++++++++ .github/scripts/series/kernel_builder.sh | 35 + .github/scripts/series/kernel_tester.sh | 43 + .github/scripts/series/kselftest_prep.sh | 10 + ...tests-bpf-Add-RISC-V-specific-config.patch | 47 + .../0002-selftests-bpf-Rename-fallback.patch | 70 + ...sts-iommu-Add-RISC-V-specific-config.patch | 30 + ...004-selftests-exec-Remove-static-pie.patch | 40 + .../0005-selftests-hid-Fix-broken-build.patch | 30 + ...Add-missing-net-lib-kselftest-target.patch | 28 + .../patches/0007-BPF-kselftest-fix.patch | 32 + .../patches/0008-BPF-selftest-install.patch | 35 + ...s-clone3-Avoid-fragile-struct-poking.patch | 66 + ...s-clone3-Avoid-fragile-struct-poking.patch | 48 + ...pf-Add-missing-per-arch-include-path.patch | 54 + .../patches/0012-Missing-iommu-config.patch | 29 + ...3-selftests-filesystems-mount-notify.patch | 34 + .../patches/0014-selftests-mm-page_frag.patch | 40 + ...sts-mount_setattr-mount_setattr_test.patch | 47 + ...0016-selftests-pid_namespace-pid_max.patch | 30 + .../0017-selftests-proc-proc-pid-vm.patch | 39 + ...-selftests-riscv-abi-pointer_masking.patch | 39 + .../series/patches/0019-selftests-ublk.patch | 31 + ...ing-utility-script-to-install-target.patch | 31 + .../series/patches/0021-ntsync-typo.patch | 25 + .github/scripts/series/post_to_squad.py | 296 + .github/scripts/series/prepare_rootfs.sh | 111 + .github/scripts/series/qemu_test_utils.sh | 95 + .github/scripts/series/selftest_builder.sh | 24 + .github/scripts/series/test_all.sh | 17 + .github/scripts/series/test_kernel.sh | 206 + .github/scripts/series/test_only_defconfig.sh | 15 + .github/scripts/series/test_only_kselftest.sh | 26 + .../scripts/series/tuxrun_to_squad_json.py | 54 + .github/scripts/series/unpack_fw.sh | 25 + .github/scripts/series/utils.sh | 48 + .github/scripts/sync.sh | 122 + .github/scripts/sync_patchwork.py | 452 ++ .github/scripts/xfstests.sh | 43 + .github/workflows/kselftest.yml | 59 + .github/workflows/patchwork.yml | 64 + .github/workflows/series.yml | 52 + .github/workflows/sync.yml | 82 + .github/workflows/testsuites.yml | 174 + 117 files changed, 11276 insertions(+) create mode 100755 .github/scripts/build_ubuntu_defconfig.sh create mode 100644 .github/scripts/ci/__init__.py create mode 100644 .github/scripts/ci/base.py create mode 100644 .github/scripts/ci/shelltest.py create mode 100755 .github/scripts/cleanup_pr.py create mode 100644 .github/scripts/config.json create mode 100755 .github/scripts/defconfig.sh create mode 100755 .github/scripts/kselftest.sh create mode 100755 .github/scripts/libhugetlbfs.sh create mode 100755 .github/scripts/libs/__init__.py create mode 100755 .github/scripts/libs/context.py create mode 100755 .github/scripts/libs/email.py create mode 100755 .github/scripts/libs/githubtool.py create mode 100755 .github/scripts/libs/patchwork.py create mode 100755 .github/scripts/libs/repotool.py create mode 100755 .github/scripts/libs/utils.py create mode 100755 .github/scripts/ltp.sh create mode 100755 .github/scripts/pw_ci.py create mode 100644 .github/scripts/pw_tests/build_rv32_defconfig.sh create mode 100644 .github/scripts/pw_tests/build_rv64_clang_allmodconfig.sh create mode 100644 .github/scripts/pw_tests/build_rv64_gcc_allmodconfig.sh create mode 100644 .github/scripts/pw_tests/build_rv64_nommu_k210_defconfig.sh create mode 100644 .github/scripts/pw_tests/build_rv64_nommu_virt_defconfig.sh create mode 100644 .github/scripts/pw_tests/checkpatch.sh create mode 100644 .github/scripts/pw_tests/dtb_warn_rv64.sh create mode 100644 .github/scripts/pw_tests/header_inline.sh create mode 100644 .github/scripts/pw_tests/kdoc.sh create mode 100644 .github/scripts/pw_tests/module_param.sh create mode 100644 .github/scripts/pw_tests/verify_fixes.sh create mode 100644 .github/scripts/pw_tests/verify_signedoff.sh create mode 100644 .github/scripts/requirements.txt create mode 100755 .github/scripts/series.sh create mode 100755 .github/scripts/series/build_all.sh create mode 100755 .github/scripts/series/build_kernel.sh create mode 100755 .github/scripts/series/build_only_defconfig.sh create mode 100755 .github/scripts/series/build_only_kselftest.sh create mode 100755 .github/scripts/series/build_selftest.sh create mode 100755 .github/scripts/series/generate_build_configs.sh create mode 100755 .github/scripts/series/generate_kconfigs.sh create mode 100644 .github/scripts/series/generate_metadata.py create mode 100644 .github/scripts/series/generate_qemu_test_configs.sh create mode 100755 .github/scripts/series/generate_test_runs.sh create mode 100644 .github/scripts/series/github_ci_squad_results.py create mode 100644 .github/scripts/series/kconfigs/defconfig/early_boot_alternative create mode 100644 .github/scripts/series/kconfigs/defconfig/early_boot_alternative_reloc create mode 100644 .github/scripts/series/kconfigs/defconfig/flatmem create mode 100644 .github/scripts/series/kconfigs/defconfig/hardened_usercopy_slub create mode 100644 .github/scripts/series/kconfigs/defconfig/kasan create mode 100644 .github/scripts/series/kconfigs/defconfig/kasan_inline create mode 100644 .github/scripts/series/kconfigs/defconfig/kasan_sparsemem_novmemmmap create mode 100644 .github/scripts/series/kconfigs/defconfig/kasan_sparsemem_vmemmmap create mode 100644 .github/scripts/series/kconfigs/defconfig/kasan_vmalloc create mode 100644 .github/scripts/series/kconfigs/defconfig/kfence create mode 100644 .github/scripts/series/kconfigs/defconfig/legacy_sbi create mode 100644 .github/scripts/series/kconfigs/defconfig/lockdep create mode 100644 .github/scripts/series/kconfigs/defconfig/medany create mode 100644 .github/scripts/series/kconfigs/defconfig/medlow create mode 100644 .github/scripts/series/kconfigs/defconfig/noc create mode 100644 .github/scripts/series/kconfigs/defconfig/nosmp create mode 100644 .github/scripts/series/kconfigs/defconfig/pmu create mode 100644 .github/scripts/series/kconfigs/defconfig/preempt create mode 100644 .github/scripts/series/kconfigs/defconfig/preempt_rt create mode 100644 .github/scripts/series/kconfigs/defconfig/qspinlock create mode 100644 .github/scripts/series/kconfigs/defconfig/randomize_base create mode 100644 .github/scripts/series/kconfigs/defconfig/rseq create mode 100644 .github/scripts/series/kconfigs/defconfig/rseq_debug create mode 100644 .github/scripts/series/kconfigs/defconfig/size create mode 100644 .github/scripts/series/kconfigs/defconfig/sparsemem create mode 100644 .github/scripts/series/kconfigs/defconfig/spinwait create mode 100644 .github/scripts/series/kconfigs/defconfig/strict_rwx create mode 100644 .github/scripts/series/kconfigs/defconfig/svnapot create mode 100644 .github/scripts/series/kconfigs/defconfig/ticket_spinlock create mode 100644 .github/scripts/series/kconfigs/defconfig/vmap_stack create mode 100644 .github/scripts/series/kconfigs/ubuntu_defconfig create mode 100755 .github/scripts/series/kernel_builder.sh create mode 100755 .github/scripts/series/kernel_tester.sh create mode 100644 .github/scripts/series/kselftest_prep.sh create mode 100644 .github/scripts/series/patches/0001-selftests-bpf-Add-RISC-V-specific-config.patch create mode 100644 .github/scripts/series/patches/0002-selftests-bpf-Rename-fallback.patch create mode 100644 .github/scripts/series/patches/0003-selftests-iommu-Add-RISC-V-specific-config.patch create mode 100644 .github/scripts/series/patches/0004-selftests-exec-Remove-static-pie.patch create mode 100644 .github/scripts/series/patches/0005-selftests-hid-Fix-broken-build.patch create mode 100644 .github/scripts/series/patches/0006-Add-missing-net-lib-kselftest-target.patch create mode 100644 .github/scripts/series/patches/0007-BPF-kselftest-fix.patch create mode 100644 .github/scripts/series/patches/0008-BPF-selftest-install.patch create mode 100644 .github/scripts/series/patches/0009-selftests-clone3-Avoid-fragile-struct-poking.patch create mode 100644 .github/scripts/series/patches/0010-selftests-clone3-Avoid-fragile-struct-poking.patch create mode 100644 .github/scripts/series/patches/0011-selftests-bpf-Add-missing-per-arch-include-path.patch create mode 100644 .github/scripts/series/patches/0012-Missing-iommu-config.patch create mode 100644 .github/scripts/series/patches/0013-selftests-filesystems-mount-notify.patch create mode 100644 .github/scripts/series/patches/0014-selftests-mm-page_frag.patch create mode 100644 .github/scripts/series/patches/0015-selftests-mount_setattr-mount_setattr_test.patch create mode 100644 .github/scripts/series/patches/0016-selftests-pid_namespace-pid_max.patch create mode 100644 .github/scripts/series/patches/0017-selftests-proc-proc-pid-vm.patch create mode 100644 .github/scripts/series/patches/0018-selftests-riscv-abi-pointer_masking.patch create mode 100644 .github/scripts/series/patches/0019-selftests-ublk.patch create mode 100644 .github/scripts/series/patches/0020-Add-missing-utility-script-to-install-target.patch create mode 100644 .github/scripts/series/patches/0021-ntsync-typo.patch create mode 100755 .github/scripts/series/post_to_squad.py create mode 100755 .github/scripts/series/prepare_rootfs.sh create mode 100644 .github/scripts/series/qemu_test_utils.sh create mode 100755 .github/scripts/series/selftest_builder.sh create mode 100755 .github/scripts/series/test_all.sh create mode 100755 .github/scripts/series/test_kernel.sh create mode 100755 .github/scripts/series/test_only_defconfig.sh create mode 100755 .github/scripts/series/test_only_kselftest.sh create mode 100644 .github/scripts/series/tuxrun_to_squad_json.py create mode 100755 .github/scripts/series/unpack_fw.sh create mode 100644 .github/scripts/series/utils.sh create mode 100755 .github/scripts/sync.sh create mode 100755 .github/scripts/sync_patchwork.py create mode 100755 .github/scripts/xfstests.sh create mode 100644 .github/workflows/kselftest.yml create mode 100644 .github/workflows/patchwork.yml create mode 100644 .github/workflows/series.yml create mode 100644 .github/workflows/sync.yml create mode 100644 .github/workflows/testsuites.yml diff --git a/.github/scripts/build_ubuntu_defconfig.sh b/.github/scripts/build_ubuntu_defconfig.sh new file mode 100755 index 00000000000000..66c9b407f781b4 --- /dev/null +++ b/.github/scripts/build_ubuntu_defconfig.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euox pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + +logs=$(get_logs_dir) +f=${logs}/build_ubuntu_defconfig.log + +date -Iseconds | tee -a ${f} +echo "Build an ubuntu kernel" | tee -a ${f} +echo "Top 16 commits" | tee -a ${f} +git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f} + +kernel_base_sha=$(git log -1 --pretty=%H $(git log -1 --reverse --pretty=%H .github)^) +echo "build_name $(git describe --tags ${kernel_base_sha})" | tee -a ${f} +build_name=$(git describe --tags ${kernel_base_sha}) + +# Build the kernel that will run LTP +export CI_TRIPLE="riscv64-linux-gnu" +cp $d/series/kconfigs/ubuntu_defconfig arch/riscv/configs/ +$d/series/kernel_builder.sh rv64 testsuites plain gcc | tee -a ${f} + +kernel_dir="/build/$(gen_kernel_name rv64 testsuites plain gcc)" +echo $build_name > $kernel_dir/kernel_version +#tar cJvf --exclude $(basename $kernel_path) modules.tar.xz /build/$(gen_kernel_name rv64 testsuites plain gcc)/ diff --git a/.github/scripts/ci/__init__.py b/.github/scripts/ci/__init__.py new file mode 100644 index 00000000000000..05c3610326d57f --- /dev/null +++ b/.github/scripts/ci/__init__.py @@ -0,0 +1,2 @@ +from .base import Base, EndTest, Verdict, submit_pw_check +from .shelltest import ShellTest diff --git a/.github/scripts/ci/base.py b/.github/scripts/ci/base.py new file mode 100644 index 00000000000000..d62af70b5141bf --- /dev/null +++ b/.github/scripts/ci/base.py @@ -0,0 +1,120 @@ +from abc import ABC, abstractmethod +from enum import Enum +import time +import sys + +from libs import utils + +sys.path.insert(0, '../libs') +from libs import log_debug + +class Verdict(Enum): + PENDING = 0 + PASS = 1 + FAIL = 2 + ERROR = 3 + SKIP = 4 + WARNING = 5 + + +class EndTest(Exception): + """ + End of Test + """ + +class Base(ABC): + """ + Base class for CI Tests. + """ + def __init__(self): + self.start_time = 0 + self.end_time = 0 + self.verdict = Verdict.PENDING + self.output = "" + + def success(self): + self.end_timer() + self.verdict = Verdict.PASS + + def error(self, msg): + self.verdict = Verdict.ERROR + self.output = msg + self.end_timer() + raise EndTest + + def warning(self, msg): + self.verdict = Verdict.WARNING + self.output = msg + self.end_timer() + + def skip(self, msg): + self.verdict = Verdict.SKIP + self.output = msg + self.end_timer() + raise EndTest + + def add_failure(self, msg): + self.verdict = Verdict.FAIL + if not self.output: + self.output = msg + else: + self.output += "\n" + msg + + def add_failure_end_test(self, msg): + self.add_failure(msg) + self.end_timer() + raise EndTest + + def start_timer(self): + self.start_time = time.time() + + def end_timer(self): + self.end_time = time.time() + + def elapsed(self): + if self.start_time == 0: + return 0 + if self.end_time == 0: + self.end_timer() + return self.end_time - self.start_time + + def log_err(self, msg): + utils.log_error(f"CI: {self.name}: {msg}") + + def log_info(self, msg): + utils.log_info(f"CI: {self.name}: {msg}") + + def log_dbg(self, msg): + utils.log_debug(f"CI: {self.name}: {msg}") + + @abstractmethod + def run(self, worktree=None): + """ + The child class should implement run() method + If the test fail, it should raise the EndTest exception + """ + pass + + @abstractmethod + def post_run(self): + """ + The child class should implement post_run() method + """ + pass + + +def submit_pw_check(pw, patch, name, verdict, desc, url=None, dry_run=False): + + utils.log_debug(f"Submitting the result to PW: dry_run={dry_run}") + + if not dry_run: + state = 0 + + if verdict == Verdict.PASS: + state = 1 + if verdict == Verdict.WARNING: + state = 2 + if verdict == Verdict.FAIL: + state = 3 + + pw.post_check(patch, name, state, desc, url) diff --git a/.github/scripts/ci/shelltest.py b/.github/scripts/ci/shelltest.py new file mode 100644 index 00000000000000..7f3f56c3ba6f52 --- /dev/null +++ b/.github/scripts/ci/shelltest.py @@ -0,0 +1,67 @@ +from gettext import install +import os +import sys + +sys.path.insert(0, '../libs') +from libs import RepoTool, cmd_run + +from ci import Base, Verdict, EndTest, submit_pw_check + +class ShellTest(Base): + """Run shell test class + This class runs a shell based test + """ + + def __init__(self, ci_data, patch, name, desc, sh): + + # Common + self.name = name + self.desc = desc + self.ci_data = ci_data + + self.sh = sh + self.patch = patch + + super().__init__() + + self.log_dbg("Initialization completed") + + def run(self, worktree=None): + + self.log_dbg("Run") + self.start_timer() + + current_script_path = os.path.dirname(os.path.abspath(__file__)) + + cwd = worktree if worktree else self.ci_data.src_dir + cmd = ["bash", f"{current_script_path}/../pw_tests/{self.sh}"] + (ret, stdout, stderr) = cmd_run(cmd, cwd=cwd) + + if ret == 0: + submit_pw_check(self.ci_data.pw, self.patch, + self.name, Verdict.PASS, + self.name, + None, self.ci_data.config['dry_run']) + self.success() + elif ret == 250: + url = self.ci_data.gh.create_gist(f"pw{self.ci_data.series['id']}-p{self.patch['id']}", + f"{self.name}-WARNING", + stdout + '\n' + stderr) + submit_pw_check(self.ci_data.pw, self.patch, + self.name, Verdict.WARNING, + self.name, + url, self.ci_data.config['dry_run']) + self.warning(stdout + '\n' + stderr) + else: + url = self.ci_data.gh.create_gist(f"pw{self.ci_data.series['id']}-p{self.patch['id']}", + f"{self.name}-FAIL", + stdout + '\n' + stderr) + submit_pw_check(self.ci_data.pw, self.patch, + self.name, Verdict.FAIL, + self.name, + url, self.ci_data.config['dry_run']) + self.error(stdout + '\n' + stderr) + + def post_run(self): + + self.log_dbg("Post Run...") diff --git a/.github/scripts/cleanup_pr.py b/.github/scripts/cleanup_pr.py new file mode 100755 index 00000000000000..691a00c7d353ab --- /dev/null +++ b/.github/scripts/cleanup_pr.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys +import logging +import argparse + +from datetime import datetime +from github import Github + +from libs import init_logger, log_debug, log_error, log_info, pr_get_sid +from libs import GithubTool + +dry_run = False + +MAGIC_LINE = "BlueZ Testbot Message:" +MAGIC_LINE_2 = "BlueZ Testbot Message #2:" +MAGIC_LINE_3 = "BlueZ Testbot Message #3:" +MAGIC_LINE_4 = "BlueZ Testbot Message #4:" + +PATCH_SUBMISSION_MSG = ''' +This is an automated message and please do not change or delete. + +Dear submitter, + +Thanks for submitting the pull request to the BlueZ github repo. +Currently, the BlueZ repo in Github is only for CI and testing purposes, +and not accepting any pull request at this moment. + +If you still want us to review your patch and merge them, please send your +patch to the Linux Bluetooth mailing list(linux-bluetooth@vger.kernel.org). + +For more detail about submitting a patch to the mailing list, +Please refer \"Submitting patches\" section in the HACKING file in the source. + +Note that this pull request will be closed in the near future. + +Best regards, +BlueZ Team +''' + +PATCH_SUBMISSION_MSG_2 = ''' +This is an automated message and please do not change or delete. + +Dear submitter, + +This is a friendly reminder that this pull request will be closed within +a week or two. + +If you already submitted the patches to the Linux Bluetooth mailing list +(linux-bluetooth@vger.kernel.org) for review, Please close this pull +request. + +If you haven't submitted the patches but still want us to review your patch, +please send your patch to the Linux Bluetooth mailing list +(linux-bluetooth@vger.kernel.org). + +For more detail about submitting a patch to the mailing list, +Please refer \"Submitting patches\" section in the HACKING file in the source. + +Note that this pull request will be closed in a week or two. + +Best regards, +BlueZ Team +''' + +PATCH_SUBMISSION_MSG_3 = ''' +This is an automated message and please do not change or delete. + +Dear submitter, + +Thanks for submitting the pull request to the BlueZ github repo. +Currently, the BlueZ repo in Github is only for CI and testing purposes, +and not accepting any pull request at this moment. + +If you still want us to review your patch and merge them, please send your +patch to the Linux Bluetooth mailing list(linux-bluetooth@vger.kernel.org). + +For more detail about submitting a patch to the mailing list, +Please refer \"Submitting patches\" section in the HACKING file in the source. + +Note that this pull request will be closed in the near future. + +Best regards, +BlueZ Team +''' + +PATCH_SUBMISSION_MSG_4 = ''' +This is an automated message and please do not change or delete. + +Closing without taking any action. + +Best regards, +BlueZ Team +''' + +def get_comment_str(magic_line): + """ + Generate the comment string including magic_line + """ + if magic_line == MAGIC_LINE: + msg = PATCH_SUBMISSION_MSG + if magic_line == MAGIC_LINE_2: + msg = PATCH_SUBMISSION_MSG_2 + if magic_line == MAGIC_LINE_3: + msg = PATCH_SUBMISSION_MSG_3 + if magic_line == MAGIC_LINE_4: + msg = PATCH_SUBMISSION_MSG_4 + + return magic_line + "\n\n" + msg + +def get_magic_line(body): + if (body.find(MAGIC_LINE) >= 0): + return MAGIC_LINE + if (body.find(MAGIC_LINE_2) >= 0): + return MAGIC_LINE_2 + if (body.find(MAGIC_LINE_3) >= 0): + return MAGIC_LINE_3 + if (body.find(MAGIC_LINE_4) >= 0): + return MAGIC_LINE_4 + return None + +def pr_add_comment(gh, pr, magic_line): + """ + Add the comment based on magic line + """ + comment = get_comment_str(magic_line) + + log_debug(f"Add PR comments{magic_line}:\n{comment}") + + if dry_run: + log_info("Dry-Run: Skip adding comment to PR") + return + + gh.pr_post_comment(pr, comment) + +def pr_close(gh, pr): + """ + Close pull request + """ + log_debug(f"Close PR{pr.number}") + + if dry_run: + log_info("Dry-Run: Skip closing PR") + return + + gh.pr_close(pr) + +def get_latest_comment(gh, pr): + """ + Search through the comments and find the latest comment + """ + comments = gh.pr_get_issue_comments(pr) + if not comments: + log_error("Unable to get the comments") + return None + + log_info(f"PR#{pr.number} Comment count: {comments.totalCount}") + + for comment in comments.reversed: + magic_line = get_magic_line(comment.body) + if magic_line != None: + log_debug(f"The most recent comment: {magic_line}") + return magic_line + + log_debug("No bluez comment found") + return None + +def update_pull_request(gh, pr, days_created): + + if days_created > 14: + log_debug("Days created > 14") + log_debug("PR is more than 2 weeks and close the PR") + pr_close(gh, pr) + +def manage_pr(gh): + + prs = gh.get_prs(force=True) + log_info(f"Pull Request count: {prs.totalCount}") + + # Handle each PR + for pr in prs: + log_debug(f"Check PR#_{pr.number}") + + # Check if this PR is created with Patchwork series. + # If yes, stop processing. + pw_sid = pr_get_sid(pr.title) + if pw_sid: + log_info(f"PR is created with Patchwork SID: {pw_sid}") + continue + + # Calcuate the number of days since PR was created + delta = datetime.now().astimezone(pr.created_at.tzinfo) - pr.created_at + days_created = delta.days + + log_debug(f"PR opended {days_created} days ago") + + # Update the PR + update_pull_request(gh, pr, days_created) + +def parse_args(): + """ Parse input argument """ + + ap = argparse.ArgumentParser(description="Clean up PR") + ap.add_argument('-d', '--dry-run', action='store_true', default=False, + help='Run it without updating the PR') + # Positional paramter + ap.add_argument("repo", + help="Name of Github repository. i.e. bluez/bluez") + return ap.parse_args() + +def main(): + + global dry_run + + init_logger("ManagePR", verbose=True) + + args = parse_args() + + # Make sure GITHUB_TOKEN exists + if 'GITHUB_TOKEN' not in os.environ: + log_error("Set GITHUB_TOKEN environment variable") + sys.exit(1) + + # Initialize github repo object + try: + gh = GithubTool(args.repo, os.environ['GITHUB_TOKEN']) + except: + log_error("Failed to initialize GithubTool class") + sys.exit(1) + + dry_run = args.dry_run + + manage_pr(gh) + +if __name__ == "__main__": + main() + diff --git a/.github/scripts/config.json b/.github/scripts/config.json new file mode 100644 index 00000000000000..050c106f8da16c --- /dev/null +++ b/.github/scripts/config.json @@ -0,0 +1,28 @@ +{ + "email": { + "server": "smtp.gmail.com", + "port": 587, + "user": "linux.riscv.bot@gmail.com", + "starttls": true, + "default-to": "linux-riscv-reports@googlegroups.com", + "only-maintainers": false, + "maintainers": [ + "linux.riscv.bot@gmail.com" + ] + }, + "patchwork": { + "url": "https://patchwork.kernel.org", + "project_name": "Linux RISC-V" + }, + "space_details": { + "kernel": { + "include": [ + ], + "exclude": [ + "pull request", + "git pull", + "git,pull" + ] + } + } +} diff --git a/.github/scripts/defconfig.sh b/.github/scripts/defconfig.sh new file mode 100755 index 00000000000000..f133b00006bb00 --- /dev/null +++ b/.github/scripts/defconfig.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + +logs=$(get_logs_dir) +f=${logs}/defconfig.log + +date -Iseconds | tee -a ${f} +echo "Top 16 commits" | tee -a ${f} +git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f} + +${d}/series/build_only_defconfig.sh | tee -a ${f} +${d}/series/test_only_defconfig.sh | tee -a ${f} diff --git a/.github/scripts/kselftest.sh b/.github/scripts/kselftest.sh new file mode 100755 index 00000000000000..19ba8442561392 --- /dev/null +++ b/.github/scripts/kselftest.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + +logs=$(get_logs_dir) +f=${logs}/kselftest.log + +date -Iseconds | tee -a ${f} +echo "Build, boot, and run kselftests on various kernels" | tee -a ${f} +echo "Top 16 commits" | tee -a ${f} +git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f} + +kernel_base_sha=$(git log -1 --pretty=%H $(git log -1 --reverse --pretty=%H .github)^) +echo "build_name $(git describe --tags ${kernel_base_sha})" | tee -a ${f} + +${d}/series/build_only_kselftest.sh | tee -a ${f} +${d}/series/test_only_kselftest.sh | tee -a ${f} diff --git a/.github/scripts/libhugetlbfs.sh b/.github/scripts/libhugetlbfs.sh new file mode 100755 index 00000000000000..fbb8d632c18476 --- /dev/null +++ b/.github/scripts/libhugetlbfs.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euox pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + +logs=$(get_logs_dir) +f=${logs}/libhugetlbfs.log + +KERNEL_PATH=$(find "$1" -name '*vmlinu[zx]*') +mv $KERNEL_PATH $KERNEL_PATH.gz +gunzip $KERNEL_PATH.gz + +build_name=$(cat "$1/kernel_version") + +# The Docker image comes with a prebuilt python environment with all tuxrun +# dependencies +source /build/.env/bin/activate + +mkdir -p /build/squad_json/ + +/build/tuxrun/run --runtime null --device qemu-riscv64 --kernel $KERNEL_PATH --tests libhugetlbfs --results /build/squad_json/libhugetlbfs.json --log-file-text /build/squad_json/libhugetlbfs.log --timeouts libhugetlbfs=480 --overlay /build/libhugetlbfs.tar.xz || true + +# Convert JSON to squad datamodel +python3 /build/my-linux/.github/scripts/series/tuxrun_to_squad_json.py --result-path /build/squad_json/libhugetlbfs.json --testsuite libhugetlbfs +python3 /build/my-linux/.github/scripts/series/generate_metadata.py --logs-path /build/squad_json/ --job-url ${GITHUB_JOB_URL} --branch ${GITHUB_BRANCH_NAME} + +curl --header "Authorization: token $SQUAD_TOKEN" --form tests=@/build/squad_json/libhugetlbfs.squad.json --form log=@/build/squad_json/libhugetlbfs.log --form metadata=@/build/squad_json/metadata.json https://squad.di.riseproject.dev/api/submit/riscv-linux/linux-all/${build_name}/qemu diff --git a/.github/scripts/libs/__init__.py b/.github/scripts/libs/__init__.py new file mode 100755 index 00000000000000..eb89c4fbbc9ca2 --- /dev/null +++ b/.github/scripts/libs/__init__.py @@ -0,0 +1,6 @@ +from .utils import init_logger, log_debug, log_error, log_info, cmd_run, pr_get_sid +from .patchwork import Patchwork, PostException +from .email import EmailTool +from .repotool import RepoTool +from .githubtool import GithubTool +from .context import Context diff --git a/.github/scripts/libs/context.py b/.github/scripts/libs/context.py new file mode 100755 index 00000000000000..dc15589e16ba66 --- /dev/null +++ b/.github/scripts/libs/context.py @@ -0,0 +1,95 @@ +import os +import json + +from libs import EmailTool, GithubTool, Patchwork, RepoTool +from libs import log_info, log_debug, log_error + + +class ContextError(Exception): + pass + + +class Context(): + """Collection of data for bzcafe. It is useful for CI""" + + def __init__(self, config_file=None, github_repo=None, src_dir=None, + patch_root=None, **kwargs): + + # Init config + log_info(f"Initialize config file: {config_file}") + self.config = None + if config_file: + with open(os.path.abspath(config_file), 'r') as f: + self.config = json.load(f) + + # Init patchwork + log_info("Initialize patchwork") + try: + self.pw = Patchwork(self.config['patchwork']['url'], + self.config['patchwork']['project_name']) + except: + log_error("Failed to initialize Patchwork class") + raise ContextError + + # If token and username is available, set it here + if 'PATCHWORK_TOKEN' in os.environ and os.environ['PATCHWORK_TOKEN'] != "": + log_debug("Found Patchwork Token in environment variable") + self.pw.set_token(os.environ['PATCHWORK_TOKEN']) + + if 'PATCHWORK_USER' in os.environ and os.environ['PATCHWORK_USER'] != "": + log_debug("Found Patchwork User in environment variable") + self.pw.set_user(int(os.environ['PATCHWORK_USER'])) + + # Init github + log_info(f"Initialize Github: {github_repo}") + if 'GITHUB_TOKEN' not in os.environ: + log_error("Set GITHUB_TOKEN environment variable") + raise ContextError + + if 'GIST_TOKEN' not in os.environ: + log_error("Set GIST_TOKEN environment variable") + raise ContextError + + try: + self.gh = GithubTool(github_repo, os.environ['GITHUB_TOKEN'], + os.environ['GIST_TOKEN']) + except: + log_error("Failed to initialize GithubTool class") + raise ContextError + + # Init email + log_info("Initailze EmailTool") + token = None + if 'EMAIL_TOKEN' in os.environ: + token = os.environ['EMAIL_TOKEN'] + log_info("Email Token is read from environment variable") + + self.email = EmailTool(token=token, config=self.config['email']) + + # Init src_dir + log_info(f"Initialize Source directory: {src_dir}") + try: + self.src_repo = RepoTool(os.path.basename(src_dir), src_dir) + except: + log_error("Failed to initialize RepoTool class") + raise ContextError + self.src_dir = self.src_repo.path() + self.patch_root = patch_root + + # Custome confguration + for kw in kwargs: + log_info(f"Storing {kw}:{kwargs[kw]}") + self.config[kw] = kwargs[kw] + + # These are the frequently used variables by CI + self.series = None + self.patches = None + self.shas = None + + log_info("Context Initialization Completed") + + def update_series(self, series, shas): + self.series = series + self.patches = series['patches'] + self.shas = shas + diff --git a/.github/scripts/libs/email.py b/.github/scripts/libs/email.py new file mode 100755 index 00000000000000..23654a351e3d43 --- /dev/null +++ b/.github/scripts/libs/email.py @@ -0,0 +1,66 @@ +from asyncio import SendfileNotAvailableError +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +import libs + +class EmailTool: + + def __init__(self, server=None, port=None, sender=None, receivers=[], + startls=True, token=None, config=None): + self._server = server + self._port = port + self._sender = sender + self._receivers = receivers + self._starttls = startls + self._token = token + self._message = MIMEMultipart() + + if config: + if 'server' in config: + self._server = config['server'] + if 'port' in config: + self._port = config['port'] + if 'user' in config: + self._sender = config['user'] + if 'startls' in config: + self._startls = config['startls'] + + def send(self): + try: + session = smtplib.SMTP(self._server, self._port) + session.ehlo() + if self._starttls: + session.starttls() + session.ehlo() + session.login(self._sender, self._token) + session.sendmail(self._sender, self._receivers, self._message.as_string()) + except Exception as e: + libs.log_error("Failed to Send email") + libs.log_error(e) + finally: + session.quit() + + libs.log_info("Email sent successfully") + + def set_receivers(self, receivers): + self._receivers = receivers + libs.log_info("Receivers are updated") + + def set_token(self, token): + self._token = token + libs.log_info("Email Token is updated") + + def _update_header(self, headers): + for key, value in headers.items(): + self._message.add_header(key, value) + + def compose(self, title, body, headers): + self._message['From'] = self._sender + self._message['To'] = ", ".join(self._receivers) + self._message['Subject'] = title + self._message.attach(MIMEText(body, 'plain')) + self._update_header(headers) + + libs.log_debug(f"EMAIL Message: \n{self._message}") diff --git a/.github/scripts/libs/githubtool.py b/.github/scripts/libs/githubtool.py new file mode 100755 index 00000000000000..97bf7cf49d1ed4 --- /dev/null +++ b/.github/scripts/libs/githubtool.py @@ -0,0 +1,77 @@ +from github import Github, InputFileContent +import re + +class GithubTool: + + def __init__(self, repo, token=None, gist_token=None): + self._repo = Github(token).get_repo(repo) + self._user = Github(gist_token).get_user() + self._pr = None + self._prs = None + + def get_pr_commits(self, pr_id): + pr = self.get_pr(pr_id, True) + + return pr.get_commits() + + def get_pr(self, pr_id, force=False): + if force or self._pr == None: + self._pr = self._repo.get_pull(pr_id) + + return self._pr + + def get_prs(self, force=False): + if force or not self._prs: + self._prs = self._repo.get_pulls() + + return self._prs + + def create_pr(self, title, body, base, head): + + return self._repo.create_pull(base, head, title=title, body=body, + maintainer_can_modify=True) + + def close_pr(self, pr_id): + pr = self.get_pr(pr_id, force=True) + pr.edit(state="closed") + + git_ref = self._repo.get_git_ref(f"heads/{pr.head.ref}") + git_ref.delete() + + def pr_exist_title(self, str): + if not self._prs: + self._prs = self.get_prs(force=True) + + for pr in self._prs: + if re.search(str, pr.title, re.IGNORECASE): + return True + + return False + + def pr_post_comment(self, pr, comment): + + try: + pr.create_issue_comment(comment) + except: + return False + + return True + + def pr_get_issue_comments(self, pr): + try: + comments = pr.get_issue_comments() + except: + return None + + return comments + + def pr_close(self, pr): + pr.edit(state="closed") + + def create_gist(self, title, test, body): + gist = self._user.create_gist( + public=True, + description=title, + files={test: InputFileContent(body)}) + + return gist.html_url diff --git a/.github/scripts/libs/patchwork.py b/.github/scripts/libs/patchwork.py new file mode 100755 index 00000000000000..c1f8cbb3abd978 --- /dev/null +++ b/.github/scripts/libs/patchwork.py @@ -0,0 +1,181 @@ +import datetime +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +import libs + +class PostException(Exception): + pass + + +class Patchwork(): + + def __init__(self, server, project_name, user=None, token=None, api=None): + self._session = requests.Session() + retry = Retry(connect=10, backoff_factor=1) + adapter = HTTPAdapter(max_retries=retry) + self._session.mount('http://', adapter) + self._session.mount('https://', adapter) + self._server = server + self._token = token + self._user = user + self._project_name = project_name + self._api = "/api" if api == None else f"/api/{api}" + self._project_id= self._get_project_id(project_name) + + libs.log_info(f"Connected to Patchwork Server: {self._server}: {self._project_id}") + + def set_token(self, token): + self._token = token + + def set_user(self, user): + self._user = user + + def _request(self, url): + libs.log_debug(f"PW GET URL: {url}") + resp = self._session.get(url) + if resp.status_code != 200: + raise requests.HTTPError(f"GET {resp.status_code}") + + return resp + + def _get(self, req): + return self._request(f'{self._server}{self._api}/{req}') + + def _get_project(self, name): + projects = self.get_all('projects') + for project in projects: + if project['name'] == name: + return project + + libs.log_error(f"No matched project found: {name}") + return None + + def _get_project_id(self, name): + project = self._get_project(name) + if project: + return project['id'] + + raise ValueError + + def _post(self, req, headers, data): + url = f'{self._server}{self._api}/{req}' + libs.log_debug(f"PW POST URL: {url}") + return self._session.post(url, headers=headers, data=data) + + def get(self, type, identifier): + return self._get(f'{type}/{identifier}/').json() + + def get_all(self, type, filters=None): + if filters is None: + filters={} + params = '' + for key, val in filters.items(): + if val is not None: + params += f'{key}={val}&' + + items = [] + + response = self._get(f'{type}/?{params}') + while response: + for entry in response.json(): + items.append(entry) + + if 'Link' not in response.headers: + break + + links = response.headers['Link'].split(',') + response = None + for link in links: + info = link.split(';') + if info[1].strip() == 'rel="next"': + response = self._request(info[0][1:-1]) + + return items + + def post_check(self, patch, context, state, desc, url=None): + headers = {} + if self._token: + headers['Authorization'] = f'Token {self._token}' + + data = { + 'user': self._user, + 'state': state, + 'target_url': url if url else "", + 'context': context, + 'description': desc + } + + resp = self._post(f"patches/{patch['id']}/checks/", data=data, + headers=headers) + if resp.status_code != 201 and resp.status_code != 200: + libs.log_error(f"PW POST failed: {resp.status_code}") + raise PostException(f"POST {resp.status_code}") + + def get_series_mbox(self, id): + url = f'{self._server}/series/{id}/mbox/' + return self._request(url).content.decode() + + def get_patch_mbox(self, id): + patch = self.get_patch(id) + return self._request(patch['mbox']).content.decode() + + def get_series(self, series_id): + return self.get('series', series_id) + + def get_patch(self, patch_id): + return self.get('patches', patch_id) + + def get_patches_by_state(self, state, archived=False, days_lookback=0): + filter = {} + + filter['project'] = self._project_id + filter['state'] = state + filter['archived'] = 'true' if archived else 'false' + if days_lookback > 0: + today = datetime.datetime.utcnow().date() + lookback = today - datetime.timedelta(days=days_lookback) + filter['since'] = lookback.strftime("%Y-%m-%dT%H:%M:%S") + + + return self.get_all('patches', filter) + + def get_series_by_state(self, state, archived=False, days_lookback=0): + series_ids = [] + series_list = [] + + patches = self.get_patches_by_state(state, archived, days_lookback) + if len(patches) == 0: + return series + + for patch in patches: + # Skip if patch has no series + if 'series' not in patch: + continue + + for series in patch['series']: + # Check if series id already exist + if series['id'] not in series_ids: + series_ids.append(series['id']) + series_list.append(self.get_series(series['id'])) + + return series_list + + def save_patch_mbox(self, patch_id, filename): + patch_mbox = self.get_patch_mbox(patch_id) + + with open(filename, 'w+') as f: + f.write(patch_mbox) + + return filename + + def save_patch_msg(self, patch_id, filename): + patch = self.get_patch(patch_id) + + with open(filename, 'w+') as f: + f.write(patch['name']) + f.write('\n\n') + f.write(patch['content']) + + return filename diff --git a/.github/scripts/libs/repotool.py b/.github/scripts/libs/repotool.py new file mode 100755 index 00000000000000..5276f465c62e98 --- /dev/null +++ b/.github/scripts/libs/repotool.py @@ -0,0 +1,92 @@ +import os +from typing import List + +import libs + + +class RepoToolNotRepo(Exception): + pass + + +class RepoTool: + + def __init__(self, name, path, remote=None): + self._name = name + self._path = os.path.abspath(path) + self._remote = "origin" + self._branch = "master" + + if remote: + self._remote = remote + + # Last executed stdout and stderr + self.stdout = None + self.stderr = None + + self._verify_repo() + libs.log_info(f'Git Repo({self._name}) verified: {self._path}') + + def path(self): + return self._path + + def git(self, args: List[str]): + (ret, self.stdout, self.stderr) = libs.cmd_run(["git"] + args, + cwd=self._path) + return ret + + def _verify_repo(self): + cmd = ["branch", "--show-current"] + + ret = self.git(cmd) + # except: + # libs.log_error("Failed to verify repo") + # raise RepoToolNotRepo + return ret + + def git_checkout(self, branch, create_branch=False): + cmd = ["checkout"] + + if create_branch: + cmd += ["-B"] + + cmd += [branch] + + return self.git(cmd) + + def git_push(self, branch, remote=None, force=False): + cmd = ["push"] + + if force: + cmd += ["-f"] + + if remote: + cmd += [remote] + else: + cmd += [self._remote] + + cmd += [branch] + + return self.git(cmd) + + def git_reset(self, target, hard=False): + cmd = ["reset", target] + + if hard: + cmd += ["--hard"] + + return self.git(cmd) + + def git_am(self, patch=None, abort=False): + cmd = ["am"] + + if abort: + cmd += ["--abort"] + else: + cmd += ["-s", patch] + + return self.git(cmd) + + def git_clean(self): + # Recursively remove all untracked files, not limited to gitignore + return self.git(["clean", "-d", "--force", "-x"]) + diff --git a/.github/scripts/libs/utils.py b/.github/scripts/libs/utils.py new file mode 100755 index 00000000000000..fada7b6c3a379c --- /dev/null +++ b/.github/scripts/libs/utils.py @@ -0,0 +1,105 @@ +import logging +import os +import subprocess +import time +import re +from typing import List, Dict, Tuple + +# Global logging object +logger = None + +def init_logger(name, verbose=False): + global logger + + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + if verbose: + logger.setLevel(logging.DEBUG) + + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s:%(levelname)-8s:%(message)s') + ch.setFormatter(formatter) + + logger.addHandler(ch) + + logger.info("Logger initialized: level=%s", + logging.getLevelName(logger.getEffectiveLevel())) + +def log_info(msg): + if logger is not None: + logger.info(msg) + +def log_error(msg): + if logger is not None: + logger.error(msg) + +def log_debug(msg): + if logger is not None: + logger.debug(msg) + +def pr_get_sid(pr_title): + """ + Parse PR title prefix and get PatchWork Series ID + PR Title Prefix = "[PW_S_ID:] XXXXX" + """ + + try: + sid = re.search(r'^\[PW_SID:([0-9]+)\]', pr_title).group(1) + except AttributeError: + log_error(f"Unable to find the series_id from title {pr_title}") + sid = None + + return sid + +def cmd_run(cmd: List[str], shell: bool = False, add_env: Dict[str, str] = None, + cwd: str = None, pass_fds=()) -> Tuple[str, str, str]: + log_info(f"------------- CMD_RUN -------------") + log_info(f"CMD: {cmd}") + + stdout = "" + + # Update ENV + env = os.environ.copy() + if add_env: + env.update(add_env) + + start_time = time.time() + + proc = subprocess.Popen(cmd, shell=shell, env=env, cwd=cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + bufsize=1, universal_newlines=True, + pass_fds=pass_fds) + log_debug(f"PROC args: {proc.args}") + + # Print the stdout in realtime + for line in proc.stdout: + log_debug("> " + line.rstrip('\n')) + stdout += line + + # STDOUT returned by proc.communicate() is empty because it was all consumed + # by the above read. + _stdout, stderr = proc.communicate() + proc.stdout.close() + proc.stderr.close() + + stderr = "\n" + stderr + if stderr[-1] == "\n": + stderr = stderr[:-1] + + log_info(f'RET: {proc.returncode}') + # No need to print STDOUT here again. It is already printed above + # log_debug(f'STDOUT:{stdout}') + # Print STDOUT only if ret != 0 + if proc.returncode: + log_debug(f'STDERR:{stderr}') + + if proc.returncode != 0: + if stderr and stderr[:-1] == "\n": + stderr = stderr[:-1] + + elapsed = time.time() - start_time + + log_info(f"------------- CMD_RUN END ({elapsed:.2f} s) -------------") + return proc.returncode, stdout, stderr + + diff --git a/.github/scripts/ltp.sh b/.github/scripts/ltp.sh new file mode 100755 index 00000000000000..6eec1bff20fbae --- /dev/null +++ b/.github/scripts/ltp.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euox pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + +logs=$(get_logs_dir) +f=${logs}/ltp.log + +KERNEL_PATH=$(find "$1" -name '*vmlinu[zx]*') +mv $KERNEL_PATH $KERNEL_PATH.gz +gunzip $KERNEL_PATH.gz + +build_name=$(cat "$1/kernel_version") + +# The Docker image comes with a prebuilt python environment with all tuxrun +# dependencies +source /build/.env/bin/activate + +# TODO ltp-controllers is too slow for now because of cgroup_fj_stress.sh +# but I haven't found an easy to skip this one from tuxrun +ltp_tests=( "ltp-commands" "ltp-syscalls" "ltp-mm" "ltp-hugetlb" "ltp-crypto" "ltp-cve" "ltp-containers" "ltp-fs" "ltp-sched" ) + +mkdir -p /build/squad_json/ +parallel_log=$(mktemp -p ${ci_root}) + +for ltp_test in ${ltp_tests[@]}; do + echo "/build/tuxrun/run --runtime null --device qemu-riscv64 --kernel $KERNEL_PATH --tests $ltp_test --results /build/squad_json/$ltp_test.json --log-file-text /build/squad_json/$ltp_test.log --timeouts $ltp_test=480 || true" +done | parallel -j $(($(nproc)/4)) --colsep ' ' --joblog ${parallel_log} + +cat ${parallel_log} +rm ${parallel_log} + +for ltp_test in ${ltp_tests[@]}; do + # Convert JSON to squad datamodel + python3 /build/my-linux/.github/scripts/series/tuxrun_to_squad_json.py --result-path /build/squad_json/$ltp_test.json --testsuite $ltp_test + python3 /build/my-linux/.github/scripts/series/generate_metadata.py --logs-path /build/squad_json/ --job-url ${GITHUB_JOB_URL} --branch ${GITHUB_BRANCH_NAME} + + curl --header "Authorization: token $SQUAD_TOKEN" --form tests=@/build/squad_json/$ltp_test.squad.json --form log=@/build/squad_json/$ltp_test.log --form metadata=@/build/squad_json/metadata.json https://squad.di.riseproject.dev/api/submit/riscv-linux/linux-all/${build_name}/qemu +done diff --git a/.github/scripts/pw_ci.py b/.github/scripts/pw_ci.py new file mode 100755 index 00000000000000..48e7067ad3955c --- /dev/null +++ b/.github/scripts/pw_ci.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys +import argparse +import tempfile + +from libs import init_logger, log_debug, log_error, log_info, pr_get_sid +from libs import Context + +import ci + +def check_args(args): + if not os.path.exists(os.path.abspath(args.config)): + log_error(f"Invalid parameter(config) {args.config}") + return False + + if not os.path.exists(os.path.abspath(args.src_dir)): + log_error(f"Invalid parameter(src_dir) {args.src_dir}") + return False + + return True + +def parse_args(): + ap = argparse.ArgumentParser(description="Run CI tests") + ap.add_argument('-c', '--config', default='./config.json', + help='Configuration file to use. default=./config.json') + ap.add_argument('-s', '--src-dir', required=True, + help='Source directory') + ap.add_argument('-d', '--dry-run', action='store_true', default=False, + help='Run it without uploading the result. default=False') + + # Positional parameter + ap.add_argument("repo", + help="Name of Github repository. i.e. bluez/bluez") + return ap.parse_args() + +# Email Message Templates + +EMAIL_MESSAGE = '''This is automated email and please do not reply to this email! + +Dear submitter, + +Thank you for submitting the patches to the Linux RISC-V mailing list. +This is a CI test results with your patch series: +PW Link:{pw_link} + +---Test result--- +{content} + +--- +Regards, +Linux RISC-V bot + +''' + +def github_pr_post_result(ci_data, i, patch, test): + pr = ci_data.gh.get_pr(ci_data.config['pr_num'], force=True) + + comment = f'Patch {i+1}: "{patch['name']}"\n' + comment += f"**{test.name}**\n" + comment += f"Desc: {test.desc}\n" + comment += f"Duration: {test.elapsed():.2f} seconds\n" + comment += f"**Result: {test.verdict.name}**\n" + + if test.output: + comment += f"Output:\n```\n{test.output}\n```" + + return ci_data.gh.pr_post_comment(pr, comment) + +def is_maintainers_only(email_config): + if 'only-maintainers' in email_config and email_config['only-maintainers']: + return True + return False + +def get_receivers(email_config, submitter): + log_debug("Get the list of email receivers") + + receivers = [] + if is_maintainers_only(email_config): + # Send only to the maintainers + receivers.extend(email_config['maintainers']) + else: + # Send to default-to and submitter + receivers.append(email_config['default-to']) + receivers.append(submitter) + + return receivers + +def send_email(ci_data, content): + headers = {} + email_config = ci_data.config['email'] + + body = EMAIL_MESSAGE.format(pw_link=ci_data.series['web_url'], + content=content) + + headers['In-Reply-To'] = ci_data.patches[0]['msgid'] + headers['References'] = ci_data.patches[0]['msgid'] + + if not is_maintainers_only(email_config): + headers['Reply-To'] = email_config['default-to'] + + receivers = get_receivers(email_config, ci_data.series['submitter']['email']) + ci_data.email.set_receivers(receivers) + ci_data.email.compose("Re: " + ci_data.series['name'], body, headers) + + if ci_data.config['dry_run']: + log_info("Dry-Run is set. Skip sending email") + return + + log_info("Sending Email...") + ci_data.email.send() + +def report_ci(ci_data, test_list): + """Generate the CI result and send email""" + results = "" + summary = "Test Summary:\n" + + line = "{head:<100}{name:<35}{result:<10}{elapsed:.2f} seconds\n" + fail_msg = '{head}\nTest: {name} - {result}\nDesc: {desc}\nOutput:\n{output}\n' + + for i in range(len(test_list)): + sha, tests = test_list[i] + patch = ci_data.patches[i] # 'name' + + for test in tests: + if test.verdict == ci.Verdict.PASS: + # No need to add result of passed tests to simplify the email + summary += line.format(head=f'Patch {i+1}: "{patch['name']}"', + name=test.name, result='PASS', + elapsed=test.elapsed()) + continue + + # Rest of the verdicts use same output format + results += "##############################\n" + results += fail_msg.format(head=f'Patch {i+1}: "{patch['name']}"', + name=test.name, result=test.verdict.name, + desc=test.desc, output=test.output) + summary += line.format(head=f'Patch {i+1}: "{patch['name']}"', + name=test.name, result=test.verdict.name, + elapsed=test.elapsed()) + + if results != "": + results = "Details\n" + results + + send_email(ci_data, summary + '\n' + results) + +def create_test_list(ci_data): + test_list = [] + # XXX ci_config = ci_data.config['space_details']['kernel']['ci'] + + ######################################## + # Test List + ######################################## + + i = 0 + for sha in ci_data.shas: + tests = [] + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "build-rv32-defconfig", + "Builds riscv32 defconfig", + "build_rv32_defconfig.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "build-rv64-clang-allmodconfig", + "Builds riscv64 allmodconfig with Clang, and checks for errors and added warnings", + "build_rv64_clang_allmodconfig.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "build-rv64-gcc-allmodconfig", + "Builds riscv64 allmodconfig with GCC, and checks for errors and added warnings", + "build_rv64_gcc_allmodconfig.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "build-rv64-nommu-k210-defconfig", + "Builds riscv64 defconfig with NOMMU for K210", + "build_rv64_nommu_k210_defconfig.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "build-rv64-nommu-k210-virt", + "Builds riscv64 defconfig with NOMMU for the virt platform", + "build_rv64_nommu_virt_defconfig.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "checkpatch", + "Runs checkpatch.pl on the patch", + "checkpatch.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "dtb-warn-rv64", + "Checks for Device Tree warnings/errors", + "dtb_warn_rv64.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "header-inline", + "Detects static functions without inline keyword in header files", + "header_inline.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "kdoc", + "Detects for kdoc errors", + "kdoc.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "module-param", + "Detect module_param changes", + "module_param.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "verify-fixes", + "Verifies that the Fixes: tags exist", + "verify_fixes.sh")) + + tests.append(ci.ShellTest(ci_data, ci_data.patches[i], + "verify-signedoff", + "Verifies that Signed-off-by: tags are correct", + "verify_signedoff.sh")) + t = (sha, tests) + test_list.append(t) + i += 1 + + return test_list + +def run_ci(ci_data): + num_fails = 0 + + test_list = create_test_list(ci_data) + + log_info(f"Test list is created: {len(test_list)}") + log_debug("+--------------------------+") + log_debug("| Run CI |") + log_debug("+--------------------------+") + + for i in range(len(test_list)): + sha, tests = test_list[i] + patch = ci_data.patches[i] # 'name' + with tempfile.TemporaryDirectory(dir="/build") as worktree: + cmd = ['worktree', 'add', worktree, sha] + if ci_data.src_repo.git(cmd): + log_error(f"Failed to create worktree") + continue + + for test in tests: + log_info("##############################") + log_info(f'## CI: Patch {i+1}: "{patch['name']}"') + log_info(f"## CI: {test.name}") + log_info("##############################") + + try: + test.run(worktree=worktree) + except ci.EndTest as e: + log_error(f"Test Ended(Failure): {test.name}:{test.verdict.name}") + except Exception as e: + log_error(f"Test Ended(Exception): {test.name}: {e.__class__}") + finally: + test.post_run() + + if test.verdict != ci.Verdict.PASS: + num_fails += 1 + + if ci_data.config['dry_run']: + log_info("Skip submitting result to Github: dry_run=True") + continue + + log_debug("Submit the result to github") + if not github_pr_post_result(ci_data, i, patch, test): + log_error("Failed to submit the result to Github") + + log_info(f"Total number of failed test: {num_fails}") + log_debug("+--------------------------+") + log_debug("| ReportCI |") + log_debug("+--------------------------+") + report_ci(ci_data, test_list) + + return num_fails + +def main(): + global config, pw, gh, src_repo, email + + init_logger("PW_CI", verbose=True) + + if 'GITHUB_REF' not in os.environ: + log_error("GITHUB_REF environment not set") + sys.exit(1) + + pr_num = int(os.environ['GITHUB_REF'].removeprefix("refs/pull/").removesuffix("/merge")) + + args = parse_args() + if not check_args(args): + sys.exit(1) + + ci_data = Context(config_file=os.path.abspath(args.config), + github_repo=args.repo, + src_dir=args.src_dir, + dry_run=args.dry_run, + pr_num=pr_num, + space='kernel') + + pr = ci_data.gh.get_pr(pr_num, force=True) + sid = pr_get_sid(pr.title) + + # If PR is not created for Patchwork (no key string), ignore this PR and + # stop running the CI + if not sid: + log_error("Not a valid PR. No need to run") + sys.exit(1) + + cmd = ['log', '-1', '--pretty=%H', '.github/scripts/sync_patchwork.py'] + if ci_data.src_repo.git(cmd): + log_error("Failed to get base commit") + sys.exit(1) + + base_sha = ci_data.src_repo.stdout.strip() + + if len(base_sha) == 0: + log_error("Failed to get base commit") + sys.exit(1) + + cmd = ['rev-list', '--reverse', f'{base_sha}..HEAD'] + if ci_data.src_repo.git(cmd): + log_error("Failed to list of commits") + sys.exit(1) + + shas = ci_data.src_repo.stdout.split() + if len(shas) == 0: + log_error("Failed to get list of commits") + sys.exit(1) + + ci_data.update_series(ci_data.pw.get_series(sid), shas) + + if len(ci_data.shas) != len(ci_data.patches): + log_error("Git and patchwork mismatch") + sys.exit(1) + + num_fails = run_ci(ci_data) + + log_debug("----- DONE -----") + + sys.exit(num_fails) + +if __name__ == "__main__": + main() diff --git a/.github/scripts/pw_tests/build_rv32_defconfig.sh b/.github/scripts/pw_tests/build_rv32_defconfig.sh new file mode 100644 index 00000000000000..45e2891b9722ff --- /dev/null +++ b/.github/scripts/pw_tests/build_rv32_defconfig.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2022 by Rivos Inc. + +tmpdir=$(mktemp -d -p /build) +tmpfile=$(mktemp -p /build) +rc=0 + +tuxmake --wrapper ccache --target-arch riscv --directory . \ + --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \ + --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \ + -o $tmpdir --toolchain llvm -z none -k rv32_defconfig \ + CROSS_COMPILE=riscv64-linux- \ + >$tmpfile 2>&1 || rc=1 + +if [ $rc -ne 0 ]; then + echo "Full log:" + cat $tmpfile + echo "warnings/errors:" + grep "\(warning\|error\):" $tmpfile +fi + +rm -rf $tmpdir $tmpfile + +exit $rc diff --git a/.github/scripts/pw_tests/build_rv64_clang_allmodconfig.sh b/.github/scripts/pw_tests/build_rv64_clang_allmodconfig.sh new file mode 100644 index 00000000000000..0a27666da8328b --- /dev/null +++ b/.github/scripts/pw_tests/build_rv64_clang_allmodconfig.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Netronome Systems, Inc. + +# Modified tests/patch/build_defconfig_warn.sh for RISC-V builds + +tmpfile_e=$(mktemp -p /build) +tmpfile_o=$(mktemp -p /build) +tmpfile_n=$(mktemp -p /build) +tmpdir_b=$(mktemp -d -p /build) +tmpdir_o=$(mktemp -d -p /build) + +rc=0 + +build() { + tuxmake --wrapper ccache --target-arch riscv -e PATH=$PATH --directory . \ + --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \ + --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \ + -o $tmpdir_o -b $tmpdir_b --toolchain llvm -z none --kconfig allmodconfig \ + -K CONFIG_WERROR=n -K CONFIG_RANDSTRUCT_NONE=y -K CONFIG_SAMPLES=n \ + -K CONFIG_DRM_WERROR=n \ + W=1 CROSS_COMPILE=riscv64-linux- \ + config default \ + >$1 2>&1 +} + +echo "Redirect to $tmpfile_o and $tmpfile_n" +echo "Tree base:" +HEAD=$(git rev-parse HEAD) +git log -1 --pretty='%h ("%s")' HEAD~ + +echo "Building the whole tree with the patch" +time build $tmpfile_e || rc=1 +if [ $rc -eq 1 ]; then + echo "error:" + grep "\(error\):" $tmpfile_e + rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b $tmpfile_e + exit $rc +fi + +git checkout -q HEAD~ +echo "Building the tree before the patch" +time build $tmpfile_o +incumbent=$(grep -c "\(warning\|error\):" $tmpfile_o) + +git checkout -q $HEAD +echo "Building the tree with the patch" +time build $tmpfile_n || rc=1 +if [ $rc -eq 1 ]; then + echo "error/warning:" + grep "\(warning\|error\):" $tmpfile_n + rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b + exit $rc +fi + +current=$(grep -c "\(warning\|error\):" $tmpfile_n) +if [ $current -gt $incumbent ]; then + echo "New errors added:" + + tmpfile_errors_before=$(mktemp -p /build) + tmpfile_errors_now=$(mktemp -p /build) + grep "\(warning\|error\):" $tmpfile_o | sort | uniq -c > $tmpfile_errors_before + grep "\(warning\|error\):" $tmpfile_n | sort | uniq -c > $tmpfile_errors_now + + diff -U 0 $tmpfile_errors_before $tmpfile_errors_now + + rm $tmpfile_errors_before $tmpfile_errors_now + + echo "Per-file breakdown" + tmpfile_fo=$(mktemp -p /build) + tmpfile_fn=$(mktemp -p /build) + + echo "error/warning file pre:" + grep "\(warning\|error\):" $tmpfile_o | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \ + > $tmpfile_fo + echo "error/warning file post:" + grep "\(warning\|error\):" $tmpfile_n | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \ + > $tmpfile_fn + + diff -U 0 $tmpfile_fo $tmpfile_fn + rm $tmpfile_fo $tmpfile_fn + echo "pre: $incumbent post: $current" + rc=1 +fi + +rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b $tmpfile_e +exit $rc diff --git a/.github/scripts/pw_tests/build_rv64_gcc_allmodconfig.sh b/.github/scripts/pw_tests/build_rv64_gcc_allmodconfig.sh new file mode 100644 index 00000000000000..36c2b8334fb416 --- /dev/null +++ b/.github/scripts/pw_tests/build_rv64_gcc_allmodconfig.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Netronome Systems, Inc. + +# Modified tests/patch/build_defconfig_warn.sh for RISC-V builds + +tmpfile_e=$(mktemp -p /build) +tmpfile_o=$(mktemp -p /build) +tmpfile_n=$(mktemp -p /build) +tmpdir_b=$(mktemp -d -p /build) +tmpdir_o=$(mktemp -d -p /build) + +rc=0 + +build() { + tuxmake --wrapper ccache --target-arch riscv -e PATH=$PATH --directory . \ + --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \ + --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \ + -o $tmpdir_o -b $tmpdir_b --toolchain gcc -z none --kconfig allmodconfig \ + -K CONFIG_WERROR=n -K CONFIG_GCC_PLUGINS=n \ + -K CONFIG_DRM_WERROR=n \ + W=1 CROSS_COMPILE=riscv64-linux- \ + config default \ + >$1 2>&1 +} + +echo "Redirect to $tmpfile_o and $tmpfile_n" +echo "Tree base:" +HEAD=$(git rev-parse HEAD) +git log -1 --pretty='%h ("%s")' HEAD~ + +echo "Building the whole tree with the patch" +time build $tmpfile_e || rc=1 +if [ $rc -eq 1 ]; then + echo "error:" + grep "\(error\):" $tmpfile_e + rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b $tmpfile_e + exit $rc +fi + +git checkout -q HEAD~ +echo "Building the tree before the patch" +time build $tmpfile_o +incumbent=$(grep -c "\(warning\|error\):" $tmpfile_o) + +git checkout -q $HEAD +echo "Building the tree with the patch" +time build $tmpfile_n || rc=1 +if [ $rc -eq 1 ]; then + echo "error/warning:" + grep "\(warning\|error\):" $tmpfile_n + rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b + exit $rc +fi + +current=$(grep -c "\(warning\|error\):" $tmpfile_n) +if [ $current -gt $incumbent ]; then + echo "New errors added:" + + tmpfile_errors_before=$(mktemp -p /build) + tmpfile_errors_now=$(mktemp -p /build) + grep "\(warning\|error\):" $tmpfile_o | sort | uniq -c > $tmpfile_errors_before + grep "\(warning\|error\):" $tmpfile_n | sort | uniq -c > $tmpfile_errors_now + + diff -U 0 $tmpfile_errors_before $tmpfile_errors_now + + rm $tmpfile_errors_before $tmpfile_errors_now + + echo "Per-file breakdown" + tmpfile_fo=$(mktemp -p /build) + tmpfile_fn=$(mktemp -p /build) + + echo "error/warning file pre:" + grep "\(warning\|error\):" $tmpfile_o | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \ + > $tmpfile_fo + grep "\(warning\|error\):" $tmpfile_n | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \ + > $tmpfile_fn + + diff -U 0 $tmpfile_fo $tmpfile_fn + rm $tmpfile_fo $tmpfile_fn + echo "pre: $incumbent post: $current" + + rc=1 +fi + +rm -rf $tmpdir_o $tmpfile_o $tmpfile_n $tmpdir_b $tmpfile_e +exit $rc diff --git a/.github/scripts/pw_tests/build_rv64_nommu_k210_defconfig.sh b/.github/scripts/pw_tests/build_rv64_nommu_k210_defconfig.sh new file mode 100644 index 00000000000000..18a610bae2df63 --- /dev/null +++ b/.github/scripts/pw_tests/build_rv64_nommu_k210_defconfig.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2022 by Rivos Inc. + +tmpdir=$(mktemp -d -p /build) +tmpfile=$(mktemp -p /build) +rc=0 + +tuxmake --wrapper ccache --target-arch riscv --directory . \ + --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \ + --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \ + -o $tmpdir --toolchain gcc -z none -k nommu_k210_defconfig \ + CROSS_COMPILE=riscv64-linux- \ + >$tmpfile 2>&1 || rc=1 + +if [ $rc -ne 0 ]; then + echo "Full log:" + cat $tmpfile + echo "warnings/errors:" + grep "\(warning\|error\):" $tmpfile +fi + +rm -rf $tmpdir $tmpfile + +exit $rc diff --git a/.github/scripts/pw_tests/build_rv64_nommu_virt_defconfig.sh b/.github/scripts/pw_tests/build_rv64_nommu_virt_defconfig.sh new file mode 100644 index 00000000000000..7e919812342c41 --- /dev/null +++ b/.github/scripts/pw_tests/build_rv64_nommu_virt_defconfig.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2022 by Rivos Inc. + +tmpdir=$(mktemp -d -p /build) +tmpfile=$(mktemp -p /build) +rc=0 + +tuxmake --wrapper ccache --target-arch riscv --directory . \ + --environment=KBUILD_BUILD_TIMESTAMP=@1621270510 \ + --environment=KBUILD_BUILD_USER=tuxmake --environment=KBUILD_BUILD_HOST=tuxmake \ + -o $tmpdir --toolchain gcc -z none -k nommu_virt_defconfig \ + CROSS_COMPILE=riscv64-linux- \ + >$tmpfile 2>&1 || rc=1 + +if [ $rc -ne 0 ]; then + echo "Full log:" + cat $tmpfile + echo "warnings/errors:" + grep "\(warning\|error\):" $tmpfile +fi + +rm -rf $tmpdir $tmpfile + +exit $rc diff --git a/.github/scripts/pw_tests/checkpatch.sh b/.github/scripts/pw_tests/checkpatch.sh new file mode 100644 index 00000000000000..ca3946efbf6038 --- /dev/null +++ b/.github/scripts/pw_tests/checkpatch.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Netronome Systems, Inc. + +IGNORED=\ +COMMIT_LOG_LONG_LINE,\ +MACRO_ARG_REUSE,\ +ALLOC_SIZEOF_STRUCT,\ +NO_AUTHOR_SIGN_OFF,\ +GIT_COMMIT_ID,\ +CAMELCASE + +tmpfile=$(mktemp -p /build) + +./scripts/checkpatch.pl --strict --ignore=$IGNORED -g HEAD | tee $tmpfile + +grep 'total: 0 errors, 0 warnings, 0 checks' $tmpfile +ret=$? + +# return 250 (warning) if there are not errors +[ $ret -ne 0 ] && grep -P 'total: 0 errors, \d+ warnings, \d+ checks' $tmpfile && ret=250 + +if [ $ret -ne 0 ]; then + grep '\(WARNING\|ERROR\|CHECK\): ' $tmpfile | LC_COLLATE=C sort -u +else + grep 'total: ' $tmpfile | LC_COLLATE=C sort -u +fi + +rm $tmpfile + +exit $ret + +# ./scripts/checkpatch.pl --ignore=SPACING_CAST,LONG_LINE,LONG_LINE_COMMENT,LONG_LINE_STRING,LINE_SPACING_STRUCT,FILE_PATH_CHANGES,CAMELCASE,OPEN_ENDED_LINE,AVOID_EXTERNS_HEADER,UNCOMMENTED_DEFINITION diff --git a/.github/scripts/pw_tests/dtb_warn_rv64.sh b/.github/scripts/pw_tests/dtb_warn_rv64.sh new file mode 100644 index 00000000000000..1ab4cfcb8c4fa3 --- /dev/null +++ b/.github/scripts/pw_tests/dtb_warn_rv64.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Netronome Systems, Inc. + +# Modified tests/patch/build_defconfig_warn.sh for RISC-V builds + +tmpfile_o=$(mktemp -p /build) +tmpfile_n=$(mktemp -p /build) + +tmpdir_o=$(mktemp -d -p /build) +tmpdir_n=$(mktemp -d -p /build) + +rc=0 + +echo "Redirect to $tmpfile_o and $tmpfile_n" + +HEAD=$(git rev-parse HEAD) + +echo "Tree base:" +git log -1 --pretty='%h ("%s")' HEAD~ + +git checkout -q HEAD~ + +echo "Building the tree before the patch" + +make -C . O=$tmpdir_o ARCH=riscv CROSS_COMPILE=riscv64-linux- \ + defconfig + +make -C . O=$tmpdir_o ARCH=riscv CROSS_COMPILE=riscv64-linux- \ + dtbs_check W=1 -j$(nproc) \ + 2> >(tee $tmpfile_o) + +incumbent=$(cat $tmpfile_o | grep -v "From schema" | wc -l) + +echo "Building the tree with the patch" + +git checkout -q $HEAD + +make -C . O=$tmpdir_n ARCH=riscv CROSS_COMPILE=riscv64-linux- \ + defconfig + +make -C . O=$tmpdir_n ARCH=riscv CROSS_COMPILE=riscv64-linux- \ + dtbs_check W=1 -j$(nproc) \ + 2> >(tee $tmpfile_n) || rc=1 + +current=$(cat $tmpfile_n | grep -v "From schema" | wc -l) + +if [ $current -gt $incumbent ]; then + echo "Errors and warnings before: $incumbent this patch: $current" + echo "New errors added" + sed -i 's|^.*arch|arch|g' $tmpfile_o + sed -i 's|^.*arch|arch|g' $tmpfile_n + diff -U 0 $tmpfile_o $tmpfile_n + + rc=1 +fi + +rm -rf $tmpdir_o $tmpdir_n $tmpfile_o $tmpfile_n + +exit $rc diff --git a/.github/scripts/pw_tests/header_inline.sh b/.github/scripts/pw_tests/header_inline.sh new file mode 100644 index 00000000000000..3b33d5ebd91ab0 --- /dev/null +++ b/.github/scripts/pw_tests/header_inline.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2020 Facebook + +inlines=$( + git show -- '*.h' | grep -C1 -P '^\+static (?!(__always_)?inline).*\('; + git show -- '*.h' | grep -C1 -P '^\+(static )?(?!(__always_)?inline )((unsigned|long|short) )*(char|bool|void|int|u[0-9]*) [0-9A-Za-z_]*\(.*\) *{' + ) + +if [ -z "$inlines" ]; then + exit 0 +fi + +msg="Detected static functions without inline keyword in header files:" +echo -e "$msg\n$inlines" +count=$( (echo "---"; echo "$inlines") | grep '^---$' | wc -l) +echo "$msg $count" +exit 1 diff --git a/.github/scripts/pw_tests/kdoc.sh b/.github/scripts/pw_tests/kdoc.sh new file mode 100644 index 00000000000000..2fc8141e84790d --- /dev/null +++ b/.github/scripts/pw_tests/kdoc.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Netronome Systems, Inc. +# Copyright (c) 2020 Facebook + +tmpfile_o=$(mktemp -p /build) +tmpfile_n=$(mktemp -p /build) +rc=0 + +files=$(git show --pretty="" --name-only HEAD) + +HEAD=$(git rev-parse HEAD) + +echo "Checking the tree before the patch" +git checkout -q HEAD~ +./scripts/kernel-doc -none $files 2> >(tee $tmpfile_o) + +incumbent=$(grep -v 'Error: Cannot open file ' $tmpfile_o | wc -l) + +echo "Checking the tree with the patch" + +git checkout -q $HEAD +./scripts/kernel-doc -none $files 2> >(tee $tmpfile_n) + +current=$(grep -v 'Error: Cannot open file ' $tmpfile_n | wc -l) + + +if [ $current -gt $incumbent ]; then + echo "Errors and warnings before: $incumbent this patch: $current" + echo "New warnings added" + diff $tmpfile_o $tmpfile_n + + echo "Per-file breakdown" + tmpfile_fo=$(mktemp -p /build) + tmpfile_fn=$(mktemp -p /build) + + grep -i "\(warn\|error\)" $tmpfile_o | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \ + >$tmpfile_fo + grep -i "\(warn\|error\)" $tmpfile_n | sed -n 's@\(^\.\./[/a-zA-Z0-9_.-]*.[ch]\):.*@\1@p' | sort | uniq -c \ + >$tmpfile_fn + + diff $tmpfile_fo $tmpfile_fn + rm $tmpfile_fo $tmpfile_fn + rc=1 +fi + +rm $tmpfile_o $tmpfile_n + +exit $rc diff --git a/.github/scripts/pw_tests/module_param.sh b/.github/scripts/pw_tests/module_param.sh new file mode 100644 index 00000000000000..3af83b63699b14 --- /dev/null +++ b/.github/scripts/pw_tests/module_param.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2020 Facebook + +params=$(git show | grep -i '^\+.*module_param') +new_params=$(git show | grep -ic '^\+.*module_param') +old_params=$(git show | grep -ic '^\-.*module_param') + +echo "Was $old_params now: $new_params" + +if [ -z "$params" ]; then + exit 0 +fi + +echo -e "Detected module_param\n$params" +if [ $new_params -eq $old_params ]; then + exit 250 +fi + +exit 1 diff --git a/.github/scripts/pw_tests/verify_fixes.sh b/.github/scripts/pw_tests/verify_fixes.sh new file mode 100644 index 00000000000000..23795485e4ffab --- /dev/null +++ b/.github/scripts/pw_tests/verify_fixes.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Stephen Rothwell +# Copyright (C) 2019 Greg Kroah-Hartman +# Copyright (c) 2020 Facebook +# +# Verify that the "Fixes:" tag is correct in a kernel commit +# +# usage: +# verify_fixes.sh GIT_RANGE +# +# To test just the HEAD commit do: +# verify_fixes.sh HEAD^..HEAD +# +# +# Thanks to Stephen Rothwell for the majority of this code +# + +# Only thing you might want to change here, the location of where Linus's git +# tree is on your system: + +########################################## +# No need to touch anything below here + +split_re='^([Cc][Oo][Mm][Mm][Ii][Tt])?[[:space:]]*([[:xdigit:]]{5,})([[:space:]]*)(.*)$' +nl=$'\n' +tab=$'\t' + +help() +{ + echo "error, git range not found" + echo "usage:" + echo " $0 GIT_RANGE" + exit 1 +} + +# Strip the leading and training spaces from a string +strip_spaces() +{ + [[ "$1" =~ ^[[:space:]]*(.*[^[:space:]])[[:space:]]*$ ]] + echo "${BASH_REMATCH[1]}" +} + +verify_fixes() +{ + git_range=$1 + error=0 + commits=$(git rev-list --no-merges -i --grep='^[[:space:]]*Fixes:' "${git_range}") + if [ -z "$commits" ]; then + echo "No Fixes tag" + return 0 + fi + + for c in $commits; do + + commit_log=$(git log -1 --format='%h ("%s")' "$c") +# commit_msg="In commit: +# $commit_log +#" + commit_msg="Commit: $commit_log +" + + fixes_lines=$(git log -1 --format='%B' "$c" | + grep -i '^[[:space:]]*Fixes:') + + while read -r fline; do + [[ "$fline" =~ ^[[:space:]]*[Ff][Ii][Xx][Ee][Ss]:[[:space:]]*(.*)$ ]] + f="${BASH_REMATCH[1]}" +# fixes_msg=" Fixes tag: +# $fline +# Has these problem(s): +#" + fixes_msg=" Fixes tag: $fline + Has these problem(s): +" + sha= + subject= + msg= + + if git log -1 --format='%B' "$c" | tr '\n' '#' | grep -qF "##$fline##"; then + msg="${msg:+${msg}${nl}}${tab}${tab}- empty lines surround the Fixes tag" + error=$(( error + 1 )) + fi + + if [[ "$f" =~ $split_re ]]; then + first="${BASH_REMATCH[1]}" + sha="${BASH_REMATCH[2]}" + spaces="${BASH_REMATCH[3]}" + subject="${BASH_REMATCH[4]}" + if [ "$first" ]; then + msg="${msg:+${msg}${nl}}${tab}${tab}- leading word '$first' unexpected" + error=$(( error + 1 )) + fi + if [ -z "$subject" ]; then + msg="${msg:+${msg}${nl}}${tab}${tab}- missing subject" + error=$(( error + 1 )) + elif [ -z "$spaces" ]; then + msg="${msg:+${msg}${nl}}${tab}${tab}- missing space between the SHA1 and the subject" + error=$(( error + 1 )) + fi + else + printf '%s%s\t\t- %s\n' "$commit_msg" "$fixes_msg" 'No SHA1 recognised' + error=$(( error + 1 )) + commit_msg='' + continue + fi + if ! git rev-parse -q --verify "$sha" >/dev/null; then + printf '%s%s\t\t- %s\n' "$commit_msg" "$fixes_msg" 'Target SHA1 does not exist' + error=$(( error + 1 )) + commit_msg='' + continue + fi + + if [ "${#sha}" -lt 12 ]; then + msg="${msg:+${msg}${nl}}${tab}${tab}- SHA1 should be at least 12 digits long${nl}${tab}${tab} Can be fixed by setting core.abbrev to 12 (or more) or (for git v2.11${nl}${tab}${tab} or later) just making sure it is not set (or set to \"auto\")." + error=$(( error + 1 )) + fi + # reduce the subject to the part between () if there + if [[ "$subject" =~ ^\((.*)\) ]]; then + subject="${BASH_REMATCH[1]}" + elif [[ "$subject" =~ ^\((.*) ]]; then + subject="${BASH_REMATCH[1]}" + msg="${msg:+${msg}${nl}}${tab}${tab}- Subject has leading but no trailing parentheses" + error=$(( error + 1 )) + fi + + # strip matching quotes at the start and end of the subject + # the unicode characters in the classes are + # U+201C LEFT DOUBLE QUOTATION MARK + # U+201D RIGHT DOUBLE QUOTATION MARK + # U+2018 LEFT SINGLE QUOTATION MARK + # U+2019 RIGHT SINGLE QUOTATION MARK + re1=$'^[\"\u201C](.*)[\"\u201D]$' + re2=$'^[\'\u2018](.*)[\'\u2019]$' + re3=$'^[\"\'\u201C\u2018](.*)$' + if [[ "$subject" =~ $re1 ]]; then + subject="${BASH_REMATCH[1]}" + elif [[ "$subject" =~ $re2 ]]; then + subject="${BASH_REMATCH[1]}" + elif [[ "$subject" =~ $re3 ]]; then + subject="${BASH_REMATCH[1]}" + msg="${msg:+${msg}${nl}}${tab}${tab}- Subject has leading but no trailing quotes" + error=$(( error + 1 )) + fi + + subject=$(strip_spaces "$subject") + + target_subject=$(git log -1 --format='%s' "$sha") + target_subject=$(strip_spaces "$target_subject") + + # match with ellipses + case "$subject" in + *...) subject="${subject%...}" + target_subject="${target_subject:0:${#subject}}" + ;; + ...*) subject="${subject#...}" + target_subject="${target_subject: -${#subject}}" + ;; + *\ ...\ *) + s1="${subject% ... *}" + s2="${subject#* ... }" + subject="$s1 $s2" + t1="${target_subject:0:${#s1}}" + t2="${target_subject: -${#s2}}" + target_subject="$t1 $t2" + ;; + esac + subject=$(strip_spaces "$subject") + target_subject=$(strip_spaces "$target_subject") + + if [ "$subject" != "${target_subject:0:${#subject}}" ]; then + msg="${msg:+${msg}${nl}}${tab}${tab}- Subject does not match target commit subject${nl}${tab}${tab} Just use${nl}${tab}${tab}${tab}git log -1 --format='Fixes: %h (\"%s\")'" + error=$(( error + 1 )) + fi + lsha=$(git rev-parse -q --verify "$sha") + if [ -z "$lsha" ]; then + count=$(git rev-list --count "$sha".."$c") + if [ "$count" -eq 0 ]; then + msg="${msg:+${msg}${nl}}${tab}${tab}- Target is not an ancestor of this commit" + error=$(( error + 1 )) + fi + fi + + if [ "$msg" ]; then + printf '%s%s%s\n' "$commit_msg" "$fixes_msg" "$msg" + commit_msg='' + # Make sure we don't accidentally miss anything. + if [ $error -eq 0 ]; then + echo 'Whoops! $error out of sync with $msg' + error=1 + fi + fi + done <<< "$fixes_lines" + done + +if [ ${error} -ne 0 ] ; then + echo "Problems with Fixes tag: $error" + exit 1 + fi + echo "Fixes tag looks correct" + return 0 +} + +git_range="HEAD~..HEAD" + +if [ "${git_range}" == "" ] ; then + help +fi + +verify_fixes "${git_range}" +exit 0 diff --git a/.github/scripts/pw_tests/verify_signedoff.sh b/.github/scripts/pw_tests/verify_signedoff.sh new file mode 100644 index 00000000000000..efd2366cb52195 --- /dev/null +++ b/.github/scripts/pw_tests/verify_signedoff.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2019 Stephen Rothwell +# Copyright (C) 2019 Greg Kroah-Hartman +# +# Verify that the signed-off-by chain looks correct for a range of git commits. +# +# usage: +# verify_signedoff.sh GIT_RANGE +# +# To test just the HEAD commit do: +# verify_signedoff.sh HEAD^..HEAD +# +# +# Thanks to Stephen Rothwell for the majority of this code +# + +help() +{ + echo "error, git range not found" + echo "usage:" + echo " $0 GIT_RANGE" + exit 1 +} + +verify_signedoff() +{ + git_range=$1 + error=false + for c in $(git rev-list --no-merges "${git_range}"); do + ae=$(git log -1 --format='%ae' "$c") + aE=$(git log -1 --format='%aE' "$c") + an=$(git log -1 --format='%an' "$c") + aN=$(git log -1 --format='%aN' "$c") + ce=$(git log -1 --format='%ce' "$c") + cE=$(git log -1 --format='%cE' "$c") + cn=$(git log -1 --format='%cn' "$c") + cN=$(git log -1 --format='%cN' "$c") + sob=$(git log -1 --format='%b' "$c" | grep -i '^[[:space:]]*Signed-off-by:') + + am=false + cm=false + grep -i -q "<$ae>" <<<"$sob" || + grep -i -q "<$aE>" <<<"$sob" || + grep -i -q ":[[:space:]]*${an}[[:space:]]*<" <<<"$sob" || + grep -i -q ":[[:space:]]*${aN}[[:space:]]*<" <<<"$sob" || + am=true + grep -i -q "<$ce>" <<<"$sob" || + grep -i -q "<$cE>" <<<"$sob" || + grep -i -q ":[[:space:]]*${cn}[[:space:]]*<" <<<"$sob" || + grep -i -q ":[[:space:]]*${cN}[[:space:]]*<" <<<"$sob" || + cm=true + + if "$am" || "$cm"; then + printf "Commit %s\n" "$(git show -s --abbrev-commit --abbrev=12 --pretty=format:"%h (\"%s\")%n" "${c}")" + "$am" && printf "\tauthor Signed-off-by missing\n" + "$cm" && printf "\tcommitter Signed-off-by missing\n" + printf "\tauthor email: %s\n" "$ae" + printf "\tcommitter email: %s\n" "$ce" + readarray -t s <<< "${sob}" + printf "\t%s\n" "${s[@]}" + printf "\n" + error=true + fi + done + if "$error"; then + echo "Errors in tree with Signed-off-by, please fix!" + exit 1 + fi + echo "Signed-off-by tag matches author and committer" +} + +git_range="HEAD~..HEAD" + +if [ "${git_range}" == "" ] ; then + help +fi + +verify_signedoff "${git_range}" +exit 0 diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt new file mode 100644 index 00000000000000..118fc389f3ad00 --- /dev/null +++ b/.github/scripts/requirements.txt @@ -0,0 +1,5 @@ +PyGithub +python-lsp-server +requests +ply +GitPython diff --git a/.github/scripts/series.sh b/.github/scripts/series.sh new file mode 100755 index 00000000000000..c7d28d7f3240cb --- /dev/null +++ b/.github/scripts/series.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + + +logs=$(get_logs_dir) +f=${logs}/series.log + +date -Iseconds | tee -a ${f} +echo "Build, and boot various kernels" | tee -a ${f} +echo "Top 16 commits" | tee -a ${f} +git log -16 --abbrev=12 --pretty="commit %h (\"%s\")" | tee -a ${f} + +kernel_base_sha=$(git log -1 --pretty=%H $(git log -1 --reverse --pretty=%H .github)^) +echo "build_name $(git describe --tags ${kernel_base_sha})" | tee -a ${f} +build_name=$(git describe --tags ${kernel_base_sha}) + +${d}/series/build_all.sh | tee -a ${f} +${d}/series/test_all.sh | tee -a ${f} + +# Some logs contain invalid bytes (not utf-8) and then makes the following +# script fail so convert them all. +for f in `ls ${logs}`; do + iconv -c -t utf-8 ${logs}/${f} > ${logs}/${f}_tmp || true + mv ${logs}/${f}_tmp ${logs}/${f} || true +done + +python3 ${d}/series/github_ci_squad_results.py --logs-path ${logs} +python3 ${d}/series/generate_metadata.py --logs-path ${logs} \ + --job-url ${GITHUB_JOB_URL} --branch ${GITHUB_BRANCH_NAME} + +curl --header "Authorization: token ${SQUAD_TOKEN}" \ + --form tests=@${logs}/squad.json \ + --form metadata=@${logs}/metadata.json \ + https://squad.di.riseproject.dev/api/submit/riscv-linux/linux-all/${build_name}/qemu diff --git a/.github/scripts/series/build_all.sh b/.github/scripts/series/build_all.sh new file mode 100755 index 00000000000000..44082e2861ef8e --- /dev/null +++ b/.github/scripts/series/build_all.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +rc=0 +while read xlen config fragment toolchain; do + ${d}/kernel_builder.sh $xlen $config $fragment $toolchain || rc=1 + if [[ $config == "kselftest" ]]; then + ${d}/selftest_builder.sh $xlen $config $fragment $toolchain || rc=1 + fi +done < <($d/generate_build_configs.sh) +exit 0 diff --git a/.github/scripts/series/build_kernel.sh b/.github/scripts/series/build_kernel.sh new file mode 100755 index 00000000000000..1d43196c68db47 --- /dev/null +++ b/.github/scripts/series/build_kernel.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -x +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +lnxroot=$(pwd) + +# E.g. build_kernel.sh rv32 defconfig /path/to/fragment llvm +# build_kernel.sh rv64 defconfig plain gcc + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 + +install=${ci_root}/$(gen_kernel_name $xlen $config $fragment $toolchain) +output=${install}_build +triple=${ci_triple} + +keep_build=0 # enable for kselftest + +make_gcc() { + make O=$output ARCH=riscv CROSS_COMPILE=${triple}- \ + "CC=ccache ${triple}-gcc" 'HOSTCC=ccache gcc' $* +} + +make_llvm() { + make O=$output ARCH=riscv CROSS_COMPILE=${triple}- \ + LLVM=1 LLVM_IAS=1 'CC=ccache clang' 'HOSTCC=ccache clang' $* +} + +make_wrap() { + if [[ $toolchain == "llvm" ]]; then + make_llvm $* + elif [[ $toolchain == "gcc-old" ]]; then + oldpath=${PATH} + export PATH=/opt/gcc-old/riscv64-linux/bin:${oldpath} + make_gcc $* + export PATH=${oldpath} + else + make_gcc $* + fi +} + +rm -rf ${output} +rm -rf ${install} +mkdir -p ${output} +mkdir -p ${install} + +if [[ $config == "allmodconfig" || $config == "randconfig" ]]; then + make_wrap KCONFIG_ALLCONFIG=$lnxroot/arch/riscv/configs/${xlen//rv/}-bit.config $config + $lnxroot/scripts/kconfig/merge_config.sh -m -O $output $output/.config \ + <(echo "CONFIG_WERROR=n") \ + <(echo "CONFIG_DRM_WERROR=n") \ + <(echo "CONFIG_GCC_PLUGINS=n") +elif [[ $config == "kselftest" ]]; then + apply_patches + trap unapply_patches EXIT + make_wrap defconfig + make_wrap kselftest-merge + $lnxroot/scripts/kconfig/merge_config.sh -y -m -O $output $output/.config \ + <(echo "CONFIG_KERNEL_UNCOMPRESSED=y") + make_wrap olddefconfig + keep_build=1 +elif [[ $config == "testsuites" ]]; then + make_wrap ubuntu_defconfig +else + if [[ $fragment == "plain" ]]; then + $lnxroot/scripts/kconfig/merge_config.sh -y -m -O $output $lnxroot/arch/riscv/configs/$config \ + $lnxroot/arch/riscv/configs/${xlen//rv/}-bit.config \ + <(echo "CONFIG_KERNEL_UNCOMPRESSED=y") + else + $lnxroot/scripts/kconfig/merge_config.sh -y -m -O $output $lnxroot/arch/riscv/configs/$config \ + $fragment \ + $lnxroot/arch/riscv/configs/${xlen//rv/}-bit.config \ + <(echo "CONFIG_KERNEL_UNCOMPRESSED=y") + fi + make_wrap olddefconfig +fi + +make_wrap -j $(nproc) -Oline + +make_wrap INSTALL_PATH=${install} install +make_wrap INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=${install} modules_install || true + +if ! (( ${keep_build} )); then + rm -rf ${output} +fi diff --git a/.github/scripts/series/build_only_defconfig.sh b/.github/scripts/series/build_only_defconfig.sh new file mode 100755 index 00000000000000..23aad429a41c2d --- /dev/null +++ b/.github/scripts/series/build_only_defconfig.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +rc=0 +${d}/kernel_builder.sh rv64 defconfig plain gcc || rc=1 +exit $rc diff --git a/.github/scripts/series/build_only_kselftest.sh b/.github/scripts/series/build_only_kselftest.sh new file mode 100755 index 00000000000000..b935e6d814a34c --- /dev/null +++ b/.github/scripts/series/build_only_kselftest.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh +. $d/kselftest_prep.sh + +rc=0 +${d}/kernel_builder.sh rv64 kselftest plain gcc || rc=1 +${d}/selftest_builder.sh rv64 kselftest plain gcc || rc=1 +exit $rc diff --git a/.github/scripts/series/build_selftest.sh b/.github/scripts/series/build_selftest.sh new file mode 100755 index 00000000000000..7e3434b7185516 --- /dev/null +++ b/.github/scripts/series/build_selftest.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -x +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +lnxroot=$(pwd) + +# E.g. build_selftest.sh rv64 kselftest-bpf plain gcc + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 + +if ! [[ "$config" =~ ^kselftest ]]; then + echo "Not a selftest config: please try kselftest kselftest-bpf kselftest-net" + exit 1 +fi + +install=${ci_root}/$(gen_kernel_name $xlen $config $fragment $toolchain) +output=${install}_build +triple=${ci_triple} + +if ! [[ -d $output ]]; then + echo "Cannot find kernel build" + exit 1 +fi + +make_gcc() { + make O=$output ARCH=riscv CROSS_COMPILE=${triple}- \ + "CC=${triple}-gcc" 'HOSTCC=gcc' $* +} + +make_llvm() { + make O=$output ARCH=riscv CROSS_COMPILE=${triple}- \ + LLVM=1 LLVM_IAS=1 'CC=clang' 'HOSTCC=clang' $* +} + +make_wrap() { + if [ $toolchain == "llvm" ]; then + make_llvm $* + else + make_gcc $* + fi +} + +apply_patches +trap unapply_patches EXIT + +make_wrap -j $(($(nproc)-1)) headers + +make_wrap SKIP_TARGETS="bpf livepatch" -j $(($(nproc)-1)) -C tools/testing/selftests install +make_wrap TARGETS="bpf" SKIP_TARGETS="" -j $(($(nproc)-1)) -C tools/testing/selftests +make_wrap TARGETS="bpf" SKIP_TARGETS="" COLLECTION="bpf" -j $(($(nproc)-1)) \ + -C tools/testing/selftests/bpf emit_tests | grep -e '^bpf:' \ + >> $output/kselftest/kselftest_install/kselftest-list.txt +cp -R $output/kselftest/bpf $output/kselftest/kselftest_install + +logs=$(get_logs_dir) + +cat $output/kselftest/kselftest_install/kselftest-list.txt \ + | awk -F : '{print "kselftest-"$1}' \ + | uniq | grep -v 'amd\|intel\|livepatch\|lkdtm\|breakpoints\|tdx\|x86' > $logs/kselftest-collections.txt diff --git a/.github/scripts/series/generate_build_configs.sh b/.github/scripts/series/generate_build_configs.sh new file mode 100755 index 00000000000000..2e657bbbb70863 --- /dev/null +++ b/.github/scripts/series/generate_build_configs.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") + +toolchains="gcc llvm" + +# SKIP_BUILD_CONFIG_TOOLCHAIN= + +do_generate_toolchain () { + local toolchain=$1 + + if [ ! -z "${SKIP_BUILD_CONFIG_TOOLCHAIN:-}" ] && echo $toolchain | egrep -wq "$SKIP_BUILD_CONFIG_TOOLCHAIN"; then + return 1 + fi + return 0 +} + +while read xlen config fragment; do + if [ $xlen == "rv32" ] && [[ "$config" =~ "k210" ]]; then + continue + fi + + if do_generate_toolchain "gcc"; then + echo $xlen $config $fragment gcc + fi + if do_generate_toolchain "llvm"; then + echo $xlen $config $fragment llvm + fi +done < <($d/generate_kconfigs.sh) + +echo rv64 allmodconfig plain gcc-old diff --git a/.github/scripts/series/generate_kconfigs.sh b/.github/scripts/series/generate_kconfigs.sh new file mode 100755 index 00000000000000..27b0f664b31ff6 --- /dev/null +++ b/.github/scripts/series/generate_kconfigs.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +lnxroot=$(pwd) +kconfigs=$d/kconfigs + +builtin_skip="rv32_defconfig" +builtin_allow="" + +# Too much? Override by uncommenting below: +# echo rv64 defconfig "" && exit 0 + +print() { + if [ ! -z "${SKIP_KCONFIG:-}" ]; then + if echo $* | egrep -q "$SKIP_KCONFIG"; then + return + fi + fi + if [ ! -z "$builtin_skip" ]; then + if echo $* | egrep -q "$builtin_skip"; then + return + fi + fi + if [ ! -z "${ALLOW_KCONFIG:-}" ]; then + if echo $* | egrep -q "$ALLOW_KCONFIG"; then + echo $* + fi + return + fi + if [ ! -z "$builtin_allow" ]; then + if echo $* | egrep -q "$builtin_allow"; then + echo $* + fi + return + fi + + echo $* +} + +defconfigs=$(find $lnxroot/arch/riscv/configs/ -type f -name '*defconfig' -printf '%f\n') +for i in $defconfigs; do + for xlen in 32 64; do + frags=$(echo $i && find $kconfigs/$i -type f -printf '%f\n' 2>/dev/null || :) + for frag in $frags; do + if [ $frag == $i ]; then + fn=${xlen}__${i} + fr="plain" + else + fn=${xlen}_${frag}__$i + fr=$(readlink -f $kconfigs/$i/$frag) + fi + + print rv$xlen $i $fr + done + done +done + +#special case set KCONFIG_ALLCONFIG +print rv32 allmodconfig "plain" +print rv64 allmodconfig "plain" diff --git a/.github/scripts/series/generate_metadata.py b/.github/scripts/series/generate_metadata.py new file mode 100644 index 00000000000000..8249df2387bbf9 --- /dev/null +++ b/.github/scripts/series/generate_metadata.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse +import json + +def parse_args(): + parser = argparse.ArgumentParser(description = 'Output Squad metadata') + parser.add_argument("--logs-path", default = "", + help = 'Path to the logs directory') + parser.add_argument("--job-url", default = "", + help = 'URL to the current Github job') + parser.add_argument("--branch", default = "", + help = 'Branch name of the current Github job') + + return parser.parse_args() + +def generate_squad_json(logs_path, job_url, branch): + dict_results = {} + + dict_results["job_url"] = job_url + dict_results["branch"] = branch + + with open(logs_path + "/" + "metadata.json", "w") as f: + json.dump(dict_results, f) + +if __name__ == "__main__": + args = parse_args() + generate_squad_json(args.logs_path, args.job_url, args.branch) + diff --git a/.github/scripts/series/generate_qemu_test_configs.sh b/.github/scripts/series/generate_qemu_test_configs.sh new file mode 100644 index 00000000000000..e62ea9fb06c916 --- /dev/null +++ b/.github/scripts/series/generate_qemu_test_configs.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 +rootfs=$5 + +rv64_cpus=( + "rv64" + "rv64,v=true,vlen=256,elen=64,h=true,zbkb=on,zbkc=on,zbkx=on,zkr=on,zkt=on,svinval=on,svnapot=on,svpbmt=on" +) + +list_cpus+=( "sifive-u54" ) + + +rv32_cpus=( + "rv32" +) diff --git a/.github/scripts/series/generate_test_runs.sh b/.github/scripts/series/generate_test_runs.sh new file mode 100755 index 00000000000000..4d987e7e7abf92 --- /dev/null +++ b/.github/scripts/series/generate_test_runs.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") + +rv64_rootfs="alpine ubuntu" +rv32_rootfs="buildroot_glibc" + +# SKIP_TEST_RUN_ROOTFS= + +print() { + if [ ! -z "${SKIP_TEST_RUN_ROOTFS:-}" ]; then + if echo $* | egrep -wq "$SKIP_TEST_RUN_ROOTFS"; then + return + fi + fi + + echo $* +} + +while read xlen config fragment image toolchain; do + if [[ "$config" =~ "nommu" ]]; then + continue + fi + if [[ "$config" =~ "allmodconfig" ]]; then + continue + fi + if [[ "$config" =~ "randconfig" ]]; then + continue + fi + if [[ "$config" =~ "kselftest" ]]; then + print $xlen $config $fragment $image $toolchain ubuntu + continue + fi + + if [[ $xlen == "rv64" ]]; then + print $xlen $config $fragment $image $toolchain alpine + print $xlen $config $fragment $image $toolchain ubuntu + else + print $xlen $config $fragment $image $toolchain buildroot_glibc + print $xlen $config $fragment $image $toolchain buildroot_musl + fi +done < <($d/generate_build_configs.sh) diff --git a/.github/scripts/series/github_ci_squad_results.py b/.github/scripts/series/github_ci_squad_results.py new file mode 100644 index 00000000000000..4da6e7607de169 --- /dev/null +++ b/.github/scripts/series/github_ci_squad_results.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import argparse +import json + +def parse_args(): + parser = argparse.ArgumentParser(description = 'Output Squad tests results for the Github CI') + parser.add_argument("--logs-path", default = "", + help = 'Path to the log files') + + return parser.parse_args() + +def generate_squad_json(logs_path): + dict_results = {} + + with open(logs_path + "/series.log", "r") as f: + logs_content = f.readlines() + + for line in logs_content: + # ::notice::OK Build kernel rv64__nommu_k210_sdcard_defconfig__plain__gcc took 15.74s + if not line.startswith("::notice::") and not line.startswith("::error::"): + continue + + # We parse only the Builds to get the build name and then + # we add all the corresponding tests + + # Either "Build" or "Test" + type_result = line.split(" ")[1] + if type_result == "Test": + break + + build_name = line.split(" ")[3] + time = line.split(" ")[-1] + + with open(logs_path + "/build_kernel___" + build_name + ".log", "r") as f: + build_content = f.read() + + if line.split(" ")[0] == "::error::FAIL": + dict_results[build_name + "/build"] = { "result": "fail", "time": time, "log": build_content } + continue + + dict_results[build_name + "/build"] = { "result": "pass", "time": time, "log": build_content } + + # The build succeeded, so look for the associated tests + for test_line in logs_content: + if not test_line.startswith("::notice::") and not test_line.startswith("::error::"): + continue + + type_result = test_line.split(" ")[1] + if type_result == "Build": + continue + + test_build_name = test_line.split(" ")[3] + if test_build_name != build_name: + continue + + rootfs = test_line.split(" ")[4] + test_type = test_line.split(" ")[5] + "__" + test_line.split(" ")[6] + "__" + test_line.split(" ")[7] + "__" + test_line.split(" ")[8] + + log_name = "test_kernel" + "___" + test_build_name + "___" + rootfs + "___" + test_type + ".log" + with open(logs_path + "/" + log_name, "r") as f: + log_content = f.read() + + if test_line.split(" ")[0] == "::error::FAIL": + result = "fail" + else: + result = "pass" + + dict_results[build_name + "/" + rootfs + "___" + test_type] = { "result": result, "log": log_content } + + + with open(logs_path + "/squad.json", "w") as f: + json.dump(dict_results, f) + +if __name__ == "__main__": + args = parse_args() + generate_squad_json(args.logs_path) + diff --git a/.github/scripts/series/kconfigs/defconfig/early_boot_alternative b/.github/scripts/series/kconfigs/defconfig/early_boot_alternative new file mode 100644 index 00000000000000..e0876280809bfd --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/early_boot_alternative @@ -0,0 +1,2 @@ +CONFIG_RISCV_ALTERNATIVE=y +CONFIG_RISCV_ALTERNATIVE_EARLY=y diff --git a/.github/scripts/series/kconfigs/defconfig/early_boot_alternative_reloc b/.github/scripts/series/kconfigs/defconfig/early_boot_alternative_reloc new file mode 100644 index 00000000000000..55bd04829bdc15 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/early_boot_alternative_reloc @@ -0,0 +1,3 @@ +CONFIG_RELOCATABLE=y +CONFIG_RISCV_ALTERNATIVE=y +CONFIG_RISCV_ALTERNATIVE_EARLY=y diff --git a/.github/scripts/series/kconfigs/defconfig/flatmem b/.github/scripts/series/kconfigs/defconfig/flatmem new file mode 100644 index 00000000000000..a0e1ed12d66f58 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/flatmem @@ -0,0 +1,3 @@ +# CONFIG_SPARSEMEM_MANUAL is not set +CONFIG_FLATMEM_MANUAL=y +CONFIG_FLATMEM=y diff --git a/.github/scripts/series/kconfigs/defconfig/hardened_usercopy_slub b/.github/scripts/series/kconfigs/defconfig/hardened_usercopy_slub new file mode 100644 index 00000000000000..d1a8be99f2f7c4 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/hardened_usercopy_slub @@ -0,0 +1,2 @@ +CONFIG_SLUB=y +CONFIG_HARDENED_USERCOPY=y diff --git a/.github/scripts/series/kconfigs/defconfig/kasan b/.github/scripts/series/kconfigs/defconfig/kasan new file mode 100644 index 00000000000000..642c037c8e96d6 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/kasan @@ -0,0 +1,2 @@ +CONFIG_KASAN=y +# CONFIG_KASAN_VMALLOC is not set diff --git a/.github/scripts/series/kconfigs/defconfig/kasan_inline b/.github/scripts/series/kconfigs/defconfig/kasan_inline new file mode 100644 index 00000000000000..d18d056f78f1fc --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/kasan_inline @@ -0,0 +1,2 @@ +CONFIG_KASAN=y +CONFIG_KASAN_INLINE=y diff --git a/.github/scripts/series/kconfigs/defconfig/kasan_sparsemem_novmemmmap b/.github/scripts/series/kconfigs/defconfig/kasan_sparsemem_novmemmmap new file mode 100644 index 00000000000000..c9b4f0c2efec94 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/kasan_sparsemem_novmemmmap @@ -0,0 +1,5 @@ +CONFIG_KASAN=y +# CONFIG_FLATMEM_MANUAL is not set +CONFIG_SPARSEMEM_MANUAL=y +CONFIG_SPARSEMEM=y +# CONFIG_SPARSEMEM_VMEMMAP is not set diff --git a/.github/scripts/series/kconfigs/defconfig/kasan_sparsemem_vmemmmap b/.github/scripts/series/kconfigs/defconfig/kasan_sparsemem_vmemmmap new file mode 100644 index 00000000000000..669064147997bb --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/kasan_sparsemem_vmemmmap @@ -0,0 +1,5 @@ +CONFIG_KASAN=y +# CONFIG_FLATMEM_MANUAL is not set +CONFIG_SPARSEMEM_MANUAL=y +CONFIG_SPARSEMEM=y +CONFIG_SPARSEMEM_VMEMMAP=y diff --git a/.github/scripts/series/kconfigs/defconfig/kasan_vmalloc b/.github/scripts/series/kconfigs/defconfig/kasan_vmalloc new file mode 100644 index 00000000000000..183e7b2d2ce285 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/kasan_vmalloc @@ -0,0 +1,2 @@ +CONFIG_KASAN=y +CONFIG_KASAN_VMALLOC=y diff --git a/.github/scripts/series/kconfigs/defconfig/kfence b/.github/scripts/series/kconfigs/defconfig/kfence new file mode 100644 index 00000000000000..9037f9b6ad4402 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/kfence @@ -0,0 +1,3 @@ +CONFIG_KFENCE=y +CONFIG_KFENCE_SAMPLE_INTERVAL=100 +CONFIG_KFENCE_NUM_OBJECTS=255 diff --git a/.github/scripts/series/kconfigs/defconfig/legacy_sbi b/.github/scripts/series/kconfigs/defconfig/legacy_sbi new file mode 100644 index 00000000000000..02ed0014719dc7 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/legacy_sbi @@ -0,0 +1,4 @@ +CONFIG_NONPORTABLE=y +CONFIG_RISCV_SBI_V01=y +CONFIG_HVC_RISCV_SBI=y +CONFIG_SERIAL_EARLYCON_RISCV_SBI=y diff --git a/.github/scripts/series/kconfigs/defconfig/lockdep b/.github/scripts/series/kconfigs/defconfig/lockdep new file mode 100644 index 00000000000000..608538fff563ab --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/lockdep @@ -0,0 +1,3 @@ +CONFIG_PROVE_LOCKING=y +CONFIG_DEBUG_LOCK_ALLOC=y +CONFIG_DEBUG_LOCKING_API_SELFTESTS=y diff --git a/.github/scripts/series/kconfigs/defconfig/medany b/.github/scripts/series/kconfigs/defconfig/medany new file mode 100644 index 00000000000000..20d4f94c1cbe25 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/medany @@ -0,0 +1,2 @@ +# CONFIG_CMODEL_MEDLOW is not set +CONFIG_CMODEL_MEDANY=y diff --git a/.github/scripts/series/kconfigs/defconfig/medlow b/.github/scripts/series/kconfigs/defconfig/medlow new file mode 100644 index 00000000000000..79cac7912772f9 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/medlow @@ -0,0 +1,2 @@ +# CONFIG_CMODEL_MEDANY is not set +CONFIG_CMODEL_MEDLOW=y diff --git a/.github/scripts/series/kconfigs/defconfig/noc b/.github/scripts/series/kconfigs/defconfig/noc new file mode 100644 index 00000000000000..9a4a7a1d838922 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/noc @@ -0,0 +1,3 @@ +# CONFIG_RISCV_ISA_C is not set +CONFIG_NONPORTABLE=y +# CONFIG_EFI is not set diff --git a/.github/scripts/series/kconfigs/defconfig/nosmp b/.github/scripts/series/kconfigs/defconfig/nosmp new file mode 100644 index 00000000000000..0c26a8626295e3 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/nosmp @@ -0,0 +1 @@ +# CONFIG_SMP is not set diff --git a/.github/scripts/series/kconfigs/defconfig/pmu b/.github/scripts/series/kconfigs/defconfig/pmu new file mode 100644 index 00000000000000..c57cb3510168fb --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/pmu @@ -0,0 +1,4 @@ +CONFIG_PERF_EVENTS=y +CONFIG_RISCV_SBI=y +CONFIG_RISCV_PMU=y +CONFIG_RISCV_PMU_SBI=y diff --git a/.github/scripts/series/kconfigs/defconfig/preempt b/.github/scripts/series/kconfigs/defconfig/preempt new file mode 100644 index 00000000000000..2b346554f869ec --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/preempt @@ -0,0 +1,2 @@ +CONFIG_PREEMPT=y +CONFIG_DEBUG_PREEMPT=y diff --git a/.github/scripts/series/kconfigs/defconfig/preempt_rt b/.github/scripts/series/kconfigs/defconfig/preempt_rt new file mode 100644 index 00000000000000..ffab30de80e489 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/preempt_rt @@ -0,0 +1 @@ +CONFIG_PREEMPT_RT=y diff --git a/.github/scripts/series/kconfigs/defconfig/qspinlock b/.github/scripts/series/kconfigs/defconfig/qspinlock new file mode 100644 index 00000000000000..7791363d351038 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/qspinlock @@ -0,0 +1 @@ +CONFIG_RISCV_QUEUED_SPINLOCKS=y diff --git a/.github/scripts/series/kconfigs/defconfig/randomize_base b/.github/scripts/series/kconfigs/defconfig/randomize_base new file mode 100644 index 00000000000000..7f9170d16348aa --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/randomize_base @@ -0,0 +1,2 @@ +CONFIG_RELOCATABLE=y +CONFIG_RANDOMIZE_BASE=y diff --git a/.github/scripts/series/kconfigs/defconfig/rseq b/.github/scripts/series/kconfigs/defconfig/rseq new file mode 100644 index 00000000000000..adc7767df654e6 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/rseq @@ -0,0 +1 @@ +CONFIG_RSEQ=y diff --git a/.github/scripts/series/kconfigs/defconfig/rseq_debug b/.github/scripts/series/kconfigs/defconfig/rseq_debug new file mode 100644 index 00000000000000..72a197a93f3465 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/rseq_debug @@ -0,0 +1,3 @@ +CONFIG_RSEQ=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_RSEQ=y diff --git a/.github/scripts/series/kconfigs/defconfig/size b/.github/scripts/series/kconfigs/defconfig/size new file mode 100644 index 00000000000000..7a224853ad00a7 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/size @@ -0,0 +1 @@ +CONFIG_CC_OPTIMIZE_FOR_SIZE=y diff --git a/.github/scripts/series/kconfigs/defconfig/sparsemem b/.github/scripts/series/kconfigs/defconfig/sparsemem new file mode 100644 index 00000000000000..cc1d59050946d7 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/sparsemem @@ -0,0 +1,3 @@ +# CONFIG_FLATMEM_MANUAL is not set +CONFIG_SPARSEMEM_MANUAL=y +CONFIG_SPARSEMEM=y diff --git a/.github/scripts/series/kconfigs/defconfig/spinwait b/.github/scripts/series/kconfigs/defconfig/spinwait new file mode 100644 index 00000000000000..270b49711e2898 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/spinwait @@ -0,0 +1 @@ +CONFIG_RISCV_BOOT_SPINWAIT=y diff --git a/.github/scripts/series/kconfigs/defconfig/strict_rwx b/.github/scripts/series/kconfigs/defconfig/strict_rwx new file mode 100644 index 00000000000000..8c57b454ad2611 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/strict_rwx @@ -0,0 +1 @@ +CONFIG_STRICT_KERNEL_RWX=y diff --git a/.github/scripts/series/kconfigs/defconfig/svnapot b/.github/scripts/series/kconfigs/defconfig/svnapot new file mode 100644 index 00000000000000..102afb0cff137a --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/svnapot @@ -0,0 +1 @@ +CONFIG_RISCV_ISA_SVNAPOT=y diff --git a/.github/scripts/series/kconfigs/defconfig/ticket_spinlock b/.github/scripts/series/kconfigs/defconfig/ticket_spinlock new file mode 100644 index 00000000000000..05b5fa71b09a80 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/ticket_spinlock @@ -0,0 +1 @@ +CONFIG_RISCV_TICKET_SPINLOCKS=y diff --git a/.github/scripts/series/kconfigs/defconfig/vmap_stack b/.github/scripts/series/kconfigs/defconfig/vmap_stack new file mode 100644 index 00000000000000..8bd986875fc752 --- /dev/null +++ b/.github/scripts/series/kconfigs/defconfig/vmap_stack @@ -0,0 +1 @@ +CONFIG_VMAP_STACK=y diff --git a/.github/scripts/series/kconfigs/ubuntu_defconfig b/.github/scripts/series/kconfigs/ubuntu_defconfig new file mode 100644 index 00000000000000..7a70f4194f0566 --- /dev/null +++ b/.github/scripts/series/kconfigs/ubuntu_defconfig @@ -0,0 +1,5503 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_WATCH_QUEUE=y +CONFIG_USELIB=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BPF_JIT=y +CONFIG_BPF_JIT_ALWAYS_ON=y +CONFIG_BPF_LSM=y +CONFIG_PREEMPT_VOLUNTARY=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_PSI=y +CONFIG_IKHEADERS=m +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_NUMA_BALANCING=y +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_RDMA=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_HUGETLB=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_MISC=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_EXPERT=y +CONFIG_SGETMASK_SYSCALL=y +CONFIG_KALLSYMS_ALL=y +CONFIG_PC104=y +CONFIG_PROFILING=y +CONFIG_SOC_MICROCHIP_POLARFIRE=y +CONFIG_SOC_SIFIVE=y +CONFIG_SOC_STARFIVE=y +CONFIG_SOC_VIRT=y +CONFIG_SMP=y +CONFIG_NR_CPUS=32 +CONFIG_NUMA=y +# CONFIG_RISCV_ISA_SVPBMT is not set +CONFIG_KEXEC=y +CONFIG_KEXEC_FILE=y +CONFIG_CRASH_DUMP=y +# CONFIG_COMPAT is not set +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y +CONFIG_CPU_IDLE_GOV_TEO=y +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_JUMP_LABEL=y +CONFIG_COMPAT_32BIT_TIME=y +# CONFIG_VMAP_STACK is not set +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MODULE_SRCVERSION_ALL=y +CONFIG_MODULE_SIG_SHA512=y +CONFIG_BLK_DEV_ZONED=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_BLK_WBT=y +CONFIG_BLK_CGROUP_FC_APPID=y +CONFIG_BLK_CGROUP_IOCOST=y +CONFIG_BLK_CGROUP_IOPRIO=y +CONFIG_BLK_SED_OPAL=y +CONFIG_BLK_INLINE_ENCRYPTION=y +CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_AIX_PARTITION=y +CONFIG_OSF_PARTITION=y +CONFIG_AMIGA_PARTITION=y +CONFIG_ATARI_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +CONFIG_SUN_PARTITION=y +CONFIG_KARMA_PARTITION=y +CONFIG_SYSV68_PARTITION=y +CONFIG_CMDLINE_PARTITION=y +CONFIG_MQ_IOSCHED_KYBER=m +CONFIG_IOSCHED_BFQ=m +CONFIG_BINFMT_FLAT=y +CONFIG_BINFMT_FLAT_OLD=y +CONFIG_BINFMT_ZFLAT=y +CONFIG_BINFMT_MISC=m +CONFIG_ZSWAP=y +CONFIG_Z3FOLD=m +CONFIG_ZSMALLOC=y +CONFIG_SLAB_FREELIST_RANDOM=y +CONFIG_SLAB_FREELIST_HARDENED=y +CONFIG_SHUFFLE_PAGE_ALLOCATOR=y +# CONFIG_COMPAT_BRK is not set +CONFIG_KSM=y +CONFIG_DEFAULT_MMAP_MIN_ADDR=65536 +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y +CONFIG_CMA=y +CONFIG_CMA_SYSFS=y +CONFIG_CMA_AREAS=7 +CONFIG_IDLE_PAGE_TRACKING=y +CONFIG_ANON_VMA_NAME=y +CONFIG_USERFAULTFD=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=m +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=m +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_SMC=m +CONFIG_SMC_DIAG=m +CONFIG_XDP_SOCKETS=y +CONFIG_XDP_SOCKETS_DIAG=m +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_IP_MROUTE=y +CONFIG_IP_MROUTE_MULTIPLE_TABLES=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_NET_IPVTI=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_ESPINTCP=y +CONFIG_INET_IPCOMP=m +CONFIG_INET_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_INET_RAW_DIAG=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_ESPINTCP=y +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_IPV6_IOAM6_LWTUNNEL=y +CONFIG_MPTCP=y +CONFIG_NETWORK_PHY_TIMESTAMPING=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NETFILTER_NETLINK_HOOK=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_XFRM=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_SYNPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NFT_REJECT_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_SECMARK=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_IPV6=y +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_PROTO_ESP=y +CONFIG_IP_VS_PROTO_AH=y +CONFIG_IP_VS_PROTO_SCTP=y +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_LC=m +CONFIG_IP_VS_WLC=m +CONFIG_IP_VS_FO=m +CONFIG_IP_VS_OVF=m +CONFIG_IP_VS_LBLC=m +CONFIG_IP_VS_LBLCR=m +CONFIG_IP_VS_DH=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_MH=m +CONFIG_IP_VS_SED=m +CONFIG_IP_VS_NQ=m +CONFIG_IP_VS_FTP=m +CONFIG_IP_VS_PE_SIP=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_IP6_NF_IPTABLES=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_TARGET_SYNPROXY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BPFILTER=y +CONFIG_IP_DCCP=m +# CONFIG_IP_DCCP_CCID3 is not set +CONFIG_SCTP_DEFAULT_COOKIE_HMAC_SHA1=y +CONFIG_SCTP_COOKIE_HMAC_MD5=y +CONFIG_RDS=m +CONFIG_RDS_RDMA=m +CONFIG_RDS_TCP=m +CONFIG_TIPC=m +CONFIG_TIPC_MEDIA_IB=y +CONFIG_ATM=m +CONFIG_ATM_CLIP=m +CONFIG_ATM_LANE=m +CONFIG_ATM_MPOA=m +CONFIG_ATM_BR2684=m +CONFIG_L2TP=m +CONFIG_L2TP_DEBUGFS=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_BRIDGE_MRP=y +CONFIG_BRIDGE_CFM=y +CONFIG_NET_DSA=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_LLC2=m +CONFIG_ATALK=m +CONFIG_DEV_APPLETALK=m +CONFIG_X25=m +CONFIG_LAPB=m +CONFIG_PHONET=m +CONFIG_6LOWPAN=m +CONFIG_IEEE802154=m +CONFIG_IEEE802154_6LOWPAN=m +CONFIG_MAC802154=m +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_TAPRIO=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_FQ_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_ETS=m +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_CANID=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_IPT=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_CTINFO=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_ACT_CT=m +CONFIG_NET_ACT_GATE=m +CONFIG_NET_TC_SKB_EXT=y +CONFIG_DCB=y +CONFIG_DNS_RESOLVER=y +CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set +CONFIG_BATMAN_ADV_NC=y +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_VIRTIO_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_MPLS_ROUTING=m +CONFIG_MPLS_IPTUNNEL=m +CONFIG_HSR=m +CONFIG_QRTR_SMD=m +CONFIG_QRTR_TUN=m +CONFIG_NET_NCSI=y +CONFIG_NCSI_OEM_CMD_GET_MAC=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_STREAM_PARSER=y +CONFIG_NET_PKTGEN=m +CONFIG_NET_DROP_MONITOR=y +CONFIG_HAMRADIO=y +CONFIG_AX25=m +CONFIG_NETROM=m +CONFIG_ROSE=m +CONFIG_MKISS=m +CONFIG_6PACK=m +CONFIG_BPQETHER=m +CONFIG_BAYCOM_SER_FDX=m +CONFIG_BAYCOM_SER_HDX=m +CONFIG_BAYCOM_PAR=m +CONFIG_YAM=m +CONFIG_CAN=m +CONFIG_CAN_J1939=m +CONFIG_CAN_ISOTP=m +CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_CMTP=m +CONFIG_BT_HIDP=m +CONFIG_BT_HS=y +CONFIG_BT_6LOWPAN=m +CONFIG_BT_LEDS=y +CONFIG_BT_MSFTEXT=y +CONFIG_BT_AOSPEXT=y +CONFIG_BT_HCIBTUSB=m +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIBTUSB_MTK=y +CONFIG_BT_HCIBTSDIO=m +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_BCSP=y +CONFIG_BT_HCIUART_ATH3K=y +CONFIG_BT_HCIUART_LL=y +CONFIG_BT_HCIUART_INTEL=y +CONFIG_BT_HCIUART_BCM=y +CONFIG_BT_HCIUART_RTL=y +CONFIG_BT_HCIUART_QCA=y +CONFIG_BT_HCIUART_AG6XX=y +CONFIG_BT_HCIUART_MRVL=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MRVL=m +CONFIG_BT_MRVL_SDIO=m +CONFIG_BT_ATH3K=m +CONFIG_BT_MTKSDIO=m +CONFIG_BT_MTKUART=m +CONFIG_BT_VIRTIO=m +CONFIG_AF_RXRPC_IPV6=y +CONFIG_RXKAD=y +CONFIG_AF_KCM=m +CONFIG_MCTP=y +CONFIG_CFG80211=m +CONFIG_CFG80211_DEBUGFS=y +CONFIG_MAC80211=m +CONFIG_MAC80211_MESH=y +CONFIG_MAC80211_MESSAGE_TRACING=y +CONFIG_RFKILL=y +CONFIG_RFKILL_INPUT=y +CONFIG_RFKILL_GPIO=m +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_NET_9P_RDMA=m +CONFIG_CAIF=m +CONFIG_CAIF_USB=m +CONFIG_CEPH_LIB_USE_DNS_RESOLVER=y +CONFIG_NFC=m +CONFIG_NFC_DIGITAL=m +CONFIG_NFC_NCI=m +CONFIG_NFC_NCI_SPI=m +CONFIG_NFC_NCI_UART=m +CONFIG_NFC_HCI=m +CONFIG_NFC_SHDLC=y +CONFIG_NFC_TRF7970A=m +CONFIG_NFC_SIM=m +CONFIG_NFC_PORT100=m +CONFIG_NFC_VIRTUAL_NCI=m +CONFIG_NFC_FDP=m +CONFIG_NFC_FDP_I2C=m +CONFIG_NFC_PN544_I2C=m +CONFIG_NFC_PN533_USB=m +CONFIG_NFC_PN533_I2C=m +CONFIG_NFC_PN532_UART=m +CONFIG_NFC_MICROREAD_I2C=m +CONFIG_NFC_MRVL_USB=m +CONFIG_NFC_MRVL_UART=m +CONFIG_NFC_MRVL_I2C=m +CONFIG_NFC_MRVL_SPI=m +CONFIG_NFC_ST21NFCA_I2C=m +CONFIG_NFC_ST_NCI_I2C=m +CONFIG_NFC_ST_NCI_SPI=m +CONFIG_NFC_NXP_NCI=m +CONFIG_NFC_NXP_NCI_I2C=m +CONFIG_NFC_S3FWRN5_I2C=m +CONFIG_NFC_S3FWRN82_UART=m +CONFIG_NFC_ST95HF=m +CONFIG_NET_IFE=m +CONFIG_PCI=y +CONFIG_PCIEAER=y +CONFIG_PCIE_DPC=y +CONFIG_PCIE_PTM=y +CONFIG_PCI_REALLOC_ENABLE_AUTO=y +CONFIG_PCI_STUB=m +CONFIG_PCI_PF_STUB=m +CONFIG_PCI_IOV=y +CONFIG_PCI_PRI=y +CONFIG_PCI_PASID=y +CONFIG_HOTPLUG_PCI_CPCI=y +CONFIG_HOTPLUG_PCI_SHPC=y +CONFIG_PCI_FTPCI100=y +CONFIG_PCI_HOST_GENERIC=y +CONFIG_PCIE_XILINX=y +CONFIG_PCIE_MICROCHIP_HOST=y +CONFIG_PCIE_DW_PLAT_HOST=y +CONFIG_PCIE_DW_PLAT_EP=y +CONFIG_PCIE_FU740=y +CONFIG_PCIE_CADENCE_PLAT_HOST=y +CONFIG_PCIE_CADENCE_PLAT_EP=y +CONFIG_PCI_J721E_HOST=y +CONFIG_PCI_J721E_EP=y +CONFIG_PCI_ENDPOINT=y +CONFIG_PCI_ENDPOINT_CONFIGFS=y +CONFIG_PCI_EPF_NTB=m +CONFIG_CXL_BUS=m +CONFIG_RAPIDIO=y +CONFIG_RAPIDIO_TSI721=m +CONFIG_RAPIDIO_DMA_ENGINE=y +CONFIG_RAPIDIO_ENUM_BASIC=m +CONFIG_RAPIDIO_CHMAN=m +CONFIG_RAPIDIO_MPORT_CDEV=m +CONFIG_RAPIDIO_CPS_XX=m +CONFIG_RAPIDIO_CPS_GEN2=m +CONFIG_RAPIDIO_RXS_GEN3=m +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_DEVTMPFS_SAFE=y +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y +CONFIG_MOXTET=m +CONFIG_MHI_BUS_PCI_GENERIC=m +CONFIG_MHI_BUS_EP=m +CONFIG_CONNECTOR=y +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_EFI_VARS_PSTORE=m +CONFIG_EFI_BOOTLOADER_CONTROL=m +CONFIG_EFI_CAPSULE_LOADER=m +CONFIG_EFI_TEST=m +CONFIG_RESET_ATTACK_MITIGATION=y +CONFIG_EFI_COCO_SECRET=y +CONFIG_GNSS=m +CONFIG_GNSS_MTK_SERIAL=m +CONFIG_GNSS_SIRF_SERIAL=m +CONFIG_GNSS_UBX_SERIAL=m +CONFIG_GNSS_USB=m +CONFIG_MTD=m +CONFIG_MTD_AR7_PARTS=m +CONFIG_MTD_CMDLINE_PARTS=m +CONFIG_MTD_REDBOOT_PARTS=m +CONFIG_MTD_BLOCK=m +CONFIG_MTD_BLOCK_RO=m +CONFIG_FTL=m +CONFIG_NFTL=m +CONFIG_NFTL_RW=y +CONFIG_INFTL=m +CONFIG_RFD_FTL=m +CONFIG_SSFDC=m +CONFIG_SM_FTL=m +CONFIG_MTD_OOPS=m +CONFIG_MTD_PSTORE=m +CONFIG_MTD_SWAP=m +CONFIG_MTD_JEDECPROBE=m +CONFIG_MTD_CFI_INTELEXT=m +CONFIG_MTD_CFI_STAA=m +CONFIG_MTD_ROM=m +CONFIG_MTD_ABSENT=m +CONFIG_MTD_PHYSMAP=m +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PHYSMAP_GPIO_ADDR=y +CONFIG_MTD_PCI=m +CONFIG_MTD_INTEL_VR_NOR=m +CONFIG_MTD_PLATRAM=m +CONFIG_MTD_PMC551=m +CONFIG_MTD_DATAFLASH=m +CONFIG_MTD_DATAFLASH_OTP=y +CONFIG_MTD_MCHP23K256=m +CONFIG_MTD_MCHP48L640=m +CONFIG_MTD_SST25L=m +CONFIG_MTD_SLRAM=m +CONFIG_MTD_PHRAM=m +CONFIG_MTD_MTDRAM=m +CONFIG_MTD_BLOCK2MTD=m +CONFIG_MTD_ONENAND=m +CONFIG_MTD_ONENAND_VERIFY_WRITE=y +CONFIG_MTD_ONENAND_GENERIC=m +CONFIG_MTD_ONENAND_2X_PROGRAM=y +CONFIG_MTD_RAW_NAND=m +CONFIG_MTD_NAND_DENALI_PCI=m +CONFIG_MTD_NAND_DENALI_DT=m +CONFIG_MTD_NAND_CAFE=m +CONFIG_MTD_NAND_MXIC=m +CONFIG_MTD_NAND_GPIO=m +CONFIG_MTD_NAND_PLATFORM=m +CONFIG_MTD_NAND_CADENCE=m +CONFIG_MTD_NAND_ARASAN=m +CONFIG_MTD_NAND_INTEL_LGM=m +CONFIG_MTD_NAND_NANDSIM=m +CONFIG_MTD_NAND_RICOH=m +CONFIG_MTD_NAND_DISKONCHIP=m +CONFIG_MTD_SPI_NAND=m +CONFIG_MTD_NAND_ECC_SW_BCH=y +CONFIG_MTD_LPDDR=m +CONFIG_MTD_SPI_NOR=m +CONFIG_MTD_UBI=m +CONFIG_MTD_UBI_FASTMAP=y +CONFIG_MTD_UBI_GLUEBI=m +CONFIG_MTD_UBI_BLOCK=y +CONFIG_MTD_HYPERBUS=m +CONFIG_OF_OVERLAY=y +CONFIG_PARPORT=m +CONFIG_PARPORT_PC=m +CONFIG_PARPORT_SERIAL=m +CONFIG_PARPORT_PC_FIFO=y +CONFIG_PARPORT_1284=y +CONFIG_BLK_DEV_NULL_BLK=m +CONFIG_BLK_DEV_PCIESSD_MTIP32XX=m +CONFIG_ZRAM=m +CONFIG_ZRAM_WRITEBACK=y +CONFIG_ZRAM_MEMORY_TRACKING=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_DRBD=m +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=m +CONFIG_BLK_DEV_RAM_SIZE=65536 +CONFIG_CDROM_PKTCDVD=m +CONFIG_ATA_OVER_ETH=m +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_RBD=m +CONFIG_BLK_DEV_RNBD_CLIENT=m +CONFIG_BLK_DEV_RNBD_SERVER=m +CONFIG_BLK_DEV_NVME=m +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_HWMON=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=m +CONFIG_NVME_TCP=m +CONFIG_NVME_TARGET=m +CONFIG_NVME_TARGET_PASSTHRU=y +CONFIG_NVME_TARGET_LOOP=m +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=m +CONFIG_NVME_TARGET_TCP=m +CONFIG_AD525X_DPOT=m +CONFIG_AD525X_DPOT_I2C=m +CONFIG_AD525X_DPOT_SPI=m +CONFIG_DUMMY_IRQ=m +CONFIG_PHANTOM=m +CONFIG_ICS932S401=m +CONFIG_ENCLOSURE_SERVICES=m +CONFIG_HI6421V600_IRQ=m +CONFIG_HP_ILO=m +CONFIG_APDS9802ALS=m +CONFIG_ISL29003=m +CONFIG_ISL29020=m +CONFIG_SENSORS_TSL2550=m +CONFIG_SENSORS_BH1770=m +CONFIG_SENSORS_APDS990X=m +CONFIG_HMC6352=m +CONFIG_DS1682=m +CONFIG_LATTICE_ECP3_CONFIG=m +CONFIG_SRAM=y +CONFIG_DW_XDATA_PCIE=m +CONFIG_OPEN_DICE=m +CONFIG_C2PORT=m +CONFIG_EEPROM_AT24=m +CONFIG_EEPROM_AT25=m +CONFIG_EEPROM_LEGACY=m +CONFIG_EEPROM_MAX6875=m +CONFIG_EEPROM_93XX46=m +CONFIG_EEPROM_IDT_89HPESX=m +CONFIG_EEPROM_EE1004=m +CONFIG_TI_ST=m +CONFIG_SENSORS_LIS3_SPI=m +CONFIG_SENSORS_LIS3_I2C=m +CONFIG_GENWQE=m +CONFIG_ECHO=m +CONFIG_BCM_VK=m +CONFIG_BCM_VK_TTY=y +CONFIG_MISC_ALCOR_PCI=m +CONFIG_MISC_RTSX_PCI=m +CONFIG_MISC_RTSX_USB=m +CONFIG_UACCE=m +CONFIG_PVPANIC=y +CONFIG_PVPANIC_MMIO=m +CONFIG_PVPANIC_PCI=m +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=m +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=m +CONFIG_SCSI_ENCLOSURE=m +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_ATA=y +CONFIG_ISCSI_TCP=m +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_SCSI_BNX2X_FCOE=m +CONFIG_BE2ISCSI=m +CONFIG_BLK_DEV_3W_XXXX_RAID=m +CONFIG_SCSI_HPSA=m +CONFIG_SCSI_3W_9XXX=m +CONFIG_SCSI_3W_SAS=m +CONFIG_SCSI_ACARD=m +CONFIG_SCSI_AACRAID=m +CONFIG_SCSI_AIC7XXX=m +CONFIG_AIC7XXX_CMDS_PER_DEVICE=8 +# CONFIG_AIC7XXX_DEBUG_ENABLE is not set +CONFIG_SCSI_AIC79XX=m +# CONFIG_AIC79XX_DEBUG_ENABLE is not set +CONFIG_SCSI_AIC94XX=m +# CONFIG_AIC94XX_DEBUG is not set +CONFIG_SCSI_MVSAS=m +# CONFIG_SCSI_MVSAS_DEBUG is not set +CONFIG_SCSI_MVUMI=m +CONFIG_SCSI_ADVANSYS=m +CONFIG_SCSI_ARCMSR=m +CONFIG_SCSI_ESAS2R=m +CONFIG_MEGARAID_NEWGEN=y +CONFIG_MEGARAID_MM=m +CONFIG_MEGARAID_MAILBOX=m +CONFIG_MEGARAID_LEGACY=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT2SAS=m +CONFIG_SCSI_MPI3MR=m +CONFIG_SCSI_SMARTPQI=m +CONFIG_SCSI_HPTIOP=m +CONFIG_SCSI_MYRB=m +CONFIG_SCSI_MYRS=m +CONFIG_LIBFC=m +CONFIG_LIBFCOE=m +CONFIG_FCOE=m +CONFIG_SCSI_SNIC=m +CONFIG_SCSI_DMX3191D=m +CONFIG_SCSI_IPS=m +CONFIG_SCSI_INITIO=m +CONFIG_SCSI_INIA100=m +CONFIG_SCSI_PPA=m +CONFIG_SCSI_IMM=m +CONFIG_SCSI_STEX=m +CONFIG_SCSI_SYM53C8XX_2=m +CONFIG_SCSI_IPR=m +CONFIG_SCSI_QLOGIC_1280=m +CONFIG_SCSI_QLA_FC=m +CONFIG_TCM_QLA2XXX=m +CONFIG_SCSI_QLA_ISCSI=m +CONFIG_QEDI=m +CONFIG_QEDF=m +CONFIG_SCSI_EFCT=m +CONFIG_SCSI_DC395x=m +CONFIG_SCSI_AM53C974=m +CONFIG_SCSI_WD719X=m +CONFIG_SCSI_DEBUG=m +CONFIG_SCSI_PMCRAID=m +CONFIG_SCSI_PM8001=m +CONFIG_SCSI_BFA_FC=m +CONFIG_SCSI_VIRTIO=m +CONFIG_SCSI_CHELSIO_FCOE=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=m +CONFIG_SCSI_DH_HP_SW=m +CONFIG_SCSI_DH_EMC=m +CONFIG_SCSI_DH_ALUA=m +CONFIG_ATA=y +CONFIG_SATA_AHCI=m +CONFIG_SATA_MOBILE_LPM_POLICY=3 +CONFIG_SATA_AHCI_PLATFORM=m +CONFIG_AHCI_CEVA=m +CONFIG_AHCI_QORIQ=m +CONFIG_SATA_INIC162X=m +CONFIG_SATA_ACARD_AHCI=m +CONFIG_SATA_SIL24=m +CONFIG_PDC_ADMA=m +CONFIG_SATA_QSTOR=m +CONFIG_SATA_SX4=m +CONFIG_ATA_PIIX=m +CONFIG_SATA_DWC=m +CONFIG_SATA_DWC_OLD_DMA=y +CONFIG_SATA_MV=m +CONFIG_SATA_NV=m +CONFIG_SATA_PROMISE=m +CONFIG_SATA_SIL=m +CONFIG_SATA_SIS=m +CONFIG_SATA_SVW=m +CONFIG_SATA_ULI=m +CONFIG_SATA_VIA=m +CONFIG_SATA_VITESSE=m +CONFIG_PATA_ALI=m +CONFIG_PATA_AMD=m +CONFIG_PATA_ARTOP=m +CONFIG_PATA_ATIIXP=m +CONFIG_PATA_ATP867X=m +CONFIG_PATA_CMD64X=m +CONFIG_PATA_CYPRESS=m +CONFIG_PATA_EFAR=m +CONFIG_PATA_HPT366=m +CONFIG_PATA_HPT37X=m +CONFIG_PATA_HPT3X2N=m +CONFIG_PATA_HPT3X3=m +CONFIG_PATA_IT8213=m +CONFIG_PATA_IT821X=m +CONFIG_PATA_JMICRON=m +CONFIG_PATA_MARVELL=m +CONFIG_PATA_NETCELL=m +CONFIG_PATA_NINJA32=m +CONFIG_PATA_NS87415=m +CONFIG_PATA_OLDPIIX=m +CONFIG_PATA_OPTIDMA=m +CONFIG_PATA_PDC2027X=m +CONFIG_PATA_PDC_OLD=m +CONFIG_PATA_RADISYS=m +CONFIG_PATA_RDC=m +CONFIG_PATA_SCH=m +CONFIG_PATA_SERVERWORKS=m +CONFIG_PATA_SIL680=m +CONFIG_PATA_TOSHIBA=m +CONFIG_PATA_TRIFLEX=m +CONFIG_PATA_VIA=m +CONFIG_PATA_WINBOND=m +CONFIG_PATA_CMD640_PCI=m +CONFIG_PATA_MPIIX=m +CONFIG_PATA_NS87410=m +CONFIG_PATA_OPTI=m +CONFIG_PATA_OF_PLATFORM=m +CONFIG_PATA_RZ1000=m +CONFIG_ATA_GENERIC=m +CONFIG_PATA_LEGACY=m +CONFIG_MD=y +CONFIG_BLK_DEV_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_MD_CLUSTER=m +CONFIG_BCACHE=m +CONFIG_BCACHE_ASYNC_REGISTRATION=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_EBS=m +CONFIG_DM_ERA=m +CONFIG_DM_CLONE=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_MULTIPATH_HST=m +CONFIG_DM_MULTIPATH_IOA=m +CONFIG_DM_DELAY=m +CONFIG_DM_INIT=y +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_DM_ZONED=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_TCM_FC=m +CONFIG_ISCSI_TARGET=m +CONFIG_ISCSI_TARGET_CXGB4=m +CONFIG_SBP_TARGET=m +CONFIG_FUSION=y +CONFIG_FUSION_SPI=m +CONFIG_FUSION_FC=m +CONFIG_FUSION_SAS=m +CONFIG_FUSION_CTL=m +CONFIG_FUSION_LAN=m +CONFIG_FUSION_LOGGING=y +CONFIG_FIREWIRE=m +CONFIG_FIREWIRE_OHCI=m +CONFIG_FIREWIRE_SBP2=m +CONFIG_FIREWIRE_NET=m +CONFIG_FIREWIRE_NOSY=m +CONFIG_BONDING=m +CONFIG_DUMMY=m +CONFIG_WIREGUARD=m +CONFIG_EQUALIZER=m +CONFIG_NET_FC=y +CONFIG_IFB=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_IPVLAN=m +CONFIG_IPVTAP=m +CONFIG_VXLAN=m +CONFIG_GENEVE=m +CONFIG_BAREUDP=m +CONFIG_GTP=m +CONFIG_AMT=m +CONFIG_MACSEC=m +CONFIG_NETCONSOLE=m +CONFIG_NETCONSOLE_DYNAMIC=y +CONFIG_NTB_NETDEV=m +CONFIG_RIONET=m +CONFIG_TUN=y +CONFIG_VETH=m +CONFIG_VIRTIO_NET=m +CONFIG_NLMON=m +CONFIG_NET_VRF=m +CONFIG_VSOCKMON=m +CONFIG_MHI_NET=m +CONFIG_ARCNET=m +CONFIG_ARCNET_1201=m +CONFIG_ARCNET_1051=m +CONFIG_ARCNET_RAW=m +CONFIG_ARCNET_CAP=m +CONFIG_ARCNET_COM90xx=m +CONFIG_ARCNET_COM90xxIO=m +CONFIG_ARCNET_RIM_I=m +CONFIG_ARCNET_COM20020=m +CONFIG_ARCNET_COM20020_PCI=m +CONFIG_ATM_DUMMY=m +CONFIG_ATM_TCP=m +CONFIG_ATM_LANAI=m +CONFIG_ATM_ENI=m +CONFIG_ATM_NICSTAR=m +CONFIG_ATM_IDT77252=m +CONFIG_ATM_IA=m +CONFIG_ATM_FORE200E=m +CONFIG_ATM_HE=m +CONFIG_ATM_HE_USE_SUNI=y +CONFIG_ATM_SOLOS=m +CONFIG_CAIF_DRIVERS=y +CONFIG_CAIF_TTY=m +CONFIG_CAIF_VIRTIO=m +CONFIG_B53_SPI_DRIVER=m +CONFIG_B53_MDIO_DRIVER=m +CONFIG_B53_MMAP_DRIVER=m +CONFIG_B53_SRAB_DRIVER=m +CONFIG_B53_SERDES=m +CONFIG_NET_DSA_BCM_SF2=m +CONFIG_NET_DSA_HIRSCHMANN_HELLCREEK=m +CONFIG_NET_DSA_LANTIQ_GSWIP=m +CONFIG_NET_DSA_MT7530=m +CONFIG_NET_DSA_MV88E6060=m +CONFIG_NET_DSA_MICROCHIP_KSZ_COMMON=m +CONFIG_NET_DSA_MICROCHIP_KSZ9477_I2C=m +CONFIG_NET_DSA_MICROCHIP_KSZ8863_SMI=m +CONFIG_NET_DSA_MV88E6XXX=m +CONFIG_NET_DSA_MV88E6XXX_PTP=y +CONFIG_NET_DSA_MSCC_SEVILLE=m +CONFIG_NET_DSA_AR9331=m +CONFIG_NET_DSA_QCA8K=m +CONFIG_NET_DSA_SJA1105=m +CONFIG_NET_DSA_SJA1105_PTP=y +CONFIG_NET_DSA_SJA1105_TAS=y +CONFIG_NET_DSA_SJA1105_VL=y +CONFIG_NET_DSA_XRS700X_I2C=m +CONFIG_NET_DSA_XRS700X_MDIO=m +CONFIG_NET_DSA_REALTEK=m +CONFIG_NET_DSA_REALTEK_RTL8365MB=m +CONFIG_NET_DSA_REALTEK_RTL8366RB=m +CONFIG_NET_DSA_SMSC_LAN9303_I2C=m +CONFIG_NET_DSA_SMSC_LAN9303_MDIO=m +CONFIG_NET_DSA_VITESSE_VSC73XX_SPI=m +CONFIG_NET_DSA_VITESSE_VSC73XX_PLATFORM=m +CONFIG_VORTEX=m +CONFIG_TYPHOON=m +CONFIG_ADAPTEC_STARFIRE=m +CONFIG_ET131X=m +CONFIG_SLICOSS=m +CONFIG_ACENIC=m +CONFIG_ALTERA_TSE=m +CONFIG_ENA_ETHERNET=m +CONFIG_AMD8111_ETH=m +CONFIG_PCNET32=m +CONFIG_AQTION=m +CONFIG_SPI_AX88796C=m +CONFIG_ATL2=m +CONFIG_ATL1=m +CONFIG_ATL1E=m +CONFIG_ATL1C=m +CONFIG_ALX=m +CONFIG_B44=m +CONFIG_BCMGENET=m +CONFIG_TIGON3=m +CONFIG_BNX2X=m +CONFIG_SYSTEMPORT=m +CONFIG_BNXT=m +CONFIG_BNXT_DCB=y +CONFIG_MACB=m +CONFIG_MACB_PCI=m +CONFIG_THUNDER_NIC_PF=m +CONFIG_THUNDER_NIC_VF=m +CONFIG_LIQUIDIO=m +CONFIG_LIQUIDIO_VF=m +CONFIG_CHELSIO_T1=m +CONFIG_CHELSIO_T1_1G=y +CONFIG_CHELSIO_T4_DCB=y +CONFIG_CHELSIO_T4_FCOE=y +CONFIG_CHELSIO_T4VF=m +CONFIG_CHELSIO_IPSEC_INLINE=m +CONFIG_CHELSIO_TLS_DEVICE=m +CONFIG_ENIC=m +CONFIG_GEMINI_ETHERNET=m +CONFIG_DM9051=m +CONFIG_DNET=m +CONFIG_NET_TULIP=y +CONFIG_DE2104X=m +CONFIG_TULIP=m +CONFIG_WINBOND_840=m +CONFIG_DM9102=m +CONFIG_ULI526X=m +CONFIG_DL2K=m +CONFIG_SUNDANCE=m +CONFIG_TSNEP=m +CONFIG_EZCHIP_NPS_MANAGEMENT_ENET=m +CONFIG_FUN_ETH=m +CONFIG_E100=m +CONFIG_E1000=m +CONFIG_E1000E=m +CONFIG_IGB=m +CONFIG_IGBVF=m +CONFIG_IXGB=m +CONFIG_IXGBE=m +CONFIG_IXGBE_DCB=y +CONFIG_IXGBEVF=m +CONFIG_I40E=m +CONFIG_I40E_DCB=y +CONFIG_I40EVF=m +CONFIG_ICE=m +CONFIG_FM10K=m +CONFIG_IGC=m +CONFIG_JME=m +CONFIG_LITEX_LITEETH=m +CONFIG_MVMDIO=m +CONFIG_SKGE=m +CONFIG_SKGE_GENESIS=y +CONFIG_SKY2=m +CONFIG_OCTEON_EP=m +CONFIG_PRESTERA=m +CONFIG_MLX4_EN=m +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLX5_CORE_IPOIB=y +CONFIG_MLX5_EN_IPSEC=y +CONFIG_MLX5_EN_TLS=y +CONFIG_MLX5_SF=y +CONFIG_MLXSW_CORE=m +CONFIG_KS8842=m +CONFIG_KS8851=m +CONFIG_KS8851_MLL=m +CONFIG_KSZ884X_PCI=m +CONFIG_ENC28J60=m +CONFIG_ENCX24J600=m +CONFIG_LAN743X=m +CONFIG_LAN966X_SWITCH=m +CONFIG_MSCC_OCELOT_SWITCH=m +CONFIG_MYRI10GE=m +CONFIG_NI_XGE_MANAGEMENT_ENET=m +CONFIG_NATSEMI=m +CONFIG_NS83820=m +CONFIG_S2IO=m +CONFIG_NFP=m +CONFIG_NE2K_PCI=m +CONFIG_FORCEDETH=m +CONFIG_ETHOC=m +CONFIG_HAMACHI=m +CONFIG_YELLOWFIN=m +CONFIG_IONIC=m +CONFIG_QLA3XXX=m +CONFIG_QLCNIC=m +CONFIG_NETXEN_NIC=m +CONFIG_QED=m +CONFIG_QEDE=m +CONFIG_BNA=m +CONFIG_QCA7000_SPI=m +CONFIG_QCA7000_UART=m +CONFIG_QCOM_EMAC=m +CONFIG_RMNET=m +CONFIG_R6040=m +CONFIG_8139CP=m +CONFIG_8139TOO=m +CONFIG_8139TOO_8129=y +CONFIG_R8169=m +CONFIG_ROCKER=m +CONFIG_SXGBE_ETH=m +CONFIG_SC92031=m +CONFIG_SIS900=m +CONFIG_SIS190=m +CONFIG_SFC=m +CONFIG_SFC_FALCON=m +CONFIG_SFC_SIENA=m +CONFIG_SFC_SIENA_SRIOV=y +CONFIG_EPIC100=m +CONFIG_SMSC911X=m +CONFIG_SMSC9420=m +CONFIG_STMMAC_ETH=m +CONFIG_DWMAC_DWC_QOS_ETH=m +CONFIG_DWMAC_LOONGSON=m +CONFIG_STMMAC_PCI=m +CONFIG_HAPPYMEAL=m +CONFIG_SUNGEM=m +CONFIG_CASSINI=m +CONFIG_NIU=m +CONFIG_DWC_XLGMAC=m +CONFIG_DWC_XLGMAC_PCI=m +CONFIG_TEHUTI=m +CONFIG_TLAN=m +CONFIG_MSE102X=m +CONFIG_VIA_RHINE=m +CONFIG_VIA_RHINE_MMIO=y +CONFIG_VIA_VELOCITY=m +CONFIG_WIZNET_W5100=m +CONFIG_WIZNET_W5300=m +CONFIG_WIZNET_W5100_SPI=m +CONFIG_XILINX_EMACLITE=m +CONFIG_XILINX_AXI_EMAC=m +CONFIG_XILINX_LL_TEMAC=m +CONFIG_FDDI=y +CONFIG_DEFXX=m +CONFIG_SKFP=m +CONFIG_PHYLIB=y +CONFIG_LED_TRIGGER_PHY=y +CONFIG_SFP=m +CONFIG_AMD_PHY=m +CONFIG_ADIN_PHY=m +CONFIG_ADIN1100_PHY=m +CONFIG_AQUANTIA_PHY=m +CONFIG_BROADCOM_PHY=m +CONFIG_BCM54140_PHY=m +CONFIG_BCM84881_PHY=m +CONFIG_BCM87XX_PHY=m +CONFIG_CICADA_PHY=m +CONFIG_CORTINA_PHY=m +CONFIG_DAVICOM_PHY=m +CONFIG_ICPLUS_PHY=m +CONFIG_LXT_PHY=m +CONFIG_INTEL_XWAY_PHY=m +CONFIG_LSI_ET1011C_PHY=m +CONFIG_MARVELL_PHY=m +CONFIG_MARVELL_10G_PHY=m +CONFIG_MARVELL_88X2222_PHY=m +CONFIG_MAXLINEAR_GPHY=m +CONFIG_MICROCHIP_T1_PHY=m +CONFIG_MICROSEMI_PHY=m +CONFIG_MOTORCOMM_PHY=m +CONFIG_NATIONAL_PHY=m +CONFIG_NXP_C45_TJA11XX_PHY=m +CONFIG_NXP_TJA11XX_PHY=m +CONFIG_AT803X_PHY=m +CONFIG_QSEMI_PHY=m +CONFIG_RENESAS_PHY=m +CONFIG_ROCKCHIP_PHY=m +CONFIG_STE10XP=m +CONFIG_TERANETICS_PHY=m +CONFIG_DP83822_PHY=m +CONFIG_DP83TC811_PHY=m +CONFIG_DP83848_PHY=m +CONFIG_DP83867_PHY=m +CONFIG_DP83869_PHY=m +CONFIG_DP83TD510_PHY=m +CONFIG_XILINX_GMII2RGMII=m +CONFIG_MICREL_KS8995MA=m +CONFIG_CAN_VCAN=m +CONFIG_CAN_VXCAN=m +CONFIG_CAN_FLEXCAN=m +CONFIG_CAN_GRCAN=m +CONFIG_CAN_JANZ_ICAN3=m +CONFIG_CAN_KVASER_PCIEFD=m +CONFIG_CAN_SLCAN=m +CONFIG_CAN_C_CAN=m +CONFIG_CAN_C_CAN_PLATFORM=m +CONFIG_CAN_C_CAN_PCI=m +CONFIG_CAN_CC770=m +CONFIG_CAN_CC770_ISA=m +CONFIG_CAN_CC770_PLATFORM=m +CONFIG_CAN_CTUCANFD_PCI=m +CONFIG_CAN_CTUCANFD_PLATFORM=m +CONFIG_CAN_IFI_CANFD=m +CONFIG_CAN_M_CAN=m +CONFIG_CAN_M_CAN_PCI=m +CONFIG_CAN_M_CAN_PLATFORM=m +CONFIG_CAN_M_CAN_TCAN4X5X=m +CONFIG_CAN_PEAK_PCIEFD=m +CONFIG_CAN_SJA1000=m +CONFIG_CAN_EMS_PCI=m +CONFIG_CAN_F81601=m +CONFIG_CAN_KVASER_PCI=m +CONFIG_CAN_PEAK_PCI=m +CONFIG_CAN_PLX_PCI=m +CONFIG_CAN_SJA1000_ISA=m +CONFIG_CAN_SJA1000_PLATFORM=m +CONFIG_CAN_SOFTING=m +CONFIG_CAN_HI311X=m +CONFIG_CAN_MCP251X=m +CONFIG_CAN_MCP251XFD=m +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ETAS_ES58X=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_MCBA_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_CAN_UCAN=m +CONFIG_MCTP_SERIAL=m +CONFIG_MDIO_GPIO=m +CONFIG_MDIO_HISI_FEMAC=m +CONFIG_MDIO_MVUSB=m +CONFIG_MDIO_OCTEON=m +CONFIG_MDIO_IPQ4019=m +CONFIG_MDIO_IPQ8064=m +CONFIG_MDIO_BUS_MUX_GPIO=m +CONFIG_MDIO_BUS_MUX_MULTIPLEXER=m +CONFIG_MDIO_BUS_MUX_MMIOREG=m +CONFIG_PLIP=m +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOATM=m +CONFIG_PPPOE=m +CONFIG_PPTP=m +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_SLIP=m +CONFIG_SLIP_COMPRESSED=y +CONFIG_SLIP_SMART=y +CONFIG_SLIP_MODE_SLIP6=y +CONFIG_USB_NET_DRIVERS=m +CONFIG_USB_CATC=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_LAN78XX=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_NET_SR9800=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=m +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_HSO=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_CDC_PHONET=m +CONFIG_USB_IPHETH=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_USB_NET_CH9200=m +CONFIG_USB_NET_AQC111=m +CONFIG_ADM8211=m +CONFIG_ATH5K=m +CONFIG_ATH9K=m +CONFIG_ATH9K_AHB=y +CONFIG_ATH9K_DEBUGFS=y +CONFIG_ATH9K_STATION_STATISTICS=y +CONFIG_ATH9K_CHANNEL_CONTEXT=y +CONFIG_ATH9K_PCI_NO_EEPROM=m +CONFIG_ATH9K_HTC=m +CONFIG_ATH9K_HTC_DEBUGFS=y +CONFIG_ATH9K_HWRNG=y +CONFIG_ATH9K_COMMON_SPECTRAL=y +CONFIG_CARL9170=m +CONFIG_CARL9170_HWRNG=y +CONFIG_ATH6KL=m +CONFIG_ATH6KL_SDIO=m +CONFIG_ATH6KL_USB=m +CONFIG_AR5523=m +CONFIG_WIL6210=m +CONFIG_WIL6210_TRACING=y +CONFIG_ATH10K=m +CONFIG_ATH10K_PCI=m +CONFIG_ATH10K_AHB=y +CONFIG_ATH10K_SDIO=m +CONFIG_ATH10K_USB=m +CONFIG_ATH10K_DEBUGFS=y +CONFIG_ATH10K_SPECTRAL=y +CONFIG_ATH10K_TRACING=y +CONFIG_WCN36XX=m +CONFIG_ATH11K=m +CONFIG_ATH11K_PCI=m +CONFIG_ATH11K_DEBUGFS=y +CONFIG_ATH11K_TRACING=y +CONFIG_ATH11K_SPECTRAL=y +CONFIG_ATMEL=m +CONFIG_PCI_ATMEL=m +CONFIG_AT76C50X_USB=m +CONFIG_B43=m +CONFIG_B43LEGACY=m +# CONFIG_B43LEGACY_DEBUG is not set +CONFIG_BRCMSMAC=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_BRCMFMAC_PCIE=y +CONFIG_BRCM_TRACING=y +CONFIG_IPW2100=m +CONFIG_IPW2100_MONITOR=y +CONFIG_IPW2200=m +CONFIG_IPW2200_MONITOR=y +CONFIG_IPW2200_PROMISCUOUS=y +CONFIG_IPW2200_QOS=y +CONFIG_IWL4965=m +CONFIG_IWL3945=m +CONFIG_IWLEGACY_DEBUGFS=y +CONFIG_IWLWIFI=m +CONFIG_IWLDVM=m +CONFIG_IWLMVM=m +CONFIG_IWLWIFI_DEBUGFS=y +CONFIG_HOSTAP=m +CONFIG_HOSTAP_FIRMWARE=y +CONFIG_HOSTAP_FIRMWARE_NVRAM=y +CONFIG_HOSTAP_PLX=m +CONFIG_HOSTAP_PCI=m +CONFIG_HERMES=m +CONFIG_PLX_HERMES=m +CONFIG_TMD_HERMES=m +CONFIG_NORTEL_HERMES=m +CONFIG_ORINOCO_USB=m +CONFIG_P54_COMMON=m +CONFIG_P54_USB=m +CONFIG_P54_PCI=m +CONFIG_P54_SPI=m +CONFIG_LIBERTAS=m +CONFIG_LIBERTAS_USB=m +CONFIG_LIBERTAS_SDIO=m +CONFIG_LIBERTAS_SPI=m +CONFIG_LIBERTAS_MESH=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MWIFIEX=m +CONFIG_MWIFIEX_SDIO=m +CONFIG_MWIFIEX_PCIE=m +CONFIG_MWIFIEX_USB=m +CONFIG_MWL8K=m +CONFIG_MT7601U=m +CONFIG_MT76x0U=m +CONFIG_MT76x0E=m +CONFIG_MT76x2E=m +CONFIG_MT76x2U=m +CONFIG_MT7603E=m +CONFIG_MT7615E=m +CONFIG_MT7663U=m +CONFIG_MT7663S=m +CONFIG_MT7915E=m +CONFIG_MT7921E=m +CONFIG_MT7921S=m +CONFIG_MT7921U=m +CONFIG_WILC1000_SDIO=m +CONFIG_WILC1000_SPI=m +CONFIG_WILC1000_HW_OOB_INTR=y +CONFIG_PLFXLC=m +CONFIG_RT2X00=m +CONFIG_RT2400PCI=m +CONFIG_RT2500PCI=m +CONFIG_RT61PCI=m +CONFIG_RT2800PCI=m +CONFIG_RT2500USB=m +CONFIG_RT73USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RT2800USB_UNKNOWN=y +CONFIG_RTL8180=m +CONFIG_RTL8187=m +CONFIG_RTL8192CE=m +CONFIG_RTL8192SE=m +CONFIG_RTL8192DE=m +CONFIG_RTL8723AE=m +CONFIG_RTL8723BE=m +CONFIG_RTL8188EE=m +CONFIG_RTL8192EE=m +CONFIG_RTL8821AE=m +CONFIG_RTL8192CU=m +# CONFIG_RTLWIFI_DEBUG is not set +CONFIG_RTL8XXXU=m +CONFIG_RTL8XXXU_UNTESTED=y +CONFIG_RTW88=m +CONFIG_RTW88_8822BE=m +CONFIG_RTW88_8822CE=m +CONFIG_RTW88_8723DE=m +CONFIG_RTW88_8821CE=m +CONFIG_RTW88_DEBUG=y +CONFIG_RTW88_DEBUGFS=y +CONFIG_RTW89=m +CONFIG_RTW89_8852AE=m +CONFIG_RTW89_8852CE=m +CONFIG_RTW89_DEBUGMSG=y +CONFIG_RTW89_DEBUGFS=y +CONFIG_RSI_91X=m +# CONFIG_RSI_DEBUGFS is not set +CONFIG_WFX=m +CONFIG_CW1200=m +CONFIG_CW1200_WLAN_SDIO=m +CONFIG_CW1200_WLAN_SPI=m +CONFIG_WL1251=m +CONFIG_WL1251_SPI=m +CONFIG_WL1251_SDIO=m +CONFIG_WL12XX=m +CONFIG_WL18XX=m +CONFIG_WLCORE_SPI=m +CONFIG_WLCORE_SDIO=m +CONFIG_USB_ZD1201=m +CONFIG_ZD1211RW=m +CONFIG_QTNFMAC_PCIE=m +CONFIG_MAC80211_HWSIM=m +CONFIG_USB_NET_RNDIS_WLAN=m +CONFIG_VIRT_WIFI=m +CONFIG_WAN=y +CONFIG_HDLC=m +CONFIG_HDLC_RAW=m +CONFIG_HDLC_RAW_ETH=m +CONFIG_HDLC_CISCO=m +CONFIG_HDLC_FR=m +CONFIG_HDLC_PPP=m +CONFIG_HDLC_X25=m +CONFIG_PCI200SYN=m +CONFIG_WANXL=m +CONFIG_PC300TOO=m +CONFIG_FARSYNC=m +CONFIG_LAPBETHER=m +CONFIG_IEEE802154_FAKELB=m +CONFIG_IEEE802154_AT86RF230=m +CONFIG_IEEE802154_MRF24J40=m +CONFIG_IEEE802154_CC2520=m +CONFIG_IEEE802154_ATUSB=m +CONFIG_IEEE802154_ADF7242=m +CONFIG_IEEE802154_CA8210=m +CONFIG_IEEE802154_CA8210_DEBUGFS=y +CONFIG_IEEE802154_MCR20A=m +CONFIG_IEEE802154_HWSIM=m +CONFIG_WWAN=m +CONFIG_WWAN_HWSIM=m +CONFIG_MHI_WWAN_CTRL=m +CONFIG_MHI_WWAN_MBIM=m +CONFIG_RPMSG_WWAN_CTRL=m +CONFIG_MTK_T7XX=m +CONFIG_VMXNET3=m +CONFIG_USB4_NET=m +CONFIG_NETDEVSIM=m +CONFIG_ISDN=y +CONFIG_MISDN=m +CONFIG_MISDN_DSP=m +CONFIG_MISDN_L1OIP=m +CONFIG_MISDN_HFCPCI=m +CONFIG_MISDN_HFCMULTI=m +CONFIG_MISDN_HFCUSB=m +CONFIG_MISDN_AVMFRITZ=m +CONFIG_MISDN_SPEEDFAX=m +CONFIG_MISDN_INFINEON=m +CONFIG_MISDN_W6692=m +CONFIG_MISDN_NETJET=m +CONFIG_INPUT_LEDS=m +CONFIG_INPUT_SPARSEKMAP=m +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_MOUSEDEV_PSAUX=y +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_EVBUG=m +CONFIG_KEYBOARD_ADC=m +CONFIG_KEYBOARD_ADP5520=m +CONFIG_KEYBOARD_ADP5588=m +CONFIG_KEYBOARD_ADP5589=m +CONFIG_KEYBOARD_QT1050=m +CONFIG_KEYBOARD_QT1070=m +CONFIG_KEYBOARD_QT2160=m +CONFIG_KEYBOARD_DLINK_DIR685=m +CONFIG_KEYBOARD_LKKBD=m +CONFIG_KEYBOARD_GPIO=m +CONFIG_KEYBOARD_GPIO_POLLED=m +CONFIG_KEYBOARD_TCA6416=m +CONFIG_KEYBOARD_TCA8418=m +CONFIG_KEYBOARD_MATRIX=m +CONFIG_KEYBOARD_LM8323=m +CONFIG_KEYBOARD_LM8333=m +CONFIG_KEYBOARD_MAX7359=m +CONFIG_KEYBOARD_MCS=m +CONFIG_KEYBOARD_MPR121=m +CONFIG_KEYBOARD_NEWTON=m +CONFIG_KEYBOARD_OPENCORES=m +CONFIG_KEYBOARD_SAMSUNG=m +CONFIG_KEYBOARD_GOLDFISH_EVENTS=m +CONFIG_KEYBOARD_STOWAWAY=m +CONFIG_KEYBOARD_SUNKBD=m +CONFIG_KEYBOARD_STMPE=m +CONFIG_KEYBOARD_IQS62X=m +CONFIG_KEYBOARD_OMAP4=m +CONFIG_KEYBOARD_TC3589X=m +CONFIG_KEYBOARD_TM2_TOUCHKEY=m +CONFIG_KEYBOARD_TWL4030=m +CONFIG_KEYBOARD_XTKBD=m +CONFIG_KEYBOARD_CAP11XX=m +CONFIG_KEYBOARD_BCM=m +CONFIG_KEYBOARD_MTK_PMIC=m +CONFIG_KEYBOARD_CYPRESS_SF=m +CONFIG_MOUSE_PS2=m +CONFIG_MOUSE_PS2_ELANTECH=y +CONFIG_MOUSE_PS2_SENTELIC=y +CONFIG_MOUSE_PS2_TOUCHKIT=y +CONFIG_MOUSE_SERIAL=m +CONFIG_MOUSE_APPLETOUCH=m +CONFIG_MOUSE_BCM5974=m +CONFIG_MOUSE_CYAPA=m +CONFIG_MOUSE_ELAN_I2C=m +CONFIG_MOUSE_ELAN_I2C_SMBUS=y +CONFIG_MOUSE_VSXXXAA=m +CONFIG_MOUSE_GPIO=m +CONFIG_MOUSE_SYNAPTICS_I2C=m +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_ANALOG=m +CONFIG_JOYSTICK_A3D=m +CONFIG_JOYSTICK_ADC=m +CONFIG_JOYSTICK_ADI=m +CONFIG_JOYSTICK_COBRA=m +CONFIG_JOYSTICK_GF2K=m +CONFIG_JOYSTICK_GRIP=m +CONFIG_JOYSTICK_GRIP_MP=m +CONFIG_JOYSTICK_GUILLEMOT=m +CONFIG_JOYSTICK_INTERACT=m +CONFIG_JOYSTICK_SIDEWINDER=m +CONFIG_JOYSTICK_TMDC=m +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=m +CONFIG_JOYSTICK_IFORCE_232=m +CONFIG_JOYSTICK_WARRIOR=m +CONFIG_JOYSTICK_MAGELLAN=m +CONFIG_JOYSTICK_SPACEORB=m +CONFIG_JOYSTICK_SPACEBALL=m +CONFIG_JOYSTICK_STINGER=m +CONFIG_JOYSTICK_TWIDJOY=m +CONFIG_JOYSTICK_ZHENHUA=m +CONFIG_JOYSTICK_DB9=m +CONFIG_JOYSTICK_GAMECON=m +CONFIG_JOYSTICK_TURBOGRAFX=m +CONFIG_JOYSTICK_AS5011=m +CONFIG_JOYSTICK_JOYDUMP=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_JOYSTICK_WALKERA0701=m +CONFIG_JOYSTICK_PSXPAD_SPI=m +CONFIG_JOYSTICK_PSXPAD_SPI_FF=y +CONFIG_JOYSTICK_PXRC=m +CONFIG_JOYSTICK_QWIIC=m +CONFIG_JOYSTICK_FSIA6B=m +CONFIG_JOYSTICK_SENSEHAT=m +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_TABLET_SERIAL_WACOM4=m +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_88PM860X=m +CONFIG_TOUCHSCREEN_ADS7846=m +CONFIG_TOUCHSCREEN_AD7877=m +CONFIG_TOUCHSCREEN_AD7879=m +CONFIG_TOUCHSCREEN_AD7879_I2C=m +CONFIG_TOUCHSCREEN_AD7879_SPI=m +CONFIG_TOUCHSCREEN_ADC=m +CONFIG_TOUCHSCREEN_AR1021_I2C=m +CONFIG_TOUCHSCREEN_ATMEL_MXT=m +CONFIG_TOUCHSCREEN_ATMEL_MXT_T37=y +CONFIG_TOUCHSCREEN_AUO_PIXCIR=m +CONFIG_TOUCHSCREEN_BU21013=m +CONFIG_TOUCHSCREEN_BU21029=m +CONFIG_TOUCHSCREEN_CHIPONE_ICN8318=m +CONFIG_TOUCHSCREEN_CY8CTMA140=m +CONFIG_TOUCHSCREEN_CY8CTMG110=m +CONFIG_TOUCHSCREEN_CYTTSP_CORE=m +CONFIG_TOUCHSCREEN_CYTTSP_I2C=m +CONFIG_TOUCHSCREEN_CYTTSP_SPI=m +CONFIG_TOUCHSCREEN_CYTTSP4_CORE=m +CONFIG_TOUCHSCREEN_CYTTSP4_I2C=m +CONFIG_TOUCHSCREEN_CYTTSP4_SPI=m +CONFIG_TOUCHSCREEN_DA9034=m +CONFIG_TOUCHSCREEN_DA9052=m +CONFIG_TOUCHSCREEN_DYNAPRO=m +CONFIG_TOUCHSCREEN_HAMPSHIRE=m +CONFIG_TOUCHSCREEN_EETI=m +CONFIG_TOUCHSCREEN_EGALAX=m +CONFIG_TOUCHSCREEN_EGALAX_SERIAL=m +CONFIG_TOUCHSCREEN_EXC3000=m +CONFIG_TOUCHSCREEN_FUJITSU=m +CONFIG_TOUCHSCREEN_GOODIX=m +CONFIG_TOUCHSCREEN_HIDEEP=m +CONFIG_TOUCHSCREEN_HYCON_HY46XX=m +CONFIG_TOUCHSCREEN_ILI210X=m +CONFIG_TOUCHSCREEN_ILITEK=m +CONFIG_TOUCHSCREEN_S6SY761=m +CONFIG_TOUCHSCREEN_GUNZE=m +CONFIG_TOUCHSCREEN_EKTF2127=m +CONFIG_TOUCHSCREEN_ELAN=m +CONFIG_TOUCHSCREEN_ELO=m +CONFIG_TOUCHSCREEN_WACOM_W8001=m +CONFIG_TOUCHSCREEN_WACOM_I2C=m +CONFIG_TOUCHSCREEN_MAX11801=m +CONFIG_TOUCHSCREEN_MCS5000=m +CONFIG_TOUCHSCREEN_MMS114=m +CONFIG_TOUCHSCREEN_MELFAS_MIP4=m +CONFIG_TOUCHSCREEN_MSG2638=m +CONFIG_TOUCHSCREEN_MTOUCH=m +CONFIG_TOUCHSCREEN_IMAGIS=m +CONFIG_TOUCHSCREEN_IMX6UL_TSC=m +CONFIG_TOUCHSCREEN_INEXIO=m +CONFIG_TOUCHSCREEN_MK712=m +CONFIG_TOUCHSCREEN_PENMOUNT=m +CONFIG_TOUCHSCREEN_EDT_FT5X06=m +CONFIG_TOUCHSCREEN_TOUCHRIGHT=m +CONFIG_TOUCHSCREEN_TOUCHWIN=m +CONFIG_TOUCHSCREEN_TI_AM335X_TSC=m +CONFIG_TOUCHSCREEN_PIXCIR=m +CONFIG_TOUCHSCREEN_WDT87XX_I2C=m +CONFIG_TOUCHSCREEN_WM831X=m +CONFIG_TOUCHSCREEN_WM97XX=m +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_MC13783=m +CONFIG_TOUCHSCREEN_TOUCHIT213=m +CONFIG_TOUCHSCREEN_TSC_SERIO=m +CONFIG_TOUCHSCREEN_TSC2004=m +CONFIG_TOUCHSCREEN_TSC2005=m +CONFIG_TOUCHSCREEN_TSC2007=m +CONFIG_TOUCHSCREEN_TSC2007_IIO=y +CONFIG_TOUCHSCREEN_PCAP=m +CONFIG_TOUCHSCREEN_RM_TS=m +CONFIG_TOUCHSCREEN_SILEAD=m +CONFIG_TOUCHSCREEN_SIS_I2C=m +CONFIG_TOUCHSCREEN_ST1232=m +CONFIG_TOUCHSCREEN_STMFTS=m +CONFIG_TOUCHSCREEN_STMPE=m +CONFIG_TOUCHSCREEN_SUR40=m +CONFIG_TOUCHSCREEN_SURFACE3_SPI=m +CONFIG_TOUCHSCREEN_SX8654=m +CONFIG_TOUCHSCREEN_TPS6507X=m +CONFIG_TOUCHSCREEN_ZET6223=m +CONFIG_TOUCHSCREEN_ZFORCE=m +CONFIG_TOUCHSCREEN_COLIBRI_VF50=m +CONFIG_TOUCHSCREEN_ROHM_BU21023=m +CONFIG_TOUCHSCREEN_IQS5XX=m +CONFIG_TOUCHSCREEN_ZINITIX=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_88PM860X_ONKEY=m +CONFIG_INPUT_88PM80X_ONKEY=m +CONFIG_INPUT_AD714X=m +CONFIG_INPUT_ARIZONA_HAPTICS=m +CONFIG_INPUT_ATC260X_ONKEY=m +CONFIG_INPUT_ATMEL_CAPTOUCH=m +CONFIG_INPUT_BMA150=m +CONFIG_INPUT_E3X0_BUTTON=m +CONFIG_INPUT_MAX77650_ONKEY=m +CONFIG_INPUT_MAX77693_HAPTIC=m +CONFIG_INPUT_MAX8925_ONKEY=m +CONFIG_INPUT_MAX8997_HAPTIC=m +CONFIG_INPUT_MC13783_PWRBUTTON=m +CONFIG_INPUT_MMA8450=m +CONFIG_INPUT_GPIO_BEEPER=m +CONFIG_INPUT_GPIO_DECODER=m +CONFIG_INPUT_GPIO_VIBRA=m +CONFIG_INPUT_CPCAP_PWRBUTTON=m +CONFIG_INPUT_ATI_REMOTE2=m +CONFIG_INPUT_KEYSPAN_REMOTE=m +CONFIG_INPUT_KXTJ9=m +CONFIG_INPUT_POWERMATE=m +CONFIG_INPUT_YEALINK=m +CONFIG_INPUT_CM109=m +CONFIG_INPUT_REGULATOR_HAPTIC=m +CONFIG_INPUT_RETU_PWRBUTTON=m +CONFIG_INPUT_TPS65218_PWRBUTTON=m +CONFIG_INPUT_AXP20X_PEK=m +CONFIG_INPUT_TWL4030_PWRBUTTON=m +CONFIG_INPUT_TWL4030_VIBRA=m +CONFIG_INPUT_TWL6040_VIBRA=m +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_PALMAS_PWRBUTTON=m +CONFIG_INPUT_PCF50633_PMU=m +CONFIG_INPUT_PCF8574=m +CONFIG_INPUT_PWM_BEEPER=m +CONFIG_INPUT_PWM_VIBRA=m +CONFIG_INPUT_RK805_PWRKEY=m +CONFIG_INPUT_GPIO_ROTARY_ENCODER=m +CONFIG_INPUT_DA7280_HAPTICS=m +CONFIG_INPUT_DA9052_ONKEY=m +CONFIG_INPUT_DA9055_ONKEY=m +CONFIG_INPUT_DA9063_ONKEY=m +CONFIG_INPUT_WM831X_ON=m +CONFIG_INPUT_PCAP=m +CONFIG_INPUT_ADXL34X=m +CONFIG_INPUT_IMS_PCU=m +CONFIG_INPUT_IQS269A=m +CONFIG_INPUT_IQS626A=m +CONFIG_INPUT_IQS7222=m +CONFIG_INPUT_CMA3000=m +CONFIG_INPUT_CMA3000_I2C=m +CONFIG_INPUT_DRV260X_HAPTICS=m +CONFIG_INPUT_DRV2665_HAPTICS=m +CONFIG_INPUT_DRV2667_HAPTICS=m +CONFIG_INPUT_RAVE_SP_PWRBUTTON=m +CONFIG_INPUT_STPMIC1_ONKEY=m +CONFIG_RMI4_I2C=m +CONFIG_RMI4_SPI=m +CONFIG_RMI4_SMB=m +CONFIG_RMI4_F34=y +CONFIG_RMI4_F3A=y +CONFIG_RMI4_F54=y +CONFIG_SERIO_SERPORT=m +CONFIG_SERIO_PARKBD=m +CONFIG_SERIO_PCIPS2=m +CONFIG_SERIO_RAW=m +CONFIG_SERIO_ALTERA_PS2=m +CONFIG_SERIO_PS2MULT=m +CONFIG_SERIO_ARC_PS2=m +CONFIG_SERIO_APBPS2=m +CONFIG_SERIO_GPIO_PS2=m +CONFIG_USERIO=m +CONFIG_GAMEPORT_NS558=m +CONFIG_GAMEPORT_L4=m +CONFIG_GAMEPORT_EMU10K1=m +CONFIG_GAMEPORT_FM801=m +CONFIG_LEGACY_PTY_COUNT=0 +CONFIG_SERIAL_8250=y +# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +CONFIG_SERIAL_8250_FINTEK=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_EXAR=m +CONFIG_SERIAL_8250_MEN_MCB=m +CONFIG_SERIAL_8250_NR_UARTS=48 +CONFIG_SERIAL_8250_RUNTIME_UARTS=32 +CONFIG_SERIAL_8250_EXTENDED=y +CONFIG_SERIAL_8250_MANY_PORTS=y +CONFIG_SERIAL_8250_SHARE_IRQ=y +CONFIG_SERIAL_8250_RSA=y +CONFIG_SERIAL_8250_DW=m +CONFIG_SERIAL_8250_RT288X=y +CONFIG_SERIAL_8250_PERICOM=m +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_KGDB_NMI=y +CONFIG_SERIAL_MAX3100=m +CONFIG_SERIAL_MAX310X=y +CONFIG_SERIAL_UARTLITE=m +CONFIG_SERIAL_JSM=m +CONFIG_SERIAL_SCCNXP=y +CONFIG_SERIAL_SCCNXP_CONSOLE=y +CONFIG_SERIAL_SC16IS7XX=m +CONFIG_SERIAL_SC16IS7XX_SPI=y +CONFIG_SERIAL_ALTERA_JTAGUART=m +CONFIG_SERIAL_ALTERA_UART=m +CONFIG_SERIAL_XILINX_PS_UART=m +CONFIG_SERIAL_ARC=m +CONFIG_SERIAL_RP2=m +CONFIG_SERIAL_FSL_LPUART=m +CONFIG_SERIAL_CONEXANT_DIGICOLOR=m +CONFIG_SERIAL_MEN_Z135=m +CONFIG_SERIAL_SPRD=m +CONFIG_SERIAL_LITEUART=m +CONFIG_SERIAL_NONSTANDARD=y +CONFIG_MOXA_INTELLIO=m +CONFIG_MOXA_SMARTIO=m +CONFIG_SYNCLINK_GT=m +CONFIG_N_HDLC=m +CONFIG_GOLDFISH_TTY=m +CONFIG_N_GSM=m +CONFIG_NOZOMI=m +CONFIG_NULL_TTY=m +CONFIG_RPMSG_TTY=m +CONFIG_SERIAL_DEV_BUS=y +CONFIG_TTY_PRINTK=y +CONFIG_PRINTER=m +CONFIG_PPDEV=m +CONFIG_VIRTIO_CONSOLE=y +CONFIG_IPMI_HANDLER=m +CONFIG_IPMI_DEVICE_INTERFACE=m +CONFIG_IPMI_SSIF=m +CONFIG_IPMI_WATCHDOG=m +CONFIG_IPMI_POWEROFF=m +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_TIMERIOMEM=m +CONFIG_HW_RANDOM_BA431=m +CONFIG_HW_RANDOM_VIRTIO=m +CONFIG_HW_RANDOM_POLARFIRE_SOC=m +CONFIG_HW_RANDOM_CCTRNG=m +CONFIG_HW_RANDOM_XIPHERA=m +CONFIG_APPLICOM=m +CONFIG_TCG_TIS=y +CONFIG_TCG_TIS_SPI=m +CONFIG_TCG_TIS_SPI_CR50=y +CONFIG_TCG_TIS_I2C_CR50=m +CONFIG_TCG_TIS_I2C_ATMEL=m +CONFIG_TCG_TIS_I2C_INFINEON=m +CONFIG_TCG_TIS_I2C_NUVOTON=m +CONFIG_TCG_ATMEL=m +CONFIG_TCG_VTPM_PROXY=m +CONFIG_TCG_TIS_ST33ZP24_I2C=m +CONFIG_TCG_TIS_ST33ZP24_SPI=m +CONFIG_XILLYBUS=m +CONFIG_XILLYBUS_PCIE=m +CONFIG_XILLYBUS_OF=m +CONFIG_XILLYUSB=m +CONFIG_I2C=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_ARB_GPIO_CHALLENGE=m +CONFIG_I2C_MUX_GPIO=m +CONFIG_I2C_MUX_GPMUX=m +CONFIG_I2C_MUX_LTC4306=m +CONFIG_I2C_MUX_PCA9541=m +CONFIG_I2C_MUX_PCA954x=m +CONFIG_I2C_MUX_PINCTRL=m +CONFIG_I2C_MUX_REG=m +CONFIG_I2C_DEMUX_PINCTRL=m +CONFIG_I2C_MUX_MLXCPLD=m +CONFIG_I2C_ALI1535=m +CONFIG_I2C_ALI1563=m +CONFIG_I2C_ALI15X3=m +CONFIG_I2C_AMD756=m +CONFIG_I2C_AMD8111=m +CONFIG_I2C_I801=m +CONFIG_I2C_ISCH=m +CONFIG_I2C_PIIX4=m +CONFIG_I2C_NFORCE2=m +CONFIG_I2C_NVIDIA_GPU=m +CONFIG_I2C_SIS5595=m +CONFIG_I2C_SIS630=m +CONFIG_I2C_SIS96X=m +CONFIG_I2C_VIA=m +CONFIG_I2C_VIAPRO=m +CONFIG_I2C_CBUS_GPIO=m +CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_I2C_DESIGNWARE_PCI=m +CONFIG_I2C_GPIO=m +CONFIG_I2C_KEMPLD=m +CONFIG_I2C_OCORES=m +CONFIG_I2C_PCA_PLATFORM=m +CONFIG_I2C_RK3X=m +CONFIG_I2C_SIMTEC=m +CONFIG_I2C_XILINX=m +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_DLN2=m +CONFIG_I2C_CP2615=m +CONFIG_I2C_PARPORT=m +CONFIG_I2C_ROBOTFUZZ_OSIF=m +CONFIG_I2C_TAOS_EVM=m +CONFIG_I2C_TINY_USB=m +CONFIG_I2C_VIPERBOARD=m +CONFIG_I2C_FSI=m +CONFIG_I2C_VIRTIO=m +CONFIG_I2C_STUB=m +CONFIG_I3C=m +CONFIG_CDNS_I3C_MASTER=m +CONFIG_DW_I3C_MASTER=m +CONFIG_SVC_I3C_MASTER=m +CONFIG_MIPI_I3C_HCI=m +CONFIG_SPI=y +CONFIG_SPI_ALTERA=m +CONFIG_SPI_ALTERA_DFL=m +CONFIG_SPI_AXI_SPI_ENGINE=m +CONFIG_SPI_BUTTERFLY=m +CONFIG_SPI_CADENCE=m +CONFIG_SPI_CADENCE_XSPI=m +CONFIG_SPI_DESIGNWARE=m +CONFIG_SPI_DW_DMA=y +CONFIG_SPI_DW_PCI=m +CONFIG_SPI_DW_MMIO=m +CONFIG_SPI_DLN2=m +CONFIG_SPI_FSI=m +CONFIG_SPI_NXP_FLEXSPI=m +CONFIG_SPI_GPIO=m +CONFIG_SPI_LM70_LLP=m +CONFIG_SPI_FSL_SPI=y +CONFIG_SPI_OC_TINY=m +CONFIG_SPI_PXA2XX=m +CONFIG_SPI_SC18IS602=m +CONFIG_SPI_SIFIVE=y +CONFIG_SPI_MXIC=m +CONFIG_SPI_XCOMM=m +CONFIG_SPI_ZYNQMP_GQSPI=m +CONFIG_SPI_AMD=m +CONFIG_SPI_MUX=m +CONFIG_SPI_SPIDEV=m +CONFIG_SPI_LOOPBACK_TEST=m +CONFIG_SPI_TLE62X0=m +CONFIG_SPI_SLAVE=y +CONFIG_SPI_SLAVE_TIME=m +CONFIG_SPI_SLAVE_SYSTEM_CONTROL=m +CONFIG_SPMI=m +CONFIG_SPMI_HISI3670=m +CONFIG_HSI=m +CONFIG_HSI_CHAR=m +CONFIG_PPS_CLIENT_LDISC=m +CONFIG_PPS_CLIENT_PARPORT=m +CONFIG_PPS_CLIENT_GPIO=m +CONFIG_DP83640_PHY=m +CONFIG_PTP_1588_CLOCK_INES=m +CONFIG_PTP_1588_CLOCK_IDT82P33=m +CONFIG_PTP_1588_CLOCK_IDTCM=m +CONFIG_PTP_1588_CLOCK_OCP=m +CONFIG_PINCTRL_AS3722=y +CONFIG_PINCTRL_AXP209=m +CONFIG_PINCTRL_DA9062=m +CONFIG_PINCTRL_MAX77620=m +CONFIG_PINCTRL_MCP23S08=m +CONFIG_PINCTRL_MICROCHIP_SGPIO=y +CONFIG_PINCTRL_OCELOT=y +CONFIG_PINCTRL_PALMAS=y +CONFIG_PINCTRL_RK805=m +CONFIG_PINCTRL_SINGLE=y +CONFIG_PINCTRL_STMFX=m +CONFIG_PINCTRL_SX150X=y +CONFIG_PINCTRL_LOCHNAGAR=m +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_74XX_MMIO=m +CONFIG_GPIO_ALTERA=m +CONFIG_GPIO_CADENCE=m +CONFIG_GPIO_DWAPB=m +CONFIG_GPIO_EXAR=m +CONFIG_GPIO_FTGPIO010=y +CONFIG_GPIO_GENERIC_PLATFORM=y +CONFIG_GPIO_GRGPIO=m +CONFIG_GPIO_HLWD=m +CONFIG_GPIO_LOGICVC=m +CONFIG_GPIO_MB86S7X=m +CONFIG_GPIO_MENZ127=m +CONFIG_GPIO_SIFIVE=y +CONFIG_GPIO_SIOX=m +CONFIG_GPIO_SYSCON=m +CONFIG_GPIO_WCD934X=m +CONFIG_GPIO_ADNP=m +CONFIG_GPIO_GW_PLD=m +CONFIG_GPIO_MAX7300=m +CONFIG_GPIO_MAX732X=m +CONFIG_GPIO_PCA953X=m +CONFIG_GPIO_PCA953X_IRQ=y +CONFIG_GPIO_PCA9570=m +CONFIG_GPIO_PCF857X=m +CONFIG_GPIO_TPIC2810=m +CONFIG_GPIO_ADP5520=m +CONFIG_GPIO_ARIZONA=m +CONFIG_GPIO_BD71815=m +CONFIG_GPIO_BD71828=m +CONFIG_GPIO_BD9571MWV=m +CONFIG_GPIO_DA9052=m +CONFIG_GPIO_DA9055=m +CONFIG_GPIO_DLN2=m +CONFIG_GPIO_JANZ_TTL=m +CONFIG_GPIO_KEMPLD=m +CONFIG_GPIO_LP3943=m +CONFIG_GPIO_LP873X=m +CONFIG_GPIO_LP87565=m +CONFIG_GPIO_MADERA=m +CONFIG_GPIO_MAX77620=m +CONFIG_GPIO_MAX77650=m +CONFIG_GPIO_PALMAS=y +CONFIG_GPIO_RC5T583=y +CONFIG_GPIO_STMPE=y +CONFIG_GPIO_TC3589X=y +CONFIG_GPIO_TPS65086=m +CONFIG_GPIO_TPS65218=m +CONFIG_GPIO_TPS6586X=y +CONFIG_GPIO_TPS65910=y +CONFIG_GPIO_TPS65912=m +CONFIG_GPIO_TWL4030=m +CONFIG_GPIO_TWL6040=m +CONFIG_GPIO_WM831X=m +CONFIG_GPIO_WM8350=m +CONFIG_GPIO_WM8994=m +CONFIG_GPIO_PCI_IDIO_16=m +CONFIG_GPIO_PCIE_IDIO_24=m +CONFIG_GPIO_RDC321X=m +CONFIG_GPIO_74X164=m +CONFIG_GPIO_MAX3191X=m +CONFIG_GPIO_MAX7301=m +CONFIG_GPIO_MC33880=m +CONFIG_GPIO_PISOSR=m +CONFIG_GPIO_XRA1403=m +CONFIG_GPIO_MOXTET=m +CONFIG_GPIO_VIPERBOARD=m +CONFIG_GPIO_AGGREGATOR=m +CONFIG_GPIO_VIRTIO=m +CONFIG_GPIO_SIM=m +CONFIG_W1_MASTER_MATROX=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_MASTER_DS2482=m +CONFIG_W1_MASTER_GPIO=m +CONFIG_W1_MASTER_SGI=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS2430=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS250X=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_W1_SLAVE_DS28E17=m +CONFIG_POWER_RESET_AS3722=y +CONFIG_POWER_RESET_ATC260X=m +CONFIG_POWER_RESET_GPIO=y +CONFIG_POWER_RESET_GPIO_RESTART=y +CONFIG_POWER_RESET_LTC2952=y +CONFIG_POWER_RESET_MT6323=y +CONFIG_POWER_RESET_REGULATOR=y +CONFIG_POWER_RESET_RESTART=y +CONFIG_POWER_RESET_TPS65086=y +CONFIG_SYSCON_REBOOT_MODE=m +CONFIG_NVMEM_REBOOT_MODE=m +CONFIG_GENERIC_ADC_BATTERY=m +CONFIG_IP5XXX_POWER=m +CONFIG_MAX8925_POWER=m +CONFIG_WM831X_BACKUP=m +CONFIG_WM831X_POWER=m +CONFIG_WM8350_POWER=m +CONFIG_TEST_POWER=m +CONFIG_BATTERY_88PM860X=m +CONFIG_CHARGER_ADP5061=m +CONFIG_BATTERY_ACT8945A=m +CONFIG_BATTERY_CW2015=m +CONFIG_BATTERY_DS2760=m +CONFIG_BATTERY_DS2780=m +CONFIG_BATTERY_DS2781=m +CONFIG_BATTERY_DS2782=m +CONFIG_BATTERY_SAMSUNG_SDI=y +CONFIG_BATTERY_SBS=m +CONFIG_CHARGER_SBS=m +CONFIG_MANAGER_SBS=m +CONFIG_BATTERY_BQ27XXX=m +CONFIG_BATTERY_DA9030=m +CONFIG_BATTERY_DA9052=m +CONFIG_CHARGER_DA9150=m +CONFIG_BATTERY_DA9150=m +CONFIG_CHARGER_AXP20X=m +CONFIG_BATTERY_AXP20X=m +CONFIG_AXP20X_POWER=m +CONFIG_BATTERY_MAX17040=m +CONFIG_BATTERY_MAX17042=m +CONFIG_BATTERY_MAX1721X=m +CONFIG_BATTERY_TWL4030_MADC=m +CONFIG_CHARGER_88PM860X=m +CONFIG_CHARGER_PCF50633=m +CONFIG_BATTERY_RX51=m +CONFIG_CHARGER_ISP1704=m +CONFIG_CHARGER_MAX8903=m +CONFIG_CHARGER_TWL4030=m +CONFIG_CHARGER_LP8727=m +CONFIG_CHARGER_LP8788=m +CONFIG_CHARGER_GPIO=m +CONFIG_CHARGER_MANAGER=y +CONFIG_CHARGER_LT3651=m +CONFIG_CHARGER_LTC4162L=m +CONFIG_CHARGER_MAX14577=m +CONFIG_CHARGER_DETECTOR_MAX14656=m +CONFIG_CHARGER_MAX77650=m +CONFIG_CHARGER_MAX77693=m +CONFIG_CHARGER_MAX77976=m +CONFIG_CHARGER_MAX8997=m +CONFIG_CHARGER_MAX8998=m +CONFIG_CHARGER_MP2629=m +CONFIG_CHARGER_MT6360=m +CONFIG_CHARGER_BQ2415X=m +CONFIG_CHARGER_BQ24190=m +CONFIG_CHARGER_BQ24257=m +CONFIG_CHARGER_BQ24735=m +CONFIG_CHARGER_BQ2515X=m +CONFIG_CHARGER_BQ25890=m +CONFIG_CHARGER_BQ25980=m +CONFIG_CHARGER_BQ256XX=m +CONFIG_CHARGER_SMB347=m +CONFIG_CHARGER_TPS65090=m +CONFIG_BATTERY_GAUGE_LTC2941=m +CONFIG_BATTERY_GOLDFISH=m +CONFIG_BATTERY_RT5033=m +CONFIG_CHARGER_RT9455=m +CONFIG_CHARGER_UCS1002=m +CONFIG_CHARGER_BD99954=m +CONFIG_RN5T618_POWER=m +CONFIG_BATTERY_UG3105=m +CONFIG_SENSORS_AD7314=m +CONFIG_SENSORS_AD7414=m +CONFIG_SENSORS_AD7418=m +CONFIG_SENSORS_ADM1025=m +CONFIG_SENSORS_ADM1026=m +CONFIG_SENSORS_ADM1029=m +CONFIG_SENSORS_ADM1031=m +CONFIG_SENSORS_ADM1177=m +CONFIG_SENSORS_ADM9240=m +CONFIG_SENSORS_ADT7310=m +CONFIG_SENSORS_ADT7410=m +CONFIG_SENSORS_ADT7411=m +CONFIG_SENSORS_ADT7462=m +CONFIG_SENSORS_ADT7470=m +CONFIG_SENSORS_ADT7475=m +CONFIG_SENSORS_AHT10=m +CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m +CONFIG_SENSORS_AS370=m +CONFIG_SENSORS_ASC7621=m +CONFIG_SENSORS_AXI_FAN_CONTROL=m +CONFIG_SENSORS_ATXP1=m +CONFIG_SENSORS_CORSAIR_CPRO=m +CONFIG_SENSORS_CORSAIR_PSU=m +CONFIG_SENSORS_DRIVETEMP=m +CONFIG_SENSORS_DS620=m +CONFIG_SENSORS_DS1621=m +CONFIG_SENSORS_DA9052_ADC=m +CONFIG_SENSORS_DA9055=m +CONFIG_SENSORS_I5K_AMB=m +CONFIG_SENSORS_F71805F=m +CONFIG_SENSORS_F71882FG=m +CONFIG_SENSORS_F75375S=m +CONFIG_SENSORS_GSC=m +CONFIG_SENSORS_MC13783_ADC=m +CONFIG_SENSORS_FTSTEUTATES=m +CONFIG_SENSORS_GL518SM=m +CONFIG_SENSORS_GL520SM=m +CONFIG_SENSORS_G760A=m +CONFIG_SENSORS_G762=m +CONFIG_SENSORS_GPIO_FAN=m +CONFIG_SENSORS_HIH6130=m +CONFIG_SENSORS_IBMAEM=m +CONFIG_SENSORS_IBMPEX=m +CONFIG_SENSORS_IIO_HWMON=m +CONFIG_SENSORS_IT87=m +CONFIG_SENSORS_JC42=m +CONFIG_SENSORS_POWR1220=m +CONFIG_SENSORS_LINEAGE=m +CONFIG_SENSORS_LOCHNAGAR=m +CONFIG_SENSORS_LTC2945=m +CONFIG_SENSORS_LTC2947_I2C=m +CONFIG_SENSORS_LTC2947_SPI=m +CONFIG_SENSORS_LTC2990=m +CONFIG_SENSORS_LTC2992=m +CONFIG_SENSORS_LTC4151=m +CONFIG_SENSORS_LTC4215=m +CONFIG_SENSORS_LTC4222=m +CONFIG_SENSORS_LTC4245=m +CONFIG_SENSORS_LTC4260=m +CONFIG_SENSORS_LTC4261=m +CONFIG_SENSORS_MAX1111=m +CONFIG_SENSORS_MAX127=m +CONFIG_SENSORS_MAX16065=m +CONFIG_SENSORS_MAX1619=m +CONFIG_SENSORS_MAX1668=m +CONFIG_SENSORS_MAX197=m +CONFIG_SENSORS_MAX31722=m +CONFIG_SENSORS_MAX31730=m +CONFIG_SENSORS_MAX6620=m +CONFIG_SENSORS_MAX6621=m +CONFIG_SENSORS_MAX6639=m +CONFIG_SENSORS_MAX6650=m +CONFIG_SENSORS_MAX6697=m +CONFIG_SENSORS_MAX31790=m +CONFIG_SENSORS_MCP3021=m +CONFIG_SENSORS_TC654=m +CONFIG_SENSORS_TPS23861=m +CONFIG_SENSORS_MENF21BMC_HWMON=m +CONFIG_SENSORS_MR75203=m +CONFIG_SENSORS_ADCXX=m +CONFIG_SENSORS_LM63=m +CONFIG_SENSORS_LM70=m +CONFIG_SENSORS_LM73=m +CONFIG_SENSORS_LM75=m +CONFIG_SENSORS_LM77=m +CONFIG_SENSORS_LM78=m +CONFIG_SENSORS_LM80=m +CONFIG_SENSORS_LM83=m +CONFIG_SENSORS_LM85=m +CONFIG_SENSORS_LM87=m +CONFIG_SENSORS_LM90=m +CONFIG_SENSORS_LM92=m +CONFIG_SENSORS_LM93=m +CONFIG_SENSORS_LM95234=m +CONFIG_SENSORS_LM95241=m +CONFIG_SENSORS_LM95245=m +CONFIG_SENSORS_PC87360=m +CONFIG_SENSORS_PC87427=m +CONFIG_SENSORS_NTC_THERMISTOR=m +CONFIG_SENSORS_NCT6683=m +CONFIG_SENSORS_NCT6775_I2C=m +CONFIG_SENSORS_NCT7802=m +CONFIG_SENSORS_NCT7904=m +CONFIG_SENSORS_NPCM7XX=m +CONFIG_SENSORS_NZXT_KRAKEN2=m +CONFIG_SENSORS_NZXT_SMART2=m +CONFIG_SENSORS_PCF8591=m +CONFIG_SENSORS_PECI_CPUTEMP=m +CONFIG_SENSORS_PECI_DIMMTEMP=m +CONFIG_PMBUS=m +CONFIG_SENSORS_ADM1266=m +CONFIG_SENSORS_ADM1275=m +CONFIG_SENSORS_BEL_PFE=m +CONFIG_SENSORS_BPA_RS600=m +CONFIG_SENSORS_DELTA_AHE50DC_FAN=m +CONFIG_SENSORS_FSP_3Y=m +CONFIG_SENSORS_IBM_CFFPS=m +CONFIG_SENSORS_DPS920AB=m +CONFIG_SENSORS_INSPUR_IPSPS=m +CONFIG_SENSORS_IR35221=m +CONFIG_SENSORS_IR36021=m +CONFIG_SENSORS_IR38064=m +CONFIG_SENSORS_IR38064_REGULATOR=y +CONFIG_SENSORS_IRPS5401=m +CONFIG_SENSORS_ISL68137=m +CONFIG_SENSORS_LM25066=m +CONFIG_SENSORS_LM25066_REGULATOR=y +CONFIG_SENSORS_LTC2978=m +CONFIG_SENSORS_LTC2978_REGULATOR=y +CONFIG_SENSORS_LTC3815=m +CONFIG_SENSORS_MAX15301=m +CONFIG_SENSORS_MAX16064=m +CONFIG_SENSORS_MAX16601=m +CONFIG_SENSORS_MAX20730=m +CONFIG_SENSORS_MAX20751=m +CONFIG_SENSORS_MAX31785=m +CONFIG_SENSORS_MAX34440=m +CONFIG_SENSORS_MAX8688=m +CONFIG_SENSORS_MP2888=m +CONFIG_SENSORS_MP2975=m +CONFIG_SENSORS_MP5023=m +CONFIG_SENSORS_PIM4328=m +CONFIG_SENSORS_PLI1209BC=m +CONFIG_SENSORS_PLI1209BC_REGULATOR=y +CONFIG_SENSORS_PM6764TR=m +CONFIG_SENSORS_PXE1610=m +CONFIG_SENSORS_Q54SJ108A2=m +CONFIG_SENSORS_STPDDC60=m +CONFIG_SENSORS_TPS40422=m +CONFIG_SENSORS_TPS53679=m +CONFIG_SENSORS_UCD9000=m +CONFIG_SENSORS_UCD9200=m +CONFIG_SENSORS_XDPE152=m +CONFIG_SENSORS_XDPE122=m +CONFIG_SENSORS_XDPE122_REGULATOR=y +CONFIG_SENSORS_ZL6100=m +CONFIG_SENSORS_PWM_FAN=m +CONFIG_SENSORS_SBTSI=m +CONFIG_SENSORS_SBRMI=m +CONFIG_SENSORS_SHT15=m +CONFIG_SENSORS_SHT21=m +CONFIG_SENSORS_SHT3x=m +CONFIG_SENSORS_SHT4x=m +CONFIG_SENSORS_SHTC1=m +CONFIG_SENSORS_SIS5595=m +CONFIG_SENSORS_DME1737=m +CONFIG_SENSORS_EMC1403=m +CONFIG_SENSORS_EMC2103=m +CONFIG_SENSORS_EMC6W201=m +CONFIG_SENSORS_SMSC47M1=m +CONFIG_SENSORS_SMSC47M192=m +CONFIG_SENSORS_SMSC47B397=m +CONFIG_SENSORS_SCH5627=m +CONFIG_SENSORS_SCH5636=m +CONFIG_SENSORS_STTS751=m +CONFIG_SENSORS_SMM665=m +CONFIG_SENSORS_ADC128D818=m +CONFIG_SENSORS_ADS7828=m +CONFIG_SENSORS_ADS7871=m +CONFIG_SENSORS_AMC6821=m +CONFIG_SENSORS_INA209=m +CONFIG_SENSORS_INA2XX=m +CONFIG_SENSORS_INA238=m +CONFIG_SENSORS_INA3221=m +CONFIG_SENSORS_TC74=m +CONFIG_SENSORS_THMC50=m +CONFIG_SENSORS_TMP102=m +CONFIG_SENSORS_TMP103=m +CONFIG_SENSORS_TMP108=m +CONFIG_SENSORS_TMP401=m +CONFIG_SENSORS_TMP421=m +CONFIG_SENSORS_TMP464=m +CONFIG_SENSORS_TMP513=m +CONFIG_SENSORS_VIA686A=m +CONFIG_SENSORS_VT1211=m +CONFIG_SENSORS_VT8231=m +CONFIG_SENSORS_W83773G=m +CONFIG_SENSORS_W83781D=m +CONFIG_SENSORS_W83791D=m +CONFIG_SENSORS_W83792D=m +CONFIG_SENSORS_W83793=m +CONFIG_SENSORS_W83795=m +CONFIG_SENSORS_W83L785TS=m +CONFIG_SENSORS_W83L786NG=m +CONFIG_SENSORS_W83627HF=m +CONFIG_SENSORS_W83627EHF=m +CONFIG_SENSORS_WM831X=m +CONFIG_SENSORS_WM8350=m +CONFIG_THERMAL_NETLINK=y +CONFIG_THERMAL_STATISTICS=y +CONFIG_THERMAL_WRITABLE_TRIPS=y +CONFIG_THERMAL_GOV_FAIR_SHARE=y +CONFIG_THERMAL_GOV_BANG_BANG=y +CONFIG_THERMAL_GOV_USER_SPACE=y +CONFIG_CPU_THERMAL=y +CONFIG_CPU_IDLE_THERMAL=y +CONFIG_DEVFREQ_THERMAL=y +CONFIG_THERMAL_EMULATION=y +CONFIG_THERMAL_MMIO=m +CONFIG_MAX77620_THERMAL=m +CONFIG_DA9062_THERMAL=m +CONFIG_GENERIC_ADC_THERMAL=m +CONFIG_WATCHDOG=y +CONFIG_WATCHDOG_CORE=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV_PANIC=m +CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP=y +CONFIG_SOFT_WATCHDOG=m +CONFIG_SOFT_WATCHDOG_PRETIMEOUT=y +CONFIG_BD957XMUF_WATCHDOG=m +CONFIG_DA9052_WATCHDOG=m +CONFIG_DA9055_WATCHDOG=m +CONFIG_DA9063_WATCHDOG=m +CONFIG_DA9062_WATCHDOG=m +CONFIG_GPIO_WATCHDOG=m +CONFIG_MENF21BMC_WATCHDOG=m +CONFIG_MENZ069_WATCHDOG=m +CONFIG_WM831X_WATCHDOG=m +CONFIG_WM8350_WATCHDOG=m +CONFIG_XILINX_WATCHDOG=m +CONFIG_ZIIRAVE_WATCHDOG=m +CONFIG_RAVE_SP_WATCHDOG=m +CONFIG_CADENCE_WATCHDOG=m +CONFIG_DW_WATCHDOG=m +CONFIG_RN5T618_WATCHDOG=m +CONFIG_TWL4030_WATCHDOG=m +CONFIG_MAX63XX_WATCHDOG=m +CONFIG_MAX77620_WATCHDOG=m +CONFIG_RETU_WATCHDOG=m +CONFIG_STPMIC1_WATCHDOG=m +CONFIG_ALIM7101_WDT=m +CONFIG_I6300ESB_WDT=m +CONFIG_KEMPLD_WDT=m +CONFIG_MEN_A21_WDT=m +CONFIG_PCIPCWATCHDOG=m +CONFIG_WDTPCI=m +CONFIG_USBPCWATCHDOG=m +CONFIG_SSB_SDIOHOST=y +CONFIG_SSB_DRIVER_GPIO=y +CONFIG_BCMA_HOST_SOC=y +CONFIG_BCMA_DRIVER_GMAC_CMN=y +CONFIG_BCMA_DRIVER_GPIO=y +CONFIG_MFD_ACT8945A=m +CONFIG_MFD_AS3711=y +CONFIG_MFD_AS3722=y +CONFIG_PMIC_ADP5520=y +CONFIG_MFD_AAT2870_CORE=y +CONFIG_MFD_ATMEL_FLEXCOM=m +CONFIG_MFD_ATMEL_HLCDC=m +CONFIG_MFD_BCM590XX=m +CONFIG_MFD_BD9571MWV=m +CONFIG_MFD_AXP20X_I2C=m +CONFIG_MFD_MADERA=m +CONFIG_MFD_MADERA_I2C=m +CONFIG_MFD_MADERA_SPI=m +CONFIG_MFD_CS47L15=y +CONFIG_MFD_CS47L35=y +CONFIG_MFD_CS47L85=y +CONFIG_MFD_CS47L90=y +CONFIG_MFD_CS47L92=y +CONFIG_PMIC_DA903X=y +CONFIG_MFD_DA9052_SPI=y +CONFIG_MFD_DA9052_I2C=y +CONFIG_MFD_DA9055=y +CONFIG_MFD_DA9062=m +CONFIG_MFD_DA9063=y +CONFIG_MFD_DA9150=m +CONFIG_MFD_DLN2=m +CONFIG_MFD_GATEWORKS_GSC=m +CONFIG_MFD_MC13XXX_SPI=m +CONFIG_MFD_MC13XXX_I2C=m +CONFIG_MFD_MP2629=m +CONFIG_MFD_HI6421_PMIC=m +CONFIG_MFD_HI6421_SPMI=m +CONFIG_LPC_ICH=m +CONFIG_MFD_IQS62X=m +CONFIG_MFD_JANZ_CMODIO=m +CONFIG_MFD_KEMPLD=m +CONFIG_MFD_88PM800=m +CONFIG_MFD_88PM805=m +CONFIG_MFD_88PM860X=y +CONFIG_MFD_MAX14577=y +CONFIG_MFD_MAX77620=y +CONFIG_MFD_MAX77650=m +CONFIG_MFD_MAX77686=y +CONFIG_MFD_MAX77693=y +CONFIG_MFD_MAX77714=m +CONFIG_MFD_MAX77843=y +CONFIG_MFD_MAX8907=m +CONFIG_MFD_MAX8925=y +CONFIG_MFD_MAX8997=y +CONFIG_MFD_MAX8998=y +CONFIG_MFD_MT6360=m +CONFIG_MFD_MT6397=m +CONFIG_MFD_MENF21BMC=m +CONFIG_EZX_PCAP=y +CONFIG_MFD_CPCAP=m +CONFIG_MFD_VIPERBOARD=m +CONFIG_MFD_NTXEC=m +CONFIG_MFD_RETU=m +CONFIG_MFD_PCF50633=m +CONFIG_PCF50633_ADC=m +CONFIG_PCF50633_GPIO=m +CONFIG_MFD_RT4831=m +CONFIG_MFD_RT5033=m +CONFIG_MFD_RC5T583=y +CONFIG_MFD_RK808=m +CONFIG_MFD_RN5T618=m +CONFIG_MFD_SEC_CORE=y +CONFIG_MFD_SI476X_CORE=m +CONFIG_MFD_SM501=m +CONFIG_MFD_SM501_GPIO=y +CONFIG_MFD_SKY81452=m +CONFIG_MFD_STMPE=y +CONFIG_STMPE_SPI=y +CONFIG_MFD_TI_AM335X_TSCADC=m +CONFIG_MFD_LP3943=m +CONFIG_MFD_LP8788=y +CONFIG_MFD_TI_LMU=m +CONFIG_MFD_PALMAS=y +CONFIG_TPS6105X=m +CONFIG_TPS65010=m +CONFIG_TPS6507X=m +CONFIG_MFD_TPS65086=m +CONFIG_MFD_TPS65090=y +CONFIG_MFD_TI_LP873X=m +CONFIG_MFD_TI_LP87565=m +CONFIG_MFD_TPS65218=m +CONFIG_MFD_TPS6586X=y +CONFIG_MFD_TPS65910=y +CONFIG_MFD_TPS65912_I2C=y +CONFIG_MFD_TPS65912_SPI=y +CONFIG_TWL4030_CORE=y +CONFIG_TWL6040_CORE=y +CONFIG_MFD_LM3533=m +CONFIG_MFD_TC3589X=y +CONFIG_MFD_VX855=m +CONFIG_MFD_LOCHNAGAR=y +CONFIG_MFD_ARIZONA_I2C=m +CONFIG_MFD_ARIZONA_SPI=m +CONFIG_MFD_CS47L24=y +CONFIG_MFD_WM5102=y +CONFIG_MFD_WM5110=y +CONFIG_MFD_WM8997=y +CONFIG_MFD_WM8998=y +CONFIG_MFD_WM8400=y +CONFIG_MFD_WM831X_I2C=y +CONFIG_MFD_WM831X_SPI=y +CONFIG_MFD_WM8350_I2C=y +CONFIG_MFD_WM8994=m +CONFIG_MFD_ROHM_BD718XX=m +CONFIG_MFD_ROHM_BD71828=m +CONFIG_MFD_ROHM_BD957XMUF=m +CONFIG_MFD_STPMIC1=m +CONFIG_MFD_WCD934X=m +CONFIG_MFD_ATC260X_I2C=m +CONFIG_MFD_QCOM_PM8008=m +CONFIG_RAVE_SP_CORE=m +CONFIG_MFD_RSMU_I2C=m +CONFIG_MFD_RSMU_SPI=m +CONFIG_REGULATOR_VIRTUAL_CONSUMER=m +CONFIG_REGULATOR_USERSPACE_CONSUMER=m +CONFIG_REGULATOR_88PG86X=m +CONFIG_REGULATOR_88PM800=m +CONFIG_REGULATOR_88PM8607=m +CONFIG_REGULATOR_ACT8865=m +CONFIG_REGULATOR_ACT8945A=m +CONFIG_REGULATOR_AD5398=m +CONFIG_REGULATOR_AAT2870=m +CONFIG_REGULATOR_ARIZONA_LDO1=m +CONFIG_REGULATOR_ARIZONA_MICSUPP=m +CONFIG_REGULATOR_AS3711=m +CONFIG_REGULATOR_AS3722=m +CONFIG_REGULATOR_ATC260X=m +CONFIG_REGULATOR_AXP20X=m +CONFIG_REGULATOR_BCM590XX=m +CONFIG_REGULATOR_BD71815=m +CONFIG_REGULATOR_BD71828=m +CONFIG_REGULATOR_BD718XX=m +CONFIG_REGULATOR_BD9571MWV=m +CONFIG_REGULATOR_BD957XMUF=m +CONFIG_REGULATOR_CPCAP=m +CONFIG_REGULATOR_DA903X=m +CONFIG_REGULATOR_DA9052=m +CONFIG_REGULATOR_DA9055=m +CONFIG_REGULATOR_DA9062=m +CONFIG_REGULATOR_DA9063=m +CONFIG_REGULATOR_DA9121=m +CONFIG_REGULATOR_DA9210=m +CONFIG_REGULATOR_DA9211=m +CONFIG_REGULATOR_FAN53555=m +CONFIG_REGULATOR_FAN53880=m +CONFIG_REGULATOR_GPIO=m +CONFIG_REGULATOR_HI6421=m +CONFIG_REGULATOR_HI6421V530=m +CONFIG_REGULATOR_HI6421V600=m +CONFIG_REGULATOR_ISL9305=m +CONFIG_REGULATOR_ISL6271A=m +CONFIG_REGULATOR_LM363X=m +CONFIG_REGULATOR_LOCHNAGAR=m +CONFIG_REGULATOR_LP3971=m +CONFIG_REGULATOR_LP3972=m +CONFIG_REGULATOR_LP872X=m +CONFIG_REGULATOR_LP873X=m +CONFIG_REGULATOR_LP8755=m +CONFIG_REGULATOR_LP87565=m +CONFIG_REGULATOR_LP8788=m +CONFIG_REGULATOR_LTC3589=m +CONFIG_REGULATOR_LTC3676=m +CONFIG_REGULATOR_MAX14577=m +CONFIG_REGULATOR_MAX1586=m +CONFIG_REGULATOR_MAX77620=m +CONFIG_REGULATOR_MAX77650=m +CONFIG_REGULATOR_MAX8649=m +CONFIG_REGULATOR_MAX8660=m +CONFIG_REGULATOR_MAX8893=m +CONFIG_REGULATOR_MAX8907=m +CONFIG_REGULATOR_MAX8925=m +CONFIG_REGULATOR_MAX8952=m +CONFIG_REGULATOR_MAX8973=m +CONFIG_REGULATOR_MAX8997=m +CONFIG_REGULATOR_MAX8998=m +CONFIG_REGULATOR_MAX20086=m +CONFIG_REGULATOR_MAX77686=m +CONFIG_REGULATOR_MAX77693=m +CONFIG_REGULATOR_MAX77802=m +CONFIG_REGULATOR_MAX77826=m +CONFIG_REGULATOR_MC13783=m +CONFIG_REGULATOR_MC13892=m +CONFIG_REGULATOR_MCP16502=m +CONFIG_REGULATOR_MP5416=m +CONFIG_REGULATOR_MP8859=m +CONFIG_REGULATOR_MP886X=m +CONFIG_REGULATOR_MPQ7920=m +CONFIG_REGULATOR_MT6311=m +CONFIG_REGULATOR_MT6315=m +CONFIG_REGULATOR_MT6323=m +CONFIG_REGULATOR_MT6358=m +CONFIG_REGULATOR_MT6359=m +CONFIG_REGULATOR_MT6360=m +CONFIG_REGULATOR_MT6397=m +CONFIG_REGULATOR_PALMAS=m +CONFIG_REGULATOR_PCA9450=m +CONFIG_REGULATOR_PCAP=m +CONFIG_REGULATOR_PCF50633=m +CONFIG_REGULATOR_PF8X00=m +CONFIG_REGULATOR_PFUZE100=m +CONFIG_REGULATOR_PV88060=m +CONFIG_REGULATOR_PV88080=m +CONFIG_REGULATOR_PV88090=m +CONFIG_REGULATOR_PWM=m +CONFIG_REGULATOR_QCOM_SPMI=m +CONFIG_REGULATOR_QCOM_USB_VBUS=m +CONFIG_REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY=m +CONFIG_REGULATOR_RC5T583=m +CONFIG_REGULATOR_RK808=m +CONFIG_REGULATOR_RN5T618=m +CONFIG_REGULATOR_RT4801=m +CONFIG_REGULATOR_RT4831=m +CONFIG_REGULATOR_RT5033=m +CONFIG_REGULATOR_RT5190A=m +CONFIG_REGULATOR_RT5759=m +CONFIG_REGULATOR_RT6160=m +CONFIG_REGULATOR_RT6245=m +CONFIG_REGULATOR_RTQ2134=m +CONFIG_REGULATOR_RTMV20=m +CONFIG_REGULATOR_RTQ6752=m +CONFIG_REGULATOR_S2MPA01=m +CONFIG_REGULATOR_S2MPS11=m +CONFIG_REGULATOR_S5M8767=m +CONFIG_REGULATOR_SKY81452=m +CONFIG_REGULATOR_SLG51000=m +CONFIG_REGULATOR_STPMIC1=m +CONFIG_REGULATOR_SY8106A=m +CONFIG_REGULATOR_SY8824X=m +CONFIG_REGULATOR_SY8827N=m +CONFIG_REGULATOR_TPS51632=m +CONFIG_REGULATOR_TPS62360=m +CONFIG_REGULATOR_TPS6286X=m +CONFIG_REGULATOR_TPS65023=m +CONFIG_REGULATOR_TPS6507X=m +CONFIG_REGULATOR_TPS65086=m +CONFIG_REGULATOR_TPS65090=m +CONFIG_REGULATOR_TPS65132=m +CONFIG_REGULATOR_TPS65218=m +CONFIG_REGULATOR_TPS6524X=m +CONFIG_REGULATOR_TPS6586X=m +CONFIG_REGULATOR_TPS65910=m +CONFIG_REGULATOR_TPS65912=m +CONFIG_REGULATOR_TWL4030=m +CONFIG_REGULATOR_VCTRL=m +CONFIG_REGULATOR_WM831X=m +CONFIG_REGULATOR_WM8350=m +CONFIG_REGULATOR_WM8400=m +CONFIG_REGULATOR_WM8994=m +CONFIG_REGULATOR_QCOM_LABIBB=m +CONFIG_RC_CORE=m +CONFIG_LIRC=y +CONFIG_RC_DECODERS=y +CONFIG_IR_IMON_DECODER=m +CONFIG_IR_JVC_DECODER=m +CONFIG_IR_MCE_KBD_DECODER=m +CONFIG_IR_NEC_DECODER=m +CONFIG_IR_RC5_DECODER=m +CONFIG_IR_RC6_DECODER=m +CONFIG_IR_RCMM_DECODER=m +CONFIG_IR_SANYO_DECODER=m +CONFIG_IR_SHARP_DECODER=m +CONFIG_IR_SONY_DECODER=m +CONFIG_IR_XMP_DECODER=m +CONFIG_RC_DEVICES=y +CONFIG_IR_GPIO_CIR=m +CONFIG_IR_GPIO_TX=m +CONFIG_IR_HIX5HD2=m +CONFIG_IR_IGORPLUGUSB=m +CONFIG_IR_IGUANA=m +CONFIG_IR_IMON=m +CONFIG_IR_IMON_RAW=m +CONFIG_IR_MCEUSB=m +CONFIG_IR_PWM_TX=m +CONFIG_IR_REDRAT3=m +CONFIG_IR_SERIAL=m +CONFIG_IR_SERIAL_TRANSMITTER=y +CONFIG_IR_SPI=m +CONFIG_IR_STREAMZAP=m +CONFIG_IR_TOY=m +CONFIG_IR_TTUSBIR=m +CONFIG_RC_ATI_REMOTE=m +CONFIG_RC_LOOPBACK=m +CONFIG_RC_XBOX_DVD=m +CONFIG_MEDIA_CEC_RC=y +CONFIG_MEDIA_CEC_SUPPORT=y +CONFIG_CEC_CH7322=m +CONFIG_USB_PULSE8_CEC=m +CONFIG_USB_RAINSHADOW_CEC=m +CONFIG_MEDIA_SUPPORT=m +CONFIG_MEDIA_SUPPORT_FILTER=y +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_ANALOG_TV_SUPPORT=y +CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y +CONFIG_MEDIA_RADIO_SUPPORT=y +CONFIG_MEDIA_SDR_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_MEDIA_TEST_SUPPORT=y +CONFIG_V4L2_FLASH_LED_CLASS=m +CONFIG_DVB_MAX_ADAPTERS=8 +CONFIG_MEDIA_USB_SUPPORT=y +CONFIG_USB_GSPCA=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_GL860=m +CONFIG_USB_M5602=m +CONFIG_USB_STV06XX=m +CONFIG_USB_PWC=m +CONFIG_USB_S2255=m +CONFIG_VIDEO_USBTV=m +CONFIG_USB_VIDEO_CLASS=m +CONFIG_VIDEO_GO7007=m +CONFIG_VIDEO_GO7007_USB=m +CONFIG_VIDEO_GO7007_USB_S2250_BOARD=m +CONFIG_VIDEO_HDPVR=m +CONFIG_VIDEO_PVRUSB2=m +CONFIG_VIDEO_STK1160_COMMON=m +CONFIG_VIDEO_AU0828=m +CONFIG_VIDEO_AU0828_RC=y +CONFIG_VIDEO_CX231XX=m +CONFIG_VIDEO_CX231XX_ALSA=m +CONFIG_VIDEO_CX231XX_DVB=m +CONFIG_DVB_AS102=m +CONFIG_DVB_B2C2_FLEXCOP_USB=m +CONFIG_DVB_USB_V2=m +CONFIG_DVB_USB_AF9015=m +CONFIG_DVB_USB_AF9035=m +CONFIG_DVB_USB_ANYSEE=m +CONFIG_DVB_USB_AU6610=m +CONFIG_DVB_USB_AZ6007=m +CONFIG_DVB_USB_CE6230=m +CONFIG_DVB_USB_DVBSKY=m +CONFIG_DVB_USB_EC168=m +CONFIG_DVB_USB_GL861=m +CONFIG_DVB_USB_LME2510=m +CONFIG_DVB_USB_MXL111SF=m +CONFIG_DVB_USB_RTL28XXU=m +CONFIG_DVB_USB_ZD1301=m +CONFIG_DVB_USB=m +CONFIG_DVB_USB_A800=m +CONFIG_DVB_USB_AF9005=m +CONFIG_DVB_USB_AF9005_REMOTE=m +CONFIG_DVB_USB_AZ6027=m +CONFIG_DVB_USB_CINERGY_T2=m +CONFIG_DVB_USB_CXUSB=m +CONFIG_DVB_USB_CXUSB_ANALOG=y +CONFIG_DVB_USB_DIB0700=m +CONFIG_DVB_USB_DIBUSB_MB=m +CONFIG_DVB_USB_DIBUSB_MC=m +CONFIG_DVB_USB_DIGITV=m +CONFIG_DVB_USB_DTT200U=m +CONFIG_DVB_USB_DTV5100=m +CONFIG_DVB_USB_DW2102=m +CONFIG_DVB_USB_GP8PSK=m +CONFIG_DVB_USB_M920X=m +CONFIG_DVB_USB_NOVA_T_USB2=m +CONFIG_DVB_USB_OPERA1=m +CONFIG_DVB_USB_PCTV452E=m +CONFIG_DVB_USB_TECHNISAT_USB2=m +CONFIG_DVB_USB_TTUSB2=m +CONFIG_DVB_USB_UMT_010=m +CONFIG_DVB_USB_VP702X=m +CONFIG_DVB_USB_VP7045=m +CONFIG_SMS_USB_DRV=m +CONFIG_DVB_TTUSB_BUDGET=m +CONFIG_DVB_TTUSB_DEC=m +CONFIG_VIDEO_EM28XX=m +CONFIG_VIDEO_EM28XX_V4L2=m +CONFIG_VIDEO_EM28XX_ALSA=m +CONFIG_VIDEO_EM28XX_DVB=m +CONFIG_USB_AIRSPY=m +CONFIG_USB_HACKRF=m +CONFIG_USB_MSI2500=m +CONFIG_MEDIA_PCI_SUPPORT=y +CONFIG_VIDEO_SOLO6X10=m +CONFIG_VIDEO_TW5864=m +CONFIG_VIDEO_TW68=m +CONFIG_VIDEO_TW686X=m +CONFIG_VIDEO_DT3155=m +CONFIG_VIDEO_IVTV=m +CONFIG_VIDEO_IVTV_ALSA=m +CONFIG_VIDEO_FB_IVTV=m +CONFIG_VIDEO_BT848=m +CONFIG_DVB_BT8XX=m +CONFIG_VIDEO_CX18=m +CONFIG_VIDEO_CX18_ALSA=m +CONFIG_VIDEO_CX23885=m +CONFIG_MEDIA_ALTERA_CI=m +CONFIG_VIDEO_CX25821=m +CONFIG_VIDEO_CX25821_ALSA=m +CONFIG_VIDEO_CX88=m +CONFIG_VIDEO_CX88_ALSA=m +CONFIG_VIDEO_CX88_BLACKBIRD=m +CONFIG_VIDEO_CX88_DVB=m +CONFIG_VIDEO_SAA7134=m +CONFIG_VIDEO_SAA7134_ALSA=m +CONFIG_VIDEO_SAA7134_DVB=m +CONFIG_VIDEO_SAA7134_GO7007=m +CONFIG_VIDEO_SAA7164=m +CONFIG_DVB_B2C2_FLEXCOP_PCI=m +CONFIG_DVB_DDBRIDGE=m +CONFIG_DVB_DM1105=m +CONFIG_MANTIS_CORE=m +CONFIG_DVB_MANTIS=m +CONFIG_DVB_HOPPER=m +CONFIG_DVB_NETUP_UNIDVB=m +CONFIG_DVB_NGENE=m +CONFIG_DVB_PLUTO2=m +CONFIG_DVB_PT1=m +CONFIG_DVB_PT3=m +CONFIG_DVB_SMIPCIE=m +CONFIG_RADIO_MAXIRADIO=m +CONFIG_RADIO_SAA7706H=m +CONFIG_RADIO_SHARK=m +CONFIG_RADIO_SHARK2=m +CONFIG_RADIO_SI4713=m +CONFIG_RADIO_SI476X=m +CONFIG_RADIO_TEA5764=m +CONFIG_RADIO_TEF6862=m +CONFIG_RADIO_WL1273=m +CONFIG_USB_DSBR=m +CONFIG_USB_KEENE=m +CONFIG_USB_MA901=m +CONFIG_USB_MR800=m +CONFIG_USB_RAREMONO=m +CONFIG_RADIO_SI470X=m +CONFIG_USB_SI470X=m +CONFIG_I2C_SI470X=m +CONFIG_USB_SI4713=m +CONFIG_PLATFORM_SI4713=m +CONFIG_RADIO_WL128X=m +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_SDR_PLATFORM_DRIVERS=y +CONFIG_DVB_PLATFORM_DRIVERS=y +CONFIG_V4L_MEM2MEM_DRIVERS=y +CONFIG_VIDEO_MEM2MEM_DEINTERLACE=m +CONFIG_VIDEO_MUX=m +CONFIG_VIDEO_CADENCE_CSI2RX=m +CONFIG_VIDEO_CADENCE_CSI2TX=m +CONFIG_VIDEO_CAFE_CCIC=m +CONFIG_VIDEO_XILINX=m +CONFIG_VIDEO_XILINX_CSI2RXSS=m +CONFIG_VIDEO_XILINX_TPG=m +CONFIG_SMS_SDIO_DRV=m +CONFIG_V4L_TEST_DRIVERS=y +CONFIG_VIDEO_VIM2M=m +CONFIG_VIDEO_VICODEC=m +CONFIG_VIDEO_VIMC=m +CONFIG_VIDEO_VIVID=m +CONFIG_VIDEO_VIVID_CEC=y +CONFIG_DVB_FIREDTV=m +CONFIG_SMS_SIANO_DEBUGFS=y +CONFIG_VIDEO_HI556=m +CONFIG_VIDEO_HI846=m +CONFIG_VIDEO_HI847=m +CONFIG_VIDEO_IMX208=m +CONFIG_VIDEO_IMX214=m +CONFIG_VIDEO_IMX219=m +CONFIG_VIDEO_IMX258=m +CONFIG_VIDEO_IMX274=m +CONFIG_VIDEO_IMX290=m +CONFIG_VIDEO_IMX319=m +CONFIG_VIDEO_IMX334=m +CONFIG_VIDEO_IMX335=m +CONFIG_VIDEO_IMX355=m +CONFIG_VIDEO_IMX412=m +CONFIG_VIDEO_MT9M001=m +CONFIG_VIDEO_MT9M032=m +CONFIG_VIDEO_MT9M111=m +CONFIG_VIDEO_MT9P031=m +CONFIG_VIDEO_MT9T001=m +CONFIG_VIDEO_MT9T112=m +CONFIG_VIDEO_MT9V032=m +CONFIG_VIDEO_MT9V111=m +CONFIG_VIDEO_NOON010PC30=m +CONFIG_VIDEO_OG01A1B=m +CONFIG_VIDEO_OV02A10=m +CONFIG_VIDEO_OV08D10=m +CONFIG_VIDEO_OV13858=m +CONFIG_VIDEO_OV13B10=m +CONFIG_VIDEO_OV2659=m +CONFIG_VIDEO_OV2680=m +CONFIG_VIDEO_OV2685=m +CONFIG_VIDEO_OV5640=m +CONFIG_VIDEO_OV5645=m +CONFIG_VIDEO_OV5647=m +CONFIG_VIDEO_OV5670=m +CONFIG_VIDEO_OV5675=m +CONFIG_VIDEO_OV5693=m +CONFIG_VIDEO_OV5695=m +CONFIG_VIDEO_OV6650=m +CONFIG_VIDEO_OV7251=m +CONFIG_VIDEO_OV772X=m +CONFIG_VIDEO_OV7740=m +CONFIG_VIDEO_OV8856=m +CONFIG_VIDEO_OV9282=m +CONFIG_VIDEO_OV9640=m +CONFIG_VIDEO_OV9650=m +CONFIG_VIDEO_RDACM20=m +CONFIG_VIDEO_RDACM21=m +CONFIG_VIDEO_RJ54N1=m +CONFIG_VIDEO_S5C73M3=m +CONFIG_VIDEO_S5K5BAF=m +CONFIG_VIDEO_S5K6A3=m +CONFIG_VIDEO_S5K6AA=m +CONFIG_VIDEO_SR030PC30=m +CONFIG_VIDEO_VS6624=m +CONFIG_VIDEO_CCS=m +CONFIG_VIDEO_ET8EK8=m +CONFIG_VIDEO_M5MOLS=m +CONFIG_VIDEO_AD5820=m +CONFIG_VIDEO_AK7375=m +CONFIG_VIDEO_DW9714=m +CONFIG_VIDEO_DW9768=m +CONFIG_VIDEO_DW9807_VCM=m +CONFIG_VIDEO_ADP1653=m +CONFIG_VIDEO_LM3560=m +CONFIG_VIDEO_LM3646=m +CONFIG_VIDEO_TDA1997X=m +CONFIG_VIDEO_TDA9840=m +CONFIG_VIDEO_TEA6415C=m +CONFIG_VIDEO_TEA6420=m +CONFIG_VIDEO_TLV320AIC23B=m +CONFIG_VIDEO_ADV7180=m +CONFIG_VIDEO_ADV7183=m +CONFIG_VIDEO_ADV748X=m +CONFIG_VIDEO_ADV7604=m +CONFIG_VIDEO_ADV7604_CEC=y +CONFIG_VIDEO_ADV7842=m +CONFIG_VIDEO_ADV7842_CEC=y +CONFIG_VIDEO_BT819=m +CONFIG_VIDEO_BT856=m +CONFIG_VIDEO_BT866=m +CONFIG_VIDEO_ISL7998X=m +CONFIG_VIDEO_KS0127=m +CONFIG_VIDEO_MAX9286=m +CONFIG_VIDEO_ML86V7667=m +CONFIG_VIDEO_SAA7110=m +CONFIG_VIDEO_TC358743=m +CONFIG_VIDEO_TC358743_CEC=y +CONFIG_VIDEO_TVP514X=m +CONFIG_VIDEO_TVP7002=m +CONFIG_VIDEO_TW9910=m +CONFIG_VIDEO_VPX3220=m +CONFIG_VIDEO_AD9389B=m +CONFIG_VIDEO_ADV7170=m +CONFIG_VIDEO_ADV7175=m +CONFIG_VIDEO_ADV7343=m +CONFIG_VIDEO_ADV7393=m +CONFIG_VIDEO_AK881X=m +CONFIG_VIDEO_SAA7185=m +CONFIG_VIDEO_THS8200=m +CONFIG_SDR_MAX2175=m +CONFIG_VIDEO_I2C=m +CONFIG_VIDEO_ST_MIPID02=m +CONFIG_VIDEO_THS7303=m +CONFIG_CXD2880_SPI_DRV=m +CONFIG_VIDEO_GS1662=m +CONFIG_DVB_TDA8261=m +CONFIG_DVB_TUA6100=m +CONFIG_DVB_VES1X93=m +CONFIG_DVB_DIB9000=m +CONFIG_DVB_L64781=m +CONFIG_DVB_S5H1432=m +CONFIG_DVB_CXD2880=m +CONFIG_DVB_MN88443X=m +CONFIG_DVB_LGS8GL5=m +CONFIG_DVB_LNBH29=m +CONFIG_DVB_DUMMY_FE=m +CONFIG_DRM=m +CONFIG_DRM_LOAD_EDID_FIRMWARE=y +CONFIG_DRM_DP_AUX_CHARDEV=y +CONFIG_DRM_DP_CEC=y +CONFIG_DRM_I2C_NXP_TDA998X=m +CONFIG_DRM_I2C_NXP_TDA9950=m +CONFIG_DRM_KOMEDA=m +CONFIG_DRM_RADEON=m +CONFIG_DRM_AMDGPU=m +CONFIG_DRM_AMDGPU_SI=y +CONFIG_DRM_AMDGPU_CIK=y +CONFIG_DRM_AMDGPU_USERPTR=y +CONFIG_DRM_AMD_ACP=y +CONFIG_DRM_AMD_DC_HDCP=y +CONFIG_DRM_AMD_DC_SI=y +CONFIG_DRM_NOUVEAU=m +CONFIG_DRM_VGEM=m +CONFIG_DRM_VKMS=m +CONFIG_DRM_UDL=m +CONFIG_DRM_AST=m +CONFIG_DRM_MGAG200=m +CONFIG_DRM_QXL=m +CONFIG_DRM_VIRTIO_GPU=m +CONFIG_DRM_PANEL_ABT_Y030XX067A=m +CONFIG_DRM_PANEL_ARM_VERSATILE=m +CONFIG_DRM_PANEL_ASUS_Z00T_TM5P5_NT35596=m +CONFIG_DRM_PANEL_BOE_BF060Y8M_AJ0=m +CONFIG_DRM_PANEL_BOE_HIMAX8279D=m +CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m +CONFIG_DRM_PANEL_DSI_CM=m +CONFIG_DRM_PANEL_LVDS=m +CONFIG_DRM_PANEL_ELIDA_KD35T133=m +CONFIG_DRM_PANEL_FEIXIN_K101_IM2BA02=m +CONFIG_DRM_PANEL_FEIYANG_FY07024DI26A30D=m +CONFIG_DRM_PANEL_ILITEK_IL9322=m +CONFIG_DRM_PANEL_ILITEK_ILI9341=m +CONFIG_DRM_PANEL_ILITEK_ILI9881C=m +CONFIG_DRM_PANEL_INNOLUX_EJ030NA=m +CONFIG_DRM_PANEL_INNOLUX_P079ZCA=m +CONFIG_DRM_PANEL_JDI_LT070ME05000=m +CONFIG_DRM_PANEL_JDI_R63452=m +CONFIG_DRM_PANEL_KHADAS_TS050=m +CONFIG_DRM_PANEL_KINGDISPLAY_KD097D04=m +CONFIG_DRM_PANEL_LEADTEK_LTK050H3146W=m +CONFIG_DRM_PANEL_LEADTEK_LTK500HD1829=m +CONFIG_DRM_PANEL_SAMSUNG_LD9040=m +CONFIG_DRM_PANEL_LG_LB035Q02=m +CONFIG_DRM_PANEL_LG_LG4573=m +CONFIG_DRM_PANEL_NEC_NL8048HL11=m +CONFIG_DRM_PANEL_NEWVISION_NV3052C=m +CONFIG_DRM_PANEL_NOVATEK_NT35510=m +CONFIG_DRM_PANEL_NOVATEK_NT35560=m +CONFIG_DRM_PANEL_NOVATEK_NT35950=m +CONFIG_DRM_PANEL_NOVATEK_NT36672A=m +CONFIG_DRM_PANEL_NOVATEK_NT39016=m +CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m +CONFIG_DRM_PANEL_OLIMEX_LCD_OLINUXINO=m +CONFIG_DRM_PANEL_ORISETECH_OTM8009A=m +CONFIG_DRM_PANEL_OSD_OSD101T2587_53TS=m +CONFIG_DRM_PANEL_PANASONIC_VVX10F034N00=m +CONFIG_DRM_PANEL_RASPBERRYPI_TOUCHSCREEN=m +CONFIG_DRM_PANEL_RAYDIUM_RM67191=m +CONFIG_DRM_PANEL_RAYDIUM_RM68200=m +CONFIG_DRM_PANEL_RONBO_RB070D30=m +CONFIG_DRM_PANEL_SAMSUNG_DB7430=m +CONFIG_DRM_PANEL_SAMSUNG_S6D16D0=m +CONFIG_DRM_PANEL_SAMSUNG_S6D27A1=m +CONFIG_DRM_PANEL_SAMSUNG_S6E3HA2=m +CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03=m +CONFIG_DRM_PANEL_SAMSUNG_S6E63M0=m +CONFIG_DRM_PANEL_SAMSUNG_S6E63M0_DSI=m +CONFIG_DRM_PANEL_SAMSUNG_S6E88A0_AMS452EF01=m +CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=m +CONFIG_DRM_PANEL_SAMSUNG_SOFEF00=m +CONFIG_DRM_PANEL_SEIKO_43WVF1G=m +CONFIG_DRM_PANEL_SHARP_LQ101R1SX01=m +CONFIG_DRM_PANEL_SHARP_LS037V7DW01=m +CONFIG_DRM_PANEL_SHARP_LS043T1LE01=m +CONFIG_DRM_PANEL_SHARP_LS060T1SX01=m +CONFIG_DRM_PANEL_SITRONIX_ST7701=m +CONFIG_DRM_PANEL_SITRONIX_ST7703=m +CONFIG_DRM_PANEL_SITRONIX_ST7789V=m +CONFIG_DRM_PANEL_SONY_ACX565AKM=m +CONFIG_DRM_PANEL_SONY_TULIP_TRULY_NT35521=m +CONFIG_DRM_PANEL_TDO_TL070WSH30=m +CONFIG_DRM_PANEL_TPO_TD028TTEC1=m +CONFIG_DRM_PANEL_TPO_TD043MTEA1=m +CONFIG_DRM_PANEL_TPO_TPG110=m +CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA=m +CONFIG_DRM_PANEL_VISIONOX_RM69299=m +CONFIG_DRM_PANEL_WIDECHIPS_WS2401=m +CONFIG_DRM_PANEL_XINPENG_XPP055C272=m +CONFIG_DRM_CHIPONE_ICN6211=m +CONFIG_DRM_CHRONTEL_CH7033=m +CONFIG_DRM_DISPLAY_CONNECTOR=m +CONFIG_DRM_ITE_IT6505=m +CONFIG_DRM_LONTIUM_LT8912B=m +CONFIG_DRM_LONTIUM_LT9211=m +CONFIG_DRM_LONTIUM_LT9611=m +CONFIG_DRM_LONTIUM_LT9611UXC=m +CONFIG_DRM_ITE_IT66121=m +CONFIG_DRM_LVDS_CODEC=m +CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW=m +CONFIG_DRM_NWL_MIPI_DSI=m +CONFIG_DRM_NXP_PTN3460=m +CONFIG_DRM_PARADE_PS8622=m +CONFIG_DRM_PARADE_PS8640=m +CONFIG_DRM_SIL_SII8620=m +CONFIG_DRM_SII902X=m +CONFIG_DRM_SII9234=m +CONFIG_DRM_SIMPLE_BRIDGE=m +CONFIG_DRM_THINE_THC63LVD1024=m +CONFIG_DRM_TOSHIBA_TC358762=m +CONFIG_DRM_TOSHIBA_TC358764=m +CONFIG_DRM_TOSHIBA_TC358767=m +CONFIG_DRM_TOSHIBA_TC358768=m +CONFIG_DRM_TOSHIBA_TC358775=m +CONFIG_DRM_TI_TFP410=m +CONFIG_DRM_TI_SN65DSI83=m +CONFIG_DRM_TI_SN65DSI86=m +CONFIG_DRM_TI_TPD12S015=m +CONFIG_DRM_ANALOGIX_ANX6345=m +CONFIG_DRM_ANALOGIX_ANX78XX=m +CONFIG_DRM_ANALOGIX_ANX7625=m +CONFIG_DRM_I2C_ADV7511=m +CONFIG_DRM_I2C_ADV7511_AUDIO=y +CONFIG_DRM_CDNS_DSI=m +CONFIG_DRM_CDNS_MHDP8546=m +CONFIG_DRM_ARCPGU=m +CONFIG_DRM_BOCHS=m +CONFIG_DRM_CIRRUS_QEMU=m +CONFIG_DRM_GM12U320=m +CONFIG_DRM_PANEL_MIPI_DBI=m +CONFIG_DRM_SIMPLEDRM=m +CONFIG_TINYDRM_HX8357D=m +CONFIG_TINYDRM_ILI9163=m +CONFIG_TINYDRM_ILI9225=m +CONFIG_TINYDRM_ILI9341=m +CONFIG_TINYDRM_ILI9486=m +CONFIG_TINYDRM_MI0283QT=m +CONFIG_TINYDRM_REPAPER=m +CONFIG_TINYDRM_ST7586=m +CONFIG_TINYDRM_ST7735R=m +CONFIG_DRM_GUD=m +CONFIG_DRM_SSD130X=m +CONFIG_DRM_SSD130X_I2C=m +CONFIG_DRM_SSD130X_SPI=m +CONFIG_FB=y +CONFIG_FIRMWARE_EDID=y +CONFIG_FB_CIRRUS=m +CONFIG_FB_PM2=m +CONFIG_FB_PM2_FIFO_DISCONNECT=y +CONFIG_FB_CYBER2000=m +CONFIG_FB_ASILIANT=y +CONFIG_FB_IMSTT=y +CONFIG_FB_UVESA=m +CONFIG_FB_EFI=y +CONFIG_FB_OPENCORES=m +CONFIG_FB_S1D13XXX=m +CONFIG_FB_NVIDIA=m +CONFIG_FB_NVIDIA_I2C=y +CONFIG_FB_RIVA=m +CONFIG_FB_RIVA_I2C=y +CONFIG_FB_I740=m +CONFIG_FB_MATROX=m +CONFIG_FB_MATROX_MILLENIUM=y +CONFIG_FB_MATROX_MYSTIQUE=y +CONFIG_FB_MATROX_G=y +CONFIG_FB_MATROX_I2C=m +CONFIG_FB_MATROX_MAVEN=m +CONFIG_FB_RADEON=m +CONFIG_FB_ATY128=m +CONFIG_FB_ATY=m +CONFIG_FB_ATY_CT=y +CONFIG_FB_ATY_GX=y +CONFIG_FB_S3=m +CONFIG_FB_SAVAGE=m +CONFIG_FB_SAVAGE_I2C=y +CONFIG_FB_SIS=m +CONFIG_FB_SIS_300=y +CONFIG_FB_SIS_315=y +CONFIG_FB_NEOMAGIC=m +CONFIG_FB_KYRO=m +CONFIG_FB_3DFX=m +# CONFIG_FB_3DFX_I2C is not set +CONFIG_FB_VOODOO1=m +CONFIG_FB_VT8623=m +CONFIG_FB_TRIDENT=m +CONFIG_FB_ARK=m +CONFIG_FB_PM3=m +CONFIG_FB_CARMINE=m +CONFIG_FB_SM501=m +CONFIG_FB_SMSCUFX=m +CONFIG_FB_UDL=m +CONFIG_FB_GOLDFISH=m +CONFIG_FB_METRONOME=m +CONFIG_FB_MB862XX=m +CONFIG_FB_SIMPLE=m +CONFIG_FB_SSD1307=m +CONFIG_FB_SM712=m +CONFIG_LCD_CLASS_DEVICE=m +CONFIG_LCD_L4F00242T03=m +CONFIG_LCD_LMS283GF05=m +CONFIG_LCD_LTV350QV=m +CONFIG_LCD_ILI922X=m +CONFIG_LCD_TDO24M=m +CONFIG_LCD_VGG2432A4=m +CONFIG_LCD_PLATFORM=m +CONFIG_LCD_AMS369FG06=m +CONFIG_LCD_LMS501KF03=m +CONFIG_LCD_HX8357=m +CONFIG_LCD_OTM3225A=m +CONFIG_BACKLIGHT_KTD253=m +CONFIG_BACKLIGHT_LM3533=m +CONFIG_BACKLIGHT_PWM=m +CONFIG_BACKLIGHT_DA903X=m +CONFIG_BACKLIGHT_DA9052=m +CONFIG_BACKLIGHT_MAX8925=m +CONFIG_BACKLIGHT_QCOM_WLED=m +CONFIG_BACKLIGHT_RT4831=m +CONFIG_BACKLIGHT_WM831X=m +CONFIG_BACKLIGHT_ADP5520=m +CONFIG_BACKLIGHT_ADP8860=m +CONFIG_BACKLIGHT_ADP8870=m +CONFIG_BACKLIGHT_88PM860X=m +CONFIG_BACKLIGHT_PCF50633=m +CONFIG_BACKLIGHT_AAT2870=m +CONFIG_BACKLIGHT_LM3630A=m +CONFIG_BACKLIGHT_LM3639=m +CONFIG_BACKLIGHT_LP855X=m +CONFIG_BACKLIGHT_LP8788=m +CONFIG_BACKLIGHT_PANDORA=m +CONFIG_BACKLIGHT_SKY81452=m +CONFIG_BACKLIGHT_AS3711=m +CONFIG_BACKLIGHT_GPIO=m +CONFIG_BACKLIGHT_LV5207LP=m +CONFIG_BACKLIGHT_BD6107=m +CONFIG_BACKLIGHT_ARCXCNN=m +CONFIG_BACKLIGHT_RAVE_SP=m +CONFIG_BACKLIGHT_LED=m +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER=y +CONFIG_SOUND=m +# CONFIG_SOUND_OSS_CORE_PRECLAIM is not set +CONFIG_SND=m +CONFIG_SND_OSSEMUL=y +CONFIG_SND_MIXER_OSS=m +CONFIG_SND_HRTIMER=m +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_DUMMY=m +CONFIG_SND_ALOOP=m +CONFIG_SND_VIRMIDI=m +CONFIG_SND_MTPAV=m +CONFIG_SND_MTS64=m +CONFIG_SND_SERIAL_U16550=m +CONFIG_SND_SERIAL_GENERIC=m +CONFIG_SND_MPU401=m +CONFIG_SND_PORTMAN2X4=m +CONFIG_SND_AC97_POWER_SAVE=y +CONFIG_SND_AD1889=m +CONFIG_SND_ATIIXP=m +CONFIG_SND_ATIIXP_MODEM=m +CONFIG_SND_AU8810=m +CONFIG_SND_AU8820=m +CONFIG_SND_AU8830=m +CONFIG_SND_AW2=m +CONFIG_SND_BT87X=m +CONFIG_SND_CA0106=m +CONFIG_SND_CMIPCI=m +CONFIG_SND_OXYGEN=m +CONFIG_SND_CS4281=m +CONFIG_SND_CS46XX=m +CONFIG_SND_CTXFI=m +CONFIG_SND_DARLA20=m +CONFIG_SND_GINA20=m +CONFIG_SND_LAYLA20=m +CONFIG_SND_DARLA24=m +CONFIG_SND_GINA24=m +CONFIG_SND_LAYLA24=m +CONFIG_SND_MONA=m +CONFIG_SND_MIA=m +CONFIG_SND_ECHO3G=m +CONFIG_SND_INDIGO=m +CONFIG_SND_INDIGOIO=m +CONFIG_SND_INDIGODJ=m +CONFIG_SND_INDIGOIOX=m +CONFIG_SND_INDIGODJX=m +CONFIG_SND_ENS1370=m +CONFIG_SND_ENS1371=m +CONFIG_SND_FM801=m +CONFIG_SND_FM801_TEA575X_BOOL=y +CONFIG_SND_HDSP=m +CONFIG_SND_HDSPM=m +CONFIG_SND_ICE1724=m +CONFIG_SND_INTEL8X0=m +CONFIG_SND_INTEL8X0M=m +CONFIG_SND_KORG1212=m +CONFIG_SND_LOLA=m +CONFIG_SND_LX6464ES=m +CONFIG_SND_MIXART=m +CONFIG_SND_NM256=m +CONFIG_SND_PCXHR=m +CONFIG_SND_RIPTIDE=m +CONFIG_SND_RME32=m +CONFIG_SND_RME96=m +CONFIG_SND_RME9652=m +CONFIG_SND_VIA82XX=m +CONFIG_SND_VIA82XX_MODEM=m +CONFIG_SND_VIRTUOSO=m +CONFIG_SND_VX222=m +CONFIG_SND_YMFPCI=m +CONFIG_SND_HDA_INTEL=m +CONFIG_SND_HDA_HWDEP=y +CONFIG_SND_HDA_INPUT_BEEP=y +CONFIG_SND_HDA_INPUT_BEEP_MODE=0 +CONFIG_SND_HDA_PATCH_LOADER=y +CONFIG_SND_HDA_CODEC_REALTEK=m +CONFIG_SND_HDA_CODEC_ANALOG=m +CONFIG_SND_HDA_CODEC_SIGMATEL=m +CONFIG_SND_HDA_CODEC_VIA=m +CONFIG_SND_HDA_CODEC_HDMI=m +CONFIG_SND_HDA_CODEC_CIRRUS=m +CONFIG_SND_HDA_CODEC_CS8409=m +CONFIG_SND_HDA_CODEC_CONEXANT=m +CONFIG_SND_HDA_CODEC_CA0110=m +CONFIG_SND_HDA_CODEC_CA0132=m +CONFIG_SND_HDA_CODEC_CMEDIA=m +CONFIG_SND_HDA_CODEC_SI3054=m +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_BCD2000=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_DICE=m +CONFIG_SND_OXFW=m +CONFIG_SND_ISIGHT=m +CONFIG_SND_FIREWORKS=m +CONFIG_SND_BEBOB=m +CONFIG_SND_FIREWIRE_DIGI00X=m +CONFIG_SND_FIREWIRE_TASCAM=m +CONFIG_SND_FIREWIRE_MOTU=m +CONFIG_SND_FIREFACE=m +CONFIG_SND_SOC=m +CONFIG_SND_SOC_ADI=m +CONFIG_SND_SOC_ADI_AXI_I2S=m +CONFIG_SND_SOC_ADI_AXI_SPDIF=m +CONFIG_SND_SOC_AMD_ACP=m +CONFIG_SND_SOC_AMD_CZ_RT5645_MACH=m +CONFIG_SND_AMD_ACP_CONFIG=m +CONFIG_SND_ATMEL_SOC=m +CONFIG_SND_SOC_MIKROE_PROTO=m +CONFIG_SND_BCM63XX_I2S_WHISTLER=m +CONFIG_SND_DESIGNWARE_I2S=m +CONFIG_SND_DESIGNWARE_PCM=y +CONFIG_SND_SOC_FSL_ASRC=m +CONFIG_SND_SOC_FSL_SAI=m +CONFIG_SND_SOC_FSL_MQS=m +CONFIG_SND_SOC_FSL_AUDMIX=m +CONFIG_SND_SOC_FSL_SSI=m +CONFIG_SND_SOC_FSL_SPDIF=m +CONFIG_SND_SOC_FSL_ESAI=m +CONFIG_SND_SOC_FSL_MICFIL=m +CONFIG_SND_SOC_FSL_EASRC=m +CONFIG_SND_SOC_FSL_XCVR=m +CONFIG_SND_SOC_FSL_RPMSG=m +CONFIG_SND_SOC_IMX_AUDMUX=m +CONFIG_SND_I2S_HI6210_I2S=m +CONFIG_SND_SOC_IMG=y +CONFIG_SND_SOC_IMG_I2S_IN=m +CONFIG_SND_SOC_IMG_I2S_OUT=m +CONFIG_SND_SOC_IMG_PARALLEL_OUT=m +CONFIG_SND_SOC_IMG_SPDIF_IN=m +CONFIG_SND_SOC_IMG_SPDIF_OUT=m +CONFIG_SND_SOC_IMG_PISTACHIO_INTERNAL_DAC=m +CONFIG_SND_SOC_MTK_BTCVSD=m +CONFIG_SND_SOC_SOF_TOPLEVEL=y +CONFIG_SND_SOC_SOF_PCI=m +CONFIG_SND_SOC_SOF_OF=m +CONFIG_SND_SOC_XILINX_I2S=m +CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m +CONFIG_SND_SOC_XILINX_SPDIF=m +CONFIG_SND_SOC_XTFPGA_I2S=m +CONFIG_SND_SOC_AC97_CODEC=m +CONFIG_SND_SOC_ADAU1372_I2C=m +CONFIG_SND_SOC_ADAU1372_SPI=m +CONFIG_SND_SOC_ADAU1701=m +CONFIG_SND_SOC_ADAU1761_I2C=m +CONFIG_SND_SOC_ADAU1761_SPI=m +CONFIG_SND_SOC_ADAU7002=m +CONFIG_SND_SOC_ADAU7118_HW=m +CONFIG_SND_SOC_ADAU7118_I2C=m +CONFIG_SND_SOC_AK4104=m +CONFIG_SND_SOC_AK4118=m +CONFIG_SND_SOC_AK4375=m +CONFIG_SND_SOC_AK4458=m +CONFIG_SND_SOC_AK4554=m +CONFIG_SND_SOC_AK4613=m +CONFIG_SND_SOC_AK4642=m +CONFIG_SND_SOC_AK5386=m +CONFIG_SND_SOC_AK5558=m +CONFIG_SND_SOC_ALC5623=m +CONFIG_SND_SOC_AW8738=m +CONFIG_SND_SOC_BD28623=m +CONFIG_SND_SOC_BT_SCO=m +CONFIG_SND_SOC_CPCAP=m +CONFIG_SND_SOC_CS35L32=m +CONFIG_SND_SOC_CS35L33=m +CONFIG_SND_SOC_CS35L34=m +CONFIG_SND_SOC_CS35L35=m +CONFIG_SND_SOC_CS35L36=m +CONFIG_SND_SOC_CS35L41_SPI=m +CONFIG_SND_SOC_CS35L41_I2C=m +CONFIG_SND_SOC_CS35L45_SPI=m +CONFIG_SND_SOC_CS35L45_I2C=m +CONFIG_SND_SOC_CS42L42=m +CONFIG_SND_SOC_CS42L51_I2C=m +CONFIG_SND_SOC_CS42L52=m +CONFIG_SND_SOC_CS42L56=m +CONFIG_SND_SOC_CS42L73=m +CONFIG_SND_SOC_CS4234=m +CONFIG_SND_SOC_CS4265=m +CONFIG_SND_SOC_CS4270=m +CONFIG_SND_SOC_CS4271_I2C=m +CONFIG_SND_SOC_CS4271_SPI=m +CONFIG_SND_SOC_CS42XX8_I2C=m +CONFIG_SND_SOC_CS43130=m +CONFIG_SND_SOC_CS4341=m +CONFIG_SND_SOC_CS4349=m +CONFIG_SND_SOC_CS53L30=m +CONFIG_SND_SOC_CX2072X=m +CONFIG_SND_SOC_DA7213=m +CONFIG_SND_SOC_DMIC=m +CONFIG_SND_SOC_ES7134=m +CONFIG_SND_SOC_ES7241=m +CONFIG_SND_SOC_ES8316=m +CONFIG_SND_SOC_ES8328_I2C=m +CONFIG_SND_SOC_ES8328_SPI=m +CONFIG_SND_SOC_GTM601=m +CONFIG_SND_SOC_ICS43432=m +CONFIG_SND_SOC_INNO_RK3036=m +CONFIG_SND_SOC_LOCHNAGAR_SC=m +CONFIG_SND_SOC_MAX98088=m +CONFIG_SND_SOC_MAX98357A=m +CONFIG_SND_SOC_MAX98504=m +CONFIG_SND_SOC_MAX9867=m +CONFIG_SND_SOC_MAX98927=m +CONFIG_SND_SOC_MAX98520=m +CONFIG_SND_SOC_MAX98373_I2C=m +CONFIG_SND_SOC_MAX98373_SDW=m +CONFIG_SND_SOC_MAX98390=m +CONFIG_SND_SOC_MAX98396=m +CONFIG_SND_SOC_MAX9860=m +CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m +CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m +CONFIG_SND_SOC_PCM1681=m +CONFIG_SND_SOC_PCM1789_I2C=m +CONFIG_SND_SOC_PCM179X_I2C=m +CONFIG_SND_SOC_PCM179X_SPI=m +CONFIG_SND_SOC_PCM186X_I2C=m +CONFIG_SND_SOC_PCM186X_SPI=m +CONFIG_SND_SOC_PCM3060_I2C=m +CONFIG_SND_SOC_PCM3060_SPI=m +CONFIG_SND_SOC_PCM3168A_I2C=m +CONFIG_SND_SOC_PCM3168A_SPI=m +CONFIG_SND_SOC_PCM5102A=m +CONFIG_SND_SOC_PCM512x_I2C=m +CONFIG_SND_SOC_PCM512x_SPI=m +CONFIG_SND_SOC_RK3328=m +CONFIG_SND_SOC_RK817=m +CONFIG_SND_SOC_RT1308_SDW=m +CONFIG_SND_SOC_RT1316_SDW=m +CONFIG_SND_SOC_RT5616=m +CONFIG_SND_SOC_RT5631=m +CONFIG_SND_SOC_RT5640=m +CONFIG_SND_SOC_RT5659=m +CONFIG_SND_SOC_RT5682_SDW=m +CONFIG_SND_SOC_RT700_SDW=m +CONFIG_SND_SOC_RT711_SDW=m +CONFIG_SND_SOC_RT711_SDCA_SDW=m +CONFIG_SND_SOC_RT715_SDW=m +CONFIG_SND_SOC_RT715_SDCA_SDW=m +CONFIG_SND_SOC_RT9120=m +CONFIG_SND_SOC_SDW_MOCKUP=m +CONFIG_SND_SOC_SGTL5000=m +CONFIG_SND_SOC_SIMPLE_AMPLIFIER=m +CONFIG_SND_SOC_SIMPLE_MUX=m +CONFIG_SND_SOC_SPDIF=m +CONFIG_SND_SOC_SSM2305=m +CONFIG_SND_SOC_SSM2518=m +CONFIG_SND_SOC_SSM2602_SPI=m +CONFIG_SND_SOC_SSM2602_I2C=m +CONFIG_SND_SOC_SSM4567=m +CONFIG_SND_SOC_STA32X=m +CONFIG_SND_SOC_STA350=m +CONFIG_SND_SOC_STI_SAS=m +CONFIG_SND_SOC_TAS2552=m +CONFIG_SND_SOC_TAS2562=m +CONFIG_SND_SOC_TAS2764=m +CONFIG_SND_SOC_TAS2770=m +CONFIG_SND_SOC_TAS5086=m +CONFIG_SND_SOC_TAS571X=m +CONFIG_SND_SOC_TAS5720=m +CONFIG_SND_SOC_TAS5805M=m +CONFIG_SND_SOC_TAS6424=m +CONFIG_SND_SOC_TDA7419=m +CONFIG_SND_SOC_TFA9879=m +CONFIG_SND_SOC_TFA989X=m +CONFIG_SND_SOC_TLV320ADC3XXX=m +CONFIG_SND_SOC_TLV320AIC23_I2C=m +CONFIG_SND_SOC_TLV320AIC23_SPI=m +CONFIG_SND_SOC_TLV320AIC31XX=m +CONFIG_SND_SOC_TLV320AIC32X4_I2C=m +CONFIG_SND_SOC_TLV320AIC32X4_SPI=m +CONFIG_SND_SOC_TLV320AIC3X_I2C=m +CONFIG_SND_SOC_TLV320AIC3X_SPI=m +CONFIG_SND_SOC_TLV320ADCX140=m +CONFIG_SND_SOC_TS3A227E=m +CONFIG_SND_SOC_TSCS42XX=m +CONFIG_SND_SOC_TSCS454=m +CONFIG_SND_SOC_UDA1334=m +CONFIG_SND_SOC_WCD9335=m +CONFIG_SND_SOC_WCD934X=m +CONFIG_SND_SOC_WCD938X_SDW=m +CONFIG_SND_SOC_WM8510=m +CONFIG_SND_SOC_WM8523=m +CONFIG_SND_SOC_WM8524=m +CONFIG_SND_SOC_WM8580=m +CONFIG_SND_SOC_WM8711=m +CONFIG_SND_SOC_WM8728=m +CONFIG_SND_SOC_WM8731_I2C=m +CONFIG_SND_SOC_WM8731_SPI=m +CONFIG_SND_SOC_WM8737=m +CONFIG_SND_SOC_WM8741=m +CONFIG_SND_SOC_WM8750=m +CONFIG_SND_SOC_WM8753=m +CONFIG_SND_SOC_WM8770=m +CONFIG_SND_SOC_WM8776=m +CONFIG_SND_SOC_WM8782=m +CONFIG_SND_SOC_WM8804_I2C=m +CONFIG_SND_SOC_WM8804_SPI=m +CONFIG_SND_SOC_WM8903=m +CONFIG_SND_SOC_WM8904=m +CONFIG_SND_SOC_WM8940=m +CONFIG_SND_SOC_WM8960=m +CONFIG_SND_SOC_WM8962=m +CONFIG_SND_SOC_WM8974=m +CONFIG_SND_SOC_WM8978=m +CONFIG_SND_SOC_WM8985=m +CONFIG_SND_SOC_WSA881X=m +CONFIG_SND_SOC_ZL38060=m +CONFIG_SND_SOC_MAX9759=m +CONFIG_SND_SOC_MT6351=m +CONFIG_SND_SOC_MT6358=m +CONFIG_SND_SOC_MT6660=m +CONFIG_SND_SOC_NAU8315=m +CONFIG_SND_SOC_NAU8540=m +CONFIG_SND_SOC_NAU8810=m +CONFIG_SND_SOC_NAU8821=m +CONFIG_SND_SOC_NAU8822=m +CONFIG_SND_SOC_NAU8824=m +CONFIG_SND_SOC_TPA6130A2=m +CONFIG_SND_SOC_LPASS_WSA_MACRO=m +CONFIG_SND_SOC_LPASS_VA_MACRO=m +CONFIG_SND_SOC_LPASS_RX_MACRO=m +CONFIG_SND_SOC_LPASS_TX_MACRO=m +CONFIG_SND_SIMPLE_CARD=m +CONFIG_SND_AUDIO_GRAPH_CARD=m +CONFIG_SND_AUDIO_GRAPH_CARD2=m +CONFIG_SND_AUDIO_GRAPH_CARD2_CUSTOM_SAMPLE=m +CONFIG_SND_TEST_COMPONENT=m +CONFIG_SND_VIRTIO=m +CONFIG_HID=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HIDRAW=y +CONFIG_UHID=m +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_ASUS=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_MACALLY=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CP2112=m +CONFIG_HID_CREATIVE_SB0540=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EZKEY=m +CONFIG_HID_FT260=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_GLORIOUS=m +CONFIG_HID_HOLTEK=m +CONFIG_HOLTEK_FF=y +CONFIG_HID_VIVALDI=m +CONFIG_HID_GT683R=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_WALTOP=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_XIAOMI=m +CONFIG_HID_GYRATION=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LETSKETCH=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MEGAWORLD_FF=m +CONFIG_HID_REDRAGON=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_NINTENDO=m +CONFIG_NINTENDO_FF=y +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_LCD=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PICOLCD_CIR=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PLAYSTATION=m +CONFIG_PLAYSTATION_FF=y +CONFIG_HID_RAZER=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_RETRODE=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SEMITEK=m +CONFIG_HID_SIGMAMICRO=m +CONFIG_HID_SONY=m +CONFIG_SONY_FF=y +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_RMI=m +CONFIG_HID_GREENASIA=m +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_WACOM=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_ZEROPLUS_FF=y +CONFIG_HID_ZYDACRON=m +CONFIG_HID_SENSOR_HUB=m +CONFIG_HID_SENSOR_CUSTOM_SENSOR=m +CONFIG_HID_ALPS=m +CONFIG_HID_MCP2221=m +CONFIG_USB_HID=m +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_USB_KBD=m +CONFIG_USB_MOUSE=m +CONFIG_I2C_HID_OF=m +CONFIG_I2C_HID_OF_GOODIX=m +CONFIG_USB_LED_TRIG=y +CONFIG_USB_CONN_GPIO=m +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_DYNAMIC_MINORS=y +CONFIG_USB_LEDS_TRIGGER_USBPORT=m +CONFIG_USB_MON=m +CONFIG_USB_C67X00_HCD=m +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_DBGCAP=y +CONFIG_USB_XHCI_PCI_RENESAS=m +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_FSL=m +CONFIG_USB_OXU210HP_HCD=m +CONFIG_USB_ISP116X_HCD=m +CONFIG_USB_MAX3421_HCD=m +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_UHCI_HCD=y +CONFIG_USB_U132_HCD=m +CONFIG_USB_SL811_HCD=m +CONFIG_USB_SL811_HCD_ISO=y +CONFIG_USB_R8A66597_HCD=m +CONFIG_USB_HCD_BCMA=m +CONFIG_USB_HCD_SSB=m +CONFIG_USB_PRINTER=m +CONFIG_USB_TMC=m +CONFIG_USB_STORAGE=m +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_UAS=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VUDC=m +CONFIG_USB_CDNS_SUPPORT=m +CONFIG_USB_CDNS3=m +CONFIG_USB_CDNS3_GADGET=y +CONFIG_USB_CDNS3_HOST=y +CONFIG_USB_MUSB_HDRC=m +CONFIG_USB_MUSB_POLARFIRE_SOC=m +CONFIG_MUSB_PIO_ONLY=y +CONFIG_USB_DWC3=m +CONFIG_USB_DWC3_ULPI=y +CONFIG_USB_DWC2=y +CONFIG_USB_DWC2_PCI=m +CONFIG_USB_CHIPIDEA=m +CONFIG_USB_CHIPIDEA_UDC=y +CONFIG_USB_CHIPIDEA_HOST=y +CONFIG_USB_ISP1760=m +CONFIG_USB_USS720=m +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_METRO=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7715_PARPORT=y +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_MXUPORT=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SERIAL_WISHBONE=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_XR=m +CONFIG_USB_SERIAL_DEBUG=m +CONFIG_USB_EMI62=m +CONFIG_USB_EMI26=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_LCD=m +CONFIG_USB_CYPRESS_CY7C63=m +CONFIG_USB_CYTHERM=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_FTDI_ELAN=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_APPLE_MFI_FASTCHARGE=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_LD=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_TEST=m +CONFIG_USB_EHSET_TEST_FIXTURE=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_YUREX=m +CONFIG_USB_HUB_USB251XB=m +CONFIG_USB_HSIC_USB3503=m +CONFIG_USB_HSIC_USB4604=m +CONFIG_USB_LINK_LAYER_TEST=m +CONFIG_USB_CHAOSKEY=m +CONFIG_USB_ATM=m +CONFIG_USB_SPEEDTOUCH=m +CONFIG_USB_CXACRU=m +CONFIG_USB_UEAGLEATM=m +CONFIG_USB_XUSBATM=m +CONFIG_USB_GPIO_VBUS=m +CONFIG_TAHVO_USB=m +CONFIG_TAHVO_USB_HOST_BY_DEFAULT=y +CONFIG_USB_ISP1301=m +CONFIG_USB_GADGET=m +CONFIG_U_SERIAL_CONSOLE=y +CONFIG_USB_GR_UDC=m +CONFIG_USB_R8A66597=m +CONFIG_USB_PXA27X=m +CONFIG_USB_MV_UDC=m +CONFIG_USB_MV_U3D=m +CONFIG_USB_SNP_UDC_PLAT=m +CONFIG_USB_BDC_UDC=m +CONFIG_USB_AMD5536UDC=m +CONFIG_USB_NET2272=m +CONFIG_USB_NET2272_DMA=y +CONFIG_USB_NET2280=m +CONFIG_USB_GOKU=m +CONFIG_USB_EG20T=m +CONFIG_USB_GADGET_XILINX=m +CONFIG_USB_MAX3420_UDC=m +CONFIG_USB_CONFIGFS=m +CONFIG_USB_CONFIGFS_SERIAL=y +CONFIG_USB_CONFIGFS_ACM=y +CONFIG_USB_CONFIGFS_OBEX=y +CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y +CONFIG_USB_CONFIGFS_ECM_SUBSET=y +CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y +CONFIG_USB_CONFIGFS_PHONET=y +CONFIG_USB_CONFIGFS_MASS_STORAGE=y +CONFIG_USB_CONFIGFS_F_LB_SS=y +CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_CONFIGFS_F_UAC1=y +CONFIG_USB_CONFIGFS_F_UAC1_LEGACY=y +CONFIG_USB_CONFIGFS_F_UAC2=y +CONFIG_USB_CONFIGFS_F_MIDI=y +CONFIG_USB_CONFIGFS_F_HID=y +CONFIG_USB_CONFIGFS_F_UVC=y +CONFIG_USB_CONFIGFS_F_PRINTER=y +CONFIG_USB_CONFIGFS_F_TCM=y +CONFIG_USB_ZERO=m +CONFIG_USB_AUDIO=m +CONFIG_GADGET_UAC1=y +CONFIG_USB_ETH=m +CONFIG_USB_ETH_EEM=y +CONFIG_USB_G_NCM=m +CONFIG_USB_GADGETFS=m +CONFIG_USB_FUNCTIONFS=m +CONFIG_USB_FUNCTIONFS_ETH=y +CONFIG_USB_FUNCTIONFS_RNDIS=y +CONFIG_USB_FUNCTIONFS_GENERIC=y +CONFIG_USB_MASS_STORAGE=m +CONFIG_USB_GADGET_TARGET=m +CONFIG_USB_G_SERIAL=m +CONFIG_USB_MIDI_GADGET=m +CONFIG_USB_G_PRINTER=m +CONFIG_USB_CDC_COMPOSITE=m +CONFIG_USB_G_NOKIA=m +CONFIG_USB_G_ACM_MS=m +CONFIG_USB_G_HID=m +CONFIG_USB_G_DBGP=m +CONFIG_USB_G_WEBCAM=m +CONFIG_USB_RAW_GADGET=m +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_TCPCI=m +CONFIG_TYPEC_RT1711H=m +CONFIG_TYPEC_MT6360=m +CONFIG_TYPEC_TCPCI_MAXIM=m +CONFIG_TYPEC_FUSB302=m +CONFIG_TYPEC_UCSI=m +CONFIG_UCSI_CCG=m +CONFIG_TYPEC_TPS6598X=m +CONFIG_TYPEC_RT1719=m +CONFIG_TYPEC_HD3SS3220=m +CONFIG_TYPEC_STUSB160X=m +CONFIG_TYPEC_WUSB3801=m +CONFIG_TYPEC_MUX_FSA4480=m +CONFIG_TYPEC_MUX_PI3USB30532=m +CONFIG_TYPEC_DP_ALTMODE=m +CONFIG_TYPEC_NVIDIA_ALTMODE=m +CONFIG_MMC=y +CONFIG_PWRSEQ_EMMC=m +CONFIG_PWRSEQ_SD8787=m +CONFIG_PWRSEQ_SIMPLE=m +CONFIG_SDIO_UART=m +CONFIG_MMC_CRYPTO=y +CONFIG_MMC_SDHCI=m +CONFIG_MMC_SDHCI_PCI=m +CONFIG_MMC_SDHCI_PLTFM=m +CONFIG_MMC_SDHCI_OF_ARASAN=m +CONFIG_MMC_SDHCI_OF_AT91=m +CONFIG_MMC_SDHCI_OF_DWCMSHC=m +CONFIG_MMC_SDHCI_CADENCE=m +CONFIG_MMC_SDHCI_F_SDH30=m +CONFIG_MMC_SDHCI_MILBEAUT=m +CONFIG_MMC_ALCOR=m +CONFIG_MMC_TIFM_SD=m +CONFIG_MMC_SPI=y +CONFIG_MMC_CB710=m +CONFIG_MMC_VIA_SDMMC=m +CONFIG_MMC_DW=m +CONFIG_MMC_DW_BLUEFIELD=m +CONFIG_MMC_DW_EXYNOS=m +CONFIG_MMC_DW_HI3798CV200=m +CONFIG_MMC_DW_K3=m +CONFIG_MMC_DW_PCI=m +CONFIG_MMC_VUB300=m +CONFIG_MMC_USHC=m +CONFIG_MMC_USDHI6ROL0=m +CONFIG_MMC_REALTEK_PCI=m +CONFIG_MMC_REALTEK_USB=m +CONFIG_MMC_HSQ=m +CONFIG_MMC_TOSHIBA_PCI=m +CONFIG_MMC_MTK=m +CONFIG_MMC_SDHCI_XENON=m +CONFIG_MMC_SDHCI_OMAP=m +CONFIG_MMC_SDHCI_AM654=m +CONFIG_MMC_LITEX=m +CONFIG_SCSI_UFSHCD=m +CONFIG_SCSI_UFS_BSG=y +CONFIG_SCSI_UFS_CRYPTO=y +CONFIG_SCSI_UFS_HPB=y +CONFIG_SCSI_UFSHCD_PCI=m +CONFIG_SCSI_UFS_DWC_TC_PCI=m +CONFIG_SCSI_UFSHCD_PLATFORM=m +CONFIG_SCSI_UFS_CDNS_PLATFORM=m +CONFIG_SCSI_UFS_DWC_TC_PLATFORM=m +CONFIG_MEMSTICK=m +CONFIG_MSPRO_BLOCK=m +CONFIG_MS_BLOCK=m +CONFIG_MEMSTICK_TIFM_MS=m +CONFIG_MEMSTICK_JMICRON_38X=m +CONFIG_MEMSTICK_R592=m +CONFIG_MEMSTICK_REALTEK_PCI=m +CONFIG_MEMSTICK_REALTEK_USB=m +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_CLASS_FLASH=m +CONFIG_LEDS_CLASS_MULTICOLOR=m +CONFIG_LEDS_BRIGHTNESS_HW_CHANGED=y +CONFIG_LEDS_88PM860X=m +CONFIG_LEDS_AN30259A=m +CONFIG_LEDS_AW2013=m +CONFIG_LEDS_BCM6328=m +CONFIG_LEDS_BCM6358=m +CONFIG_LEDS_CPCAP=m +CONFIG_LEDS_CR0014114=m +CONFIG_LEDS_EL15203000=m +CONFIG_LEDS_LM3530=m +CONFIG_LEDS_LM3532=m +CONFIG_LEDS_LM3533=m +CONFIG_LEDS_LM3642=m +CONFIG_LEDS_LM3692X=m +CONFIG_LEDS_MT6323=m +CONFIG_LEDS_PCA9532=m +CONFIG_LEDS_PCA9532_GPIO=y +CONFIG_LEDS_GPIO=m +CONFIG_LEDS_LP3944=m +CONFIG_LEDS_LP3952=m +CONFIG_LEDS_LP50XX=m +CONFIG_LEDS_LP55XX_COMMON=m +CONFIG_LEDS_LP5521=m +CONFIG_LEDS_LP5523=m +CONFIG_LEDS_LP5562=m +CONFIG_LEDS_LP8501=m +CONFIG_LEDS_LP8788=m +CONFIG_LEDS_LP8860=m +CONFIG_LEDS_PCA955X=m +CONFIG_LEDS_PCA955X_GPIO=y +CONFIG_LEDS_PCA963X=m +CONFIG_LEDS_WM831X_STATUS=m +CONFIG_LEDS_WM8350=m +CONFIG_LEDS_DA903X=m +CONFIG_LEDS_DA9052=m +CONFIG_LEDS_DAC124S085=m +CONFIG_LEDS_PWM=m +CONFIG_LEDS_REGULATOR=m +CONFIG_LEDS_BD2802=m +CONFIG_LEDS_LT3593=m +CONFIG_LEDS_ADP5520=m +CONFIG_LEDS_MC13783=m +CONFIG_LEDS_TCA6507=m +CONFIG_LEDS_TLC591XX=m +CONFIG_LEDS_MAX77650=m +CONFIG_LEDS_MAX8997=m +CONFIG_LEDS_LM355x=m +CONFIG_LEDS_MENF21BMC=m +CONFIG_LEDS_IS31FL319X=m +CONFIG_LEDS_IS31FL32XX=m +CONFIG_LEDS_BLINKM=m +CONFIG_LEDS_SYSCON=y +CONFIG_LEDS_MLXREG=m +CONFIG_LEDS_USER=m +CONFIG_LEDS_SPI_BYTE=m +CONFIG_LEDS_TI_LMU_COMMON=m +CONFIG_LEDS_LM3697=m +CONFIG_LEDS_LM36274=m +CONFIG_LEDS_AAT1290=m +CONFIG_LEDS_AS3645A=m +CONFIG_LEDS_KTD2692=m +CONFIG_LEDS_LM3601X=m +CONFIG_LEDS_MAX77693=m +CONFIG_LEDS_MT6360=m +CONFIG_LEDS_RT4505=m +CONFIG_LEDS_RT8515=m +CONFIG_LEDS_SGM3140=m +CONFIG_LEDS_PWM_MULTICOLOR=m +CONFIG_LEDS_QCOM_LPG=m +CONFIG_LEDS_TRIGGER_TIMER=m +CONFIG_LEDS_TRIGGER_ONESHOT=m +CONFIG_LEDS_TRIGGER_DISK=y +CONFIG_LEDS_TRIGGER_MTD=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=m +CONFIG_LEDS_TRIGGER_BACKLIGHT=m +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_ACTIVITY=m +CONFIG_LEDS_TRIGGER_GPIO=m +CONFIG_LEDS_TRIGGER_DEFAULT_ON=m +CONFIG_LEDS_TRIGGER_TRANSIENT=m +CONFIG_LEDS_TRIGGER_CAMERA=m +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_LEDS_TRIGGER_NETDEV=m +CONFIG_LEDS_TRIGGER_PATTERN=m +CONFIG_LEDS_TRIGGER_TTY=m +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_BNXT_RE=m +CONFIG_INFINIBAND_CXGB4=m +CONFIG_INFINIBAND_EFA=m +CONFIG_INFINIBAND_IRDMA=m +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_INFINIBAND_OCRDMA=m +CONFIG_INFINIBAND_QEDR=m +CONFIG_INFINIBAND_VMWARE_PVRDMA=m +CONFIG_RDMA_RXE=m +CONFIG_RDMA_SIW=m +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y +# CONFIG_INFINIBAND_IPOIB_DEBUG is not set +CONFIG_INFINIBAND_SRP=m +CONFIG_INFINIBAND_SRPT=m +CONFIG_INFINIBAND_ISER=m +CONFIG_INFINIBAND_ISERT=m +CONFIG_INFINIBAND_RTRS_CLIENT=m +CONFIG_INFINIBAND_RTRS_SERVER=m +CONFIG_EDAC=y +# CONFIG_EDAC_LEGACY_SYSFS is not set +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_88PM860X=m +CONFIG_RTC_DRV_88PM80X=m +CONFIG_RTC_DRV_ABB5ZES3=m +CONFIG_RTC_DRV_ABEOZ9=m +CONFIG_RTC_DRV_ABX80X=m +CONFIG_RTC_DRV_AS3722=m +CONFIG_RTC_DRV_DS1307=m +CONFIG_RTC_DRV_DS1307_CENTURY=y +CONFIG_RTC_DRV_DS1374=m +CONFIG_RTC_DRV_DS1374_WDT=y +CONFIG_RTC_DRV_DS1672=m +CONFIG_RTC_DRV_HYM8563=m +CONFIG_RTC_DRV_LP8788=m +CONFIG_RTC_DRV_MAX6900=m +CONFIG_RTC_DRV_MAX8907=m +CONFIG_RTC_DRV_MAX8925=m +CONFIG_RTC_DRV_MAX8998=m +CONFIG_RTC_DRV_MAX8997=m +CONFIG_RTC_DRV_MAX77686=m +CONFIG_RTC_DRV_RK808=m +CONFIG_RTC_DRV_RS5C372=m +CONFIG_RTC_DRV_ISL1208=m +CONFIG_RTC_DRV_ISL12022=m +CONFIG_RTC_DRV_ISL12026=m +CONFIG_RTC_DRV_X1205=m +CONFIG_RTC_DRV_PCF8523=m +CONFIG_RTC_DRV_PCF85063=m +CONFIG_RTC_DRV_PCF85363=m +CONFIG_RTC_DRV_PCF8563=m +CONFIG_RTC_DRV_PCF8583=m +CONFIG_RTC_DRV_M41T80=m +CONFIG_RTC_DRV_M41T80_WDT=y +CONFIG_RTC_DRV_BD70528=m +CONFIG_RTC_DRV_BQ32K=m +CONFIG_RTC_DRV_TWL4030=m +CONFIG_RTC_DRV_PALMAS=m +CONFIG_RTC_DRV_TPS6586X=m +CONFIG_RTC_DRV_TPS65910=m +CONFIG_RTC_DRV_RC5T583=m +CONFIG_RTC_DRV_RC5T619=m +CONFIG_RTC_DRV_S35390A=m +CONFIG_RTC_DRV_FM3130=m +CONFIG_RTC_DRV_RX8010=m +CONFIG_RTC_DRV_RX8581=m +CONFIG_RTC_DRV_RX8025=m +CONFIG_RTC_DRV_EM3027=m +CONFIG_RTC_DRV_RV3028=m +CONFIG_RTC_DRV_RV3032=m +CONFIG_RTC_DRV_RV8803=m +CONFIG_RTC_DRV_S5M=m +CONFIG_RTC_DRV_SD3078=m +CONFIG_RTC_DRV_M41T93=m +CONFIG_RTC_DRV_M41T94=m +CONFIG_RTC_DRV_DS1302=m +CONFIG_RTC_DRV_DS1305=m +CONFIG_RTC_DRV_DS1343=m +CONFIG_RTC_DRV_DS1347=m +CONFIG_RTC_DRV_DS1390=m +CONFIG_RTC_DRV_MAX6916=m +CONFIG_RTC_DRV_R9701=m +CONFIG_RTC_DRV_RX4581=m +CONFIG_RTC_DRV_RS5C348=m +CONFIG_RTC_DRV_MAX6902=m +CONFIG_RTC_DRV_PCF2123=m +CONFIG_RTC_DRV_MCP795=m +CONFIG_RTC_DRV_DS3232=m +CONFIG_RTC_DRV_PCF2127=m +CONFIG_RTC_DRV_RV3029C2=m +CONFIG_RTC_DRV_RX6110=m +CONFIG_RTC_DRV_DS1286=m +CONFIG_RTC_DRV_DS1511=m +CONFIG_RTC_DRV_DS1553=m +CONFIG_RTC_DRV_DS1685_FAMILY=m +CONFIG_RTC_DRV_DS1742=m +CONFIG_RTC_DRV_DS2404=m +CONFIG_RTC_DRV_DA9052=m +CONFIG_RTC_DRV_DA9055=m +CONFIG_RTC_DRV_DA9063=m +CONFIG_RTC_DRV_EFI=m +CONFIG_RTC_DRV_STK17TA8=m +CONFIG_RTC_DRV_M48T86=m +CONFIG_RTC_DRV_M48T35=m +CONFIG_RTC_DRV_M48T59=m +CONFIG_RTC_DRV_MSM6242=m +CONFIG_RTC_DRV_BQ4802=m +CONFIG_RTC_DRV_RP5C01=m +CONFIG_RTC_DRV_WM831X=m +CONFIG_RTC_DRV_WM8350=m +CONFIG_RTC_DRV_PCF50633=m +CONFIG_RTC_DRV_ZYNQMP=m +CONFIG_RTC_DRV_NTXEC=m +CONFIG_RTC_DRV_CADENCE=m +CONFIG_RTC_DRV_FTRTC010=m +CONFIG_RTC_DRV_PCAP=m +CONFIG_RTC_DRV_MC13XXX=m +CONFIG_RTC_DRV_MT6397=m +CONFIG_RTC_DRV_R7301=m +CONFIG_RTC_DRV_CPCAP=m +CONFIG_RTC_DRV_HID_SENSOR_TIME=m +CONFIG_DMADEVICES=y +CONFIG_ALTERA_MSGDMA=m +CONFIG_DW_AXI_DMAC=m +CONFIG_FSL_EDMA=m +CONFIG_INTEL_IDMA64=m +CONFIG_PLX_DMA=m +CONFIG_XILINX_ZYNQMP_DPDMA=m +CONFIG_QCOM_HIDMA_MGMT=m +CONFIG_QCOM_HIDMA=m +CONFIG_DW_DMAC=m +CONFIG_DW_DMAC_PCI=m +CONFIG_DW_EDMA=m +CONFIG_DW_EDMA_PCIE=m +CONFIG_SF_PDMA=m +CONFIG_ASYNC_TX_DMA=y +CONFIG_SW_SYNC=y +CONFIG_UDMABUF=y +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_DMABUF_HEAPS_CMA=y +CONFIG_HD44780=m +CONFIG_KS0108=m +CONFIG_IMG_ASCII_LCD=m +CONFIG_HT16K33=m +CONFIG_LCD2S=m +CONFIG_PANEL=m +CONFIG_UIO_CIF=m +CONFIG_UIO_PDRV_GENIRQ=m +CONFIG_UIO_DMEM_GENIRQ=m +CONFIG_UIO_AEC=m +CONFIG_UIO_SERCOS3=m +CONFIG_UIO_PCI_GENERIC=m +CONFIG_UIO_NETX=m +CONFIG_UIO_PRUSS=m +CONFIG_UIO_MF624=m +CONFIG_UIO_DFL=m +CONFIG_VFIO=m +CONFIG_VFIO_NOIOMMU=y +CONFIG_VFIO_PCI=m +CONFIG_MLX5_VFIO_PCI=m +CONFIG_VIRT_DRIVERS=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_VDPA=m +CONFIG_VIRTIO_PMEM=m +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=m +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y +CONFIG_VDPA=m +CONFIG_VDPA_SIM=m +CONFIG_VDPA_SIM_NET=m +CONFIG_VDPA_SIM_BLOCK=m +CONFIG_VDPA_USER=m +CONFIG_IFCVF=m +CONFIG_MLX5_VDPA_NET=m +CONFIG_VP_VDPA=m +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_VDPA=m +CONFIG_COMEDI=m +CONFIG_COMEDI_MISC_DRIVERS=y +CONFIG_COMEDI_BOND=m +CONFIG_COMEDI_TEST=m +CONFIG_COMEDI_PARPORT=m +CONFIG_COMEDI_ISA_DRIVERS=y +CONFIG_COMEDI_PCL711=m +CONFIG_COMEDI_PCL724=m +CONFIG_COMEDI_PCL726=m +CONFIG_COMEDI_PCL730=m +CONFIG_COMEDI_PCL812=m +CONFIG_COMEDI_PCL816=m +CONFIG_COMEDI_PCL818=m +CONFIG_COMEDI_PCM3724=m +CONFIG_COMEDI_AMPLC_DIO200_ISA=m +CONFIG_COMEDI_AMPLC_PC236_ISA=m +CONFIG_COMEDI_AMPLC_PC263_ISA=m +CONFIG_COMEDI_RTI800=m +CONFIG_COMEDI_RTI802=m +CONFIG_COMEDI_DAC02=m +CONFIG_COMEDI_DAS16M1=m +CONFIG_COMEDI_DAS08_ISA=m +CONFIG_COMEDI_DAS16=m +CONFIG_COMEDI_DAS800=m +CONFIG_COMEDI_DAS1800=m +CONFIG_COMEDI_DAS6402=m +CONFIG_COMEDI_DT2801=m +CONFIG_COMEDI_DT2811=m +CONFIG_COMEDI_DT2814=m +CONFIG_COMEDI_DT2815=m +CONFIG_COMEDI_DT2817=m +CONFIG_COMEDI_DT282X=m +CONFIG_COMEDI_DMM32AT=m +CONFIG_COMEDI_FL512=m +CONFIG_COMEDI_AIO_AIO12_8=m +CONFIG_COMEDI_AIO_IIRO_16=m +CONFIG_COMEDI_II_PCI20KC=m +CONFIG_COMEDI_C6XDIGIO=m +CONFIG_COMEDI_MPC624=m +CONFIG_COMEDI_ADQ12B=m +CONFIG_COMEDI_NI_AT_A2150=m +CONFIG_COMEDI_NI_AT_AO=m +CONFIG_COMEDI_NI_ATMIO=m +CONFIG_COMEDI_NI_ATMIO16D=m +CONFIG_COMEDI_NI_LABPC_ISA=m +CONFIG_COMEDI_PCMAD=m +CONFIG_COMEDI_PCMDA12=m +CONFIG_COMEDI_PCMMIO=m +CONFIG_COMEDI_PCMUIO=m +CONFIG_COMEDI_MULTIQ3=m +CONFIG_COMEDI_S526=m +CONFIG_COMEDI_PCI_DRIVERS=m +CONFIG_COMEDI_8255_PCI=m +CONFIG_COMEDI_ADDI_APCI_1032=m +CONFIG_COMEDI_ADDI_APCI_1500=m +CONFIG_COMEDI_ADDI_APCI_1516=m +CONFIG_COMEDI_ADDI_APCI_1564=m +CONFIG_COMEDI_ADDI_APCI_16XX=m +CONFIG_COMEDI_ADDI_APCI_2032=m +CONFIG_COMEDI_ADDI_APCI_2200=m +CONFIG_COMEDI_ADDI_APCI_3120=m +CONFIG_COMEDI_ADDI_APCI_3501=m +CONFIG_COMEDI_ADDI_APCI_3XXX=m +CONFIG_COMEDI_ADL_PCI6208=m +CONFIG_COMEDI_ADL_PCI7X3X=m +CONFIG_COMEDI_ADL_PCI8164=m +CONFIG_COMEDI_ADL_PCI9111=m +CONFIG_COMEDI_ADL_PCI9118=m +CONFIG_COMEDI_ADV_PCI1710=m +CONFIG_COMEDI_ADV_PCI1720=m +CONFIG_COMEDI_ADV_PCI1723=m +CONFIG_COMEDI_ADV_PCI1724=m +CONFIG_COMEDI_ADV_PCI1760=m +CONFIG_COMEDI_ADV_PCI_DIO=m +CONFIG_COMEDI_AMPLC_DIO200_PCI=m +CONFIG_COMEDI_AMPLC_PC236_PCI=m +CONFIG_COMEDI_AMPLC_PC263_PCI=m +CONFIG_COMEDI_AMPLC_PCI224=m +CONFIG_COMEDI_AMPLC_PCI230=m +CONFIG_COMEDI_CONTEC_PCI_DIO=m +CONFIG_COMEDI_DAS08_PCI=m +CONFIG_COMEDI_DT3000=m +CONFIG_COMEDI_DYNA_PCI10XX=m +CONFIG_COMEDI_GSC_HPDI=m +CONFIG_COMEDI_MF6X4=m +CONFIG_COMEDI_ICP_MULTI=m +CONFIG_COMEDI_DAQBOARD2000=m +CONFIG_COMEDI_JR3_PCI=m +CONFIG_COMEDI_KE_COUNTER=m +CONFIG_COMEDI_CB_PCIDAS64=m +CONFIG_COMEDI_CB_PCIDAS=m +CONFIG_COMEDI_CB_PCIDDA=m +CONFIG_COMEDI_CB_PCIMDAS=m +CONFIG_COMEDI_CB_PCIMDDA=m +CONFIG_COMEDI_ME4000=m +CONFIG_COMEDI_ME_DAQ=m +CONFIG_COMEDI_NI_6527=m +CONFIG_COMEDI_NI_65XX=m +CONFIG_COMEDI_NI_660X=m +CONFIG_COMEDI_NI_670X=m +CONFIG_COMEDI_NI_LABPC_PCI=m +CONFIG_COMEDI_NI_PCIDIO=m +CONFIG_COMEDI_NI_PCIMIO=m +CONFIG_COMEDI_RTD520=m +CONFIG_COMEDI_S626=m +CONFIG_COMEDI_USB_DRIVERS=m +CONFIG_COMEDI_DT9812=m +CONFIG_COMEDI_NI_USB6501=m +CONFIG_COMEDI_USBDUX=m +CONFIG_COMEDI_USBDUXFAST=m +CONFIG_COMEDI_USBDUXSIGMA=m +CONFIG_COMEDI_VMK80XX=m +CONFIG_COMEDI_8255_SA=m +CONFIG_COMEDI_TESTS=m +CONFIG_COMEDI_TESTS_EXAMPLE=m +CONFIG_COMEDI_TESTS_NI_ROUTES=m +CONFIG_STAGING=y +CONFIG_PRISM2_USB=m +CONFIG_RTL8192U=m +CONFIG_RTLLIB=m +CONFIG_RTL8192E=m +CONFIG_RTL8723BS=m +CONFIG_R8712U=m +CONFIG_R8188EU=m +CONFIG_RTS5208=m +CONFIG_VT6655=m +CONFIG_VT6656=m +CONFIG_ADIS16203=m +CONFIG_ADIS16240=m +CONFIG_AD7816=m +CONFIG_ADT7316=m +CONFIG_ADT7316_I2C=m +CONFIG_AD9832=m +CONFIG_AD9834=m +CONFIG_AD5933=m +CONFIG_ADE7854=m +CONFIG_AD2S1210=m +CONFIG_FB_SM750=m +CONFIG_STAGING_MEDIA=y +CONFIG_VIDEO_MAX96712=m +CONFIG_LTE_GDM724X=m +CONFIG_FB_TFT=m +CONFIG_FB_TFT_AGM1264K_FL=m +CONFIG_FB_TFT_BD663474=m +CONFIG_FB_TFT_HX8340BN=m +CONFIG_FB_TFT_HX8347D=m +CONFIG_FB_TFT_HX8353D=m +CONFIG_FB_TFT_HX8357D=m +CONFIG_FB_TFT_ILI9163=m +CONFIG_FB_TFT_ILI9320=m +CONFIG_FB_TFT_ILI9325=m +CONFIG_FB_TFT_ILI9340=m +CONFIG_FB_TFT_ILI9341=m +CONFIG_FB_TFT_ILI9481=m +CONFIG_FB_TFT_ILI9486=m +CONFIG_FB_TFT_PCD8544=m +CONFIG_FB_TFT_RA8875=m +CONFIG_FB_TFT_S6D02A1=m +CONFIG_FB_TFT_S6D1121=m +CONFIG_FB_TFT_SEPS525=m +CONFIG_FB_TFT_SH1106=m +CONFIG_FB_TFT_SSD1289=m +CONFIG_FB_TFT_SSD1305=m +CONFIG_FB_TFT_SSD1306=m +CONFIG_FB_TFT_SSD1331=m +CONFIG_FB_TFT_SSD1351=m +CONFIG_FB_TFT_ST7735R=m +CONFIG_FB_TFT_ST7789V=m +CONFIG_FB_TFT_TINYLCD=m +CONFIG_FB_TFT_TLS8204=m +CONFIG_FB_TFT_UC1611=m +CONFIG_FB_TFT_UC1701=m +CONFIG_FB_TFT_UPD161704=m +CONFIG_MOST_COMPONENTS=m +CONFIG_MOST_NET=m +CONFIG_MOST_VIDEO=m +CONFIG_MOST_DIM2=m +CONFIG_MOST_I2C=m +CONFIG_KS7010=m +CONFIG_PI433=m +CONFIG_XIL_AXIS_FIFO=m +CONFIG_FIELDBUS_DEV=m +CONFIG_HMS_ANYBUSS_BUS=m +CONFIG_ARCX_ANYBUS_CONTROLLER=m +CONFIG_HMS_PROFINET=m +CONFIG_QLGE=m +CONFIG_VME_BUS=y +CONFIG_VME_TSI148=m +CONFIG_VME_FAKE=m +CONFIG_VME_USER=m +CONFIG_GOLDFISH_PIPE=m +CONFIG_COMMON_CLK_WM831X=m +CONFIG_LMK04832=m +CONFIG_COMMON_CLK_MAX77686=m +CONFIG_COMMON_CLK_MAX9485=m +CONFIG_COMMON_CLK_RK808=m +CONFIG_COMMON_CLK_SI5341=m +CONFIG_COMMON_CLK_SI5351=m +CONFIG_COMMON_CLK_SI514=m +CONFIG_COMMON_CLK_SI544=m +CONFIG_COMMON_CLK_SI570=m +CONFIG_COMMON_CLK_CDCE706=m +CONFIG_COMMON_CLK_CDCE925=m +CONFIG_COMMON_CLK_CS2000_CP=m +CONFIG_COMMON_CLK_S2MPS11=m +CONFIG_CLK_TWL6040=m +CONFIG_COMMON_CLK_AXI_CLKGEN=m +CONFIG_COMMON_CLK_LOCHNAGAR=m +CONFIG_COMMON_CLK_PALMAS=m +CONFIG_COMMON_CLK_PWM=m +CONFIG_COMMON_CLK_RS9_PCIE=m +CONFIG_COMMON_CLK_VC5=m +CONFIG_COMMON_CLK_BD718XX=m +CONFIG_COMMON_CLK_FIXED_MMIO=y +CONFIG_XILINX_VCU=m +CONFIG_HWSPINLOCK=y +CONFIG_MAILBOX=y +CONFIG_PLATFORM_MHU=m +CONFIG_ALTERA_MBOX=m +CONFIG_MAILBOX_TEST=m +CONFIG_POLARFIRE_SOC_MAILBOX=m +CONFIG_RPMSG_CHAR=m +CONFIG_RPMSG_CTRL=m +CONFIG_RPMSG_QCOM_GLINK_RPM=m +CONFIG_RPMSG_VIRTIO=m +CONFIG_SOUNDWIRE=m +CONFIG_SOUNDWIRE_QCOM=m +CONFIG_LITEX_SOC_CONTROLLER=m +CONFIG_POLARFIRE_SOC_SYS_CTRL=m +CONFIG_SOC_TI=y +CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y +CONFIG_DEVFREQ_GOV_PERFORMANCE=y +CONFIG_DEVFREQ_GOV_POWERSAVE=y +CONFIG_DEVFREQ_GOV_USERSPACE=y +CONFIG_DEVFREQ_GOV_PASSIVE=y +CONFIG_PM_DEVFREQ_EVENT=y +CONFIG_EXTCON_ADC_JACK=m +CONFIG_EXTCON_FSA9480=m +CONFIG_EXTCON_GPIO=m +CONFIG_EXTCON_MAX14577=m +CONFIG_EXTCON_MAX3355=m +CONFIG_EXTCON_MAX77693=m +CONFIG_EXTCON_MAX77843=m +CONFIG_EXTCON_MAX8997=m +CONFIG_EXTCON_PALMAS=m +CONFIG_EXTCON_PTN5150=m +CONFIG_EXTCON_RT8973A=m +CONFIG_EXTCON_SM5502=m +CONFIG_EXTCON_USB_GPIO=m +CONFIG_EXTCON_USBC_TUSB320=m +CONFIG_MEMORY=y +CONFIG_FPGA_DFL_EMIF=m +CONFIG_IIO_SW_DEVICE=m +CONFIG_IIO_SW_TRIGGER=m +CONFIG_ADIS16201=m +CONFIG_ADIS16209=m +CONFIG_ADXL313_I2C=m +CONFIG_ADXL313_SPI=m +CONFIG_ADXL355_I2C=m +CONFIG_ADXL355_SPI=m +CONFIG_ADXL367_SPI=m +CONFIG_ADXL367_I2C=m +CONFIG_ADXL372_SPI=m +CONFIG_ADXL372_I2C=m +CONFIG_BMA220=m +CONFIG_BMA400=m +CONFIG_BMC150_ACCEL=m +CONFIG_BMI088_ACCEL=m +CONFIG_DA280=m +CONFIG_DA311=m +CONFIG_DMARD06=m +CONFIG_DMARD09=m +CONFIG_DMARD10=m +CONFIG_FXLS8962AF_I2C=m +CONFIG_FXLS8962AF_SPI=m +CONFIG_HID_SENSOR_ACCEL_3D=m +CONFIG_KXSD9=m +CONFIG_KXCJK1013=m +CONFIG_MC3230=m +CONFIG_MMA7455_I2C=m +CONFIG_MMA7455_SPI=m +CONFIG_MMA7660=m +CONFIG_MMA8452=m +CONFIG_MMA9551=m +CONFIG_MMA9553=m +CONFIG_MXC4005=m +CONFIG_MXC6255=m +CONFIG_SCA3000=m +CONFIG_SCA3300=m +CONFIG_STK8312=m +CONFIG_STK8BA50=m +CONFIG_AD7091R5=m +CONFIG_AD7124=m +CONFIG_AD7192=m +CONFIG_AD7266=m +CONFIG_AD7280=m +CONFIG_AD7291=m +CONFIG_AD7292=m +CONFIG_AD7298=m +CONFIG_AD7476=m +CONFIG_AD7606_IFACE_PARALLEL=m +CONFIG_AD7606_IFACE_SPI=m +CONFIG_AD7766=m +CONFIG_AD7768_1=m +CONFIG_AD7780=m +CONFIG_AD7791=m +CONFIG_AD7793=m +CONFIG_AD7887=m +CONFIG_AD7923=m +CONFIG_AD7949=m +CONFIG_AD799X=m +CONFIG_AD9467=m +CONFIG_ADI_AXI_ADC=m +CONFIG_AXP20X_ADC=m +CONFIG_AXP288_ADC=m +CONFIG_CC10001_ADC=m +CONFIG_CPCAP_ADC=m +CONFIG_DA9150_GPADC=m +CONFIG_DLN2_ADC=m +CONFIG_ENVELOPE_DETECTOR=m +CONFIG_HI8435=m +CONFIG_HX711=m +CONFIG_INA2XX_ADC=m +CONFIG_LP8788_ADC=m +CONFIG_LTC2471=m +CONFIG_LTC2485=m +CONFIG_LTC2496=m +CONFIG_LTC2497=m +CONFIG_MAX1027=m +CONFIG_MAX11100=m +CONFIG_MAX1118=m +CONFIG_MAX1241=m +CONFIG_MAX1363=m +CONFIG_MAX9611=m +CONFIG_MCP320X=m +CONFIG_MCP3422=m +CONFIG_MCP3911=m +CONFIG_MEDIATEK_MT6360_ADC=m +CONFIG_MEN_Z188_ADC=m +CONFIG_MP2629_ADC=m +CONFIG_NAU7802=m +CONFIG_PALMAS_GPADC=m +CONFIG_QCOM_SPMI_IADC=m +CONFIG_QCOM_SPMI_VADC=m +CONFIG_QCOM_SPMI_ADC5=m +CONFIG_RN5T618_ADC=m +CONFIG_SD_ADC_MODULATOR=m +CONFIG_STMPE_ADC=m +CONFIG_TI_ADC081C=m +CONFIG_TI_ADC0832=m +CONFIG_TI_ADC084S021=m +CONFIG_TI_ADC12138=m +CONFIG_TI_ADC108S102=m +CONFIG_TI_ADC128S052=m +CONFIG_TI_ADC161S626=m +CONFIG_TI_ADS1015=m +CONFIG_TI_ADS7950=m +CONFIG_TI_ADS8344=m +CONFIG_TI_ADS8688=m +CONFIG_TI_ADS124S08=m +CONFIG_TI_ADS131E08=m +CONFIG_TI_AM335X_ADC=m +CONFIG_TI_TLC4541=m +CONFIG_TI_TSC2046=m +CONFIG_TWL4030_MADC=m +CONFIG_TWL6030_GPADC=m +CONFIG_VF610_ADC=m +CONFIG_VIPERBOARD_ADC=m +CONFIG_XILINX_XADC=m +CONFIG_AD74413R=m +CONFIG_IIO_RESCALE=m +CONFIG_AD8366=m +CONFIG_ADA4250=m +CONFIG_HMC425=m +CONFIG_AD7150=m +CONFIG_AD7746=m +CONFIG_ATLAS_PH_SENSOR=m +CONFIG_ATLAS_EZO_SENSOR=m +CONFIG_BME680=m +CONFIG_CCS811=m +CONFIG_IAQCORE=m +CONFIG_PMS7003=m +CONFIG_SCD30_CORE=m +CONFIG_SCD30_I2C=m +CONFIG_SCD30_SERIAL=m +CONFIG_SCD4X=m +CONFIG_SENSIRION_SGP30=m +CONFIG_SENSIRION_SGP40=m +CONFIG_SPS30_I2C=m +CONFIG_SPS30_SERIAL=m +CONFIG_SENSEAIR_SUNRISE_CO2=m +CONFIG_VZ89X=m +CONFIG_IIO_SSP_SENSORS_COMMONS=m +CONFIG_IIO_SSP_SENSORHUB=m +CONFIG_AD3552R=m +CONFIG_AD5064=m +CONFIG_AD5360=m +CONFIG_AD5380=m +CONFIG_AD5421=m +CONFIG_AD5446=m +CONFIG_AD5449=m +CONFIG_AD5592R=m +CONFIG_AD5593R=m +CONFIG_AD5504=m +CONFIG_AD5624R_SPI=m +CONFIG_LTC2688=m +CONFIG_AD5686_SPI=m +CONFIG_AD5696_I2C=m +CONFIG_AD5755=m +CONFIG_AD5758=m +CONFIG_AD5761=m +CONFIG_AD5764=m +CONFIG_AD5766=m +CONFIG_AD5770R=m +CONFIG_AD5791=m +CONFIG_AD7293=m +CONFIG_AD7303=m +CONFIG_AD8801=m +CONFIG_DPOT_DAC=m +CONFIG_DS4424=m +CONFIG_LTC1660=m +CONFIG_LTC2632=m +CONFIG_M62332=m +CONFIG_MAX517=m +CONFIG_MAX5821=m +CONFIG_MCP4725=m +CONFIG_MCP4922=m +CONFIG_TI_DAC082S085=m +CONFIG_TI_DAC5571=m +CONFIG_TI_DAC7311=m +CONFIG_TI_DAC7612=m +CONFIG_VF610_DAC=m +CONFIG_IIO_SIMPLE_DUMMY=m +CONFIG_ADMV8818=m +CONFIG_AD9523=m +CONFIG_ADF4350=m +CONFIG_ADF4371=m +CONFIG_ADMV1013=m +CONFIG_ADMV1014=m +CONFIG_ADMV4420=m +CONFIG_ADRF6780=m +CONFIG_ADIS16080=m +CONFIG_ADIS16130=m +CONFIG_ADIS16136=m +CONFIG_ADIS16260=m +CONFIG_ADXRS290=m +CONFIG_ADXRS450=m +CONFIG_BMG160=m +CONFIG_FXAS21002C=m +CONFIG_HID_SENSOR_GYRO_3D=m +CONFIG_MPU3050_I2C=m +CONFIG_IIO_ST_GYRO_3AXIS=m +CONFIG_ITG3200=m +CONFIG_AFE4403=m +CONFIG_AFE4404=m +CONFIG_MAX30100=m +CONFIG_MAX30102=m +CONFIG_AM2315=m +CONFIG_DHT11=m +CONFIG_HDC100X=m +CONFIG_HDC2010=m +CONFIG_HID_SENSOR_HUMIDITY=m +CONFIG_HTS221=m +CONFIG_HTU21=m +CONFIG_SI7005=m +CONFIG_SI7020=m +CONFIG_ADIS16400=m +CONFIG_ADIS16460=m +CONFIG_ADIS16475=m +CONFIG_ADIS16480=m +CONFIG_BMI160_I2C=m +CONFIG_BMI160_SPI=m +CONFIG_FXOS8700_I2C=m +CONFIG_FXOS8700_SPI=m +CONFIG_KMX61=m +CONFIG_INV_ICM42600_I2C=m +CONFIG_INV_ICM42600_SPI=m +CONFIG_INV_MPU6050_I2C=m +CONFIG_INV_MPU6050_SPI=m +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_IIO_ST_LSM9DS0=m +CONFIG_ADJD_S311=m +CONFIG_ADUX1020=m +CONFIG_AL3010=m +CONFIG_AL3320A=m +CONFIG_APDS9300=m +CONFIG_APDS9960=m +CONFIG_AS73211=m +CONFIG_BH1750=m +CONFIG_BH1780=m +CONFIG_CM32181=m +CONFIG_CM3232=m +CONFIG_CM3323=m +CONFIG_CM3605=m +CONFIG_CM36651=m +CONFIG_GP2AP002=m +CONFIG_GP2AP020A00F=m +CONFIG_IQS621_ALS=m +CONFIG_SENSORS_ISL29018=m +CONFIG_SENSORS_ISL29028=m +CONFIG_ISL29125=m +CONFIG_HID_SENSOR_ALS=m +CONFIG_HID_SENSOR_PROX=m +CONFIG_JSA1212=m +CONFIG_RPR0521=m +CONFIG_SENSORS_LM3533=m +CONFIG_LTR501=m +CONFIG_LV0104CS=m +CONFIG_MAX44000=m +CONFIG_MAX44009=m +CONFIG_NOA1305=m +CONFIG_OPT3001=m +CONFIG_PA12203001=m +CONFIG_SI1133=m +CONFIG_SI1145=m +CONFIG_STK3310=m +CONFIG_ST_UVIS25=m +CONFIG_TCS3414=m +CONFIG_TCS3472=m +CONFIG_SENSORS_TSL2563=m +CONFIG_TSL2583=m +CONFIG_TSL2591=m +CONFIG_TSL2772=m +CONFIG_TSL4531=m +CONFIG_US5182D=m +CONFIG_VCNL4000=m +CONFIG_VCNL4035=m +CONFIG_VEML6030=m +CONFIG_VEML6070=m +CONFIG_VL6180=m +CONFIG_ZOPT2201=m +CONFIG_AK8974=m +CONFIG_AK09911=m +CONFIG_BMC150_MAGN_I2C=m +CONFIG_BMC150_MAGN_SPI=m +CONFIG_MAG3110=m +CONFIG_HID_SENSOR_MAGNETOMETER_3D=m +CONFIG_MMC35240=m +CONFIG_SENSORS_HMC5843_I2C=m +CONFIG_SENSORS_HMC5843_SPI=m +CONFIG_SENSORS_RM3100_I2C=m +CONFIG_SENSORS_RM3100_SPI=m +CONFIG_YAMAHA_YAS530=m +CONFIG_IIO_MUX=m +CONFIG_HID_SENSOR_INCLINOMETER_3D=m +CONFIG_HID_SENSOR_DEVICE_ROTATION=m +CONFIG_IIO_HRTIMER_TRIGGER=m +CONFIG_IIO_INTERRUPT_TRIGGER=m +CONFIG_IIO_TIGHTLOOP_TRIGGER=m +CONFIG_IIO_SYSFS_TRIGGER=m +CONFIG_IQS624_POS=m +CONFIG_HID_SENSOR_CUSTOM_INTEL_HINGE=m +CONFIG_AD5110=m +CONFIG_AD5272=m +CONFIG_DS1803=m +CONFIG_MAX5432=m +CONFIG_MAX5481=m +CONFIG_MAX5487=m +CONFIG_MCP4018=m +CONFIG_MCP4131=m +CONFIG_MCP4531=m +CONFIG_MCP41010=m +CONFIG_TPL0102=m +CONFIG_LMP91000=m +CONFIG_ABP060MG=m +CONFIG_BMP280=m +CONFIG_DLHL60D=m +CONFIG_DPS310=m +CONFIG_HID_SENSOR_PRESS=m +CONFIG_HP03=m +CONFIG_ICP10100=m +CONFIG_MPL115_I2C=m +CONFIG_MPL115_SPI=m +CONFIG_MPL3115=m +CONFIG_MS5611=m +CONFIG_MS5611_I2C=m +CONFIG_MS5611_SPI=m +CONFIG_MS5637=m +CONFIG_IIO_ST_PRESS=m +CONFIG_T5403=m +CONFIG_HP206C=m +CONFIG_ZPA2326=m +CONFIG_AS3935=m +CONFIG_ISL29501=m +CONFIG_LIDAR_LITE_V2=m +CONFIG_MB1232=m +CONFIG_PING=m +CONFIG_RFD77402=m +CONFIG_SRF04=m +CONFIG_SX9310=m +CONFIG_SX9324=m +CONFIG_SX9360=m +CONFIG_SX9500=m +CONFIG_SRF08=m +CONFIG_VCNL3020=m +CONFIG_VL53L0X_I2C=m +CONFIG_AD2S90=m +CONFIG_AD2S1200=m +CONFIG_IQS620AT_TEMP=m +CONFIG_LTC2983=m +CONFIG_MAXIM_THERMOCOUPLE=m +CONFIG_HID_SENSOR_TEMP=m +CONFIG_MLX90614=m +CONFIG_MLX90632=m +CONFIG_TMP006=m +CONFIG_TMP007=m +CONFIG_TMP117=m +CONFIG_TSYS01=m +CONFIG_TSYS02D=m +CONFIG_MAX31856=m +CONFIG_MAX31865=m +CONFIG_NTB=m +CONFIG_NTB_MSI=y +CONFIG_NTB_IDT=m +CONFIG_NTB_EPF=m +CONFIG_NTB_SWITCHTEC=m +CONFIG_NTB_PINGPONG=m +CONFIG_NTB_TOOL=m +CONFIG_NTB_PERF=m +CONFIG_NTB_TRANSPORT=m +CONFIG_PWM=y +CONFIG_PWM_ATMEL_HLCDC_PWM=m +CONFIG_PWM_ATMEL_TCB=m +CONFIG_PWM_DWC=m +CONFIG_PWM_FSL_FTM=m +CONFIG_PWM_IQS620A=m +CONFIG_PWM_LP3943=m +CONFIG_PWM_NTXEC=m +CONFIG_PWM_PCA9685=m +CONFIG_PWM_SIFIVE=m +CONFIG_PWM_STMPE=y +CONFIG_PWM_TWL=m +CONFIG_PWM_TWL_LED=m +CONFIG_PWM_XILINX=m +CONFIG_AL_FIC=y +CONFIG_XILINX_INTC=y +CONFIG_IPACK_BUS=m +CONFIG_BOARD_TPCI200=m +CONFIG_SERIAL_IPOCTAL=m +CONFIG_RESET_TI_SYSCON=m +CONFIG_PHY_CAN_TRANSCEIVER=m +CONFIG_BCM_KONA_USB2_PHY=m +CONFIG_PHY_CADENCE_TORRENT=m +CONFIG_PHY_CADENCE_DPHY=m +CONFIG_PHY_CADENCE_DPHY_RX=m +CONFIG_PHY_CADENCE_SIERRA=m +CONFIG_PHY_CADENCE_SALVO=m +CONFIG_PHY_PXA_28NM_HSIC=m +CONFIG_PHY_PXA_28NM_USB2=m +CONFIG_PHY_LAN966X_SERDES=m +CONFIG_PHY_CPCAP_USB=m +CONFIG_PHY_MAPPHONE_MDM6600=m +CONFIG_PHY_OCELOT_SERDES=m +CONFIG_PHY_QCOM_USB_HS=m +CONFIG_PHY_QCOM_USB_HSIC=m +CONFIG_PHY_SAMSUNG_USB2=m +CONFIG_PHY_TUSB1210=m +CONFIG_POWERCAP=y +CONFIG_IDLE_INJECT=y +CONFIG_DTPM=y +CONFIG_MCB=m +CONFIG_MCB_PCI=m +CONFIG_MCB_LPC=m +CONFIG_USB4=m +CONFIG_LIBNVDIMM=y +CONFIG_BLK_DEV_PMEM=m +CONFIG_OF_PMEM=m +CONFIG_DAX=y +CONFIG_DEV_DAX=m +CONFIG_NVMEM_RMEM=m +CONFIG_NVMEM_SPMI_SDAM=m +CONFIG_STM=m +CONFIG_STM_PROTO_BASIC=m +CONFIG_STM_PROTO_SYS_T=m +CONFIG_STM_DUMMY=m +CONFIG_STM_SOURCE_CONSOLE=m +CONFIG_STM_SOURCE_HEARTBEAT=m +CONFIG_STM_SOURCE_FTRACE=m +CONFIG_INTEL_TH=m +CONFIG_INTEL_TH_PCI=m +CONFIG_INTEL_TH_GTH=m +CONFIG_INTEL_TH_STH=m +CONFIG_INTEL_TH_MSU=m +CONFIG_INTEL_TH_PTI=m +CONFIG_FPGA=m +CONFIG_ALTERA_PR_IP_CORE=m +CONFIG_ALTERA_PR_IP_CORE_PLAT=m +CONFIG_FPGA_MGR_ALTERA_PS_SPI=m +CONFIG_FPGA_MGR_ALTERA_CVP=m +CONFIG_FPGA_MGR_XILINX_SPI=m +CONFIG_FPGA_MGR_ICE40_SPI=m +CONFIG_FPGA_MGR_MACHXO2_SPI=m +CONFIG_ALTERA_FREEZE_BRIDGE=m +CONFIG_XILINX_PR_DECOUPLER=m +CONFIG_OF_FPGA_REGION=m +CONFIG_FPGA_DFL=m +CONFIG_FPGA_DFL_FME=m +CONFIG_FPGA_DFL_FME_MGR=m +CONFIG_FPGA_DFL_FME_BRIDGE=m +CONFIG_FPGA_DFL_FME_REGION=m +CONFIG_FPGA_DFL_AFU=m +CONFIG_FPGA_DFL_NIOS_INTEL_PAC_N3000=m +CONFIG_FPGA_DFL_PCI=m +CONFIG_FSI=m +CONFIG_FSI_MASTER_GPIO=m +CONFIG_FSI_MASTER_HUB=m +CONFIG_FSI_MASTER_ASPEED=m +CONFIG_FSI_SCOM=m +CONFIG_FSI_SBEFIFO=m +CONFIG_FSI_OCC=m +CONFIG_MUX_ADG792A=m +CONFIG_MUX_ADGS1408=m +CONFIG_MUX_GPIO=m +CONFIG_MUX_MMIO=m +CONFIG_SIOX=m +CONFIG_SIOX_BUS_GPIO=m +CONFIG_SLIM_QCOM_CTRL=m +CONFIG_INTERCONNECT=y +CONFIG_MOST=m +CONFIG_MOST_USB_HDM=m +CONFIG_MOST_CDEV=m +CONFIG_MOST_SND=m +CONFIG_PECI=m +CONFIG_VALIDATE_FS_PARSER=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_REISERFS_FS=m +CONFIG_REISERFS_FS_XATTR=y +CONFIG_REISERFS_FS_POSIX_ACL=y +CONFIG_REISERFS_FS_SECURITY=y +CONFIG_JFS_FS=m +CONFIG_JFS_POSIX_ACL=y +CONFIG_JFS_SECURITY=y +CONFIG_JFS_STATISTICS=y +CONFIG_XFS_FS=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_RT=y +CONFIG_GFS2_FS=m +CONFIG_GFS2_FS_LOCKING_DLM=y +CONFIG_OCFS2_FS=m +CONFIG_BTRFS_FS=m +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_NILFS2_FS=m +CONFIG_F2FS_FS=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_COMPRESSION=y +# CONFIG_F2FS_IOSTAT is not set +CONFIG_F2FS_UNFAIR_RWSEM=y +CONFIG_ZONEFS_FS=m +CONFIG_FS_ENCRYPTION=y +CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y +CONFIG_FS_VERITY=y +CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +# CONFIG_PRINT_QUOTA_WARNING is not set +CONFIG_QFMT_V1=m +CONFIG_QFMT_V2=y +CONFIG_AUTOFS4_FS=m +CONFIG_FUSE_FS=y +CONFIG_CUSE=m +CONFIG_VIRTIO_FS=m +CONFIG_OVERLAY_FS=m +CONFIG_OVERLAY_FS_XINO_AUTO=y +CONFIG_FSCACHE=m +CONFIG_FSCACHE_STATS=y +CONFIG_CACHEFILES=m +CONFIG_CACHEFILES_ERROR_INJECTION=y +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=m +CONFIG_MSDOS_FS=m +CONFIG_VFAT_FS=y +CONFIG_EXFAT_FS=m +CONFIG_NTFS_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y +CONFIG_NTFS3_FS_POSIX_ACL=y +CONFIG_PROC_KCORE=y +CONFIG_PROC_VMCORE_DEVICE_DUMP=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_TMPFS_INODE64=y +CONFIG_HUGETLBFS=y +CONFIG_EFIVAR_FS=y +CONFIG_ORANGEFS_FS=m +CONFIG_ADFS_FS=m +CONFIG_AFFS_FS=m +CONFIG_ECRYPT_FS=y +CONFIG_ECRYPT_FS_MESSAGING=y +CONFIG_HFS_FS=m +CONFIG_HFSPLUS_FS=m +CONFIG_BEFS_FS=m +CONFIG_BFS_FS=m +CONFIG_EFS_FS=m +CONFIG_JFFS2_FS=m +CONFIG_JFFS2_FS_XATTR=y +CONFIG_JFFS2_COMPRESSION_OPTIONS=y +CONFIG_JFFS2_LZO=y +CONFIG_JFFS2_CMODE_FAVOURLZO=y +CONFIG_UBIFS_FS=m +CONFIG_UBIFS_FS_AUTHENTICATION=y +CONFIG_CRAMFS=m +CONFIG_CRAMFS_MTD=y +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_FILE_DIRECT=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_VXFS_FS=m +CONFIG_MINIX_FS=m +CONFIG_OMFS_FS=m +CONFIG_HPFS_FS=m +CONFIG_QNX4FS_FS=m +CONFIG_QNX6FS_FS=m +CONFIG_ROMFS_FS=m +CONFIG_PSTORE=y +CONFIG_PSTORE_RAM=m +CONFIG_PSTORE_BLK=m +CONFIG_SYSV_FS=m +CONFIG_UFS_FS=m +CONFIG_EROFS_FS=m +CONFIG_NFS_FS=m +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=m +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_BLOCKLAYOUT=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_FLEXFILELAYOUT=y +CONFIG_NFSD_V4_2_INTER_SSC=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_SUNRPC_DEBUG=y +CONFIG_CEPH_FS=m +CONFIG_CEPH_FSCACHE=y +CONFIG_CEPH_FS_POSIX_ACL=y +CONFIG_CEPH_FS_SECURITY_LABEL=y +CONFIG_CIFS=m +# CONFIG_CIFS_STATS2 is not set +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_SWN_UPCALL=y +CONFIG_CIFS_FSCACHE=y +CONFIG_SMB_SERVER=m +CONFIG_SMB_SERVER_SMBDIRECT=y +CONFIG_SMB_SERVER_KERBEROS5=y +CONFIG_CODA_FS=m +CONFIG_AFS_FS=m +CONFIG_AFS_FSCACHE=y +CONFIG_9P_FS=y +CONFIG_9P_FSCACHE=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_DLM=m +CONFIG_UNICODE=y +CONFIG_KEYS_REQUEST_CACHE=y +CONFIG_PERSISTENT_KEYRINGS=y +CONFIG_TRUSTED_KEYS=y +CONFIG_USER_DECRYPTED_DATA=y +CONFIG_KEY_DH_OPERATIONS=y +CONFIG_KEY_NOTIFICATIONS=y +CONFIG_SECURITY_DMESG_RESTRICT=y +CONFIG_SECURITY=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_LSM_MMAP_MIN_ADDR=0 +CONFIG_HARDENED_USERCOPY=y +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_SELINUX_BOOTPARAM=y +CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1 +CONFIG_SECURITY_SMACK=y +CONFIG_SECURITY_SMACK_NETFILTER=y +CONFIG_SECURITY_SMACK_APPEND_SIGNALS=y +CONFIG_SECURITY_TOMOYO=y +CONFIG_SECURITY_APPARMOR=y +CONFIG_SECURITY_YAMA=y +CONFIG_SECURITY_SAFESETID=y +CONFIG_SECURITY_LOCKDOWN_LSM=y +CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y +CONFIG_SECURITY_LANDLOCK=y +CONFIG_INTEGRITY_SIGNATURE=y +CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y +CONFIG_INTEGRITY_PLATFORM_KEYRING=y +CONFIG_INTEGRITY_MACHINE_KEYRING=y +CONFIG_IMA=y +CONFIG_IMA_KEXEC=y +CONFIG_IMA_APPRAISE=y +CONFIG_IMA_ARCH_POLICY=y +CONFIG_IMA_APPRAISE_MODSIG=y +CONFIG_EVM=y +CONFIG_EVM_EXTRA_SMACK_XATTRS=y +CONFIG_EVM_ADD_XATTRS=y +CONFIG_DEFAULT_SECURITY_APPARMOR=y +CONFIG_LSM="landlock,lockdown,yama,integrity,apparmor" +CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y +CONFIG_CRYPTO_USER=m +CONFIG_CRYPTO_PCRYPT=m +CONFIG_CRYPTO_CRYPTD=m +CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_DH_RFC7919_GROUPS=y +CONFIG_CRYPTO_ECDSA=m +CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m +CONFIG_CRYPTO_CURVE25519=m +CONFIG_CRYPTO_AES_TI=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_DES=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_CFB=m +CONFIG_CRYPTO_KEYWRAP=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_OFB=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_842=m +CONFIG_CRYPTO_LZ4=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ANSI_CPRNG=m +CONFIG_CRYPTO_DRBG_HASH=y +CONFIG_CRYPTO_DRBG_CTR=y +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +# CONFIG_CRYPTO_USER_API_ENABLE_OBSOLETE is not set +CONFIG_CRYPTO_STATS=y +CONFIG_CRYPTO_DEV_NITROX_CNN55XX=m +CONFIG_CRYPTO_DEV_CHELSIO=m +CONFIG_CRYPTO_DEV_VIRTIO=m +CONFIG_CRYPTO_DEV_CCREE=m +CONFIG_CRYPTO_DEV_AMLOGIC_GXL=m +CONFIG_PKCS8_PRIVATE_KEY_PARSER=m +CONFIG_PKCS7_TEST_KEY=m +CONFIG_SIGNED_PE_FILE_VERIFICATION=y +CONFIG_SYSTEM_EXTRA_CERTIFICATE=y +CONFIG_SECONDARY_TRUSTED_KEYRING=y +CONFIG_SYSTEM_BLACKLIST_KEYRING=y +CONFIG_SYSTEM_REVOCATION_LIST=y +CONFIG_XZ_DEC_MICROLZMA=y +CONFIG_XZ_DEC_TEST=m +CONFIG_DMA_RESTRICTED_POOL=y +CONFIG_DMA_CMA=y +CONFIG_DMA_PERNUMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=32 +CONFIG_FONTS=y +CONFIG_FONT_8x8=y +CONFIG_FONT_ACORN_8x8=y +CONFIG_FONT_6x10=y +CONFIG_FONT_TER16x32=y +CONFIG_PRINTK_TIME=y +CONFIG_BOOT_PRINTK_DELAY=y +CONFIG_DYNAMIC_DEBUG=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y +CONFIG_GDB_SCRIPTS=y +CONFIG_FRAME_WARN=1024 +CONFIG_VMLINUX_MAP=y +CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x01b6 +CONFIG_KGDB=y +CONFIG_KGDB_KDB=y +CONFIG_KDB_KEYBOARD=y +CONFIG_PAGE_POISONING=y +CONFIG_DEBUG_WX=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_KFENCE=y +CONFIG_KFENCE_SAMPLE_INTERVAL=0 +CONFIG_SOFTLOCKUP_DETECTOR=y +CONFIG_SCHEDSTATS=y +CONFIG_RCU_CPU_STALL_TIMEOUT=60 +# CONFIG_RCU_TRACE is not set +CONFIG_BOOTTIME_TRACING=y +CONFIG_FUNCTION_PROFILER=y +CONFIG_STACK_TRACER=y +CONFIG_SCHED_TRACER=y +CONFIG_HWLAT_TRACER=y +CONFIG_FTRACE_SYSCALLS=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_BPF_KPROBE_OVERRIDE=y +CONFIG_SYNTH_EVENTS=y +CONFIG_TRACE_EVENT_INJECT=y +CONFIG_SAMPLES=y +CONFIG_SAMPLE_TRACE_PRINTK=m +CONFIG_SAMPLE_TRACE_ARRAY=m +CONFIG_NOTIFIER_ERROR_INJECTION=m +CONFIG_FUNCTION_ERROR_INJECTION=y +CONFIG_TEST_BPF=m +CONFIG_TEST_BLACKHOLE_DEV=m +CONFIG_MEMTEST=y +CONFIG_RELOCATABLE=y +CONFIG_RANDOMIZE_BASE=y +CONFIG_ERRATA_THEAD_MAE=y +CONFIG_ERRATA_THEAD=y +# DEBUG configs +CONFIG_DEBUG_ATOMIC_SLEEP=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_LOCK_ALLOC=y +CONFIG_DEBUG_RWSEMS=y +CONFIG_DEBUG_LOCKDEP=y +CONFIG_DEBUG_IRQFLAGS=y +CONFIG_DEBUG_VM=y +CONFIG_PROVE_LOCKING=y +CONFIG_SLUB_DEBUG=y +CONFIG_SLUB_DEBUG_ON=y +CONFIG_DEBUG_LIST=y +CONFIG_DEBUG_PAGEALLOC=y +CONFIG_DEBUG_PAGEALLOC_DEFAULT=y +CONFIG_DEBUG_STACK_USAGE=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y diff --git a/.github/scripts/series/kernel_builder.sh b/.github/scripts/series/kernel_builder.sh new file mode 100755 index 00000000000000..98837bf7d76e03 --- /dev/null +++ b/.github/scripts/series/kernel_builder.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 + +tm=$(mktemp -p ${ci_root}) +n=$(gen_kernel_name $xlen $config $fragment $toolchain) +logs=$(get_logs_dir) +rc=0 +log="build_kernel___${n}.log" +\time --quiet -o $tm -f "took %es" \ + $d/build_kernel.sh "${xlen}" "${config}" "${fragment}" "${toolchain}" &> "${logs}/${log}" || rc=$? + +if grep -a ": warning:" "${logs}/${log}" | grep -qv "frame size"; then + # TODO Can't get rid of LLVM "warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]" + if [[ ! "${log}" =~ "nommu" ]]; then + echo "::error::FAIL WARNINGS kernel ${n} \"${log}\" $(cat $tm)" + fi +elif (( $rc )); then + echo "::error::FAIL Build kernel ${n} \"${log}\" $(cat $tm)" +else + echo "::notice::OK Build kernel ${n} $(cat $tm)" +fi +rm $tm +exit $rc diff --git a/.github/scripts/series/kernel_tester.sh b/.github/scripts/series/kernel_tester.sh new file mode 100755 index 00000000000000..d3e5bd99e9a9d7 --- /dev/null +++ b/.github/scripts/series/kernel_tester.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh +. $d/qemu_test_utils.sh + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 +rootfs=$5 + +generate_qemu_subtests $xlen $config $fragment $toolchain $rootfs + +tm=$(mktemp -p ${ci_root}) +n=$(gen_kernel_name $xlen $config $fragment $toolchain) +logs=$(get_logs_dir) +tot=${#qemu_subtests[@]} +allrc=0 +for i in $(seq $tot); do + rc=0 + tstn=$(get_qemu_test_name ${qemu_subtests[$(($i - 1))]}) + tst=${qemu_subtests[$(($i - 1))]} + + log="test_kernel___${n}___${rootfs}___${tstn}.log" + log=${log//\//-} + \time --quiet -o $tm -f "took %es" \ + $d/test_kernel.sh "${xlen}" "${config}" "${fragment}" "${toolchain}" "${rootfs}" \ + $tst &> "${logs}/${log}" || rc=$? + if (( $rc )); then + allrc=1 + echo "::error::FAIL Test kernel ${n} ${rootfs} ${tst} $i/$tot \"${log}\" $(cat $tm)" + else + echo "::notice::OK Test kernel ${n} ${rootfs} ${tst} $i/$tot $(cat $tm)" + fi +done +rm $tm +exit $allrc diff --git a/.github/scripts/series/kselftest_prep.sh b/.github/scripts/series/kselftest_prep.sh new file mode 100644 index 00000000000000..5f6a5adf1a68ca --- /dev/null +++ b/.github/scripts/series/kselftest_prep.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Please source me! + +git config --global user.email "you@example.com" +git config --global user.name "Your Name" + +export PATH=$(echo $PATH | tr : "\n"| grep -v ^/opt | tr "\n" :) + +export CI_TRIPLE=riscv64-linux-gnu diff --git a/.github/scripts/series/patches/0001-selftests-bpf-Add-RISC-V-specific-config.patch b/.github/scripts/series/patches/0001-selftests-bpf-Add-RISC-V-specific-config.patch new file mode 100644 index 00000000000000..a4aec87eb24532 --- /dev/null +++ b/.github/scripts/series/patches/0001-selftests-bpf-Add-RISC-V-specific-config.patch @@ -0,0 +1,47 @@ +From 611f4103494e5847190d8d8b22c15a496441c075 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 27 Aug 2024 17:21:12 +0000 +Subject: [PATCH 1/6] selftests: bpf: Add RISC-V specific config +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/bpf/config.riscv64 | 22 ++++++++++++++++++++++ + 1 file changed, 22 insertions(+) + create mode 100644 tools/testing/selftests/bpf/config.riscv64 + +diff --git a/tools/testing/selftests/bpf/config.riscv64 b/tools/testing/selftests/bpf/config.riscv64 +new file mode 100644 +index 000000000000..9124d1549a1a +--- /dev/null ++++ b/tools/testing/selftests/bpf/config.riscv64 +@@ -0,0 +1,22 @@ ++CONFIG_DEBUG_CREDENTIALS=y ++CONFIG_DEBUG_INFO_BTF=y ++CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y ++CONFIG_DEBUG_MEMORY_INIT=y ++CONFIG_EXPERT=y ++CONFIG_RUNTIME_TESTING_MENU=y ++CONFIG_TLS=y ++CONFIG_IP_ROUTE_MULTIPATH=y ++CONFIG_MACSEC=y ++CONFIG_CRYPTO_CHACHA20POLY1305=y ++CONFIG_CRYPTO_CHACHA20=y ++CONFIG_CRYPTO_CCM=y ++CONFIG_MACVTAP=y ++CONFIG_L2TP=y ++CONFIG_TEST_FPU=y ++CONFIG_IOMMUFD=y ++CONFIG_IPV6_SEG6_LWTUNNEL=y ++CONFIG_XFRM_SUB_POLICY=y ++CONFIG_XFRM_USER=y ++CONFIG_OPENVSWITCH=y ++CONFIG_NONPORTABLE=y ++CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS=y + +base-commit: 8d8d276ba2fb5f9ac4984f5c10ae60858090babc +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0002-selftests-bpf-Rename-fallback.patch b/.github/scripts/series/patches/0002-selftests-bpf-Rename-fallback.patch new file mode 100644 index 00000000000000..2fef6472ad3fa9 --- /dev/null +++ b/.github/scripts/series/patches/0002-selftests-bpf-Rename-fallback.patch @@ -0,0 +1,70 @@ +From af4e24f21714ca1ffdedeba5fd2e587328b35823 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Thu, 22 Aug 2024 16:26:22 +0000 +Subject: [PATCH 2/6] selftests: bpf: Rename fallback +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +There's a clash: + +arch/riscv/lib/crc32.c +154:typedef u32 (*fallback)(u32 crc, unsigned char const *p, size_t len); + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c | 2 +- + tools/testing/selftests/bpf/progs/bpf_dctcp.c | 8 ++++---- + 2 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +index 63422f4f3896..860df52717f0 100644 +--- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c ++++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +@@ -285,7 +285,7 @@ static void test_dctcp_fallback(void) + dctcp_skel = bpf_dctcp__open(); + if (!ASSERT_OK_PTR(dctcp_skel, "dctcp_skel")) + return; +- strcpy(dctcp_skel->rodata->fallback, "cubic"); ++ strcpy(dctcp_skel->rodata->fallback_s, "cubic"); + if (!ASSERT_OK(bpf_dctcp__load(dctcp_skel), "bpf_dctcp__load")) + goto done; + +diff --git a/tools/testing/selftests/bpf/progs/bpf_dctcp.c b/tools/testing/selftests/bpf/progs/bpf_dctcp.c +index 02f552e7fd4d..aaa74d5fbd86 100644 +--- a/tools/testing/selftests/bpf/progs/bpf_dctcp.c ++++ b/tools/testing/selftests/bpf/progs/bpf_dctcp.c +@@ -26,7 +26,7 @@ static bool before(__u32 seq1, __u32 seq2) + + char _license[] SEC("license") = "GPL"; + +-volatile const char fallback[TCP_CA_NAME_MAX]; ++volatile const char fallback_s[TCP_CA_NAME_MAX]; + const char bpf_dctcp[] = "bpf_dctcp"; + const char tcp_cdg[] = "cdg"; + char cc_res[TCP_CA_NAME_MAX]; +@@ -71,10 +71,10 @@ void BPF_PROG(bpf_dctcp_init, struct sock *sk) + struct bpf_dctcp *ca = inet_csk_ca(sk); + int *stg; + +- if (!(tp->ecn_flags & TCP_ECN_OK) && fallback[0]) { ++ if (!(tp->ecn_flags & TCP_ECN_OK) && fallback_s[0]) { + /* Switch to fallback */ + if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, +- (void *)fallback, sizeof(fallback)) == -EBUSY) ++ (void *)fallback_s, sizeof(fallback_s)) == -EBUSY) + ebusy_cnt++; + + /* Switch back to myself and the recurred bpf_dctcp_init() +@@ -87,7 +87,7 @@ void BPF_PROG(bpf_dctcp_init, struct sock *sk) + + /* Switch back to fallback */ + if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, +- (void *)fallback, sizeof(fallback)) == -EBUSY) ++ (void *)fallback_s, sizeof(fallback_s)) == -EBUSY) + ebusy_cnt++; + + /* Expecting -ENOTSUPP for tcp_cdg_res */ +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0003-selftests-iommu-Add-RISC-V-specific-config.patch b/.github/scripts/series/patches/0003-selftests-iommu-Add-RISC-V-specific-config.patch new file mode 100644 index 00000000000000..f99e2824232452 --- /dev/null +++ b/.github/scripts/series/patches/0003-selftests-iommu-Add-RISC-V-specific-config.patch @@ -0,0 +1,30 @@ +From 06586b3f06e5f16504967407126684b5d78aed60 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 27 Aug 2024 17:27:18 +0000 +Subject: [PATCH 3/6] selftests: iommu: Add RISC-V specific config +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/iommu/config.riscv64 | 7 +++++++ + 1 file changed, 7 insertions(+) + create mode 100644 tools/testing/selftests/iommu/config.riscv64 + +diff --git a/tools/testing/selftests/iommu/config.riscv64 b/tools/testing/selftests/iommu/config.riscv64 +new file mode 100644 +index 000000000000..d218811ccadf +--- /dev/null ++++ b/tools/testing/selftests/iommu/config.riscv64 +@@ -0,0 +1,7 @@ ++# IOMMUFD ++CONFIG_IOMMUFD=y ++CONFIG_VFIO=y ++CONFIG_VFIO_CONTAINER=n ++CONFIG_IOMMUFD_VFIO_CONTAINER=y ++CONFIG_FAULT_INJECTION=y ++CONFIG_IOMMUFD_TEST=y +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0004-selftests-exec-Remove-static-pie.patch b/.github/scripts/series/patches/0004-selftests-exec-Remove-static-pie.patch new file mode 100644 index 00000000000000..58bfa6065b5077 --- /dev/null +++ b/.github/scripts/series/patches/0004-selftests-exec-Remove-static-pie.patch @@ -0,0 +1,40 @@ +From 0e22720f7f4c1cbed83c032ca4b8bc50d2f7aeb5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Thu, 22 Aug 2024 16:27:16 +0000 +Subject: [PATCH 4/6] selftests: exec: Remove static-pie +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +static-pie is broken on RV pre libc6-dev 2.40 + +WIP: Fix patch + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/exec/Makefile | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/tools/testing/selftests/exec/Makefile b/tools/testing/selftests/exec/Makefile +index ba012bc5aab9..fcf2db80cc40 100644 +--- a/tools/testing/selftests/exec/Makefile ++++ b/tools/testing/selftests/exec/Makefile +@@ -5,7 +5,7 @@ CFLAGS += -Wno-nonnull + ALIGNS := 0x1000 0x200000 0x1000000 + ALIGN_PIES := $(patsubst %,load_address.%,$(ALIGNS)) + ALIGN_STATIC_PIES := $(patsubst %,load_address.static.%,$(ALIGNS)) +-ALIGNMENT_TESTS := $(ALIGN_PIES) $(ALIGN_STATIC_PIES) ++ALIGNMENT_TESTS := $(ALIGN_PIES) + + TEST_PROGS := binfmt_script.py + TEST_GEN_PROGS := execveat non-regular $(ALIGNMENT_TESTS) +@@ -35,6 +35,3 @@ $(OUTPUT)/execveat.denatured: $(OUTPUT)/execveat + $(OUTPUT)/load_address.0x%: load_address.c + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=$(lastword $(subst ., ,$@)) \ + -fPIE -pie $< -o $@ +-$(OUTPUT)/load_address.static.0x%: load_address.c +- $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=$(lastword $(subst ., ,$@)) \ +- -fPIE -static-pie $< -o $@ +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0005-selftests-hid-Fix-broken-build.patch b/.github/scripts/series/patches/0005-selftests-hid-Fix-broken-build.patch new file mode 100644 index 00000000000000..d8c1ec966ac76b --- /dev/null +++ b/.github/scripts/series/patches/0005-selftests-hid-Fix-broken-build.patch @@ -0,0 +1,30 @@ +From 350c424f2379572fcdc9e6470a02e61e2da5441a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Thu, 22 Aug 2024 16:28:10 +0000 +Subject: [PATCH 5/6] selftests: hid: Fix broken build +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +kexec prototypes are messed up in vmlinux.h w/o the define + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/hid/progs/hid_bpf_helpers.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h +index e5db897586bb..76e06cab1b9d 100644 +--- a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h ++++ b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h +@@ -22,6 +22,7 @@ + #define HID_REQ_SET_IDLE HID_REQ_SET_IDLE___not_used + #define HID_REQ_SET_PROTOCOL HID_REQ_SET_PROTOCOL___not_used + ++#define BPF_NO_KFUNC_PROTOTYPES + #include "vmlinux.h" + + #undef hid_bpf_ctx +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0006-Add-missing-net-lib-kselftest-target.patch b/.github/scripts/series/patches/0006-Add-missing-net-lib-kselftest-target.patch new file mode 100644 index 00000000000000..41011c5b9a466f --- /dev/null +++ b/.github/scripts/series/patches/0006-Add-missing-net-lib-kselftest-target.patch @@ -0,0 +1,28 @@ +From a28da2534fd8273eab88228749786ddfdd463184 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Wed, 11 Sep 2024 09:53:21 +0200 +Subject: [PATCH 6/6] Add missing net/lib kselftest target +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/Makefile | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile +index bc8fe9e8f7f2..cf166c64a5d3 100644 +--- a/tools/testing/selftests/Makefile ++++ b/tools/testing/selftests/Makefile +@@ -64,6 +64,7 @@ TARGETS += net + TARGETS += net/af_unix + TARGETS += net/forwarding + TARGETS += net/hsr ++TARGETS += net/lib + TARGETS += net/mptcp + TARGETS += net/openvswitch + TARGETS += net/tcp_ao +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0007-BPF-kselftest-fix.patch b/.github/scripts/series/patches/0007-BPF-kselftest-fix.patch new file mode 100644 index 00000000000000..5298f4916b916c --- /dev/null +++ b/.github/scripts/series/patches/0007-BPF-kselftest-fix.patch @@ -0,0 +1,32 @@ +From 597d0700c8c55ef6a627ff6ecbc2d3c51185d570 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Sat, 14 Sep 2024 09:49:14 +0200 +Subject: [PATCH 1/2] BPF kselftest fix +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Remove dead script + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/bpf/Makefile | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile +index f04af11df8eb..df75f1beb731 100644 +--- a/tools/testing/selftests/bpf/Makefile ++++ b/tools/testing/selftests/bpf/Makefile +@@ -132,7 +132,6 @@ TEST_PROGS := test_kmod.sh \ + test_tunnel.sh \ + test_lwt_seg6local.sh \ + test_lirc_mode2.sh \ +- test_skb_cgroup_id.sh \ + test_flow_dissector.sh \ + test_xdp_vlan_mode_generic.sh \ + test_xdp_vlan_mode_native.sh \ + +base-commit: 5277d130947ba8c0d54c16eed89eb97f0b6d2e5a +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0008-BPF-selftest-install.patch b/.github/scripts/series/patches/0008-BPF-selftest-install.patch new file mode 100644 index 00000000000000..c10f52e25bc8b4 --- /dev/null +++ b/.github/scripts/series/patches/0008-BPF-selftest-install.patch @@ -0,0 +1,35 @@ +From 5fd0e251ba091bddb302fd56436007dbf50f450f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Sun, 15 Sep 2024 16:58:21 +0200 +Subject: [PATCH 2/2] BPF selftest install +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix deps Makefile + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/bpf/Makefile | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile +index df75f1beb731..f6e38b9d07f6 100644 +--- a/tools/testing/selftests/bpf/Makefile ++++ b/tools/testing/selftests/bpf/Makefile +@@ -627,6 +627,12 @@ $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TR + $(notdir %.skel.h): $(TRUNNER_OUTPUT)/%.skel.h + @true + ++$(notdir %.lskel.h): $(TRUNNER_OUTPUT)/%.lskel.h ++ @true ++ ++$(notdir %.subskel.h): $(TRUNNER_OUTPUT)/%.subskel.h ++ @true ++ + endif + + # ensure we set up tests.h header generation rule just once +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0009-selftests-clone3-Avoid-fragile-struct-poking.patch b/.github/scripts/series/patches/0009-selftests-clone3-Avoid-fragile-struct-poking.patch new file mode 100644 index 00000000000000..d4c5d315f13dac --- /dev/null +++ b/.github/scripts/series/patches/0009-selftests-clone3-Avoid-fragile-struct-poking.patch @@ -0,0 +1,66 @@ +From c70c39b2ee707075b40c91e5047cf33406f30f30 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Thu, 12 Sep 2024 19:20:34 +0000 +Subject: [PATCH] selftests: clone3: Avoid fragile struct poking +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +To: Adrian Reber , + Christian Brauner , + Shuah Khan , + linux-kselftest@vger.kernel.org +Cc: linux-kernel@vger.kernel.org + +The struct libcap in the selftest, and the what libcap is actually +using doesn't match, so the test fail. Libcap internals have changed, +but the selftest hasn't. + +Fix by avoid poking the libcap internals. + +Signed-off-by: Björn Töpel +--- + .../clone3/clone3_cap_checkpoint_restore.c | 17 +++-------------- + 1 file changed, 3 insertions(+), 14 deletions(-) + +diff --git a/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c b/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c +index 31b56d625655..9e6ee8c5316e 100644 +--- a/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c ++++ b/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c +@@ -89,14 +89,9 @@ static int test_clone3_set_tid(struct __test_metadata *_metadata, + return ret; + } + +-struct libcap { +- struct __user_cap_header_struct hdr; +- struct __user_cap_data_struct data[2]; +-}; +- + static int set_capability(void) + { +- cap_value_t cap_values[] = { CAP_SETUID, CAP_SETGID }; ++ cap_value_t cap_values[] = { CAP_SETUID, CAP_SETGID, CAP_CHECKPOINT_RESTORE }; + struct libcap *cap; + int ret = -1; + cap_t caps; +@@ -113,14 +108,8 @@ static int set_capability(void) + goto out; + } + +- cap_set_flag(caps, CAP_EFFECTIVE, 2, cap_values, CAP_SET); +- cap_set_flag(caps, CAP_PERMITTED, 2, cap_values, CAP_SET); +- +- cap = (struct libcap *) caps; +- +- /* 40 -> CAP_CHECKPOINT_RESTORE */ +- cap->data[1].effective |= 1 << (40 - 32); +- cap->data[1].permitted |= 1 << (40 - 32); ++ cap_set_flag(caps, CAP_EFFECTIVE, 3, cap_values, CAP_SET); ++ cap_set_flag(caps, CAP_PERMITTED, 3, cap_values, CAP_SET); + + if (cap_set_proc(caps)) { + perror("cap_set_proc"); + +base-commit: 075dbe9f6e3c21596c5245826a4ee1f1c1676eb8 +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0010-selftests-clone3-Avoid-fragile-struct-poking.patch b/.github/scripts/series/patches/0010-selftests-clone3-Avoid-fragile-struct-poking.patch new file mode 100644 index 00000000000000..4800187f5ec109 --- /dev/null +++ b/.github/scripts/series/patches/0010-selftests-clone3-Avoid-fragile-struct-poking.patch @@ -0,0 +1,48 @@ +From dd9191caeafccf7fd4b282aa8a516e594b2cc4fe Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Fri, 27 Sep 2024 15:01:25 +0200 +Subject: [PATCH bpf-next 1/2] libbpf: Add missing per-arch include path +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +To: Andrii Nakryiko , + Eduard Zingerman , + Mykola Lysenko , + bpf@vger.kernel.org, + netdev@vger.kernel.org +Cc: linux-kselftest@vger.kernel.org, + linux-kernel@vger.kernel.org, + linux-riscv@lists.infradead.org, + Charlie Jenkins + +libbpf does not include the per-arch tools include path, e.g. +tools/arch/riscv/include. Some architectures depend those files to +build properly. + +Include tools/arch/$(SUBARCH)/include in the libbpf build. + +Fixes: 6d74d178fe6e ("tools: Add riscv barrier implementation") +Signed-off-by: Björn Töpel +--- + tools/lib/bpf/Makefile | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile +index 1b22f0f37288..857a5f7b413d 100644 +--- a/tools/lib/bpf/Makefile ++++ b/tools/lib/bpf/Makefile +@@ -61,7 +61,8 @@ ifndef VERBOSE + endif + + INCLUDES = -I$(or $(OUTPUT),.) \ +- -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi ++ -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi \ ++ -I$(srctree)/tools/arch/$(SRCARCH)/include + + export prefix libdir src obj + + +base-commit: db5ca265e3334b48c4e3fa07eef79e8bc578c430 +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0011-selftests-bpf-Add-missing-per-arch-include-path.patch b/.github/scripts/series/patches/0011-selftests-bpf-Add-missing-per-arch-include-path.patch new file mode 100644 index 00000000000000..8430c2762afe76 --- /dev/null +++ b/.github/scripts/series/patches/0011-selftests-bpf-Add-missing-per-arch-include-path.patch @@ -0,0 +1,54 @@ +From 14733cc5a0896de29d8f4f25d80bb74793659bb1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Fri, 27 Sep 2024 15:05:32 +0200 +Subject: [PATCH bpf-next 2/2] selftests: bpf: Add missing per-arch include + path +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +To: Andrii Nakryiko , + Eduard Zingerman , + Mykola Lysenko , + bpf@vger.kernel.org, + netdev@vger.kernel.org +Cc: linux-kselftest@vger.kernel.org, + linux-kernel@vger.kernel.org, + linux-riscv@lists.infradead.org, + Charlie Jenkins + +The prog_tests programs do not include the per-arch tools include +path, e.g. tools/arch/riscv/include. Some architectures depend those +files to build properly. + +Include tools/arch/$(SUBARCH)/include in the selftests bpf build. + +Fixes: 6d74d178fe6e ("tools: Add riscv barrier implementation") +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/bpf/Makefile | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile +index 365740f24d2e..d6a53afa449f 100644 +--- a/tools/testing/selftests/bpf/Makefile ++++ b/tools/testing/selftests/bpf/Makefile +@@ -10,6 +10,7 @@ TOOLSDIR := $(abspath ../../..) + LIBDIR := $(TOOLSDIR)/lib + BPFDIR := $(LIBDIR)/bpf + TOOLSINCDIR := $(TOOLSDIR)/include ++TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include + BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool + APIDIR := $(TOOLSINCDIR)/uapi + ifneq ($(O),) +@@ -44,7 +45,7 @@ CFLAGS += -g $(OPT_FLAGS) -rdynamic \ + -Wall -Werror -fno-omit-frame-pointer \ + $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ + -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ +- -I$(TOOLSINCDIR) -I$(APIDIR) -I$(OUTPUT) ++ -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) + LDFLAGS += $(SAN_LDFLAGS) + LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread + +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0012-Missing-iommu-config.patch b/.github/scripts/series/patches/0012-Missing-iommu-config.patch new file mode 100644 index 00000000000000..aeb956c38564a7 --- /dev/null +++ b/.github/scripts/series/patches/0012-Missing-iommu-config.patch @@ -0,0 +1,29 @@ +From cf68142d04abb6d1c848930bf4b19097497cd924 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Mon, 30 Sep 2024 09:47:56 +0200 +Subject: [PATCH] Missing iommu config +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/iommu/config.riscv64 | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/tools/testing/selftests/iommu/config.riscv64 b/tools/testing/selftests/iommu/config.riscv64 +index d218811ccadf..efd7fe60553f 100644 +--- a/tools/testing/selftests/iommu/config.riscv64 ++++ b/tools/testing/selftests/iommu/config.riscv64 +@@ -1,4 +1,5 @@ + # IOMMUFD ++CONFIG_RUNTIME_TESTING_MENU=y + CONFIG_IOMMUFD=y + CONFIG_VFIO=y + CONFIG_VFIO_CONTAINER=n + +base-commit: e32cde8d2bd7d251a8f9b434143977ddf13dcec6 +prerequisite-patch-id: 0b6c42032915388333abd2e1fa71082780e474d1 +-- +2.43.0 + diff --git a/.github/scripts/series/patches/0013-selftests-filesystems-mount-notify.patch b/.github/scripts/series/patches/0013-selftests-filesystems-mount-notify.patch new file mode 100644 index 00000000000000..d4033542597713 --- /dev/null +++ b/.github/scripts/series/patches/0013-selftests-filesystems-mount-notify.patch @@ -0,0 +1,34 @@ +From 1405dc9a853bd1ccd2bb66f8b26c113bdb199222 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:15:12 +0000 +Subject: [PATCH 1/7] selftests: filesystems/mount-notify +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix warning from not handling return value of chdir(). + +Signed-off-by: Björn Töpel +--- + .../selftests/filesystems/mount-notify/mount-notify_test.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c +index 59a71f22fb11..c64acbbbd562 100644 +--- a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c ++++ b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c +@@ -487,7 +487,8 @@ TEST_F(fanotify, rmdir) + ASSERT_GE(ret, 0); + + if (ret == 0) { +- chdir("/"); ++ ret = chdir("/"); ++ ASSERT_EQ(ret, 0); + unshare(CLONE_NEWNS); + mount("", "/", NULL, MS_REC|MS_PRIVATE, NULL); + umount2("/a", MNT_DETACH); + +base-commit: 01f95500a162fca88cefab9ed64ceded5afabc12 +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0014-selftests-mm-page_frag.patch b/.github/scripts/series/patches/0014-selftests-mm-page_frag.patch new file mode 100644 index 00000000000000..c5443aa1e3d0bd --- /dev/null +++ b/.github/scripts/series/patches/0014-selftests-mm-page_frag.patch @@ -0,0 +1,40 @@ +From 69963d4336b06285ecd4f2bc81ed9d7ee2678a63 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:16:01 +0000 +Subject: [PATCH 2/7] selftests: mm/page_frag +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix build error for cross-builds. Make sure that correct kernel, and +make flags are picked up. + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/mm/page_frag/Makefile | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/tools/testing/selftests/mm/page_frag/Makefile b/tools/testing/selftests/mm/page_frag/Makefile +index 8c8bb39ffa28..7e60c04dadf7 100644 +--- a/tools/testing/selftests/mm/page_frag/Makefile ++++ b/tools/testing/selftests/mm/page_frag/Makefile +@@ -1,5 +1,5 @@ + PAGE_FRAG_TEST_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +-KDIR ?= /lib/modules/$(shell uname -r)/build ++KDIR ?= $(abspath $(PAGE_FRAG_TEST_DIR)/../../../../..) + + ifeq ($(V),1) + Q = +@@ -12,7 +12,7 @@ MODULES = page_frag_test.ko + obj-m += page_frag_test.o + + all: +- +$(Q)make -C $(KDIR) M=$(PAGE_FRAG_TEST_DIR) modules ++ +$(Q)$(MAKE) -C $(KDIR) M=$(PAGE_FRAG_TEST_DIR) modules + + clean: +- +$(Q)make -C $(KDIR) M=$(PAGE_FRAG_TEST_DIR) clean ++ +$(Q)$(MAKE) -C $(KDIR) M=$(PAGE_FRAG_TEST_DIR) clean +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0015-selftests-mount_setattr-mount_setattr_test.patch b/.github/scripts/series/patches/0015-selftests-mount_setattr-mount_setattr_test.patch new file mode 100644 index 00000000000000..026fcdc337046b --- /dev/null +++ b/.github/scripts/series/patches/0015-selftests-mount_setattr-mount_setattr_test.patch @@ -0,0 +1,47 @@ +From 9db1ec73a41c555846a72d463b9d675c191f37e1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:16:55 +0000 +Subject: [PATCH 3/7] selftests: mount_setattr/mount_setattr_test +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix build error: Some syscalls wrappers are already included via +"filesystems" kselftests. + +Signed-off-by: Björn Töpel +--- + .../mount_setattr/mount_setattr_test.c | 17 ----------------- + 1 file changed, 17 deletions(-) + +diff --git a/tools/testing/selftests/mount_setattr/mount_setattr_test.c b/tools/testing/selftests/mount_setattr/mount_setattr_test.c +index 48a000cabc97..7b624cbf3c55 100644 +--- a/tools/testing/selftests/mount_setattr/mount_setattr_test.c ++++ b/tools/testing/selftests/mount_setattr/mount_setattr_test.c +@@ -161,23 +161,6 @@ static inline int sys_mount_setattr(int dfd, const char *path, unsigned int flag + return syscall(__NR_mount_setattr, dfd, path, flags, attr, size); + } + +-#ifndef OPEN_TREE_CLONE +-#define OPEN_TREE_CLONE 1 +-#endif +- +-#ifndef OPEN_TREE_CLOEXEC +-#define OPEN_TREE_CLOEXEC O_CLOEXEC +-#endif +- +-#ifndef AT_RECURSIVE +-#define AT_RECURSIVE 0x8000 /* Apply to the entire subtree */ +-#endif +- +-static inline int sys_open_tree(int dfd, const char *filename, unsigned int flags) +-{ +- return syscall(__NR_open_tree, dfd, filename, flags); +-} +- + static ssize_t write_nointr(int fd, const void *buf, size_t count) + { + ssize_t ret; +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0016-selftests-pid_namespace-pid_max.patch b/.github/scripts/series/patches/0016-selftests-pid_namespace-pid_max.patch new file mode 100644 index 00000000000000..c6e3bce0ff07f6 --- /dev/null +++ b/.github/scripts/series/patches/0016-selftests-pid_namespace-pid_max.patch @@ -0,0 +1,30 @@ +From 1f035044cdf5a5619b49cc43dce96e7e1705f3ca Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:18:29 +0000 +Subject: [PATCH 4/7] selftests: pid_namespace/pid_max +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix build error by including glibc headers. + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/pid_namespace/pid_max.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/tools/testing/selftests/pid_namespace/pid_max.c b/tools/testing/selftests/pid_namespace/pid_max.c +index 51c414faabb0..972bedc475f1 100644 +--- a/tools/testing/selftests/pid_namespace/pid_max.c ++++ b/tools/testing/selftests/pid_namespace/pid_max.c +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + + #include "../kselftest_harness.h" + #include "../pidfd/pidfd.h" +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0017-selftests-proc-proc-pid-vm.patch b/.github/scripts/series/patches/0017-selftests-proc-proc-pid-vm.patch new file mode 100644 index 00000000000000..18dd8094ec282f --- /dev/null +++ b/.github/scripts/series/patches/0017-selftests-proc-proc-pid-vm.patch @@ -0,0 +1,39 @@ +From a4bfc538aa2d9dc960e075c6b298bbc2f03314fe Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:19:24 +0000 +Subject: [PATCH 5/7] selftests: proc/proc-pid-vm +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix warning for unused variables by moving the x86 ifdef. + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/proc/proc-pid-vm.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/tools/testing/selftests/proc/proc-pid-vm.c b/tools/testing/selftests/proc/proc-pid-vm.c +index d04685771952..f5d011ec296b 100644 +--- a/tools/testing/selftests/proc/proc-pid-vm.c ++++ b/tools/testing/selftests/proc/proc-pid-vm.c +@@ -212,6 +212,8 @@ static int make_exe(const uint8_t *payload, size_t len) + } + #endif + ++#ifdef __x86_64__ ++ + /* + * 0: vsyscall VMA doesn't exist vsyscall=none + * 1: vsyscall VMA is --xp vsyscall=xonly +@@ -226,7 +228,6 @@ static const char str_vsyscall_1[] = + static const char str_vsyscall_2[] = + "ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"; + +-#ifdef __x86_64__ + static void sigaction_SIGSEGV(int _, siginfo_t *__, void *___) + { + _exit(g_vsyscall); +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0018-selftests-riscv-abi-pointer_masking.patch b/.github/scripts/series/patches/0018-selftests-riscv-abi-pointer_masking.patch new file mode 100644 index 00000000000000..c2bfcb1499edca --- /dev/null +++ b/.github/scripts/series/patches/0018-selftests-riscv-abi-pointer_masking.patch @@ -0,0 +1,39 @@ +From 427d10dede741757a985e8e5a8128040cba0c328 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:20:17 +0000 +Subject: [PATCH 6/7] selftests: riscv/abi/pointer_masking +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix warning. + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/riscv/abi/pointer_masking.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tools/testing/selftests/riscv/abi/pointer_masking.c b/tools/testing/selftests/riscv/abi/pointer_masking.c +index 059d2e87eb1f..4627bb7e6c01 100644 +--- a/tools/testing/selftests/riscv/abi/pointer_masking.c ++++ b/tools/testing/selftests/riscv/abi/pointer_masking.c +@@ -213,14 +213,14 @@ static void test_tagged_addr_abi_sysctl(void) + + value = '1'; + if (!pwrite_wrapper(fd, &value, 1, "write '1'")) +- ksft_test_result_fail(err_pwrite_msg); ++ ksft_test_result_fail("%s", err_pwrite_msg); + else + ksft_test_result(set_tagged_addr_ctrl(min_pmlen, true) == -EINVAL, + "sysctl disabled\n"); + + value = '0'; + if (!pwrite_wrapper(fd, &value, 1, "write '0'")) +- ksft_test_result_fail(err_pwrite_msg); ++ ksft_test_result_fail("%s", err_pwrite_msg); + else + ksft_test_result(set_tagged_addr_ctrl(min_pmlen, true) == 0, + "sysctl enabled\n"); +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0019-selftests-ublk.patch b/.github/scripts/series/patches/0019-selftests-ublk.patch new file mode 100644 index 00000000000000..0ea338de438384 --- /dev/null +++ b/.github/scripts/series/patches/0019-selftests-ublk.patch @@ -0,0 +1,31 @@ +From 4e73635a35e39135a6c92c3ceb6c8036de707d79 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 6 May 2025 07:20:35 +0000 +Subject: [PATCH 7/7] selftests: ublk +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fix build by making sure the correct headers are included in the +build. + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/ublk/Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile +index f34ac0bac696..4aa339e222bd 100644 +--- a/tools/testing/selftests/ublk/Makefile ++++ b/tools/testing/selftests/ublk/Makefile +@@ -1,6 +1,6 @@ + # SPDX-License-Identifier: GPL-2.0 + +-CFLAGS += -O3 -Wl,-no-as-needed -Wall -I $(top_srcdir) ++CFLAGS += -O3 -Wl,-no-as-needed -Wall $(KHDR_INCLUDES) -I $(top_srcdir) + LDLIBS += -lpthread -lm -luring + + TEST_PROGS := test_generic_01.sh +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0020-Add-missing-utility-script-to-install-target.patch b/.github/scripts/series/patches/0020-Add-missing-utility-script-to-install-target.patch new file mode 100644 index 00000000000000..e51d2d958c1e4c --- /dev/null +++ b/.github/scripts/series/patches/0020-Add-missing-utility-script-to-install-target.patch @@ -0,0 +1,31 @@ +From 566f501224c76e10d4b88dbd88720458add6bf3d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Tue, 13 May 2025 10:23:18 +0000 +Subject: [PATCH] Add missing utility script to install target +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/ublk/Makefile | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile +index f34ac0bac696..127ab3f17658 100644 +--- a/tools/testing/selftests/ublk/Makefile ++++ b/tools/testing/selftests/ublk/Makefile +@@ -29,6 +29,8 @@ TEST_PROGS += test_stress_03.sh + TEST_PROGS += test_stress_04.sh + TEST_PROGS += test_stress_05.sh + ++TEST_FILES := test_common.sh ++ + TEST_GEN_PROGS_EXTENDED = kublk + + include ../lib.mk + +base-commit: e9565e23cd89d4d5cd4388f8742130be1d6f182d +-- +2.48.1 + diff --git a/.github/scripts/series/patches/0021-ntsync-typo.patch b/.github/scripts/series/patches/0021-ntsync-typo.patch new file mode 100644 index 00000000000000..f2b3fa5f244c5c --- /dev/null +++ b/.github/scripts/series/patches/0021-ntsync-typo.patch @@ -0,0 +1,25 @@ +From 7a2aa1a0420d6b6fae31b6a3cb59965ebca7bb6f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= +Date: Wed, 14 May 2025 12:27:17 +0000 +Subject: [PATCH] ntsync: typo +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Björn Töpel +--- + tools/testing/selftests/drivers/ntsync/config | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/testing/selftests/drivers/ntsync/config b/tools/testing/selftests/drivers/ntsync/config +index 60539c826d06..0aa68de147af 100644 +--- a/tools/testing/selftests/drivers/ntsync/config ++++ b/tools/testing/selftests/drivers/ntsync/config +@@ -1 +1 @@ +-CONFIG_WINESYNC=y ++CONFIG_NTSYNC=y + +base-commit: 9f35e33144ae5377d6a8de86dd3bd4d995c6ac65 +-- +2.48.1 + diff --git a/.github/scripts/series/post_to_squad.py b/.github/scripts/series/post_to_squad.py new file mode 100755 index 00000000000000..425e776c389073 --- /dev/null +++ b/.github/scripts/series/post_to_squad.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +# +# Pulls build/run results from the kselftest kernel logs, and +# publishes them to SQUAD. +# + +import argparse +import json +import os +import os.path +import pathlib +import pprint +import re +import requests +import sys +import tempfile + +from pathlib import Path +from tap import parser + +SQUAD_TOKEN = os.getenv('SQUAD_TOKEN') +SQUAD_URL = "https://squad.di.riseproject.dev/api/submit" +SQUAD_GROUP = "riscv-linux" +SQUAD_PROJECT = "linux-all" +SQUAD_CI_ENV = "qemu" + +def parse_bpf_kselftest(kselftest_file, results): + bpf_start_pat = re.compile(r'^#\s+selftests:\s+([a-z_\-/]+):\s+([a-z_\-/]+)') + bpf_end_pat = re.compile(r'^(not ok|ok)\s+([0-9]+)\s+selftests:\s+([a-z_\-/]+):\s+([a-z_\-/]+)') + bpf_test_pat = re.compile(r'^#\s+#([0-9]+)\s+([a-z_\-/]+):(OK|FAIL)') + + curr_start = None + curr_end = None + for i in kselftest_file: + if (m := bpf_start_pat.match(i)): + group = m.group(1) + test = m.group(2) + if curr_start and not curr_end: + print(f"PREMATURE START: group: {group} test: {test}", file=sys.stderr) + curr_start = m + curr_end = None + continue + + if (m := bpf_end_pat.match(i)): + group = "kselftest-" + m.group(3) + test = group + "/" + m.group(4) + "__allsubtests" + res = False if m.group(1) == "not ok" else True + if not curr_start or curr_start.group(1) != m.group(3): + print(f"PREMATURE END: group end: {group} test: {test}", file=sys.stderr) + res = False + curr_end = m + + if group not in results: + results[group] = {} + if "tests" not in results[group]: + results[group]["tests"] = {} + if test not in results[group]["tests"]: + results[group]["tests"][test] = {} + + results[group]["tests"][test]["result"] = "pass" if res else "fail" + results[group]["log"] = kselftest_file.name + continue + + if (m := bpf_test_pat.match(i)): + sub_test_id = m.group(1) + sub_test_name = m.group(2) + sub_res = False if m.group(3) == "FAIL" else True + if not curr_start: + print(f"PREMATURE TEST: test: {sub_test_name}", file=sys.stderr) + continue + + group = "kselftest-" + curr_start.group(1) + test = group + "/" + curr_start.group(2) + "__" + sub_test_name + + if "tests" not in results[group]: + results[group]["tests"] = {} + if test not in results[group]["tests"]: + results[group]["tests"][test] = {} + + results[group]["tests"][test]["result"] = "pass" if sub_res else "fail" + results[group]["log"] = kselftest_file.name + + if curr_start and not curr_end: + group = "kselftest-" + curr_start.group(1) + test = group + "/" + curr_start.group(2) + "__allsubtests" + print(f"END MISSING: group end: {group} test: {test}", file=sys.stderr) + if test not in results[group]["tests"]: + results[group]["tests"][test] = {} + results[group]["tests"][test]["result"] = "fail" + + return results + +def parse_kselftest(kselftest_file, results): + description_pat = re.compile(r"selftests:\s+(\S+)\s+(\S+)") + + p = parser.Parser() + for l in p.parse(kselftest_file): + if l.category == "test": + description = description_pat.match(l.description) + if not description: + print(f"BAD SELFTEST STRING: {l.description}", file=sys.stderr) + continue + + group = "kselftest-" + description.group(1)[:-1] + test = group + "/" + description.group(2) + + if group not in results: + results[group] = {} + if "tests" not in results[group]: + results[group]["tests"] = {} + if test not in results[group]["tests"]: + results[group]["tests"][test] = {} + + # XXX Use l.directive.skip? + + results[group]["tests"][test]["result"] = "pass" if l.ok else "fail" + results[group]["log"] = kselftest_file.name + + return results + +# +# The toplevel file needs special handling, when submitting the +# results to SQUAD. The SQUAD REST API only allows for *one* full log +# per POST, but the toplevel log is a collection of builds/tests. This +# means that each test in the toplevel, that has a log needs a POST of +# its own. +# +def parse_toplevel(top_file, results): + build_fail_pat = re.compile(r"^::error::FAIL Build (kernel|selftest) (\S+) \"(\S+)\"") + build_ok_pat = re.compile(r"^::notice::OK Build (kernel|selftest) (\S+)") + test_fail_pat = re.compile(r"^::error::FAIL Test kernel (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) \S+ \"(\S+)\"") + test_ok_pat = re.compile(r"^::notice::OK Test kernel (\S+) (\S+) (\S+) (\S+) (\S+) (\S+)") + build_name_pat = re.compile(r"^build_name (\S+)") + + for i in top_file: + group = None + test = None + log = None + res = False + + if (m := build_name_pat.match(i)): + results["build_name"] = m.group(1) + continue + + if (m := build_fail_pat.match(i)): + group = m.group(2) + test = group + "/" + "build_" + m.group(1) + log = m.group(3) + res = False + + if (m := build_ok_pat.match(i)): + group = m.group(2) + test = group + "/" + "build_" + m.group(1) + res = True + + if (m := test_fail_pat.match(i)): + group = m.group(1) + test = group + "/" + m.group(2) + "__" + m.group(3) + "__" + m.group(4)\ + + "__" + m.group(5) + "__" + m.group(6) + log = m.group(7) + res = False + + if (m := test_ok_pat.match(i)): + group = m.group(1) + test = group + "/" + m.group(2) + "__" + m.group(3) + "__" + m.group(4)\ + + "__" + m.group(5) + "__" + m.group(6) + res = True + + if group and test: + if group not in results: + results[group] = {} + + results[group]["log_per_test"] = True + + if "tests" not in results[group]: + results[group]["tests"] = {} + if "tests-log" not in results[group]: + results[group]["tests-log"] = {} + if test not in results[group]["tests"]: + results[group]["tests"][test] = {} + + results[group]["tests"][test]["result"] = "pass" if res else "fail" + if log: + if "tests-log" not in results[group]: + results[group]["tests-log"] = {} + + l = os.path.dirname(top_file.name) + "/" + log + results[group]["tests-log"][test] = l + + return results + +def parse_args(): + parser = argparse.ArgumentParser(description = 'Output Squad json from kselftest runs') + + parser.add_argument("--fake-curl", action="store_true", help = 'Dry run showing curl equivalent') + parser.add_argument("--branch", default="please_set_me", help = 'Git branch ref') + parser.add_argument("--job-url", default="http://example.com/notset", help = 'Job URL') + parser.add_argument("--selftest-bpf-log", help = 'BPF kselftest log file') + parser.add_argument("--selftest-log-dir", default=None, help = 'Kselftest log files directory') + parser.add_argument("--toplevel-log", required=True, help = 'Toplevel "kselftest.log" file') + + return parser.parse_args() + +def submit_fake_curl(testsuite, tests_dict, log, job_url, branch, build_name, o_path): + metadata_json = o_path / (testsuite.replace("/", "_") + "--metadata.json") + jstr = json.dumps({"job_url" : job_url, + "branch" : branch + }, indent=4) + metadata_json.write_text(jstr) + print(f"metadata: {jstr}") + + tests_json = o_path / (testsuite.replace("/", "_") + "--tests.json") + jstr = json.dumps(tests_dict, indent=4) + tests_json.write_text(jstr) + print(f"tests: {jstr}") + + print(f'curl --header "Authorization: token {SQUAD_TOKEN}" \\') + print(f' --form tests=@{str(o_path.absolute()) + "/" + tests_json.name} \\') + print(f' --form metadata=@{str(o_path.absolute()) + "/" + metadata_json.name} \\') + if log: + print(f' --form log=@{log} \\') + print(f' {SQUAD_URL}/{SQUAD_GROUP}/{SQUAD_PROJECT}/{build_name}/{SQUAD_CI_ENV}') + +def submit_squad(testsuite, tests_dict, log, job_url, branch, build_name, o_path): + full_url = f"{SQUAD_URL}/{SQUAD_GROUP}/{SQUAD_PROJECT}/{build_name}/{SQUAD_CI_ENV}" + + metadata_json = o_path / (testsuite.replace("/", "_") + "--metadata.json") + jstr = json.dumps({"job_url" : job_url, + "branch" : branch + }, indent=4) + metadata_json.write_text(jstr) + # print(f"metadata: {jstr}") + + tests_json = o_path / (testsuite.replace("/", "_") + "--tests.json") + jstr = json.dumps(tests_dict, indent=4) + tests_json.write_text(jstr) + # print(f"tests: {jstr}") + + headers = { + "Authorization": f"token {SQUAD_TOKEN}" + } + + files = { + "tests": tests_json.open(mode='rb'), + "metadata": metadata_json.open(mode='rb') + } + + if log: + files["log"] = open(f"{log}", 'rb') + + response = requests.post(full_url, headers=headers, files=files) + print(f"Request completed with status code: {response.status_code} text: {response.text}") + +if __name__ == "__main__": + args = parse_args() + + results = {} + + t = os.path.expanduser(args.toplevel_log) + with open(t, 'r') as top: + results = parse_toplevel(top, results) + + if args.selftest_log_dir: + for f in Path(args.selftest_log_dir).glob("test_kernel__*.log"): + a = os.path.expanduser(f) + with open(a, 'r') as all: + results = parse_kselftest(all, results) + + if args.selftest_bpf_log: + b = os.path.expanduser(args.selftest_bpf_log) + with open(b, 'r') as bpf: + results = parse_bpf_kselftest(bpf, results) + + submit = submit_fake_curl if args.fake_curl else submit_squad + + with tempfile.TemporaryDirectory() as temp_dir: + o_path = Path(temp_dir) + + for testsuite in results: + if testsuite == "build_name": + continue + + if results[testsuite].get("log_per_test", False): + for test in results[testsuite]["tests"]: + log = results[testsuite]["tests-log"].get(test, None) + submit(testsuite, {test : results[testsuite]["tests"][test] },\ + log, args.job_url, args.branch, results["build_name"], o_path) + else: + log = results[testsuite].get("log", None) + submit(testsuite, results[testsuite]["tests"], log, args.job_url,\ + args.branch, results["build_name"], o_path) diff --git a/.github/scripts/series/prepare_rootfs.sh b/.github/scripts/series/prepare_rootfs.sh new file mode 100755 index 00000000000000..e77699ea752442 --- /dev/null +++ b/.github/scripts/series/prepare_rootfs.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +# Prepares a VM image, from a kernel tar-ball and a rootfs. + +set -x +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh +. $d/qemu_test_utils.sh + +kernelpath=$1 +rootfs=$2 +tst=$3 +imagename=$4 + +cleanup() { + rm -rf "$tmp" +} + +tmp=$(mktemp -d -p ${ci_root}) +trap cleanup EXIT + +unzstd --keep --stdout $rootfs > $tmp/$(basename $rootfs .zst) + +rootfs="$tmp/$(basename $rootfs .zst)" +modpath=$(find $kernelpath -wholename '*/lib/modules') +vmlinuz=$(find $kernelpath -name '*vmlinu[zx]*') + +kselftestpath=${kernelpath}_build/kselftest/kselftest_install + +imsz=0 +if [[ $tst =~ kselftest ]]; then + sz=$(du -B 1G -s "$kselftestpath" | awk '{print $1}') + imsz=$(( ${imsz} + $sz )) +fi + +if [[ -n $modpath ]]; then + sz=$(du -B 1G -s "$modpath" | awk '{print $1}') + imsz=$(( ${imsz} + $sz )) +fi +imsz=$(( ${imsz} + 2 )) + +# aarch64 export LIBGUESTFS_BACKEND_SETTINGS=force_tcg +eval "$(guestfish --listen)" + +rm -rf $imagename +guestfish --remote -- \ + disk-create "$imagename" raw ${imsz}G : \ + add-drive "$imagename" format:raw : \ + launch : \ + part-init /dev/sda gpt : \ + part-add /dev/sda primary 2048 526336 : \ + part-add /dev/sda primary 526337 -34 : \ + part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B : \ + mkfs ext4 /dev/sda2 : \ + mount /dev/sda2 / : \ + mkdir /boot : \ + mkdir /boot/efi : \ + mkfs vfat /dev/sda1 : \ + mount /dev/sda1 /boot/efi : \ + tar-in $rootfs / : \ + copy-in $vmlinuz /boot/efi/ : \ + mv /boot/efi/$(basename $vmlinuz) /boot/efi/Image + + +if [[ -n $modpath ]]; then + guestfish --remote -- copy-in $modpath /lib/ +fi + +if [[ $tst =~ kselftest ]]; then + guestfish --remote -- \ + copy-in $kselftestpath / + + subtest=$(echo ${tst} | cut -f2- -d'-') + + if [[ "$subtest" =~ bpf ]]; then + timeout=9000 + else + timeout=3600 + fi + + touch $tmp/dotest + chmod +x $tmp/dotest + cat >$tmp/dotest <Hello kselftest" > /dev/kmsg +cd /kselftest_install +export PATH=${PATH}:/kselftest_install/bpf/tools/sbin + +echo "TEST ${subtest}" +./run_kselftest.sh -o ${timeout} -c ${subtest} +EOF + + echo "dotest:" + cat $tmp/dotest + echo "dotest end" + guestfish --remote -- \ + copy-in $tmp/dotest / +fi + +guestfish --remote -- \ + sync : \ + umount /boot/efi : \ + umount / : \ + exit diff --git a/.github/scripts/series/qemu_test_utils.sh b/.github/scripts/series/qemu_test_utils.sh new file mode 100644 index 00000000000000..5bc13b74e3c87c --- /dev/null +++ b/.github/scripts/series/qemu_test_utils.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +qemu_subtests=() + +generate_qemu_subtests() { + local xlen=$1 + local config=$2 + local fragment=$3 + local toolchain=$4 + local rootfs=$5 + + local n=$(gen_kernel_name $xlen $config $fragment $toolchain) + local cpu_sifive=0 + local fw_uefi=0 + local hw_acpi=0 + local kernel_config=$(find ${ci_root}/${n} -name 'config-*' 2>/dev/null || echo "/dev/null") + + qemu_subtests=() + + if [[ $xlen == "rv64" ]]; then + if ls ${kernel_config} &> /dev/null; then + if grep -q 'CONFIG_RISCV_ALTERNATIVE_EARLY=y' ${kernel_config}; then + cpu_sifive=1 + fi + if grep -q 'CONFIG_EFI=y' ${kernel_config}; then + fw_uefi=1 + fi + if grep -q 'CONFIG_ACPI=y' ${kernel_config}; then + hw_acpi=1 + fi + fi + + for cpu in default64 server64 max64 sifive; do + if [[ $cpu =~ sifive ]]; then + if ! (( ${cpu_sifive} )); then + continue + fi + fi + + for fw in no_uefi uboot_uefi; do + if ! [[ $fw == no_uefi ]]; then + if ! (( ${fw_uefi} )); then + continue + fi + + if [[ $fw == edk2_uefi ]]; then + continue + fi + fi + + # For now, we're only doing selftest on DT. + if [[ $config =~ ^kselftest ]]; then + if [[ $cpu == server64 && $fw == uboot_uefi && $rootfs == ubuntu ]]; then + if (( ${ci_test_selftests} )); then + qemu_subtests+=( "$cpu $fw dt $config" ) + fi + fi + continue + fi + + for hw in dt acpi; do + if [[ $hw == acpi ]]; then + if [[ $fw == no_uefi ]]; then + continue + fi + + if ! (( ${hw_acpi} )); then + continue + fi + fi + + qemu_subtests+=( "$cpu $fw $hw boot" ) + + done + done + done + else + qemu_subtests+=( "default32 no_uefi dt boot" ) + fi + return 0 +} + +get_qemu_test_name() { + local cpu=$1 + local fw=$2 + local hw=$3 + local tst=$4 + + echo "${cpu}__${fw}__${hw}__${tst}" +} diff --git a/.github/scripts/series/selftest_builder.sh b/.github/scripts/series/selftest_builder.sh new file mode 100755 index 00000000000000..51b0ac69fcb9a6 --- /dev/null +++ b/.github/scripts/series/selftest_builder.sh @@ -0,0 +1,24 @@ +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 + +tm=$(mktemp -p ${ci_root}) +n=$(gen_kernel_name $xlen $config $fragment $toolchain) +logs=$(get_logs_dir) +rc=0 +log="build_selftest___${n}.log" +\time --quiet -o $tm -f "took %es" \ + $d/build_selftest.sh "${xlen}" "${config}" "${fragment}" "${toolchain}" &> "${logs}/${log}" || rc=$? +if (( $rc )); then + echo "::error::FAIL Build selftest ${n} \"${log}\" $(cat $tm)" +else + echo "::notice::OK Build selftest ${n} $(cat $tm)" +fi +rm $tm +exit $rc diff --git a/.github/scripts/series/test_all.sh b/.github/scripts/series/test_all.sh new file mode 100755 index 00000000000000..b9329e55cffead --- /dev/null +++ b/.github/scripts/series/test_all.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +$d/unpack_fw.sh + +parallel_log=$(mktemp -p ${ci_root}) +parallel -j $(($(nproc)/4)) --colsep ' ' --joblog ${parallel_log} \ + ${d}/kernel_tester.sh {1} {2} {3} {4} {5} :::: <($d/generate_test_runs.sh) || true +cat ${parallel_log} +rm ${parallel_log} diff --git a/.github/scripts/series/test_kernel.sh b/.github/scripts/series/test_kernel.sh new file mode 100755 index 00000000000000..2e668bfc3feb6d --- /dev/null +++ b/.github/scripts/series/test_kernel.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +# Executes the VMs, and report. + +set -x +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh +. $d/qemu_test_utils.sh + +xlen=$1 +config=$2 +fragment=$3 +toolchain=$4 +rootfs=$5 + +cpu=$6 +fw=$7 +hw=$8 +tst=$9 + +cpu_to_qemu() { + local cpu=$1 + + case "$cpu" in + "default32") + echo "rv32" + ;; + "default64") + echo "rv64" + ;; + "server64") + echo "rv64,v=true,vlen=256,elen=64,h=true,zbkb=on,zbkc=on,zbkx=on,zkr=on,zkt=on,svinval=on,svnapot=on,svpbmt=on,zabha=on,zacas=on" + ;; + "max64") + echo "max" + ;; + "sifive") + echo "sifive-u54" + ;; + *) + echo "BADCPU" + ;; + esac +} + +fw_to_qemu() { + local hw=$1 + local fw=$2 + local vmlinuz=$3 + + case "$fw" in + "no_uefi") + echo "$vmlinuz" + ;; + "uboot_uefi") + if [[ ${hw} == "acpi" ]]; then + echo "${ci_root}/firmware/rv64/rv64-u-boot-acpi.bin" + else + echo "${ci_root}/firmware/rv64/rv64-u-boot.bin" + fi + ;; + *) + echo "BADFW" + ;; + esac +} + +qemu_kernel_append="root=/dev/vda2 rw earlycon console=tty0 console=ttyS0 panic=-1 oops=panic sysctl.vm.panic_on_oom=1" + +qemu_rv64 () { + local qemu_to=$1 + local qemu_log=$2 + local qemu_bios=$3 + local qemu_kernel=$4 + local qemu_cpu=$5 + local qemu_acpi=$6 + local qemu_aia=$7 + local qemu_image=$8 + + timeout --foreground ${qemu_to}s qemu-system-riscv64 \ + -no-reboot \ + -nodefaults \ + -nographic \ + -machine virt,acpi=${qemu_acpi},aia=${qemu_aia} \ + -smp 4 \ + -bios ${qemu_bios} \ + -cpu ${qemu_cpu} \ + -kernel ${qemu_kernel} \ + -append "${qemu_kernel_append}" \ + -m 8G \ + -object rng-random,filename=/dev/urandom,id=rng0 \ + -device virtio-rng-device,rng=rng0 \ + -chardev stdio,id=char0,mux=on,signal=off,logfile="${qemu_log}" \ + -serial chardev:char0 \ + -drive if=none,file=${qemu_image},format=raw,id=hd0 \ + -device virtio-blk-pci,drive=hd0 +} + +qemu_rv32 () { + local qemu_to=$1 + local qemu_log=$2 + local qemu_bios=$3 + local qemu_kernel=$4 + local qemu_cpu=$5 + local qemu_acpi=$6 + local qemu_aia=$7 + local qemu_image=$8 + + timeout --foreground ${qemu_to}s qemu-system-riscv32 \ + -no-reboot \ + -nodefaults \ + -nographic \ + -machine virt \ + -cpu rv32 \ + -smp 4 \ + -bios ${qemu_bios} \ + -kernel ${qemu_kernel} \ + -append "${qemu_kernel_append}" \ + -m 1G \ + -object rng-random,filename=/dev/urandom,id=rng0 \ + -device virtio-rng-device,rng=rng0 \ + -chardev stdio,id=char0,mux=on,signal=off,logfile="${qemu_log}" \ + -serial chardev:char0 \ + -drive if=none,file=${qemu_image},format=raw,id=hd0 \ + -device virtio-blk-pci,drive=hd0 +} + +check_shutdown () { + local image=$1 + local rc=0 + + shutdown="$(guestfish --ro -a "$image" -i cat /shutdown-status 2>/dev/null)" + if [[ $shutdown == "clean" ]]; then + f=$(mktemp -p ${tmp}) + guestfish --rw -a "$image" -i download /dmesg $f + fail_str=( "\-+\\[ cut here \\]-+\\s+(.*\\s+-+\\[ end trace (\\w*) \\]-+)" "(Unhandled fault.*)\\r\\n" "Kernel panic - (.*) end Kernel panic" "Stack:\\s+(.*\\s+-+\\[ end trace (\\w*) \\]-+)" "^[^\\n]+WARNING:.*?$" "^[^\\n]+Oops(?: -|:).*?$" "^[^\\n]+BUG:.*?$" ) + for fail in "${fail_str[@]}"; do + if grep -E "$fail" $f; then + rc=1 + fi + done + else + rc=1 + fi + + return $rc +} + +tmp=$(mktemp -d -p "${ci_root}") +trap 'rm -rf "$tmp"' EXIT + +kernelpath=${ci_root}/$(gen_kernel_name $xlen $config $fragment $toolchain) +vmlinuz=$(find $kernelpath -name '*vmlinu[zx]*') +rootfs_tar=$(echo ${ci_rootfs_root}/rootfs_${xlen}_${rootfs}_*.tar.zst) +qemu_image=$tmp/rootfs.img + +rc=0 +\time --quiet -f "took prepare_rootfs %e" \ + $d/prepare_rootfs.sh $kernelpath $rootfs_tar $tst $qemu_image || rc=$? +if (( $rc )); then + echo "Failed preparing rootfs image" + exit 1 +fi + +qemu_to=120 +if [[ $rootfs == "ubuntu" ]]; then + qemu_to=$(( $qemu_to * 3 )) + if [[ $fragment =~ nosmp || $fragment =~ lockdep || $fragment =~ kasan || $fragment =~ kfence ]]; then + qemu_to=$(( $qemu_to * 10 )) + fi +else + if [[ $fragment =~ lockdep ]]; then + qemu_to=$(( $qemu_to * 10 )) + fi +fi + +if [[ $config =~ kselftest ]]; then + qemu_to=$((2 * 24 * 3600)) # 40h +fi + +qemu_log=${tmp}/qemu.log +qemu_bios=${ci_root}/firmware/${xlen}/fw_dynamic.bin +qemu_kernel=$(fw_to_qemu $hw $fw $vmlinuz) +qemu_cpu=$(cpu_to_qemu $cpu) + +qemu_acpi=off +if [[ ${hw} == "acpi" ]]; then + qemu_acpi=on + qemu_kernel_append="${qemu_kernel_append} acpi=force" +fi +qemu_aia=none +if [[ ${cpu} == "server64" || ${cpu} == "max64" ]]; then + qemu_aia="aplic-imsic" +fi + +export TIMEFORMAT="took qemu %0R" +time qemu_${xlen} ${qemu_to} ${qemu_log} ${qemu_bios} ${qemu_kernel} ${qemu_cpu} ${qemu_acpi} ${qemu_aia} ${qemu_image} + +export TIMEFORMAT="took check_shutdown %0R" +time check_shutdown $qemu_image || rc=$? +exit $rc diff --git a/.github/scripts/series/test_only_defconfig.sh b/.github/scripts/series/test_only_defconfig.sh new file mode 100755 index 00000000000000..b7551995268bef --- /dev/null +++ b/.github/scripts/series/test_only_defconfig.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +$d/unpack_fw.sh + +rc=0 +${d}/kernel_tester.sh rv64 defconfig plain gcc ubuntu || rc=1 +exit $rc diff --git a/.github/scripts/series/test_only_kselftest.sh b/.github/scripts/series/test_only_kselftest.sh new file mode 100755 index 00000000000000..0663895a19555d --- /dev/null +++ b/.github/scripts/series/test_only_kselftest.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh +. $d/kselftest_prep.sh + +$d/unpack_fw.sh +rc=0 + +logs=$(get_logs_dir) +subtests=${logs}/kselftest-collections.txt +readarray -t kselftest_subtests < ${subtests} + +parallel_log=$(mktemp -p ${ci_root}) + +for subtest in "${kselftest_subtests[@]}"; do + echo "${d}/kernel_tester.sh rv64 ${subtest} plain gcc ubuntu" +done | parallel -j$(($(nproc)/8)) --colsep ' ' --joblog ${parallel_log} || true + +cat ${parallel_log} +rm ${parallel_log} diff --git a/.github/scripts/series/tuxrun_to_squad_json.py b/.github/scripts/series/tuxrun_to_squad_json.py new file mode 100644 index 00000000000000..434c5ed78d3bc4 --- /dev/null +++ b/.github/scripts/series/tuxrun_to_squad_json.py @@ -0,0 +1,54 @@ +import argparse +import json + +""" +Turn +{ + "testsuiteA": { + "test1": {} + }, + "testsuiteB": { + "test1": {} + } +} + +into + +{ + "testsuiteA/test1": {} + "testsuiteB/test1": {} +} + +which is what is expected by Squad. +""" + +def parse_args(): + parser = argparse.ArgumentParser(description = 'Output Squad tests results for tuxrun LTP') + parser.add_argument("--result-path", default = "", + help = 'Path to the tuxrun JSON result file') + parser.add_argument("--testsuite", default = "", + help = 'Testsuite name') + + return parser.parse_args() + +def generate_squad_json(result_path, testsuite): + dict_results = {} + + with open(result_path, "r") as f: + dict_initial = json.loads(f.read()) + + # Search only the first dimension for keys starting with "ltp-" + for k, v in dict_initial.items(): + if k.startswith(testsuite): + for ltp_key, ltp_value in v.items(): + dict_results[k + "/" + ltp_key] = ltp_value + + print(dict_results) + + with open(result_path.replace(".json", ".squad.json"), "w") as f: + json.dump(dict_results, f) + +if __name__ == "__main__": + args = parse_args() + generate_squad_json(args.result_path, args.testsuite) + diff --git a/.github/scripts/series/unpack_fw.sh b/.github/scripts/series/unpack_fw.sh new file mode 100755 index 00000000000000..6a6114ad556ba0 --- /dev/null +++ b/.github/scripts/series/unpack_fw.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +shopt -s extglob + +d=$(dirname "${BASH_SOURCE[0]}") +. $d/utils.sh + +firmware_dir=${ci_root}/firmware + +fw_rv32_opensbi=$(echo ${ci_fw_root}/firmware_rv32_opensbi_+([a-f0-9]).tar.zst) +fw_rv64_opensbi=$(echo ${ci_fw_root}/firmware_rv64_opensbi_+([a-f0-9]).tar.zst) +fw_rv64_uboot=$(echo ${ci_fw_root}/firmware_rv64_uboot_+([a-f0-9]).tar.zst) +fw_rv64_uboot_acpi=$(echo ${ci_fw_root}/firmware_rv64_uboot_acpi_+([a-f0-9]).tar.zst) + +mkdir -p ${firmware_dir}/rv32 +mkdir -p ${firmware_dir}/rv64 + +tar -C ${firmware_dir}/rv32 -xf $fw_rv32_opensbi +tar -C ${firmware_dir}/rv64 -xf $fw_rv64_opensbi +tar -C ${firmware_dir}/rv64 -xf $fw_rv64_uboot +tar -C ${firmware_dir}/rv64 -xf $fw_rv64_uboot_acpi diff --git a/.github/scripts/series/utils.sh b/.github/scripts/series/utils.sh new file mode 100644 index 00000000000000..55b5e008fcef30 --- /dev/null +++ b/.github/scripts/series/utils.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +ci_root=${CI_ROOT:-"/build"} +ci_triple=${CI_TRIPLE:-"riscv64-linux"} +ci_patches=${CI_PATCHES:-"$d/patches"} +ci_fw_root=${CI_FW_ROOT:-"/firmware"} +ci_rootfs_root=${CI_ROOTFS_ROOT:-"/rootfs"} + +ci_test_selftests=1 + +apply_patches() { + if [[ -d ${ci_patches} ]]; then + if ls ${ci_patches}/*.patch &> /dev/null; then + for i in ${ci_patches}/*.patch; do + if git apply --check -q $i; then + git apply --index -q $i + fi + done + fi + fi + git commit --allow-empty -m "OOT" +} + +unapply_patches() { + git reset --hard HEAD^ +} + +gen_kernel_name() { + local xlen=$1 + local config=$2 + local fragment=$3 + local toolchain=$4 + + if [[ "$config" =~ ^kselftest ]]; then + config="kselftest" + fi + + echo "${xlen}__${config}__$(basename $fragment)__${toolchain}" +} + +get_logs_dir() { + logs=${ci_root}/logs + mkdir -p ${logs} + echo ${logs} +} diff --git a/.github/scripts/sync.sh b/.github/scripts/sync.sh new file mode 100755 index 00000000000000..6a92064f85134c --- /dev/null +++ b/.github/scripts/sync.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +set -euo pipefail + +# Assumptions: +# Run from git source tree, e.g. /build/linux +# +# This script syncs remote "upstream" branches to the local git repo. +# The script will create a $ORIGIN_BRANCH, which is a clone of of the +# upstream $UPSTREAM_BRANCH, and a $WORKFLOW_BRANCH which is the +# upstream with the latest CI applied as on commit on top. +# +# Subsequent jobs will use these branches, e.g. to run Patchwork CI. + +echo "Environment Variables:" +echo " Workflow: ${GITHUB_WORKFLOW:-notset}" +echo " Action: ${GITHUB_ACTION:-notset}" +echo " Actor: ${GITHUB_ACTOR:-notset}" +echo " Repository: ${GITHUB_REPOSITORY:-notset}" +echo " Event-name: ${GITHUB_EVENT_NAME:-notset}" +echo " Event-path: ${GITHUB_EVENT_PATH:-notset}" +echo " Workspace: ${GITHUB_WORKSPACE:-notset}" +echo " SHA: ${GITHUB_SHA:-notset}" +echo " REF: ${GITHUB_REF:-notset}" +echo " HEAD-REF: ${GITHUB_HEAD_REF:-notset}" +echo " BASE-REF: ${GITHUB_BASE_REF:-notset}" +echo " PWD: $(pwd)" +echo " Repo: ${GITHUB_REPOSITORY:-notset}" + +tmpdir=$(mktemp -d -p /build) + +cleanup() { + git remote remove upstream + echo "bye $tmpdir" + rm -rf $tmpdir +} + +trap cleanup EXIT + +# e.g. "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" +UPSTREAM_REPO=$1 +# e.g. "master" +UPSTREAM_BRANCH=$2 +# e.g. "master" +ORIGIN_BRANCH=$3 +# e.g. workflow +WORKFLOW_BRANCH=$4 +#e.g. https://github.com/linux-riscv/github-ci.git +CI_REPO=$5 + +echo ">>> Setup repo" +echo "$ git remote set-url origin $GITHUB_REPOSITORY" +git remote set-url origin "https://$GITHUB_ACTOR:$ACTION_TOKEN@github.com/$GITHUB_REPOSITORY" +echo "$ git remote add upstream $UPSTREAM_REPO" +git remote add upstream "$UPSTREAM_REPO" +echo "$ git fetch upstream $UPSTREAM_BRANCH" +git fetch upstream --tags $UPSTREAM_BRANCH + +echo ">>> Check Origin and Upstream" +if git rev-parse --verify --quiet origin/$ORIGIN_BRANCH &>/dev/null; then + ORIGIN_HEAD=$(git log -1 --format=%H origin/$ORIGIN_BRANCH) +else + ORIGIN_HEAD=noneexisting +fi +echo "ORIGIN_HEAD: $ORIGIN_HEAD" +UPSTREAM_HEAD=$(git log -1 --format=%H upstream/$UPSTREAM_BRANCH) +echo "UPSTREAM_HEAD: $UPSTREAM_HEAD" + +if [[ "$ORIGIN_HEAD" != "$UPSTREAM_HEAD" ]]; then + echo "Repos are NOT synced. Need to merge..." + echo ">>> Sync origin with upstream" + echo "$ git remote set-branches origin *" + git remote set-branches origin '*' + echo "$ git fetch origin" + git fetch origin + echo "$ git push -f origin refs/remotes/upstream/$UPSTREAM_BRANCH:refs/heads/$ORIGIN_BRANCH" + git push -f origin "refs/remotes/upstream/$UPSTREAM_BRANCH:refs/heads/$ORIGIN_BRANCH" + echo "$ git push -f origin refs/tags/*" + git push -f origin "refs/tags/*" +fi + +echo ">>> Prepare CI repo for workflow" +echo "$ git clone $CI_REPO $tmpdir/ci" +git clone $CI_REPO $tmpdir/ci + +if ! git rev-parse --verify --quiet origin/$WORKFLOW_BRANCH &>/dev/null; then + echo "$ git checkout -B $ORIGIN_BRANCH origin/$ORIGIN_BRANCH" + git checkout -B $ORIGIN_BRANCH origin/$ORIGIN_BRANCH +else + echo "$ git checkout -B $WORKFLOW_BRANCH origin/$WORKFLOW_BRANCH" + git checkout -B $WORKFLOW_BRANCH origin/$WORKFLOW_BRANCH +fi + +update() { + echo "$ git checkout -B $WORKFLOW_BRANCH origin/$ORIGIN_BRANCH" + git checkout -B $WORKFLOW_BRANCH origin/$ORIGIN_BRANCH + echo "$ cp -R $tmpdir/ci/.github ." + cp -R $tmpdir/ci/.github . + echo "$ git add --all --force .github" + git add --all --force .github + echo "$ git commit --all --message \"Adding CI files\"" + git commit --all --message "Adding CI files" + echo "$ git branch" + git branch + echo "$ git push -f origin $WORKFLOW_BRANCH" + git push -f origin $WORKFLOW_BRANCH +} + +if ! git diff --no-index .github -- $tmpdir/ci/.github &> /dev/null; then + echo ">>> Workflow has changed, pulling in" + update +fi + +master_commit=$(git log -1 --format=%H origin/$ORIGIN_BRANCH) +workflow_commit=$(git log -1 --format=%H origin/${WORKFLOW_BRANCH}^) +echo ">>> Assert master/workflow commits are same: $master_commit $workflow_commit" +if [[ "$master_commit" != "$workflow_commit" ]]; then + echo ">>> Updating workflow" + update +fi + +echo ">>> Done Exit" diff --git a/.github/scripts/sync_patchwork.py b/.github/scripts/sync_patchwork.py new file mode 100755 index 00000000000000..cc8c5d34f4dfd3 --- /dev/null +++ b/.github/scripts/sync_patchwork.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys +import json +import re +import argparse +import tempfile + +from github import Github + +from libs import init_logger, log_debug, log_error, log_info, pr_get_sid +from libs import Patchwork, GithubTool, RepoTool, EmailTool, Context + +def patch_get_new_file_list(patch): + """ + Parse patch to get the file that is newly added + """ + + file_list = [] + + # If patch has no contents, return empty file + if patch == None: + log_error("WARNING: No file found in patch") + return file_list + + # split patch(in string) to list of string by newline + lines = patch.split('\n') + iter_lines = iter(lines) + for line in iter_lines: + try: + if re.search(r'^\-\-\- ', line): + if line.find('dev/null') >= 0: + # Detect new file. Read next line to get the filename + line2 = next(iter_lines) + file_list.append(line2[line2.find('/')+1:]) + except StopIteration: + # End of iteration or no next line. Nothing to do. Just pass + pass + + log_debug(f"New file in the patch: {file_list}") + + return file_list + +def patch_get_file_list(patch): + """ + Parse patch to get the file list + """ + + file_list = [] + + # If patch has no contents, return empty file + if patch == None: + log_error("WARNING: No file found in patch") + return file_list + + # split patch(in string) to list of string by newline + lines = patch.split('\n') + for line in lines: + # Use --- (before) instead of +++ (after). + # If new file is added, --- is /dev/null and can be ignored + # If file is removed, file in --- still exists in the tree + # The corner case is if the patch adds new files. Even in this case + # even if new files are ignored, Makefile should be changed as well + # so it still can be checked. + if re.search(r'^\-\-\- ', line): + # For new file, it should be dev/null. Ignore the file. + if line.find('dev/null') >= 0: + log_debug("New file is added. Ignore in the file list") + continue + + # Trim the '--- /' + file_list.append(line[line.find('/')+1:]) + + log_debug(f"files found in the patch: {file_list}") + + return file_list + +def series_get_file_list(ci_data, series, ignore_new_file=False): + """ + Get the list of files from the patches in the series. + """ + + file_list = [] + new_file_list = [] + + for patch in series['patches']: + full_patch = ci_data.pw.get_patch(patch['id']) + file_list += patch_get_file_list(full_patch['diff']) + if ignore_new_file: + new_file_list += patch_get_new_file_list(full_patch['diff']) + + if ignore_new_file == False or len(new_file_list) == 0: + return file_list + + log_debug("Check if new file is in the file list") + new_list = [] + for filename in file_list: + if filename in new_file_list: + log_debug(f"file:{filename} is in new_file_list. Don't count.") + continue + new_list.append(filename) + + return new_list + +def filter_repo_space(ci_data, space_details, series, src_dir): + """ + Check if the series belong to this repository + + if the series[name] has exclude string + return False + if the series[name] has include string + return True + get file list from the patch in series + if the file exist + return True + else + return False + """ + + log_debug(f"Check repo space for this series[{series['id']}]") + + # Check Exclude string + for str in space_details['exclude']: + if re.search(str, series['name'], re.IGNORECASE): + log_debug(f"Found EXCLUDE string: {str}") + return False + + # Check Include string + for str in space_details['include']: + if re.search(str, series['name'], re.IGNORECASE): + log_debug(f"Found INCLUDE string: {str}") + return True + + # Skip the rest of the test for now + return True + + # Get file list from the patches in the series + file_list = series_get_file_list(ci_data, series, ignore_new_file=True) + if len(file_list) == 0: + # Something is not right. + log_error("ERROR: No files found in the series/patch") + return False + log_debug(f"Files in series={file_list}") + + # File exist in source tree? + for filename in file_list: + file_path = os.path.join(src_dir, filename) + if not os.path.exists(file_path): + log_error(f"File not found: {filename}") + return False + + # Files exist in the source tree + log_info("Files exist in the source tree.") + return True + +EMAIL_MESSAGE = '''This is an automated email and please do not reply to this email. + +Dear Submitter, + +Thank you for submitting the patches to the Linux RISC-V mailing list. + +While preparing the CI tests, the patches you submitted couldn't be +applied to any of the current repository workflow branches. + +----- Output ----- +{content} + +Please resolve the issue and submit the patches again. + +--- +Regards, +Linux RISC-V bot + +''' + +def is_maintainers_only(email_config): + if 'only-maintainers' in email_config and email_config['only-maintainers']: + return True + return False + +def get_receivers(email_config, submitter): + log_debug("Get the list of email receivers") + + receivers = [] + if is_maintainers_only(email_config): + # Send only to the maintainers + receivers.extend(email_config['maintainers']) + else: + # Send to default-to and submitter + receivers.append(email_config['default-to']) + receivers.append(submitter) + + return receivers + +def send_email(ci_data, series, content): + + headers = {} + email_config = ci_data.config['email'] + + body = EMAIL_MESSAGE.format(content=content) + + patch_1 = series['patches'][0] + headers['In-Reply-To'] = patch_1['msgid'] + headers['References'] = patch_1['msgid'] + + if not is_maintainers_only(email_config): + headers['Reply-To'] = email_config['default-to'] + + receivers = get_receivers(email_config, + series['submitter']['email']) + ci_data.email.set_receivers(receivers) + + ci_data.email.compose(f"Re: {series['name']}", body, headers) + + if ci_data.config['dry_run']: + log_info("Dry-Run: Skip sending email") + return + + log_info("Sending Email...") + ci_data.email.send() + +PR_BODY = '''PR for series {sid} applied to {branch} + +Name: {name} +URL: {url} +Version: {version} +''' + +def series_check_patches(ci_data, series): + + series_dir = os.path.join(ci_data.config['temp_root'], f"{series['id']}") + if not os.path.exists(series_dir): + os.makedirs(series_dir) + + series_mbox = ci_data.pw.get_series_mbox(series['id']) + series_mbox_file = os.path.join(series_dir, "series.mbox") + with open(series_mbox_file, "w") as f: + f.write(series_mbox) + + already_checked = False + patch_1 = ci_data.pw.get_patch(series['patches'][0]['id']) + if patch_1['check'] != 'pending': + already_checked = True + log_info("This series is already checked") + + applied_branch = None + content = "" + for branch in ci_data.config['branch']: + ci_data.src_repo.git_checkout(branch); + ci_data.src_repo.git_checkout(f"pw{series['id']}", create_branch=True) + if ci_data.src_repo.git_am(series_mbox_file): + log_info(f"Failed to apply series {series['id']} to {branch}") + content += f"Failed to apply series {series['id']} to {branch}:\n\n" + content += ci_data.src_repo.stdout + content += ci_data.src_repo.stderr + content += "\n---\n" + ci_data.src_repo.git_am(abort=True) + continue + + log_info(f"Applied series {series['id']} to {branch}") + applied_branch = branch + # git am success + break + + if not applied_branch: + if ci_data.config['dry_run'] or already_checked: + log_info(f"Skip submitting the result to PW") + else: + url = ci_data.gh.create_gist(f"pw{series['id']}", "pre-ci_am-FAILED", content) + for patch in series['patches']: + ci_data.pw.post_check(patch, "pre-ci_am", 3, "Failed to apply series", + url=url) + + log_info("PRE-CI AM failed. Notify the submitter") + if ci_data.config['dry_run'] or already_checked: + log_info(f"Skip sending email: {content}") + return False + + send_email(ci_data, series, content) + return False + + if ci_data.config['disable_pr']: + log_info("Disable PR: Skip creating PR") + return True + + # Create Pull Request + if ci_data.src_repo.git_push(f"pw{series['id']}"): + log_error("Failed to push the source to Github") + return False + + title = f"[PW_SID:{series['id']}] {series['name']}" + + pr_body = PR_BODY.format(sid=series['id'], branch=applied_branch, name=series['name'], + url=series['web_url'], version=series['version']) + log_info(f"Creating PR: {title}") + if (pr := ci_data.gh.create_pr(title, pr_body, applied_branch, f"pw{series['id']}")): + if ci_data.config['dry_run'] or already_checked: + log_info("Skip submitting the result to PW: Success") + else: + for patch in series['patches']: + ci_data.pw.post_check(patch, "pre-ci_am", 1, "Success", url=pr.html_url) + + return True + + return False + +def run_series(ci_data, new_series): + + log_debug("##### Processing Series #####") + + space_details = ci_data.config['space_details'][ci_data.config['space']] + + # Process the series + for series in new_series: + log_info(f"\n### Process Series: {series['id']} ###") + + # If the series subject doesn't have the key-str, ignore it. + # Sometimes, the name have null value. If that's the case, use the + # name from the first patch and update to series name + if series['name'] == None: + patch_1 = series['patches'][0] + series['name'] = patch_1['name'] + log_debug(f"updated series name: {series['name']}") + + if not series['received_all']: + log_info(f"Series is NOT fully received") + continue + + # Filter the series by include/exclude string + if not filter_repo_space(ci_data, space_details, series, + ci_data.src_dir): + log_info(f"Series is NOT for this repo") + continue + + # Check if PR already exist + if ci_data.gh.pr_exist_title(f"PW_SID:{series['id']}"): + log_info("PR exists already") + continue + + # This series is ready to create PR + series_check_patches(ci_data, series) + + log_debug("##### processing Series Done #####") + +def sid_in_series_list(sid, series_list): + + log_debug(f"Search PW SID({sid} in the series list") + for series in series_list: + if int(sid) == series['id']: + log_debug("Found matching PW_SID in series list") + return series + + log_debug("No found matching PW_SID in series list") + + return None + +def cleanup_pullrequest(ci_data, new_series): + + log_debug("##### Clean Up Pull Request #####") + + prs = ci_data.gh.get_prs(force=True) + log_debug(f"Current PR: {prs}") + for pr in prs: + log_debug(f"PR: {pr}") + pw_sid = pr_get_sid(pr.title) + if not pw_sid: + log_debug(f"Not a valid PR title: {pr.title}. Skip PR") + continue + + log_debug(f"PW_SID: {pw_sid}") + + if sid_in_series_list(pw_sid, new_series): + log_debug(f"PW_SID:{pw_sid} found in PR list. Keep PR") + continue + + log_debug(f"PW_SID:{pw_sid} not found in PR list. Close PR") + + ci_data.gh.close_pr(pr.number) + + log_debug("##### Clean Up Pull Request Done #####") + +def check_args(args): + + if not os.path.exists(os.path.abspath(args.config)): + log_error(f"Invalid parameter(config) {args.config}") + return False + + if not os.path.exists(os.path.abspath(args.src_dir)): + log_error(f"Invalid parameter(src_dir) {args.src_dir}") + return False + + args.branch = args.branch or ["workflow"] + return True + +def parse_args(): + ap = argparse.ArgumentParser(description= + "Manage patch series in Patchwork and create PR") + ap.add_argument('-c', '--config', default='./config.json', + help='Configuration file to use') + ap.add_argument("-b", "--branch", default=None, action="append", + help="Name of branch in base_repo where the PR is pushed. " + "Use format. i.e. workflow") + ap.add_argument('-s', '--src-dir', required=True, + help='Source directory') + ap.add_argument('-d', '--dry-run', action='store_true', default=False, + help='Run it without uploading the result') + ap.add_argument('-p', '--disable-pr', action='store_true', default=False, + help='Disable creating pull request') + + ap.add_argument("repo", + help="Name of Github repository. i.e. linux-riscv/linux-riscv") + return ap.parse_args() + +def main(): + + init_logger("SyncPatchwork", verbose=True) + + args = parse_args() + if not check_args(args): + sys.exit(1) + + # Set temp workspace + temp_root = tempfile.TemporaryDirectory().name + log_info(f"Temp Root Dir: {temp_root}") + + ci_data = Context(config_file=os.path.abspath(args.config), + github_repo=args.repo, + src_dir=args.src_dir, + branch=args.branch, + space='kernel', + dry_run=args.dry_run, + disable_pr=args.disable_pr, + temp_root=temp_root) + + + # Process the series, state 1 = NEW + new_series = ci_data.pw.get_series_by_state(1, days_lookback=7) + if len(new_series) == 0: + log_info("No new patches/series found. Done. Exit") + return + + # Process Series + run_series(ci_data, new_series) + + # Cleanup PR + cleanup_pullrequest(ci_data, new_series) + + log_debug("----- DONE -----") + +if __name__ == "__main__": + main() diff --git a/.github/scripts/xfstests.sh b/.github/scripts/xfstests.sh new file mode 100755 index 00000000000000..11555ff4fc760f --- /dev/null +++ b/.github/scripts/xfstests.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2025 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euox pipefail +d=$(dirname "${BASH_SOURCE[0]}") +. $d/series/utils.sh + +logs=$(get_logs_dir) +f=${logs}/xfstests.log + +KERNEL_PATH=$(find "$1" -name '*vmlinu[zx]*') +mv $KERNEL_PATH $KERNEL_PATH.gz +gunzip $KERNEL_PATH.gz + +ROOTFS_PATH=$(find /rootfs/ -name 'rootfs_rv64_ubuntu*.ext4') + +build_name=$(cat "$1/kernel_version") + +# The Docker image comes with a prebuilt python environment with all tuxrun +# dependencies +source /build/.env/bin/activate + +xfs_tests=( "xfstests-ext4" "xfstests-btrfs" "xfstests-f2fs" "xfstests-xfs" ) + +mkdir -p /build/squad_json/ +parallel_log=$(mktemp -p ${ci_root}) + +for xfs_test in ${xfs_tests[@]}; do + echo "/build/tuxrun/run --runtime null --device qemu-riscv64 --kernel $KERNEL_PATH --tests ${xfs_test} --results /build/squad_json/${xfs_test}.json --log-file-text /build/squad_json/${xfs_test}.log --timeouts ${xfs_test}=480 --overlay /build/xfstests.tar.xz --rootfs $ROOTFS_PATH --boot-args \"rw\" || true" +done | parallel -j $(($(nproc)/4)) --colsep ' ' --joblog ${parallel_log} + +cat ${parallel_log} +rm ${parallel_log} + +for xfs_test in ${xfs_tests[@]}; do + # Convert JSON to squad datamodel + python3 /build/my-linux/.github/scripts/series/tuxrun_to_squad_json.py --result-path /build/squad_json/${xfs_test}.json --testsuite ${xfs_test} + python3 /build/my-linux/.github/scripts/series/generate_metadata.py --logs-path /build/squad_json/ --job-url ${GITHUB_JOB_URL} --branch ${GITHUB_BRANCH_NAME} + + curl --header "Authorization: token $SQUAD_TOKEN" --form tests=@/build/squad_json/${xfs_test}.squad.json --form log=@/build/squad_json/${xfs_test}.log --form metadata=@/build/squad_json/metadata.json https://squad.di.riseproject.dev/api/submit/riscv-linux/linux-all/${build_name}/qemu +done diff --git a/.github/workflows/kselftest.yml b/.github/workflows/kselftest.yml new file mode 100644 index 00000000000000..cf71d6d410b4fe --- /dev/null +++ b/.github/workflows/kselftest.yml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +name: linux-riscv-ci-kselftest + +defaults: + run: + shell: bash -leo pipefail {0} + +on: pull_request + +concurrency: + group: ci-test-${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + build-series: + if: ${{ startsWith(github.head_ref, 'linus') || endsWith(github.head_ref, '_manual') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/pw-builder:latest + volumes: + - /home/github/ramdisk/build:/build + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + steps: + - name: Configure git + run: | + git config --global --add safe.directory '*' + - name: Checkout git + run: | + mkdir -p /build/my-linux + cd /build/my-linux + git clone --filter=tree:0 --reference /build/gitref https://github.com/${{ github.repository }} . + git fetch origin ${{ github.event.pull_request.head.sha }} + git reset --hard ${{ github.event.pull_request.head.sha }} + git log -1 + - name: Run checks + continue-on-error: true + run: | + cd /build/my-linux && bash .github/scripts/kselftest.sh + - name: Collect logs + uses: actions/upload-artifact@v4 + with: + name: test-logs + path: /build/logs/* + - name: Publish to SQUAD + env: + SQUAD_TOKEN: ${{ secrets.SQUAD_TOKEN }} + GITHUB_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + /build/my-linux/.github/scripts/series/post_to_squad.py \ + --toplevel-log=/build/logs/kselftest.log \ + --selftest-bpf-log=/build/logs/test_kernel___rv64__kselftest__plain__gcc___ubuntu___server64__uboot_uefi__dt__kselftest-bpf.log \ + --selftest-log-dir=/build/logs/ \ + --job-url="${GITHUB_JOB_URL}" \ + --branch="${{ github.head_ref || github.ref_name }}" diff --git a/.github/workflows/patchwork.yml b/.github/workflows/patchwork.yml new file mode 100644 index 00000000000000..593de064624f8d --- /dev/null +++ b/.github/workflows/patchwork.yml @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +name: patchwork + +defaults: + run: + shell: bash -leo pipefail {0} + +on: pull_request + +concurrency: + group: ci-test-${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + patchwork_ci: + if: ${{ startsWith(github.head_ref, 'pw') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/pw-builder:latest + volumes: + - /home/github/ramdisk/build:/build + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + steps: + - name: Configure git + run: | + git config --global --add safe.directory '*' + git config --global user.name "Linux RISC-V bot" + git config --global user.email "linux.riscv.bot@gmail.com" + + - name: Checkout git + run: | + mkdir -p /build/linux + cd /build/linux + git clone --filter=tree:0 --reference /build/gitref https://github.com/${{ github.repository }} . + git fetch origin ${{ github.event.pull_request.head.sha }} + git reset --hard ${{ github.event.pull_request.head.sha }} + git log -1 + + - name: Create and activate virtual environment + run: | + python -m venv venv + source venv/bin/activate + echo "PATH=$PATH" >> $GITHUB_ENV + + - name: Install dependencies + run: | + pip install -r /build/linux/.github/scripts/requirements.txt + + - name: Run checks + env: + GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }} + GIST_TOKEN: ${{ secrets.GIST_TOKEN }} + PATCHWORK_TOKEN: ${{ secrets.PATCHWORK_TOKEN }} + PATCHWORK_USER: "" + EMAIL_TOKEN: ${{ secrets.EMAIL_TOKEN }} + run: | + cd /build/linux + ./.github/scripts/pw_ci.py -c ./.github/scripts/config.json -s /build/linux linux-riscv/linux + diff --git a/.github/workflows/series.yml b/.github/workflows/series.yml new file mode 100644 index 00000000000000..ba62596b47a899 --- /dev/null +++ b/.github/workflows/series.yml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +name: linux-riscv-ci-series + +defaults: + run: + shell: bash -leo pipefail {0} + +on: pull_request + +concurrency: + group: ci-test-${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + build-series: + if: ${{ endsWith(github.head_ref, '_test') || endsWith(github.head_ref, '_manual') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/pw-builder:latest + volumes: + - /home/github/ramdisk/build:/build + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + steps: + - name: Configure git + run: | + git config --global --add safe.directory '*' + - name: Checkout git + run: | + mkdir -p /build/my-linux + cd /build/my-linux + git clone --filter=tree:0 --reference /build/gitref https://github.com/${{ github.repository }} . + git fetch origin ${{ github.event.pull_request.head.sha }} + git reset --hard ${{ github.event.pull_request.head.sha }} + git log -1 + - name: Run checks + env: + SQUAD_TOKEN: ${{ secrets.SQUAD_TOKEN }} + GITHUB_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + continue-on-error: true + run: | + cd /build/my-linux && bash .github/scripts/series.sh + - name: Collect logs + uses: actions/upload-artifact@v4 + with: + name: test-logs + path: /build/logs/* diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 00000000000000..f787f02e3a866c --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,82 @@ +name: sync + +defaults: + run: + shell: bash -leo pipefail {0} + +on: + schedule: + - cron: "*/30 * * * *" + workflow_dispatch: + +concurrency: + group: ci-test-${{ github.workflow }}-${{ github.ref_name }} + +jobs: + sync_repo: + runs-on: ubuntu-latest + container: + image: ghcr.io/linux-riscv/pw-builder:latest + volumes: + - /home/github/ramdisk/build:/build + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + steps: + - name: Configure git + run: | + git config --global --add safe.directory '*' + git config --global user.name "Linux RISC-V bot" + git config --global user.email "linux.riscv.bot@gmail.com" + - name: Checkout git + run: | + mkdir -p /build/linux + cd /build/linux + git clone https://github.com/${{ github.repository }} . + git log --oneline -4 + - name: Sync Linus master + env: + ACTION_TOKEN: ${{ secrets.ACTION_TOKEN }} + run: | + cd /build/linux + ./.github/scripts/sync.sh "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" master master workflow "https://github.com/linux-riscv/github-ci.git" + - name: Sync RISC-V for-next + env: + ACTION_TOKEN: ${{ secrets.ACTION_TOKEN }} + run: | + cd /build/linux + ./.github/scripts/sync.sh "git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git" for-next master__riscv__for-next workflow__riscv__for-next "https://github.com/linux-riscv/github-ci.git" + - name: Sync RISC-V fixes + env: + ACTION_TOKEN: ${{ secrets.ACTION_TOKEN }} + run: | + cd /build/linux + ./.github/scripts/sync.sh "git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git" fixes master__riscv__fixes workflow__riscv__fixes "https://github.com/linux-riscv/github-ci.git" + + - name: Create and activate virtual environment + run: | + python -m venv venv + source venv/bin/activate + echo "PATH=$PATH" >> $GITHUB_ENV + + - name: Install dependencies + run: | + pip install -r /build/linux/.github/scripts/requirements.txt + + - name: Sync Patchwork + env: + GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }} + GIST_TOKEN: ${{ secrets.GIST_TOKEN }} + PATCHWORK_TOKEN: ${{ secrets.PATCHWORK_TOKEN }} + PATCHWORK_USER: "" + EMAIL_TOKEN: ${{ secrets.EMAIL_TOKEN }} + run: | + cd /build/linux + ./.github/scripts/sync_patchwork.py -c ./.github/scripts/config.json -s /build/linux -b workflow__riscv__fixes -b workflow__riscv__for-next -b workflow linux-riscv/linux + + - name: Clean PRs + env: + GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }} + GIST_TOKEN: ${{ secrets.GIST_TOKEN }} + run: | + cd /build/linux + ./.github/scripts/cleanup_pr.py linux-riscv/linux diff --git a/.github/workflows/testsuites.yml b/.github/workflows/testsuites.yml new file mode 100644 index 00000000000000..5844fc7d747cee --- /dev/null +++ b/.github/workflows/testsuites.yml @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: 2024 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +name: linux-riscv-ci-testsuites + +defaults: + run: + shell: bash -leo pipefail {0} + +on: pull_request + +concurrency: + group: ci-test-${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + build-kernel: + if: ${{ endsWith(github.head_ref, '_manual') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/linaro-tuxrun-dispatcher-riscv64:latest + volumes: + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + steps: + - name: Configure git + run: | + git config --global --add safe.directory '*' + - name: Checkout git + run: | + mkdir -p /build/my-linux + cd /build/my-linux + git clone --filter=tree:0 --reference /build/gitref https://github.com/${{ github.repository }} . + git fetch origin ${{ github.event.pull_request.head.sha }} + git reset --hard ${{ github.event.pull_request.head.sha }} + git log -1 + - name: Build kernel + continue-on-error: true + run: | + mkdir -p /build/logs/ + cd /build/my-linux && bash .github/scripts/build_ubuntu_defconfig.sh + - name: Collect build log + uses: actions/upload-artifact@v4 + with: + name: test-logs + path: /build/logs/* + - name: Publish kernel + uses: actions/upload-artifact@v4 + # TODO create modules directory and kernel separately + with: + name: test-kernel + path: /build/rv64__testsuites__plain__gcc/* + - name: Publish CI files + uses: actions/upload-artifact@v4 + with: + include-hidden-files: true + name: ci-files + path: /build/my-linux/.github/* + + run-ltp: + needs: build-kernel + if: ${{ endsWith(github.head_ref, '_manual') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/linaro-tuxrun-dispatcher-riscv64:latest + volumes: + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + - /tmp:/tmp + steps: + - name: Download pre-built kernel + uses: actions/download-artifact@v4 + with: + name: test-kernel + path: /build/test-kernel + - name: Download CI files + uses: actions/download-artifact@v4 + with: + name: ci-files + path: /build/my-linux/.github + - name: Run checks + env: + SQUAD_TOKEN: ${{ secrets.SQUAD_TOKEN }} + GITHUB_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + mkdir -p /build/logs/ + cd /build/my-linux && bash .github/scripts/ltp.sh /build/test-kernel | tee -i /build/logs/all.log + - name: Collect logs and json squad + uses: actions/upload-artifact@v4 + with: + name: test-logs-ltp + path: | + /build/logs/* + /build/squad_json/* + + run-libhugetlbfs: + needs: build-kernel + if: ${{ endsWith(github.head_ref, '_manual') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/linaro-tuxrun-dispatcher-riscv64:latest + volumes: + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + - /tmp:/tmp + steps: + - name: Download pre-built kernel + uses: actions/download-artifact@v4 + with: + name: test-kernel + path: /build/test-kernel + - name: Download CI files + uses: actions/download-artifact@v4 + with: + name: ci-files + path: /build/my-linux/.github + - name: Run checks + env: + SQUAD_TOKEN: ${{ secrets.SQUAD_TOKEN }} + GITHUB_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + mkdir -p /build/logs/ + cd /build/my-linux && bash .github/scripts/libhugetlbfs.sh /build/test-kernel | tee -i /build/logs/all.log + - name: Collect logs and json squad + uses: actions/upload-artifact@v4 + with: + name: test-logs-libhugetlbfs + path: | + /build/logs/* + /build/squad_json/* + + run-xfstests: + needs: build-kernel + if: ${{ endsWith(github.head_ref, '_manual') }} + runs-on: self-hosted + timeout-minutes: 50400 # 35 days + container: + image: ghcr.io/linux-riscv/linaro-tuxrun-dispatcher-riscv64:latest + volumes: + - /home/github/ccache:/build/ccache + - /home/github/gitref:/build/gitref + - /tmp:/tmp + steps: + - name: Download pre-built kernel + uses: actions/download-artifact@v4 + with: + name: test-kernel + path: /build/test-kernel + - name: Download CI files + uses: actions/download-artifact@v4 + with: + name: ci-files + path: /build/my-linux/.github + - name: Run checks + env: + SQUAD_TOKEN: ${{ secrets.SQUAD_TOKEN }} + GITHUB_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + mkdir -p /build/logs/ + cd /build/my-linux && bash .github/scripts/xfstests.sh /build/test-kernel | tee -i /build/logs/all.log + - name: Collect logs and json squad + uses: actions/upload-artifact@v4 + with: + name: test-logs-xfstests + path: | + /build/logs/* + /build/squad_json/* From 1bd2c38c020563c86a06dfe600371d6a95a7d207 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:19 -0700 Subject: [PATCH 03/21] dt-bindings: crypto: add Rambus CryptoManager Hub Add device tree binding schema for the CRI CryptoManager Hub (CMH) hardware crypto accelerator. The binding covers the parent SoC-level node with register region, interrupt, DMA properties, and per-core child nodes identified by compatible string and unit address. Register the 'cri' vendor prefix for Cryptography Research, Inc. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- .../devicetree/bindings/crypto/cri,cmh.yaml | 222 ++++++++++++++++++ .../devicetree/bindings/vendor-prefixes.yaml | 2 + 2 files changed, 224 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/cri,cmh.yaml diff --git a/Documentation/devicetree/bindings/crypto/cri,cmh.yaml b/Documentation/devicetree/bindings/crypto/cri,cmh.yaml new file mode 100644 index 00000000000000..db41132e0591d5 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/cri,cmh.yaml @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/cri,cmh.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: CRI CryptoManager Hub (CMH) Hardware Crypto Accelerator + +maintainers: + - Alex Ousherovitch + - Saravanakrishnan Krishnamoorthy + - Joel Wittenauer + +description: | + The CRI CryptoManager Hub (CMH) is a hardware cryptographic accelerator accessed + via a mailbox-based VCQ (Virtual Command Queue) interface. The host + writes VCQ command sequences into per-mailbox DMA queue buffers and + rings a doorbell; the CMH eSW processes them and signals completion + via interrupt. + + Supported algorithm families: SHA-2, SHA-3, SM3, AES, SM4, + ChaCha20-Poly1305, RSA, ECDSA, EdDSA, ECDH, SM2, ML-KEM, ML-DSA, + SLH-DSA, LMS, XMSS, DRBG. + +properties: + compatible: + const: cri,cmh + + reg: + maxItems: 1 + description: + SIC (System Interface Controller) MMIO region. Mailbox instance + registers are at offsets N * 0x1000 within this region. + + interrupts: + minItems: 1 + maxItems: 64 + description: + Per-mailbox completion/error interrupts from the CryptoManager Hub, + matching the real CMH ch_sys_interrupt_mbx[N-1:0] topology. + Entry i corresponds to MBX instance i. The driver maps each + configured mailbox (cri,mbx-instances) to its DT interrupt + index and registers a separate threaded IRQ handler per MBX. + + interrupt-names: + minItems: 1 + maxItems: 64 + items: + pattern: '^mbx[0-9]+$' + description: + Names for each mailbox interrupt, matching the interrupts array. + Format is "mbxN" where N is the mailbox instance index. + + cri,mbx-instances: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 64 + description: + Array of 0-based mailbox instance indices to configure. + Each index N maps to register offset N * 0x1000 within the + SIC region. If absent, defaults to instances 0 and 1. + + cri,mbx-slots-log2: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 64 + description: + Per-mailbox slot count as log2. Valid range 1..15. + Array length must match cri,mbx-instances. + Default is 5 (32 slots). + + cri,mbx-strides-log2: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 64 + description: + Per-mailbox stride (bytes per slot) as log2. Valid range 7..10. + Array length must match cri,mbx-instances. + Default is 7 (128 bytes per slot). + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^(hc|aes|sm4|sm3|hcq|qse|pke|drbg|ccp)@[0-9a-f]+$": + type: object + description: + Per-core-type child nodes. Each child represents one crypto core + instance available in the hardware. The driver enumerates these at + probe to discover which algorithm families are present. + + properties: + reg: + maxItems: 1 + description: + Hardware core ID for this core type (e.g. 0x02 for HC, 0x03 for AES). + Must match the CORE_ID_* values defined by the CMH hardware. + + cri,mbx: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Pin this core instance to a specific mailbox instance index. + Multiple child nodes of the same core type may each specify a + different cri,mbx value to spread instances across mailboxes. + When absent, the driver auto-assigns a mailbox via round-robin + across the instances listed in cri,mbx-instances. + + required: + - reg + + additionalProperties: false + +required: + - compatible + - reg + - interrupts + - "#address-cells" + - "#size-cells" + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + crypto@a4800000 { + compatible = "cri,cmh"; + reg = <0x0 0xa4800000 0x0 0x41000>; + interrupts = <1 2>; + interrupt-names = "mbx0", "mbx1"; + cri,mbx-instances = <0 1>; + cri,mbx-slots-log2 = <5 5>; + cri,mbx-strides-log2 = <7 7>; + #address-cells = <1>; + #size-cells = <0>; + + hc@2 { + reg = <0x02>; + }; + + aes@3 { + reg = <0x03>; + }; + + sm4@4 { + reg = <0x04>; + }; + + sm3@5 { + reg = <0x05>; + }; + + hcq@8 { + reg = <0x08>; + }; + + qse@9 { + reg = <0x09>; + }; + + pke@a { + reg = <0x0a>; + cri,mbx = <1>; + }; + + drbg@f { + reg = <0x0f>; + }; + + ccp@18 { + reg = <0x18>; + }; + }; + }; + + - | + /* Multi-instance: two AES cores on separate MBXes (future eSW support) */ + soc { + #address-cells = <2>; + #size-cells = <2>; + + crypto@a4800000 { + compatible = "cri,cmh"; + reg = <0x0 0xa4800000 0x0 0x41000>; + interrupts = <1 2>; + interrupt-names = "mbx0", "mbx1"; + cri,mbx-instances = <0 1>; + cri,mbx-slots-log2 = <5 5>; + cri,mbx-strides-log2 = <7 7>; + #address-cells = <1>; + #size-cells = <0>; + + hc@2 { + reg = <0x02>; + }; + + aes@3 { + reg = <0x03>; + cri,mbx = <0>; + }; + + /* Second AES instance at core ID 0x06, pinned to MBX 1 */ + aes@6 { + reg = <0x06>; + cri,mbx = <1>; + }; + + pke@a { + reg = <0x0a>; + cri,mbx = <1>; + }; + + drbg@f { + reg = <0x0f>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 396044f368e7cf..8b7187ea01943b 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -375,6 +375,8 @@ patternProperties: description: Crane Connectivity Solutions "^creative,.*": description: Creative Technology Ltd + "^cri,.*": + description: Cryptography Research, Inc. "^crystalfontz,.*": description: Crystalfontz America, Inc. "^csky,.*": From 2f3e2177b6f8798b1dc48f40d56058d5cb12e0b5 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:20 -0700 Subject: [PATCH 04/21] crypto: cmh - add core platform driver Add the CRI CryptoManager Hub (CMH) hardware crypto accelerator core platform driver. This patch provides: - Platform driver registration and probe/remove lifecycle - Hardware configuration and core discovery - Mailbox Queue Interface (MQI) for VCQ command submission - Transaction manager with async completion and backlog support - Result handler with threaded-IRQ completion - DMA buffer management - Debugfs instrumentation (when CONFIG_CRYPTO_DEV_CMH_DEBUG=y) - Sysfs attributes (fw_version, hw_version, product, algorithms) - Kconfig and Makefile integration No crypto algorithms are registered yet -- those follow in subsequent patches. The driver communicates with the hardware via a mailbox-based VCQ (Virtual Command Queue) interface. Each crypto operation is packed into VCQ command entries, submitted to a mailbox, and completed asynchronously via interrupt. MODULE_IMPORT_NS(CRYPTO_INTERNAL) imports two symbols: - crypto_cipher_setkey() [crypto/cipher.c, EXPORT_SYMBOL_NS_GPL] - crypto_cipher_encrypt_one() [crypto/cipher.c, EXPORT_SYMBOL_NS_GPL] These are the single-block cipher API used for software-fallback paths: CCM empty-input tag computation (2 ECB encryptions + XOR) and XCBC(SM4) empty-message workaround (3 ECB encryptions + XOR). No public wrapper exists; this is the same pattern used by in-tree crypto/ccm.c, crypto/cmac.c, and crypto/xcbc.c. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- Documentation/ABI/testing/debugfs-driver-cmh | 155 ++ Documentation/ABI/testing/sysfs-driver-cmh | 66 + Documentation/crypto/device_drivers/cmh.rst | 356 +++ Documentation/crypto/device_drivers/index.rst | 1 + drivers/crypto/Kconfig | 1 + drivers/crypto/Makefile | 1 + drivers/crypto/cmh/Kconfig | 46 + drivers/crypto/cmh/Makefile | 25 + drivers/crypto/cmh/cmh_config.c | 476 ++++ drivers/crypto/cmh/cmh_debugfs.c | 286 +++ drivers/crypto/cmh/cmh_dma.c | 373 ++++ drivers/crypto/cmh/cmh_main.c | 371 ++++ drivers/crypto/cmh/cmh_mqi.c | 355 +++ drivers/crypto/cmh/cmh_rh.c | 1145 ++++++++++ drivers/crypto/cmh/cmh_sysfs.c | 108 + drivers/crypto/cmh/cmh_txn.c | 1978 +++++++++++++++++ drivers/crypto/cmh/include/cmh.h | 27 + drivers/crypto/cmh/include/cmh_aes_abi.h | 98 + drivers/crypto/cmh/include/cmh_ccp_abi.h | 108 + drivers/crypto/cmh/include/cmh_config.h | 91 + drivers/crypto/cmh/include/cmh_debugfs.h | 90 + drivers/crypto/cmh/include/cmh_dma.h | 219 ++ drivers/crypto/cmh/include/cmh_drbg_abi.h | 67 + drivers/crypto/cmh/include/cmh_eac_abi.h | 44 + drivers/crypto/cmh/include/cmh_hc_abi.h | 162 ++ drivers/crypto/cmh/include/cmh_hcq_abi.h | 221 ++ drivers/crypto/cmh/include/cmh_kic_abi.h | 77 + drivers/crypto/cmh/include/cmh_mqi.h | 36 + drivers/crypto/cmh/include/cmh_pke_abi.h | 272 +++ drivers/crypto/cmh/include/cmh_qse_abi.h | 181 ++ drivers/crypto/cmh/include/cmh_registers.h | 146 ++ drivers/crypto/cmh/include/cmh_rh.h | 93 + drivers/crypto/cmh/include/cmh_rng.h | 31 + drivers/crypto/cmh/include/cmh_sm3_abi.h | 79 + drivers/crypto/cmh/include/cmh_sm4_abi.h | 101 + drivers/crypto/cmh/include/cmh_sys_abi.h | 148 ++ drivers/crypto/cmh/include/cmh_sysfs.h | 14 + drivers/crypto/cmh/include/cmh_txn.h | 463 ++++ drivers/crypto/cmh/include/cmh_vcq.h | 283 +++ 39 files changed, 8794 insertions(+) create mode 100644 Documentation/ABI/testing/debugfs-driver-cmh create mode 100644 Documentation/ABI/testing/sysfs-driver-cmh create mode 100644 Documentation/crypto/device_drivers/cmh.rst create mode 100644 drivers/crypto/cmh/Kconfig create mode 100644 drivers/crypto/cmh/Makefile create mode 100644 drivers/crypto/cmh/cmh_config.c create mode 100644 drivers/crypto/cmh/cmh_debugfs.c create mode 100644 drivers/crypto/cmh/cmh_dma.c create mode 100644 drivers/crypto/cmh/cmh_main.c create mode 100644 drivers/crypto/cmh/cmh_mqi.c create mode 100644 drivers/crypto/cmh/cmh_rh.c create mode 100644 drivers/crypto/cmh/cmh_sysfs.c create mode 100644 drivers/crypto/cmh/cmh_txn.c create mode 100644 drivers/crypto/cmh/include/cmh.h create mode 100644 drivers/crypto/cmh/include/cmh_aes_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_ccp_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_config.h create mode 100644 drivers/crypto/cmh/include/cmh_debugfs.h create mode 100644 drivers/crypto/cmh/include/cmh_dma.h create mode 100644 drivers/crypto/cmh/include/cmh_drbg_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_eac_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_hc_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_hcq_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_kic_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_mqi.h create mode 100644 drivers/crypto/cmh/include/cmh_pke_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_qse_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_registers.h create mode 100644 drivers/crypto/cmh/include/cmh_rh.h create mode 100644 drivers/crypto/cmh/include/cmh_rng.h create mode 100644 drivers/crypto/cmh/include/cmh_sm3_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_sm4_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_sys_abi.h create mode 100644 drivers/crypto/cmh/include/cmh_sysfs.h create mode 100644 drivers/crypto/cmh/include/cmh_txn.h create mode 100644 drivers/crypto/cmh/include/cmh_vcq.h diff --git a/Documentation/ABI/testing/debugfs-driver-cmh b/Documentation/ABI/testing/debugfs-driver-cmh new file mode 100644 index 00000000000000..3bbf903a45114c --- /dev/null +++ b/Documentation/ABI/testing/debugfs-driver-cmh @@ -0,0 +1,155 @@ +What: /sys/kernel/debug/cmh/mbx/vcqs_submitted +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Total number of VCQ command entries submitted to + mailbox N since the driver was loaded. + +What: /sys/kernel/debug/cmh/mbx/vcqs_completed +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Total number of VCQ command completions received + from mailbox N. + +What: /sys/kernel/debug/cmh/mbx/vcqs_errors +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Total number of error completions received from + mailbox N. + +What: /sys/kernel/debug/cmh/mbx/queue_full_count +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Number of times the transaction manager skipped + mailbox N because its in-flight queue was full. + +What: /sys/kernel/debug/cmh/mbx/max_queue_depth +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) High-water mark of in-flight transactions on + mailbox N. + +What: /sys/kernel/debug/cmh/mbx/inject_abort +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (WO) Write any value to inject an MBX_COMMAND_ABORT on + mailbox N. The abort triggers error-IRQ handling that + completes all in-flight transactions with -EIO and then + issues MBX_COMMAND_RESTART to resume the mailbox. + Only available when CONFIG_CRYPTO_DEV_CMH_DEBUG is enabled. + +What: /sys/kernel/debug/cmh/mbx/force_drain +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (WO) Write any value to unconditionally FLUSH and drain + all pending transactions on mailbox N, completing each + with -ECANCELED, and reset all recovery bookkeeping + (including the wedged flag). The mailbox is re-enabled + for new work immediately; no hardware health verification + is performed. Use as a last-resort recovery when the eSW + is unresponsive and normal ABORT/RESTART escalation has + not recovered the mailbox. + Only available when CONFIG_CRYPTO_DEV_CMH_DEBUG is enabled. + +What: /sys/kernel/debug/cmh/tm/cmq_posts +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Total number of cmh_tm_post_command() calls (one + per crypto request submitted to the transaction manager). + +What: /sys/kernel/debug/cmh/tm/cmq_depth_max +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) High-water mark of the command queue length. + +What: /sys/kernel/debug/cmh/tm/cmq_eagain_count +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Number of times the command queue was full and + returned -EAGAIN to the caller. + +What: /sys/kernel/debug/cmh/tm/backoff_count +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Number of times the transaction manager backed off + because all mailbox queues were full. + +What: /sys/kernel/debug/cmh/tm/async_timeout_count +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RO) Number of async crypto requests that timed out + waiting for hardware completion. + +What: /sys/kernel/debug/cmh/config/async_timeout_ms +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RW) Async request timeout in milliseconds. On timeout + the driver issues MBX_COMMAND_ABORT; if the eSW is + unresponsive, the watchdog escalates through RESTART, + FLUSH, and force-drain to bound D-state duration. + +What: /sys/kernel/debug/cmh/config/vcq_timeout_ms +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RW) VCQ command timeout in milliseconds. + +What: /sys/kernel/debug/cmh/config/slow_op_timeout_ms +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RW) Slow-operation timeout in milliseconds. Used for + operations known to take longer (e.g. RSA key generation, + PQC key generation). + +What: /sys/kernel/debug/cmh/config/drain_timeout_ms +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RW) Drain timeout in milliseconds. Maximum time to wait + for all in-flight transactions to complete during driver + removal or suspend. + +What: /sys/kernel/debug/cmh/config/watchdog_ms +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RW) Result-handler watchdog interval in milliseconds. + Detects missed IRQs, stuck mailboxes, and abort-stall + conditions. Clamped to a 10 ms minimum. + +What: /sys/kernel/debug/cmh/config/drbg_timeout_ms +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + (RW) DRBG self-seed timeout in milliseconds. diff --git a/Documentation/ABI/testing/sysfs-driver-cmh b/Documentation/ABI/testing/sysfs-driver-cmh new file mode 100644 index 00000000000000..62e593fac6fec3 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-cmh @@ -0,0 +1,66 @@ +What: /sys/devices/platform//fw_version +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + Reports the CryptoManager Hub embedded software (eSW) firmware + version as a 32-bit hexadecimal value read from the SIC + SW_VERSION register. + + Example: "0x00010002" + + Read-only. + +What: /sys/devices/platform//hw_version +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + Reports the CryptoManager Hub hardware version as a 32-bit + hexadecimal value read from the SIC HW_VERSION0 register. + + Example: "0x00000000" + + Read-only. + +What: /sys/devices/platform//boot_status +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + Reports the CryptoManager Hub boot status register as a 32-bit + hexadecimal value. This reflects the firmware boot + progress and final state: + + 0x00000066 - firmware booted (post-self-test) + other - firmware boot in progress or failed + + Read-only. + +What: /sys/devices/platform//mbx_available +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + Reports the mailbox availability bitmap as a 32-bit + hexadecimal value read from the SIC MBX_AVAILABILITY + register. Each set bit indicates a hardware mailbox + instance that the firmware has made available. + + Example: "0x00000003" (mailboxes 0 and 1 available) + + Read-only. + +What: /sys/devices/platform//mbx_count +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + Reports the number of mailboxes the driver has configured, + as a decimal integer. This reflects the driver's active + configuration (from DT properties or module parameters), + which may be fewer than illustrated by mbx_available. + + Example: "2" + + Read-only. diff --git a/Documentation/crypto/device_drivers/cmh.rst b/Documentation/crypto/device_drivers/cmh.rst new file mode 100644 index 00000000000000..4319b9ff1ab1a3 --- /dev/null +++ b/Documentation/crypto/device_drivers/cmh.rst @@ -0,0 +1,356 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================================== +CRI CryptoManager Hub (CMH) Driver +==================================== + +Overview +======== + +The ``cmh`` driver supports the CRI CryptoManager Hub hardware cryptographic +accelerator. The hardware is accessed through a mailbox-based VCQ +(Virtual Command Queue) interface: the driver writes command sequences +into per-mailbox DMA queue buffers and rings a doorbell register; the +CryptoManager Hub embedded software (eSW) processes the commands and signals +completion via a per-mailbox interrupt. + +The driver registers algorithms with the Linux kernel crypto subsystem +and exposes a management character device (``/dev/cmh_mgmt``) for +operations that have no standard crypto API binding. + +Hardware Interface +================== + +The CryptoManager Hub is presented as a platform device matched via Device Tree +(compatible ``"cri,cmh"``). The driver maps a single MMIO region +(the SIC -- System Interface Controller) whose sub-regions contain +per-mailbox doorbell, status, and command queue registers. + +The driver manages a configurable number of mailboxes (default 2). +Each mailbox has a configurable number of slots (default 32) and a +configurable stride (default 128 bytes per slot). The driver allocates +DMA-coherent memory for each mailbox queue during probe. + +Interrupts are per-mailbox completion/error interrupts. The driver +registers a threaded IRQ handler for each configured mailbox. + +The eSW is loaded independently of this driver -- typically by the +boot firmware or a platform-specific loader -- so the driver does not +use ``request_firmware()``. Instead it waits for the eSW to reach +mission mode during probe, bounded by ``fw_ready_timeout_ms``. + +Supported Algorithms +==================== + +The driver registers the following algorithm families: + +Hash (ahash) + SHA-224, SHA-256, SHA-384, SHA-512, SHA3-224, SHA3-256, SHA3-384, + SHA3-512, SHAKE-128, SHAKE-256, cSHAKE-128, cSHAKE-256, KMAC-128, + KMAC-256, SM3 (10 hash + 2 cSHAKE + 2 KMAC + 1 SM3 = 15 algorithms) + +HMAC (ahash) + HMAC-SHA-224, HMAC-SHA-256, HMAC-SHA-384, HMAC-SHA-512, + HMAC-SHA3-224, HMAC-SHA3-256, HMAC-SHA3-384, HMAC-SHA3-512 + (8 algorithms) + +Symmetric Ciphers (skcipher) + AES: ECB, CBC, CTR, CFB, XTS (5 algorithms) + SM4: ECB, CBC, CTR, CFB, XTS (5 algorithms) + ChaCha20 (1 algorithm) + +AEAD + AES-GCM, AES-CCM (2 algorithms) + SM4-GCM, SM4-CCM (2 algorithms) + ``rfc7539(chacha20,poly1305)``, ``rfc7539esp(chacha20,poly1305)`` + (2 algorithms) + +MAC (ahash) + CMAC(AES) (1 algorithm) + CMAC(SM4), XCBC(SM4) (2 algorithms) + Poly1305 (1 algorithm) + +Public-Key, Key Agreement, and PQC Signatures + RSA (akcipher, 1 algorithm) + ECDSA P-256, P-384, P-521 (sig, 3 algorithms) + SM2 (sig, verify-only, 1 algorithm) + ECDH P-256, P-384, X25519 (kpp, 3 algorithms) + ML-DSA-44, ML-DSA-65, ML-DSA-87 (sig, 3 algorithms) + SLH-DSA: all 12 parameter sets (sig, 12 algorithms) + LMS, LMS-HSS (sig, verify-only, 2 algorithms) + XMSS, XMSS-MT (sig, verify-only, 2 algorithms) + (ML-KEM keygen/encaps/decaps is available via ``/dev/cmh_mgmt`` + only -- see `Limitations`_.) + +Hardware RNG + DRBG-backed hwrng (``/dev/hwrng``, 1 algorithm) + +All algorithm driver names use the ``cri-cmh-`` prefix (e.g. +``cri-cmh-sha256``, ``cri-cmh-ecb-aes``, ``cri-cmh-gcm-aes``, +``cri-cmh-mldsa44``). Names generally follow the kernel's hyphenated +template name; families that have no kernel template (e.g. ML-DSA) use +the concatenated upstream algorithm name (``mldsa44``). + +Most algorithms register at priority 300 (301 for AES-CCM). +The ML-DSA ``sig`` algorithms register at priority 5001 to +outrank the kernel's generic software ML-DSA (priority 5000, which is +verify-only); the CMH driver provides full hardware sign and verify. + +Request model +------------- + +All crypto API operations are asynchronous: the driver queues each +request to its transaction-manager kthread and returns +``-EINPROGRESS``, invoking the caller's completion callback when the +hardware finishes. Requests that set ``CRYPTO_TFM_REQ_MAY_BACKLOG`` +are queued on a backlog of up to ``backlog_max_depth`` entries when the +command queue is full; without that flag a full queue is reported as +``-EBUSY``. Hardware or eSW failures surface as ``-EIO``, malformed +requests as ``-EINVAL``, oversized requests as ``-EMSGSIZE`` or +``-EINVAL`` (see `Data-Size Limits`_), and unresponsive hardware as +``-ETIMEDOUT``. The ``/dev/cmh_mgmt`` ioctls are, by contrast, +synchronous -- each ioctl blocks until the hardware completes. + +Driver Architecture +=================== + +The driver is structured as follows: + +Platform Driver + Matches DT compatible ``"cri,cmh"``. Probe initializes all + subsystems in order; remove tears them down in reverse. + +Configuration + Parses DT properties and module parameter overrides. Validates + mailbox counts, slot sizes, and stride values. + +MQI (Mailbox Queue Interface) + Allocates DMA-coherent queue memory per mailbox. Manages slot + allocation, VCQ command writing, and doorbell ringing. + +Transaction Manager + A dedicated kthread dequeues crypto requests from a central command + queue, builds VCQ command sequences, and submits them to mailbox + slots. Completion is signaled via wait queues. + +Response Handler + Per-mailbox threaded IRQ handlers walk completed slots, parse + results, and fire request completions. A configurable watchdog + timer (the ``watchdog_ms`` debugfs knob, default 200 ms) detects + stuck requests and escalates through ABORT, RESTART, and FLUSH + recovery. + +Key Management (``/dev/cmh_mgmt``) + A misc character device providing ioctl-based access to datastore + key CRUD, key derivation (KIC), PKE operations (EdDSA, SM2), + PQC operations (ML-KEM, ML-DSA, SLH-DSA), + EAC error register readback, and DRBG runtime configuration. + See ``Documentation/ABI/testing/cmh-mgmt`` for the full ioctl list. + +Power Management + The driver implements ``DEFINE_SIMPLE_DEV_PM_OPS`` suspend/resume. + On suspend, the transaction-manager kthread is stopped and pending + transactions are drained, waiting up to ``drain_timeout_ms`` + (default 10000 ms); resume restarts the kthread. + +Module Parameters +================= + +The driver defines four production module parameters and five +debug-only parameters (compiled only with +``CONFIG_CRYPTO_DEV_CMH_DEBUG``). In production, all mailbox topology, +per-core affinity, slot counts, strides, and timeout tuning are taken +from Device Tree properties, not module parameters. The debug-only +parameters exist solely to force alternate geometries at ``insmod`` +time during bringup and validation (for example, to drive the +mailbox-contention and cross-mailbox dispatch paths without +rebuilding the Device Tree); they default to "use the DT value" +and have no effect in a production build. + +Production: + +``fw_ready_timeout_ms`` (uint, default 5000, RO) + Timeout in milliseconds to wait for CMH eSW to reach mission mode + during probe. + +``cmq_max_depth`` (uint, default 256, RO) + Maximum number of pending commands in the central Command Message + Queue. + +``backlog_max_depth`` (uint, default 1024, RO) + Maximum depth of the backlog queue for ``CRYPTO_TFM_REQ_MAY_BACKLOG`` + requests. Set to 0 to disable backlogs. + +``hwrng_quality`` (int, default 0, RO) + Quality value passed to ``hwrng_register()``. 0 disables kernel CRNG + seeding; 1-1024 sets the quality directly. + +Debug-only (``CONFIG_CRYPTO_DEV_CMH_DEBUG``): + +``mbx_count_override`` (uint, default 0, RO) + Override the DT mailbox count (0 = use DT) to force fewer + mailboxes than the hardware provides. + +``mbx_slots_override`` (uint, default 0, RO) + Override all MBX slots_log2 values (0 = use DT). + +``mbx_round_robin`` (bool, default false, RO) + Ignore DT ``cri,mbx`` affinity pins and round-robin all cores + across the configured mailboxes (0 = use DT affinity). Restores + the unpinned dispatch that exercises cross-mailbox distribution. + +``drbg_config`` (charp, default "auto", RO) + DRBG configuration at probe: ``"auto"`` (normal) or ``"skip"`` + (skip initial DRBG configuration). + +``skip_fw_check`` (bool, default false, RO) + Skip the SIC boot status and eSW mission-mode checks at probe. + Allows the module to load before the eSW has booted. + +Runtime-tunable timeout knobs are exposed via debugfs rather than +module parameters; see `debugfs Counters`_ below. + +sysfs Attributes +================ + +The driver exposes five read-only attributes under the platform +device sysfs directory: ``fw_version``, ``hw_version``, +``boot_status``, ``mbx_available``, and ``mbx_count``. See +``Documentation/ABI/testing/sysfs-driver-cmh`` for the authoritative +per-attribute description. + +debugfs Counters +================ + +When built with ``CONFIG_CRYPTO_DEV_CMH_DEBUG``, the driver creates +``/sys/kernel/debug/cmh/`` with three groups: per-mailbox counters +(``mbxN/``), transaction-manager statistics (``tm/``), and +runtime-tunable timeout knobs (``config/``, including +``drain_timeout_ms`` and ``watchdog_ms``). See +``Documentation/ABI/testing/debugfs-driver-cmh`` for the authoritative +per-file description. + +Device Tree Binding +=================== + +See ``Documentation/devicetree/bindings/crypto/cri,cmh.yaml`` for the +full DT binding schema and complete, schema-validated examples +(including the per-mailbox topology properties ``cri,mbx-instances``, +``cri,mbx-slots-log2``, and ``cri,mbx-strides-log2``). When those +properties are omitted the driver falls back to two mailboxes +(instances 0 and 1) with the slot/stride defaults described above. + +User-Space Interfaces +===================== + +``/dev/cmh_mgmt`` + Management character device. Opening it requires ``CAP_SYS_ADMIN``. + See ``Documentation/ABI/testing/cmh-mgmt`` for ioctl documentation. + The UAPI header is ````. + +In-kernel crypto API + All algorithms register with the standard kernel crypto API and are + consumed by in-kernel users (dm-crypt, fscrypt, IPsec, kTLS, etc.). + + Keys provisioned inside the hardware via ``/dev/cmh_mgmt`` are + referenced by an opaque hardware key identifier and are operated on + through the ``/dev/cmh_mgmt`` ioctl interface, without ever exposing + plaintext key material to user space. See + ``Documentation/ABI/testing/cmh-mgmt`` for key provisioning. + +``/dev/hwrng`` + The DRBG-backed hardware RNG is available as a standard hwrng device. + +Limitations +=========== + +- LMS and XMSS support verify-only (no sign/keygen in hardware for + stateful hash-based signatures). +- SM2 sig registration is verify-only (sign via ``/dev/cmh_mgmt`` ioctl). +- EdDSA (Ed25519/Ed448) is available only through ``/dev/cmh_mgmt`` + ioctls; no kernel ``sig`` registration. +- ML-KEM operations (encapsulate/decapsulate/keygen) are available only + through ``/dev/cmh_mgmt`` ioctls; no standard kernel crypto API + binding exists for KEM. + +Data-Size Limits +================ + +The driver imposes data-size limits on several APIs. These are +driver-level safety caps for kernel memory allocation unless noted +otherwise. + +Symmetric / AEAD / MAC linearization caps: + +============================== ======= ======================================= +Scope Limit Origin +============================== ======= ======================================= +AES skcipher 32 MiB Driver-imposed DMA linearization cap +SM4 skcipher 32 MiB Driver-imposed DMA linearization cap +All AEAD + ChaCha20 skcipher 1 MiB Driver-imposed DMA linearization cap +============================== ======= ======================================= + +MAC and keyed-hash algorithms that buffer all input in kernel memory +(hardware lacks context save/restore): + +==================== ======= ============================================= +Algorithm Limit Reason +==================== ======= ============================================= +``cmac(aes)`` 64 KiB AES core has no external save/restore +``cmac(sm4)`` 64 KiB SM4 core has no external save/restore +``xcbc(sm4)`` 64 KiB SM4 core has no external save/restore +``poly1305`` 64 KiB CCP core has no external save/restore +``hmac(sha*)`` 64 KiB HMAC save/restore not supported (see below) +``hmac(sha3-*)`` 64 KiB HMAC save/restore not supported (see below) +``kmac128`` 64 KiB eSW rejects save when outlen != 0 +``kmac256`` 64 KiB eSW rejects save when outlen != 0 +==================== ======= ============================================= + +HMAC save/restore is unsupported by the eSW firmware. For HMAC-SHA3, +exposing the Keccak sponge state would allow key recovery because the +sponge permutation is invertible; HMAC-SHA2 save/restore is likewise +not exposed by the eSW. + +HMAC ``.export()``/``.import()`` (used for request cloning) is limited +to a single-page accumulated-data window of 4092 bytes (one page minus +a 4-byte length header), since the crypto subsystem pre-allocates the +state buffer per request. Cloning a request that has accumulated more +input than this window fails. + +Requests exceeding the limit are rejected with ``-EINVAL``. Pure hash +algorithms (SHA-2, SHA-3, SHAKE, cSHAKE, SM3) have no data limit because +the hardware supports incremental save/restore. + +cSHAKE uses save/restore for ``.export()``/``.import()`` but accumulates +data in ``.update()`` by design (the Keccak sponge has no block-alignment +boundary to trigger per-update HW submission, and HC_CMD_GATHER amortizes +the cost into a single finalize-time submission). + +Asymmetric / PQC algorithm limits: + +============================== ========= ==================================== +Scope Limit Origin +============================== ========= ==================================== +RSA key size 4096 bit HW-imposed +ML-DSA message 10 KiB eSW-imposed (QSE ABI) +SLH-DSA message 128 B eSW-imposed (HCQ ABI) +SLH-DSA context 255 B Spec-imposed (FIPS 205) +LMS public key 60 B eSW-imposed (HCQ ABI) +LMS message 256 B eSW-imposed (HCQ ABI) +LMS signature 13,364 B eSW-imposed (HCQ ABI) +XMSS public key 136 B eSW-imposed (HCQ ABI) +XMSS message 64 B eSW-imposed (HCQ ABI) +XMSS signature 27,688 B eSW-imposed (HCQ ABI) +SM2 encrypt message 32 B eSW KDF (single SM3 block) +============================== ========= ==================================== + +Miscellaneous limits: + +============================== ========= ==================================== +Scope Limit Origin +============================== ========= ==================================== +cSHAKE/KMAC customization 256 B VCQ slot layout constraint +KIC HKDF key 64 B Partially eSW-derived +KIC HKDF label 56 B VCQ slot layout constraint +Key/blob mgmt ioctls 256 KiB Driver-imposed sanity cap +============================== ========= ==================================== diff --git a/Documentation/crypto/device_drivers/index.rst b/Documentation/crypto/device_drivers/index.rst index c81d311ac61b89..c0247fc97bf851 100644 --- a/Documentation/crypto/device_drivers/index.rst +++ b/Documentation/crypto/device_drivers/index.rst @@ -6,4 +6,5 @@ Hardware Device Driver Specific Documentation .. toctree:: :maxdepth: 1 + cmh octeontx2 diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 216a00bad5d706..777c12496bdfd1 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -839,4 +839,5 @@ source "drivers/crypto/starfive/Kconfig" source "drivers/crypto/inside-secure/eip93/Kconfig" source "drivers/crypto/ti/Kconfig" +source "drivers/crypto/cmh/Kconfig" endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 5a950c7abc393d..7ad69348f0f0cd 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -47,3 +47,4 @@ obj-y += intel/ obj-y += starfive/ obj-y += cavium/ obj-y += ti/ +obj-$(CONFIG_CRYPTO_DEV_CMH) += cmh/ diff --git a/drivers/crypto/cmh/Kconfig b/drivers/crypto/cmh/Kconfig new file mode 100644 index 00000000000000..fa5adeca251240 --- /dev/null +++ b/drivers/crypto/cmh/Kconfig @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# CRI CryptoManager Hub (CMH) hardware crypto accelerator +# + +config CRYPTO_DEV_CMH + tristate "CRI CryptoManager Hub (CMH) hardware crypto accelerator" + depends on CRYPTO && OF && HAS_IOMEM && (64BIT || COMPILE_TEST) + select CRYPTO_HASH + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_AKCIPHER + select CRYPTO_SIG + select CRYPTO_KPP + select CRYPTO_ECC + select CRYPTO_RSA + select CRYPTO_AES + select CRYPTO_CCM + select CRYPTO_SM4_GENERIC + select HW_RANDOM + help + Driver for the CRI CryptoManager Hub (CMH) hardware crypto accelerator. + Accesses the hardware via a mailbox-based VCQ (Virtual Command + Queue) interface and registers algorithms with the kernel + crypto subsystem. + + Supported algorithm families: AES (ECB/CBC/CTR/XTS/CFB), + SM4 (ECB/CBC/CTR/XTS/CFB), ChaCha20-Poly1305, AES-GCM, AES-CCM, + SHA-2, SHA-3, SHAKE, CSHAKE, KMAC, SM3, HMAC, AES-CMAC, + SM4-CMAC, SM4-XCBC, RSA, ECDSA, ECDH, SM2, and DRBG (hwrng). + Ioctl-only algorithms: EdDSA, ML-KEM. + + To compile this driver as a module, choose M here. + +config CRYPTO_DEV_CMH_DEBUG + bool "CMH debug instrumentation (debugfs counters)" + depends on CRYPTO_DEV_CMH && DEBUG_FS + help + Enable per-mailbox debugfs counters under + /sys/kernel/debug/cmh/ for the CMH driver. + Exposes VCQ submit/complete/error counts, queue depth + high-water marks, and transaction manager backoff statistics. + + Useful for bringup, validation, and performance analysis. + Not recommended for production. + diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile new file mode 100644 index 00000000000000..0a4591c9fd8608 --- /dev/null +++ b/drivers/crypto/cmh/Makefile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the CRI CryptoManager Hub (CMH) hardware crypto accelerator driver. +# + +obj-$(CONFIG_CRYPTO_DEV_CMH) += cmh.o + +cmh-y := \ + cmh_main.o \ + cmh_config.o \ + cmh_mqi.o \ + cmh_txn.o \ + cmh_rh.o \ + cmh_dma.o \ + cmh_sysfs.o + +ccflags-y += -I$(src)/include + +# Suppress -Woverride-init for the [0 ... N] = -1 range-initializer pattern +# (standard kernel idiom for sparse lookup tables with a default value). +CFLAGS_cmh_config.o += -Wno-override-init + +# Debug instrumentation: per-mailbox debugfs counters. +# cmh_debugfs.o is linked into the composite cmh.o (same tristate). +cmh-$(CONFIG_CRYPTO_DEV_CMH_DEBUG) += cmh_debugfs.o diff --git a/drivers/crypto/cmh/cmh_config.c b/drivers/crypto/cmh/cmh_config.c new file mode 100644 index 00000000000000..4631eebb1556a3 --- /dev/null +++ b/drivers/crypto/cmh/cmh_config.c @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Configuration from Device Tree + * + * The CMH device tree node provides: + * - reg: SIC base + size (mandatory) + * - interrupts: per-MBX IRQs (mandatory for IRQ mode) + * - cri,mbx-instances: array of MBX instance IDs + * - cri,mbx-slots-log2: per-MBX slot count as log2 + * - cri,mbx-strides-log2: per-MBX stride as log2 + * + * Per-core-type child nodes (e.g. aes@3, pke@a): + * - reg: hardware core ID (CORE_ID_* from cmh_vcq.h) + * - cri,mbx: (optional) pin to a specific MBX index + * + * Module parameters (non-topology): + * - fw_ready_timeout_ms: CMH eSW mission-mode boot timeout + * (hwrng_quality, cmq_max_depth, backlog_max_depth live in other files) + */ + +#include +#include +#include +#include + +#include "cmh_config.h" +#include "cmh_dma.h" + +/* -- Module parameters ------------------------------------------------- */ + +static unsigned int fw_ready_timeout_ms = CMH_DEFAULT_FW_READY_TIMEOUT_MS; +module_param(fw_ready_timeout_ms, uint, 0444); +MODULE_PARM_DESC(fw_ready_timeout_ms, + "Timeout in ms to wait for CMH eSW mission mode (default 5000)"); + +/* + * Debug-only MBX overrides for stress testing. + * When non-zero, these override the corresponding DT values, enabling + * contention stress tests to force a minimal MBX config + * (e.g. mbx_count_override=1 mbx_slots_override=1 for 1 MBX, 2 slots). + */ +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +static unsigned int mbx_count_override; +module_param(mbx_count_override, uint, 0444); +MODULE_PARM_DESC(mbx_count_override, + "[debug] Override DT MBX count (0 = use DT, default: 0)"); + +static unsigned int mbx_slots_override; +module_param(mbx_slots_override, uint, 0444); +MODULE_PARM_DESC(mbx_slots_override, + "[debug] Override all MBX slots_log2 (0 = use DT, default: 0)"); + +static bool mbx_round_robin; +module_param(mbx_round_robin, bool, 0444); +MODULE_PARM_DESC(mbx_round_robin, + "[debug] Ignore DT cri,mbx pins and round-robin all cores across MBXes (0 = use DT affinity, default: 0)"); +#endif + +/* -- Core ID -> core_type lookup --------------------------------------- */ + +/* + * Map hardware core IDs (from DT child "reg") to enum cmh_core_type. + * + * Entries set to -1 are not dispatchable crypto cores: system cores + * (SYS, DMA, KIC, TIC, MPU, EMC, EAC) and the DRBG singleton + * (handled separately in cmh_rng.c). + */ +static const int core_id_to_type[CORE_ID_NUM] = { + [0 ... CORE_ID_NUM - 1] = -1, + [CORE_ID_HC] = CMH_CORE_HC, + [CORE_ID_AES] = CMH_CORE_AES, + [CORE_ID_SM4] = CMH_CORE_SM4, + [CORE_ID_SM3] = CMH_CORE_SM3, + [CORE_ID_CCP] = CMH_CORE_CCP, + [CORE_ID_PKE] = CMH_CORE_PKE, + [CORE_ID_QSE] = CMH_CORE_QSE, + [CORE_ID_HCQ] = CMH_CORE_HCQ, +}; + +/* Human-readable names for error messages */ +static const char * const core_type_names[CMH_NUM_CORE_TYPES] = { + [CMH_CORE_HC] = "hc", + [CMH_CORE_AES] = "aes", + [CMH_CORE_SM4] = "sm4", + [CMH_CORE_SM3] = "sm3", + [CMH_CORE_CCP] = "ccp", + [CMH_CORE_PKE] = "pke", + [CMH_CORE_QSE] = "qse", + [CMH_CORE_HCQ] = "hcq", +}; + +/* -- DT child node enumeration ----------------------------------------- */ + +static int cmh_config_populate_cores(struct cmh_config *cfg, + struct device_node *np) +{ + struct device_node *child; + u32 core_id, mbx_val; + int type, ret; + + for_each_child_of_node(np, child) { + ret = of_property_read_u32(child, "reg", &core_id); + if (ret) { + dev_warn(cmh_dev(), + "DT child %pOFn: missing 'reg', skipping\n", + child); + continue; + } + + if (core_id >= CORE_ID_NUM) { + dev_info(cmh_dev(), + "DT child %pOFn: core_id 0x%02x unknown, skipping\n", + child, core_id); + continue; + } + + type = core_id_to_type[core_id]; + if (type < 0) { + /* Not a dispatchable crypto core (DRBG, SYS, etc.) */ + dev_dbg(cmh_dev(), + "DT child %pOFn: core_id 0x%02x not dispatchable\n", + child, core_id); + continue; + } + + if (cfg->core_types[type].num_instances >= + CMH_MAX_CORE_INSTANCES) { + dev_err(cmh_dev(), + "DT: too many instances for %s (max %u)\n", + core_type_names[type], + CMH_MAX_CORE_INSTANCES); + of_node_put(child); + return -EINVAL; + } + + { + struct cmh_core_type_cfg *ct = &cfg->core_types[type]; + u32 idx = ct->num_instances; + + ct->core_ids[idx] = core_id; + ret = of_property_read_u32(child, "cri,mbx", &mbx_val); + ct->mbx[idx] = ret ? -1 : (s32)mbx_val; +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG + /* + * Debug knob for the cross-core stress test: drop the + * DT MBX pin so cmh_core_select_instance() round-robins + * this core across all configured MBXes (the unpinned + * dispatch behaviour exercised before cri,mbx affinity + * was added to the baseline device tree). + */ + if (mbx_round_robin) + ct->mbx[idx] = -1; +#endif + ct->num_instances++; + } + } + + return 0; +} + +/* -- Validation -------------------------------------------------------- */ + +static int cmh_config_validate_core_types(struct cmh_config *cfg) +{ + unsigned int i, j, k; + + for (i = 0; i < CMH_NUM_CORE_TYPES; i++) { + struct cmh_core_type_cfg *ct = &cfg->core_types[i]; + const char *name = core_type_names[i]; + + /* Zero instances is valid -- core absent from DT */ + if (ct->num_instances == 0) + continue; + + if (ct->num_instances > CMH_MAX_CORE_INSTANCES) { + dev_err(cmh_dev(), "%s: num_instances %u > max %u\n", + name, ct->num_instances, + CMH_MAX_CORE_INSTANCES); + return -EINVAL; + } + + /* Validate MBX indices */ + for (j = 0; j < ct->num_instances; j++) { + if (ct->mbx[j] >= 0 && + (u32)ct->mbx[j] >= cfg->mbx_count) { +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG + if (mbx_count_override > 0) { + dev_info(cmh_dev(), + "%s: mbx[%u]=%d >= overridden mbx_count %u, auto-assigning\n", + name, j, ct->mbx[j], + cfg->mbx_count); + ct->mbx[j] = -1; + continue; + } +#endif + dev_err(cmh_dev(), "%s: mbx[%u]=%d >= mbx_count %u\n", + name, j, ct->mbx[j], + cfg->mbx_count); + return -EINVAL; + } + } + + /* No duplicate core IDs within this type */ + for (j = 1; j < ct->num_instances; j++) { + for (k = 0; k < j; k++) { + if (ct->core_ids[j] == ct->core_ids[k]) { + dev_err(cmh_dev(), + "%s: duplicate core_id 0x%02x at [%u] and [%u]\n", + name, ct->core_ids[j], + k, j); + return -EINVAL; + } + } + } + + /* No duplicate MBX within this type (if explicit) */ + for (j = 1; j < ct->num_instances; j++) { + if (ct->mbx[j] < 0) + continue; + for (k = 0; k < j; k++) { + if (ct->mbx[k] == ct->mbx[j]) { + dev_err(cmh_dev(), + "%s: duplicate mbx %d at [%u] and [%u]\n", + name, ct->mbx[j], k, j); + return -EINVAL; + } + } + } + + /* All core IDs must fit in VCQ 8-bit field */ + for (j = 0; j < ct->num_instances; j++) { + if (ct->core_ids[j] > CORE_ID_MAX) { + dev_err(cmh_dev(), + "%s: core_ids[%u]=0x%02x > CORE_ID_MAX\n", + name, j, ct->core_ids[j]); + return -EINVAL; + } + } + } + + /* Cross-type: no core ID used by more than one type */ + for (i = 0; i < CMH_NUM_CORE_TYPES; i++) { + struct cmh_core_type_cfg *ct_i = &cfg->core_types[i]; + + for (j = i + 1; j < CMH_NUM_CORE_TYPES; j++) { + struct cmh_core_type_cfg *ct_j = &cfg->core_types[j]; + + for (k = 0; k < ct_i->num_instances; k++) { + unsigned int m; + + for (m = 0; m < ct_j->num_instances; m++) { + if (ct_i->core_ids[k] != + ct_j->core_ids[m]) + continue; + dev_err(cmh_dev(), + "core_id 0x%02x conflict: %s[%u] and %s[%u]\n", + ct_i->core_ids[k], + core_type_names[i], k, + core_type_names[j], m); + return -EINVAL; + } + } + } + } + + return 0; +} + +static int cmh_config_validate(struct cmh_config *cfg) +{ + unsigned int i, j; + unsigned long max_instance_end; + + if (cfg->mbx_count == 0 || cfg->mbx_count > CMH_MAX_CONFIGURED_MBX) { + dev_err(cmh_dev(), "mbx_count %u out of range (1..%u)\n", + cfg->mbx_count, CMH_MAX_CONFIGURED_MBX); + return -EINVAL; + } + + for (i = 0; i < cfg->mbx_count; i++) { + struct cmh_mbx_config *m = &cfg->mailboxes[i]; + + if (m->instance >= CMH_MAX_MBX_INSTANCES) { + dev_err(cmh_dev(), "mbx_instances[%u]=%u >= %u\n", + i, m->instance, CMH_MAX_MBX_INSTANCES); + return -EINVAL; + } + + if (m->slots_log2 < CMH_MBX_SLOTS_LOG2_MIN || + m->slots_log2 > CMH_MBX_SLOTS_LOG2_MAX) { + dev_err(cmh_dev(), "mbx_slots[%u]=%u out of range (%u..%u)\n", + i, m->slots_log2, + CMH_MBX_SLOTS_LOG2_MIN, CMH_MBX_SLOTS_LOG2_MAX); + return -EINVAL; + } + + if (m->stride_log2 < CMH_MBX_STRIDE_LOG2_MIN || + m->stride_log2 > CMH_MBX_STRIDE_LOG2_MAX) { + dev_err(cmh_dev(), "mbx_strides[%u]=%u out of range (%u..%u)\n", + i, m->stride_log2, + CMH_MBX_STRIDE_LOG2_MIN, CMH_MBX_STRIDE_LOG2_MAX); + return -EINVAL; + } + + /* Check for duplicate instance indices */ + for (j = 0; j < i; j++) { + if (cfg->mailboxes[j].instance == m->instance) { + dev_err(cmh_dev(), "duplicate instance %u at indices %u and %u\n", + m->instance, j, i); + return -EINVAL; + } + } + } + + /* Ensure SIC region is large enough for all requested instances */ + max_instance_end = 0; + for (i = 0; i < cfg->mbx_count; i++) { + unsigned long end = ((unsigned long)cfg->mailboxes[i].instance + 1) + << CMH_MBX_INSTANCE_SHIFT; + if (end > max_instance_end) + max_instance_end = end; + } + + if (max_instance_end > cfg->sic_size) { + dev_err(cmh_dev(), "sic_size 0x%zx too small for instance requiring 0x%lx\n", + cfg->sic_size, max_instance_end); + return -EINVAL; + } + + return 0; +} + +/* -- Public Interface -------------------------------------------------- */ + +/** + * cmh_config_init() - Initialize device configuration from platform/DT data + * @cfg: Configuration structure to populate + * @pdev: Platform device providing DT node and resources + * + * Parse the "cri,cmh" device tree node for MMIO base address, interrupt + * specifiers, and per-mailbox properties (instance indices, slot counts, + * strides). When DT properties are absent, fall back to module parameter + * arrays. Populate per-core-type instance configuration from module + * parameters, then validate the complete configuration. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_config_init(struct cmh_config *cfg, struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct resource *res; + unsigned int i; + int ret, irq, nr; + + if (!np) { + dev_err(&pdev->dev, "device tree node required\n"); + return -ENODEV; + } + + cfg->of_node = np; + + /* SIC base + size from DT "reg" property (mandatory) */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(cmh_dev(), "missing DT reg resource\n"); + return -EINVAL; + } + cfg->sic_base = res->start; + cfg->sic_size = resource_size(res); + + /* + * IRQ resolution order: + * 1. Platform-level IRQ from the first DT "interrupts" entry. + * 2. If absent (cfg->irq == -1), cmh_rh_resolve_irqs() tries + * per-MBX of_irq_get() for per-mailbox interrupt routing. + * 3. If no IRQs are available at all, the response handler + * falls back to watchdog-timer polling (200 ms default). + */ + irq = platform_get_irq_optional(pdev, 0); + cfg->irq = (irq >= 0) ? irq : -1; + + cfg->sic_mapped = NULL; + cfg->fw_ready_timeout_ms = fw_ready_timeout_ms; + + /* -- MBX configuration from DT --------------------------------- */ + + nr = of_property_count_u32_elems(np, "cri,mbx-instances"); + if (nr <= 0) { + dev_err(cmh_dev(), "missing or empty cri,mbx-instances in DT\n"); + return -EINVAL; + } + if ((unsigned int)nr > CMH_MAX_CONFIGURED_MBX) { + dev_err(cmh_dev(), "too many MBX instances in DT (%d > %u)\n", + nr, CMH_MAX_CONFIGURED_MBX); + return -EINVAL; + } + cfg->mbx_count = nr; + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG + if (mbx_count_override > 0) { + if (mbx_count_override > cfg->mbx_count) { + dev_err(cmh_dev(), + "mbx_count_override %u > DT count %u\n", + mbx_count_override, cfg->mbx_count); + return -EINVAL; + } + dev_info(cmh_dev(), "[debug] overriding mbx_count: %u -> %u\n", + cfg->mbx_count, mbx_count_override); + cfg->mbx_count = mbx_count_override; + } +#endif + + for (i = 0; i < cfg->mbx_count; i++) { + struct cmh_mbx_config *m = &cfg->mailboxes[i]; + u32 val; + + ret = of_property_read_u32_index(np, "cri,mbx-instances", + i, &val); + if (ret) { + dev_err(cmh_dev(), "missing cri,mbx-instances[%u] in DT\n", + i); + return ret; + } + m->instance = val; + + ret = of_property_read_u32_index(np, "cri,mbx-slots-log2", + i, &val); + if (ret) { + m->slots_log2 = CMH_DEFAULT_SLOTS_LOG2; + dev_info(cmh_dev(), + "MBX[%u]: cri,mbx-slots-log2 absent, using default %u\n", + i, CMH_DEFAULT_SLOTS_LOG2); + } else { + m->slots_log2 = val; + } + + ret = of_property_read_u32_index(np, "cri,mbx-strides-log2", + i, &val); + if (ret) { + m->stride_log2 = CMH_DEFAULT_STRIDE_LOG2; + dev_info(cmh_dev(), + "MBX[%u]: cri,mbx-strides-log2 absent, using default %u\n", + i, CMH_DEFAULT_STRIDE_LOG2); + } else { + m->stride_log2 = val; + } + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG + if (mbx_slots_override > 0) { + m->slots_log2 = mbx_slots_override; + if (i == 0) + dev_info(cmh_dev(), + "[debug] overriding slots_log2 -> %u\n", + mbx_slots_override); + } +#endif + + m->queue_size = (1UL << m->slots_log2) << m->stride_log2; + m->dma_handle = 0; + m->virt_addr = NULL; + m->reg_base = NULL; + } + + /* -- Core-type enumeration from DT child nodes ----------------- */ + + ret = cmh_config_populate_cores(cfg, np); + if (ret) + return ret; + + ret = cmh_config_validate(cfg); + if (ret) + return ret; + + return cmh_config_validate_core_types(cfg); +} diff --git a/drivers/crypto/cmh/cmh_debugfs.c b/drivers/crypto/cmh/cmh_debugfs.c new file mode 100644 index 00000000000000..bd7b083b9ef1b0 --- /dev/null +++ b/drivers/crypto/cmh/cmh_debugfs.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- debugfs Per-MBX Counters and Fault Injection + * + * Creates the /sys/kernel/debug/cmh/ tree with: + * mbxN/vcqs_submitted (ro) Total VCQs sent to MBX N + * mbxN/vcqs_completed (ro) Total completions received + * mbxN/vcqs_errors (ro) Total error completions + * mbxN/queue_full_count (ro) Times select_mailbox() skipped this MBX + * mbxN/max_queue_depth (ro) High-water mark of in-flight transactions + * mbxN/inject_abort (wo) Write any value to inject MBX_COMMAND_ABORT + * mbxN/force_drain (wo) Write any value to force-drain all pending txns + * tm/cmq_posts (ro) Total cmh_tm_post_command() calls + * tm/cmq_depth_max (ro) High-water mark of CMQ length + * tm/cmq_eagain_count (ro) Times CMQ was full (-EAGAIN) + * tm/backoff_count (ro) Times TM backed off (all MBX queues full) + * tm/async_timeout_count (ro) Async requests that timed out + * + * This file is only compiled when CONFIG_CRYPTO_DEV_CMH_DEBUG=y (see Kbuild). + * Requires CONFIG_DEBUG_FS=y in the kernel (standard for dev builds). + */ + +#include +#include +#include +#include + +#include "cmh_debugfs.h" +#include "cmh_config.h" +#include "cmh_registers.h" +#include "cmh_dma.h" +#include "cmh_txn.h" +#include "cmh_rh.h" +#include "cmh_rng.h" + +/* -- Module State ---------------------------------------------------------- */ + +static struct { + struct dentry *root; /* /sys/kernel/debug/cmh/ */ + struct cmh_mbx_stats *mbx; /* array[mbx_count] */ + struct cmh_tm_stats tm; + struct cmh_config *cfg; /* for inject_abort register access */ + u32 mbx_count; +} dbgfs; + +/* -- debugfs file ops for atomic64_t --------------------------------------- */ + +static int cmh_dbgfs_u64_get(void *data, u64 *val) +{ + *val = (u64)atomic64_read((atomic64_t *)data); + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(cmh_dbgfs_u64_ro_fops, + cmh_dbgfs_u64_get, NULL, "%llu\n"); + +/* -- Per-MBX directory ----------------------------------------------------- */ + +/* + * inject_abort -- write-only debugfs file for fault injection. + * + * Writing any value triggers MBX_COMMAND_ABORT on this mailbox. + * The eSW calls mbx_abort() -> mbx_cmd_error(mbx, -EPIPE), fires the + * error IRQ, and the LKM RH completes in-flight transactions with -EIO + * then issues MBX_COMMAND_RESTART to resume the mailbox. + * + * Private data points to the MBX index (cast to void *). + */ +static ssize_t inject_abort_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + u32 idx = (u32)(unsigned long)file->private_data; + void __iomem *base; + + if (!dbgfs.cfg || idx >= dbgfs.cfg->mbx_count) + return -EINVAL; + + base = dbgfs.cfg->mailboxes[idx].reg_base; + dev_warn(cmh_dev(), "debugfs: injecting ABORT on mbx[%u]\n", idx); + cmh_reg_write32(MBX_COMMAND_ABORT, base, R_MBX_COMMAND); + + return count; +} + +static const struct file_operations inject_abort_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = inject_abort_write, + .llseek = noop_llseek, +}; + +/* + * force_drain -- write-only debugfs file for administrative recovery. + * + * Writing any value issues MBX_COMMAND_FLUSH, drains all pending + * transactions on this mailbox (completing each with -ECANCELED), + * and resets all recovery bookkeeping (abort_stall_ticks, + * restart_pending, restart_retries, flush_count, wedged). + * + * Use this to recover D-state processes when the eSW is dead and + * normal ABORT/RESTART escalation has not recovered the mailbox. + */ +static ssize_t force_drain_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + u32 idx = (u32)(unsigned long)file->private_data; + + if (!dbgfs.cfg || idx >= dbgfs.cfg->mbx_count) + return -EINVAL; + + cmh_rh_force_drain_mbx(idx); + return count; +} + +static const struct file_operations force_drain_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = force_drain_write, + .llseek = noop_llseek, +}; + +static void create_mbx_dir(u32 idx, struct dentry *parent) +{ + struct cmh_mbx_stats *s = &dbgfs.mbx[idx]; + struct dentry *d; + char name[16]; + + snprintf(name, sizeof(name), "mbx%u", idx); + d = debugfs_create_dir(name, parent); + + debugfs_create_file("vcqs_submitted", 0444, d, + &s->vcqs_submitted, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("vcqs_completed", 0444, d, + &s->vcqs_completed, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("vcqs_errors", 0444, d, + &s->vcqs_errors, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("queue_full_count", 0444, d, + &s->queue_full_count, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("max_queue_depth", 0444, d, + &s->max_queue_depth, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("inject_abort", 0200, d, + (void *)(uintptr_t)idx, &inject_abort_fops); + debugfs_create_file("force_drain", 0200, d, + (void *)(uintptr_t)idx, &force_drain_fops); +} + +/* -- TM directory ---------------------------------------------------------- */ + +static void create_tm_dir(struct dentry *parent) +{ + struct cmh_tm_stats *s = &dbgfs.tm; + struct dentry *d; + + d = debugfs_create_dir("tm", parent); + + debugfs_create_file("cmq_posts", 0444, d, + &s->cmq_posts, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("cmq_depth_max", 0444, d, + &s->cmq_depth_max, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("cmq_eagain_count", 0444, d, + &s->cmq_eagain_count, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("backoff_count", 0444, d, + &s->backoff_count, &cmh_dbgfs_u64_ro_fops); + debugfs_create_file("async_timeout_count", 0444, d, + &s->async_timeout_count, &cmh_dbgfs_u64_ro_fops); +} + +/* -- Config directory: timeout tuning ---------------------------------- */ + +static void create_config_dir(struct dentry *parent) +{ + struct dentry *d; + + d = debugfs_create_dir("config", parent); + + /* TM timeouts */ + debugfs_create_u32("async_timeout_ms", 0644, d, + cmh_tm_timeout_async_ptr()); + debugfs_create_u32("vcq_timeout_ms", 0644, d, + cmh_tm_timeout_vcq_ptr()); + debugfs_create_u32("slow_op_timeout_ms", 0644, d, + cmh_tm_timeout_slow_op_ptr()); + debugfs_create_u32("drain_timeout_ms", 0644, d, + cmh_tm_timeout_drain_ptr()); + + /* RH watchdog */ + debugfs_create_u32("watchdog_ms", 0644, d, + cmh_rh_timeout_watchdog_ptr()); + + /* DRBG timeout */ + debugfs_create_u32("drbg_timeout_ms", 0644, d, + cmh_rng_timeout_drbg_ptr()); +} + +/* -- Public Interface ------------------------------------------------------ */ + +/** + * cmh_debugfs_init() - Create debugfs directory hierarchy for CMH + * @cfg: Platform configuration containing mailbox count and register bases. + * + * Allocates per-mailbox statistics and creates the /sys/kernel/debug/cmh/ + * tree with per-mailbox counters, fault-injection files, and transaction + * manager statistics. debugfs is optional; failure to create entries does + * not prevent module initialisation. + * + * Return: 0 on success (always returns 0 -- debugfs is best-effort). + */ +int cmh_debugfs_init(struct cmh_config *cfg) +{ + u32 mbx_count = cfg->mbx_count; + u32 i; + + dbgfs.root = debugfs_create_dir("cmh", NULL); + if (IS_ERR_OR_NULL(dbgfs.root)) { + if (!IS_ERR(dbgfs.root)) + dev_warn(cmh_dev(), "debugfs: creation returned NULL -- counters disabled\n"); + else + dev_warn(cmh_dev(), "debugfs: creation failed (rc=%ld) -- counters disabled\n", + PTR_ERR(dbgfs.root)); + dbgfs.root = NULL; + return 0; /* debugfs is optional -- never fail module init */ + } + + dbgfs.mbx_count = mbx_count; + dbgfs.cfg = cfg; + dbgfs.mbx = kcalloc(mbx_count, sizeof(*dbgfs.mbx), GFP_KERNEL); + if (!dbgfs.mbx) { + debugfs_remove_recursive(dbgfs.root); + dbgfs.root = NULL; + return 0; + } + + for (i = 0; i < mbx_count; i++) + create_mbx_dir(i, dbgfs.root); + + create_tm_dir(dbgfs.root); + + create_config_dir(dbgfs.root); + + dev_dbg(cmh_dev(), "debugfs: initialized (%u mailboxes)\n", mbx_count); + return 0; +} + +/** + * cmh_debugfs_cleanup() - Remove all CMH debugfs entries + * + * Tears down the /sys/kernel/debug/cmh/ tree and frees per-mailbox + * statistics memory. Safe to call even if cmh_debugfs_init() was never + * called or failed. + */ +void cmh_debugfs_cleanup(void) +{ + debugfs_remove_recursive(dbgfs.root); + dbgfs.root = NULL; + kfree(dbgfs.mbx); + dbgfs.mbx = NULL; + dev_dbg(cmh_dev(), "debugfs: cleaned up\n"); +} + +/** + * cmh_debugfs_mbx_stats() - Return per-mailbox statistics pointer + * @mbx_idx: Zero-based mailbox index. + * + * Return: Pointer to the statistics structure for @mbx_idx, or NULL if + * debugfs is disabled or @mbx_idx is out of range. + */ +struct cmh_mbx_stats *cmh_debugfs_mbx_stats(u32 mbx_idx) +{ + if (!dbgfs.mbx || mbx_idx >= dbgfs.mbx_count) + return NULL; + return &dbgfs.mbx[mbx_idx]; +} + +/** + * cmh_debugfs_tm_stats() - Return transaction manager statistics pointer + * + * Return: Pointer to the singleton TM statistics structure. The pointer + * is always valid (points to static storage). + */ +struct cmh_tm_stats *cmh_debugfs_tm_stats(void) +{ + return &dbgfs.tm; +} diff --git a/drivers/crypto/cmh/cmh_dma.c b/drivers/crypto/cmh/cmh_dma.c new file mode 100644 index 00000000000000..36ea277420cfff --- /dev/null +++ b/drivers/crypto/cmh/cmh_dma.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- DMA Operations + * + * Implements the cmh_dma.h interface using the kernel DMA API + * (dma_map_single, dma_alloc_coherent, etc.). + * + * Scatterlist linearization rationale + * ------------------------------------ + * The eSW firmware supports SCATTERGATHER commands for all core + * types (AES_CMD_SCATTERGATHER, SM4_CMD_SCATTERGATHER, + * CCP_CMD_SCATTERGATHER, HC_CMD_GATHER), using a proprietary + * linked-list-item (LLI) descriptor chain format. The hash driver + * already uses this via cmh_dma_build_sg() + HC_CMD_GATHER. + * + * For symmetric cipher and AEAD commands, the LKM currently + * linearizes scatterlist input into contiguous bounce buffers via + * scatterwalk_map_and_copy() rather than building LLI chains from + * kernel scatterlists. This is a deliberate first-submission + * simplification with a concrete technical justification: + * + * - The hash SG path is unidirectional (DMA_TO_DEVICE gather only). + * Skcipher and AEAD require bidirectional handling: separate src + * and dst scatterlists (which may alias for in-place operations), + * plus AAD and authentication tag regions with distinct DMA + * directions and alignment constraints. + * - The CMH LLI format requires 64-byte aligned descriptor chain + * pointers (the .lli field) with 32-bit length fields. This + * alignment is automatically satisfied by dma_alloc_coherent() + * for the descriptor array; data buffer addresses have no + * hardware alignment requirement. Kernel SG entries have no + * alignment guarantee for data, so direct SG-to-LLI translation + * requires per-segment validation, potential splitting at + * descriptor boundaries, and separate chains for src/dst/AAD -- + * substantially more complex than the unidirectional hash + * gather case. + * - Each skcipher/AEAD driver caps linearization at + * CMH_AES_MAX_CRYPTLEN / CMH_SM4_MAX_CRYPTLEN (32 MiB). + * Requests exceeding this cap are rejected with -EINVAL. + * In practice, crypto API callers (dm-crypt, IPsec, kernel TLS) + * send page-sized or smaller buffers, so the bounce allocation + * is typically <= PAGE_SIZE and succeeds even under GFP_ATOMIC. + * + * A shared SG-to-LLI adapter handling bidirectional mappings, + * alignment splitting, and in-place src==dst detection for the + * skcipher/AEAD/MAC paths is planned as a follow-up series once the + * core driver is accepted. + * + * This linearization pattern is consistent with other upstream HW + * crypto drivers that use bounce buffers in their initial + * submissions (e.g. ccree, sa2ul, omap-aes). + */ + +#include +#include +#include +#include +#include +#include + +#include "cmh_dma.h" + +/* Module-global device pointer, set in cmh_dma_init() */ +static struct device *cmh_device; + +/** + * cmh_dma_init() - Initialize the standard DMA backend + * @pdev: Platform device providing the struct device for DMA ops + * + * Stores the device pointer for use by all DMA wrapper functions. + * + * Return: 0 (always succeeds for the standard backend). + */ +int cmh_dma_init(struct platform_device *pdev) +{ + cmh_device = &pdev->dev; + return 0; +} + +/** + * cmh_dma_cleanup() - Tear down the standard DMA backend + * + * Clears the stored device pointer. + */ +void cmh_dma_cleanup(void) +{ + cmh_device = NULL; +} + +/** + * cmh_dev() - Return the platform device pointer + * + * Return: struct device pointer, or NULL outside probe/remove lifecycle. + */ +struct device *cmh_dev(void) +{ + return cmh_device; +} + +/* -- Streaming DMA -------------------------------------------------------- */ + +/** + * cmh_dma_map_single() - Map a kernel buffer for streaming DMA + * @buf: Kernel virtual address + * @size: Buffer length in bytes + * @dir: DMA direction + * + * Return: DMA address, or a DMA_MAPPING_ERROR value on failure. + */ +dma_addr_t cmh_dma_map_single(void *buf, size_t size, + enum dma_data_direction dir) +{ + return dma_map_single(cmh_device, buf, size, dir); +} + +/** + * cmh_dma_unmap_single() - Unmap a streaming DMA buffer + * @addr: DMA address returned by cmh_dma_map_single() + * @size: Buffer length in bytes + * @dir: DMA direction (must match the map call) + */ +void cmh_dma_unmap_single(dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + dma_unmap_single(cmh_device, addr, size, dir); +} + +/** + * cmh_dma_sync_for_cpu() - Sync a DMA buffer for CPU access + * @addr: DMA address of the mapped buffer + * @size: Region length in bytes + * @dir: DMA direction + */ +void cmh_dma_sync_for_cpu(dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + dma_sync_single_for_cpu(cmh_device, addr, size, dir); +} + +/** + * cmh_dma_sync_for_device() - Sync a DMA buffer for device access + * @addr: DMA address of the mapped buffer + * @size: Region length in bytes + * @dir: DMA direction + */ +void cmh_dma_sync_for_device(dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + dma_sync_single_for_device(cmh_device, addr, size, dir); +} + +/** + * cmh_dma_map_error() - Check whether a DMA mapping failed + * @addr: DMA address to check + * + * Return: Non-zero if @addr indicates a mapping error. + */ +int cmh_dma_map_error(dma_addr_t addr) +{ + return dma_mapping_error(cmh_device, addr); +} + +/* -- Coherent DMA --------------------------------------------------------- */ + +/** + * cmh_dma_alloc() - Allocate coherent DMA memory + * @size: Allocation size in bytes + * @handle: Output DMA address + * @gfp: GFP allocation flags + * + * Return: Kernel virtual address, or NULL on failure. + */ +void *cmh_dma_alloc(size_t size, dma_addr_t *handle, gfp_t gfp) +{ + return dma_alloc_coherent(cmh_device, size, handle, gfp); +} + +/** + * cmh_dma_free() - Free coherent DMA memory + * @size: Allocation size (must match cmh_dma_alloc) + * @virt: Kernel virtual address + * @handle: DMA address + */ +void cmh_dma_free(size_t size, void *virt, dma_addr_t handle) +{ + dma_free_coherent(cmh_device, size, virt, handle); +} + +/* -- Buffer write helpers ------------------------------------------------- */ + +/** + * cmh_dma_write() - Copy data into a DMA buffer + * @dst: Destination (from cmh_dma_alloc) + * @src: Source kernel buffer + * @len: Byte count + */ +void cmh_dma_write(void *dst, const void *src, size_t len) +{ + memcpy(dst, src, len); +} + +/** + * cmh_dma_fence() - No-op on standard DMA API platforms (coherent) + * @ptr: Unused -- present for interface compatibility + */ +void cmh_dma_fence(void *ptr) +{ + /* Standard DMA API: coherent memory, no cross-slave fence needed */ +} + +/** + * cmh_dma_zero() - Zero a DMA buffer + * @dst: Destination (from cmh_dma_alloc) + * @len: Byte count + */ +void cmh_dma_zero(void *dst, size_t len) +{ + memset(dst, 0, len); +} + +/** + * cmh_dma_build_sg() - Build a scatter-gather DMA mapping + * @bufs: Array of buffer descriptors to map + * @count: Number of entries in @bufs + * @gfp: GFP flags for memory allocation + * + * Allocates a streaming-DMA descriptor array and maps each buffer in @bufs + * for DMA-to-device transfer, filling CMH eSW-format scatter-gather + * descriptors with linked-list pointers. + * + * The descriptor array uses streaming DMA (kmalloc + dma_map_single) rather + * than dma_alloc_coherent so that cmh_dma_free_sg() -- which calls + * dma_unmap_single + kfree -- is safe from any context including BH-disabled + * completion callbacks. + * + * Return: Pointer to the allocated cmh_sg_map on success, NULL on failure. + */ +struct cmh_sg_map *cmh_dma_build_sg(const struct cmh_dma_buf *bufs, u32 count, + gfp_t gfp) +{ + struct cmh_sg_map *sgm; + u32 i; + + if (!count) + return NULL; + + sgm = kzalloc(struct_size(sgm, bufs, count), gfp); + if (!sgm) + return NULL; + + sgm->count = count; + sgm->items_size = array_size(count, sizeof(*sgm->items)); + if (sgm->items_size == SIZE_MAX) + goto err_free_sgm; + + /* + * Allocate descriptor array with kmalloc and map for streaming DMA. + * We map first to obtain items_dma (needed for .lli pointers), + * then sync-for-cpu, fill descriptors, and sync-for-device. + */ + sgm->items = kzalloc(sgm->items_size, gfp); + if (!sgm->items) + goto err_free_sgm; + + sgm->items_dma = cmh_dma_map_single(sgm->items, sgm->items_size, + DMA_TO_DEVICE); + if (cmh_dma_map_error(sgm->items_dma)) + goto err_free_items; + + /* Map each source buffer for device read */ + for (i = 0; i < count; i++) { + dma_addr_t dma; + + if (!bufs[i].len) + goto err_unmap; + sgm->bufs[i].len = bufs[i].len; + dma = cmh_dma_map_single(bufs[i].data, bufs[i].len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(dma)) + goto err_unmap; + sgm->bufs[i].dma = dma; + } + + /* + * Reclaim CPU ownership of the descriptor buffer. After + * dma_map_single the device owns the mapping; we must call + * sync_for_cpu before writing regardless of direction. The + * direction matches the original mapping (DMA_TO_DEVICE) -- + * this tells the DMA layer which cache operations apply: + * invalidate so the CPU sees coherent data before we fill + * the SG descriptors and later sync_for_device. + */ + cmh_dma_sync_for_cpu(sgm->items_dma, sgm->items_size, + DMA_TO_DEVICE); + + /* Fill CMH eSW SG descriptors */ + for (i = 0; i < count; i++) { + u64 lli_val; + + if (i + 1 < count) + lli_val = (u64)(sgm->items_dma + + (i + 1) * sizeof(*sgm->items)); + else + lli_val = 0; + + sgm->items[i].lli = lli_val; + sgm->items[i].src = (u64)sgm->bufs[i].dma; + sgm->items[i].dst = 0; + sgm->items[i].len = (u64)bufs[i].len; + } + + /* Flush descriptor writes to device */ + cmh_dma_sync_for_device(sgm->items_dma, sgm->items_size, + DMA_TO_DEVICE); + + return sgm; + +err_unmap: + while (i--) + cmh_dma_unmap_single(sgm->bufs[i].dma, + sgm->bufs[i].len, DMA_TO_DEVICE); + cmh_dma_unmap_single(sgm->items_dma, sgm->items_size, + DMA_TO_DEVICE); +err_free_items: + kfree(sgm->items); +err_free_sgm: + kfree(sgm); + return NULL; +} + +/** + * cmh_dma_free_sg() - Unmap and free a scatter-gather mapping + * @sgm: Scatter-gather mapping created by cmh_dma_build_sg(), or NULL + * + * Unmaps all DMA-mapped buffers, unmaps and frees the descriptor array, + * and releases the cmh_sg_map structure. Safe to call from any context + * (including BH-disabled completion callbacks) because it uses only + * dma_unmap_single + kfree -- no vunmap/dma_free_coherent. + */ +void cmh_dma_free_sg(struct cmh_sg_map *sgm) +{ + u32 i; + + if (!sgm) + return; + + for (i = 0; i < sgm->count; i++) + cmh_dma_unmap_single(sgm->bufs[i].dma, + sgm->bufs[i].len, DMA_TO_DEVICE); + + cmh_dma_unmap_single(sgm->items_dma, sgm->items_size, + DMA_TO_DEVICE); + kfree(sgm->items); + kfree(sgm); +} + +/** + * cmh_dma_orphan_free() - Orphan cleanup callback for abandoned DMA buffers + * @data: Pointer to a struct cmh_dma_orphan describing the orphaned mapping + * + * Called by the transaction manager when a synchronous operation times out + * and the caller has already returned. Unmaps the DMA buffer and frees + * the backing memory and the orphan descriptor itself. + */ +void cmh_dma_orphan_free(void *data) +{ + struct cmh_dma_orphan *o = data; + + cmh_dma_unmap_single(o->addr, o->len, o->dir); + kfree_sensitive(o->buf); + kfree(o); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c new file mode 100644 index 00000000000000..de065a425a2cf5 --- /dev/null +++ b/drivers/crypto/cmh/cmh_main.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Platform Driver Entry and Exit + * + * Responsibilities: + * - Match "cri,cmh" DT node via platform_driver + * - Parse device-tree properties via cmh_config_init() + * - ioremap the SIC region + * - Verify CMH boot status (sanity check) + * - Compute per-instance register bases + * - Initialize MBX queues (MQI) + * - Start Transaction Manager kthread + * - Register Response Handler IRQ + * - Register Kernel Crypto API hash algorithms + * - Clean up in reverse order on exit or error + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmh.h" +#include "cmh_dma.h" +#include "cmh_mqi.h" +#include "cmh_txn.h" +#include "cmh_rh.h" +#include "cmh_registers.h" +#include "cmh_debugfs.h" +#include "cmh_sysfs.h" + +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Alex Ousherovitch "); +MODULE_AUTHOR("Saravanakrishnan Krishnamoorthy "); +MODULE_AUTHOR("Joel Wittenauer "); +MODULE_DESCRIPTION("CRI CryptoManager Hub (CMH) hardware crypto accelerator"); +MODULE_ALIAS("platform:cmh"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +static bool skip_fw_check; +module_param(skip_fw_check, bool, 0444); +MODULE_PARM_DESC(skip_fw_check, + "[debug] Skip eSW boot status check at probe (default: false)"); +#else +#define skip_fw_check false +#endif + +/* Global device state (single-instance module) */ + +static struct cmh_device *g_cmh_dev; + +/* SIC Sanity Check */ + +static int cmh_check_sic(struct cmh_config *cfg) +{ + const u32 ready = SIC_SW_BOOT_STATUS_MISSION | + SIC_SW_BOOT_STATUS_MISSION2; + u32 boot_status; + u32 hw_version; + u32 sw_boot; + int ret; + + boot_status = cmh_reg_read32(cfg->sic_mapped, R_SIC_BOOT_STATUS); + hw_version = cmh_reg_read32(cfg->sic_mapped, R_SIC_HW_VERSION0); + + dev_info(cmh_dev(), "SIC boot_status=0x%08x hw_version=0x%08x\n", + boot_status, hw_version); + + if ((boot_status & SIC_BOOT_STATUS_MASK) != SIC_BOOT_STATUS_PASS) { + dev_err(cmh_dev(), "SIC boot status check failed (0x%02x != 0x%02x)\n", + boot_status & SIC_BOOT_STATUS_MASK, SIC_BOOT_STATUS_PASS); + return -EIO; + } + + /* + * Wait for eSW readiness: MISSION signals the primary VCQ engine, + * MISSION2 the sidecar engine (set asynchronously). The driver + * uses both, so require both bits. + */ + ret = read_poll_timeout(ioread32, sw_boot, + (sw_boot & ready) == ready, + 1000, + (unsigned long)cfg->fw_ready_timeout_ms * 1000UL, + false, + cfg->sic_mapped + R_SIC_SW_BOOT_STATUS); + if (ret) { + sw_boot = cmh_reg_read32(cfg->sic_mapped, R_SIC_SW_BOOT_STATUS); + dev_err(cmh_dev(), "CMH eSW not ready (sw_boot_status=0x%08x, timeout=%ums)\n", + sw_boot, cfg->fw_ready_timeout_ms); + return -ETIMEDOUT; + } + + dev_info(cmh_dev(), "CMH eSW fully operational (sw_boot_status=0x%08x)\n", + sw_boot); + + return 0; +} + +/* Module Init -- platform driver probe */ + +static int cmh_probe(struct platform_device *pdev) +{ + struct cmh_device *dev; + struct cmh_config *cfg; + unsigned int i; + int ret; + + /* Single-instance guard: reject if already probed */ + if (g_cmh_dev) + return -EBUSY; + + dev_info(&pdev->dev, "loading v%s\n", CMH_VERSION); + + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + dev->dev = &pdev->dev; + cfg = &dev->config; + + /* Declare DMA addressing capability */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(&pdev->dev, "dma_set_mask_and_coherent failed (rc=%d)\n", + ret); + goto err_free_dev; + } + + /* Initialize DMA backend (standard API or FPGA pool) */ + ret = cmh_dma_init(pdev); + if (ret) { + dev_err(&pdev->dev, "DMA init failed (rc=%d)\n", ret); + goto err_free_dev; + } + + /* Step 1: Parse and validate configuration (DT + module params) */ + ret = cmh_config_init(cfg, pdev); + if (ret) + goto err_dma_init; + + dev_info(cmh_dev(), "sic_base=0x%llx size=0x%zx mbx_count=%u irq=%d\n", + (unsigned long long)cfg->sic_base, cfg->sic_size, + cfg->mbx_count, cfg->irq); + + /* Step 2: ioremap the SIC region */ + cfg->sic_mapped = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(cfg->sic_mapped)) { + ret = PTR_ERR(cfg->sic_mapped); + cfg->sic_mapped = NULL; + dev_err(cmh_dev(), "ioremap failed for SIC region (rc=%d)\n", + ret); + goto err_dma_init; + } + + /* Step 3: Verify CMH is alive */ + if (skip_fw_check) { + dev_info(cmh_dev(), "skipping eSW boot check (skip_fw_check=1)\n"); + } else { + ret = cmh_check_sic(cfg); + if (ret) + goto err_dma_init; + } + + /* Step 4: Compute per-instance register bases */ + for (i = 0; i < cfg->mbx_count; i++) { + struct cmh_mbx_config *m = &cfg->mailboxes[i]; + + m->reg_base = cmh_mbx_instance_base(cfg->sic_mapped, + m->instance); + + dev_dbg(cmh_dev(), "mbx[%u] instance=%u reg_base=%p\n", + i, m->instance, m->reg_base); + } + + (void)cmh_debugfs_init(cfg); + + /* Initialise mailbox queue interface */ + ret = cmh_mqi_init(cfg); + if (ret) + goto err_mqi_init; + + /* Initialise transaction manager */ + ret = cmh_tm_init(cfg); + if (ret) + goto err_tm_init; + + /* Initialise response handler */ + ret = cmh_rh_init(cfg); + if (ret) + goto err_rh_init; + + g_cmh_dev = dev; + platform_set_drvdata(pdev, dev); + + dev_info(cmh_dev(), "initialized successfully\n"); + return 0; + +err_rh_init: + cmh_tm_cleanup(); +err_tm_init: + cmh_mqi_cleanup(cfg); +err_mqi_init: + cmh_debugfs_cleanup(); +err_dma_init: + cmh_dma_cleanup(); +err_free_dev: + return ret; +} + +/* Module Exit -- platform driver remove */ + +static void cmh_remove(struct platform_device *pdev) +{ + struct cmh_device *dev = platform_get_drvdata(pdev); + struct cmh_config *cfg; + + if (!dev) + return; + + cfg = &dev->config; + + cmh_rh_cleanup(cfg); + cmh_tm_cleanup(); + cmh_mqi_cleanup(cfg); + cmh_debugfs_cleanup(); + cmh_dma_cleanup(); + + dev_info(&pdev->dev, "unloaded successfully\n"); + + g_cmh_dev = NULL; +} + +static const struct of_device_id cmh_of_match[] = { + { .compatible = "cri,cmh" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, cmh_of_match); + +/* + * PM suspend/resume. + * + * Suspend: drain the TM first (while the RH is still active and can + * deliver completions for in-flight transactions), then quiesce the + * RH (cancel watchdog, mask HW interrupts). This ordering ensures + * the drain_timeout_ms wait in cmh_tm_quiesce() can actually succeed + * -- if we suspended RH first, no completions would be delivered and + * the drain would always hit the force-cancel path. + * + * IRQ handlers remain registered (standard PM pattern: the kernel + * disables the IRQ lines during suspend, no need to free/re-request). + * + * Resume: re-check the SIC/SW boot status, re-synchronise the RH + * with hardware (head positions, interrupt masks, watchdog), then + * restart the TM kthread. + */ + +static int cmh_suspend(struct device *dev) +{ + struct cmh_device *cmh = dev_get_drvdata(dev); + + if (!cmh) + return 0; + + dev_info(dev, "suspending\n"); + cmh_tm_quiesce(); + cmh_rh_suspend(&cmh->config); + return 0; +} + +static int cmh_resume(struct device *dev) +{ + struct cmh_device *cmh = dev_get_drvdata(dev); + int ret; + + if (!cmh) + return 0; + + ret = cmh_check_sic(&cmh->config); + if (ret) { + dev_err(dev, "resume: CMH eSW health check failed (%d)\n", + ret); + return ret; + } + + /* + * cmh_rh_resume() is void: it only re-syncs MMIO head pointers, + * clears stale interrupt status bits (W1C), re-enables interrupt + * masks, and re-arms the watchdog timer -- none of which can fail + * after the SIC health check above has confirmed HW accessibility. + */ + cmh_rh_resume(&cmh->config); + + ret = cmh_tm_resume(); + if (ret) { + dev_err(dev, "resume: TM restart failed (%d)\n", ret); + return ret; + } + dev_info(dev, "resumed successfully\n"); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cmh_pm_ops, + cmh_suspend, + cmh_resume); + +/* + * Runtime PM is intentionally not implemented. The CMH hardware does + * not expose HLOS-accessible clock gates or power domains -- the eSW + * firmware manages HW power state independently. There is no mechanism + * for the kernel to idle, gate clocks, or power down the accelerator + * block from HLOS. If a future platform variant exposes power control + * to HLOS (e.g. via a SCMI power domain), runtime PM support can be + * added at that time using SET_RUNTIME_PM_OPS and pm_runtime_get/put + * around VCQ submission paths. + * + * System sleep (suspend/resume) is supported via DEFINE_SIMPLE_DEV_PM_OPS + * above: suspend quiesces the TM and masks IRQs; resume re-verifies + * eSW health (SIC status) and restarts the TM thread. + */ + +static struct platform_driver cmh_driver = { + .probe = cmh_probe, + .remove = cmh_remove, + .driver = { + .name = CMH_DRV_NAME, + .of_match_table = cmh_of_match, + .dev_groups = cmh_sysfs_groups, + .pm = pm_sleep_ptr(&cmh_pm_ops), + }, +}; + +static int __init cmh_init(void) +{ + int ret; + + ret = platform_driver_register(&cmh_driver); + if (ret) + return ret; + + /* + * platform_driver_register() does not propagate probe() errors. + * If a DT node matched but probe() failed (e.g. bad module params), + * g_cmh_dev will not have been set. Detect this and unregister. + * + * This is intentional for a non-discoverable accelerator with no + * hotplug or deferred-probe scenarios -- the device is either + * present at boot or not. Leaving the driver registered after a + * probe failure would silently produce a non-functional module. + */ + if (!g_cmh_dev) { + platform_driver_unregister(&cmh_driver); + return -ENODEV; + } + + return 0; +} + +static void __exit cmh_exit(void) +{ + platform_driver_unregister(&cmh_driver); +} + +module_init(cmh_init); +module_exit(cmh_exit); diff --git a/drivers/crypto/cmh/cmh_mqi.c b/drivers/crypto/cmh/cmh_mqi.c new file mode 100644 index 00000000000000..9a135be585625a --- /dev/null +++ b/drivers/crypto/cmh/cmh_mqi.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Mailbox Queue Initializer + * + * Responsibilities: + * - Allocate queue buffers for each configured mailbox + * - Execute the MBX lock/setup/enable register sequence + * - Readback-verify all critical register writes + * - Hold lock for MBX lifetime (CMH eSW requires it for host access) + * - Clean up (flush + unlock + free) on exit or error + * + * Register sequence per instance (per CMH MBX hardware specification): + * 1. Read R_MBX_LOCK -> non-zero = ownership token acquired + * 2. W1C stale R_MBX_INTERRUPT bits (avoids spurious error cascade) + * 3. Set R_MBX_INTERRUPT_MASK = MBX_IRQ_MASK + * 4. Write QUEUE_LO/HI, SLOTS, STRIDE (queue address + geometry) + * 5. Sync TAIL = HEAD (CMH eSW owns HEAD; avoids stale-queue parse) + * 6. Readback verify QUEUE_LO/HI/SLOTS/STRIDE + * 7. Write HOST_INFO (signals CMH eSW "MBX configured") + * 8. Write COMMAND = MBX_COMMAND_RUN + * 9. Lock stays held -- released only in teardown + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_mqi.h" +#include "cmh_dma.h" +#include "cmh_registers.h" +#include "cmh_config.h" + +/* Flush polling: eSW clears R_MBX_COMMAND to 0 when flush completes */ +#define MBX_FLUSH_POLL_US 50 +#define MBX_FLUSH_TIMEOUT_US 1000000 /* 1 second */ + +/* MBX Lock / Unlock */ + +/* + * Attempt to acquire the MBX hardware lock. + * Returns the lock token (non-zero) on success, 0 on timeout. + */ +static u32 cmh_mbx_lock(void __iomem *reg_base, u32 instance) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(MBX_LOCK_TIMEOUT_MS); + u32 lock; + + while (time_before(jiffies, deadline)) { + lock = cmh_reg_read32(reg_base, R_MBX_LOCK); + if (lock) { + dev_dbg(cmh_dev(), "mbx %u lock acquired (token=0x%08x)\n", + instance, lock); + return lock; + } + /* HW lock may be held by CMH eSW -- back off before retry */ + usleep_range(MBX_LOCK_POLL_MIN_US, MBX_LOCK_POLL_MAX_US); + } + + return 0; +} + +/* Release the MBX lock: clear interrupt mask, write token back */ +static void cmh_mbx_unlock(void __iomem *reg_base, u32 lock_val) +{ + cmh_reg_write32(0, reg_base, R_MBX_INTERRUPT_MASK); + cmh_reg_write32(lock_val, reg_base, R_MBX_LOCK); +} + +/* Register Readback Verification */ + +static int cmh_verify_reg(void __iomem *base, u32 offset, u32 expected, + const char *name, u32 instance) +{ + u32 actual = cmh_reg_read32(base, offset); + + if (actual != expected) { + dev_err(cmh_dev(), "mbx %u %s readback mismatch: 0x%08x != 0x%08x\n", + instance, name, actual, expected); + return -EIO; + } + return 0; +} + +/* Clear any stale interrupt bits left from a prior module lifecycle. */ +static void cmh_mbx_clear_stale_irqs(void __iomem *base, u32 instance) +{ + u32 stale = cmh_reg_read32(base, R_MBX_INTERRUPT); + + if (stale) { + cmh_reg_write32(stale, base, R_MBX_INTERRUPT); + dev_dbg(cmh_dev(), "mbx %u cleared stale irq bits=0x%x\n", + instance, stale); + } +} + +/* Read CMH eSW HEAD and set TAIL = HEAD so the queue appears empty. */ +static void cmh_mbx_sync_tail_to_head(void __iomem *base, u32 instance) +{ + u32 fw_head = cmh_reg_read32(base, R_MBX_QUEUE_HEAD); + + cmh_reg_write32(fw_head, base, R_MBX_QUEUE_TAIL); + if (fw_head) + dev_dbg(cmh_dev(), "mbx %u synced tail=%u to fw head\n", + instance, fw_head); +} + +/* Per-Mailbox Setup */ + +static int cmh_mbx_setup_one(struct cmh_mbx_config *mbx) +{ + void __iomem *base = mbx->reg_base; + u32 addr_lo = lower_32_bits(mbx->dma_handle); + u32 addr_hi = upper_32_bits(mbx->dma_handle); + u32 lock_val; + int ret; + + /* Step 1: Acquire exclusive access */ + lock_val = cmh_mbx_lock(base, mbx->instance); + if (!lock_val) { + dev_err(cmh_dev(), "mbx %u lock timeout after %u ms\n", + mbx->instance, MBX_LOCK_TIMEOUT_MS); + return -ETIMEDOUT; + } + + /* + * Step 1.5: Clear stale interrupt bits from a prior module lifecycle. + * + * After rmmod, the CMH eSW may leave ERROR_IRQ set in + * R_MBX_INTERRUPT even though STATUS is IDLE. If we enable + * the mask first, the stale bits immediately trigger the + * CMH eSW interrupt chain, which can cascade into ERROR + * status before the first hash operation. W1C-clear them first. + */ + cmh_mbx_clear_stale_irqs(base, mbx->instance); + + /* Step 2: Program interrupt mask (enable DONE/ERROR interrupts) */ + cmh_reg_write32(MBX_IRQ_MASK, base, R_MBX_INTERRUPT_MASK); + + /* Step 3: Configure queue address (64-bit split) */ + cmh_reg_write32(addr_lo, base, R_MBX_QUEUE_LO); + cmh_reg_write32(addr_hi, base, R_MBX_QUEUE_HI); + + /* Step 4: Configure queue geometry */ + cmh_reg_write32(mbx->slots_log2, base, R_MBX_QUEUE_SLOTS); + cmh_reg_write32(mbx->stride_log2, base, R_MBX_QUEUE_STRIDE); + + /* + * Step 5: Synchronise TAIL to CMH eSW's HEAD. + * + * R_MBX_QUEUE_HEAD is read-only from the host side -- only the + * CMH eSW can write it. On a fresh boot HEAD is 0; after an + * rmmod/insmod cycle it retains the value from the previous + * session (e.g. 44). Writing 0 from the host is silently + * dropped by the MBX HW. + * + * If we set TAIL=0 while HEAD=44 the CMH eSW sees a non-empty + * queue (head != tail with wrap-around) and immediately tries + * to load a VCQ at the old head offset into our freshly-zeroed + * DMA buffer, causing an "Invalid VCQ" EFAULT -> ECHILD cascade. + * + * Fix: read HEAD and set TAIL = HEAD so the queue looks empty. + */ + cmh_mbx_sync_tail_to_head(base, mbx->instance); + + /* + * Step 6: Readback verify critical registers. + * HOST_INFO is deliberately deferred to after verification -- writing + * it tells the CMH eSW "MBX is ready" and the CMH eSW may inspect + * (and clear) the queue registers immediately. + */ + ret = cmh_verify_reg(base, R_MBX_QUEUE_LO, addr_lo, + "QUEUE_LO", mbx->instance); + if (ret) + goto err_unlock; + + ret = cmh_verify_reg(base, R_MBX_QUEUE_HI, addr_hi, + "QUEUE_HI", mbx->instance); + if (ret) + goto err_unlock; + + ret = cmh_verify_reg(base, R_MBX_QUEUE_SLOTS, mbx->slots_log2, + "QUEUE_SLOTS", mbx->instance); + if (ret) + goto err_unlock; + + ret = cmh_verify_reg(base, R_MBX_QUEUE_STRIDE, mbx->stride_log2, + "QUEUE_STRIDE", mbx->instance); + if (ret) + goto err_unlock; + + /* + * Step 7: Host identification -- signals CMH eSW that MBX is configured. + * Must come after readback verification (CMH eSW may inspect the MBX + * immediately) and before COMMAND_RUN. + */ + cmh_reg_write32(MBX_HOST_INFO_LKM, base, R_MBX_HOST_INFO); + + /* Step 8: Enable -- start the mailbox */ + cmh_reg_write32(MBX_COMMAND_RUN, base, R_MBX_COMMAND); + + /* Read status while we still hold the lock */ + dev_dbg(cmh_dev(), "mbx %u setup: dma=0x%08x%08x slots=%u stride=%u status=0x%08x\n", + mbx->instance, addr_hi, addr_lo, + mbx->slots_log2, mbx->stride_log2, + cmh_reg_read32(base, R_MBX_STATUS)); + + /* + * Lock stays held for the lifetime of this MBX session. + * + * mbx->lock_val is the ownership token returned by R_MBX_LOCK at + * acquisition time. The CMH eSW validates this token on every + * register access and requires it to be written back to release. + * It is NOT a transient mutex -- it persists until teardown. + */ + mbx->lock_val = lock_val; + + return 0; + +err_unlock: + cmh_mbx_unlock(base, lock_val); + return ret; +} + +/* Per-Mailbox Teardown */ + +static void cmh_mbx_teardown_one(struct cmh_mbx_config *mbx) +{ + void __iomem *base = mbx->reg_base; + u32 status; + + if (!base || !mbx->lock_val) + return; + + if (MBX_STATUS_CODE(cmh_reg_read32(base, R_MBX_STATUS)) != + MBX_STATUS_OFFLINE) { + cmh_reg_write32(MBX_COMMAND_FLUSH, base, R_MBX_COMMAND); + + /* + * Wait for the eSW to process the flush before releasing + * the DMA buffer. The eSW clears R_MBX_COMMAND to zero + * upon completion; if it doesn't within 1 s, log a + * warning and proceed (best-effort teardown). + * + * DMA safety: by this point the RH and TM are already + * shut down (remove order: algos -> RH -> TM -> MQI), + * so no new transactions can be submitted and no + * completions are in flight. The queue buffer is only + * read by the eSW during active command processing; + * after flush the eSW will not touch it again. + */ + if (read_poll_timeout(cmh_reg_read32, status, + status == 0, + MBX_FLUSH_POLL_US, + MBX_FLUSH_TIMEOUT_US, + true, base, R_MBX_COMMAND)) + dev_warn(cmh_dev(), + "mbx %u flush timeout during teardown (status=0x%08x)\n", + mbx->instance, + cmh_reg_read32(base, R_MBX_STATUS)); + } + + cmh_mbx_unlock(base, mbx->lock_val); + mbx->lock_val = 0; +} + +/* Public Interface */ + +/** + * cmh_mqi_init() - Initialize all mailbox queues + * @cfg: CMH configuration describing the mailboxes to set up + * + * Allocates DMA queue buffers for each configured mailbox, then executes + * the MBX lock/setup/enable register sequence. On failure, all + * successfully initialized mailboxes are torn down and buffers freed. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mqi_init(struct cmh_config *cfg) +{ + unsigned int i, j; + int ret; + + /* Allocate queue buffers */ + for (i = 0; i < cfg->mbx_count; i++) { + struct cmh_mbx_config *m = &cfg->mailboxes[i]; + + m->virt_addr = cmh_dma_alloc(m->queue_size, &m->dma_handle, + GFP_KERNEL); + if (!m->virt_addr) { + ret = -ENOMEM; + goto err_free_bufs; + } + + dev_dbg(cmh_dev(), "mqi[%u] alloc %zu bytes @ virt=%pK dma=%pad\n", + i, m->queue_size, m->virt_addr, &m->dma_handle); + } + + /* Lock/setup/enable each mailbox */ + for (i = 0; i < cfg->mbx_count; i++) { + ret = cmh_mbx_setup_one(&cfg->mailboxes[i]); + if (ret) { + dev_err(cmh_dev(), "mqi[%u] setup failed (rc=%d)\n", + i, ret); + goto err_teardown; + } + } + + dev_info(cmh_dev(), "MQI init complete (%u mailboxes)\n", cfg->mbx_count); + return 0; + +err_teardown: + for (j = 0; j < i; j++) + cmh_mbx_teardown_one(&cfg->mailboxes[j]); +err_free_bufs: + for (j = 0; j < cfg->mbx_count; j++) { + if (cfg->mailboxes[j].virt_addr) + cmh_dma_free(cfg->mailboxes[j].queue_size, + cfg->mailboxes[j].virt_addr, + cfg->mailboxes[j].dma_handle); + cfg->mailboxes[j].virt_addr = NULL; + cfg->mailboxes[j].dma_handle = 0; + } + return ret; +} + +/** + * cmh_mqi_cleanup() - Clean up all mailbox queues + * @cfg: CMH configuration describing the mailboxes to tear down + * + * Tears down each mailbox (flush + unlock) and frees the DMA queue + * buffers allocated by cmh_mqi_init(). + */ +void cmh_mqi_cleanup(struct cmh_config *cfg) +{ + unsigned int i; + + for (i = 0; i < cfg->mbx_count; i++) { + struct cmh_mbx_config *m = &cfg->mailboxes[i]; + + cmh_mbx_teardown_one(m); + + if (m->virt_addr) + cmh_dma_free(m->queue_size, m->virt_addr, + m->dma_handle); + m->virt_addr = NULL; + m->dma_handle = 0; + } + + dev_info(cmh_dev(), "MQI cleanup complete\n"); +} diff --git a/drivers/crypto/cmh/cmh_rh.c b/drivers/crypto/cmh/cmh_rh.c new file mode 100644 index 00000000000000..48cb51d24a5e08 --- /dev/null +++ b/drivers/crypto/cmh/cmh_rh.c @@ -0,0 +1,1145 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Response Handler + * + * IRQ-driven completion processing using request_threaded_irq(): + * + * Hardirq: For each MBX, read R_MBX_INTERRUPT. If any bit is set, + * W1C-clear it and mark the MBX for threaded processing. + * Return IRQ_WAKE_THREAD if any MBX had work. + * + * Thread: For each pending MBX, read R_MBX_QUEUE_HEAD. Walk the + * per-MBX transaction queue (oldest first): for every txn + * whose last_vcq_id < new_head, check status, fire the + * completion callback, and free the transaction object. + * + * The DT "cri,cmh" node declares one PLIC interrupt per mailbox, + * matching the real CMH ch_sys_interrupt_mbx[N-1:0] topology. + * Each MBX gets its own Linux virq; the same hardirq/thread pair + * is registered for all of them. The handler still scans all + * mailboxes on every invocation -- this is intentional, as it + * provides robustness against coalesced or missed edges. + * + * IRQ source: resolved from the "cri,cmh" DT node at init time. + * The module's irq= parameter can override with a single shared IRQ. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_rh.h" +#include "cmh_txn.h" +#include "cmh_registers.h" +#include "cmh_config.h" +#include "cmh_debugfs.h" +#include "cmh_dma.h" + +/* Per-mailbox IRQ bookkeeping */ +struct cmh_rh_mbx { + u32 last_head; /* last-observed MBX head position */ + atomic_t irq_bits; /* interrupt bits saved by hardirq (atomic_or) */ + bool pending; /* threaded handler should process this MBX */ + bool restart_pending; /* RESTART issued, awaiting eSW ack */ + u32 restart_retries; /* watchdog ticks since RESTART issued */ + u32 flush_count; /* consecutive failed FLUSH escalations */ + bool wedged; /* recovery failed, MBX offline */ + u32 abort_stall_ticks; /* ticks since async timeout ABORT issued */ +}; + +/* Module-level RH state */ +static struct { + struct cmh_config *cfg; + int irqs[CMH_MAX_CONFIGURED_MBX]; /* per-MBX virqs */ + u32 nirqs; /* number of registered IRQs */ + struct cmh_rh_mbx *mbx; /* array[cfg->mbx_count] */ + atomic_t irq_count; /* hardirq invocation counter */ + bool active; +} rh; + +/* + * Serialise the read-last_head / process_mbx / update-last_head + * sequence between the threaded IRQ handler (process context) and + * the watchdog timer (softirq context). Without this, a timer + * softirq can preempt the kthread mid-sequence, causing both paths + * to process the same head advance and prematurely complete a + * subsequent transaction before the CMH eSW has written its DMA + * output -- leading to data corruption and SLAB freelist poisoning. + * + * The kthread acquires with spin_lock_bh (disables softirqs), the + * watchdog acquires with spin_lock (already in softirq context). + */ +static DEFINE_SPINLOCK(rh_process_lock); + +/* + * Watchdog timer -- missed-IRQ recovery. + * + * Fires every watchdog_ms while rh.active. Reads MBX head registers; + * if any head has advanced without an IRQ, processes completions and + * logs a notice. Standard kernel pattern, analogous to NIC watchdog + * timers. + * + * Safe from timer/softirq context: cmh_reg_read32() is an MMIO read, + * cmh_tm_pop_transaction() uses spin_lock_irqsave(), and TM completion + * callbacks (crypto_request_complete et al.) are documented safe from + * any context including softirq. rh_process_lock serialises the + * head-read / process / head-update sequence against the threaded + * IRQ handler to prevent double-processing of the same completion. + * + * Default 200 ms (5 fires/s) provides ~10 recovery attempts within + * the default vcq_timeout_ms (2 s). Tune via debugfs config/watchdog_ms + * for platforms where interrupt delivery is more reliable (e.g. MSI on + * FPGA/silicon -- 500 ms--1 s may suffice as a safety net). + */ +#define CMH_RH_WATCHDOG_MS_DEFAULT 200 + +/* + * Floor for watchdog_ms to prevent a zero/near-zero value from + * spinning the timer in a tight softirq loop. Enforced at the + * point of use so debugfs writes are never rejected. + */ +#define CMH_RH_WATCHDOG_MS_MIN 10 + +/* + * Maximum watchdog ticks to wait for the eSW to process RESTART + * before escalating to FLUSH. At the default 200 ms interval, + * 5 retries = 1 s -- generous for an operation that should take + * microseconds. If the eSW hasn't responded by then, issue + * MBX_COMMAND_FLUSH to hard-reset the mailbox state. + */ +#define CMH_RH_RESTART_MAX_RETRIES 5 + +/* + * Maximum consecutive FLUSH escalations before marking the MBX as + * wedged. Each FLUSH cycle takes RESTART_MAX_RETRIES watchdog ticks + * (~1 s at default interval). Two failed FLUSHes (~2 s total) + * strongly indicate the eSW is not processing MBX commands at all. + */ +#define CMH_RH_FLUSH_MAX_FAILURES 2 + +/* + * Time budget (ms) after an async timeout ABORT before escalating + * to FLUSH + force-drain. Converted to watchdog ticks at runtime + * via abort_stall_ms / watchdog_ms, so the actual wall-clock bound + * stays constant regardless of watchdog_ms tuning. + * + * The stall detector fires when: + * - The head-of-queue transaction is in TXN_TIMED_OUT state + * - HEAD hasn't advanced (eSW didn't process the ABORT) + * - abort_stall_ticks exceeds the derived threshold + * + * At that point we issue FLUSH + force-drain, completing all pending + * transactions with -ETIMEDOUT and waking any blocked waiters. + * + * Default 5000 ms bounds worst-case D-state to + * async_timeout (2 s) + abort_stall (5 s) = ~7 s. + */ +#define CMH_RH_ABORT_STALL_MS 5000 + +static unsigned int watchdog_ms = CMH_RH_WATCHDOG_MS_DEFAULT; + +/* + * Re-poke R_MBX_QUEUE_TAIL to generate a fresh interrupt to the eSW. + * Writing the current value back is a queue no-op but guarantees a + * SIC interrupt edge, ensuring the eSW wakes from WFI. + */ +static void cmh_rh_poke_tail(void __iomem *base) +{ + u32 tail = cmh_reg_read32(base, R_MBX_QUEUE_TAIL); + + cmh_reg_write32(tail, base, R_MBX_QUEUE_TAIL); +} + +/* + * Drain all remaining in-flight transactions for a mailbox, completing + * each with the given error code. Called after FLUSH (which discards + * all queued VCQs) or when marking a mailbox as wedged. Updates + * last_head to the current hardware HEAD so subsequent polls don't + * re-process the same (now-dead) VCQ IDs as successful completions. + * + * Caller must hold rh_process_lock. + */ +static void cmh_rh_drain_mbx(u32 mbx_idx, int error) +{ + struct transaction_obj *txn; + + while ((txn = cmh_tm_pop_transaction(mbx_idx)) != NULL) { + dev_dbg(cmh_dev(), "rh: mbx[%u] drain vcq=%u..%u err=%d\n", + mbx_idx, txn->first_vcq_id, + txn->last_vcq_id, error); + cmh_txn_finish(txn, error); + cmh_tm_txq_completion_notify(); + } + + rh.mbx[mbx_idx].last_head = + cmh_reg_read32(rh.cfg->mailboxes[mbx_idx].reg_base, + R_MBX_QUEUE_HEAD); +} + +/** + * cmh_rh_force_drain_mbx() - FLUSH + drain a mailbox from external context + * @mbx_idx: Mailbox index to drain + * + * Issues MBX_COMMAND_FLUSH to the eSW, drains all pending transactions + * (completing each with -ECANCELED), and resets all recovery bookkeeping + * including the wedged flag. This is an administrative last-resort + * recovery path exposed via debugfs. + * + * Context: process context. Acquires rh_process_lock internally. + */ +void cmh_rh_force_drain_mbx(u32 mbx_idx) +{ + void __iomem *base; + + if (!rh.cfg || !rh.mbx || mbx_idx >= rh.cfg->mbx_count) + return; + + base = rh.cfg->mailboxes[mbx_idx].reg_base; + + dev_warn(cmh_dev(), "rh: force-drain mbx[%u] (debugfs)\n", mbx_idx); + spin_lock_bh(&rh_process_lock); + cmh_reg_write32(MBX_IRQ_MASK, base, R_MBX_INTERRUPT); + cmh_reg_write32(MBX_COMMAND_FLUSH, base, R_MBX_COMMAND); + cmh_rh_poke_tail(base); + cmh_rh_drain_mbx(mbx_idx, -ECANCELED); + rh.mbx[mbx_idx].abort_stall_ticks = 0; + WRITE_ONCE(rh.mbx[mbx_idx].restart_pending, false); + rh.mbx[mbx_idx].restart_retries = 0; + rh.mbx[mbx_idx].flush_count = 0; + WRITE_ONCE(rh.mbx[mbx_idx].wedged, false); + spin_unlock_bh(&rh_process_lock); +} + +/** + * cmh_rh_mbx_is_wedged() - Check if a mailbox is permanently wedged + * @mbx_idx: Mailbox index to check + * + * Return: true if the mailbox has failed recovery and is offline. + */ +bool cmh_rh_mbx_is_wedged(u32 mbx_idx) +{ + if (!rh.mbx || !rh.cfg || mbx_idx >= rh.cfg->mbx_count) + return false; + + return READ_ONCE(rh.mbx[mbx_idx].wedged); +} + +/** + * cmh_rh_abort_mbx() - Issue MBX_COMMAND_ABORT under rh_process_lock + * @mbx_idx: Mailbox index to abort + * + * Serialises the ABORT write with RESTART/FLUSH commands issued by the + * watchdog, preventing command-register clobber races. Safe to call + * from any context (uses spin_lock_bh). + */ +void cmh_rh_abort_mbx(u32 mbx_idx) +{ + void __iomem *base; + + if (!rh.cfg || !rh.mbx || mbx_idx >= rh.cfg->mbx_count) + return; + + base = rh.cfg->mailboxes[mbx_idx].reg_base; + + spin_lock_bh(&rh_process_lock); + cmh_reg_write32(MBX_COMMAND_ABORT, base, R_MBX_COMMAND); + spin_unlock_bh(&rh_process_lock); +} + +static struct timer_list rh_watchdog; + +/* + * Hardirq handler -- runs with interrupts disabled. + * + * Read and W1C-clear R_MBX_INTERRUPT for each mailbox. + * If any MBX had a pending interrupt, return IRQ_WAKE_THREAD. + * Shared-IRQ safe: returns IRQ_NONE if we didn't handle anything. + */ +static irqreturn_t cmh_rh_hardirq(int irq, void *data) +{ + struct cmh_config *cfg = data; + bool handled = false; + u32 i; + + for (i = 0; i < cfg->mbx_count; i++) { + void __iomem *base = cfg->mailboxes[i].reg_base; + u32 bits; + + bits = cmh_reg_read32(base, R_MBX_INTERRUPT); + if (!bits) + continue; + + /* W1C: write back the set bits to clear them */ + cmh_reg_write32(bits, base, R_MBX_INTERRUPT); + + /* + * Accumulate bits atomically so a second hardirq + * firing while the threaded handler runs does not + * overwrite the first set of bits. + */ + atomic_or((int)bits, &rh.mbx[i].irq_bits); + WRITE_ONCE(rh.mbx[i].pending, true); + handled = true; + } + + /* + * Ordering: the kernel IRQ threading infrastructure + * performs a full barrier between hardirq return and + * the threaded handler invocation. + */ + if (handled) + atomic_inc(&rh.irq_count); + + return handled ? IRQ_WAKE_THREAD : IRQ_NONE; +} + +/* + * Process completions for a single mailbox. + * + * Walk the per-MBX transaction queue FIFO. For each transaction + * whose last_vcq_id is strictly less than the new head, fire the + * completion callback and free the object. + * + * "Strictly less than" using signed (s32) arithmetic handles wrap-around: + * the CMH eSW uses monotonically increasing 32-bit VCQ IDs. + */ +static void cmh_rh_process_mbx(u32 mbx_idx, u32 new_head, u32 irq_bits) +{ + struct transaction_obj *txn; + int error = 0; + + /* Determine error state from saved IRQ bits */ + if (irq_bits & MBX_ERROR_IRQ) { + void __iomem *base = rh.cfg->mailboxes[mbx_idx].reg_base; + u32 status = cmh_reg_read32(base, R_MBX_STATUS); + + error = -EIO; + dev_dbg(cmh_dev(), "rh: mbx[%u] error status=0x%08x (code=%u cmd_idx=%u)\n", + mbx_idx, status, + MBX_STATUS_ERROR_CODE(status), + MBX_STATUS_CMD_INDEX(status)); + + /* + * ECHILD (10) in the parent status means a child VCQ + * failed internally. Read R_MBX_CHILD for the actual + * root cause (real errno, child core ID, child cmd idx). + */ + if (MBX_STATUS_ERROR_CODE(status) == ECHILD) { + u32 child = cmh_reg_read32(base, R_MBX_CHILD); + + dev_dbg(cmh_dev(), + "rh: mbx[%u] child error=0x%08x (core=%u code=%u cmd_idx=%u)\n", + mbx_idx, child, + MBX_STATUS_CORE_ID(child), + MBX_STATUS_ERROR_CODE(child), + MBX_STATUS_CMD_INDEX(child)); + } + + /* + * CMH eSW does not advance head on error -- the MBX is + * stuck in ERROR state until the host issues a recovery + * command. However, HEAD may have advanced past one or + * more already-completed transactions before the error + * occurred (their completion IRQ may not have been + * processed yet). Retire those normally first, then + * force-complete the NEXT transaction (the one that + * actually failed) with -EIO. + * + * MBX command semantics after ERROR: + * CONTINUE -- re-run the same VCQ at HEAD (retry) + * RESTART -- advance HEAD+1, skip failed, resume + * FLUSH -- HEAD=TAIL, flush all HWCs, discard queue + */ + + /* First: retire transactions completed before the error */ + while ((txn = cmh_tm_peek_transaction(mbx_idx)) != NULL) { + if ((s32)(new_head - txn->last_vcq_id) <= 0) + break; + txn = cmh_tm_pop_transaction(mbx_idx); + if (!txn) + break; + dev_dbg(cmh_dev(), + "rh: mbx[%u] pre-error complete vcq=%u..%u\n", + mbx_idx, txn->first_vcq_id, + txn->last_vcq_id); + cmh_txn_finish(txn, 0); + cmh_tm_txq_completion_notify(); + } + + /* Now pop and fail the transaction that actually errored */ + txn = cmh_tm_pop_transaction(mbx_idx); + if (txn) { + dev_dbg(cmh_dev(), "rh: mbx[%u] error-complete vcq=%u..%u\n", + mbx_idx, txn->first_vcq_id, + txn->last_vcq_id); + cmh_txn_finish(txn, error); + cmh_tm_txq_completion_notify(); + } else { + u32 head_reg, tail_reg; + + head_reg = cmh_reg_read32(base, R_MBX_QUEUE_HEAD); + tail_reg = cmh_reg_read32(base, R_MBX_QUEUE_TAIL); + dev_warn_ratelimited(cmh_dev(), + "rh: mbx[%u] ERROR with empty txn queue (orphaned) status=0x%08x head=%u tail=%u core=%u ecode=%u cmd_idx=%u\n", + mbx_idx, status, + head_reg, tail_reg, + MBX_STATUS_CORE_ID(status), + MBX_STATUS_ERROR_CODE(status), + MBX_STATUS_CMD_INDEX(status)); + } + { + struct cmh_mbx_stats *s = cmh_debugfs_mbx_stats(mbx_idx); + + if (s) + atomic64_inc(&s->vcqs_errors); + } + + /* + * W1C-clear R_MBX_INTERRUPT before issuing RESTART. + * + * The eSW sets MBX_ERROR_IRQ in R_MBX_INTERRUPT when + * it writes ERROR status. On platforms where the + * hardirq handler runs (IRQ wired to GIC), this bit + * is cleared there. On polling-only platforms (no + * IRQ line), it must be cleared explicitly before + * issuing a recovery command to de-assert the + * MBX-to-SIC interrupt line. + */ + cmh_reg_write32(MBX_IRQ_MASK, base, R_MBX_INTERRUPT); + cmh_reg_write32(MBX_COMMAND_RESTART, base, R_MBX_COMMAND); + + /* + * Poke R_MBX_QUEUE_TAIL to guarantee the eSW receives + * an interrupt. + * + * Writing R_MBX_COMMAND alone may not produce a new + * SIC interrupt edge if the MBX-to-SIC line is still + * asserted from prior error processing. The eSW RUN + * handler re-writes ERROR_IRQ to R_MBX_INTERRUPT on + * every spurious wakeup while in ERROR state, which + * can keep the SIC line high on level-triggered HW. + * + * R_MBX_QUEUE_TAIL writes always generate a fresh + * interrupt to the eSW (this is the normal VCQ + * submission path). Writing the current TAIL value + * back is a no-op from the queue perspective but + * ensures the eSW wakes from WFI and processes the + * RESTART command. + */ + cmh_rh_poke_tail(base); + WRITE_ONCE(rh.mbx[mbx_idx].restart_pending, true); + rh.mbx[mbx_idx].restart_retries = 0; + return; + } + + /* + * Pop completed transactions. A transaction is complete when + * the CMH eSW has advanced head past its last VCQ ID: + * (s32)(new_head - txn->last_vcq_id) > 0 + * Using signed comparison for correct wrap-around handling. + * + * Multi-VCQ note: transactions spanning multiple slots (e.g. + * SLH-DSA with 3+ VCQs) are treated atomically -- either the + * head has passed all of them or none. The CMH eSW processes + * multi-VCQ groups sequentially within a single mailbox and + * only advances HEAD after the entire group completes. Per-slot + * progress validation (checking intermediate HEAD positions + * within a multi-VCQ group) is not implemented because: + * 1. The eSW guarantees atomic group completion semantics + * 2. Partial progress is only observable during processing, + * never at a completion boundary + * 3. Adding intermediate checks would require tracking + * per-slot status with no correctness benefit + * + * A defensive WARN_ON_ONCE detects eSW misbehavior: if HEAD + * lands between first_vcq_id and last_vcq_id of a multi-VCQ + * transaction, the eSW violated its atomic group contract. + */ + while ((txn = cmh_tm_peek_transaction(mbx_idx)) != NULL) { + if ((s32)(new_head - txn->last_vcq_id) <= 0) { + /* + * Not yet complete. For multi-VCQ transactions, + * assert HEAD hasn't partially advanced into the + * group -- that would indicate eSW firmware bug. + */ + WARN_ON_ONCE(txn->first_vcq_id != txn->last_vcq_id && + (s32)(new_head - txn->first_vcq_id) > 0); + break; + } + + txn = cmh_tm_pop_transaction(mbx_idx); + if (!txn) + break; + + dev_dbg(cmh_dev(), "rh: mbx[%u] complete vcq=%u..%u err=%d\n", + mbx_idx, txn->first_vcq_id, txn->last_vcq_id, + error); + + { + struct cmh_mbx_stats *s = cmh_debugfs_mbx_stats(mbx_idx); + + if (s) { + u32 n = txn->last_vcq_id - + txn->first_vcq_id + 1; + + atomic64_add(n, &s->vcqs_completed); + } + } + + cmh_txn_finish(txn, error); + cmh_tm_txq_completion_notify(); + } +} + +/* + * Threaded IRQ handler -- runs in process context. + * + * Walk all MBXes that had pending interrupts. After processing the + * pending set, do a final hardware poll of all MBX head registers to + * catch completions whose PLIC interrupt was consumed during an + * earlier register access (e.g. an inline interrupt notification + * during MMIO can cause the PLIC edge to be claimed before the + * hardirq sees it). + */ +static irqreturn_t cmh_rh_thread(int irq, void *data) +{ + struct cmh_config *cfg = data; + u32 i; + bool recheck; + + do { + recheck = false; + + for (i = 0; i < cfg->mbx_count; i++) { + u32 new_head, irq_bits; + + if (!READ_ONCE(rh.mbx[i].pending)) + continue; + + irq_bits = (u32)atomic_xchg(&rh.mbx[i].irq_bits, 0); + WRITE_ONCE(rh.mbx[i].pending, false); + + spin_lock_bh(&rh_process_lock); + new_head = cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_HEAD); + + if (new_head == rh.mbx[i].last_head && !irq_bits) { + spin_unlock_bh(&rh_process_lock); + continue; + } + + cmh_rh_process_mbx(i, new_head, irq_bits); + rh.mbx[i].last_head = new_head; + spin_unlock_bh(&rh_process_lock); + } + + /* + * Re-check: if the hardirq fired again while we were + * processing, pending flags will be set again. + */ + for (i = 0; i < cfg->mbx_count; i++) { + if (READ_ONCE(rh.mbx[i].pending)) { + recheck = true; + break; + } + } + } while (recheck); + + /* + * Final hardware poll: read every MBX head register and status + * to catch completions or errors whose interrupt was missed. + */ + for (i = 0; i < cfg->mbx_count; i++) { + u32 new_head; + u32 status; + u32 poll_irq_bits = 0; + + spin_lock_bh(&rh_process_lock); + new_head = cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_HEAD); + status = cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_STATUS); + + if (MBX_STATUS_CODE(status) == MBX_STATUS_ERROR) { + if (READ_ONCE(rh.mbx[i].wedged)) { + spin_unlock_bh(&rh_process_lock); + continue; + } + if (READ_ONCE(rh.mbx[i].restart_pending)) { + /* + * HEAD advanced while restart_pending means + * RESTART worked but next VCQ also failed. + * Clear restart state and process new error. + */ + if (new_head != rh.mbx[i].last_head) { + WRITE_ONCE(rh.mbx[i].restart_pending, + false); + rh.mbx[i].restart_retries = 0; + } else { + spin_unlock_bh(&rh_process_lock); + continue; + } + } + poll_irq_bits = MBX_ERROR_IRQ; + } else { + WRITE_ONCE(rh.mbx[i].restart_pending, false); + rh.mbx[i].restart_retries = 0; + rh.mbx[i].flush_count = 0; + } + + if (new_head != rh.mbx[i].last_head || poll_irq_bits) { + cmh_rh_process_mbx(i, new_head, poll_irq_bits); + rh.mbx[i].last_head = new_head; + } + spin_unlock_bh(&rh_process_lock); + } + + return IRQ_HANDLED; +} + +/* + * Watchdog timer callback -- missed-IRQ recovery. + * + * Reads all MBX head registers. If any head advanced without a + * corresponding IRQ, process the completions here. Re-arms itself + * while rh.active is true. + */ +static void cmh_rh_watchdog_fn(struct timer_list *t) +{ + u32 i; + + if (!rh.active || !rh.cfg || !rh.mbx) + return; + + for (i = 0; i < rh.cfg->mbx_count; i++) { + u32 new_head; + u32 status; + u32 irq_bits = 0; + + spin_lock(&rh_process_lock); + new_head = cmh_reg_read32(rh.cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_HEAD); + status = cmh_reg_read32(rh.cfg->mailboxes[i].reg_base, + R_MBX_STATUS); + + if (MBX_STATUS_CODE(status) == MBX_STATUS_ERROR) { + if (READ_ONCE(rh.mbx[i].wedged)) { + spin_unlock(&rh_process_lock); + continue; + } + /* + * Back-to-back failure scenario: the crypto API + * (e.g. testmgr) may submit requests continuously. + * If RESTART succeeds but the next VCQ also fails, + * the entire RESTART->IDLE->RUN->ERROR cycle can + * complete within a single 200ms watchdog period. + * Without the HEAD-advance check below, the watchdog + * would mistake the new error for a failed RESTART, + * increment restart_retries, and eventually escalate + * to FLUSH -- wedging the mailbox unnecessarily. + */ + if (READ_ONCE(rh.mbx[i].restart_pending)) { + void __iomem *base = + rh.cfg->mailboxes[i].reg_base; + + /* + * HEAD advanced since RESTART was issued: + * RESTART succeeded, this is a fresh error. + * Clear recovery state and process normally. + */ + if (new_head != rh.mbx[i].last_head) { + dev_dbg(cmh_dev(), + "rh: watchdog: mbx[%u] head advanced %u->%u during restart -- new error\n", + i, rh.mbx[i].last_head, + new_head); + WRITE_ONCE(rh.mbx[i].restart_pending, + false); + rh.mbx[i].restart_retries = 0; + goto new_error; + } + + rh.mbx[i].restart_retries++; + if (rh.mbx[i].restart_retries > + CMH_RH_RESTART_MAX_RETRIES) { + rh.mbx[i].flush_count++; + if (rh.mbx[i].flush_count >= + CMH_RH_FLUSH_MAX_FAILURES) { + u32 hb, ei, cmd; + + cmd = cmh_reg_read32(base, R_MBX_COMMAND); + hb = cmh_reg_read32(rh.cfg->sic_mapped, + R_SIC_SW_HEARTBEAT); + ei = cmh_reg_read32(rh.cfg->sic_mapped, + R_SIC_SW_ERROR_INFO); + dev_crit(cmh_dev(), + "rh: mbx[%u] wedged after %u FLUSHes (cmd=0x%x status=0x%x hb=0x%x err=0x%x)\n", + i, + rh.mbx[i].flush_count, + cmd, status, + hb, ei); + WRITE_ONCE(rh.mbx[i].wedged, + true); + cmh_rh_drain_mbx(i, -EIO); + spin_unlock(&rh_process_lock); + continue; + } + /* + * Backstop: eSW did not respond + * to RESTART within the retry + * budget. Escalate to FLUSH + * which is a harder reset of + * the eSW mailbox state. + */ + dev_err(cmh_dev(), + "rh: watchdog: mbx[%u] RESTART unresponsive after %u ticks, escalating to FLUSH (attempt %u/%u)\n", + i, rh.mbx[i].restart_retries, + rh.mbx[i].flush_count, + CMH_RH_FLUSH_MAX_FAILURES); + cmh_reg_write32(MBX_IRQ_MASK, + base, + R_MBX_INTERRUPT); + cmh_reg_write32(MBX_COMMAND_FLUSH, + base, + R_MBX_COMMAND); + cmh_rh_poke_tail(base); + cmh_rh_drain_mbx(i, -EIO); + WRITE_ONCE(rh.mbx[i].restart_pending, + false); + rh.mbx[i].restart_retries = 0; + spin_unlock(&rh_process_lock); + continue; + } + /* + * RESTART was already issued on a prior + * tick but the eSW hasn't cleared the + * ERROR status yet. Do NOT pop another + * transaction -- that would cascade-kill + * unrelated in-flight work. Re-poke TAIL + * in case the eSW missed the interrupt. + */ + cmh_rh_poke_tail(base); + dev_dbg_ratelimited(cmh_dev(), + "rh: watchdog: mbx[%u] restart pending (%u/%u) status=0x%08x, re-poke\n", + i, + rh.mbx[i].restart_retries, + CMH_RH_RESTART_MAX_RETRIES, + status); + spin_unlock(&rh_process_lock); + continue; + } +new_error: + dev_dbg_ratelimited(cmh_dev(), + "rh: watchdog: mbx[%u] error status=0x%08x (missed error IRQ) head=%u tail=%u core=%u ecode=%u cmd_idx=%u\n", + i, status, new_head, + cmh_reg_read32(rh.cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_TAIL), + MBX_STATUS_CORE_ID(status), + MBX_STATUS_ERROR_CODE(status), + MBX_STATUS_CMD_INDEX(status)); + irq_bits = MBX_ERROR_IRQ; + } else { + /* eSW cleared ERROR -- recovery succeeded */ + WRITE_ONCE(rh.mbx[i].restart_pending, false); + rh.mbx[i].restart_retries = 0; + rh.mbx[i].flush_count = 0; + } + + if (new_head != rh.mbx[i].last_head || irq_bits) { + if (new_head != rh.mbx[i].last_head) + dev_dbg_ratelimited(cmh_dev(), + "rh: watchdog: mbx[%u] head %u->%u (missed IRQ recovery)\n", + i, rh.mbx[i].last_head, + new_head); + cmh_rh_process_mbx(i, new_head, irq_bits); + rh.mbx[i].last_head = new_head; + rh.mbx[i].abort_stall_ticks = 0; + } + + /* + * Abort-stall detector: if the head-of-queue transaction + * timed out (state == TXN_TIMED_OUT) but the eSW hasn't + * responded (HEAD didn't advance, no ERROR status): + * + * tick 1: issue MBX_COMMAND_ABORT (serialised + * under rh_process_lock -- safe against + * concurrent RESTART/FLUSH) + * ticks 2..N-1: wait for eSW to respond with ERROR + * tick N: escalate to FLUSH + force-drain + * + * If the eSW responds with ERROR between ticks, the ERROR + * status branch above handles RESTART recovery and resets + * abort_stall_ticks via the restart_pending guard. + */ + if (!READ_ONCE(rh.mbx[i].wedged) && + !READ_ONCE(rh.mbx[i].restart_pending)) { + struct transaction_obj *head_txn; + + head_txn = cmh_tm_peek_transaction(i); + if (head_txn && + atomic_read(&head_txn->state) == TXN_TIMED_OUT) { + unsigned int stall_max; + void __iomem *base = + rh.cfg->mailboxes[i].reg_base; + + rh.mbx[i].abort_stall_ticks++; + + if (rh.mbx[i].abort_stall_ticks == 1) { + dev_warn(cmh_dev(), + "rh: watchdog: mbx[%u] head txn timed out, issuing ABORT\n", + i); + cmh_reg_write32(MBX_COMMAND_ABORT, + base, + R_MBX_COMMAND); + } + + stall_max = DIV_ROUND_UP(CMH_RH_ABORT_STALL_MS, + max(watchdog_ms, + CMH_RH_WATCHDOG_MS_MIN)); + if (rh.mbx[i].abort_stall_ticks >= + stall_max) { + dev_err(cmh_dev(), + "rh: watchdog: mbx[%u] abort stall (%u ticks) -- FLUSH + drain\n", + i, rh.mbx[i].abort_stall_ticks); + cmh_reg_write32(MBX_COMMAND_FLUSH, + base, R_MBX_COMMAND); + cmh_rh_drain_mbx(i, -ETIMEDOUT); + rh.mbx[i].abort_stall_ticks = 0; + } + } else { + rh.mbx[i].abort_stall_ticks = 0; + } + } + spin_unlock(&rh_process_lock); + } + + if (rh.active) { + unsigned int wdog = max(watchdog_ms, CMH_RH_WATCHDOG_MS_MIN); + + mod_timer(&rh_watchdog, + jiffies + msecs_to_jiffies(wdog)); + } +} + +/* + * Resolve per-MBX Linux virqs for the CMH interrupt lines. + * + * Strategy: + * 1. If cfg->irq >= 0, use it as a shared IRQ for all MBXes + * 2. Otherwise, find the "cri,cmh" DT node and map one IRQ per + * active mailbox using cfg->mailboxes[i].instance as the DT + * interrupt index (matching the per-MBX PLIC wiring) + * + * Populates rh.irqs[] and rh.nirqs. Returns 0 on success, or a + * negative errno if no IRQs could be resolved (polling-only mode). + */ +static int cmh_rh_resolve_irqs(struct cmh_config *cfg) +{ + struct device_node *np; + u32 i; + + rh.nirqs = 0; + + /* Single legacy IRQ from DT: shared across all MBXes */ + if (cfg->irq >= 0) { + rh.irqs[0] = cfg->irq; + rh.nirqs = 1; + dev_dbg(cmh_dev(), "rh: using single DT IRQ %d for all %u MBXes\n", + cfg->irq, cfg->mbx_count); + return 0; + } + + np = cfg->of_node; + if (!np) { + dev_warn(cmh_dev(), "rh: no DT node -- IRQ disabled\n"); + return -ENODEV; + } + + for (i = 0; i < cfg->mbx_count; i++) { + int dt_idx = cfg->mailboxes[i].instance; + int virq = of_irq_get(np, dt_idx); + + if (virq <= 0) { + dev_warn(cmh_dev(), "rh: failed to map IRQ for MBX%u (DT index %d, rc=%d)\n", + i, dt_idx, virq); + return -ENODEV; + } + rh.irqs[i] = virq; + dev_dbg(cmh_dev(), "rh: MBX%u -> IRQ %d (DT index %d)\n", + i, virq, dt_idx); + } + + rh.nirqs = cfg->mbx_count; + return 0; +} + +/** + * cmh_rh_init() - Initialize the response handler + * @cfg: Device configuration (mailbox count, MMIO bases, IRQ info) + * + * Resolve per-mailbox IRQs from the device tree (or module parameter + * override), register threaded IRQ handlers (hardirq + kthread), and + * arm the missed-IRQ software watchdog timer. If no IRQs can be + * resolved, falls back to watchdog-only polling mode. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_rh_init(struct cmh_config *cfg) +{ + int ret; + u32 i; + + rh.cfg = cfg; + rh.nirqs = 0; + rh.active = false; + atomic_set(&rh.irq_count, 0); + + /* Allocate per-MBX tracking */ + rh.mbx = kcalloc(cfg->mbx_count, sizeof(*rh.mbx), GFP_KERNEL); + if (!rh.mbx) + return -ENOMEM; + + /* Resolve per-MBX IRQs */ + if (cmh_rh_resolve_irqs(cfg) < 0) { + /* + * No IRQs available. The watchdog timer provides + * a polling fallback: it reads MBX head registers + * periodically and processes completions. This is + * slower than IRQ-driven completion but functional. + * + * Completion latency in polling-only mode is bounded + * by the watchdog interval (default 200 ms, tunable + * via debugfs config/watchdog_ms). + */ + dev_warn(cmh_dev(), + "rh: no IRQs -- using watchdog polling (interval %u ms)\n", + watchdog_ms); + + /* Seed last_head from HW before first watchdog tick */ + for (i = 0; i < cfg->mbx_count; i++) + rh.mbx[i].last_head = + cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_HEAD); + + rh.active = true; + timer_setup(&rh_watchdog, cmh_rh_watchdog_fn, 0); + mod_timer(&rh_watchdog, jiffies + + msecs_to_jiffies(max(watchdog_ms, + CMH_RH_WATCHDOG_MS_MIN))); + return 0; + } + + /* Initialize per-MBX state: read current head positions */ + for (i = 0; i < cfg->mbx_count; i++) + rh.mbx[i].last_head = cmh_reg_read32(rh.cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_HEAD); + + /* + * Register threaded IRQ handlers. + * + * DT per-MBX path: one distinct virq per MBX, nirqs == mbx_count. + * DT single-IRQ path: one shared IRQ, nirqs == 1. The handler + * scans all mailboxes unconditionally, so a single registration + * suffices. + * + * Use IRQF_SHARED only for the single-IRQ path where one line + * is shared across all MBXes. Dedicated per-MBX virqs need no + * sharing flag. + */ + { + unsigned long irqflags = (rh.nirqs == 1 && cfg->mbx_count > 1) + ? IRQF_SHARED : 0; + + for (i = 0; i < rh.nirqs; i++) { + ret = request_threaded_irq(rh.irqs[i], + cmh_rh_hardirq, + cmh_rh_thread, + irqflags, + "cmh", cfg); + if (ret) { + dev_err(cmh_dev(), "rh: request_threaded_irq(%d) for MBX%u failed (rc=%d)\n", + rh.irqs[i], i, ret); + /* Unwind previously registered IRQs */ + while (i--) + free_irq(rh.irqs[i], cfg); + rh.nirqs = 0; + kfree(rh.mbx); + rh.mbx = NULL; + return ret; + } + } + } + + rh.active = true; + + /* Enable MBX completion interrupts (DONE + ERROR) */ + for (i = 0; i < cfg->mbx_count; i++) { + u32 stale; + + /* + * W1C any interrupt bits that accumulated between + * MQI setup and now (e.g. CMH eSW processing stale + * commands) before enabling the mask. + */ + stale = cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT); + if (stale) + cmh_reg_write32(stale, cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT); + + cmh_reg_write32(MBX_IRQ_MASK, + cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT_MASK); + } + + dev_info(cmh_dev(), "rh: initialized (%u IRQs, %u mailboxes, watchdog %u ms)\n", + rh.nirqs, cfg->mbx_count, watchdog_ms); + + /* Arm missed-IRQ watchdog timer */ + timer_setup(&rh_watchdog, cmh_rh_watchdog_fn, 0); + mod_timer(&rh_watchdog, jiffies + + msecs_to_jiffies(max(watchdog_ms, + CMH_RH_WATCHDOG_MS_MIN))); + + return 0; +} + +/** + * cmh_rh_suspend() - Suspend the response handler + * @cfg: Device configuration + * + * Stop the watchdog timer and mask mailbox interrupts at the hardware + * level. The IRQ handlers remain registered so that resume can + * re-enable them without re-requesting. + */ +void cmh_rh_suspend(struct cmh_config *cfg) +{ + u32 i; + + if (!rh.active) + return; + + /* Stop the watchdog before masking HW interrupts */ + timer_delete_sync(&rh_watchdog); + + /* Mask MBX interrupts at the hardware level */ + for (i = 0; i < cfg->mbx_count; i++) + cmh_reg_write32(0, cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT_MASK); + + /* + * Ensure no threaded IRQ handler is still in-flight. + * After masking, a handler may already have been scheduled. + * synchronize_irq() waits for it to complete before we + * proceed with suspend (which tears down TM state). + */ + for (i = 0; i < rh.nirqs; i++) + synchronize_irq(rh.irqs[i]); + + rh.active = false; + dev_dbg(cmh_dev(), "rh: suspended\n"); +} + +/** + * cmh_rh_resume() - Resume the response handler after suspend + * @cfg: Device configuration + * + * Re-synchronize per-mailbox head tracking with hardware, clear stale + * interrupt bits accumulated during the power transition, re-enable + * mailbox completion interrupts, and re-arm the watchdog timer. + */ +void cmh_rh_resume(struct cmh_config *cfg) +{ + u32 i; + + if (!rh.mbx || !cfg) + return; + + /* Re-sync per-MBX head tracking with hardware */ + for (i = 0; i < cfg->mbx_count; i++) { + u32 stale; + + rh.mbx[i].last_head = + cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_QUEUE_HEAD); + + /* W1C any stale interrupt bits from the power transition */ + stale = cmh_reg_read32(cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT); + if (stale) + cmh_reg_write32(stale, cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT); + + /* Re-enable MBX completion interrupts */ + cmh_reg_write32(MBX_IRQ_MASK, cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT_MASK); + } + + rh.active = true; + + /* Re-arm the watchdog */ + mod_timer(&rh_watchdog, jiffies + + msecs_to_jiffies(max(watchdog_ms, + CMH_RH_WATCHDOG_MS_MIN))); + dev_dbg(cmh_dev(), "rh: resumed\n"); +} + +/** + * cmh_rh_cleanup() - Clean up the response handler + * @cfg: Device configuration + * + * Stop the watchdog timer, mask mailbox interrupts at the hardware + * level, release all registered IRQ handlers, and free per-mailbox + * tracking state. Safe to call even if init was never completed. + */ +void cmh_rh_cleanup(struct cmh_config *cfg) +{ + if (rh.active) { + u32 i; + + /* Cancel watchdog before disabling interrupts */ + timer_delete_sync(&rh_watchdog); + + /* Disable MBX interrupts before releasing handlers */ + for (i = 0; i < cfg->mbx_count; i++) + cmh_reg_write32(0, + cfg->mailboxes[i].reg_base, + R_MBX_INTERRUPT_MASK); + + /* Release all per-MBX IRQs */ + for (i = 0; i < rh.nirqs; i++) + free_irq(rh.irqs[i], cfg); + dev_dbg(cmh_dev(), "rh: %u IRQs released\n", rh.nirqs); + rh.nirqs = 0; + rh.active = false; + } + + dev_dbg(cmh_dev(), "rh: %u IRQs handled\n", + atomic_read(&rh.irq_count)); + + kfree(rh.mbx); + rh.mbx = NULL; + + dev_info(cmh_dev(), "rh: cleaned up\n"); +} + +/* -- debugfs timeout accessor ------------------------------------------ */ + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +/** + * cmh_rh_timeout_watchdog_ptr() - Return pointer to watchdog_ms for debugfs + * + * Exposes the Response Handler watchdog timeout for runtime tuning + * via debugfs config/ directory. + * + * Return: pointer to the static watchdog_ms variable. + */ +unsigned int *cmh_rh_timeout_watchdog_ptr(void) { return &watchdog_ms; } +#endif diff --git a/drivers/crypto/cmh/cmh_sysfs.c b/drivers/crypto/cmh/cmh_sysfs.c new file mode 100644 index 00000000000000..ab482a222167c6 --- /dev/null +++ b/drivers/crypto/cmh/cmh_sysfs.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- sysfs Device Attributes + * + * Exposes hardware identity and status as read-only sysfs attributes + * under /sys/devices/platform/cmh/. Wired via .dev_groups in the + * platform_driver struct -- the driver core creates and removes these + * automatically around .probe() / .remove(). + * + * Because .dev_groups is used (not manual sysfs_create_group), the + * driver core guarantees that attributes are created after .probe() + * sets drvdata and removed before .remove() clears it. Therefore + * platform_get_drvdata() cannot return NULL in any show callback and + * no NULL check is needed. Same pattern as caam/ctrl.c and + * ccree/cc_sysfs.c. + */ + +#include +#include +#include + +#include "cmh.h" +#include "cmh_registers.h" +#include "cmh_sysfs.h" + +static ssize_t fw_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cmh_device *cmh = platform_get_drvdata(to_platform_device(dev)); + struct cmh_config *cfg = &cmh->config; + + if (!cfg->sic_mapped) + return -ENODEV; + + return sysfs_emit(buf, "0x%08x\n", + cmh_reg_read32(cfg->sic_mapped, R_SIC_SW_VERSION)); +} +static DEVICE_ATTR_RO(fw_version); + +static ssize_t hw_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cmh_device *cmh = platform_get_drvdata(to_platform_device(dev)); + struct cmh_config *cfg = &cmh->config; + + if (!cfg->sic_mapped) + return -ENODEV; + + return sysfs_emit(buf, "0x%08x\n", + cmh_reg_read32(cfg->sic_mapped, R_SIC_HW_VERSION0)); +} +static DEVICE_ATTR_RO(hw_version); + +static ssize_t boot_status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cmh_device *cmh = platform_get_drvdata(to_platform_device(dev)); + struct cmh_config *cfg = &cmh->config; + + if (!cfg->sic_mapped) + return -ENODEV; + + return sysfs_emit(buf, "0x%08x\n", + cmh_reg_read32(cfg->sic_mapped, R_SIC_BOOT_STATUS)); +} +static DEVICE_ATTR_RO(boot_status); + +static ssize_t mbx_available_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cmh_device *cmh = platform_get_drvdata(to_platform_device(dev)); + struct cmh_config *cfg = &cmh->config; + + if (!cfg->sic_mapped) + return -ENODEV; + + return sysfs_emit(buf, "0x%08x\n", + cmh_reg_read32(cfg->sic_mapped, R_SIC_MBX_AVAILABILITY)); +} +static DEVICE_ATTR_RO(mbx_available); + +static ssize_t mbx_count_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cmh_device *cmh = platform_get_drvdata(to_platform_device(dev)); + + return sysfs_emit(buf, "%u\n", cmh->config.mbx_count); +} +static DEVICE_ATTR_RO(mbx_count); + +static struct attribute *cmh_sysfs_attrs[] = { + &dev_attr_fw_version.attr, + &dev_attr_hw_version.attr, + &dev_attr_boot_status.attr, + &dev_attr_mbx_available.attr, + &dev_attr_mbx_count.attr, + NULL, +}; + +static const struct attribute_group cmh_sysfs_group = { + .attrs = cmh_sysfs_attrs, +}; + +const struct attribute_group *cmh_sysfs_groups[] = { + &cmh_sysfs_group, + NULL, +}; diff --git a/drivers/crypto/cmh/cmh_txn.c b/drivers/crypto/cmh/cmh_txn.c new file mode 100644 index 00000000000000..3c696a8baac5b2 --- /dev/null +++ b/drivers/crypto/cmh/cmh_txn.c @@ -0,0 +1,1978 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Transaction Manager + * + * Dedicated kthread that dequeues command messages, builds VCQs in + * DMA queue slots, and rings the MBX doorbell. + * + * Command flow: + * 1. Caller posts command_msg via cmh_tm_post_command() + * 2. TM thread wakes, dequeues msg from CMQ + * 3. Selects mailbox (core-to-MBX affinity, or caller-pinned) + * 4. Copies pre-built VCQ entries into DMA slot at tail + * 5. Creates transaction_obj, appends to per-MBX txn queue + * 6. Writes tail+1 -> R_MBX_QUEUE_TAIL (doorbell) + * + * The Response Handler (cmh_rh.c) walks per-MBX txn queues + * when an IRQ fires and the head advances, firing completion callbacks. + * + * Transaction state machine + * ------------------------- + * Each async transaction moves through the following states. DMA + * buffers remain mapped and owned by the HW until the COMPLETE state + * is reached -- only then are they safe to unmap/free. + * + * QUEUED --[TM posts to HW]--> INFLIGHT + * (cmq) | | \ + * | | \--[timer fires]--> + * | | TIMED_OUT + * | | | + * | [HW completes / [HW completes / + * | RH pops txn] RH pops txn] + * | | | + * | v v + * | COMPLETE COMPLETE + * | (err=HW rc) (err=-ETIMEDOUT) + * | + * +--[pre-submit fail]--> freed (callback never fires) + * + * Note: QUEUED is the command_msg phase (sitting in the CMQ list, + * not yet a transaction_obj). The transaction_obj states tracked + * by atomic_cmpxchg are INFLIGHT, TIMED_OUT, and COMPLETE only. + * + * Completion callback context guarantee: + * The crypto_request_complete() callback is invoked from one of: + * - The RH threaded IRQ handler (process context, BH disabled) + * - The watchdog timer (softirq / timer context) + * - The TM kthread during queue drain/cleanup (process context) + * + * It is NEVER invoked from hardirq context. + * + * The watchdog path runs from timer softirq because it must recover + * missed IRQs without sleeping. This is crypto-API-compliant: + * crypto_request_complete() is documented safe from any context + * (including softirq). Callers must NOT assume process context in + * their completion callbacks -- all operations therein must be + * softirq-safe (no mutex, no GFP_KERNEL, no sleeping locks). + * + * For backlog promotion (-EINPROGRESS callbacks), the callback runs + * under the CMQ spinlock with IRQs disabled -- callers must handle + * this per the crypto API backlog contract. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_txn.h" +#include "cmh_rh.h" +#include "cmh_registers.h" +#include "cmh_config.h" +#include "cmh_vcq.h" +#include "cmh_debugfs.h" +#include "cmh_dma.h" + +/* Module State */ + +static struct { + struct cmh_config *cfg; + struct task_struct *thread; + bool running; + + /* Command Message Queue (CMQ) */ + struct list_head cmq; + spinlock_t cmq_lock; /* protects cmq + backlog lists */ + wait_queue_head_t cmq_waitq; + + /* Backlog queue for CRYPTO_TFM_REQ_MAY_BACKLOG requests */ + struct list_head backlog; + u32 backlog_depth; + + /* Per-mailbox transaction queues */ + struct cmh_mbx_txq *txqs; /* array[cfg->mbx_count] */ + + /* Round-robin mailbox selector */ + u32 next_mbx; +} tm; + +static unsigned int cmq_max_depth = 256; +module_param(cmq_max_depth, uint, 0444); +MODULE_PARM_DESC(cmq_max_depth, + "Max pending commands in the Command Message Queue (default: 256)"); + +static unsigned int backlog_max_depth = 1024; +module_param(backlog_max_depth, uint, 0444); +MODULE_PARM_DESC(backlog_max_depth, + "Max pending commands in the backlog queue (0 = disable backlog, default: 1024)"); + +static unsigned int async_timeout_ms = 2000; + +#define CMH_TM_BACKOFF_MIN_US 100 /* queue-full backoff range (us) */ +#define CMH_TM_BACKOFF_MAX_US 500 +static unsigned int cmq_depth; /* current CMQ depth, protected by tm.cmq_lock */ + +/* + * Monotonically increasing counter bumped by cmh_tm_txq_completion_notify(). + * Used as a generation check in the queue-full backoff predicate so that + * wait_event_interruptible_timeout() returns immediately when a TXQ + * completion frees a slot, rather than sleeping for the full timeout. + */ +static atomic_t txq_completion_gen; + +/* -- Debugfs stat helpers (avoid anonymous compound blocks) ------------- */ + +static void cmh_stat_inc_mbx_queue_full(u32 mbx_idx) +{ + struct cmh_mbx_stats *s = cmh_debugfs_mbx_stats(mbx_idx); + + if (s) + atomic64_inc(&s->queue_full_count); +} + +static void cmh_stat_record_vcq_submit(u32 mbx_idx, u32 num_vcqs, u32 depth) +{ + struct cmh_mbx_stats *s = cmh_debugfs_mbx_stats(mbx_idx); + + if (s) { + atomic64_add(num_vcqs, &s->vcqs_submitted); + cmh_stat_update_max(&s->max_queue_depth, (s64)depth); + } +} + +static void cmh_stat_inc_tm_backoff(void) +{ + struct cmh_tm_stats *s = cmh_debugfs_tm_stats(); + + if (s) + atomic64_inc(&s->backoff_count); +} + +static void cmh_stat_inc_cmq_eagain(void) +{ + struct cmh_tm_stats *s = cmh_debugfs_tm_stats(); + + if (s) + atomic64_inc(&s->cmq_eagain_count); +} + +static void cmh_stat_record_cmq_post(u32 depth) +{ + struct cmh_tm_stats *s = cmh_debugfs_tm_stats(); + + if (s) { + atomic64_inc(&s->cmq_posts); + cmh_stat_update_max(&s->cmq_depth_max, (s64)depth); + } +} + +static void cmh_stat_inc_async_timeout(void) +{ + struct cmh_tm_stats *s = cmh_debugfs_tm_stats(); + + if (s) + atomic64_inc(&s->async_timeout_count); +} + +/* + * Drop one reference on a command_msg; free when the last ref is dropped. + * Used by cmh_tm_submit_sync() to share msg ownership between the + * waiter (caller) and the TM subsystem (thread or cleanup drain). + */ +static void command_msg_put(struct command_msg *msg) +{ + if (refcount_dec_and_test(&msg->refs)) { + kfree(msg->vcq_data); + kfree(msg); + } +} + +/* + * Drop one reference on a transaction_obj; free when the last ref drops. + * Two references are held when the per-request timeout timer is armed: + * one for the TXQ owner (RH/cleanup), one for the timer callback. + * When no timer is armed, only the owner ref exists. + */ +static void txn_put(struct transaction_obj *txn) +{ + if (refcount_dec_and_test(&txn->refs)) + kfree(txn); +} + +/* + * Per-request async timeout callback (runs in softirq / timer context). + * + * This function ONLY marks the transaction state as TIMED_OUT via + * atomic cmpxchg and drops the timer reference. It does NOT fire + * the completion callback, does NOT touch DMA buffers, and does NOT + * write any MBX registers. + * + * Rationale: the HW may still be writing to DMA buffers at this + * point. Unmapping or freeing them here would be a use-after-free. + * The actual -ETIMEDOUT completion fires later, from process + * context, when the RH threaded IRQ pops the transaction after the + * HW finishes (or after MBX abort/drain on rmmod/suspend). + * + * MBX_COMMAND_ABORT is NOT issued here. It is issued by the RH + * watchdog abort-stall detector under rh_process_lock, which + * serialises it against RESTART/FLUSH recovery commands. Writing + * ABORT from timer softirq without the lock caused a race where + * concurrent timeouts clobbered an in-progress RESTART, wedging + * the mailbox. + * + * Context: softirq (timer). Must not sleep. + */ +static void txn_timeout_fn(struct timer_list *t) +{ + struct transaction_obj *txn = timer_container_of(txn, t, timeout_timer); + int old; + + old = atomic_cmpxchg(&txn->state, TXN_INFLIGHT, TXN_TIMED_OUT); + if (old == TXN_INFLIGHT) { + dev_err_ratelimited(cmh_dev(), + "tm: async timeout vcq=%u..%u mbx=%u cmd_id=0x%08x\n", + txn->first_vcq_id, + txn->last_vcq_id, txn->mailbox_idx, + txn->command_id); + cmh_stat_inc_async_timeout(); + } + + txn_put(txn); /* drop timer ref */ +} + +/** + * cmh_txn_finish() - Complete a popped transaction with FSM + timer cleanup + * @txn: Transaction object to complete + * @error: Error code from HW (0 on success) + * + * Three cases: + * 1. Normal: state INFLIGHT -> COMPLETE. Fire callback with HW error. + * 2. Timed out: state already TXN_TIMED_OUT (timer marked it). + * Fire callback with -ETIMEDOUT. DMA is now safe because the + * HW has finished and HEAD has advanced past this VCQ. + * 3. Force-cancel (drain/quiesce): handled by caller, not here. + */ +void cmh_txn_finish(struct transaction_obj *txn, int error) +{ + int old; + + old = atomic_cmpxchg(&txn->state, TXN_INFLIGHT, TXN_COMPLETE); + + /* Dequeue the timer if still pending; drop timer ref if we did */ + if (timer_delete(&txn->timeout_timer)) + txn_put(txn); + + if (old == TXN_INFLIGHT) { + /* HW completion (may carry error) */ + if (txn->complete) + txn->complete(txn->completion_data, error); + } else if (old == TXN_TIMED_OUT) { + /* Timer won earlier; now HW is done -- deliver -ETIMEDOUT */ + if (txn->complete) + txn->complete(txn->completion_data, -ETIMEDOUT); + } + + txn_put(txn); /* drop owner ref */ +} + +/* Mailbox Slot Addressing */ + +/* + * Return a kernel-virtual pointer to the VCQ slot for the given vcqid. + * Mirrors CMH eSW's mbx_queue_addr() but uses the kernel virt_addr. + */ +static void *mbx_slot_ptr(struct cmh_mbx_config *mbx, u32 vcqid) +{ + u32 slot_mask = (1U << mbx->slots_log2) - 1U; + u32 slot_offset = (vcqid & slot_mask) << mbx->stride_log2; + + return (u8 *)mbx->virt_addr + slot_offset; +} + +/* + * Return the number of free slots in a mailbox queue. + */ +static u32 mbx_free_slots(struct cmh_mbx_config *mbx) +{ + u32 head = cmh_reg_read32(mbx->reg_base, R_MBX_QUEUE_HEAD); + u32 tail = cmh_reg_read32(mbx->reg_base, R_MBX_QUEUE_TAIL); + u32 size = 1U << mbx->slots_log2; + + return size - (u32)(tail - head); +} + +/** + * cmh_tm_max_cmds_per_vcq() - Return max commands per VCQ slot + * + * Scans all mailbox configurations and returns the minimum number of + * VCQ command entries that fit in a single slot, clamped to the + * MIN_VCQ_CMDS..MAX_VCQ_CMDS range. + * + * Return: Maximum usable VCQ command count per slot. + */ +u32 cmh_tm_max_cmds_per_vcq(void) +{ + u32 i, min_cmds = MAX_VCQ_CMDS; + + for (i = 0; i < tm.cfg->mbx_count; i++) { + u32 stride = 1U << tm.cfg->mailboxes[i].stride_log2; + u32 cmds = stride / (u32)sizeof(struct vcq_cmd); + + if (cmds < min_cmds) + min_cmds = cmds; + } + + if (min_cmds < MIN_VCQ_CMDS) + min_cmds = MIN_VCQ_CMDS; + + return min_cmds; +} + +/** + * cmh_tm_mbx_count() - Return the number of configured mailboxes + * + * Return: Number of mailboxes in the current configuration. + */ +u32 cmh_tm_mbx_count(void) +{ + return tm.cfg->mbx_count; +} + +/* Core-to-MBX Affinity -- Config-Driven Multi-Instance Support */ + +/* + * Per-core-type configuration table. Each entry holds one or more + * (core_id, mbx_idx) instances. Defaults: single instance per core + * type with the standard CORE_ID_* and MBX auto-assigned on first use + * (mbx_idx = -1). Module params can override for explicit assignment + * and multi-instance support. + * + * Round-robin across instances for each new crypto operation. + */ + +struct core_instance_info { + u32 core_id; /* VCQ dispatch core_id */ + /* + * Assigned MBX index, or -1 (sentinel) for auto-assign on first + * use. Uses atomic_t for a lockless once-only latch: the first + * caller does atomic_cmpxchg(&mbx_idx, -1, new_mbx); all later + * callers see the winning value via atomic_read(). + */ + atomic_t mbx_idx; +}; + +struct core_type_info { + u32 num_instances; + struct core_instance_info instances[CMH_MAX_CORE_INSTANCES]; + atomic_t next_instance; /* round-robin counter */ +}; + +static struct core_type_info core_types[CMH_NUM_CORE_TYPES] = { + [CMH_CORE_HC] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_HC, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_AES] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_AES, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_SM4] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_SM4, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_SM3] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_SM3, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_CCP] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_CCP, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_PKE] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_PKE, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_QSE] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_QSE, + .mbx_idx = ATOMIC_INIT(-1) } } }, + [CMH_CORE_HCQ] = { .num_instances = 1, + .instances = { { .core_id = CORE_ID_HCQ, + .mbx_idx = ATOMIC_INIT(-1) } } }, +}; + +/* Round-robin counter for auto-assigning MBXes to core instances */ +static atomic_t affinity_next_mbx = ATOMIC_INIT(0); + +/** + * cmh_tm_affinity_reset() - Reset core-to-MBX affinity state + * + * Clears all auto-assigned MBX bindings and resets round-robin + * counters for both the global MBX allocator and per-core-type + * instance selectors. + */ +void cmh_tm_affinity_reset(void) +{ + u32 i, j; + + atomic_set(&affinity_next_mbx, 0); + + /* Reset multi-instance table */ + for (i = 0; i < CMH_NUM_CORE_TYPES; i++) { + struct core_type_info *ct = &core_types[i]; + + atomic_set(&ct->next_instance, 0); + for (j = 0; j < ct->num_instances; j++) + atomic_set(&ct->instances[j].mbx_idx, -1); + } +} + +/** + * cmh_core_default_id() - Return default core_id for a core type + * @type: Core type selector + * + * Returns the first-instance core_id for @type without advancing the + * round-robin counter. Used by callers pinned to a fixed MBX (e.g. + * mgmt ioctls on MGMT_MBX) that only need the VCQ core_id field. + * + * Return: VCQ core_id value for the default instance of @type. + */ +u32 cmh_core_default_id(enum cmh_core_type type) +{ + if (WARN_ON_ONCE(type >= CMH_NUM_CORE_TYPES)) + return 0; + + return core_types[type].instances[0].core_id; +} + +/** + * cmh_core_select_instance() - Select a core instance via round-robin + * @type: Core type selector + * + * Round-robin across configured instances, each permanently pinned to + * its MBX (auto-assigned on first use if mbx_idx was -1). + * + * Uses atomic_inc_return (pre-increment), so the very first call for a + * given type returns instance[1 % N]. Over the lifetime of the module + * the distribution is perfectly balanced; the off-by-one only affects + * the first cycle. + * + * The (u32) cast before the modulo ensures correct behaviour across + * the INT_MAX -> INT_MIN wraparound of atomic_t: (u32)INT_MIN = + * 0x80000000, and 0x80000000 % N still yields a valid index. + * + * Return: A core_dispatch with (core_id, mbx_idx) for the selected + * instance. + */ +struct core_dispatch cmh_core_select_instance(enum cmh_core_type type) +{ + struct core_type_info *ct; + struct core_instance_info *inst; + struct core_dispatch d; + u32 idx, count; + s32 mbx, new_mbx, old; + + if (WARN_ON_ONCE(type >= CMH_NUM_CORE_TYPES)) + return (struct core_dispatch){ .core_id = 0, .mbx_idx = -1 }; + + ct = &core_types[type]; + idx = (u32)atomic_inc_return(&ct->next_instance) % ct->num_instances; + inst = &ct->instances[idx]; + + d.core_id = inst->core_id; + + mbx = atomic_read(&inst->mbx_idx); + if (mbx >= 0) { + d.mbx_idx = mbx; + return d; + } + + /* Auto-assign on first use */ + count = tm.cfg->mbx_count; + new_mbx = (s32)((u32)atomic_inc_return(&affinity_next_mbx) % count); + old = atomic_cmpxchg(&inst->mbx_idx, -1, new_mbx); + + if (old >= 0) { + d.mbx_idx = old; + } else { + d.mbx_idx = new_mbx; + dev_info(cmh_dev(), + "tm: core 0x%02x -> mbx %d (auto)\n", + inst->core_id, new_mbx); + } + + return d; +} + +/** + * cmh_core_num_instances() - Return instance count for a core type + * @type: Core type selector + * + * Return: Number of configured instances for @type. + */ +u32 cmh_core_num_instances(enum cmh_core_type type) +{ + if (WARN_ON_ONCE(type >= CMH_NUM_CORE_TYPES)) + return 1; + + return core_types[type].num_instances; +} + +/** + * cmh_core_get_instance() - Get dispatch info for a specific instance + * @type: Core type selector + * @idx: Instance index within @type + * + * Returns (core_id, mbx_idx) for a specific instance by index, + * without advancing the round-robin counter. Triggers MBX auto-assign + * on first use if the instance has no MBX yet. + * + * Return: A core_dispatch with (core_id, mbx_idx) for instance @idx. + */ +struct core_dispatch cmh_core_get_instance(enum cmh_core_type type, u32 idx) +{ + struct core_type_info *ct; + struct core_instance_info *inst; + struct core_dispatch d; + u32 count; + s32 mbx, new_mbx, old; + + if (WARN_ON_ONCE(type >= CMH_NUM_CORE_TYPES)) + return (struct core_dispatch){ .core_id = 0, .mbx_idx = -1 }; + + ct = &core_types[type]; + if (WARN_ON_ONCE(idx >= ct->num_instances)) + return (struct core_dispatch){ .core_id = 0, .mbx_idx = -1 }; + + inst = &ct->instances[idx]; + d.core_id = inst->core_id; + + mbx = atomic_read(&inst->mbx_idx); + if (mbx >= 0) { + d.mbx_idx = mbx; + return d; + } + + /* Auto-assign on first use */ + count = tm.cfg->mbx_count; + new_mbx = (s32)((u32)atomic_inc_return(&affinity_next_mbx) % count); + old = atomic_cmpxchg(&inst->mbx_idx, -1, new_mbx); + + if (old >= 0) { + d.mbx_idx = old; + } else { + d.mbx_idx = new_mbx; + dev_info(cmh_dev(), + "tm: core 0x%02x -> mbx %d (auto)\n", + inst->core_id, new_mbx); + } + + return d; +} + +/** + * cmh_tm_txq_completion_notify() - Wake TM thread after RH completion + * + * Wakes the TM thread after the Response Handler completes a + * transaction. This unblocks the TM if it is waiting for a free MBX + * slot. The generation counter bump ensures the wait_event predicate + * evaluates to true on the next check. + */ +void cmh_tm_txq_completion_notify(void) +{ + atomic_inc(&txq_completion_gen); + wake_up_interruptible(&tm.cmq_waitq); +} + +/* Mailbox Selection */ + +/* + * Select a mailbox with at least @slots_needed free slots (round-robin). + * Returns mailbox index, or -EAGAIN if no mailbox qualifies. + * + * Note: the free-slot check here is advisory -- actual slot availability + * is enforced by the ring arithmetic under dispatch_lock in submit_vcq(). + * A TOCTOU gap exists between this check and the subsequent slot write, + * but it is safe: the worst case is a spurious -EAGAIN / backoff, never + * a ring overcommit. + */ +static int select_mailbox(u32 slots_needed) +{ + u32 count = tm.cfg->mbx_count; + u32 start = tm.next_mbx; + u32 i; + + for (i = 0; i < count; i++) { + u32 idx = (start + i) % count; + + if (cmh_rh_mbx_is_wedged(idx)) + continue; + + if (mbx_free_slots(&tm.cfg->mailboxes[idx]) >= slots_needed) { + tm.next_mbx = (idx + 1) % count; + return (int)idx; + } + cmh_stat_inc_mbx_queue_full(idx); + } + + return -EAGAIN; +} + +/* + * Resolve the target mailbox for a command message. + * + * If the message has a pinned MBX and it has enough free slots, use it. + * Otherwise fall back to round-robin selection. Returns mailbox index, + * or -EAGAIN when no MBX has enough free slots or all are wedged. + */ +static int resolve_mbx(struct command_msg *msg) +{ + u32 slots = msg->num_vcqs > 0 ? msg->num_vcqs : 1; + + if (msg->target_mbx >= 0 && + (u32)msg->target_mbx < tm.cfg->mbx_count) { + if (cmh_rh_mbx_is_wedged((u32)msg->target_mbx)) + return -EAGAIN; + if (mbx_free_slots(&tm.cfg->mailboxes[msg->target_mbx]) >= + slots) + return msg->target_mbx; + return -EAGAIN; /* pinned MBX full, retry */ + } + + return select_mailbox(slots); +} + +/* VCQ Submission */ + +/* + * Write VCQ(s) into consecutive DMA slots and ring the doorbell. + * + * A command_msg may carry one or more VCQs (num_vcqs field). For a + * multi-VCQ message the flat vcq_data array contains N VCQs laid out + * contiguously, each starting with its own header whose cmds field + * gives that VCQ's entry count. All VCQs are written to consecutive + * MBX slots and tracked by a single transaction_obj. + * + * Returns 0 on success, negative errno on failure. + */ +static int submit_vcq(struct command_msg *msg, u32 mbx_idx) +{ + struct cmh_mbx_config *mbx = &tm.cfg->mailboxes[mbx_idx]; + struct cmh_mbx_txq *txq = &tm.txqs[mbx_idx]; + struct transaction_obj *txn; + const struct vcq_cmd *cmds = msg->vcq_data; + u32 num_vcqs = msg->num_vcqs > 0 ? msg->num_vcqs : 1; + u32 tail, stride_bytes, offset = 0; + unsigned long flags; + u32 v; + + mutex_lock(&txq->dispatch_lock); + + /* Read current tail (first VCQ ID) */ + tail = cmh_reg_read32(mbx->reg_base, R_MBX_QUEUE_TAIL); + stride_bytes = 1U << mbx->stride_log2; + + /* Allocate transaction tracking object */ + txn = kzalloc_obj(*txn, GFP_KERNEL); + if (!txn) { + mutex_unlock(&txq->dispatch_lock); + return -ENOMEM; + } + + /* Write each VCQ into a consecutive DMA slot */ + for (v = 0; v < num_vcqs; v++) { + u32 vcq_cmds, copy_size; + void *slot; + + /* + * For single-VCQ messages (backward compat) use the + * msg-level vcq_count. For multi-VCQ, parse the per-VCQ + * header to find each VCQ's command count. + */ + if (num_vcqs == 1) { + vcq_cmds = msg->vcq_count; + } else { + const struct vcq_hdr *hdr = + (const struct vcq_hdr *)&cmds[offset].hwc; + vcq_cmds = hdr->cmds; + } + + copy_size = vcq_cmds * sizeof(struct vcq_cmd); + if (copy_size > stride_bytes) { + dev_err(cmh_dev(), "tm: VCQ %u too large (%u bytes > stride %u)\n", + v, copy_size, stride_bytes); + mutex_unlock(&txq->dispatch_lock); + kfree(txn); + return -EMSGSIZE; + } + + if (vcq_cmds < MIN_VCQ_CMDS || vcq_cmds > MAX_VCQ_CMDS) { + dev_err(cmh_dev(), "tm: invalid vcq_count %u (range %u..%u)\n", + vcq_cmds, MIN_VCQ_CMDS, MAX_VCQ_CMDS); + mutex_unlock(&txq->dispatch_lock); + kfree(txn); + return -EINVAL; + } + + /* Copy pre-built VCQ into DMA slot */ + slot = mbx_slot_ptr(mbx, tail + v); + cmh_dma_write(slot, &cmds[offset], copy_size); + + /* Zero remaining slot bytes to avoid stale data */ + if (copy_size < stride_bytes) + cmh_dma_zero((u8 *)slot + copy_size, + stride_bytes - copy_size); + + offset += vcq_cmds; + } + + /* Ensure VCQ data is visible in memory before advancing tail */ + wmb(); + /* FPGA: confirm DRAM accepted writes before SIC doorbell (cross-slave) */ + cmh_dma_fence(mbx_slot_ptr(mbx, tail + num_vcqs - 1)); + + /* Fill in transaction spanning all VCQs */ + txn->first_vcq_id = tail; + txn->last_vcq_id = tail + num_vcqs - 1; + txn->mailbox_idx = mbx_idx; + txn->command_id = msg->command_id; + txn->error_code = 0; + txn->complete = msg->complete; + txn->completion_data = msg->completion_data; + atomic_set(&txn->state, TXN_INFLIGHT); + timer_setup(&txn->timeout_timer, txn_timeout_fn, 0); + INIT_LIST_HEAD(&txn->list); + + /* + * Set refcount: 2 if a per-txn timer will be armed (one ref for + * the TXQ owner that pops it, one for the timer callback), or 1 + * if no timer (sync paths, or async_timeout_ms == 0). + */ + if (msg->timeout_jiffies) + refcount_set(&txn->refs, 2); + else + refcount_set(&txn->refs, 1); + + /* Enqueue transaction under spinlock */ + spin_lock_irqsave(&txq->lock, flags); + list_add_tail(&txn->list, &txq->head); + txq->depth++; + spin_unlock_irqrestore(&txq->lock, flags); + + /* Ring doorbell: advance tail by number of VCQs submitted */ + cmh_reg_write32(tail + num_vcqs, mbx->reg_base, R_MBX_QUEUE_TAIL); + + /* Arm per-request timeout after doorbell (async only) */ + if (msg->timeout_jiffies) + mod_timer(&txn->timeout_timer, + jiffies + msg->timeout_jiffies); + + mutex_unlock(&txq->dispatch_lock); + + cmh_stat_record_vcq_submit(mbx_idx, num_vcqs, txq->depth); + + dev_dbg(cmh_dev(), "tm: submitted %u vcq(s) id=%u..%u to mbx[%u] tail_now=%u\n", + num_vcqs, tail, tail + num_vcqs - 1, mbx_idx, + tail + num_vcqs); + + return 0; +} + +/* TM Thread */ + +static int cmh_tm_thread(void *data) +{ + struct command_msg *msg; + unsigned long flags; + int mbx_idx, ret; + + dev_info(cmh_dev(), "tm: thread started\n"); + + while (!kthread_should_stop()) { + /* Wait for work or stop signal */ + wait_event_interruptible(tm.cmq_waitq, + !list_empty(&tm.cmq) || kthread_should_stop()); + + if (kthread_should_stop()) + break; + + /* Dequeue one command message */ + spin_lock_irqsave(&tm.cmq_lock, flags); + if (list_empty(&tm.cmq)) { + spin_unlock_irqrestore(&tm.cmq_lock, flags); + continue; + } + msg = list_first_entry(&tm.cmq, struct command_msg, list); + list_del_init(&msg->list); + cmq_depth--; + + /* + * Promote one backlogged request into the CMQ now that + * there is room. Notify the crypto consumer with + * -EINPROGRESS so it knows the request has left backlog. + */ + if (!list_empty(&tm.backlog)) { + struct command_msg *bl; + + bl = list_first_entry(&tm.backlog, + struct command_msg, list); + list_move_tail(&bl->list, &tm.cmq); + tm.backlog_depth--; + cmq_depth++; + cmh_stat_record_cmq_post(cmq_depth); + /* + * Signal -EINPROGRESS while still under cmq_lock + * so the consumer sees it before the final + * completion. The callback must be IRQ-safe + * (required by the async contract anyway). + */ + if (bl->complete) + bl->complete(bl->completion_data, + -EINPROGRESS); + } + + spin_unlock_irqrestore(&tm.cmq_lock, flags); + + /* Select a mailbox: pinned or round-robin */ + mbx_idx = resolve_mbx(msg); + + if (mbx_idx < 0) { + /* + * Queue full -- re-enqueue at front and wait. + * + * Sleep on cmq_waitq with a short timeout. The RH + * calls cmh_tm_txq_completion_notify() after each + * completed transaction, which bumps the generation + * counter and wakes us immediately. The timeout is + * a safety net for missed wakeups. + */ + int gen = atomic_read(&txq_completion_gen); + unsigned long tmo; + + spin_lock_irqsave(&tm.cmq_lock, flags); + list_add(&msg->list, &tm.cmq); + cmq_depth++; + spin_unlock_irqrestore(&tm.cmq_lock, flags); + + tmo = usecs_to_jiffies(CMH_TM_BACKOFF_MAX_US); + wait_event_interruptible_timeout(tm.cmq_waitq, + kthread_should_stop() || + atomic_read(&txq_completion_gen) != gen, + tmo ?: 1); + cmh_stat_inc_tm_backoff(); + continue; + } + + /* Submit VCQ to selected mailbox */ + WRITE_ONCE(msg->actual_mbx, mbx_idx); + ret = submit_vcq(msg, mbx_idx); + if (ret && msg->complete) + msg->complete(msg->completion_data, ret); + command_msg_put(msg); + } + + dev_info(cmh_dev(), "tm: thread stopped\n"); + return 0; +} + +/* Public Interface */ + +/** + * cmh_tm_init() - Initialize the Transaction Manager subsystem + * @cfg: Hardware configuration describing mailboxes and core types + * + * Allocates per-mailbox transaction queues, applies core-type + * configuration, and starts the TM kthread. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_tm_init(struct cmh_config *cfg) +{ + u32 i, j; + + if (cmq_max_depth == 0) { + dev_warn(cmh_dev(), + "tm: cmq_max_depth=0 invalid, clamping to 1\n"); + cmq_max_depth = 1; + } + + tm.cfg = cfg; + tm.next_mbx = 0; + cmq_depth = 0; + + cmh_tm_affinity_reset(); + + /* Apply per-core-type config from DT child nodes */ + for (i = 0; i < CMH_NUM_CORE_TYPES; i++) { + struct cmh_core_type_cfg *src = &cfg->core_types[i]; + struct core_type_info *ct = &core_types[i]; + + ct->num_instances = src->num_instances; + for (j = 0; j < src->num_instances; j++) { + ct->instances[j].core_id = src->core_ids[j]; + if (src->mbx[j] >= 0) + atomic_set(&ct->instances[j].mbx_idx, + src->mbx[j]); + } + } + + /* Initialize CMQ and backlog */ + INIT_LIST_HEAD(&tm.cmq); + INIT_LIST_HEAD(&tm.backlog); + tm.backlog_depth = 0; + spin_lock_init(&tm.cmq_lock); + init_waitqueue_head(&tm.cmq_waitq); + + /* Allocate per-mailbox transaction queues */ + tm.txqs = kcalloc(cfg->mbx_count, sizeof(*tm.txqs), GFP_KERNEL); + if (!tm.txqs) + return -ENOMEM; + + for (i = 0; i < cfg->mbx_count; i++) { + INIT_LIST_HEAD(&tm.txqs[i].head); + spin_lock_init(&tm.txqs[i].lock); + mutex_init(&tm.txqs[i].dispatch_lock); + tm.txqs[i].depth = 0; + } + + /* Start TM thread */ + tm.thread = kthread_run(cmh_tm_thread, NULL, "cmh_tm"); + if (IS_ERR(tm.thread)) { + int ret = PTR_ERR(tm.thread); + + dev_err(cmh_dev(), "tm: failed to start thread (rc=%d)\n", ret); + tm.thread = NULL; + kfree(tm.txqs); + tm.txqs = NULL; + return ret; + } + + WRITE_ONCE(tm.running, true); + dev_info(cmh_dev(), + "tm: initialized (%u mailboxes, cmq_depth=%u backlog=%u)\n", + cfg->mbx_count, cmq_max_depth, backlog_max_depth); + + return 0; +} + +/* + * cmh_tm_stop_and_drain_cmq() - Stop TM thread and drain CMQ/backlog + * + * Shared preamble for cmh_tm_cleanup() and cmh_tm_quiesce(): stops the + * kthread, marks the TM as not running, then splices the CMQ and backlog + * to local lists and cancels every pending command_msg outside the lock. + */ +static void cmh_tm_stop_and_drain_cmq(void) +{ + struct command_msg *msg, *tmp_msg; + unsigned long flags; + LIST_HEAD(cmq_drain); + LIST_HEAD(backlog_drain); + + if (tm.thread) { + kthread_stop(tm.thread); + tm.thread = NULL; + } + WRITE_ONCE(tm.running, false); + + spin_lock_irqsave(&tm.cmq_lock, flags); + list_splice_init(&tm.cmq, &cmq_drain); + cmq_depth = 0; + list_splice_init(&tm.backlog, &backlog_drain); + tm.backlog_depth = 0; + spin_unlock_irqrestore(&tm.cmq_lock, flags); + + list_for_each_entry_safe(msg, tmp_msg, &cmq_drain, list) { + list_del(&msg->list); + if (msg->complete) + msg->complete(msg->completion_data, -ECANCELED); + command_msg_put(msg); + } + list_for_each_entry_safe(msg, tmp_msg, &backlog_drain, list) { + list_del(&msg->list); + if (msg->complete) + msg->complete(msg->completion_data, -ECANCELED); + command_msg_put(msg); + } +} + +/** + * cmh_tm_cleanup() - Tear down the Transaction Manager subsystem + * + * Stops the TM kthread, drains the CMQ, backlog, and all per-mailbox + * transaction queues, notifying waiters with -ECANCELED or -ETIMEDOUT. + * Frees all TM-owned resources. + */ +void cmh_tm_cleanup(void) +{ + struct transaction_obj *txn, *tmp_txn; + unsigned long flags; + u32 i; + + cmh_tm_stop_and_drain_cmq(); + + /* Drain per-mailbox transaction queues */ + if (tm.txqs) { + for (i = 0; i < tm.cfg->mbx_count; i++) { + LIST_HEAD(drain); + int old; + + spin_lock_irqsave(&tm.txqs[i].lock, flags); + list_splice_init(&tm.txqs[i].head, &drain); + tm.txqs[i].depth = 0; + spin_unlock_irqrestore(&tm.txqs[i].lock, flags); + + list_for_each_entry_safe(txn, tmp_txn, &drain, list) { + list_del(&txn->list); + + if (timer_delete_sync(&txn->timeout_timer)) + txn_put(txn); + + old = atomic_cmpxchg(&txn->state, + TXN_INFLIGHT, + TXN_COMPLETE); + if (txn->complete) { + if (old == TXN_INFLIGHT) + txn->complete(txn->completion_data, + -ECANCELED); + else if (old == TXN_TIMED_OUT) + txn->complete(txn->completion_data, + -ETIMEDOUT); + } + + txn_put(txn); + } + } + kfree(tm.txqs); + tm.txqs = NULL; + } + + dev_info(cmh_dev(), "tm: cleaned up\n"); +} + +/* + * Default drain timeout for suspend/quiesce (milliseconds). + * Covers all symmetric + PKE operations. PQC callers (SLH-DSA sign + * at up to 120 s) should complete before system suspend is requested. + */ +static unsigned int drain_timeout_ms = 10000; + +/** + * cmh_tm_quiesce() - Quiesce the TM for suspend or shutdown + * + * Stops the TM kthread, drains the CMQ and backlog, then waits up to + * drain_timeout_ms for in-flight transactions to complete via the + * Response Handler. Any remaining transactions after the deadline + * are force-cancelled. + */ +void cmh_tm_quiesce(void) +{ + struct transaction_obj *txn, *tmp_txn; + unsigned long deadline; + unsigned long flags; + u32 i; + bool drained = true; + + cmh_tm_stop_and_drain_cmq(); + + /* Wait for in-flight TXQ transactions to complete via RH */ + if (!tm.txqs) + goto out; + + deadline = jiffies + msecs_to_jiffies(drain_timeout_ms); + do { + drained = true; + for (i = 0; i < tm.cfg->mbx_count; i++) { + if (READ_ONCE(tm.txqs[i].depth)) { + drained = false; + break; + } + } + if (drained) + break; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + + if (!drained) { + dev_warn(cmh_dev(), + "tm: quiesce drain timeout (%u ms), cancelling remaining transactions\n", + drain_timeout_ms); + for (i = 0; i < tm.cfg->mbx_count; i++) { + LIST_HEAD(drain); + int old; + + spin_lock_irqsave(&tm.txqs[i].lock, flags); + list_splice_init(&tm.txqs[i].head, &drain); + tm.txqs[i].depth = 0; + spin_unlock_irqrestore(&tm.txqs[i].lock, flags); + + list_for_each_entry_safe(txn, tmp_txn, &drain, list) { + list_del(&txn->list); + + if (timer_delete_sync(&txn->timeout_timer)) + txn_put(txn); + + old = atomic_cmpxchg(&txn->state, + TXN_INFLIGHT, + TXN_COMPLETE); + if (txn->complete) { + if (old == TXN_INFLIGHT) + txn->complete(txn->completion_data, + -ECANCELED); + else if (old == TXN_TIMED_OUT) + txn->complete(txn->completion_data, + -ETIMEDOUT); + } + + txn_put(txn); + } + } + } + +out: + dev_info(cmh_dev(), "tm: quiesced%s\n", + drained ? "" : " (forced)"); +} + +/** + * cmh_tm_resume() - Resume the TM after suspend + * + * Restarts the TM kthread after a prior cmh_tm_quiesce(). + * + * Return: 0 on success, negative errno if kthread creation fails. + */ +int cmh_tm_resume(void) +{ + if (tm.thread || !tm.cfg) + return 0; + + tm.thread = kthread_run(cmh_tm_thread, NULL, "cmh_tm"); + if (IS_ERR(tm.thread)) { + int ret = PTR_ERR(tm.thread); + + dev_err(cmh_dev(), "tm: resume kthread_run failed (%d)\n", + ret); + tm.thread = NULL; + return ret; + } + WRITE_ONCE(tm.running, true); + dev_info(cmh_dev(), "tm: resumed\n"); + return 0; +} + +/** + * cmh_tm_try_cancel_command() - Cancel a queued command message + * @msg: Command message to cancel + * + * Attempts to remove @msg from the CMQ before the TM thread dequeues + * it. Must be called while @msg is still valid (before the caller's + * stack frame that owns it is freed). + * + * Return: true if @msg was removed, false if already consumed by TM. + */ +bool cmh_tm_try_cancel_command(struct command_msg *msg) +{ + unsigned long flags; + bool cancelled = false; + + spin_lock_irqsave(&tm.cmq_lock, flags); + if (!list_empty(&msg->list)) { + list_del_init(&msg->list); + cmq_depth--; + cancelled = true; + } + spin_unlock_irqrestore(&tm.cmq_lock, flags); + + return cancelled; +} + +/** + * cmh_tm_post_command() - Post a command message to the CMQ + * @msg: Pre-built command message to enqueue + * + * Enqueues @msg on the Command Message Queue and wakes the TM thread. + * If the CMQ is full, the message may be placed on the backlog queue + * (returning -EBUSY) if @msg->backlog_ok is set, or rejected with + * -EAGAIN. + * + * Return: 0 on success, -EBUSY if backlogged, -EAGAIN if full, + * -ENODEV if TM is not running. + */ +int cmh_tm_post_command(struct command_msg *msg) +{ + unsigned long flags; + + if (!READ_ONCE(tm.running)) + return -ENODEV; + + spin_lock_irqsave(&tm.cmq_lock, flags); + if (cmq_depth >= cmq_max_depth) { + if (msg->backlog_ok && + tm.backlog_depth < backlog_max_depth) { + list_add_tail(&msg->list, &tm.backlog); + tm.backlog_depth++; + spin_unlock_irqrestore(&tm.cmq_lock, flags); + return -EBUSY; + } + spin_unlock_irqrestore(&tm.cmq_lock, flags); + cmh_stat_inc_cmq_eagain(); + return -EAGAIN; + } + INIT_LIST_HEAD(&msg->list); + list_add_tail(&msg->list, &tm.cmq); + cmq_depth++; + cmh_stat_record_cmq_post(cmq_depth); + spin_unlock_irqrestore(&tm.cmq_lock, flags); + + wake_up_interruptible(&tm.cmq_waitq); + return 0; +} + +/* Synchronous Submit (refcounted completion + timeout) */ + +/* + * Heap-allocated sync context with refcounting. + * + * The completion callback may fire after the waiter has timed out and + * returned (e.g. during cmh_tm_cleanup on rmmod). If the struct lived + * on the waiter's stack, the callback would touch freed memory -- + * triggering a "BUG: spinlock bad magic" on the completion's spinlock. + * + * Two references are held: one by the waiter, one by the callback. + * Whichever runs last frees the struct. + */ +struct cmh_sync_ctx { + struct completion done; + int error; + refcount_t refs; /* 2: waiter + callback */ + + /* Optional orphan cleanup -- called when the last ref drops after + * the waiter abandoned an in-flight VCQ (noabort path). Lets the + * caller defer DMA-buffer cleanup until the eSW finishes writing. + */ + void (*orphan_cb)(void *data); + void *orphan_data; +}; + +static void cmh_sync_ctx_put(struct cmh_sync_ctx *ctx) +{ + if (refcount_dec_and_test(&ctx->refs)) { + if (ctx->orphan_cb) + ctx->orphan_cb(ctx->orphan_data); + kfree(ctx); + } +} + +static void cmh_sync_complete(void *data, int error) +{ + struct cmh_sync_ctx *ctx = data; + + ctx->error = error; + complete(&ctx->done); + cmh_sync_ctx_put(ctx); +} + +/* + * Default VCQ completion timeout (milliseconds), tunable via debugfs + * config/vcq_timeout_ms. Only affects the default timeout used by cmh_tm_submit_sync() + * and cmh_tm_submit_sync_mbx(); callers that pass an explicit timeout_hz + * (e.g. RSA keygen) are not affected. + */ +static unsigned int vcq_timeout_ms = 2000; + +/* + * Extended timeout for slow crypto operations: RSA keygen, PQC + * keygen/sign/verify. Tunable via debugfs config/slow_op_timeout_ms. + */ +static unsigned int slow_op_timeout_ms = 300000; + +/** + * cmh_tm_submit_sync_tmo() - Synchronous VCQ submit with timeout + * @vcq_cmds: Array of pre-built VCQ command entries + * @vcq_count: Total number of entries in @vcq_cmds + * @num_vcqs: Number of VCQs packed in @vcq_cmds + * @target_mbx: Pinned mailbox index, or -1 for round-robin + * @timeout_hz: Completion timeout in jiffies + * + * Posts a VCQ command to the TM, waits for completion up to + * @timeout_hz. On timeout, issues MBX_COMMAND_ABORT if the VCQ is + * already in-flight. Must be called from process context. + * + * Return: 0 on success, -ETIMEDOUT, or negative errno. + */ +int cmh_tm_submit_sync_tmo(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, s32 target_mbx, + unsigned long timeout_hz) +{ + struct cmh_sync_ctx *sync; + struct command_msg *msg; + unsigned long left; + int ret; + + /* + * This path sleeps (GFP_KERNEL allocations + wait_for_completion) + * and is not safe from atomic / non-sleepable contexts. All + * current callers run in process context (crypto API userspace or + * ioctl), so this is never violated today. Catch it loudly if + * a future caller gets this wrong. + */ + WARN_ON_ONCE(!in_task()); + + sync = kzalloc_obj(*sync, GFP_KERNEL); + if (!sync) + return -ENOMEM; + + msg = kzalloc_obj(*msg, GFP_KERNEL); + if (!msg) { + kfree(sync); + return -ENOMEM; + } + + init_completion(&sync->done); + sync->error = 0; + refcount_set(&sync->refs, 2); /* waiter + callback */ + + /* + * Heap-copy the caller's VCQ array so the msg owns its data. + * This decouples VCQ lifetime from the caller's stack frame, + * which matters when the TM thread backs off (resolve_mbx + * returns -1) and re-enqueues the msg after the caller's + * wait_for_completion_timeout expires. + */ + msg->vcq_data = kmemdup(vcq_cmds, vcq_count * sizeof(*vcq_cmds), + GFP_KERNEL); + if (!msg->vcq_data) { + kfree(msg); + kfree(sync); + return -ENOMEM; + } + + INIT_LIST_HEAD(&msg->list); + if (WARN_ON_ONCE(vcq_count < MIN_VCQ_CMDS)) { + ret = -EINVAL; + goto err_free; + } + msg->command_id = vcq_cmds[1].id; /* first real command's ID */ + msg->vcq_count = vcq_count; + msg->num_vcqs = num_vcqs; + msg->target_mbx = target_mbx; + msg->actual_mbx = -1; + msg->complete = cmh_sync_complete; + msg->completion_data = sync; + refcount_set(&msg->refs, 2); /* waiter + TM subsystem */ + + ret = cmh_tm_post_command(msg); + if (ret) { +err_free: + kfree(msg->vcq_data); + kfree(msg); + kfree(sync); /* callback will never fire */ + return ret; + } + + dev_dbg(cmh_dev(), "tm: submit_sync posted cmd 0x%08x, waiting...\n", + msg->command_id); + + left = wait_for_completion_timeout(&sync->done, timeout_hz); + if (!left) { + dev_err(cmh_dev(), + "tm: submit_sync timeout (%lums) cmd=0x%08x\n", + timeout_hz * 1000 / HZ, msg->command_id); + if (cmh_tm_try_cancel_command(msg)) { + /* + * Msg was still queued -- TM never saw it. + * Drop the callback ref (no txn will fire it) + * and free msg directly (sole owner). + */ + cmh_sync_ctx_put(sync); /* no txn -> drop cb ref */ + cmh_sync_ctx_put(sync); /* drop waiter ref */ + command_msg_put(msg); /* matches refcount_set(2) */ + command_msg_put(msg); + } else { + /* + * TM has dequeued msg and the VCQ is in-flight. + * Issue MBX_COMMAND_ABORT to force-stop the VCQ; + * the RH will fire MBX_ERROR_IRQ, complete the + * transaction with -EIO, and issue RESTART. + * + * cmh_rh_abort_mbx() serialises the write under + * rh_process_lock, preventing clobber of a + * concurrent RESTART/FLUSH from the watchdog. + */ + s32 abrt_mbx = READ_ONCE(msg->actual_mbx); + + if (abrt_mbx >= 0 && + (u32)abrt_mbx < tm.cfg->mbx_count) { + dev_warn(cmh_dev(), + "tm: aborting mbx[%d] cmd=0x%08x\n", + abrt_mbx, msg->command_id); + cmh_rh_abort_mbx((u32)abrt_mbx); + } + + /* + * Wait for the RH completion (ABORT triggers + * MBX_ERROR_IRQ within microseconds). Fixed + * 5 s ceiling -- not configurable because if + * ABORT doesn't complete in this window the + * HW is wedged and more waiting won't help. + */ + left = wait_for_completion_timeout(&sync->done, + 5 * HZ); + if (!left) { + /* + * ABORT did not complete within 5 s -- HW + * is wedged. The eSW may still be writing + * to DMA buffers owned by the caller, so we + * cannot let the caller free them. Transfer + * ownership to the sync_ctx orphan mechanism; + * the RH callback (if it ever fires) will + * free via orphan_cb. If it never fires, the + * buffers leak -- acceptable for a wedged HW + * path that should never occur in practice. + */ + dev_err(cmh_dev(), + "tm: abort timeout (5s) cmd=0x%08x - DMA buffers orphaned\n", + msg->command_id); + } + cmh_sync_ctx_put(sync); /* drop waiter ref */ + command_msg_put(msg); /* drop waiter ref on msg */ + } + return -ETIMEDOUT; + } + + ret = sync->error; + cmh_sync_ctx_put(sync); /* drop waiter ref */ + command_msg_put(msg); /* drop waiter ref on msg */ + return ret; +} + +/** + * cmh_tm_submit_sync_mbx() - Synchronous VCQ submit on a target MBX + * @vcq_cmds: Array of pre-built VCQ command entries + * @vcq_count: Total number of entries in @vcq_cmds + * @num_vcqs: Number of VCQs packed in @vcq_cmds + * @target_mbx: Pinned mailbox index, or -1 for round-robin + * + * Convenience wrapper around cmh_tm_submit_sync_tmo() using the + * default vcq_timeout_ms module parameter. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_tm_submit_sync_mbx(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, s32 target_mbx) +{ + return cmh_tm_submit_sync_tmo(vcq_cmds, vcq_count, num_vcqs, + target_mbx, + msecs_to_jiffies(vcq_timeout_ms)); +} + +/** + * cmh_tm_async_timeout_jiffies() - Default async per-request timeout + * + * Return: Timeout in jiffies from the async_timeout_ms module param, + * or 0 if async timeouts are disabled. + */ +unsigned long cmh_tm_async_timeout_jiffies(void) +{ + return async_timeout_ms ? msecs_to_jiffies(async_timeout_ms) : 0; +} + +/** + * cmh_tm_slow_op_timeout_jiffies() - Timeout for slow crypto ops + * + * Returns the extended timeout used for RSA keygen, PQC keygen/sign, + * and similar long-running operations. + * + * Return: Timeout in jiffies from the slow_op_timeout_ms module param. + */ +unsigned long cmh_tm_slow_op_timeout_jiffies(void) +{ + return msecs_to_jiffies(slow_op_timeout_ms); +} + +/** + * cmh_tm_submit_async() - Asynchronous VCQ submission + * @vcq_cmds: Array of pre-built VCQ command entries + * @vcq_count: Total number of entries in @vcq_cmds + * @num_vcqs: Number of VCQs packed in @vcq_cmds + * @target_mbx: Pinned mailbox index, or -1 for round-robin + * @callback: Completion callback (see context note below) + * @callback_data: Opaque data passed to @callback + * @backlog_ok: Allow backlogging if CMQ is full + * @timeout_jiffies: Per-request timeout (0 = no timeout) + * + * Builds a command_msg, heap-copies the VCQ data, and posts it to the + * CMQ via cmh_tm_post_command(). + * + * Callback context guarantee: + * The @callback may be invoked from one of: + * - RH threaded IRQ handler (process context, BH disabled) + * - RH watchdog timer (softirq / timer context) + * - TM kthread if submit_vcq() fails post-dequeue + * - cmh_tm_cleanup()/cmh_tm_quiesce() during drain (process context) + * It is NEVER invoked from hardirq context. + * + * Because the watchdog path runs from timer softirq, callbacks + * MUST be safe in atomic/softirq context: no mutex, no GFP_KERNEL, + * no sleeping locks. crypto_request_complete() is safe (documented + * callable from any context). kfree_sensitive() and + * scatterwalk_map_and_copy() are also safe (non-sleeping). + * Callers must not assume thread affinity (callback may run on any CPU). + * + * Unlike the _sync variants, this function: + * - Does NOT allocate a cmh_sync_ctx or wait for completion + * - Uses GFP_ATOMIC for internal allocations because the crypto API + * may call ->encrypt/->decrypt/->hash_final from softirq context + * (e.g. network stack via IPsec/TLS); GFP_KERNEL would deadlock. + * + * The command_msg is single-owner (refcount 1) -- the TM subsystem + * owns it after post and frees it after dispatching to the HW. + * + * DMA buffer ownership: the caller transfers ownership to the callback + * on return of 0 or -EBUSY. On any other return, the caller must + * clean up DMA buffers itself -- the callback will never fire. + * + * Return: 0 on successful post, -EBUSY if backlogged, -ENOMEM, + * -EINVAL, -EAGAIN, or -ENODEV on failure. + */ +int cmh_tm_submit_async(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, s32 target_mbx, + cmh_completion_fn callback, void *callback_data, + bool backlog_ok, unsigned long timeout_jiffies) +{ + struct command_msg *msg; + int ret; + + msg = kzalloc_obj(*msg, GFP_ATOMIC); + if (!msg) + return -ENOMEM; + + msg->vcq_data = kmemdup(vcq_cmds, + array_size(vcq_count, sizeof(*vcq_cmds)), + GFP_ATOMIC); + if (!msg->vcq_data) { + kfree(msg); + return -ENOMEM; + } + + INIT_LIST_HEAD(&msg->list); + if (WARN_ON_ONCE(vcq_count < MIN_VCQ_CMDS)) { + kfree(msg->vcq_data); + kfree(msg); + return -EINVAL; + } + msg->command_id = vcq_cmds[1].id; + msg->vcq_count = vcq_count; + msg->num_vcqs = num_vcqs; + msg->target_mbx = target_mbx; + msg->actual_mbx = -1; + msg->complete = callback; + msg->completion_data = callback_data; + msg->backlog_ok = backlog_ok; + msg->timeout_jiffies = timeout_jiffies; + refcount_set(&msg->refs, 1); /* sole owner: TM subsystem */ + + ret = cmh_tm_post_command(msg); + if (ret && ret != -EBUSY) { + kfree(msg->vcq_data); + kfree(msg); + } + return ret; +} + +/** + * cmh_tm_submit_sync_noabort() - Sync submit without MBX abort on timeout + * @vcq_cmds: Array of pre-built VCQ command entries + * @vcq_count: Total number of entries in @vcq_cmds + * @num_vcqs: Number of VCQs packed in @vcq_cmds + * @timeout_hz: Completion timeout in jiffies + * @orphan_cb: Optional cleanup callback for abandoned DMA buffers + * @orphan_data: Opaque data passed to @orphan_cb + * + * On timeout, if the command was still queued it is cancelled and + * -EAGAIN is returned (caller may free all resources). If the VCQ is + * already in-flight, the waiter drops its refs and returns -EINPROGRESS + * -- the RH callback will fire when the eSW finishes the VCQ and free + * the sync_ctx / msg via the refcount mechanism. + * + * @orphan_cb is invoked when the last ref on the sync_ctx drops after + * the waiter abandoned an in-flight VCQ, allowing the caller to defer + * DMA-buffer cleanup until the eSW finishes writing. + * + * This prevents a short-timeout command (e.g. DRBG GENERATE from the + * hwrng kthread) from aborting the entire MBX and killing unrelated + * long-running operations (e.g. SLH-DSA sign at 120 s). + * + * Return: 0 on success, -EAGAIN if cancelled from queue, + * -EINPROGRESS if left in-flight, or negative errno. + */ +int cmh_tm_submit_sync_noabort(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, unsigned long timeout_hz, + void (*orphan_cb)(void *), void *orphan_data) +{ + struct cmh_sync_ctx *sync; + struct command_msg *msg; + unsigned long left; + int ret; + + WARN_ON_ONCE(!in_task()); + + sync = kzalloc_obj(*sync, GFP_KERNEL); + if (!sync) + return -ENOMEM; + + msg = kzalloc_obj(*msg, GFP_KERNEL); + if (!msg) { + kfree(sync); + return -ENOMEM; + } + + init_completion(&sync->done); + sync->error = 0; + refcount_set(&sync->refs, 2); + + INIT_LIST_HEAD(&msg->list); + if (WARN_ON_ONCE(vcq_count < MIN_VCQ_CMDS)) { + kfree(msg); + kfree(sync); + return -EINVAL; + } + msg->command_id = vcq_cmds[1].id; + msg->vcq_data = kmemdup(vcq_cmds, vcq_count * sizeof(*vcq_cmds), + GFP_KERNEL); + if (!msg->vcq_data) { + kfree(msg); + kfree(sync); + return -ENOMEM; + } + msg->vcq_count = vcq_count; + msg->num_vcqs = num_vcqs; + msg->target_mbx = -1; + msg->actual_mbx = -1; + msg->complete = cmh_sync_complete; + msg->completion_data = sync; + refcount_set(&msg->refs, 2); + + ret = cmh_tm_post_command(msg); + if (ret) { + kfree(msg->vcq_data); + kfree(msg); + kfree(sync); + return ret; + } + + left = wait_for_completion_timeout(&sync->done, timeout_hz); + if (!left) { + if (cmh_tm_try_cancel_command(msg)) { + /* Still queued -- TM never saw it, clean up fully */ + cmh_sync_ctx_put(sync); /* drop cb ref */ + cmh_sync_ctx_put(sync); /* drop waiter ref */ + command_msg_put(msg); /* matches refcount_set(2) */ + command_msg_put(msg); + return -EAGAIN; + } + + /* + * In-flight: skip ABORT. Transfer orphan cleanup + * ownership to sync_ctx -- the RH callback will + * eventually complete this VCQ, and when the last + * ref drops, orphan_cb frees any DMA buffers the + * eSW was still writing to. + */ + dev_dbg_ratelimited(cmh_dev(), + "tm: noabort timeout (%lums) cmd=0x%08x, leaving in-flight\n", + timeout_hz * 1000 / HZ, + msg->command_id); + sync->orphan_cb = orphan_cb; + sync->orphan_data = orphan_data; + cmh_sync_ctx_put(sync); + command_msg_put(msg); + return -EINPROGRESS; + } + + ret = sync->error; + cmh_sync_ctx_put(sync); + command_msg_put(msg); + return ret; +} + +/** + * cmh_tm_submit_sync() - Synchronous VCQ submit with default timeout + * @vcq_cmds: Array of pre-built VCQ command entries + * @vcq_count: Total number of entries in @vcq_cmds + * @num_vcqs: Number of VCQs packed in @vcq_cmds + * + * Convenience wrapper: submits via round-robin MBX selection with the + * default vcq_timeout_ms. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_tm_submit_sync(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs) +{ + return cmh_tm_submit_sync_mbx(vcq_cmds, vcq_count, num_vcqs, -1); +} + +#define MBX_FLUSH_TIMEOUT_MS 1000 +#define MBX_FLUSH_POLL_MIN_US 10 +#define MBX_FLUSH_POLL_MAX_US 50 + +/** + * cmh_tm_flush_mbx() - Issue MBX_COMMAND_FLUSH and wait for completion + * @mbx_idx: Mailbox index to flush + * + * Resets the eSW child mailbox state: clears the VCQ command queue, + * resets head/tail, and -- critically -- resets the child temp stack + * via mbx_hdr_init() (sets hdr->temp back to &cmds[MAX_VCQ_CMDS]). + * + * Why this is needed: + * KIC derivation commands that output to SYS_REF_TEMP allocate on the + * per-MBX child temp LIFO stack (mbx_alloc_temp, each costing + * ROUND_UP(len,4)+56 bytes). These allocations persist across VCQ + * completions because mbx_vcq_done() does NOT reset the temp stack. + * Without an explicit flush, sequential KIC-TEMP ioctls exhaust the + * ~960-byte temp area and subsequent derives fail with ENOMEM. + * + * What is NOT affected: + * KIC HW keys, datastore objects, DRBG state -- these survive the + * flush. Only the queue pointers and temp stack are reset. + * + * Concurrency: + * Acquires the per-MBX dispatch_lock mutex to serialise with VCQ + * dispatch in submit_vcq(). This prevents the flush from resetting + * head/tail while the TM kthread is writing a VCQ to a DMA slot on + * the same MBX. The eSW clears R_MBX_COMMAND to zero once the flush + * completes. + * + * Return: 0 on success, -EINVAL, -ENODEV, -EBUSY, or -ETIMEDOUT. + */ +int cmh_tm_flush_mbx(s32 mbx_idx) +{ + struct cmh_mbx_config *mbx; + struct cmh_mbx_txq *txq; + void __iomem *base; + u32 reg; + int ret; + + if (!tm.cfg || mbx_idx < 0 || (u32)mbx_idx >= tm.cfg->mbx_count) + return -EINVAL; + + mbx = &tm.cfg->mailboxes[mbx_idx]; + base = mbx->reg_base; + if (!base) + return -ENODEV; + + txq = &tm.txqs[mbx_idx]; + mutex_lock(&txq->dispatch_lock); + + /* Ensure no command is already pending */ + if (cmh_reg_read32(base, R_MBX_COMMAND) != 0) { + mutex_unlock(&txq->dispatch_lock); + return -EBUSY; + } + + cmh_reg_write32(MBX_COMMAND_FLUSH, base, R_MBX_COMMAND); + + /* Poll until eSW clears the command register */ + ret = read_poll_timeout(cmh_reg_read32, reg, reg == 0, + MBX_FLUSH_POLL_MIN_US, + MBX_FLUSH_TIMEOUT_MS * 1000, + true, base, R_MBX_COMMAND); + if (ret) + dev_err(cmh_dev(), "mbx %u flush timeout (cmd=0x%08x)\n", + mbx->instance, + cmh_reg_read32(base, R_MBX_COMMAND)); + + mutex_unlock(&txq->dispatch_lock); + return ret; +} + +/** + * cmh_vcq_pack_and_submit() - Pack payload into VCQs and submit sync + * @payload: Array of VCQ command entries (without headers) + * @count: Number of entries in @payload + * @packed: Caller-provided output buffer for packed VCQ data + * @max_packed: Size of @packed buffer in vcq_cmd entries + * @target_mbx: Pinned mailbox index, or -1 for round-robin + * + * Splits @payload into VCQ-sized chunks, prepends headers, and submits + * synchronously. + * + * Return: 0 on success, -EMSGSIZE if @packed is too small, or + * negative errno from submit. + */ +int cmh_vcq_pack_and_submit(const struct vcq_cmd *payload, u32 count, + struct vcq_cmd *packed, u32 max_packed, + s32 target_mbx) +{ + u32 max_per_vcq = cmh_tm_max_cmds_per_vcq(); + u32 max_payload_per = max_per_vcq - 1; + u32 num_vcqs = 0, total = 0, i = 0; + + while (i < count) { + u32 chunk = min_t(u32, count - i, max_payload_per); + u32 vcq_cmds = chunk + 1; + + if (total + vcq_cmds > max_packed) + return -EMSGSIZE; + + vcq_set_header(&packed[total], vcq_cmds); + memcpy(&packed[total + 1], &payload[i], + chunk * sizeof(struct vcq_cmd)); + + total += vcq_cmds; + i += chunk; + num_vcqs++; + } + + return cmh_tm_submit_sync_mbx(packed, total, num_vcqs, target_mbx); +} + +/** + * cmh_vcq_pack_and_submit_async() - Pack payload and submit async + * @payload: Array of VCQ command entries (without headers) + * @count: Number of entries in @payload + * @packed: Caller-provided output buffer for packed VCQ data + * @max_packed: Size of @packed buffer in vcq_cmd entries + * @target_mbx: Pinned mailbox index, or -1 for round-robin + * @callback: Completion callback + * @callback_data: Opaque data passed to @callback + * @backlog_ok: Allow backlogging if CMQ is full + * @timeout_jiffies: Per-request timeout (0 = no timeout) + * + * Asynchronous variant of cmh_vcq_pack_and_submit(). Splits @payload + * into VCQ-sized chunks, prepends headers, and submits via + * cmh_tm_submit_async(). + * + * Return: 0 on success, -EBUSY if backlogged, -EMSGSIZE if @packed + * is too small, or negative errno from submit. + */ +int cmh_vcq_pack_and_submit_async(const struct vcq_cmd *payload, u32 count, + struct vcq_cmd *packed, u32 max_packed, + s32 target_mbx, + cmh_completion_fn callback, + void *callback_data, + bool backlog_ok, + unsigned long timeout_jiffies) +{ + u32 max_per_vcq = cmh_tm_max_cmds_per_vcq(); + u32 max_payload_per = max_per_vcq - 1; + u32 num_vcqs = 0, total = 0, i = 0; + + while (i < count) { + u32 chunk = min_t(u32, count - i, max_payload_per); + u32 vcq_cmds = chunk + 1; + + if (total + vcq_cmds > max_packed) + return -EMSGSIZE; + + vcq_set_header(&packed[total], vcq_cmds); + memcpy(&packed[total + 1], &payload[i], + chunk * sizeof(struct vcq_cmd)); + + total += vcq_cmds; + i += chunk; + num_vcqs++; + } + + return cmh_tm_submit_async(packed, total, num_vcqs, target_mbx, + callback, callback_data, backlog_ok, + timeout_jiffies); +} + +/** + * cmh_tm_peek_transaction() - Peek at the head of a mailbox TXQ + * @mbx_idx: Mailbox index to inspect + * + * Returns a pointer to the oldest in-flight transaction without + * removing it from the queue. The caller must not free the returned + * object. + * + * Return: Pointer to the head transaction, or NULL if empty. + */ +struct transaction_obj *cmh_tm_peek_transaction(u32 mbx_idx) +{ + struct cmh_mbx_txq *txq; + struct transaction_obj *txn = NULL; + unsigned long flags; + + if (!tm.txqs || mbx_idx >= tm.cfg->mbx_count) + return NULL; + + txq = &tm.txqs[mbx_idx]; + + spin_lock_irqsave(&txq->lock, flags); + if (!list_empty(&txq->head)) + txn = list_first_entry(&txq->head, struct transaction_obj, + list); + spin_unlock_irqrestore(&txq->lock, flags); + + return txn; +} + +/** + * cmh_tm_pop_transaction() - Remove and return the head of a MBX TXQ + * @mbx_idx: Mailbox index to pop from + * + * Dequeues the oldest in-flight transaction from the per-mailbox + * transaction queue. The caller takes ownership and must eventually + * call cmh_txn_finish() or txn_put(). + * + * Return: Pointer to the dequeued transaction, or NULL if empty. + */ +struct transaction_obj *cmh_tm_pop_transaction(u32 mbx_idx) +{ + struct cmh_mbx_txq *txq; + struct transaction_obj *txn; + unsigned long flags; + + if (!tm.txqs || mbx_idx >= tm.cfg->mbx_count) + return NULL; + + txq = &tm.txqs[mbx_idx]; + + spin_lock_irqsave(&txq->lock, flags); + if (list_empty(&txq->head)) { + spin_unlock_irqrestore(&txq->lock, flags); + return NULL; + } + txn = list_first_entry(&txq->head, struct transaction_obj, list); + list_del_init(&txn->list); + txq->depth--; + spin_unlock_irqrestore(&txq->lock, flags); + + return txn; +} + +/* -- debugfs timeout accessors ----------------------------------------- */ + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +/** + * cmh_tm_timeout_async_ptr() - Return pointer to async_timeout_ms for debugfs + * + * Return: pointer to the static async_timeout_ms variable. + */ +unsigned int *cmh_tm_timeout_async_ptr(void) { return &async_timeout_ms; } + +/** + * cmh_tm_timeout_vcq_ptr() - Return pointer to vcq_timeout_ms for debugfs + * + * Return: pointer to the static vcq_timeout_ms variable. + */ +unsigned int *cmh_tm_timeout_vcq_ptr(void) { return &vcq_timeout_ms; } + +/** + * cmh_tm_timeout_slow_op_ptr() - Return pointer to slow_op_timeout_ms for debugfs + * + * Return: pointer to the static slow_op_timeout_ms variable. + */ +unsigned int *cmh_tm_timeout_slow_op_ptr(void) { return &slow_op_timeout_ms; } + +/** + * cmh_tm_timeout_drain_ptr() - Return pointer to drain_timeout_ms for debugfs + * + * Return: pointer to the static drain_timeout_ms variable. + */ +unsigned int *cmh_tm_timeout_drain_ptr(void) { return &drain_timeout_ms; } +#endif diff --git a/drivers/crypto/cmh/include/cmh.h b/drivers/crypto/cmh/include/cmh.h new file mode 100644 index 00000000000000..18150ba39129ea --- /dev/null +++ b/drivers/crypto/cmh/include/cmh.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Top-level Device Structure + */ + +#ifndef CMH_H +#define CMH_H + +#include + +#include "cmh_config.h" + +#define CMH_DRV_NAME "cmh" +#define CMH_VERSION "1.0.0" + +/** + * struct cmh_device - Top-level driver state for a CMH hardware instance + * @config: Hardware configuration (core mappings, MBX layout, feature flags) + * @dev: Platform or parent device used for DMA and logging + */ +struct cmh_device { + struct cmh_config config; + struct device *dev; +}; + +#endif /* CMH_H */ diff --git a/drivers/crypto/cmh/include/cmh_aes_abi.h b/drivers/crypto/cmh/include/cmh_aes_abi.h new file mode 100644 index 00000000000000..0b876dd677731b --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_aes_abi.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- AES Core ABI Definitions + * + * Kernel-side definitions for the CMH AES ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_AES_ABI_H +#define CMH_AES_ABI_H + +#include + +/* AES Block Size */ + +#define CMH_AES_BLOCK_SIZE 16U +#define CMH_AES_IV_SIZE 16U + +/* AES Modes (per CMH AES ABI) */ + +#define AES_MODE_ECB 1U +#define AES_MODE_CBC 2U +#define AES_MODE_CTR 3U +#define AES_MODE_CFB 4U +#define AES_MODE_GCM 5U +#define AES_MODE_CMAC 6U +#define AES_MODE_CCM 7U +#define AES_MODE_XTS 8U + +/* AES Operations (per CMH AES ABI) */ + +#define AES_OP_DECRYPT 1U +#define AES_OP_ENCRYPT 2U + +/* AES Command IDs */ + +#define AES_CMD_INIT 0x01U +#define AES_CMD_AAD_UPDATE 0x02U +#define AES_CMD_AAD_FINAL 0x03U +#define AES_CMD_UPDATE 0x04U +#define AES_CMD_FINAL 0x05U +#define AES_CMD_SCATTERGATHER 0x06U +#define AES_CMD_CCM_INIT 0x0AU +#define AES_CMD_AAD_FINAL_AUTH 0x0EU + +/* AES Command Structures */ + +struct aes_cmd_init { + u64 key; /* datastore reference for the key */ + u64 iv; /* DMA address of the IV (or nonce in CCM) */ + u32 keylen; /* key length in bytes */ + u32 ivlen; /* IV length in bytes (0..16) */ + u32 mode; /* AES mode (AES_MODE_*) */ + u32 op; /* AES operation (AES_OP_*) */ + u32 aadlen; /* AAD length or 0 */ + u32 iolen; /* plaintext/ciphertext length */ + u32 taglen; /* tag length or 0 */ + u32 xts_offset; /* XTS block index j; 0 for the skcipher path */ +}; + +struct aes_cmd_aad_final { + u64 data; /* DMA address of AAD data */ + u32 datalen; /* AAD data length */ +}; + +struct aes_cmd_aad_final_auth { + u64 data; /* DMA address of final AAD data */ + u32 datalen; /* final AAD data length */ + u64 tag; /* DMA address of tag */ + u32 taglen; /* tag length */ +}; + +struct aes_cmd_update { + u64 input; /* DMA address of input data */ + u64 output; /* DMA address of output data */ + u32 iolen; /* input/output data length */ +}; + +struct aes_cmd_final { + u64 input; /* DMA address of last input data */ + u64 output; /* DMA address of last output data */ + u64 tag; /* DMA address of tag (AEAD only) */ + u32 iolen; /* last input/output data length */ + u32 taglen; /* tag length (AEAD only) */ +}; + +/* AES Command Union */ + +union aes_cmd { + struct aes_cmd_init cmd_init; + struct aes_cmd_update cmd_update; + struct aes_cmd_final cmd_final; + struct aes_cmd_aad_final cmd_aad_final; + struct aes_cmd_aad_final_auth cmd_aad_final_auth; +}; + +#endif /* CMH_AES_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_ccp_abi.h b/drivers/crypto/cmh/include/cmh_ccp_abi.h new file mode 100644 index 00000000000000..4e3eb9feaec9d4 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_ccp_abi.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- CCP Core ABI Definitions + * + * Kernel-side definitions for the CMH CCP ABI. + * All constants and layouts derived from the CMH eSW ABI. + * + * The CCP core provides three modes: + * - ChaCha20 stream cipher (skcipher) + * - Poly1305 one-time authenticator (shash) + * - ChaCha20-Poly1305 AEAD (RFC 7539) + */ + +#ifndef CMH_CCP_ABI_H +#define CMH_CCP_ABI_H + +#include + +/* CCP Block Sizes */ + +#define CCP_CHACHA_BLOCK_SIZE 64U /* ChaCha20 block = 512 bits */ +#define CCP_POLY_BLOCK_SIZE 16U /* Poly1305 block = 128 bits */ +#define CCP_CTRNONCE_SIZE 16U /* 4-byte LE counter + 12-byte nonce */ +#define CCP_POLY_KEY_SIZE 16U /* r_key and s_key each 16 bytes */ +#define CCP_POLY_TAG_SIZE 16U /* Poly1305 tag = 128 bits */ +#define CCP_CHACHA_CTR_LEN 4U /* 32-bit counter */ + +/* CCP Operations (per CMH CCP ABI) */ + +#define CCP_OP_DECRYPT 1U +#define CCP_OP_ENCRYPT 2U + +/* CCP Command IDs */ + +#define CCP_CMD_CHACHA20_INIT 0x01U +#define CCP_CMD_POLY1305_INIT 0x02U +#define CCP_CMD_AEAD_INIT 0x03U +#define CCP_CMD_AAD_UPDATE 0x04U +#define CCP_CMD_AAD_FINAL 0x05U +#define CCP_CMD_UPDATE 0x06U +#define CCP_CMD_FINAL 0x07U +#define CCP_CMD_SCATTERGATHER 0x08U +/* CCP_CMD_FLUSH = VCQ_CMD_FLUSH (0xFF) -- defined in cmh_vcq.h */ + +/* CCP Command Structures */ + +struct ccp_cmd_chacha { + u64 key; /* datastore reference for the key */ + u64 ctrnonce; /* DMA address of the 16-byte counter+nonce */ + u32 keylen; /* key length: 16 or 32 bytes */ + u32 ctrnoncelen; /* always 16 */ + u32 ctrlen; /* counter length: 4 bytes */ + u32 op; /* CCP_OP_ENCRYPT or CCP_OP_DECRYPT */ +}; + +struct ccp_cmd_poly { + u64 rkey; /* datastore reference for the r key */ + u64 skey; /* datastore reference for the s key */ + u32 rkeylen; /* always 16 */ + u32 skeylen; /* always 16 */ +}; + +struct ccp_cmd_aead { + u64 key; /* datastore reference for the key */ + u64 ctrnonce; /* DMA address of the 16-byte counter+nonce */ + u32 keylen; /* key length: 32 bytes */ + u32 ctrnoncelen; /* always 16 */ + u32 op; /* CCP_OP_ENCRYPT or CCP_OP_DECRYPT */ +}; + +struct ccp_cmd_aad_update { + u64 aad; /* DMA address of AAD data */ + u32 aadlen; /* AAD length (must be multiple of 16) */ +}; + +struct ccp_cmd_aad_final { + u64 aad; /* DMA address of last AAD data */ + u32 aadlen; /* last AAD length (any size) */ +}; + +struct ccp_cmd_update { + u64 input; /* DMA address of input data */ + u64 output; /* DMA address of output data */ + u32 iolen; /* input/output length */ +}; + +struct ccp_cmd_final { + u64 input; /* DMA address of last input data */ + u64 output; /* DMA address of last output data */ + u64 tag; /* DMA address of the 16-byte tag */ + u32 iolen; /* last input/output data length */ + u32 taglen; /* tag length (always 16) */ +}; + +/* CCP Command Union */ + +union ccp_cmd { + struct ccp_cmd_chacha cmd_chacha; + struct ccp_cmd_poly cmd_poly; + struct ccp_cmd_aead cmd_aead; + struct ccp_cmd_aad_update cmd_aad_update; + struct ccp_cmd_aad_final cmd_aad_final; + struct ccp_cmd_update cmd_update; + struct ccp_cmd_final cmd_final; +}; + +#endif /* CMH_CCP_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_config.h b/drivers/crypto/cmh/include/cmh_config.h new file mode 100644 index 00000000000000..6a9e629ed353ed --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_config.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Configuration Structures and Defaults + */ + +#ifndef CMH_CONFIG_H +#define CMH_CONFIG_H + +#include +#include + +#include "cmh_registers.h" +#include "cmh_vcq.h" + +/* Limits */ + +/* + * Max mailboxes the driver manages simultaneously. The hardware address + * space supports CMH_MAX_MBX_INSTANCES (64) instance indices, but this + * compile-time constant caps how many the driver allocates DMA queues, + * IRQ slots, and per-transform cache entries for. To manage more + * mailboxes (up to the HW max), increase this value and rebuild the LKM + * -- it cannot be changed via module parameters at runtime. + */ +#define CMH_MAX_CONFIGURED_MBX 16 +#define CMH_MAX_CORE_INSTANCES 8 + +/* MBX setup parameter ranges (per CMH hardware specification) */ +#define CMH_MBX_SLOTS_LOG2_MIN 1 +#define CMH_MBX_SLOTS_LOG2_MAX 15 +#define CMH_MBX_STRIDE_LOG2_MIN 7 +#define CMH_MBX_STRIDE_LOG2_MAX 10 + +/* Default Configuration Values */ + +#define CMH_DEFAULT_MBX_COUNT 2 +#define CMH_DEFAULT_SLOTS_LOG2 5 /* 2^5 = 32 slots */ +#define CMH_DEFAULT_STRIDE_LOG2 7 /* 2^7 = 128 bytes per slot */ +#define CMH_DEFAULT_IRQ (-1) /* polling mode */ +#define CMH_DEFAULT_FW_READY_TIMEOUT_MS 5000 /* 5s for mission mode */ + +/* Per-Core-Type Instance Configuration */ + +struct cmh_core_type_cfg { + u32 num_instances; + u32 core_ids[CMH_MAX_CORE_INSTANCES]; + s32 mbx[CMH_MAX_CORE_INSTANCES]; /* -1 = auto-assign */ +}; + +/* Per-Mailbox Configuration */ + +struct cmh_mbx_config { + u32 instance; /* 0-based MBX instance index (0..63) */ + u32 slots_log2; /* log2(slot count), range 1..15 */ + u32 stride_log2; /* log2(bytes per slot), range 7..10 */ + u32 lock_val; /* MBX lock token (non-zero while held) */ + dma_addr_t dma_handle; /* DMA bus address from dma_alloc_coherent */ + void *virt_addr; /* kernel virtual address of MBXQ buffer */ + size_t queue_size; /* total queue buffer size in bytes */ + void __iomem *reg_base; /* ioremap'd register base for this instance */ +}; + +/* Global Device Configuration */ + +struct cmh_config { + phys_addr_t sic_base; + size_t sic_size; + void __iomem *sic_mapped; /* ioremap'd SIC region */ + struct device_node *of_node; /* DT node (may be NULL) */ + u32 mbx_count; + struct cmh_mbx_config mailboxes[CMH_MAX_CONFIGURED_MBX]; + int irq; /* -1 = poll, else IRQ line */ + unsigned int fw_ready_timeout_ms; /* FW mission-mode timeout */ + struct cmh_core_type_cfg core_types[CMH_NUM_CORE_TYPES]; +}; + +/* Module Parameter Interface */ + +struct platform_device; + +/** + * cmh_config_init() - Populate config from module params and device-tree + * @cfg: Configuration structure to fill + * @pdev: Platform device (for DT properties and IRQ lookup) + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_config_init(struct cmh_config *cfg, struct platform_device *pdev); + +#endif /* CMH_CONFIG_H */ diff --git a/drivers/crypto/cmh/include/cmh_debugfs.h b/drivers/crypto/cmh/include/cmh_debugfs.h new file mode 100644 index 00000000000000..abaa837470c5ea --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_debugfs.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- debugfs Per-MBX and TM Counters + * + * Exposes diagnostic counters under /sys/kernel/debug/cmh/: + * + * mbxN/vcqs_submitted Total VCQs sent to MBX N + * mbxN/vcqs_completed Total completions received + * mbxN/vcqs_errors Total error completions + * mbxN/queue_full_count Times select_mailbox() skipped this MBX + * mbxN/max_queue_depth High-water mark of in-flight transactions + * + * tm/cmq_posts Total cmh_tm_post_command() calls + * tm/cmq_depth_max High-water mark of CMQ length + * tm/cmq_eagain_count Times CMQ was full (-EAGAIN) + * tm/backoff_count Times TM backed off (all MBX queues full) + * tm/async_timeout_count Async requests that timed out + * + * Counters are atomic64_t -- safe to read from any context. + * When CONFIG_CRYPTO_DEV_CMH_DEBUG is off, all functions become no-ops and the + * compiler eliminates the counter code entirely. + */ + +#ifndef CMH_DEBUGFS_H +#define CMH_DEBUGFS_H + +#include +#include + +/* Per-Mailbox Statistics */ + +struct cmh_mbx_stats { + atomic64_t vcqs_submitted; + atomic64_t vcqs_completed; + atomic64_t vcqs_errors; + atomic64_t queue_full_count; + atomic64_t max_queue_depth; +}; + +/* TM-Level Statistics */ + +struct cmh_tm_stats { + atomic64_t cmq_posts; + atomic64_t cmq_depth_max; + atomic64_t cmq_eagain_count; + atomic64_t backoff_count; + atomic64_t async_timeout_count; +}; + +/** + * cmh_stat_update_max() - Atomically update a high-water mark counter + * @counter: atomic64_t counter to update + * @val: New candidate value + * + * Updates @counter to @val if @val exceeds the current maximum. + * Lock-free via atomic cmpxchg loop. + */ +static inline void cmh_stat_update_max(atomic64_t *counter, s64 val) +{ + s64 cur; + + do { + cur = atomic64_read(counter); + if (val <= cur) + return; + } while (atomic64_cmpxchg(counter, cur, val) != cur); +} + +/* Interface (stub when CONFIG_CRYPTO_DEV_CMH_DEBUG is off) */ + +struct cmh_config; + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG + +int cmh_debugfs_init(struct cmh_config *cfg); +void cmh_debugfs_cleanup(void); + +struct cmh_mbx_stats *cmh_debugfs_mbx_stats(u32 mbx_idx); +struct cmh_tm_stats *cmh_debugfs_tm_stats(void); + +#else /* !CONFIG_CRYPTO_DEV_CMH_DEBUG */ + +static inline int cmh_debugfs_init(struct cmh_config *c) { return 0; } +static inline void cmh_debugfs_cleanup(void) {} +static inline struct cmh_mbx_stats *cmh_debugfs_mbx_stats(u32 i) { return NULL; } +static inline struct cmh_tm_stats *cmh_debugfs_tm_stats(void) { return NULL; } + +#endif /* CONFIG_CRYPTO_DEV_CMH_DEBUG */ +#endif /* CMH_DEBUGFS_H */ diff --git a/drivers/crypto/cmh/include/cmh_dma.h b/drivers/crypto/cmh/include/cmh_dma.h new file mode 100644 index 00000000000000..7dd0d8311785d2 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_dma.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- DMA Interface + * + * Platform-independent DMA operations for the CMH crypto accelerator. + * All functions are implemented in cmh_dma.c (standard kernel DMA API). + * + * Alternate backends may be linked in place of cmh_dma.c for + * non-standard platforms. Such backends must implement the same + * symbol set and may use different allocation and mapping semantics + * (e.g. pool-based alloc/free instead of address translation). + */ + +#ifndef CMH_DMA_H +#define CMH_DMA_H + +#include +#include + +#include "cmh_vcq.h" + +struct platform_device; + +/** + * cmh_dma_init() - Initialize the DMA backend + * @pdev: Platform device (provides struct device for DMA ops) + * + * Called early in .probe(). The standard backend stores the device + * pointer; alternate backends may set up additional resources. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_dma_init(struct platform_device *pdev); + +/** + * cmh_dma_cleanup() - Tear down the DMA backend + * + * Called in .remove() and error paths. Releases any resources + * allocated by cmh_dma_init(). + */ +void cmh_dma_cleanup(void); + +/** + * cmh_dev() - Global device accessor + * + * Returns the struct device * associated with the platform_driver instance. + * Valid only between cmh_dma_init() and cmh_dma_cleanup(). + * + * Return: Platform device pointer, or NULL outside lifecycle. + */ +struct device *cmh_dev(void); + +/* Streaming DMA map / unmap (short-lived per-request buffers) */ + +dma_addr_t cmh_dma_map_single(void *buf, size_t size, + enum dma_data_direction dir); +void cmh_dma_unmap_single(dma_addr_t addr, size_t size, + enum dma_data_direction dir); + +/* + * Sync a DMA_FROM_DEVICE buffer so the CPU sees device-written data. + * + * Required before reading *buf when SWIOTLB bounce buffering is active + * (e.g. arm64 without IOMMU): the device writes to the bounce buffer, + * not the original allocation, so the CPU must sync before access. + * On architectures without bounce buffers (e.g. rv64) this is a no-op. + * + * Call between cmh_tm_submit_sync() and the first CPU read of the buffer, + * while the mapping is still live (before cmh_dma_unmap_single). + */ +void cmh_dma_sync_for_cpu(dma_addr_t addr, size_t size, + enum dma_data_direction dir); + +/* + * Sync a DMA_TO_DEVICE buffer so the device sees CPU-written data. + * + * Required after CPU writes to a mapped streaming buffer (e.g. SG + * descriptor arrays that need items_dma for .lli pointer calculation + * before content is written). Must be called before the device reads. + */ +void cmh_dma_sync_for_device(dma_addr_t addr, size_t size, + enum dma_data_direction dir); + +int cmh_dma_map_error(dma_addr_t addr); + +/* Coherent DMA alloc / free (long-lived MBX queue buffers) */ + +void *cmh_dma_alloc(size_t size, dma_addr_t *handle, gfp_t gfp); +void cmh_dma_free(size_t size, void *virt, dma_addr_t handle); + +/** + * cmh_dma_write() - Copy data into a DMA-allocated buffer + * @dst: Destination pointer (from cmh_dma_alloc) + * @src: Source kernel buffer + * @len: Number of bytes to copy + * + * Copies @len bytes from @src to @dst. @dst must have been obtained + * from cmh_dma_alloc(). Abstracted to allow platforms with non-standard + * DMA buffer access semantics. + */ +void cmh_dma_write(void *dst, const void *src, size_t len); + +/** + * cmh_dma_fence() - Fence preceding writes to DMA-allocated memory + * @ptr: Any pointer into the region that was written + * + * Ensures all preceding CPU writes to DMA memory are committed to the + * target memory controller before subsequent MMIO register writes. + * + * Required on FPGA platforms where DMA memory and device control + * registers reside on different AXI slaves -- a CPU-side wmb() only + * orders store dispatch, not arrival at the target. A read from the + * DMA memory slave forces the memory controller to serialize behind + * all preceding writes from this CPU before responding, guaranteeing + * the data is committed before the doorbell register write is issued. + * + * On standard DMA API platforms (cache-coherent), this is a no-op. + */ +void cmh_dma_fence(void *ptr); + +/** + * cmh_dma_zero() - Zero a DMA-allocated buffer + * @dst: Destination pointer (from cmh_dma_alloc) + * @len: Number of bytes to zero + */ +void cmh_dma_zero(void *dst, size_t len); + +/* + * CMH eSW scatter-gather chain -- built with proper DMA mappings. + * + * The CMH eSW DMAC walks a linked list of dma_scattergather_item + * descriptors. Each .src is the DMA address of an input buffer; + * each .lli is the DMA address of the next descriptor (0 = end). + * + * The descriptor array uses streaming DMA (kmalloc + dma_map_single) + * so that cmh_dma_free_sg() is safe from any context -- including + * BH-disabled completion callbacks where dma_free_coherent's + * vunmap() path would crash on non-coherent architectures. + */ + +/* Input descriptor for cmh_dma_build_sg() -- one per data buffer */ +struct cmh_dma_buf { + void *data; + u32 len; +}; + +/* Opaque handle returned by cmh_dma_build_sg(); pass to cmh_dma_free_sg() */ +struct cmh_sg_map { + struct dma_scattergather_item *items; /* CPU virtual address */ + dma_addr_t items_dma; /* DMA address (pass to GATHER cmd) */ + size_t items_size; /* allocation size */ + u32 count; + struct { + dma_addr_t dma; + u32 len; + } bufs[]; /* per-entry source DMA handles */ +}; + +/** + * cmh_dma_build_sg() - Build a DMA-mapped CMH eSW SG chain + * @bufs: Array of kernel buffer descriptors (data pointer + length) + * @count: Number of entries in @bufs (must be > 0; returns NULL for 0) + * @gfp: Allocation flags (GFP_KERNEL or GFP_ATOMIC) + * + * Allocates a dma_scattergather_item chain using streaming DMA + * (kmalloc + dma_map_single), DMA-maps each source buffer, and + * links the descriptors. + * The returned cmh_sg_map->items_dma is the address to pass to + * vcq_add_hc_gather() (or any core's scatter-gather command). + * + * Caller contract: + * - Each bufs[i].data must point to DMA-mappable memory (kmalloc, + * page-allocated, or vmalloc with DMA support). Stack buffers + * are NOT safe. + * - Each bufs[i].len must be > 0. + * - The returned cmh_sg_map must remain alive (not freed) until + * the hardware completes the scatter-gather operation. Only then + * may cmh_dma_free_sg() be called. + * - There is no hardware-imposed limit on @count, but callers are + * responsible for bounding it to avoid excessive DMA mappings. + * In practice, hash uses <= 2 entries (partial + new data). + * + * Return: Opaque cmh_sg_map handle, or NULL on allocation/mapping failure. + */ +struct cmh_sg_map *cmh_dma_build_sg(const struct cmh_dma_buf *bufs, u32 count, + gfp_t gfp); + +/** + * cmh_dma_free_sg() - Unmap all buffers and free the SG chain + * @sgm: Handle from cmh_dma_build_sg(), or NULL (no-op) + */ +void cmh_dma_free_sg(struct cmh_sg_map *sgm); + +/* + * Orphan-DMA context -- generic helper for the noabort submit path. + * + * When cmh_tm_submit_sync_noabort() times out with a VCQ still + * in-flight, the eSW will continue writing to DMA buffers after the + * caller returns. Callers wrap their DMA state in this struct and + * pass cmh_dma_orphan_free as the orphan_cb -- the RH callback frees + * the mapping + buffer when the VCQ eventually completes. + * + * Drain guarantee: cmh_tm_cleanup() calls timer_delete_sync() on each + * TXN timeout timer and splices all TXQ entries before invoking their + * completion callbacks. This ensures no orphan callback can race with + * or run after TM cleanup completes -- by that point every in-flight + * transaction has been force-completed and its orphan_cb invoked. + */ +struct cmh_dma_orphan { + void *buf; + dma_addr_t addr; + size_t len; + enum dma_data_direction dir; +}; + +void cmh_dma_orphan_free(void *data); + +#endif /* CMH_DMA_H */ diff --git a/drivers/crypto/cmh/include/cmh_drbg_abi.h b/drivers/crypto/cmh/include/cmh_drbg_abi.h new file mode 100644 index 00000000000000..d4cebfe83d4bf9 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_drbg_abi.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- DRBG Core ABI Definitions + * + * Kernel-side definitions for the CMH DRBG ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_DRBG_ABI_H +#define CMH_DRBG_ABI_H + +#include + +/* DRBG Commands */ + +#define DRBG_CMD_CONFIG 0x01U +#define DRBG_CMD_GENERATE 0x02U +#define DRBG_CMD_DATASTORE 0x03U +#define DRBG_CMD_RESET 0x04U + +/* DRBG Entropy Ratio (per CMH DRBG ABI) */ + +#define DRBG_ENTROPY_RATIO_ONE 0U +#define DRBG_ENTROPY_RATIO_ONE_HALF 1U +#define DRBG_ENTROPY_RATIO_ONE_THIRD 2U +#define DRBG_ENTROPY_RATIO_ONE_FOURTH 3U + +/* DRBG Security Strength (per CMH DRBG ABI) */ + +#define DRBG_SECURITY_STRENGTH_128 0x00U +#define DRBG_SECURITY_STRENGTH_256 0x10U + +/* DRBG Personalization Data Length */ + +#define DRBG_PADATA_LEN 16U + +/* DRBG Command Structures */ + +struct drbg_cmd_config { + u32 entropy_ratio; /* drbg_entropy_ratio value */ + u32 security_strength; /* drbg_security_strength value */ + u8 padata[DRBG_PADATA_LEN]; +}; + +struct drbg_cmd_generate { + u64 dst; /* DMA physical address for output */ + u32 len; /* requested output length in bytes */ + u8 padata[DRBG_PADATA_LEN]; +}; + +struct drbg_cmd_datastore { + u64 ref; /* datastore reference */ + u32 len; /* data length in bytes */ + u32 type; /* datastore type */ + u8 padata[DRBG_PADATA_LEN]; +}; + +/* DRBG Command Union */ + +union drbg_cmd { + struct drbg_cmd_config cmd_config; + struct drbg_cmd_generate cmd_generate; + struct drbg_cmd_datastore cmd_datastore; +}; + +#endif /* CMH_DRBG_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_eac_abi.h b/drivers/crypto/cmh/include/cmh_eac_abi.h new file mode 100644 index 00000000000000..f0ebd3de1fb497 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_eac_abi.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- EAC (Error and Alarm Controller) ABI Definitions + * + * Kernel-side definitions for the CMH EAC ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_EAC_ABI_H +#define CMH_EAC_ABI_H + +#include + +/* EAC Commands */ + +#define EAC_CMD_READ 0x01U + +/* EAC Read Response -- eSW writes this to the DMA destination buffer */ + +struct eac_read_rsp { + u64 mailbox_notification; /* bitmask: MBX that raised safety notif */ + u32 hw_error; /* bitmask: HWC that raised error */ + u32 hw_nmi; /* bitmask: HWC that raised NMI */ + u32 hw_panic; /* bitmask: HWC that raised HW panic */ + u32 safety_fatal; /* bitmask: HWC that raised fatal safety */ + u32 safety_notification; /* bitmask: HWC that raised safety notif */ + u32 sw_info0; /* eSW tracing information */ + u32 sw_info1; /* eSW tracing information */ + u32 sram_bank_errors[4]; /* correctable ECC error counts per bank */ +}; + +/* EAC Command Structures */ + +struct eac_cmd_read { + u64 dst; /* DMA destination for eac_read_rsp */ + u32 len; /* must be >= sizeof(struct eac_read_rsp) */ +}; + +union eac_cmd { + struct eac_cmd_read cmd_read; +}; + +#endif /* CMH_EAC_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_hc_abi.h b/drivers/crypto/cmh/include/cmh_hc_abi.h new file mode 100644 index 00000000000000..4e8c5ea3c69c1c --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_hc_abi.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Hash Core (HC) ABI Definitions + * + * Kernel-side definitions for the CMH HC (Hash Core) ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_HC_ABI_H +#define CMH_HC_ABI_H + +#include +#include + +/* HC Commands */ + +#define HC_CMD_INIT 0x01U +#define HC_CMD_HMAC 0x02U +#define HC_CMD_UPDATE 0x03U +#define HC_CMD_FINAL 0x04U +#define HC_CMD_UPDATE2D 0x05U +#define HC_CMD_SQUEEZE 0x07U +#define HC_CMD_GATHER 0x08U +#define HC_CMD_CSHAKE 0x09U +#define HC_CMD_KMAC 0x0AU +#define HC_CMD_SAVE 0x0BU +#define HC_CMD_RESTORE 0x0CU + +/* HC Algorithms (per CMH HC ABI) */ + +#define HC_ALGO_SHA2_224 1U +#define HC_ALGO_SHA2_256 2U +#define HC_ALGO_SHA2_384 3U +#define HC_ALGO_SHA2_512 4U +#define HC_ALGO_SHA3_224 5U +#define HC_ALGO_SHA3_256 6U +#define HC_ALGO_SHA3_384 7U +#define HC_ALGO_SHA3_512 8U +#define HC_ALGO_SHAKE128 9U +#define HC_ALGO_SHAKE256 10U + +/* HC Algo Flags */ + +#define HC_ALGO_FLAG_SCA_KEY BIT(18) /* SCA key in 2 shares */ +#define HC_ALGO_FLAG_SCA_OUT BIT(19) /* SCA output in 2 shares */ + +#define HC_ALGO_SET(flags, algo) (((flags) & 0xFF0000UL) | ((algo) & 0xFFUL)) +#define HC_ALGO_GET(algo) ((algo) & 0xFFU) + +/* Hash Digest Sizes */ + +#define CMH_SHA224_DIGEST_SIZE 28U +#define CMH_SHA256_DIGEST_SIZE 32U +#define CMH_SHA384_DIGEST_SIZE 48U +#define CMH_SHA512_DIGEST_SIZE 64U + +/* SHA-3 digest sizes are the same as SHA-2 for matching output widths */ +#define CMH_SHA3_224_DIGEST_SIZE 28U +#define CMH_SHA3_256_DIGEST_SIZE 32U +#define CMH_SHA3_384_DIGEST_SIZE 48U +#define CMH_SHA3_512_DIGEST_SIZE 64U + +/* SHAKE default output lengths (fixed-output ahash registration) */ +#define CMH_SHAKE128_DIGEST_SIZE 32U /* 128-bit security -> 32 bytes */ +#define CMH_SHAKE256_DIGEST_SIZE 64U /* 256-bit security -> 64 bytes */ + +/* HC Context (for SAVE/RESTORE) */ + +#define HC_CONTEXT_WORDS 149U +#define HC_CONTEXT_SIZE (HC_CONTEXT_WORDS * 4 + 4) /* ctx[149] + crc */ + +/* cSHAKE function name max length */ + +#define HC_CSHAKE_MAX_NAMELEN 36U + +/* + * Maximum customization string (S) length for cSHAKE / KMAC. + * + * S is packed as inline VCQ data after the CSHAKE/KMAC command slot. + * The worst-case VCQ layout (KMAC with raw key + GATHER) uses 5 fixed + * slots out of CMH_KMAC_MAX_PAYLOAD (9), leaving 4 inline slots. + * Each VCQ slot is 64 bytes, so the safe limit is 4 * 64 = 256 bytes. + */ +#define HC_CSHAKE_MAX_CUSTOMLEN 256U + +/* HC Command Structures */ + +struct hc_cmd_init { + u32 algo; /* hc_algo value, optionally ORed with HC_ALGO_FLAG_* */ +}; + +struct hc_cmd_hmac { + u64 key; /* datastore reference for HMAC key */ + u32 keylen; /* key length in bytes */ + u32 algo; /* hc_algo value */ +}; + +struct hc_cmd_update { + u64 input; /* DMA physical address of input data */ + u32 inlen; /* input data length in bytes */ +}; + +struct hc_cmd_final { + u64 digest; /* DMA physical address for output digest */ + u32 outlen; /* digest length in bytes */ +}; + +struct hc_cmd_update2d { + u64 input; /* DMA source address for input data */ + u64 output; /* DMA destination address for pass-through data */ + u32 iolen; /* input/pass-through data length in bytes */ +}; + +struct hc_cmd_gather { + u64 lista; /* DMA address of dma_scattergather_item chain */ + u32 sgcmd; /* HC sub-command: HC_CMD_UPDATE or HC_CMD_UPDATE2D */ +}; + +struct hc_cmd_cshake { + u64 custom; /* DMA address for the customization string */ + u32 customlen; /* length of the customization string */ + u32 algo; /* HC_ALGO_SHAKE128 or HC_ALGO_SHAKE256 */ + u32 namelen; /* length of the function name string */ + u8 name[HC_CSHAKE_MAX_NAMELEN]; /* function name string (inline) */ +}; + +struct hc_cmd_kmac { + u64 key; /* datastore reference for KMAC key */ + u64 custom; /* DMA address for the customization string */ + u32 keylen; /* key length in bytes */ + u32 customlen; /* length of the customization string */ + u32 algo; /* HC_ALGO_SHAKE128 or HC_ALGO_SHAKE256 */ + u32 outlen; /* requested output digest length */ +}; + +struct hc_cmd_save { + u64 output; /* DMA physical address for saved context */ + u32 outlen; /* must be HC_CONTEXT_SIZE */ +}; + +struct hc_cmd_restore { + u64 input; /* DMA physical address of saved context */ + u32 inlen; /* must be HC_CONTEXT_SIZE */ +}; + +/* HC Command Union */ + +union hc_cmd { + struct hc_cmd_init cmd_init; + struct hc_cmd_hmac cmd_hmac; + struct hc_cmd_cshake cmd_cshake; + struct hc_cmd_kmac cmd_kmac; + struct hc_cmd_update cmd_update; + struct hc_cmd_final cmd_final; + struct hc_cmd_update2d cmd_update2d; + struct hc_cmd_gather cmd_gather; + struct hc_cmd_save cmd_save; + struct hc_cmd_restore cmd_restore; +}; + +#endif /* CMH_HC_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_hcq_abi.h b/drivers/crypto/cmh/include/cmh_hcq_abi.h new file mode 100644 index 00000000000000..b9fc2a80a4086e --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_hcq_abi.h @@ -0,0 +1,221 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- HCQ Core ABI Definitions + * + * Kernel-side definitions for the CMH HCQ ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_HCQ_ABI_H +#define CMH_HCQ_ABI_H + +#include +#include + +/* VCQ layout: header + [SYS cmds] + HCQ_CMD + [sys_read] + flush */ +#define HCQ_VCQ_CMDS_MIN 3 /* header + cmd + flush */ +#define HCQ_VCQ_CMDS_MAX 6 /* keygen: hdr+new+write+cmd+read+flush */ + +/* HCQ Command IDs */ +#define HCQ_CMD_XMSS_VERIFY 0x03U +#define HCQ_CMD_LMS_VERIFY 0x04U +#define HCQ_CMD_SLHDSA_VERIFY_INTERNAL 0x05U +#define HCQ_CMD_SLHDSA_VERIFY 0x06U +#define HCQ_CMD_SLHDSA_VERIFY_PREHASH 0x07U +#define HCQ_CMD_SLHDSA_VERIFY_PREHASH_DIGEST 0x08U +#define HCQ_CMD_SLHDSA_KEYGEN 0x09U +#define HCQ_CMD_SLHDSA_SIGN_INTERNAL 0x10U +#define HCQ_CMD_SLHDSA_SIGN 0x11U +#define HCQ_CMD_SLHDSA_SIGN_PREHASH 0x12U +#define HCQ_CMD_SLHDSA_SIGN_PREHASH_DIGEST 0x13U +#define HCQ_CMD_SLHDSA_PUBGEN 0x14U + +/* SLH-DSA Parameter Set IDs */ +#define HCQ_SLHDSA_SHAKE_128S 1U +#define HCQ_SLHDSA_SHAKE_128F 2U +#define HCQ_SLHDSA_SHAKE_192S 3U +#define HCQ_SLHDSA_SHAKE_192F 4U +#define HCQ_SLHDSA_SHAKE_256S 5U +#define HCQ_SLHDSA_SHAKE_256F 6U +#define HCQ_SLHDSA_SHA2_128S 7U +#define HCQ_SLHDSA_SHA2_128F 8U +#define HCQ_SLHDSA_SHA2_192S 9U +#define HCQ_SLHDSA_SHA2_192F 10U +#define HCQ_SLHDSA_SHA2_256S 11U +#define HCQ_SLHDSA_SHA2_256F 12U +#define HCQ_SLHDSA_PARAM_MAX 12U + +/* SLH-DSA Prehash Algorithm IDs */ +#define HCQ_SLHDSA_PREHASH_SHA256 1U +#define HCQ_SLHDSA_PREHASH_SHA512 2U +#define HCQ_SLHDSA_PREHASH_SHAKE128 3U +#define HCQ_SLHDSA_PREHASH_SHAKE256 4U + +/* SLH-DSA size limits */ +#define SLHDSA_MAX_PK_SIZE 64U /* 2*n, n=32 */ +#define SLHDSA_MAX_SK_SIZE 128U /* 4*n, n=32 */ +#define SLHDSA_MAX_SEED_SIZE 96U /* 3*n, n=32 */ +#define SLHDSA_MAX_SIG_SIZE 49856U /* SHAKE-256f / SHA2-256f */ +#define SLHDSA_MAX_MSG_LEN 128U +#define SLHDSA_MAX_CTX_LEN 255U + +/* LMS/HSS size limits -- derived from eSW HCQ ABI constraints */ +#define LMS_MAX_PK_LEN 60U /* eSW public-key buffer */ +#define LMS_MAX_MSG_LEN 256U /* SHS_LMS_MESSAGE_LEN_MAX */ +#define LMS_MAX_SIG_LEN 13364U /* eSW signature buffer */ + +/* XMSS/XMSS-MT size limits -- derived from eSW HCQ ABI constraints */ +#define XMSS_MAX_PK_LEN 136U /* eSW public-key buffer */ +#define XMSS_MAX_MSG_LEN 64U /* SHS_XMSS_MESSAGE_LEN_MAX */ +#define XMSS_MAX_SIG_LEN 27688U /* eSW signature buffer */ + +/* SLH-DSA n-value for each parameter set (index = param_set - 1) */ +extern const u32 slhdsa_n[]; + +/* SLH-DSA signature sizes (index = param_set - 1) */ +extern const u32 slhdsa_sig_size[]; + +/* Derive PK/SK/seed sizes from n */ +static inline u32 slhdsa_pk_size(u32 param_set) +{ + if (param_set < 1U || param_set > HCQ_SLHDSA_PARAM_MAX) + return 0; + return 2U * slhdsa_n[param_set - 1U]; +} + +static inline u32 slhdsa_sk_size(u32 param_set) +{ + if (param_set < 1U || param_set > HCQ_SLHDSA_PARAM_MAX) + return 0; + return 4U * slhdsa_n[param_set - 1U]; +} + +static inline u32 slhdsa_seed_size(u32 param_set) +{ + if (param_set < 1U || param_set > HCQ_SLHDSA_PARAM_MAX) + return 0; + return 3U * slhdsa_n[param_set - 1U]; +} + +static inline u32 slhdsa_get_sig_size(u32 param_set) +{ + if (param_set < 1U || param_set > HCQ_SLHDSA_PARAM_MAX) + return 0; + return slhdsa_sig_size[param_set - 1U]; +} + +/* HCQ Command Structures -- match CMH eSW ABI exactly */ + +struct hcq_cmd_xmss_verify { + u32 xmss_mt; /* 0 = XMSS, 1 = XMSS-MT */ + u32 pk_len; + u32 sig_len; + u32 dig_len; + u64 pk; + u64 sig; + u64 dig; +}; + +struct hcq_cmd_lms_verify { + u32 lms_hss; /* 0 = LMS, 1 = LMS-HSS */ + u32 pk_len; + u32 sig_len; + u32 dig_len; + u64 pk; + u64 sig; + u64 dig; +}; + +struct hcq_cmd_slhdsa_verify_internal { + u32 parameter_set; + u32 message_len; + u64 message; + u64 pk; + u64 sig; +}; + +struct hcq_cmd_slhdsa_verify { + u32 parameter_set; + u32 message_len; + u64 message; + u64 context; + u64 pk; + u64 sig; + u32 context_len; +}; + +struct hcq_cmd_slhdsa_verify_prehash { + u32 parameter_set; + u32 prehash_algo; + u32 message_len; + u32 context_len; + u64 message; + u64 context; + u64 pk; + u64 sig; +}; + +struct hcq_cmd_slhdsa_keygen { + u32 parameter_set; + u32 seed_len; + u32 pk_len; + u32 sk_len; + u64 seed; /* DS reference */ + u64 pk; /* extmem addr */ + u64 sk; /* DS reference */ +}; + +struct hcq_cmd_slhdsa_sign_internal { + u32 parameter_set; + u32 message_len; + u64 add_random; /* extmem addr, 0 = none */ + u64 message; + u64 sk; /* DS reference */ + u64 sig; /* extmem addr */ +}; + +struct hcq_cmd_slhdsa_sign { + u32 parameter_set; + u32 message_len; + u64 add_random; + u64 message; + u64 context; + u64 sk; /* DS reference */ + u64 sig; /* extmem addr */ + u32 context_len; +}; + +struct hcq_cmd_slhdsa_sign_prehash { + u32 parameter_set; + u32 prehash_algo; + u32 message_len; + u32 context_len; + u64 add_random; + u64 message; + u64 context; + u64 sk; /* DS reference */ + u64 sig; /* extmem addr */ +}; + +struct hcq_cmd_slhdsa_pubgen { + u32 parameter_set; + u32 sk_len; + u64 sk; /* DS reference */ + u64 pk; /* extmem addr */ +}; + +union hcq_cmd { + struct hcq_cmd_xmss_verify cmd_xmss_verify; + struct hcq_cmd_lms_verify cmd_lms_verify; + struct hcq_cmd_slhdsa_verify_internal cmd_slhdsa_verify_internal; + struct hcq_cmd_slhdsa_verify cmd_slhdsa_verify; + struct hcq_cmd_slhdsa_verify_prehash cmd_slhdsa_verify_prehash; + struct hcq_cmd_slhdsa_keygen cmd_slhdsa_keygen; + struct hcq_cmd_slhdsa_sign_internal cmd_slhdsa_sign_internal; + struct hcq_cmd_slhdsa_sign cmd_slhdsa_sign; + struct hcq_cmd_slhdsa_sign_prehash cmd_slhdsa_sign_prehash; + struct hcq_cmd_slhdsa_pubgen cmd_slhdsa_pubgen; +}; + +#endif /* CMH_HCQ_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_kic_abi.h b/drivers/crypto/cmh/include/cmh_kic_abi.h new file mode 100644 index 00000000000000..7f4fe3b9fd8976 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_kic_abi.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- KIC Core ABI Definitions + * + * Kernel-side definitions for the CMH KIC ABI (KIC commands only). + * Derived from the CMH eSW ABI. + */ + +#ifndef CMH_KIC_ABI_H +#define CMH_KIC_ABI_H + +#include + +/* KIC Commands */ + +#define KIC_CMD_HKDF1 0x06U +#define KIC_CMD_HKDF2 0x07U +#define KIC_CMD_AES_CMAC_KDF 0x08U +#define KIC_CMD_DKEK_DERIVE 0x09U + +/* Maximum key size for KIC operations (bytes) */ +#define KIC_KEY_SIZE 32U + +/* + * KIC Command Structures + * + * Field names (llen, len) mirror the CMH eSW ABI register layout. + * llen = label length, len = output key length. + */ + +struct kic_cmd_hkdf1 { + u64 dst; /* DS ref for derived key (SYS_REF_LAST) */ + u64 base; /* base key reference (e.g., KIC_KEY1) */ + u64 label; /* label pointer (0 for inline-next-slot) */ + u32 llen; /* label length */ + u32 len; /* output key length */ + u32 type; /* SYS_TYPE_SET(flags, core_id) */ +}; + +struct kic_cmd_hkdf2 { + u64 dst; /* DS ref for derived key */ + u64 base; /* base key reference */ + u64 salt; /* salt key reference (SYS_REF_NONE = no salt) */ + u64 label; /* label pointer */ + u32 llen; /* label length */ + u32 len; /* output key length */ + u32 type; /* SYS_TYPE_SET(flags, core_id) */ +}; + +struct kic_cmd_aes_cmac_kdf { + u64 base_key; /* KIC/DS reference for base key */ + u64 out_key; /* DS reference for derived key */ + u64 label; /* label DMA address */ + u32 key_len; /* base & output key length (must be 32) */ + u32 label_len; /* label length */ + u32 type; /* SYS_TYPE_SET(flags, core_id) for output */ +}; + +struct kic_cmd_dkek_derive { + u64 base_key; /* KIC base key reference */ + u64 out_key; /* DS reference for the derived KEK */ + u32 host_id; /* host ID (0 = caller's own) */ + u32 metadata_len; /* metadata length */ + u64 metadata; /* metadata DMA address */ +}; + +/* KIC Command Union */ + +union kic_cmd { + struct kic_cmd_hkdf1 cmd_hkdf1; + struct kic_cmd_hkdf2 cmd_hkdf2; + struct kic_cmd_aes_cmac_kdf cmd_aes_cmac_kdf; + struct kic_cmd_dkek_derive cmd_dkek_derive; +}; + +#endif /* CMH_KIC_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_mqi.h b/drivers/crypto/cmh/include/cmh_mqi.h new file mode 100644 index 00000000000000..93b84785995394 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_mqi.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Mailbox Queue Initializer + * + * Allocates DMA-capable queue buffers and programs MBX registers + * via the MBX lock/setup/enable/unlock register sequence. + */ + +#ifndef CMH_MQI_H +#define CMH_MQI_H + +#include "cmh_config.h" + +#define MBX_LOCK_TIMEOUT_MS 1000 +#define MBX_LOCK_POLL_MIN_US 10 +#define MBX_LOCK_POLL_MAX_US 50 +#define MBX_HOST_INFO_LKM 0x4C4B4D00U /* "LKM\0" as host identifier */ + +/** + * cmh_mqi_init() - Allocate MBX queue buffers and program registers + * @cfg: Global device configuration + * + * Performs the lock/setup/enable/unlock sequence for each configured MBX. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mqi_init(struct cmh_config *cfg); + +/** + * cmh_mqi_cleanup() - Free MBX queue buffers and release locks + * @cfg: Global device configuration + */ +void cmh_mqi_cleanup(struct cmh_config *cfg); + +#endif /* CMH_MQI_H */ diff --git a/drivers/crypto/cmh/include/cmh_pke_abi.h b/drivers/crypto/cmh/include/cmh_pke_abi.h new file mode 100644 index 00000000000000..e0e7b946b4e392 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_pke_abi.h @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- PKE Core ABI Definitions + * + * Kernel-side definitions for the CMH PKE ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_PKE_ABI_H +#define CMH_PKE_ABI_H + +#include + +/* PKE Command IDs */ + +#define PKE_CMD_ECDSA_VERIFY 0x03U +#define PKE_CMD_ECDSA_SIGN 0x04U +#define PKE_CMD_ECDSA_PUBGEN 0x05U +#define PKE_CMD_ECDSA_KEYGEN 0x06U +#define PKE_CMD_EDDSA_VERIFY 0x07U +#define PKE_CMD_EDDSA_SIGN 0x08U +#define PKE_CMD_EDDSA_PUBGEN 0x09U +#define PKE_CMD_ECDH_KEYGEN 0x0AU +#define PKE_CMD_ECDH 0x0BU +#define PKE_CMD_RSA_ENC 0x0CU +#define PKE_CMD_RSA_DEC 0x0DU +#define PKE_CMD_RSA_KEYGEN 0x0EU +#define PKE_CMD_RSA_CRT_DEC 0x0FU +#define PKE_CMD_SM2_ECDH_KEYGEN 0x16U +#define PKE_CMD_SM2_ECDH 0x17U +#define PKE_CMD_SM2_DEC_POINT 0x18U +#define PKE_CMD_SM2_ENC_POINT 0x19U +#define PKE_CMD_SM2_ID_DIGEST 0x1AU +#define PKE_CMD_SM2_ECDH_HASH 0x1BU +#define PKE_CMD_SM2_DEC_HASH 0x1CU +#define PKE_CMD_SM2_ENC_HASH 0x1DU +#define PKE_CMD_EDDSA_PRIV_KEYGEN_SCA 0x21U +#define PKE_CMD_FLUSH 0xFFU + +/* EC Curve IDs (per CMH PKE ABI) */ + +#define PKE_CURVE_P192 0x01U +#define PKE_CURVE_P224 0x02U +#define PKE_CURVE_P256 0x03U +#define PKE_CURVE_P384 0x04U +#define PKE_CURVE_P521 0x05U +#define PKE_CURVE_SECP256K1 0x07U +#define PKE_CURVE_BP192R1 0x11U +#define PKE_CURVE_BP224R1 0x12U +#define PKE_CURVE_BP256R1 0x13U +#define PKE_CURVE_BP320R1 0x14U +#define PKE_CURVE_BP384R1 0x15U +#define PKE_CURVE_BP512R1 0x16U +#define PKE_CURVE_ANSSI_FRP256V1 0x17U +#define PKE_CURVE_SM2 0x18U +#define PKE_CURVE_25519 0x21U +#define PKE_CURVE_448 0x22U + +/* PKE Command Structures -- match CMH eSW ABI exactly */ + +struct pke_cmd_ecdsa_verify { + u32 curve; + u32 digest_len; + u64 public_key; + u64 digest; + u64 signature; + u64 rprime; +}; + +struct pke_cmd_ecdsa_sign { + u32 curve; + u32 secret_key_len; + u64 digest; + u64 signature; + u64 secret_key; /* DS reference */ + u32 digest_len; +}; + +struct pke_cmd_ecdsa_pubgen { + u32 curve; + u32 secret_key_len; + u64 public_key; + u64 secret_key; /* DS reference */ +}; + +struct pke_cmd_ecdsa_keygen { + u32 curve; + u32 secret_key_len; + u64 secret_key; /* DS reference */ + u32 secret_key_type; +}; + +struct pke_cmd_eddsa_verify { + u32 curve; + u32 digest_len; + u64 public_key_y; + u64 digest; + u64 signature; + u64 rprime; +}; + +struct pke_cmd_eddsa_sign { + u32 curve; + u32 secret_key_len; + u64 digest; + u64 signature; + u64 secret_key; /* DS reference */ + u32 digest_len; +}; + +struct pke_cmd_eddsa_pubgen { + u32 curve; + u32 secret_key_len; + u64 public_key_y; + u64 secret_key; /* DS reference */ +}; + +struct pke_cmd_ecdh_keygen { + u32 curve; + u32 secret_key_len; + u64 public_key_x; + u64 secret_key; /* DS reference */ +}; + +struct pke_cmd_ecdh { + u32 curve; + u32 secret_key_len; + u32 shared_secret_len; + u32 shared_secret_type; + u64 peer_key_x; + u64 secret_key; /* DS reference */ + u64 shared_secret; /* DS reference for result */ +}; + +struct pke_cmd_rsa_enc { + u32 bits; + u32 e_len; + u64 e; + u64 n; + u64 m; + u64 c; +}; + +struct pke_cmd_rsa_dec { + u32 bits; + u32 e_len; + u64 e; + u64 n; + u64 c; + u64 m; + u64 d; /* DS reference */ +}; + +struct pke_cmd_rsa_crt_dec { + u32 bits; + u32 e_len; + u64 e; + u64 n; + u64 c; + u64 m; + u64 crt; /* DS reference */ +}; + +struct pke_cmd_rsa_keygen { + u32 bits; + u32 d_type; + u64 e; + u64 n; + u64 d; /* DS reference */ + u64 crt; /* DS reference */ + u32 crt_type; +}; + +struct pke_cmd_eddsa_keygen_sca { + u32 curve; + u64 secret_key; /* DS reference: input normal SK */ + u64 sca_secret_key; /* DS reference: output blinded SK */ +}; + +/* SM2 Command Structures */ + +struct pke_cmd_sm2_ecdh_keygen { + u64 nonce; /* DMA addr (32B input or output) */ + u64 session_key; /* DMA addr output (64B) */ + u32 nonce_len; /* 0 = HW generates, 32 = caller provides */ +}; + +struct pke_cmd_sm2_ecdh { + u32 nonce_len; /* 0 or 32 */ + u32 private_key_len; /* must be 32 */ + u64 nonce; /* DMA addr (32B) */ + u64 peer_public_key; /* DMA addr (64B) */ + u64 peer_session_key; /* DMA addr (64B) */ + u64 private_key; /* DS reference */ + u64 shared_point; /* DS reference (output, 64B) */ + u32 shared_point_type; /* SYS_TYPE_SET(flags, CORE_ID_PKE) */ +}; + +struct pke_cmd_sm2_dec_point { + u32 ciphertext_len; /* total CT length (97..128) */ + u32 private_key_len; /* must be 32 */ + u64 ciphertext; /* DMA addr (64B: C1 point) */ + u64 dec_point; /* DMA addr output (64B) */ + u64 private_key; /* DS reference */ +}; + +struct pke_cmd_sm2_enc_point { + u64 nonce; /* DMA addr (32B, optional) */ + u64 public_key; /* DMA addr (64B) */ + u64 ciphertext; /* DMA addr output (64B: C1) */ + u64 enc_point; /* DMA addr output (64B) */ + u32 nonce_len; /* 0 or 32 */ +}; + +struct pke_cmd_sm2_id_digest { + u64 id; /* DMA addr (identity, <=32B) */ + u64 public_key; /* DMA addr (64B) */ + u64 digest; /* DMA addr output (32B) */ + u32 id_len; /* identity length in bytes */ +}; + +struct pke_cmd_sm2_ecdh_hash { + u64 peer_id_digest; /* DMA addr (32B) */ + u64 id_digest; /* DMA addr (32B) */ + u64 shared_point; /* DS reference (64B input) */ + u64 shared_key; /* DS reference (16B output) */ + u32 shared_key_type; /* SYS_TYPE_SET(flags, CORE_ID_PKE) */ +}; + +struct pke_cmd_sm2_dec_hash { + u64 ciphertext; /* DMA addr (full ciphertext) */ + u64 dec_point; /* DMA addr (64B) */ + u64 plaintext; /* DMA addr output (ct_len - 96 bytes) */ + u32 ciphertext_len; /* 97..128 */ +}; + +struct pke_cmd_sm2_enc_hash { + u64 message; /* DMA addr (plaintext) */ + u64 enc_point; /* DMA addr (64B) */ + u64 ciphertext; /* DMA addr output (96 + msg_len) */ + u32 message_len; /* 1..32 */ +}; + +/* PKE Command Union */ + +union pke_cmd { + struct pke_cmd_ecdsa_verify cmd_ecdsa_verify; + struct pke_cmd_ecdsa_sign cmd_ecdsa_sign; + struct pke_cmd_ecdsa_pubgen cmd_ecdsa_pubgen; + struct pke_cmd_ecdsa_keygen cmd_ecdsa_keygen; + struct pke_cmd_eddsa_verify cmd_eddsa_verify; + struct pke_cmd_eddsa_sign cmd_eddsa_sign; + struct pke_cmd_eddsa_pubgen cmd_eddsa_pubgen; + struct pke_cmd_ecdh_keygen cmd_ecdh_keygen; + struct pke_cmd_ecdh cmd_ecdh; + struct pke_cmd_rsa_enc cmd_rsa_enc; + struct pke_cmd_rsa_dec cmd_rsa_dec; + struct pke_cmd_rsa_crt_dec cmd_rsa_crt_dec; + struct pke_cmd_rsa_keygen cmd_rsa_keygen; + struct pke_cmd_eddsa_keygen_sca cmd_eddsa_keygen_sca; + struct pke_cmd_sm2_ecdh_keygen cmd_sm2_ecdh_keygen; + struct pke_cmd_sm2_ecdh cmd_sm2_ecdh; + struct pke_cmd_sm2_dec_point cmd_sm2_dec_point; + struct pke_cmd_sm2_enc_point cmd_sm2_enc_point; + struct pke_cmd_sm2_id_digest cmd_sm2_id_digest; + struct pke_cmd_sm2_ecdh_hash cmd_sm2_ecdh_hash; + struct pke_cmd_sm2_dec_hash cmd_sm2_dec_hash; + struct pke_cmd_sm2_enc_hash cmd_sm2_enc_hash; +}; + +#endif /* CMH_PKE_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_qse_abi.h b/drivers/crypto/cmh/include/cmh_qse_abi.h new file mode 100644 index 00000000000000..9834620e21d7b8 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_qse_abi.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- QSE Core ABI Definitions + * + * Kernel-side definitions for the CMH QSE ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_QSE_ABI_H +#define CMH_QSE_ABI_H + +#include +#include +#include + +/* VCQ layout: header + [SYS_NEW] + QSE_CMD + flush */ +#define QSE_VCQ_CMDS_MIN 3 /* header + cmd + flush */ +#define QSE_VCQ_CMDS_MAX 4 /* header + sys_new + cmd + flush */ + +/* QSE Flags */ +#define QSE_FLAG_USE_REF BIT(0) +#define QSE_FLAG_USE_RNG BIT(1) + +/* QSE Command IDs */ +#define QSE_CMD_ML_KEM_KEYGEN 0x01U +#define QSE_CMD_ML_KEM_ENC 0x02U +#define QSE_CMD_ML_KEM_DEC 0x03U +#define QSE_CMD_ML_DSA_KEYGEN 0x04U +#define QSE_CMD_ML_DSA_SIGN 0x05U +#define QSE_CMD_ML_DSA_VERIFY 0x06U +#define QSE_CMD_ML_KEM_KEYGEN_MASKED 0x07U +#define QSE_CMD_ML_KEM_ENC_MASKED 0x08U +#define QSE_CMD_ML_KEM_DEC_MASKED 0x09U +#define QSE_CMD_ML_DSA_KEYGEN_MASKED 0x0AU +#define QSE_CMD_ML_DSA_SIGN_MASKED 0x0BU + +/* ML-KEM category values */ +#define ML_KEM_K_512 2U +#define ML_KEM_K_768 3U +#define ML_KEM_K_1024 4U + +/* ML-DSA mode values */ +#define ML_DSA_MODE_44 2U +#define ML_DSA_MODE_65 3U +#define ML_DSA_MODE_87 5U + +/* ML-DSA special message length for externalMu (pre-hashed 64-byte input) */ +#define ML_DSA_MLEN_EXTERNAL_MU 0xFFFFFFFFU +#define ML_DSA_EXTMU_LEN 64U /* actual copy size for externalMu */ + +/* ML-DSA maximum message length */ +#define ML_DSA_MAX_MLEN 10240U + +/* Shared secret size */ +#define ML_KEM_SS_LEN 32U +#define ML_KEM_SS_LEN_MASKED 64U + +/* Seed sizes */ +#define QSE_SEED_LEN 32U +#define QSE_SEED_LEN_MASKED 64U + +/* + * ML-KEM size tables -- indexed by (k - 2). + * [0] = ML-KEM-512 (k=2) + * [1] = ML-KEM-768 (k=3) + * [2] = ML-KEM-1024 (k=4) + */ +#define ML_KEM_LEVELS 3U + +#define ML_KEM_EK_SIZE(k) (384U * (k) + 32U) +#define ML_KEM_DK_SIZE(k) (768U * (k) + 96U) +#define ML_KEM_DK_SIZE_MASKED(k) (1152U * (k) + 128U) + +static inline u32 ml_kem_ct_size(u32 k) +{ + u32 du = (k == 4U) ? 11U : 10U; + u32 dv = (k == 4U) ? 5U : 4U; + + return 32U * (k * du + dv); +} + +#define ML_KEM_CT_SIZE(k) ml_kem_ct_size(k) + +/* + * ML-DSA size tables -- indexed by mode. + * Mode values: 2 (ML-DSA-44), 3 (ML-DSA-65), 5 (ML-DSA-87). + */ +extern const u32 ml_dsa_pk_size[]; +extern const u32 ml_dsa_sk_size[]; +extern const u32 ml_dsa_sk_size_masked[]; +extern const u32 ml_dsa_sig_size[]; + +/* Map ML-DSA mode (2/3/5) -> table index (0/1/2) */ +static inline int ml_dsa_mode_idx(u32 mode) +{ + switch (mode) { + case 2: return 0; + case 3: return 1; + case 5: return 2; + default: return -1; + } +} + +/* Map ML-KEM k (2/3/4) -> table index (0/1/2), or -1 if invalid */ +static inline int ml_kem_k_idx(u32 k) +{ + if (k >= 2U && k <= 4U) + return (int)(k - 2U); + return -1; +} + +/* QSE Command Structures -- match CMH eSW ABI exactly */ + +struct qse_cmd_ml_kem_keygen { + u32 k; + u32 flags; + u64 seed; + u64 z; + u64 ek; + u64 dk; + u32 dk_type; +}; + +struct qse_cmd_ml_kem_enc { + u32 k; + u32 flags; + u64 coin; + u64 ek; + u64 ct; + u64 ss; + u32 ss_type; +}; + +struct qse_cmd_ml_kem_dec { + u32 k; + u32 flags; + u64 ct; + u64 dk; + u64 ss; + u32 ss_type; +}; + +struct qse_cmd_ml_dsa_keygen { + u32 mode; + u32 flags; + u64 seed; + u64 pk; + u64 sk; + u32 sk_type; +}; + +struct qse_cmd_ml_dsa_sign { + u32 mode; + u32 flags; + u64 rnd; + u64 m; + u64 sk; + u64 sig; + u32 mlen; +}; + +struct qse_cmd_ml_dsa_verify { + u32 mode; + u32 flags; + u64 m; + u64 pk; + u64 sig; + u32 mlen; +}; + +union qse_cmd { + struct qse_cmd_ml_kem_keygen cmd_ml_kem_keygen; + struct qse_cmd_ml_kem_enc cmd_ml_kem_enc; + struct qse_cmd_ml_kem_dec cmd_ml_kem_dec; + struct qse_cmd_ml_dsa_keygen cmd_ml_dsa_keygen; + struct qse_cmd_ml_dsa_sign cmd_ml_dsa_sign; + struct qse_cmd_ml_dsa_verify cmd_ml_dsa_verify; +}; + +#endif /* CMH_QSE_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_registers.h b/drivers/crypto/cmh/include/cmh_registers.h new file mode 100644 index 00000000000000..cba4dc0e8258fe --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_registers.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Hardware Register Definitions + * + * Derived from the CMH hardware register specification. + * All offsets are taken directly from the hardware documentation. + */ + +#ifndef CMH_REGISTERS_H +#define CMH_REGISTERS_H + +#include +#include + +/* MBX Instance Addressing */ + +#define CMH_MBX_INSTANCE_SHIFT 12 +#define CMH_MBX_INSTANCE_SIZE BIT(CMH_MBX_INSTANCE_SHIFT) /* 0x1000 */ +#define CMH_MAX_MBX_INSTANCES 64U + +/* MBX Per-Instance Register Offsets */ + +#define R_MBX_LOCK 0x000U +#define R_MBX_HOST_INFO 0x004U +#define R_MBX_QUEUE_LO 0x008U +#define R_MBX_QUEUE_HI 0x00CU +#define R_MBX_QUEUE_SLOTS 0x010U +#define R_MBX_QUEUE_STRIDE 0x014U +#define R_MBX_QUEUE_HEAD 0x018U +#define R_MBX_QUEUE_TAIL 0x01CU +#define R_MBX_INTERRUPT 0x020U +#define R_MBX_INTERRUPT_MASK 0x024U +#define R_MBX_COMMAND 0x028U +#define R_MBX_STATUS 0x02CU +#define R_MBX_CHILD 0x030U +#define R_MBX_ID 0x034U +#define R_MBX_HOST_CONFIG 0x038U +#define R_MBX_SCRATCH 0x03CU + +#define MBX_QUEUE_ALIGNMENT 0x4U + +/* MBX Interrupt Bits */ + +#define MBX_DONE_IRQ BIT(0) +#define MBX_ERROR_IRQ BIT(1) +#define MBX_IRQ_MASK (MBX_DONE_IRQ | MBX_ERROR_IRQ) + +/* MBX Command Values */ + +#define MBX_COMMAND_RUN 0x000U +#define MBX_COMMAND_PAUSE 0xC2FU +#define MBX_COMMAND_CONTINUE 0x5DBU +#define MBX_COMMAND_RESTART 0xB78U +#define MBX_COMMAND_ABORT 0x6F6U +#define MBX_COMMAND_FLUSH 0x3A5U + +/* MBX Status Values */ + +#define MBX_STATUS_IDLE 0x01U +#define MBX_STATUS_BUSY 0x10U +#define MBX_STATUS_HOLD 0x20U +#define MBX_STATUS_PAUSED 0x28U +#define MBX_STATUS_SUCCESS 0x40U +#define MBX_STATUS_ERROR 0x80U +#define MBX_STATUS_OFFLINE 0x88U /* ERROR | 0x08: offline/stopped */ + +#define MBX_MASK_DONE (MBX_STATUS_IDLE | MBX_STATUS_SUCCESS) +#define MBX_MASK_RUNNING (MBX_STATUS_BUSY | MBX_STATUS_HOLD) +#define MBX_MASK_STOPPED MBX_STATUS_OFFLINE + +/* MBX Status Field Extraction */ + +#define MBX_STATUS_CODE(v) ((v) & 0xFFU) +#define MBX_STATUS_CORE_ID(v) (((v) >> 8) & 0xFFU) +#define MBX_STATUS_ERROR_CODE(v) (((v) >> 16) & 0xFFU) +#define MBX_STATUS_CMD_INDEX(v) (((v) >> 24) & 0xFFU) + +/* SIC Register Offsets (relative to SIC base / instance 0 base) */ + +#define R_SIC_BOOT_STATUS 0x100U +#define SIC_BOOT_STATUS_MASK 0x77U +#define SIC_BOOT_STATUS_PASS 0x66U + +#define R_SIC_MBX_AVAILABILITY 0x104U +#define R_SIC_MBX_AVAILABILITY2 0x108U + +#define R_SIC_SW_BOOT_STATUS 0x12CU +#define SIC_SW_BOOT_STATUS_STARTED BIT(0) +#define SIC_SW_BOOT_STATUS_READY BIT(1) +#define SIC_SW_BOOT_STATUS_MISSION BIT(6) +#define SIC_SW_BOOT_STATUS_MISSION2 BIT(7) + +#define R_SIC_SW_ERROR_INFO 0x130U +#define R_SIC_SW_HEARTBEAT 0x154U + +#define R_SIC_GPINTERRUPT 0x160U + +#define R_SIC_HW_VERSION0 0x200U +#define R_SIC_SW_VERSION 0x218U +#define R_SIC_CORE_ENABLE 0x22CU + +/* Register Access Helpers */ + +static inline u32 cmh_reg_read32(void __iomem *base, u32 offset) +{ + return ioread32((u8 __iomem *)base + offset); +} + +static inline void cmh_reg_write32(u32 value, void __iomem *base, u32 offset) +{ + iowrite32(value, (u8 __iomem *)base + offset); +} + +/* + * 64-bit register access via two 32-bit reads/writes. Only correct for + * register pairs where split access is defined (e.g. QUEUE_LO/HI). + * Do not use for registers requiring atomic 64-bit access. + * + * No explicit barrier between the two halves is needed: ioread32/iowrite32 + * include implicit ordering guarantees on all supported architectures + * (MMIO accessors are strongly ordered with respect to each other). + */ +static inline u64 cmh_reg_read64(void __iomem *base, u32 offset) +{ + u32 lo = ioread32((u8 __iomem *)base + offset); + u32 hi = ioread32((u8 __iomem *)base + offset + 4); + + return ((u64)hi << 32) | lo; +} + +static inline void cmh_reg_write64(u64 value, void __iomem *base, u32 offset) +{ + iowrite32((u32)value, (u8 __iomem *)base + offset); + iowrite32((u32)(value >> 32), (u8 __iomem *)base + offset + 4); +} + +/* Return the ioremap'd base for MBX instance N within the SIC region */ +static inline void __iomem *cmh_mbx_instance_base(void __iomem *sic_mapped, + u32 instance) +{ + return (u8 __iomem *)sic_mapped + + ((unsigned long)instance << CMH_MBX_INSTANCE_SHIFT); +} + +#endif /* CMH_REGISTERS_H */ diff --git a/drivers/crypto/cmh/include/cmh_rh.h b/drivers/crypto/cmh/include/cmh_rh.h new file mode 100644 index 00000000000000..b182c203a4754b --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_rh.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Response Handler + * + * IRQ-driven completion processing. Uses request_threaded_irq(): + * - Hardirq: read+clear MBX interrupt registers, wake thread + * - Threaded handler: walk per-MBX transaction queues, + * fire completion callbacks, free transaction objects + * + * The Response Handler consumes transaction_obj entries enqueued + * by the Transaction Manager (cmh_txn.c) on each per-mailbox txq. + */ + +#ifndef CMH_RH_H +#define CMH_RH_H + +#include "cmh_config.h" + +/** + * cmh_rh_init() - Register IRQ handler and start response processing + * @cfg: Global device configuration + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_rh_init(struct cmh_config *cfg); + +/** + * cmh_rh_cleanup() - Free IRQ and stop response processing + * @cfg: Global device configuration + */ +void cmh_rh_cleanup(struct cmh_config *cfg); + +/** + * cmh_rh_suspend() - Quiesce RH for system suspend + * @cfg: Global device configuration + * + * Cancels the watchdog timer and masks MBX interrupts at the hardware + * level. IRQ handlers remain registered (standard PM pattern). + * The threaded IRQ handler stays active so that cmh_tm_quiesce() + * (called after this) can still drain in-flight transactions via + * IRQ-driven completions. + */ +void cmh_rh_suspend(struct cmh_config *cfg); + +/** + * cmh_rh_resume() - Restart RH after system resume + * @cfg: Global device configuration + * + * Re-synchronises per-MBX head tracking with hardware, clears stale + * interrupt bits, re-enables MBX interrupt masks, and re-arms the + * watchdog timer. Must be called before cmh_tm_resume(). + */ +void cmh_rh_resume(struct cmh_config *cfg); + +/* debugfs timeout accessor (debug builds only) */ +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +unsigned int *cmh_rh_timeout_watchdog_ptr(void); +#endif + +/** + * cmh_rh_force_drain_mbx() - FLUSH + drain all pending transactions on a MBX + * @mbx_idx: Mailbox index to drain + * + * Issues MBX_COMMAND_FLUSH, drains all pending transactions with + * -ECANCELED, and resets all recovery bookkeeping (including the + * wedged flag). Safe to call at any time; acquires rh_process_lock. + * Intended for debugfs last-resort recovery. + */ +void cmh_rh_force_drain_mbx(u32 mbx_idx); + +/** + * cmh_rh_mbx_is_wedged() - Check if a mailbox is permanently wedged + * @mbx_idx: Mailbox index to check + * + * Returns true if the mailbox has failed RESTART+FLUSH recovery and + * is offline. Used by the TM to avoid submitting new work to a dead + * mailbox. + * + * Return: true if wedged, false otherwise (including out-of-range idx). + */ +bool cmh_rh_mbx_is_wedged(u32 mbx_idx); + +/** + * cmh_rh_abort_mbx() - Issue MBX_COMMAND_ABORT under rh_process_lock + * @mbx_idx: Mailbox index to abort + * + * Serialises the ABORT write with RESTART/FLUSH commands issued by the + * watchdog, preventing command-register clobber races. + */ +void cmh_rh_abort_mbx(u32 mbx_idx); + +#endif /* CMH_RH_H */ diff --git a/drivers/crypto/cmh/include/cmh_rng.h b/drivers/crypto/cmh/include/cmh_rng.h new file mode 100644 index 00000000000000..1a886a0d82c1ee --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_rng.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Hardware RNG (DRBG) Driver + * + * Registers a struct hwrng backed by the CMH DRBG core. + * Each .read() builds a VCQ with DRBG_CMD_GENERATE and submits it + * through the Transaction Manager for synchronous completion. + * + * The DRBG must be configured (CONFIG command) by the management host + * before the LKM is loaded -- the LKM only issues GENERATE requests. + * + * CRNG seeding control: + * - Module param "hwrng_quality" (0=disabled, 1-1024=enable) + * - Default: 0 (conservative -- no automatic kernel CRNG seeding) + */ + +#ifndef CMH_RNG_H +#define CMH_RNG_H + +struct platform_device; + +int cmh_rng_register(struct platform_device *pdev); +void cmh_rng_unregister(void); + +/* debugfs timeout accessor (debug builds only) */ +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +unsigned int *cmh_rng_timeout_drbg_ptr(void); +#endif + +#endif /* CMH_RNG_H */ diff --git a/drivers/crypto/cmh/include/cmh_sm3_abi.h b/drivers/crypto/cmh/include/cmh_sm3_abi.h new file mode 100644 index 00000000000000..cbbe80fe18d66d --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sm3_abi.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM3 Hash Core ABI Definitions + * + * Kernel-side definitions for the CMH SM3 ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_SM3_ABI_H +#define CMH_SM3_ABI_H + +#include + +/* SM3 Commands */ + +#define SM3_CMD_INIT 0x01U +#define SM3_CMD_UPDATE 0x02U +#define SM3_CMD_FINAL 0x03U +#define SM3_CMD_UPDATE2D 0x04U +#define SM3_CMD_GATHER 0x06U +#define SM3_CMD_SAVE 0x07U +#define SM3_CMD_RESTORE 0x08U + +/* SM3 Digest / Block Sizes */ + +#define CMH_SM3_DIGEST_SIZE 32U +#define CMH_SM3_BLOCK_SIZE 64U + +/* SM3 Context (for SAVE/RESTORE) */ + +#define SM3_CONTEXT_WORDS 29U +#define SM3_CONTEXT_SIZE (SM3_CONTEXT_WORDS * 4 + 4) /* ctx[29] + crc */ + +/* SM3 Command Structures */ + +struct sm3_cmd_update { + u64 input; /* DMA physical address of input data */ + u32 inlen; /* input data length in bytes */ +}; + +struct sm3_cmd_final { + u64 digest; /* DMA physical address for output digest */ + u32 outlen; /* digest length in bytes */ +}; + +struct sm3_cmd_update2d { + u64 input; /* DMA source address for input data */ + u64 output; /* DMA destination address for pass-through data */ + u32 iolen; /* input/pass-through data length in bytes */ +}; + +struct sm3_cmd_gather { + u64 lista; /* DMA address of dma_scattergather_item chain */ + u32 sgcmd; /* SM3 sub-command: SM3_CMD_UPDATE or SM3_CMD_UPDATE2D */ +}; + +struct sm3_cmd_save { + u64 output; /* DMA physical address for saved context */ + u32 outlen; /* must be SM3_CONTEXT_SIZE */ +}; + +struct sm3_cmd_restore { + u64 input; /* DMA physical address of saved context */ + u32 inlen; /* must be SM3_CONTEXT_SIZE */ +}; + +/* SM3 Command Union */ + +union sm3_cmd { + struct sm3_cmd_update cmd_update; + struct sm3_cmd_final cmd_final; + struct sm3_cmd_update2d cmd_update2d; + struct sm3_cmd_gather cmd_gather; + struct sm3_cmd_save cmd_save; + struct sm3_cmd_restore cmd_restore; +}; + +#endif /* CMH_SM3_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_sm4_abi.h b/drivers/crypto/cmh/include/cmh_sm4_abi.h new file mode 100644 index 00000000000000..a34faea613dc67 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sm4_abi.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM4 Core ABI Definitions + * + * Kernel-side definitions for the CMH SM4 ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_SM4_ABI_H +#define CMH_SM4_ABI_H + +#include + +/* SM4 Block Size */ + +#define CMH_SM4_BLOCK_SIZE 16U +#define CMH_SM4_IV_SIZE 16U +#define CMH_SM4_KEY_SIZE 16U /* SM4 always uses 128-bit keys */ + +/* SM4 Modes (per CMH SM4 ABI) */ + +#define SM4_MODE_ECB 1U +#define SM4_MODE_CBC 2U +#define SM4_MODE_CTR 3U +#define SM4_MODE_CFB 5U +#define SM4_MODE_GCM 6U +#define SM4_MODE_CMAC 7U +#define SM4_MODE_CCM 8U +#define SM4_MODE_XTS 9U +#define SM4_MODE_XCBC 10U + +/* SM4 Operations (per CMH SM4 ABI) */ + +#define SM4_OP_DECRYPT 1U +#define SM4_OP_ENCRYPT 2U + +/* SM4 Command IDs */ + +#define SM4_CMD_INIT 0x01U +#define SM4_CMD_AAD_UPDATE 0x02U +#define SM4_CMD_AAD_FINAL 0x03U +#define SM4_CMD_UPDATE 0x04U +#define SM4_CMD_FINAL 0x05U +#define SM4_CMD_SCATTERGATHER 0x06U +#define SM4_CMD_CCM_INIT 0x09U + +/* SM4 Command Structures */ + +struct sm4_cmd_init { + u64 key; /* datastore reference for the key */ + u64 iv; /* DMA address of the IV */ + u32 keylen; /* key length in bytes (16, or 32 for XTS) */ + u32 ivlen; /* IV length in bytes (0..16) */ + u32 mode; /* SM4 mode (SM4_MODE_*) */ + u32 op; /* SM4 operation (SM4_OP_*) */ + u32 aadlen; /* AAD length or 0 */ + u32 iolen; /* plaintext/ciphertext length */ +}; + +struct sm4_cmd_update { + u64 input; /* DMA address of input data */ + u64 output; /* DMA address of output data */ + u32 iolen; /* input/output data length */ +}; + +struct sm4_cmd_final { + u64 input; /* DMA address of last input data */ + u64 output; /* DMA address of last output data */ + u64 tag; /* DMA address of tag (AEAD only) */ + u32 iolen; /* last input/output data length */ + u32 taglen; /* tag length (AEAD only) */ +}; + +struct sm4_cmd_aad_final { + u64 data; /* DMA address of AAD data */ + u32 datalen; /* AAD data length */ +}; + +struct sm4_cmd_ccm_init { + u64 key; /* datastore reference for the key */ + u64 nonce; /* DMA address of the nonce */ + u32 keylen; /* key length in bytes (always 16) */ + u32 noncelen; /* nonce length (15 - L) */ + u32 op; /* SM4 operation (SM4_OP_*) */ + u32 aadlen; /* AAD length */ + u32 iolen; /* plaintext/ciphertext length */ + u32 taglen; /* tag length */ +}; + +/* SM4 Command Union */ + +union sm4_cmd { + struct sm4_cmd_init cmd_init; + struct sm4_cmd_update cmd_update; + struct sm4_cmd_final cmd_final; + struct sm4_cmd_aad_final cmd_aad_final; + struct sm4_cmd_ccm_init cmd_ccm_init; +}; + +#endif /* CMH_SM4_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_sys_abi.h b/drivers/crypto/cmh/include/cmh_sys_abi.h new file mode 100644 index 00000000000000..64110311e5520b --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sys_abi.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SYS Core ABI Definitions + * + * Kernel-side definitions for the CMH SYS ABI. + * All constants and layouts derived from the CMH eSW ABI. + */ + +#ifndef CMH_SYS_ABI_H +#define CMH_SYS_ABI_H + +#include +#include + +/* SYS Commands (per CMH SYS ABI) */ + +#define SYS_CMD_RUN 0x01U +#define SYS_CMD_NOP 0x02U +#define SYS_CMD_IMPORT 0x07U +#define SYS_CMD_EXPORT 0x08U +#define SYS_CMD_NEW 0x0AU +#define SYS_CMD_READ 0x0BU +#define SYS_CMD_WRITE 0x0CU +#define SYS_CMD_GRANT 0x0DU +#define SYS_CMD_LIST 0x0EU +#define SYS_CMD_FIND 0x0FU +#define SYS_CMD_DATA 0x11U + +/* SYS Reference Constants */ + +#define SYS_REF_NONE 0x0000000000000000ULL +#define SYS_REF_TEMP 0x1111111111111111ULL +#define SYS_REF_LAST 0xFFFFFFFFFFFFFFFFULL + +typedef u64 sys_ref_t; + +/* SYS CID */ + +#define SYS_CID_NONE 0x0000000000000000ULL + +/* SYS Type Encoding -- bits [7:0] = core_id, bits [23:16] = flags */ + +#define SYS_TYPE_FLAG_PT BIT(16) /* can be read as plaintext */ +#define SYS_TYPE_FLAG_XC BIT(17) /* can be exported over XC bus */ +#define SYS_TYPE_FLAG_SCA BIT(18) /* SCA key in 2 shares */ + +#define SYS_TYPE_SET(flags, core) \ + (((flags) & 0xFF0000UL) | ((core) & 0xFFUL)) +#define SYS_TYPE_CORE(type) ((type) & 0xFFU) +#define SYS_TYPE_FLAGS(type) ((type) & 0xFF0000U) +#define SYS_TYPE_NONE 0U /* DMA output, no DS storage */ + +#define SYS_WRAP_HDR_SIZE 16 /* sys_read plaintext header */ + +/* SYS Command Structures */ + +struct sys_cmd_new { + u64 cid; /* caller id (name) for the object */ + u64 ref; /* DMA address -- CMH eSW writes back reference here */ + u32 len; /* size of the new object in bytes */ +}; + +struct sys_cmd_write { + u64 ref; /* object datastore reference */ + u64 src; /* DMA source address of key data */ + u64 key; /* wrapping key reference (SYS_REF_NONE = plaintext) */ + u32 len; /* source buffer length */ + u32 type; /* SYS_TYPE_SET(flags, core_id) */ +}; + +struct sys_cmd_read { + u64 ref; /* object datastore reference */ + u64 dst; /* DMA destination for key data */ + u64 key; /* wrapping key reference (SYS_REF_NONE = plaintext) */ + u32 len; /* destination buffer length */ +}; + +struct sys_cmd_data { + u64 ref; /* object datastore reference */ + u64 dst; /* DMA destination for object data */ + u32 len; /* destination buffer length */ +}; + +struct sys_cmd_find { + u64 cid; /* caller id to search for */ + u64 dst; /* DMA destination for struct sys_list_item */ + u32 len; /* destination buffer length */ +}; + +struct sys_cmd_list { + u64 ref; /* starting DS reference (SYS_REF_NONE = first) */ + u64 dst; /* DMA destination for struct sys_list_item */ + u32 len; /* destination buffer length */ +}; + +struct sys_cmd_grant { + u64 ref; /* object datastore reference */ + u64 read; /* bitfield: allow read for mailboxes */ + u64 write; /* bitfield: allow write for mailboxes */ + u64 execute; /* bitfield: allow use for mailboxes */ +}; + +struct sys_cmd_export { + u64 cid; /* caller id for the response */ + u64 dst; /* DMA destination for the export blob */ + u64 key; /* wrapping key datastore reference */ + u32 len; /* destination buffer length */ +}; + +struct sys_cmd_import { + u64 src; /* DMA source address of import blob */ + u64 key; /* wrapping key datastore reference */ + u32 len; /* source buffer length */ +}; + +/* SYS List/Find Response Item */ + +struct sys_list_item { + u64 ref; /* object datastore reference */ + u64 cid; /* caller id */ + u32 len; /* object length */ + u32 type; /* object type (SYS_TYPE_SET packed) */ +}; + +/* Wrapped-read header (prepended to SYS_CMD_READ responses) */ + +struct sys_wrap_hdr { + u64 cid; /* caller id */ + u32 wrap; /* wrap data length following this header */ + u32 len; /* object data length following wrap data */ +}; + +/* SYS Command Union */ + +union sys_cmd { + struct sys_cmd_new cmd_new; + struct sys_cmd_write cmd_write; + struct sys_cmd_read cmd_read; + struct sys_cmd_data cmd_data; + struct sys_cmd_find cmd_find; + struct sys_cmd_list cmd_list; + struct sys_cmd_grant cmd_grant; + struct sys_cmd_export cmd_export; + struct sys_cmd_import cmd_import; +}; + +#endif /* CMH_SYS_ABI_H */ diff --git a/drivers/crypto/cmh/include/cmh_sysfs.h b/drivers/crypto/cmh/include/cmh_sysfs.h new file mode 100644 index 00000000000000..864cf1c8fa0021 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sysfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- sysfs Device Attributes + */ + +#ifndef CMH_SYSFS_H +#define CMH_SYSFS_H + +struct attribute_group; + +extern const struct attribute_group *cmh_sysfs_groups[]; + +#endif /* CMH_SYSFS_H */ diff --git a/drivers/crypto/cmh/include/cmh_txn.h b/drivers/crypto/cmh/include/cmh_txn.h new file mode 100644 index 00000000000000..6131f0b2224f57 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_txn.h @@ -0,0 +1,463 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Transaction Manager + * + * Dedicated kthread managing concurrent VCQ submissions. + * + * Callers post command_msg objects into the Command Message Queue (CMQ). + * The TM thread dequeues them, selects a mailbox, builds VCQ(s) in the + * DMA queue slot, creates a transaction_obj, and rings the doorbell. + * + * The Response Handler (cmh_rh.c) walks per-mailbox transaction queues + * when an IRQ fires and fires completion callbacks. + */ + +#ifndef CMH_TXN_H +#define CMH_TXN_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_config.h" +#include "cmh_vcq.h" + +/* Command Message (caller -> TM) */ + +typedef void (*cmh_completion_fn)(void *data, int error); + +struct command_msg { + struct list_head list; /* CMQ linked list node */ + u32 command_id; /* VCQ_CMD_ID(core, flags, span, cmd) */ + void *vcq_data; /* heap-owned copy of VCQ entries */ + u32 vcq_count; /* total vcq_cmd entries across all VCQs */ + u32 num_vcqs; /* how many VCQs in vcq_data (0 or 1 = single) */ + s32 target_mbx; /* MBX index from core affinity, or -1 fallback */ + s32 actual_mbx; /* MBX selected by TM thread, -1 until dispatched */ + cmh_completion_fn complete; /* completion callback (may be NULL) */ + void *completion_data; + refcount_t refs; /* submit_sync: 2 = waiter + TM */ + bool backlog_ok; /* accept into backlog when CMQ is full */ + unsigned long timeout_jiffies;/* per-txn async timeout (0 = none) */ +}; + +/* Transaction Object (TM -> RH) */ + +/* Per-transaction FSM states for async timeout resolution */ +#define TXN_INFLIGHT 0 +#define TXN_COMPLETE 1 +#define TXN_TIMED_OUT 2 + +struct transaction_obj { + struct list_head list; /* per-mailbox txn queue node */ + u32 first_vcq_id; + u32 last_vcq_id; + u32 mailbox_idx; /* index into cfg->mailboxes[] */ + u32 command_id; /* VCQ_CMD_ID from first payload cmd */ + int error_code; + cmh_completion_fn complete; + void *completion_data; + atomic_t state; /* TXN_INFLIGHT / COMPLETE / TIMED_OUT */ + struct timer_list timeout_timer; /* per-request async timeout */ + refcount_t refs; /* owner + timer (if armed) */ +}; + +/* Per-Mailbox Transaction Queue */ + +struct cmh_mbx_txq { + struct list_head head; + spinlock_t lock; /* protects head list + depth */ + u32 depth; /* number of in-flight transactions */ + struct mutex dispatch_lock; /* serialises VCQ dispatch + MBX flush */ +}; + +/* Public Interface */ + +/** + * cmh_tm_init() - Initialise the Transaction Manager + * @cfg: Global device configuration (mailbox layout, IRQ, etc.) + * + * Starts the TM kthread and initialises per-mailbox transaction queues. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_tm_init(struct cmh_config *cfg); + +/** + * cmh_tm_cleanup() - Stop the TM kthread and drain all queues + */ +void cmh_tm_cleanup(void); + +/** + * cmh_tm_quiesce() - Stop TM kthread and drain in-flight transactions + * + * Stops the TM kthread, rejects new posts, then waits (with a + * configurable timeout) for all per-MBX transaction queues to drain. + * If the timeout fires, remaining transactions are cancelled with + * -ECANCELED. + */ +void cmh_tm_quiesce(void); + +/** + * cmh_tm_resume() - Restart the TM kthread after resume + * + * Return: 0 on success, negative errno if the kthread fails to start. + */ +int cmh_tm_resume(void); + +/** + * cmh_tm_post_command() - Post a command to the TM for submission + * @msg: Command message with pre-built VCQ data and completion callback + * + * Round-robin selects the next MBX with enough free slots for + * msg->num_vcqs VCQs. All VCQs in a message are written to + * consecutive slots on the same MBX (back-to-back). + * The caller retains ownership of @msg until the completion callback fires. + * + * Return: 0 on success, -EAGAIN if queue full, -ENODEV if TM stopped. + */ +int cmh_tm_post_command(struct command_msg *msg); + +/* + * Synchronous submit -- post one or more VCQs and wait for completion. + * + * Combines post_command + refcounted wait + timeout + cancel into one + * call. This is the standard pattern for all synchronous crypto ops. + * + * Context: must be called from a sleepable (task) context. + * Performs GFP_KERNEL allocations and sleeps on + * wait_for_completion_timeout(). A WARN_ON_ONCE fires + * if called from atomic / IRQ / softirq context. + * + * vcq_cmds: pre-built VCQ array (headers + commands, contiguous) + * vcq_count: total number of vcq_cmd entries across all VCQs + * num_vcqs: number of VCQs in the array (0 or 1 = single VCQ) + * + * For multi-VCQ submissions, the array contains multiple VCQs laid + * out contiguously, each starting with its own header. All VCQs are + * written to consecutive MBX slots and share one transaction object. + * + * Returns 0 on success, -ETIMEDOUT, or CMH eSW error code. + */ +int cmh_tm_submit_sync(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs); + +/* + * Synchronous submit pinned to a specific mailbox. + * target_mbx: -1 = round-robin, >= 0 = pin to that MBX index. + */ +int cmh_tm_submit_sync_mbx(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, s32 target_mbx); + +/* + * Synchronous submit with explicit timeout. + * timeout_hz: completion timeout in jiffies (use msecs_to_jiffies()). + */ + +/* + * Extended timeout for slow crypto operations: RSA keygen, PQC + * keygen/sign/verify. Controlled by the slow_op_timeout_ms module + * parameter. + */ +unsigned long cmh_tm_slow_op_timeout_jiffies(void); + +int cmh_tm_submit_sync_tmo(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, s32 target_mbx, + unsigned long timeout_hz); + +/* + * Synchronous submit that never issues MBX_COMMAND_ABORT on timeout. + * Returns -EAGAIN if cancelled from queue, -EINPROGRESS if the VCQ is + * left in-flight. On -EINPROGRESS, @orphan_cb(@orphan_data) will be + * called when the VCQ eventually completes (RH callback fires and the + * last sync_ctx ref drops). Use this to defer DMA cleanup. + * Safe for background/kthread callers that must not disrupt other MBX work. + */ +int cmh_tm_submit_sync_noabort(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, unsigned long timeout_hz, + void (*orphan_cb)(void *), + void *orphan_data); + +/* + * Asynchronous submit -- post VCQs and return immediately. + * + * On successful return (0), the provided @callback may be invoked from + * either the RH threaded IRQ context (normal completion path) or the TM + * kthread (if VCQ dispatch to the HW ring fails after the message was + * posted to the CMQ). The caller must not assume a specific callback + * context. + * + * After a successful post, the caller must NOT touch VCQ buffers -- + * ownership transfers to the TM. If this function returns non-zero, + * the message was not posted, the callback will NOT fire, and the caller + * must perform cleanup. + * + * Uses GFP_ATOMIC internally -- the crypto API may invoke driver ops + * from softirq context (e.g. IPsec), so GFP_KERNEL would deadlock. + * + * If @backlog_ok is true and the CMQ is full, the message is placed on + * an overflow backlog queue and -EBUSY is returned. The caller must + * treat -EBUSY as "accepted" (like -EINPROGRESS): the callback WILL + * fire once the request is promoted from backlog and completes. When + * @backlog_ok is false, CMQ-full returns -EAGAIN (caller must clean up). + * + * Returns: 0 on successful post, -EBUSY (backlogged -- callback will + * fire), -ENOMEM, -EINVAL (bad vcq_count), -EAGAIN (CMQ full, + * no backlog), -ENODEV. + */ +int cmh_tm_submit_async(struct vcq_cmd *vcq_cmds, u32 vcq_count, + u32 num_vcqs, s32 target_mbx, + cmh_completion_fn callback, void *callback_data, + bool backlog_ok, unsigned long timeout_jiffies); + +/** + * cmh_tm_async_timeout_jiffies() - Default per-request async timeout + * + * Returns the debugfs-configurable timeout for symmetric data-path + * ops (async_timeout_ms converted to jiffies). Akcipher/kpp callers + * should pass 0 instead (no per-request timeout; vcq_timeout_ms is the + * safety net). + */ +unsigned long cmh_tm_async_timeout_jiffies(void); + +/** + * cmh_tm_flush_mbx() - Issue MBX_COMMAND_FLUSH and wait for completion + * @mbx_idx: Mailbox index + * + * Resets the eSW child mailbox state including the temp stack. + * Must be called when no VCQ submission is in progress on @mbx_idx. + * + * Return: 0 on success, -ETIMEDOUT if eSW does not clear the command, + * -EBUSY if a command is already pending. + */ +int cmh_tm_flush_mbx(s32 mbx_idx); + +/** + * cmh_tm_try_cancel_command() - Try to cancel a queued command + * @msg: Command message to cancel + * + * Return: true if removed from CMQ, false if already consumed by the TM thread. + */ +bool cmh_tm_try_cancel_command(struct command_msg *msg); + +/** + * cmh_tm_peek_transaction() - Peek at the oldest transaction on a mailbox + * @mbx_idx: Mailbox index + * + * For use by the Response Handler. Caller must hold txq->lock or call + * from a context where no concurrent pop is possible (e.g. threaded IRQ). + * + * Return: Pointer to the oldest transaction_obj, or NULL if empty. + */ +struct transaction_obj *cmh_tm_peek_transaction(u32 mbx_idx); + +/** + * cmh_tm_pop_transaction() - Remove and return the oldest transaction + * @mbx_idx: Mailbox index + * + * Return: Pointer to the removed transaction_obj, or NULL if empty. + */ +struct transaction_obj *cmh_tm_pop_transaction(u32 mbx_idx); + +/** + * cmh_txn_finish() - Complete a transaction with FSM + timer handling + * @txn: Transaction popped from the TXQ + * @error: Error code (0 for success, negative errno) + * + * Resolves the timer-vs-completion race via atomic cmpxchg, cancels + * the per-txn timeout timer if still pending, fires the completion + * callback (if this path wins the race), and drops the owner reference. + * The transaction is freed when the last reference is dropped. + * + * Called by the Response Handler after popping a completed transaction. + */ +void cmh_txn_finish(struct transaction_obj *txn, int error); + +/** + * cmh_tm_max_cmds_per_vcq() - Max vcq_cmd entries per MBX slot + * + * Returns the minimum across all configured MBXes so callers can pack + * VCQs without knowing which MBX will be selected. + * + * Return: At least MIN_VCQ_CMDS (2). + */ +u32 cmh_tm_max_cmds_per_vcq(void); + +/** + * cmh_tm_mbx_count() - Return the number of configured mailboxes + * + * Return: cfg->mbx_count. + */ +u32 cmh_tm_mbx_count(void); + +/** + * cmh_core_default_id() - Return the default core_id for a core type + * @type: Logical core type enum + * + * Returns the core_id of the first (index-0) instance without advancing + * the round-robin counter. Intended for callers pinned to a fixed MBX + * (e.g. mgmt ioctls on MGMT_MBX) that only need the VCQ core_id field. + * + * In multi-instance configurations the returned core_id is always that + * of instance[0], regardless of which MBX instance[0] is assigned to. + * Mgmt callers submit on MGMT_MBX (0) -- the eSW accepts any valid + * core_id on any MBX for command dispatch. + * + * Return: u32 core_id. + */ +u32 cmh_core_default_id(enum cmh_core_type type); + +/** + * cmh_core_select_instance() - Multi-instance core dispatch selection + * @type: Logical core type enum + * + * Returns the next (core_id, mbx_idx) pair for @type using round-robin + * across configured instances. On first use for an instance whose MBX + * is not pre-assigned, atomically assigns the next available MBX. + * + * With single-instance defaults, this degenerates to the same behaviour + * as the old single-entry core_to_mbx[] table -- one core type, one MBX. + * + * Return: struct core_dispatch with core_id and mbx_idx. + */ +struct core_dispatch cmh_core_select_instance(enum cmh_core_type type); + +/** + * cmh_core_num_instances() - Return count of configured instances + * @type: Logical core type enum + * + * Return: Number of instances (>= 1) for @type. + */ +u32 cmh_core_num_instances(enum cmh_core_type type); + +/** + * cmh_core_get_instance() - Get a specific instance by index + * @type: Logical core type enum + * @idx: Instance index (0-based, must be < cmh_core_num_instances()) + * + * Returns (core_id, mbx_idx) for the given instance without advancing + * the round-robin counter. Triggers auto-assign if the instance has + * no MBX yet. + * + * Return: struct core_dispatch with core_id and mbx_idx. + */ +struct core_dispatch cmh_core_get_instance(enum cmh_core_type type, u32 idx); + +/** + * cmh_tm_affinity_reset() - Reset all core-to-MBX assignments + * + * Called during init and cleanup. + */ +void cmh_tm_affinity_reset(void); + +/** + * cmh_tm_txq_completion_notify() - Wake TM thread after TXQ completion + * + * Called by the Response Handler after completing a transaction to + * unblock the TM thread if it is waiting for a free MBX slot. + */ +void cmh_tm_txq_completion_notify(void); + +/* + * Pack @count payload commands (no headers) into one or more VCQs + * respecting the per-slot size limit, then submit synchronously. + * + * @payload: flat array of vcq_cmd entries (no headers) + * @count: number of entries in @payload + * @packed: caller-provided scratch buffer for the packed output + * @max_packed: size of @packed in vcq_cmd entries + * @target_mbx: -1 = round-robin, >= 0 = pin to this MBX index + * + * Each VCQ gets its own header. All VCQs are submitted as a single + * back-to-back transaction on the same MBX. + */ +int cmh_vcq_pack_and_submit(const struct vcq_cmd *payload, u32 count, + struct vcq_cmd *packed, u32 max_packed, + s32 target_mbx); + +/** + * cmh_vcq_pack_and_submit_async() - Pack payload commands and submit async + * @payload: Flat array of VCQ command entries (no headers) + * @count: Number of entries in @payload + * @packed: Caller-provided scratch buffer for packed output + * @max_packed: Size of @packed in vcq_cmd entries + * @target_mbx: Mailbox index (-1 for round-robin) + * @callback: Completion callback + * @callback_data: Opaque data passed to @callback + * @backlog_ok: If true, accept into backlog when CMQ is full + * @timeout_jiffies: Per-request timeout (0 to disable) + * + * Async variant of cmh_vcq_pack_and_submit(). Returns 0 on successful + * post; after a successful post, @callback may run from RH threaded IRQ + * context on normal completion, from the TM kthread if VCQ dispatch + * fails after posting, or from TM teardown paths such as + * cmh_tm_cleanup() / cmh_tm_quiesce() when queued or in-flight work is + * cancelled. Callers must not assume a single callback context. On + * non-zero return, the callback will NOT fire. + * + * @payload: flat array of vcq_cmd entries (no headers) + * @count: number of entries in @payload + * @packed: caller-provided scratch buffer for the packed output + * @max_packed: size of @packed in vcq_cmd entries + * @target_mbx: -1 = round-robin, >= 0 = pin to this MBX index + * @callback: completion callback (may run from IRQ or TM context) + * @callback_data: opaque pointer passed to @callback + * @backlog_ok: if true, queue the request when all MBXs are busy + * @timeout_jiffies: maximum wait time for MBX slot (0 = no wait) + * + * Return: 0 on successful post, -EBUSY (backlogged), negative errno on failure. + */ +int cmh_vcq_pack_and_submit_async(const struct vcq_cmd *payload, u32 count, + struct vcq_cmd *packed, u32 max_packed, + s32 target_mbx, + cmh_completion_fn callback, + void *callback_data, + bool backlog_ok, + unsigned long timeout_jiffies); + +/* debugfs timeout accessors (debug builds only) */ +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +unsigned int *cmh_tm_timeout_async_ptr(void); +unsigned int *cmh_tm_timeout_vcq_ptr(void); +unsigned int *cmh_tm_timeout_slow_op_ptr(void); +unsigned int *cmh_tm_timeout_drain_ptr(void); +#endif + +/* -- Crypto request completion helper ---------------------------------- */ + +struct device *cmh_dev(void); + +/** + * cmh_complete() - Complete a crypto request with optional error logging + * @req: The async crypto request to complete + * @err: Error code (0 = success, -EINPROGRESS = backlog promotion signal) + * + * Logs a rate-limited diagnostic on genuine errors, then hands the + * request back to the crypto framework. -EINPROGRESS is excluded from + * logging -- it is the crypto API's backlog promotion notification, not + * an error. Centralizes error reporting so individual algorithm drivers + * do not need per-callback logging. + */ +static inline void cmh_complete(struct crypto_async_request *req, int err) +{ + if (err && err != -EINPROGRESS) { + /* + * For template instances (e.g. hmac(sha3-512-cmh)) the + * driver name will be the outer template's, not ours. + * Still useful for triage -- identifies the failing tfm. + */ + dev_dbg_ratelimited(cmh_dev(), "op error: alg=%s err=%d\n", + crypto_tfm_alg_driver_name(req->tfm), + err); + } + crypto_request_complete(req, err); +} + +#endif /* CMH_TXN_H */ diff --git a/drivers/crypto/cmh/include/cmh_vcq.h b/drivers/crypto/cmh/include/cmh_vcq.h new file mode 100644 index 00000000000000..a9d04635d819aa --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_vcq.h @@ -0,0 +1,283 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- VCQ (Virtual Command Queue) Definitions + * + * Kernel-side definitions for the CMH VCQ and DMA scatter-gather ABI, + * so the LKM can build VCQs without depending on CMH eSW headers. + * + * All constants and layouts are derived from the CMH eSW ABI. + * + * Per-core command definitions live in their own ABI headers (cmh_hc_abi.h, + * cmh_aes_abi.h, etc.) and are included here to form the hwc_cmd union. + */ + +#ifndef CMH_VCQ_H +#define CMH_VCQ_H + +#include +#include +#include +#include + +#include "cmh_hc_abi.h" +#include "cmh_sm3_abi.h" +#include "cmh_drbg_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_kic_abi.h" +#include "cmh_aes_abi.h" +#include "cmh_sm4_abi.h" +#include "cmh_ccp_abi.h" +#include "cmh_pke_abi.h" +#include "cmh_qse_abi.h" +#include "cmh_hcq_abi.h" +#include "cmh_eac_abi.h" + +/* VCQ Magic Numbers */ + +#define VCQ_HDR_MAGIC 0x01514356U /* 'V' 'C' 'Q' 0x01 */ +#define VCQ_CMD_MAGIC 0x01444D43U /* 'C' 'M' 'D' 0x01 */ + +/* VCQ Command ID Encoding */ + +#define VCQ_CMD_MASK 0x000000FFU +#define VCQ_SPAN_MASK 0x0000FF00U +#define VCQ_FLAG_MASK 0x00FF0000U +#define VCQ_CORE_MASK 0xFF000000U + +#define VCQ_CMD_ID(core, flags, span, cmd) \ + (((u32)(core) << 24) | ((flags) & VCQ_FLAG_MASK) | \ + (((u32)(span) << 8) & VCQ_SPAN_MASK) | ((cmd) & VCQ_CMD_MASK)) + +/* Core IDs (per CMH hardware specification) */ + +#define CORE_ID_SYS 0x00U +#define CORE_ID_DMA 0x01U +#define CORE_ID_HC 0x02U +#define CORE_ID_AES 0x03U +#define CORE_ID_SM4 0x04U +#define CORE_ID_SM3 0x05U +#define CORE_ID_XC 0x07U +#define CORE_ID_HCQ 0x08U +#define CORE_ID_QSE 0x09U +#define CORE_ID_PKE 0x0AU +#define CORE_ID_TIC 0x0BU +#define CORE_ID_KIC 0x0CU +#define CORE_ID_MPU 0x0EU +#define CORE_ID_DRBG 0x0FU +#define CORE_ID_EMC 0x11U +#define CORE_ID_CCP 0x18U +#define CORE_ID_EAC 0x1EU +#define CORE_ID_NUM 0x1FU /* eSW g_drvs[] array size sentinel */ +#define CORE_ID_MAX 0xFFU /* VCQ encoding limit (8-bit field) */ + +/** + * enum cmh_core_type - Logical core type for multi-instance dispatch + * @CMH_CORE_HC: Hash / HMAC / CSHAKE / KMAC (CORE_ID_HC) + * @CMH_CORE_AES: AES (CORE_ID_AES) + * @CMH_CORE_SM4: SM4 (CORE_ID_SM4) + * @CMH_CORE_SM3: SM3 (CORE_ID_SM3) + * @CMH_CORE_CCP: ChaCha20 / Poly1305 (CORE_ID_CCP) + * @CMH_CORE_PKE: RSA / ECDSA / ECDH / EdDSA / SM2 (CORE_ID_PKE) + * @CMH_CORE_QSE: ML-KEM / ML-DSA (CORE_ID_QSE) + * @CMH_CORE_HCQ: SLH-DSA / LMS / XMSS (CORE_ID_HCQ) + * @CMH_NUM_CORE_TYPES: Number of core types (array sizing sentinel) + * + * Algorithm drivers use this enum (not raw CORE_ID_* constants) for + * MBX selection and VCQ dispatch. Each value indexes into a config + * table that maps to one or more (core_id, mbx) pairs. + * + * Raw CORE_ID_* defines remain for: + * - SYS_TYPE_SET() key-type tags in datastore operations + * - DT child node ``reg`` values (hardware core identity for config lookup) + * - Singleton system cores (SYS, KIC, DRBG, EAC) not in this enum + */ +enum cmh_core_type { + CMH_CORE_HC = 0, + CMH_CORE_AES, + CMH_CORE_SM4, + CMH_CORE_SM3, + CMH_CORE_CCP, + CMH_CORE_PKE, + CMH_CORE_QSE, + CMH_CORE_HCQ, + CMH_NUM_CORE_TYPES +}; + +/** + * struct core_dispatch - VCQ dispatch target returned by core selection + * @core_id: Hardware core ID to encode in VCQ_CMD_ID() + * @mbx_idx: Mailbox index to submit the VCQ to + */ +struct core_dispatch { + u32 core_id; + s32 mbx_idx; +}; + +/* Common VCQ Command (per CMH VCQ ABI) */ + +#define VCQ_CMD_FLUSH 0xFFU + +/** + * struct vcq_hdr - VCQ header occupying the first slot of every VCQ + * @cmds: Total number of commands including the header itself + * @rsvd: Reserved -- used internally by CMH eSW firmware + */ +struct vcq_hdr { + u32 cmds; + u32 rsvd[13]; +}; + +/* DMA Scatter-Gather Item (per CMH DMAC hardware specification) */ + +/** + * struct dma_scattergather_item - DMA scatter-gather descriptor node + * @lli: Next descriptor address (0 = end of list) + * @src: Source address for input particle + * @dst: Destination address for output particle + * @len: Particle length (low 32 bits used by hardware) + * + * Linked-list node walked by the DMAC hardware. @lli chains to the + * next item or is zero for end-of-list. + */ +struct dma_scattergather_item { + u64 lli; + u64 src; + u64 dst; + u64 len; +}; + +/* Unified HWC Command Union */ +/* + * Each per-core ABI header defines a union _cmd. + * Add new cores here as they are implemented. + */ + +union hwc_cmd { + struct vcq_hdr hdr; + union hc_cmd hc; + union sm3_cmd sm3; + union drbg_cmd drbg; + union sys_cmd sys; + union kic_cmd kic; + union aes_cmd aes; + union sm4_cmd sm4; + union ccp_cmd ccp; + union pke_cmd pke; + union qse_cmd qse; + union hcq_cmd hcq; + union eac_cmd eac; +}; + +/** + * struct vcq_cmd - Single VCQ command entry (always 64 bytes) + * @magic: VCQ_HDR_MAGIC for the header slot, VCQ_CMD_MAGIC for commands + * @id: Encoded command ID built via VCQ_CMD_ID(core, flags, span, cmd) + * @hwc: Per-core command payload union + */ +struct vcq_cmd { + u32 magic; + u32 id; + union hwc_cmd hwc; +}; + +static_assert(sizeof(struct vcq_cmd) == 64, + "struct vcq_cmd must be exactly 64 bytes (one VCQ slot)"); + +/** + * vcq_set_header() - Write the standard VCQ header at slot[0] + * @slot: Pointer to the first VCQ slot + * @total_cmds: Total number of commands including the header + */ +static inline void vcq_set_header(struct vcq_cmd *slot, u32 total_cmds) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_HDR_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_RUN); + slot->hwc.hdr.cmds = total_cmds; +} + +/* VCQ Command Limits */ + +#define MIN_VCQ_CMDS 2U /* header + at least one command */ +#define MAX_VCQ_CMDS 15U /* including the header */ +#define MAX_VCQ_SIZE (MAX_VCQ_CMDS * sizeof(struct vcq_cmd)) + +/** + * vcq_add_inline_data() - Pack inline data into consecutive VCQ slots + * @slot: Pointer to the command slot preceding the inline data + * @data: Source data to copy into subsequent slots + * @data_len: Length of @data in bytes + * + * Appends data starting at slot+1 and updates the span field in + * slot->id. The caller must ensure enough slots are reserved. + * + * Return: Total number of slots consumed (1 + inline slots). + */ +static inline u32 vcq_add_inline_data(struct vcq_cmd *slot, + const void *data, u32 data_len) +{ + u32 inline_slots, total_span; + + if (!data_len) + return 1; + + inline_slots = (data_len + sizeof(struct vcq_cmd) - 1) / + sizeof(struct vcq_cmd); + total_span = 1 + inline_slots; + + /* Zero the inline slots, then copy data */ + memset(slot + 1, 0, inline_slots * sizeof(struct vcq_cmd)); + memcpy(slot + 1, data, data_len); + + /* Update span in the command's id field */ + slot->id = (slot->id & ~VCQ_SPAN_MASK) | + (((u32)total_span << 8) & VCQ_SPAN_MASK); + + return total_span; +} + +/** + * vcq_add_flush() - Build a generic VCQ_CMD_FLUSH command + * @slot: Pointer to the VCQ slot to populate + * @core_id: Hardware core ID for the flush command + */ +static inline void vcq_add_flush(struct vcq_cmd *slot, u32 core_id) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, VCQ_CMD_FLUSH); +} + +/* Shared HC VCQ Builders -- used by hash, hmac, cshake, kmac drivers */ + +static inline void vcq_add_hc_init(struct vcq_cmd *slot, u32 core_id, + u32 algo) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_INIT); + slot->hwc.hc.cmd_init.algo = algo; +} + +static inline void vcq_add_hc_final(struct vcq_cmd *slot, u32 core_id, + u64 digest_phys, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_FINAL); + slot->hwc.hc.cmd_final.digest = digest_phys; + slot->hwc.hc.cmd_final.outlen = outlen; +} + +static inline void vcq_add_hc_gather(struct vcq_cmd *slot, u32 core_id, + u64 lista_phys, u32 sgcmd) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_GATHER); + slot->hwc.hc.cmd_gather.lista = lista_phys; + slot->hwc.hc.cmd_gather.sgcmd = sgcmd; +} + +#endif /* CMH_VCQ_H */ From 233c4c120c0977c16aaab4fbbefc4967dba59d10 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:21 -0700 Subject: [PATCH 05/21] crypto: cmh - add key provisioning and management Add the CMH key management subsystem: - Key provisioning: create, import, derive, and destroy hardware keys stored in the CMH datastore - System object management: allocate and free CMH system objects - Management ioctl interface (/dev/cmh_mgmt): ioctl commands covering key lifecycle, KIC key derivation, PKE operations (RSA, ECDSA, ECDH, EdDSA), PQC operations (ML-KEM, ML-DSA, SLH-DSA), SM2, EAC, and DRBG reseeding - SM2 ioctl handlers: SM2 encrypt, decrypt, sign, and key exchange -- operations that require multi-step protocol flows not expressible through the standard crypto API sig interface - UAPI header: cmh_mgmt_ioctl.h (ioctl definitions and structures) The misc device requires CAP_SYS_ADMIN for open(). Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- Documentation/ABI/testing/cmh-mgmt | 136 ++ drivers/crypto/cmh/Kconfig | 19 + drivers/crypto/cmh/Makefile | 11 +- drivers/crypto/cmh/cmh_key.c | 164 +++ drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/cmh_mgmt.c | 1607 ++++++++++++++++++++++ drivers/crypto/cmh/cmh_mgmt_pke.c | 1100 +++++++++++++++ drivers/crypto/cmh/cmh_mgmt_pqc.c | 1279 +++++++++++++++++ drivers/crypto/cmh/cmh_pke_sm2.c | 827 +++++++++++ drivers/crypto/cmh/cmh_sys.c | 376 +++++ drivers/crypto/cmh/include/cmh_key.h | 82 ++ drivers/crypto/cmh/include/cmh_mgmt.h | 62 + drivers/crypto/cmh/include/cmh_pke.h | 245 ++++ drivers/crypto/cmh/include/cmh_pke_sm2.h | 30 + drivers/crypto/cmh/include/cmh_pqc.h | 25 + drivers/crypto/cmh/include/cmh_sys.h | 111 ++ include/uapi/linux/cmh_mgmt_ioctl.h | 895 ++++++++++++ 17 files changed, 6977 insertions(+), 1 deletion(-) create mode 100644 Documentation/ABI/testing/cmh-mgmt create mode 100644 drivers/crypto/cmh/cmh_key.c create mode 100644 drivers/crypto/cmh/cmh_mgmt.c create mode 100644 drivers/crypto/cmh/cmh_mgmt_pke.c create mode 100644 drivers/crypto/cmh/cmh_mgmt_pqc.c create mode 100644 drivers/crypto/cmh/cmh_pke_sm2.c create mode 100644 drivers/crypto/cmh/cmh_sys.c create mode 100644 drivers/crypto/cmh/include/cmh_key.h create mode 100644 drivers/crypto/cmh/include/cmh_mgmt.h create mode 100644 drivers/crypto/cmh/include/cmh_pke.h create mode 100644 drivers/crypto/cmh/include/cmh_pke_sm2.h create mode 100644 drivers/crypto/cmh/include/cmh_pqc.h create mode 100644 drivers/crypto/cmh/include/cmh_sys.h create mode 100644 include/uapi/linux/cmh_mgmt_ioctl.h diff --git a/Documentation/ABI/testing/cmh-mgmt b/Documentation/ABI/testing/cmh-mgmt new file mode 100644 index 00000000000000..2c6fce7ae00905 --- /dev/null +++ b/Documentation/ABI/testing/cmh-mgmt @@ -0,0 +1,136 @@ +What: /dev/cmh_mgmt +Date: June 2026 +KernelVersion: 7.1 +Contact: linux-crypto@vger.kernel.org +Description: + Character device (misc) providing a management and + key-operations ioctl interface to the CRI CryptoManager Hub + hardware crypto accelerator. Used for operations that + cannot be represented through the standard in-kernel + crypto API: datastore key CRUD, key derivation, + asymmetric crypto (EdDSA, SM2), and post-quantum crypto + (ML-KEM, ML-DSA, SLH-DSA). + + The ioctl magic is 'J'. All struct arguments are + versioned via a leading __u32 version field set to + CMH_MGMT_V1 (1). + + Ioctl commands are grouped by function: + + **Key Management (0x01-0x0E):** + + - CMH_IOCTL_KEY_NEW (0x01): Allocate a new datastore slot. + Accepts ds_type (CMH_DS_* constant), key length, flags, + and caller ID. Returns a 64-bit key reference. + + - CMH_IOCTL_KEY_WRITE (0x02): Write key material into + a previously allocated datastore slot. Supports + plaintext or wrapped key import via a wrapping key ref. + + - CMH_IOCTL_KEY_READ (0x03): Read key material from + a datastore slot, optionally wrapped. Returns data + plus a 16-byte SYS header for wrapped reads. + + - CMH_IOCTL_KEY_FIND (0x04): Look up a key reference + by caller ID (CID). + + - CMH_IOCTL_KEY_GRANT (0x05): Grant access to a key. + + - CMH_IOCTL_KEY_DELETE (0x06): Delete a datastore slot. + + - CMH_IOCTL_DS_EXPORT (0x07): Export the entire datastore + as an encrypted blob. + + - CMH_IOCTL_DS_IMPORT (0x08): Import a previously exported + datastore blob. + + - CMH_IOCTL_KEY_NEW_RANDOM (0x0B): Allocate a datastore + slot and fill it with hardware-generated random data. + + - CMH_IOCTL_KEY_LIST (0x0E): List active datastore entries, + returning CIDs, types, lengths, and flags. + + **Key Derivation -- KIC (0x09-0x0D):** + + - CMH_IOCTL_KIC_HKDF1 (0x09): HKDF-Extract step. + - CMH_IOCTL_KIC_HKDF2 (0x0A): HKDF-Expand step. + - CMH_IOCTL_KIC_AES_CMAC_KDF (0x0C): AES-CMAC KDF. + - CMH_IOCTL_KIC_DKEK_DERIVE (0x0D): DKEK derivation. + + **EAC -- Error and Alarm (0x0F):** + + - CMH_IOCTL_EAC_READ (0x0F): Read and clear hardware + error, alarm, and safety notification registers. + + **PKE -- Public Key Engine (0x10-0x1C):** + + - CMH_IOCTL_PKE_RSA_ENC (0x10): RSA public-key encrypt. + - CMH_IOCTL_PKE_RSA_DEC (0x11): RSA private-key decrypt. + - CMH_IOCTL_PKE_RSA_CRT_DEC (0x12): RSA-CRT decrypt. + - CMH_IOCTL_PKE_RSA_KEYGEN (0x13): RSA key pair generation. + - CMH_IOCTL_PKE_ECDSA_SIGN (0x14): ECDSA sign. + - CMH_IOCTL_PKE_ECDH (0x16): ECDH shared secret. + - CMH_IOCTL_PKE_ECDH_KEYGEN (0x17): ECDH key pair generation. + - CMH_IOCTL_PKE_EDDSA_SIGN (0x18): EdDSA sign (Ed25519/Ed448). + - CMH_IOCTL_PKE_EDDSA_VERIFY (0x19): EdDSA verify. + - CMH_IOCTL_PKE_EC_KEYGEN (0x1A): EC key pair generation. + - CMH_IOCTL_PKE_EC_PUBGEN (0x1B): EC public key derivation. + - CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA (0x1C): EdDSA SCA-protected + key generation. + + **PQC -- Post-Quantum Crypto (0x20-0x2D):** + + - CMH_IOCTL_ML_KEM_KEYGEN (0x20): ML-KEM key pair generation + (modes 512/768/1024). + - CMH_IOCTL_ML_KEM_ENC (0x21): ML-KEM encapsulation. + - CMH_IOCTL_ML_KEM_DEC (0x22): ML-KEM decapsulation. + - CMH_IOCTL_ML_DSA_KEYGEN (0x23): ML-DSA key pair generation + (modes 44/65/87). + - CMH_IOCTL_ML_DSA_SIGN (0x24): ML-DSA sign. + - CMH_IOCTL_SLHDSA_KEYGEN (0x28): SLH-DSA key pair generation + (12 parameter sets). + - CMH_IOCTL_SLHDSA_SIGN (0x29): SLH-DSA sign. + - CMH_IOCTL_SLHDSA_SIGN_PREHASH (0x2D): SLH-DSA prehash sign. + + **SM2 Operations (0x30-0x37):** + + - CMH_IOCTL_SM2_ECDH_KEYGEN (0x30): SM2 ephemeral key gen. + - CMH_IOCTL_SM2_ECDH (0x31): SM2 key exchange. + - CMH_IOCTL_SM2_DEC_POINT (0x32): SM2 decrypt (point step). + - CMH_IOCTL_SM2_ENC_POINT (0x33): SM2 encrypt (point step). + - CMH_IOCTL_SM2_ID_DIGEST (0x34): SM2 ID digest (ZA). + - CMH_IOCTL_SM2_ECDH_HASH (0x35): SM2 key exchange hash step. + - CMH_IOCTL_SM2_DEC_HASH (0x36): SM2 decrypt (hash step). + - CMH_IOCTL_SM2_ENC_HASH (0x37): SM2 encrypt (hash step). + + The SM2 encrypt/decrypt hash-step ioctls accept payloads + of at most 32 bytes. The underlying hardware KDF emits a + single 32-byte SM3 block, so longer messages cannot be + processed in a single command and are rejected with + -EINVAL. + + **DRBG Management (0x40):** + + - CMH_IOCTL_DRBG_CONFIG (0x40): Configure the hardware + DRBG entropy ratio and security strength. Normally + called once at system start-up before hwrng reads. + + All structs contain ``__reserved`` fields that must be + zero; the driver returns ``-EINVAL`` if any reserved field + is non-zero. This ensures forward compatibility when + reserved fields gain meaning in future versions. + + All ioctls return 0 on success or a negative errno on + failure. Common errors: + + - EINVAL: Invalid version, parameter, key type, or + non-zero reserved field. + - ENOENT: Key reference not found in datastore. + - ENOMEM: DMA allocation failure. + - EBUSY: Hardware mailbox full. + - ETIMEDOUT: VCQ operation timed out. + - EFAULT: Bad user-space pointer. + + The ioctl UAPI header is . + All structures, constants, and type definitions are + documented in that header file. diff --git a/drivers/crypto/cmh/Kconfig b/drivers/crypto/cmh/Kconfig index fa5adeca251240..c607014f8fbc8c 100644 --- a/drivers/crypto/cmh/Kconfig +++ b/drivers/crypto/cmh/Kconfig @@ -44,3 +44,22 @@ config CRYPTO_DEV_CMH_DEBUG Useful for bringup, validation, and performance analysis. Not recommended for production. + +config CRYPTO_DEV_CMH_MGMT + bool "CMH management ioctl device (/dev/cmh_mgmt)" + depends on CRYPTO_DEV_CMH + default n + help + Expose /dev/cmh_mgmt, a misc device providing ioctl commands + for operations that have no kernel crypto API binding: hardware + key lifecycle (create, import, derive, destroy), KIC key + derivation, PQC keygen/encaps/decaps (ML-KEM, ML-DSA, SLH-DSA), + EdDSA sign/verify, SM2 key exchange, and DRBG + configuration. + + The device requires CAP_SYS_ADMIN. Disabling this option + removes the ioctl interface but all kernel crypto API + algorithms (consumed by in-kernel users and validated by the + crypto test manager) remain fully functional. + + If unsure, say N. diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 0a4591c9fd8608..1492e575598cab 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -12,7 +12,16 @@ cmh-y := \ cmh_txn.o \ cmh_rh.o \ cmh_dma.o \ - cmh_sysfs.o + cmh_sysfs.o \ + cmh_key.o \ + cmh_sys.o + +# Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. +cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ + cmh_mgmt.o \ + cmh_mgmt_pke.o \ + cmh_mgmt_pqc.o \ + cmh_pke_sm2.o ccflags-y += -I$(src)/include diff --git a/drivers/crypto/cmh/cmh_key.c b/drivers/crypto/cmh/cmh_key.c new file mode 100644 index 00000000000000..fde8be50b25cc7 --- /dev/null +++ b/drivers/crypto/cmh/cmh_key.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Dual Key Path Implementation + * + * Two key provisioning paths are supported: + * + * Raw key: key bytes -> stored in tfm context -> + * SYS_CMD_WRITE(SYS_REF_TEMP) packed into every crypto VCQ. + * The raw key buffer is DMA-mapped once at setkey time and remains + * mapped for the lifetime of the transform (unmapped in destroy). + * + * Raw key DMA lifetime rationale + * ------------------------------ + * Raw keys are DMA-mapped at setkey time and the mapping persists + * until the transform is destroyed (cmh_key_destroy). This is a + * deliberate design choice, consistent with upstream HW crypto + * drivers (CAAM, ccree, CCP) that also map keys at setkey for + * transform-lifetime reuse: + * + * - The Linux crypto framework expects setkey to prepare the + * transform for repeated encrypt/decrypt calls. Remapping the + * same key on every request would add DMA API overhead per crypto + * operation with no security benefit. + * - On destroy, kfree_sensitive() scrubs the key buffer and the + * DMA mapping is released. For key-by-ID (persistent), the + * per-MBX ref cache is zeroed with memzero_explicit(). + * - No key material is ever logged; dev_dbg() messages only show + * CIDs (content identifiers), not key bytes. + * + * Hardware-required behaviors (not driver policy) + * ------------------------------------------------ + * - SYS_REF_TEMP lifetime: the eSW firmware reclaims temporary + * datastore objects when the mailbox slot is reused. This is a + * hardware constraint; the driver packs SYS_CMD_WRITE into every + * VCQ to re-provision the raw key for each operation. + * - Mailbox flush (SYS_CMD_FLUSH): reclaims temp-stack space on the + * target MBX. Required by HW to prevent temp-stack exhaustion + * across multi-VCQ operations. + */ + +#include +#include +#include + +#include "cmh_key.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_sys_abi.h" +#include + +/** + * cmh_ds_type_to_core_id() - Map a datastore type to a logical core ID + * @ds_type: Datastore type constant (e.g. CMH_DS_AES_KEY, CMH_DS_SM4_KEY) + * + * Returns the algorithm-family identity (e.g. CORE_ID_AES = 0x03), NOT the + * VCQ dispatch core_id. With multi-instance, a second AES engine dispatches + * at CORE_ID_AES2 (0x06) but keys are still tagged with CORE_ID_AES (0x03) + * -- the eSW validates against the logical identity, not the dispatch ID. + * + * Return: Logical core ID on success, CORE_ID_NUM for unknown @ds_type. + */ +u32 cmh_ds_type_to_core_id(u32 ds_type) +{ + switch (ds_type) { + case CMH_DS_AES_KEY: + case CMH_DS_AES_XTS_KEY: + return CORE_ID_AES; + case CMH_DS_SM4_KEY: + return CORE_ID_SM4; + case CMH_DS_HMAC_KEY: + case CMH_DS_KMAC_KEY: + return CORE_ID_HC; + case CMH_DS_CHACHA20_KEY: + return CORE_ID_CCP; + case CMH_DS_RSA_PRIV_KEY: + case CMH_DS_RSA_PUB_KEY: + case CMH_DS_RSA_CRT_KEY: + case CMH_DS_ECDSA_PRIV_KEY: + case CMH_DS_ECDSA_PUB_KEY: + case CMH_DS_ECDH_PRIV_KEY: + case CMH_DS_EDDSA_PRIV_KEY: + case CMH_DS_SHARED_SECRET: + case CMH_DS_SM2_PRIV_KEY: + return CORE_ID_PKE; + case CMH_DS_ML_KEM_DK: + case CMH_DS_ML_DSA_SK: + return CORE_ID_QSE; + case CMH_DS_SLHDSA_SK: + return CORE_ID_HCQ; + default: + return CORE_ID_NUM; + } +} + +/** + * cmh_key_setkey_raw() - Store a raw key in the key context + * @ctx: Key context to populate + * @key: Pointer to the raw key bytes + * @keylen: Length of @key in bytes + * @core_id: Logical core ID for SYS_TYPE tagging + * + * Duplicates the raw key, DMA-maps the copy for the lifetime of the + * transform, and stores the mapping in @ctx. Any previously held key + * is destroyed first. + * + * The DMA mapping persists until cmh_key_destroy() is called (typically + * from the algorithm .exit_tfm callback). This avoids per-request DMA + * mapping overhead and matches the setkey-to-destroy lifetime model used + * by other upstream HW crypto drivers (CAAM, ccree, CCP). The key + * buffer is freed via kfree_sensitive() on destroy. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_key_setkey_raw(struct cmh_key_ctx *ctx, const u8 *key, + u32 keylen, u32 core_id) +{ + dma_addr_t dma; + u8 *copy; + + if (!keylen || !key) + return -EINVAL; + + copy = kmemdup(key, keylen, GFP_KERNEL); + if (!copy) + return -ENOMEM; + + /* Pre-map for the lifetime of the transform */ + dma = cmh_dma_map_single(copy, keylen, DMA_TO_DEVICE); + if (cmh_dma_map_error(dma)) { + kfree_sensitive(copy); + return -ENOMEM; + } + + /* Clean up any previous key */ + cmh_key_destroy(ctx); + + ctx->mode = CMH_KEY_RAW; + ctx->raw.data = copy; + ctx->raw.len = keylen; + ctx->raw.dma = dma; + ctx->raw.sys_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, core_id); + + return 0; +} + +/** + * cmh_key_destroy() - Destroy and zero-fill a key context + * @ctx: Key context to destroy + * + * For raw keys, unmaps the DMA buffer and securely frees the key material. + * Resets the key mode to CMH_KEY_NONE. + */ +void cmh_key_destroy(struct cmh_key_ctx *ctx) +{ + if (ctx->mode == CMH_KEY_RAW && ctx->raw.data) { + cmh_dma_unmap_single(ctx->raw.dma, ctx->raw.len, + DMA_TO_DEVICE); + kfree_sensitive(ctx->raw.data); + memzero_explicit(&ctx->raw, sizeof(ctx->raw)); + } + ctx->mode = CMH_KEY_NONE; +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index de065a425a2cf5..7673ed3e586189 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -29,6 +29,7 @@ #include "cmh_mqi.h" #include "cmh_txn.h" #include "cmh_rh.h" +#include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" #include "cmh_sysfs.h" @@ -196,12 +197,19 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_rh_init; + /* Register key management device (/dev/cmh_mgmt) */ + ret = cmh_mgmt_register(); + if (ret) + goto err_mgmt_register; + g_cmh_dev = dev; platform_set_drvdata(pdev, dev); dev_info(cmh_dev(), "initialized successfully\n"); return 0; +err_mgmt_register: + cmh_rh_cleanup(cfg); err_rh_init: cmh_tm_cleanup(); err_tm_init: @@ -226,6 +234,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; + cmh_mgmt_unregister(); cmh_rh_cleanup(cfg); cmh_tm_cleanup(); cmh_mqi_cleanup(cfg); diff --git a/drivers/crypto/cmh/cmh_mgmt.c b/drivers/crypto/cmh/cmh_mgmt.c new file mode 100644 index 00000000000000..d228213f7850ce --- /dev/null +++ b/drivers/crypto/cmh/cmh_mgmt.c @@ -0,0 +1,1607 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Key Management misc_device (/dev/cmh_mgmt) + * + * Provides ioctl interface for key provisioning (NEW, NEW_RANDOM, WRITE, READ, + * FIND, GRANT, DELETE) and datastore lifecycle (EXPORT, IMPORT). + * + * Each ioctl handler: copy_from_user -> validate -> DMA alloc -> + * build VCQ -> cmh_tm_submit_sync -> copy_to_user -> DMA free. + * + * Access requires CAP_SYS_ADMIN (checked in open()). The device node + * is mode 0660; DAC further limits access to owner/group. + * CMH eSW enforces per-MBX access control on top of this. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_mgmt.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_key.h" +#include "cmh_dma.h" +#include "cmh_config.h" +#include "cmh_sys_abi.h" +#include "cmh_pke.h" +#include "cmh_pke_sm2.h" +#include "cmh_qse_abi.h" +#include "cmh_hcq_abi.h" +#include + +#include + +/* + * Pin all mgmt ioctls to a single management mailbox (MBX 0). + * + * This is a deliberate, structural choice -- not a performance default. + * The /dev/cmh_mgmt path is *stateful* with respect to the eSW datastore, + * and that state is per-mailbox, so every step of a key's lifecycle must + * land on the same mailbox: + * + * 1. Datastore access control is per-mailbox AND opaque to the driver. + * SYS_CMD_NEW grants the creating mailbox a (1 << mbx_id) access mask + * (read/write/execute). Crucially, the returned 64-bit ref encodes a + * randomised offset -- NOT the owning mailbox -- so given only a ref + * (as KEY_GRANT/READ/DELETE/DS_EXPORT receive), the driver cannot + * recover which mailbox owns the object. A fixed management mailbox + * is therefore the only way to guarantee that NEW, WRITE, GRANT, READ + * and the subsequent hardware-held-key compute ops all share the + * mailbox that holds the access rights, without exposing mailbox + * identity in the UABI. (User space may still widen access to other + * mailboxes explicitly via KEY_GRANT.) + * + * 2. The eSW SYS_REF_TEMP scratch store is per-mailbox and persists + * across ioctl calls. A derivation that writes SYS_REF_TEMP (e.g. a + * KIC_* derive) must be consumed by a later ioctl on the *same* + * mailbox (e.g. DS_EXPORT with wrap_key=SYS_REF_TEMP). + * + * Device-tree per-core ``cri,mbx`` affinity applies to the *stateless* + * registered crypto API path (cmh_core_select_instance()), which carries + * no datastore state across calls and is free to balance across mailboxes. + * + * Note: MBX 0 is NOT reserved exclusively for mgmt -- registered crypto + * operations may also land here via TM round-robin (target_mbx = -1). + * This is safe because those ops do not allocate from the temp store. + */ + +/* VCQ layout: header + command + flush = 3 entries */ +#define MGMT_VCQ_CMDS 3 + +/* + * Tracks whether any operation has left residual state in the device's + * per-mailbox temporary key store since the last flush. The device + * reclaims temp storage only on a full mailbox flush (MBX_COMMAND_FLUSH), + * which also terminates any executing command queue with -EPIPE. + * + * To avoid killing concurrent in-flight operations, the flush in + * cmh_mgmt_ioctl() is conditional: it fires only when this flag is set. + * Operations that allocate temp storage (currently: KIC derivations + * targeting SYS_REF_TEMP) set this flag on success. + */ +static atomic_t mgmt_temp_dirty = ATOMIC_INIT(0); + +/* -- KEY_NEW -------------------------- */ + +static int cmh_mgmt_key_new(void __user *argp) +{ + struct cmh_ioctl_key_new req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + u64 *ref_buf; + dma_addr_t ref_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.len) + return -EINVAL; + + /* DMA buffer for CMH eSW to write back the ref */ + ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + + *ref_buf = 0; + ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.len); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + /* + * Unmap before CPU read: single-phase operation (no re-use of + * the DMA mapping), so unmap transfers ownership back to the + * CPU. On SWIOTLB systems the unmap copies the bounce buffer + * to the original allocation. This is the correct pattern for + * single-shot sync submits where the buffer is not re-mapped. + */ + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), DMA_FROM_DEVICE); + + if (ret) { + kfree(ref_buf); + return ret; + } + + req.ref = *ref_buf; + kfree(ref_buf); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), "mgmt: KEY_NEW cid=0x%llx len=%u -> ref=0x%llx\n", + req.cid, req.len, req.ref); + return 0; +} + +/* -- KEY_WRITE ------------------------- */ + +static int cmh_mgmt_key_write(void __user *argp) +{ + struct cmh_ioctl_key_write req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + void *dmabuf; + dma_addr_t dma_addr; + u32 core_id, sys_type; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + + core_id = cmh_ds_type_to_core_id(req.ds_type); + if (core_id == CORE_ID_NUM) + return -EINVAL; + sys_type = SYS_TYPE_SET(req.flags, core_id); + + dmabuf = kmalloc(req.len, GFP_KERNEL); + if (!dmabuf) + return -ENOMEM; + + if (copy_from_user(dmabuf, u64_to_user_ptr(req.data), + req.len)) { + kfree_sensitive(dmabuf); + return -EFAULT; + } + + dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_TO_DEVICE); + if (cmh_dma_map_error(dma_addr)) { + kfree_sensitive(dmabuf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_write(&vcq[1], req.ref, dma_addr, req.wrap_key, + req.len, sys_type); + /* + * PKE keys on Weierstrass curves and RSA keys must be byte-swapped + * when stored in the DS so they match the internal big-endian + * representation used by the PKE sidecar. Edwards curve keys + * (EdDSA) use native byte order and must NOT be swapped. + */ + switch (req.ds_type) { + case CMH_DS_RSA_PRIV_KEY: + case CMH_DS_RSA_PUB_KEY: + case CMH_DS_RSA_CRT_KEY: + case CMH_DS_ECDSA_PRIV_KEY: + case CMH_DS_ECDSA_PUB_KEY: + case CMH_DS_ECDH_PRIV_KEY: + case CMH_DS_SHARED_SECRET: + case CMH_DS_SM2_PRIV_KEY: + vcq[1].id |= PKE_SWAP_FLAGS; + break; + default: + /* EdDSA, symmetric keys -- no swap */ + break; + } + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(dma_addr, req.len, DMA_TO_DEVICE); + kfree_sensitive(dmabuf); + + if (ret) + return ret; + + dev_dbg(cmh_dev(), "mgmt: KEY_WRITE ref=0x%llx len=%u type=0x%x\n", + req.ref, req.len, sys_type); + return 0; +} + +/* -- KEY_READ -------------------------- */ + +static int cmh_mgmt_key_read(void __user *argp) +{ + struct cmh_ioctl_key_read req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + void *dmabuf; + dma_addr_t dma_addr; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + + dmabuf = kzalloc(req.len, GFP_KERNEL); + if (!dmabuf) + return -ENOMEM; + + dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(dma_addr)) { + kfree(dmabuf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_read(&vcq[1], req.ref, dma_addr, req.wrap_key, req.len); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(dma_addr, req.len, DMA_FROM_DEVICE); + + if (ret) { + kfree_sensitive(dmabuf); + return ret; + } + + if (copy_to_user(u64_to_user_ptr(req.data), + dmabuf, req.len)) { + kfree_sensitive(dmabuf); + return -EFAULT; + } + + req.out_len = req.len; + kfree_sensitive(dmabuf); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), "mgmt: KEY_READ ref=0x%llx len=%u\n", + req.ref, req.out_len); + return 0; +} + +/* -- KEY_FIND -------------------------- */ + +static int cmh_mgmt_key_find(void __user *argp) +{ + struct cmh_ioctl_key_find req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + struct sys_list_item *item; + dma_addr_t item_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + + item = kzalloc_obj(*item, GFP_KERNEL); + if (!item) + return -ENOMEM; + + item_dma = cmh_dma_map_single(item, sizeof(*item), DMA_FROM_DEVICE); + if (cmh_dma_map_error(item_dma)) { + kfree(item); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_find(&vcq[1], req.cid, item_dma, sizeof(*item)); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(item_dma, sizeof(*item), DMA_FROM_DEVICE); + + if (ret) { + kfree(item); + return ret; + } + + req.ref = item->ref; + req.len = item->len; + req.type = item->type; + kfree(item); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), "mgmt: KEY_FIND cid=0x%llx -> ref=0x%llx\n", + req.cid, req.ref); + return 0; +} + +/* -- KEY_LIST ------------------------- */ + +static int cmh_mgmt_key_list(void __user *argp) +{ + struct cmh_ioctl_key_list req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + struct sys_list_item *item; + dma_addr_t item_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + + if (req.__reserved) + return -EINVAL; + + item = kzalloc_obj(*item, GFP_KERNEL); + if (!item) + return -ENOMEM; + + item_dma = cmh_dma_map_single(item, sizeof(*item), DMA_FROM_DEVICE); + if (cmh_dma_map_error(item_dma)) { + kfree(item); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_list(&vcq[1], req.start_ref, item_dma, sizeof(*item)); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(item_dma, sizeof(*item), DMA_FROM_DEVICE); + + if (ret) { + kfree(item); + return ret; + } + + req.ref = item->ref; + req.cid = item->cid; + req.len = item->len; + req.type = item->type; + kfree(item); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + return 0; +} + +/* -- KEY_GRANT / KEY_DELETE --------------------- */ + +static int cmh_mgmt_key_grant(void __user *argp, bool is_delete) +{ + struct cmh_ioctl_key_grant req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + + /* DELETE = GRANT with all permissions zeroed */ + if (is_delete) { + req.read = 0; + req.write = 0; + req.execute = 0; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_grant(&vcq[1], req.ref, req.read, req.write, req.execute); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + if (ret) + return ret; + + dev_dbg(cmh_dev(), "mgmt: KEY_%s ref=0x%llx r=0x%llx w=0x%llx x=0x%llx\n", + is_delete ? "DELETE" : "GRANT", + req.ref, req.read, req.write, req.execute); + return 0; +} + +/* -- DS_EXPORT ------------------------- */ + +static int cmh_mgmt_ds_export(void __user *argp) +{ + struct cmh_ioctl_ds_export req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + void *dmabuf; + dma_addr_t dma_addr; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + + /* + * req.len is the exact DMA buffer size given to the eSW. + * Userspace must size it to at least the export blob: + * + * wrapped: sizeof(sys_wrap_hdr) + 2*AES_BLOCK_SIZE + obj_len + * = 16 + 32 + obj_len = 48 + obj_len + * plaintext: sizeof(sys_wrap_hdr) + obj_len + * = 16 + obj_len + * + * obj_len is known from KEY_NEW or KEY_FIND. If req.len is + * too small, the eSW rejects the command and we return -EIO. + */ + dmabuf = kzalloc(req.len, GFP_KERNEL); + if (!dmabuf) + return -ENOMEM; + + dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(dma_addr)) { + kfree(dmabuf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_export(&vcq[1], req.cid, dma_addr, req.wrap_key, req.len); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(dma_addr, req.len, DMA_FROM_DEVICE); + + if (ret) { + kfree_sensitive(dmabuf); + return ret; + } + + /* Parse actual blob size from the eSW-written header */ + { + struct sys_wrap_hdr *hdr = (struct sys_wrap_hdr *)dmabuf; + u64 actual; + + if (check_add_overflow((u64)sizeof(*hdr), (u64)hdr->wrap, + &actual) || + check_add_overflow(actual, (u64)hdr->len, &actual) || + actual > req.len) { + kfree_sensitive(dmabuf); + return -EIO; + } + req.out_len = (u32)actual; + } + + if (copy_to_user(u64_to_user_ptr(req.data), + dmabuf, req.out_len)) { + kfree_sensitive(dmabuf); + return -EFAULT; + } + + kfree_sensitive(dmabuf); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), "mgmt: DS_EXPORT wrap_key=0x%llx len=%u\n", + req.wrap_key, req.out_len); + return 0; +} + +/* -- DS_IMPORT ------------------------- */ + +static int cmh_mgmt_ds_import(void __user *argp) +{ + struct cmh_ioctl_ds_import req; + struct vcq_cmd vcq[MGMT_VCQ_CMDS]; + void *dmabuf; + dma_addr_t dma_addr; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.len || req.len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + + dmabuf = kmalloc(req.len, GFP_KERNEL); + if (!dmabuf) + return -ENOMEM; + + if (copy_from_user(dmabuf, u64_to_user_ptr(req.data), + req.len)) { + kfree_sensitive(dmabuf); + return -EFAULT; + } + + dma_addr = cmh_dma_map_single(dmabuf, req.len, DMA_TO_DEVICE); + if (cmh_dma_map_error(dma_addr)) { + kfree_sensitive(dmabuf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], MGMT_VCQ_CMDS); + vcq_add_sys_import(&vcq[1], dma_addr, req.wrap_key, req.len); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, MGMT_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(dma_addr, req.len, DMA_TO_DEVICE); + kfree_sensitive(dmabuf); + + if (ret) + return ret; + + dev_dbg(cmh_dev(), "mgmt: DS_IMPORT wrap_key=0x%llx len=%u\n", + req.wrap_key, req.len); + return 0; +} + +/* -- KIC key derivation ioctls -------- + * + * All four KIC derivation handlers (HKDF1, HKDF2, AES-CMAC-KDF, + * DKEK-derive) share the same two-mode structure and temp-flush pattern. + * + * Temp-storage flush rationale: + * + * The device maintains a small per-mailbox temporary key store + * (~960 bytes, LIFO). A derivation targeting SYS_REF_TEMP allocates + * from this store; the allocation persists across command-queue + * boundaries until either (a) a subsequent command consumes it or + * (b) a mailbox flush resets the store. + * + * Our single-derivation ioctls produce a temp key with no consumer + * in the same queue -- the key is consumed by a *later* ioctl + * (e.g. DS_EXPORT with wrap_key=SYS_REF_TEMP). If no consumer + * follows, the allocation persists. Sequential temp derivations + * accumulate allocations until the store is exhausted (3--8 calls + * depending on key size), after which the device returns ENOMEM. + * + * A mailbox flush (cmh_tm_flush_mbx / MBX_COMMAND_FLUSH) resets the + * temp store. It does NOT destroy persistent keys, datastore + * objects, or DRBG state -- only the command queue and temp store. + * + * Safe for cross-ioctl temp flows (e.g. export-to-file: + * HKDF1->TEMP in ioctl 1, then DS_EXPORT with wrap_key=TEMP in + * ioctl 2): the flush only happens in derivation handlers and in + * the pre-PKE dispatch path, not in DS_EXPORT/DS_IMPORT, so the + * temp key survives until consumed. + * + * The ioctl dispatch also flushes before PKE/SM2/PQC ioctls to + * protect them from temp residue left by earlier derivations on the + * same mailbox. The per-handler flushes here remain necessary + * because sequential temp derivations (without an intervening + * PKE/SM2/PQC ioctl) would still exhaust the store. + */ + +/* -- KIC_HKDF1 ------------------------- */ + +/* + * Derive a key from a KIC base key via one-step HKDF. + * + * Two modes controlled by CMH_KIC_FLAG_TEMP: + * + * TEMP (flag set) -- 3-command VCQ: + * [0] SYS header + * [1] KIC_CMD_HKDF1 (dst=SYS_REF_TEMP) + * [2] flush + * Returns SYS_REF_TEMP as ref. No DS entry created. + * + * Persistent (flag clear) -- 4-command VCQ: + * [0] SYS header + * [1] SYS_CMD_NEW (allocate DS slot, CMH eSW writes ref) + * [2] KIC_CMD_HKDF1 (dst=SYS_REF_LAST = just-allocated slot) + * [3] flush + * Returns the new DS reference. + */ +#define KDF_VCQ_MAX 4 +#define KDF_MAX_KEY_LEN 64 +#define KDF_MAX_LABEL_LEN 56 + +static int cmh_mgmt_kic_hkdf1(void __user *argp) +{ + struct cmh_ioctl_kic_hkdf1 req; + struct vcq_cmd vcq[KDF_VCQ_MAX]; + bool temp; + u64 *ref_buf = NULL; + void *label_buf = NULL; + dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR; + unsigned int n_cmds; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.key_len || req.key_len > KDF_MAX_KEY_LEN) + return -EINVAL; + if (req.label_len > KDF_MAX_LABEL_LEN) + return -EINVAL; + + temp = !!(req.flags & CMH_KIC_FLAG_TEMP); + + /* + * Persistent path: need DMA buffer for CMH eSW to write the + * newly-allocated DS reference. + */ + if (!temp) { + ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + *ref_buf = 0; + ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + } + + /* DMA buffer for label data (CMH eSW DMA-reads it) */ + if (req.label_len > 0) { + label_buf = kzalloc(req.label_len, GFP_KERNEL); + if (!label_buf) { + ret = -ENOMEM; + goto out_ref; + } + if (copy_from_user(label_buf, + u64_to_user_ptr(req.label), + req.label_len)) { + ret = -EFAULT; + goto out_label; + } + label_dma = cmh_dma_map_single(label_buf, req.label_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(label_dma)) { + ret = -ENOMEM; + goto out_label; + } + } + + /* Build VCQ */ + memset(vcq, 0, sizeof(vcq)); + + if (temp) { + /* Flush MBX to reset temp stack -- see KIC section comment */ + ret = cmh_tm_flush_mbx(MGMT_MBX); + if (ret) + goto out_unmap_label; + + n_cmds = 3; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_kic_hkdf1(&vcq[1], SYS_REF_TEMP, req.base_key, + label_dma, req.key_len, req.label_len, + SYS_TYPE_SET(0, CORE_ID_AES)); + vcq_add_sys_flush(&vcq[2]); + } else { + n_cmds = 4; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len); + vcq_add_kic_hkdf1(&vcq[2], SYS_REF_LAST, req.base_key, + label_dma, req.key_len, req.label_len, + SYS_TYPE_SET(0, CORE_ID_AES)); + vcq_add_sys_flush(&vcq[3]); + } + + ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX); + + /* Cleanup label DMA */ + if (label_buf) { + cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE); + kfree(label_buf); + label_buf = NULL; + } + + if (ret) + goto out_ref; + + if (temp) { + req.ref = SYS_REF_TEMP; + atomic_set(&mgmt_temp_dirty, 1); + } else { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + req.ref = *ref_buf; + kfree(ref_buf); + ref_buf = NULL; + } + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), + "mgmt: KIC_HKDF1 base=0x%llx len=%u flags=0x%x -> ref=0x%llx\n", + req.base_key, req.key_len, req.flags, req.ref); + return 0; + +out_unmap_label: + if (label_buf && !cmh_dma_map_error(label_dma) && label_dma) + cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE); +out_label: + kfree(label_buf); +out_ref: + if (ref_buf) { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + kfree(ref_buf); + } + return ret; +} + +/* -- KIC_HKDF2 ------------------------- */ + +/* + * Two-step HKDF key derivation. Same as HKDF1 but adds a salt key + * reference: Step 1: HMAC(salt, base) -> PRK; Step 2: HMAC(PRK, label) -> key. + */ + +static int cmh_mgmt_kic_hkdf2(void __user *argp) +{ + struct cmh_ioctl_kic_hkdf2 req; + struct vcq_cmd vcq[KDF_VCQ_MAX]; + bool temp; + u64 *ref_buf = NULL; + void *label_buf = NULL; + dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR; + unsigned int n_cmds; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.key_len || req.key_len > KDF_MAX_KEY_LEN) + return -EINVAL; + if (req.label_len > KDF_MAX_LABEL_LEN) + return -EINVAL; + + temp = !!(req.flags & CMH_KIC_FLAG_TEMP); + + if (!temp) { + ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + *ref_buf = 0; + ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + } + + if (req.label_len > 0) { + label_buf = kzalloc(req.label_len, GFP_KERNEL); + if (!label_buf) { + ret = -ENOMEM; + goto out_ref2; + } + if (copy_from_user(label_buf, + u64_to_user_ptr(req.label), + req.label_len)) { + ret = -EFAULT; + goto out_label2; + } + label_dma = cmh_dma_map_single(label_buf, req.label_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(label_dma)) { + ret = -ENOMEM; + goto out_label2; + } + } + + memset(vcq, 0, sizeof(vcq)); + + if (temp) { + /* Flush MBX to reset temp stack -- see KIC section comment */ + ret = cmh_tm_flush_mbx(MGMT_MBX); + if (ret) + goto out_unmap_label2; + + n_cmds = 3; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_kic_hkdf2(&vcq[1], SYS_REF_TEMP, req.base_key, + req.salt_key, label_dma, + req.key_len, req.label_len, + SYS_TYPE_SET(0, CORE_ID_AES)); + vcq_add_sys_flush(&vcq[2]); + } else { + n_cmds = 4; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len); + vcq_add_kic_hkdf2(&vcq[2], SYS_REF_LAST, req.base_key, + req.salt_key, label_dma, + req.key_len, req.label_len, + SYS_TYPE_SET(0, CORE_ID_AES)); + vcq_add_sys_flush(&vcq[3]); + } + + ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX); + + if (label_buf) { + cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE); + kfree(label_buf); + label_buf = NULL; + } + + if (ret) + goto out_ref2; + + if (temp) { + req.ref = SYS_REF_TEMP; + atomic_set(&mgmt_temp_dirty, 1); + } else { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + req.ref = *ref_buf; + kfree(ref_buf); + ref_buf = NULL; + } + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), + "mgmt: KIC_HKDF2 base=0x%llx salt=0x%llx len=%u flags=0x%x -> ref=0x%llx\n", + req.base_key, req.salt_key, req.key_len, req.flags, req.ref); + return 0; + +out_unmap_label2: + if (label_buf && !cmh_dma_map_error(label_dma) && label_dma) + cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE); +out_label2: + kfree(label_buf); +out_ref2: + if (ref_buf) { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + kfree(ref_buf); + } + return ret; +} + +/* -- KIC_AES_CMAC_KDF ------------------ */ + +/* + * Derive a key using AES-CMAC-based KDF (NIST SP800-108 style). + * Base key must be 32 bytes. Output is always non-PT (the hub driver + * rejects SYS_TYPE_FLAG_PT). + * + * VCQ layout matches HKDF: TEMP mode uses 3 commands, persistent uses 4. + */ +#define CMAC_KDF_KEY_LEN 32 + +static int cmh_mgmt_kic_aes_cmac_kdf(void __user *argp) +{ + struct cmh_ioctl_kic_aes_cmac_kdf req; + struct vcq_cmd vcq[KDF_VCQ_MAX]; + bool temp; + u64 *ref_buf = NULL; + void *label_buf = NULL; + dma_addr_t ref_dma = DMA_MAPPING_ERROR, label_dma = DMA_MAPPING_ERROR; + unsigned int n_cmds; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.key_len != CMAC_KDF_KEY_LEN) + return -EINVAL; + if (req.label_len > KDF_MAX_LABEL_LEN) + return -EINVAL; + + temp = !!(req.flags & CMH_KIC_FLAG_TEMP); + + if (!temp) { + ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + *ref_buf = 0; + ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + } + + if (req.label_len > 0) { + label_buf = kzalloc(req.label_len, GFP_KERNEL); + if (!label_buf) { + ret = -ENOMEM; + goto out_ref_cmac; + } + if (copy_from_user(label_buf, + u64_to_user_ptr(req.label), + req.label_len)) { + ret = -EFAULT; + goto out_label_cmac; + } + label_dma = cmh_dma_map_single(label_buf, req.label_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(label_dma)) { + ret = -ENOMEM; + goto out_label_cmac; + } + } + + memset(vcq, 0, sizeof(vcq)); + + if (temp) { + /* Flush MBX to reset temp stack -- see KIC section comment */ + ret = cmh_tm_flush_mbx(MGMT_MBX); + if (ret) + goto out_unmap_label_cmac; + + n_cmds = 3; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_kic_aes_cmac_kdf(&vcq[1], SYS_REF_TEMP, + req.base_key, label_dma, + req.key_len, req.label_len, + SYS_TYPE_SET(0, CORE_ID_AES)); + vcq_add_sys_flush(&vcq[2]); + } else { + n_cmds = 4; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.key_len); + vcq_add_kic_aes_cmac_kdf(&vcq[2], SYS_REF_LAST, + req.base_key, label_dma, + req.key_len, req.label_len, + SYS_TYPE_SET(0, CORE_ID_AES)); + vcq_add_sys_flush(&vcq[3]); + } + + ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX); + + if (label_buf) { + cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE); + kfree(label_buf); + label_buf = NULL; + } + + if (ret) + goto out_ref_cmac; + + if (temp) { + req.ref = SYS_REF_TEMP; + atomic_set(&mgmt_temp_dirty, 1); + } else { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + req.ref = *ref_buf; + kfree(ref_buf); + ref_buf = NULL; + } + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), + "mgmt: KIC_AES_CMAC_KDF base=0x%llx len=%u flags=0x%x -> ref=0x%llx\n", + req.base_key, req.key_len, req.flags, req.ref); + return 0; + +out_unmap_label_cmac: + if (label_buf && !cmh_dma_map_error(label_dma) && label_dma) + cmh_dma_unmap_single(label_dma, req.label_len, DMA_TO_DEVICE); +out_label_cmac: + kfree(label_buf); +out_ref_cmac: + if (ref_buf) { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + kfree(ref_buf); + } + return ret; +} + +/* -- KIC_DKEK_DERIVE ------------------- */ + +/* + * Derive a Key Encryption Key (KEK) from a KIC base key. + * Output is tagged CORE_ID_KIC (usable for further derivation only). + * host_id=0 means the caller's own host; non-zero requires management + * host privilege (eSW enforces this). + */ +#define DKEK_VCQ_MAX 4 + +static int cmh_mgmt_kic_dkek_derive(void __user *argp) +{ + struct cmh_ioctl_kic_dkek_derive req; + struct vcq_cmd vcq[DKEK_VCQ_MAX]; + bool temp; + u64 *ref_buf = NULL; + void *meta_buf = NULL; + dma_addr_t ref_dma = DMA_MAPPING_ERROR, meta_dma = DMA_MAPPING_ERROR; + unsigned int n_cmds; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.metadata_len > KIC_DKEK_MAX_METADATA) + return -EINVAL; + + temp = !!(req.flags & CMH_KIC_FLAG_TEMP); + + if (!temp) { + ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + *ref_buf = 0; + ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + } + + if (req.metadata_len > 0) { + meta_buf = kzalloc(req.metadata_len, GFP_KERNEL); + if (!meta_buf) { + ret = -ENOMEM; + goto out_ref_dkek; + } + if (copy_from_user(meta_buf, + u64_to_user_ptr(req.metadata), + req.metadata_len)) { + ret = -EFAULT; + goto out_meta; + } + meta_dma = cmh_dma_map_single(meta_buf, req.metadata_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(meta_dma)) { + ret = -ENOMEM; + goto out_meta; + } + } + + memset(vcq, 0, sizeof(vcq)); + + if (temp) { + /* Flush MBX to reset temp stack -- see KIC section comment */ + ret = cmh_tm_flush_mbx(MGMT_MBX); + if (ret) + goto out_unmap_meta; + + n_cmds = 3; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_kic_dkek_derive(&vcq[1], SYS_REF_TEMP, + req.base_key, req.host_id, + meta_dma, req.metadata_len); + vcq_add_sys_flush(&vcq[2]); + } else { + n_cmds = 4; + vcq_set_header(&vcq[0], n_cmds); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, KIC_KEY_SIZE); + vcq_add_kic_dkek_derive(&vcq[2], SYS_REF_LAST, + req.base_key, req.host_id, + meta_dma, req.metadata_len); + vcq_add_sys_flush(&vcq[3]); + } + + ret = cmh_tm_submit_sync_mbx(vcq, n_cmds, 1, MGMT_MBX); + + if (meta_buf) { + cmh_dma_unmap_single(meta_dma, req.metadata_len, + DMA_TO_DEVICE); + kfree(meta_buf); + meta_buf = NULL; + } + + if (ret) + goto out_ref_dkek; + + if (temp) { + req.ref = SYS_REF_TEMP; + atomic_set(&mgmt_temp_dirty, 1); + } else { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + req.ref = *ref_buf; + kfree(ref_buf); + ref_buf = NULL; + } + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), + "mgmt: KIC_DKEK_DERIVE base=0x%llx host=%u meta_len=%u flags=0x%x -> ref=0x%llx\n", + req.base_key, req.host_id, req.metadata_len, req.flags, + req.ref); + return 0; + +out_unmap_meta: + if (meta_buf && !cmh_dma_map_error(meta_dma) && meta_dma) + cmh_dma_unmap_single(meta_dma, req.metadata_len, DMA_TO_DEVICE); +out_meta: + kfree(meta_buf); +out_ref_dkek: + if (ref_buf) { + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), + DMA_FROM_DEVICE); + kfree(ref_buf); + } + return ret; +} + +/* -- KEY_NEW_RANDOM -- DRBG-backed key generation --- */ + +/* + * Allocate a new datastore slot and fill it with DRBG-generated + * random key material in a single atomic VCQ submission: + * + * [0] SYS header(5) + * [1] SYS_CMD_NEW -- allocate DS slot (CMH eSW writes ref) + * [2] DRBG_CMD_DATASTORE(SYS_REF_LAST) -- fill with random data + * [3] DRBG flush -- release DRBG core ownership + * [4] SYS flush + * + * The DRBG must be configured before this ioctl is used. + * Reuses struct cmh_ioctl_key_new (ds_type, flags, cid, len, ref). + */ +#define DRBG_KEYGEN_VCQ_CMDS 5 + +static int cmh_mgmt_key_new_random(void __user *argp) +{ + struct cmh_ioctl_key_new req; + struct vcq_cmd vcq[DRBG_KEYGEN_VCQ_CMDS]; + u64 *ref_buf; + dma_addr_t ref_dma; + u32 core_id, sys_type; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.len) + return -EINVAL; + + core_id = cmh_ds_type_to_core_id(req.ds_type); + if (core_id == CORE_ID_NUM) + return -EINVAL; + sys_type = SYS_TYPE_SET(req.flags, core_id); + + ref_buf = kmalloc_obj(*ref_buf, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + + *ref_buf = 0; + ref_dma = cmh_dma_map_single(ref_buf, sizeof(*ref_buf), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], DRBG_KEYGEN_VCQ_CMDS); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, req.len); + vcq_add_drbg_datastore(&vcq[2], SYS_REF_LAST, req.len, sys_type); + vcq_add_flush(&vcq[3], CORE_ID_DRBG); + vcq_add_sys_flush(&vcq[4]); + + ret = cmh_tm_submit_sync_mbx(vcq, DRBG_KEYGEN_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(ref_dma, sizeof(*ref_buf), DMA_FROM_DEVICE); + + if (ret) { + kfree(ref_buf); + return ret; + } + + req.ref = *ref_buf; + kfree(ref_buf); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + dev_dbg(cmh_dev(), + "mgmt: KEY_NEW_RANDOM cid=0x%llx len=%u type=0x%x -> ref=0x%llx\n", + req.cid, req.len, sys_type, req.ref); + return 0; +} + +#define EAC_VCQ_CMDS 3 /* header + EAC_READ + flush */ + +static long cmh_mgmt_eac_read(void __user *argp) +{ + struct cmh_ioctl_eac_read req; + struct eac_read_rsp *rsp; + struct vcq_cmd vcq[EAC_VCQ_CMDS]; + dma_addr_t rsp_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved != 0) + return -EINVAL; + if (req.__pad != 0) + return -EINVAL; + + rsp = kmalloc_obj(*rsp, GFP_KERNEL); + if (!rsp) + return -ENOMEM; + + rsp_dma = cmh_dma_map_single(rsp, sizeof(*rsp), DMA_FROM_DEVICE); + if (cmh_dma_map_error(rsp_dma)) { + kfree(rsp); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], EAC_VCQ_CMDS); + vcq_add_eac_read(&vcq[1], rsp_dma, sizeof(*rsp)); + vcq_add_flush(&vcq[2], CORE_ID_EAC); + + ret = cmh_tm_submit_sync_mbx(vcq, EAC_VCQ_CMDS, 1, MGMT_MBX); + + cmh_dma_unmap_single(rsp_dma, sizeof(*rsp), DMA_FROM_DEVICE); + + if (ret) { + kfree(rsp); + return ret; + } + + /* Copy response fields into ioctl struct */ + req.mailbox_notification = rsp->mailbox_notification; + req.hw_error = rsp->hw_error; + req.hw_nmi = rsp->hw_nmi; + req.hw_panic = rsp->hw_panic; + req.safety_fatal = rsp->safety_fatal; + req.safety_notification = rsp->safety_notification; + req.sw_info0 = rsp->sw_info0; + req.sw_info1 = rsp->sw_info1; + memcpy(req.sram_bank_errors, rsp->sram_bank_errors, + sizeof(req.sram_bank_errors)); + req.__pad = 0; + + kfree(rsp); + + if (copy_to_user(argp, &req, sizeof(req))) + return -EFAULT; + + return 0; +} + +/* -- DRBG CONFIG (management) ------------ */ + +#define DRBG_CONFIG_VCQ_CMDS 4 /* header + RESET + CONFIG + flush */ + +static long cmh_mgmt_drbg_config(void __user *argp) +{ + struct cmh_ioctl_drbg_config req; + struct vcq_cmd vcq[DRBG_CONFIG_VCQ_CMDS]; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved != 0) + return -EINVAL; + if (req.entropy_ratio > 3) + return -EINVAL; + if (req.security_strength != CMH_DRBG_STRENGTH_128 && + req.security_strength != CMH_DRBG_STRENGTH_256) + return -EINVAL; + + vcq_set_header(&vcq[0], DRBG_CONFIG_VCQ_CMDS); + vcq_add_drbg_reset(&vcq[1]); + vcq_add_drbg_config(&vcq[2], req.entropy_ratio, + req.security_strength); + vcq_add_flush(&vcq[3], CORE_ID_DRBG); + + ret = cmh_tm_submit_sync_mbx(vcq, DRBG_CONFIG_VCQ_CMDS, 1, MGMT_MBX); + if (ret) + dev_warn(cmh_dev(), "mgmt: DRBG CONFIG failed (rc=%d)\n", ret); + else + dev_info(cmh_dev(), "mgmt: DRBG configured (ratio=%u strength=0x%x)\n", + req.entropy_ratio, req.security_strength); + + return ret; +} + +/* -- ioctl dispatch ------------------------ */ + +/* + * PKE, SM2, and PQC ioctls use device-internal temporary storage for + * intermediate results. Residual allocations in the per-mailbox temp + * store (left by prior operations that targeted SYS_REF_TEMP) reduce + * the space available and can cause the device to return ENOMEM. + * + * Flush the mailbox before these operations to reset the temp store, + * but ONLY when the store is actually dirty (mgmt_temp_dirty flag). + * Unconditional flushing would kill in-flight command queues from + * concurrent callers on the same mailbox -- MBX_COMMAND_FLUSH + * terminates any executing queue with -EPIPE and discards all queued + * submissions. + * + * The conditional flush is safe: PKE/SM2/PQC ioctls do not consume + * SYS_REF_TEMP from a prior ioctl (unlike DS_EXPORT/DS_IMPORT which + * may reference a temp key produced by a preceding derivation), so + * clearing the temp store before them loses no needed state. + */ +static inline bool cmh_mgmt_needs_temp_flush(unsigned int cmd) +{ + unsigned int nr = _IOC_NR(cmd); + + /* + * Range invariant: all PKE/SM2/PQC ioctls must have consecutive + * NR values between PKE_RSA_ENC (0x10) and SM2_ENC_HASH (0x37). + * If a new ioctl is added outside this range, update the bounds + * and adjust these assertions. + */ + BUILD_BUG_ON(_IOC_NR(CMH_IOCTL_PKE_RSA_ENC) != 0x10); + BUILD_BUG_ON(_IOC_NR(CMH_IOCTL_SM2_ENC_HASH) != 0x37); + + return nr >= _IOC_NR(CMH_IOCTL_PKE_RSA_ENC) && + nr <= _IOC_NR(CMH_IOCTL_SM2_ENC_HASH); +} + +static long cmh_mgmt_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + void __user *argp = (void __user *)arg; + int ret; + + if (cmh_mgmt_needs_temp_flush(cmd) && + atomic_xchg(&mgmt_temp_dirty, 0)) { + ret = cmh_tm_flush_mbx(MGMT_MBX); + if (ret) + return ret; + } + + switch (cmd) { + case CMH_IOCTL_KEY_NEW: + return cmh_mgmt_key_new(argp); + case CMH_IOCTL_KEY_WRITE: + return cmh_mgmt_key_write(argp); + case CMH_IOCTL_KEY_READ: + return cmh_mgmt_key_read(argp); + case CMH_IOCTL_KEY_FIND: + return cmh_mgmt_key_find(argp); + case CMH_IOCTL_KEY_GRANT: + return cmh_mgmt_key_grant(argp, false); + case CMH_IOCTL_KEY_DELETE: + return cmh_mgmt_key_grant(argp, true); + case CMH_IOCTL_DS_EXPORT: + return cmh_mgmt_ds_export(argp); + case CMH_IOCTL_DS_IMPORT: + return cmh_mgmt_ds_import(argp); + case CMH_IOCTL_KIC_HKDF1: + return cmh_mgmt_kic_hkdf1(argp); + case CMH_IOCTL_KIC_HKDF2: + return cmh_mgmt_kic_hkdf2(argp); + case CMH_IOCTL_KEY_NEW_RANDOM: + return cmh_mgmt_key_new_random(argp); + case CMH_IOCTL_KIC_AES_CMAC_KDF: + return cmh_mgmt_kic_aes_cmac_kdf(argp); + case CMH_IOCTL_KIC_DKEK_DERIVE: + return cmh_mgmt_kic_dkek_derive(argp); + case CMH_IOCTL_KEY_LIST: + return cmh_mgmt_key_list(argp); + case CMH_IOCTL_EAC_READ: + return cmh_mgmt_eac_read(argp); + /* PKE operations */ + case CMH_IOCTL_PKE_RSA_ENC: + return cmh_mgmt_pke_rsa_enc(argp); + case CMH_IOCTL_PKE_RSA_DEC: + return cmh_mgmt_pke_rsa_dec(argp); + case CMH_IOCTL_PKE_RSA_CRT_DEC: + return cmh_mgmt_pke_rsa_crt_dec(argp); + case CMH_IOCTL_PKE_RSA_KEYGEN: + return cmh_mgmt_pke_rsa_keygen(argp); + case CMH_IOCTL_PKE_ECDSA_SIGN: + return cmh_mgmt_pke_ecdsa_sign(argp); + case CMH_IOCTL_PKE_ECDH: + return cmh_mgmt_pke_ecdh(argp); + case CMH_IOCTL_PKE_ECDH_KEYGEN: + return cmh_mgmt_pke_ecdh_keygen(argp); + case CMH_IOCTL_PKE_EDDSA_SIGN: + return cmh_mgmt_pke_eddsa_sign(argp); + case CMH_IOCTL_PKE_EDDSA_VERIFY: + return cmh_mgmt_pke_eddsa_verify(argp); + case CMH_IOCTL_PKE_EC_KEYGEN: + return cmh_mgmt_pke_ec_keygen(argp); + case CMH_IOCTL_PKE_EC_PUBGEN: + return cmh_mgmt_pke_ec_pubgen(argp); + case CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA: + return cmh_mgmt_pke_eddsa_keygen_sca(argp); + /* SM2 operations */ + case CMH_IOCTL_SM2_ECDH_KEYGEN: + return cmh_mgmt_sm2_ecdh_keygen(argp); + case CMH_IOCTL_SM2_ECDH: + return cmh_mgmt_sm2_ecdh(argp); + case CMH_IOCTL_SM2_DEC_POINT: + return cmh_mgmt_sm2_dec_point(argp); + case CMH_IOCTL_SM2_ENC_POINT: + return cmh_mgmt_sm2_enc_point(argp); + case CMH_IOCTL_SM2_ID_DIGEST: + return cmh_mgmt_sm2_id_digest(argp); + case CMH_IOCTL_SM2_ECDH_HASH: + return cmh_mgmt_sm2_ecdh_hash(argp); + case CMH_IOCTL_SM2_DEC_HASH: + return cmh_mgmt_sm2_dec_hash(argp); + case CMH_IOCTL_SM2_ENC_HASH: + return cmh_mgmt_sm2_enc_hash(argp); + /* PQC operations */ + case CMH_IOCTL_ML_KEM_KEYGEN: + return cmh_mgmt_ml_kem_keygen(argp); + case CMH_IOCTL_ML_KEM_ENC: + return cmh_mgmt_ml_kem_enc(argp); + case CMH_IOCTL_ML_KEM_DEC: + return cmh_mgmt_ml_kem_dec(argp); + case CMH_IOCTL_ML_DSA_KEYGEN: + return cmh_mgmt_ml_dsa_keygen(argp); + case CMH_IOCTL_ML_DSA_SIGN: + return cmh_mgmt_ml_dsa_sign(argp); + case CMH_IOCTL_SLHDSA_KEYGEN: + return cmh_mgmt_slhdsa_keygen(argp); + case CMH_IOCTL_SLHDSA_SIGN: + return cmh_mgmt_slhdsa_sign(argp); + case CMH_IOCTL_SLHDSA_SIGN_PREHASH: + return cmh_mgmt_slhdsa_sign_prehash(argp); + /* DRBG management */ + case CMH_IOCTL_DRBG_CONFIG: + return cmh_mgmt_drbg_config(argp); + default: + return -ENOTTY; + } +} + +/* -- File operations ----------------------- */ + +/* + * Capability is checked once at open time. A privileged process may + * pass the resulting fd to an unprivileged helper -- this delegation + * model is intentional and mirrors /dev/kvm, /dev/loop-control, etc. + */ +static int cmh_mgmt_open(struct inode *inode, struct file *file) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + return 0; +} + +static const struct file_operations cmh_mgmt_fops = { + .owner = THIS_MODULE, + .open = cmh_mgmt_open, + .unlocked_ioctl = cmh_mgmt_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice cmh_mgmt_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "cmh_mgmt", + .fops = &cmh_mgmt_fops, + .mode = 0660, +}; + +static bool cmh_mgmt_registered; + +/** + * cmh_mgmt_register() - Register the /dev/cmh_mgmt misc device + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_register(void) +{ + int ret; + + /* + * ABI size guards -- catch silent layout changes at compile time. + * All ioctl structs use only __u32 and __u64 with explicit padding, + * guaranteeing identical layout on 32-bit and 64-bit (compat_ptr_ioctl). + */ + BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_new) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_write) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_read) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_find) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_list) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_key_grant) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ds_export) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ds_import) != 24); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_hkdf1) != 48); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_hkdf2) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_aes_cmac_kdf) != 48); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_kic_dkek_derive) != 48); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_enc) != 48); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_dec) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_crt_dec) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_rsa_keygen) != 64); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdsa_sign) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdh) != 48); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ecdh_keygen) != 24); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_sign) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_verify) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ec_keygen) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_ec_pubgen) != 24); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_pke_eddsa_keygen_sca) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_keygen) != 64); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_enc) != 64); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_kem_dec) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_dsa_keygen) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_ml_dsa_sign) != 48); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_keygen) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_sign) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_slhdsa_sign_prehash) != 64); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh_keygen) != 24); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh) != 56); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_dec_point) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_enc_point) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_id_digest) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_ecdh_hash) != 40); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_dec_hash) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_sm2_enc_hash) != 32); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_eac_read) != 64); + BUILD_BUG_ON(sizeof(struct cmh_ioctl_drbg_config) != 16); + + ret = misc_register(&cmh_mgmt_dev); + if (ret) { + dev_err(cmh_dev(), "mgmt: misc_register failed (rc=%d)\n", ret); + return ret; + } + + cmh_mgmt_registered = true; + dev_info(cmh_dev(), "mgmt: registered /dev/cmh_mgmt\n"); + return 0; +} + +/** + * cmh_mgmt_unregister() - Unregister the /dev/cmh_mgmt misc device + */ +void cmh_mgmt_unregister(void) +{ + if (!cmh_mgmt_registered) + return; + + misc_deregister(&cmh_mgmt_dev); + cmh_mgmt_registered = false; + dev_info(cmh_dev(), "mgmt: unregistered /dev/cmh_mgmt\n"); +} diff --git a/drivers/crypto/cmh/cmh_mgmt_pke.c b/drivers/crypto/cmh/cmh_mgmt_pke.c new file mode 100644 index 00000000000000..6954832fa8ac54 --- /dev/null +++ b/drivers/crypto/cmh/cmh_mgmt_pke.c @@ -0,0 +1,1100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH -- PKE ioctl handlers for /dev/cmh_mgmt + * + * RSA encrypt/decrypt/CRT/keygen, ECDSA sign, ECDH/keygen, + * EdDSA sign/verify, EC keygen/pubgen. + * + * Split from cmh_mgmt.c for maintainability. + */ + +#include +#include +#include +#include + +#include "cmh_mgmt.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_key.h" +#include "cmh_dma.h" +#include "cmh_config.h" +#include "cmh_pke.h" +#include "cmh_pke_abi.h" +#include "cmh_sys_abi.h" +#include + +#include + +/* -- PKE ioctl helpers ------------------- */ + +/* + * Maximum PKE operand size: 512 bytes (RSA 4096-bit), + * or 2 * 68 = 136 bytes (P-521 coordinate pair). + */ +#define PKE_MAX_OPERAND 512 + +/* Validate curve ID and return coordinate length; 0 = invalid */ +static u32 cmh_pke_validate_curve(u32 curve) +{ + return pke_curve_clen(curve); +} + +/** + * cmh_mgmt_pke_rsa_enc() - Handle CMH_MGMT_IOC_PKE_RSA_ENC ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_rsa_enc(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_rsa_enc req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 n_len, e_padded; + u8 *e_buf, *n_buf, *m_buf, *c_buf; + dma_addr_t e_dma, n_dma, m_dma, c_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS) + return -EINVAL; + if (!req.e_len || req.e_len > PKE_MAX_OPERAND) + return -EINVAL; + + n_len = req.bits / 8; + e_padded = ALIGN(req.e_len, 4); + + e_buf = kzalloc(e_padded, GFP_KERNEL); + n_buf = kmalloc(n_len, GFP_KERNEL); + m_buf = kmalloc(n_len, GFP_KERNEL); + c_buf = kzalloc(n_len, GFP_KERNEL); + if (!e_buf || !n_buf || !m_buf || !c_buf) { + ret = -ENOMEM; + goto out_free; + } + + /* Right-align exponent in zero-padded buffer for DMA alignment */ + if (copy_from_user(e_buf + e_padded - req.e_len, + u64_to_user_ptr(req.e), req.e_len) || + copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) || + copy_from_user(m_buf, u64_to_user_ptr(req.input), n_len)) { + ret = -EFAULT; + goto out_free; + } + + e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE); + n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, n_len, DMA_TO_DEVICE); + c_dma = cmh_dma_map_single(c_buf, n_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) || + cmh_dma_map_error(m_dma) || cmh_dma_map_error(c_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_rsa_enc(&vcq[1], pke_cid, req.bits, e_padded, + e_dma, n_dma, m_dma, c_dma, PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(c_dma)) + cmh_dma_unmap_single(c_dma, n_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, n_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(n_dma)) + cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(e_dma)) + cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.output), c_buf, n_len)) + ret = -EFAULT; + } + +out_free: + kfree(c_buf); + kfree_sensitive(m_buf); + kfree(n_buf); + kfree(e_buf); + return ret; +} + +/** + * cmh_mgmt_pke_rsa_dec() - Handle CMH_MGMT_IOC_PKE_RSA_DEC ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_rsa_dec(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_rsa_dec req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 n_len, e_padded; + u8 *e_buf, *n_buf, *c_buf, *m_buf; + dma_addr_t e_dma, n_dma, c_dma, m_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS) + return -EINVAL; + if (!req.e_len || req.e_len > PKE_MAX_OPERAND) + return -EINVAL; + + n_len = req.bits / 8; + e_padded = ALIGN(req.e_len, 4); + + e_buf = kzalloc(e_padded, GFP_KERNEL); + n_buf = kmalloc(n_len, GFP_KERNEL); + c_buf = kmalloc(n_len, GFP_KERNEL); + m_buf = kzalloc(n_len, GFP_KERNEL); + if (!e_buf || !n_buf || !c_buf || !m_buf) { + ret = -ENOMEM; + goto out_free; + } + + /* Right-align exponent in zero-padded buffer for DMA alignment */ + if (copy_from_user(e_buf + e_padded - req.e_len, + u64_to_user_ptr(req.e), req.e_len) || + copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) || + copy_from_user(c_buf, u64_to_user_ptr(req.input), n_len)) { + ret = -EFAULT; + goto out_free; + } + + e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE); + n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE); + c_dma = cmh_dma_map_single(c_buf, n_len, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, n_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) || + cmh_dma_map_error(c_dma) || cmh_dma_map_error(m_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_rsa_dec(&vcq[1], pke_cid, req.bits, e_padded, + e_dma, n_dma, c_dma, m_dma, req.key_ref, + PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, n_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(c_dma)) + cmh_dma_unmap_single(c_dma, n_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(n_dma)) + cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(e_dma)) + cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.output), m_buf, n_len)) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(m_buf); + kfree(c_buf); + kfree(n_buf); + kfree(e_buf); + return ret; +} + +/** + * cmh_mgmt_pke_rsa_crt_dec() - Handle CMH_MGMT_IOC_PKE_RSA_CRT_DEC ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_rsa_crt_dec(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_rsa_crt_dec req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 n_len, e_padded; + u8 *e_buf, *n_buf, *c_buf, *m_buf; + dma_addr_t e_dma, n_dma, c_dma, m_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS) + return -EINVAL; + if (!req.e_len || req.e_len > PKE_MAX_OPERAND) + return -EINVAL; + + n_len = req.bits / 8; + e_padded = ALIGN(req.e_len, 4); + + e_buf = kzalloc(e_padded, GFP_KERNEL); + n_buf = kmalloc(n_len, GFP_KERNEL); + c_buf = kmalloc(n_len, GFP_KERNEL); + m_buf = kzalloc(n_len, GFP_KERNEL); + if (!e_buf || !n_buf || !c_buf || !m_buf) { + ret = -ENOMEM; + goto out_free; + } + + /* Right-align exponent in zero-padded buffer for DMA alignment */ + if (copy_from_user(e_buf + e_padded - req.e_len, + u64_to_user_ptr(req.e), req.e_len) || + copy_from_user(n_buf, u64_to_user_ptr(req.n), n_len) || + copy_from_user(c_buf, u64_to_user_ptr(req.input), n_len)) { + ret = -EFAULT; + goto out_free; + } + + e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE); + n_dma = cmh_dma_map_single(n_buf, n_len, DMA_TO_DEVICE); + c_dma = cmh_dma_map_single(c_buf, n_len, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, n_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) || + cmh_dma_map_error(c_dma) || cmh_dma_map_error(m_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_rsa_crt_dec(&vcq[1], pke_cid, req.bits, e_padded, + e_dma, n_dma, c_dma, m_dma, req.crt_ref, + PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, n_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(c_dma)) + cmh_dma_unmap_single(c_dma, n_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(n_dma)) + cmh_dma_unmap_single(n_dma, n_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(e_dma)) + cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.output), m_buf, n_len)) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(m_buf); + kfree(c_buf); + kfree(n_buf); + kfree(e_buf); + return ret; +} + +/** + * cmh_mgmt_pke_rsa_keygen() - Handle CMH_MGMT_IOC_PKE_RSA_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_rsa_keygen(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_rsa_keygen req; + /* + * When has_crt, we use a two-VCQ approach (CRI pattern): + * VCQ #1: header + SYS_NEW(d) + SYS_NEW(crt) + SYS_FLUSH (4 slots) + * VCQ #2: header + RSA_KEYGEN + PKE_FLUSH + SYS_FLUSH (4 slots) + * Without CRT, single VCQ: + * header + SYS_NEW(d) + RSA_KEYGEN + PKE_FLUSH + SYS_FLUSH (5 slots) + */ + struct vcq_cmd vcq[5]; + u32 n_len, e_padded, key_flags, d_ds_len, crt_ds_len; + u8 *e_buf, *n_buf; + u64 *d_ref_buf, *crt_ref_buf; + dma_addr_t e_dma, n_dma, d_ref_dma, crt_ref_dma; + int idx, ret; + bool has_crt, is_sca; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.bits < PKE_RSA_MIN_BITS || req.bits > PKE_RSA_MAX_BITS) + return -EINVAL; + if (!req.e_len || req.e_len > PKE_MAX_OPERAND) + return -EINVAL; + if (req.flags & ~CMH_FLAG_MASK) + return -EINVAL; + + n_len = req.bits / 8; + has_crt = (req.crt_cid != 0); + e_padded = ALIGN(req.e_len, 4); + key_flags = req.flags & CMH_FLAG_MASK; + is_sca = !!(req.flags & CMH_FLAG_SCA); + + /* + * SCA keys are stored in 2 shares -- DS allocation must be enlarged. + * CRI reference formulas: cmh_pke_rsa_private_key_size(). + */ + if (is_sca) { + d_ds_len = n_len * 2; + crt_ds_len = (7 + n_len / 2) * 4; + } else { + d_ds_len = n_len; + crt_ds_len = 5 * (n_len / 2); + } + + e_buf = kzalloc(e_padded, GFP_KERNEL); + n_buf = kzalloc(n_len, GFP_KERNEL); + d_ref_buf = kzalloc_obj(u64, GFP_KERNEL); + crt_ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!e_buf || !n_buf || !d_ref_buf || !crt_ref_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(e_buf + e_padded - req.e_len, + u64_to_user_ptr(req.e), req.e_len)) { + ret = -EFAULT; + goto out_free; + } + + e_dma = cmh_dma_map_single(e_buf, e_padded, DMA_TO_DEVICE); + n_dma = cmh_dma_map_single(n_buf, n_len, DMA_FROM_DEVICE); + d_ref_dma = cmh_dma_map_single(d_ref_buf, sizeof(u64), DMA_FROM_DEVICE); + crt_ref_dma = cmh_dma_map_single(crt_ref_buf, sizeof(u64), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(e_dma) || cmh_dma_map_error(n_dma) || + cmh_dma_map_error(d_ref_dma) || cmh_dma_map_error(crt_ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + if (has_crt) { + /* + * Two-VCQ approach (CRI pattern): SYS_REF_LAST can only + * refer to the most recently created DS object. When we + * need both d and crt refs, we must first allocate DS + * objects, read back the opaque refs, then pass them by + * value in the keygen VCQ. + * + * VCQ #1: allocate both DS objects. + */ + idx = 0; + vcq_set_header(&vcq[idx++], 4); + vcq_add_sys_new(&vcq[idx++], req.d_cid, d_ref_dma, d_ds_len); + vcq_add_sys_new(&vcq[idx++], req.crt_cid, crt_ref_dma, + crt_ds_len); + vcq_add_sys_flush(&vcq[idx++]); + + ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX); + if (ret) + goto out_unmap; + + /* Sync DMA so we can read back the opaque refs */ + cmh_dma_unmap_single(d_ref_dma, sizeof(u64), DMA_FROM_DEVICE); + cmh_dma_unmap_single(crt_ref_dma, sizeof(u64), + DMA_FROM_DEVICE); + d_ref_dma = 0; + crt_ref_dma = 0; + + /* + * VCQ #2: keygen with resolved refs. + */ + idx = 0; + memset(vcq, 0, sizeof(vcq)); + vcq_set_header(&vcq[idx++], 4); + + vcq[idx].magic = VCQ_CMD_MAGIC; + vcq[idx].id = VCQ_CMD_ID(pke_cid, PKE_SWAP_FLAGS, 1, + PKE_CMD_RSA_KEYGEN); + vcq[idx].hwc.pke.cmd_rsa_keygen.bits = req.bits; + vcq[idx].hwc.pke.cmd_rsa_keygen.e = e_dma; + vcq[idx].hwc.pke.cmd_rsa_keygen.n = n_dma; + vcq[idx].hwc.pke.cmd_rsa_keygen.d = *d_ref_buf; + vcq[idx].hwc.pke.cmd_rsa_keygen.d_type = + SYS_TYPE_SET(key_flags, CORE_ID_PKE); + vcq[idx].hwc.pke.cmd_rsa_keygen.crt = *crt_ref_buf; + vcq[idx].hwc.pke.cmd_rsa_keygen.crt_type = + SYS_TYPE_SET(key_flags, CORE_ID_PKE); + idx++; + + vcq_add_pke_flush(&vcq[idx++], pke_cid); + vcq_add_sys_flush(&vcq[idx++]); + + ret = cmh_tm_submit_sync_tmo(vcq, 4, 1, MGMT_MBX, + cmh_tm_slow_op_timeout_jiffies()); + } else { + /* + * Single-VCQ: only d, so SYS_REF_LAST is unambiguous. + */ + idx = 0; + vcq_set_header(&vcq[idx++], 5); + vcq_add_sys_new(&vcq[idx++], req.d_cid, d_ref_dma, d_ds_len); + + vcq[idx].magic = VCQ_CMD_MAGIC; + vcq[idx].id = VCQ_CMD_ID(pke_cid, PKE_SWAP_FLAGS, 1, + PKE_CMD_RSA_KEYGEN); + vcq[idx].hwc.pke.cmd_rsa_keygen.bits = req.bits; + vcq[idx].hwc.pke.cmd_rsa_keygen.e = e_dma; + vcq[idx].hwc.pke.cmd_rsa_keygen.n = n_dma; + vcq[idx].hwc.pke.cmd_rsa_keygen.d = SYS_REF_LAST; + vcq[idx].hwc.pke.cmd_rsa_keygen.d_type = + SYS_TYPE_SET(key_flags, CORE_ID_PKE); + vcq[idx].hwc.pke.cmd_rsa_keygen.crt = SYS_REF_NONE; + vcq[idx].hwc.pke.cmd_rsa_keygen.crt_type = 0; + idx++; + + vcq_add_pke_flush(&vcq[idx++], pke_cid); + vcq_add_sys_flush(&vcq[idx++]); + + ret = cmh_tm_submit_sync_tmo(vcq, 5, 1, MGMT_MBX, + cmh_tm_slow_op_timeout_jiffies()); + } + +out_unmap: + if (crt_ref_dma && !cmh_dma_map_error(crt_ref_dma)) + cmh_dma_unmap_single(crt_ref_dma, sizeof(u64), + DMA_FROM_DEVICE); + if (d_ref_dma && !cmh_dma_map_error(d_ref_dma)) + cmh_dma_unmap_single(d_ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!cmh_dma_map_error(n_dma)) + cmh_dma_unmap_single(n_dma, n_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(e_dma)) + cmh_dma_unmap_single(e_dma, e_padded, DMA_TO_DEVICE); + + if (!ret) { + /* Copy generated modulus and refs back */ + if (copy_to_user(u64_to_user_ptr(req.n), n_buf, n_len)) { + ret = -EFAULT; + goto out_free; + } + req.d_ref = *d_ref_buf; + req.crt_ref = has_crt ? *crt_ref_buf : 0; + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + +out_free: + kfree(crt_ref_buf); + kfree(d_ref_buf); + kfree(n_buf); + kfree(e_buf); + return ret; +} + +/** + * cmh_mgmt_pke_ecdsa_sign() - Handle CMH_MGMT_IOC_PKE_ECDSA_SIGN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_ecdsa_sign(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_ecdsa_sign req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 clen, sig_len, dig_map_len; + u8 *dig_buf, *sig_buf; + dma_addr_t dig_dma, sig_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + clen = cmh_pke_validate_curve(req.curve); + if (!clen || !req.digest_len || + req.digest_len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + + sig_len = 2 * clen; + + /* + * eSW requires digest_len >= clen. Zero-pad shorter hashes. + */ + dig_map_len = max_t(u32, req.digest_len, clen); + + dig_buf = kzalloc(dig_map_len, GFP_KERNEL); + sig_buf = kzalloc(sig_len, GFP_KERNEL); + if (!dig_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(dig_buf, u64_to_user_ptr(req.digest), + req.digest_len)) { + ret = -EFAULT; + goto out_free; + } + + dig_dma = cmh_dma_map_single(dig_buf, dig_map_len, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(dig_dma) || cmh_dma_map_error(sig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_ecdsa_sign(&vcq[1], pke_cid, req.curve, clen, + dig_dma, sig_dma, req.key_ref, + dig_map_len, pke_swap_flags(req.curve)); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(dig_dma)) + cmh_dma_unmap_single(dig_dma, dig_map_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.signature), + sig_buf, sig_len)) + ret = -EFAULT; + } + +out_free: + kfree(sig_buf); + kfree(dig_buf); + return ret; +} + +/** + * cmh_mgmt_pke_ecdh() - Handle CMH_MGMT_IOC_PKE_ECDH ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_ecdh(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_ecdh req; + /* Phase 1: hdr + sys_new + pke_ecdh + pke_flush; reused for Phase 2 */ + struct vcq_cmd vcq[4]; + u32 clen, swap, ss_type; + u8 *peer_buf, *ss_buf; + u64 *ref_buf; + dma_addr_t peer_dma, ss_dma, ref_dma; + int ret, idx; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + clen = cmh_pke_validate_curve(req.curve); + if (!clen) + return -EINVAL; + + swap = PKE_SWAP_FLAGS; + ss_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE); + + peer_buf = kmalloc(clen, GFP_KERNEL); + ss_buf = kzalloc(clen, GFP_KERNEL); + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!peer_buf || !ss_buf || !ref_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(peer_buf, u64_to_user_ptr(req.peer_key_x), clen)) { + ret = -EFAULT; + goto out_free; + } + + peer_dma = cmh_dma_map_single(peer_buf, clen, DMA_TO_DEVICE); + ss_dma = cmh_dma_map_single(ss_buf, clen, DMA_FROM_DEVICE); + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE); + if (cmh_dma_map_error(peer_dma) || cmh_dma_map_error(ss_dma) || + cmh_dma_map_error(ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + idx = 0; + vcq_set_header(&vcq[idx++], 4); + vcq_add_sys_new(&vcq[idx++], 0, ref_dma, clen); + vcq_add_pke_ecdh(&vcq[idx++], pke_cid, req.curve, clen, clen, + ss_type, peer_dma, req.key_ref, + SYS_REF_LAST, swap); + vcq_add_pke_flush(&vcq[idx++], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX); + if (ret) + goto out_unmap; + + /* Sync bounce buffer so CPU sees the DMA-written ref */ + cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + + /* Phase 2: extract shared secret from DS via actual ref */ + vcq_set_header(&vcq[0], 3); + vcq_add_sys_data(&vcq[1], *ref_buf, ss_dma, clen); + vcq[1].id |= pke_swap_flags(req.curve); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(ref_dma)) + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ss_dma)) + cmh_dma_unmap_single(ss_dma, clen, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(peer_dma)) + cmh_dma_unmap_single(peer_dma, clen, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.output), ss_buf, clen)) + ret = -EFAULT; + } + +out_free: + kfree(ref_buf); + kfree_sensitive(ss_buf); + kfree(peer_buf); + return ret; +} + +/** + * cmh_mgmt_pke_ecdh_keygen() - Handle CMH_MGMT_IOC_PKE_ECDH_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_ecdh_keygen(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_ecdh_keygen req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 clen, out_len; + u8 *pkx_buf; + dma_addr_t pkx_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + clen = cmh_pke_validate_curve(req.curve); + if (!clen) + return -EINVAL; + + /* + * ECDH_KEYGEN always outputs both X and Y coordinates + * (2 * clen bytes total) even though only X is useful for + * the ECDH exchange. Allocate the full output size to avoid + * a DMA buffer overflow, but copy only X back to userspace. + */ + out_len = 2 * clen; + + pkx_buf = kzalloc(out_len, GFP_KERNEL); + if (!pkx_buf) + return -ENOMEM; + + pkx_dma = cmh_dma_map_single(pkx_buf, out_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(pkx_dma)) { + kfree(pkx_buf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_ecdh_keygen(&vcq[1], pke_cid, req.curve, clen, + pkx_dma, req.key_ref, + PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + + cmh_dma_unmap_single(pkx_dma, out_len, DMA_FROM_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.public_key_x), + pkx_buf, clen)) + ret = -EFAULT; + } + + kfree(pkx_buf); + return ret; +} + +/** + * cmh_mgmt_pke_eddsa_sign() - Handle CMH_MGMT_IOC_PKE_EDDSA_SIGN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_eddsa_sign(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_eddsa_sign req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 klen, sig_len; + u8 *msg_buf, *sig_buf; + dma_addr_t msg_dma, sig_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (!cmh_pke_validate_curve(req.curve) || !req.digest_len || + req.digest_len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + if (!pke_curve_is_edwards(req.curve)) + return -EINVAL; + + klen = pke_eddsa_key_len(req.curve); + sig_len = 2 * klen; + + msg_buf = kmalloc(req.digest_len, GFP_KERNEL); + sig_buf = kzalloc(sig_len, GFP_KERNEL); + if (!msg_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(msg_buf, u64_to_user_ptr(req.digest), + req.digest_len)) { + ret = -EFAULT; + goto out_free; + } + + msg_dma = cmh_dma_map_single(msg_buf, req.digest_len, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(sig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_eddsa_sign(&vcq[1], pke_cid, req.curve, klen, + msg_dma, sig_dma, req.key_ref, + req.digest_len, pke_swap_flags(req.curve)); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(msg_dma)) + cmh_dma_unmap_single(msg_dma, req.digest_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.signature), + sig_buf, sig_len)) + ret = -EFAULT; + } + +out_free: + kfree(sig_buf); + kfree(msg_buf); + return ret; +} + +/** + * cmh_mgmt_pke_eddsa_verify() - Handle CMH_MGMT_IOC_PKE_EDDSA_VERIFY ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_eddsa_verify(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_eddsa_verify req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 clen, klen, sig_len; + u8 *msg_buf, *sig_buf, *pky_buf, *rp_buf; + dma_addr_t msg_dma, sig_dma, pky_dma, rp_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + clen = cmh_pke_validate_curve(req.curve); + if (!clen || !req.digest_len || + req.digest_len > CMH_MGMT_MAX_DATA_LEN) + return -EINVAL; + if (!pke_curve_is_edwards(req.curve)) + return -EINVAL; + + klen = pke_eddsa_key_len(req.curve); + sig_len = 2 * klen; + + msg_buf = kmalloc(req.digest_len, GFP_KERNEL); + sig_buf = kmalloc(sig_len, GFP_KERNEL); + pky_buf = kmalloc(klen, GFP_KERNEL); + rp_buf = kzalloc(clen, GFP_KERNEL); + if (!msg_buf || !sig_buf || !pky_buf || !rp_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(msg_buf, u64_to_user_ptr(req.digest), + req.digest_len) || + copy_from_user(sig_buf, u64_to_user_ptr(req.signature), + sig_len) || + copy_from_user(pky_buf, u64_to_user_ptr(req.public_key_y), + klen)) { + ret = -EFAULT; + goto out_free; + } + + msg_dma = cmh_dma_map_single(msg_buf, req.digest_len, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_buf, sig_len, DMA_TO_DEVICE); + pky_dma = cmh_dma_map_single(pky_buf, klen, DMA_TO_DEVICE); + rp_dma = cmh_dma_map_single(rp_buf, clen, DMA_FROM_DEVICE); + if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(sig_dma) || + cmh_dma_map_error(pky_dma) || cmh_dma_map_error(rp_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_eddsa_verify(&vcq[1], pke_cid, req.curve, req.digest_len, + pky_dma, msg_dma, sig_dma, rp_dma, + pke_swap_flags(req.curve)); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(rp_dma)) + cmh_dma_unmap_single(rp_dma, clen, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(pky_dma)) + cmh_dma_unmap_single(pky_dma, klen, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(msg_dma)) + cmh_dma_unmap_single(msg_dma, req.digest_len, DMA_TO_DEVICE); + +out_free: + kfree(rp_buf); + kfree(pky_buf); + kfree(sig_buf); + kfree(msg_buf); + return ret; +} + +/** + * cmh_mgmt_pke_ec_keygen() - Handle CMH_MGMT_IOC_PKE_EC_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_ec_keygen(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_ec_keygen req; + /* header + SYS_NEW + ECDSA_KEYGEN + flush_pke + flush_sys */ + struct vcq_cmd vcq[5]; + u32 clen, key_flags, ds_len; + u64 *ref_buf; + dma_addr_t ref_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (req.flags & ~CMH_FLAG_MASK) + return -EINVAL; + clen = cmh_pke_validate_curve(req.curve); + if (!clen) + return -EINVAL; + + key_flags = req.flags & CMH_FLAG_MASK; + /* SCA keys are stored in 2 shares -- allocate double the curve length */ + ds_len = (req.flags & CMH_FLAG_SCA) ? clen * 2 : clen; + + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], 5); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, ds_len); + vcq_add_pke_ecdsa_keygen(&vcq[2], pke_cid, req.curve, clen, + SYS_REF_LAST, + SYS_TYPE_SET(key_flags, CORE_ID_PKE), + pke_swap_flags(req.curve)); + vcq_add_pke_flush(&vcq[3], pke_cid); + vcq_add_sys_flush(&vcq[4]); + + ret = cmh_tm_submit_sync_mbx(vcq, 5, 1, MGMT_MBX); + + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + + if (!ret) { + req.ref = *ref_buf; + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + + kfree(ref_buf); + return ret; +} + +/** + * cmh_mgmt_pke_ec_pubgen() - Handle CMH_MGMT_IOC_PKE_EC_PUBGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_ec_pubgen(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_ec_pubgen req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 clen, pk_len; + u8 *pk_buf; + dma_addr_t pk_dma; + bool is_ed; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + clen = cmh_pke_validate_curve(req.curve); + if (!clen) + return -EINVAL; + + is_ed = pke_curve_is_edwards(req.curve); + pk_len = is_ed ? pke_eddsa_key_len(req.curve) : 2 * clen; + + pk_buf = kzalloc(pk_len, GFP_KERNEL); + if (!pk_buf) + return -ENOMEM; + + pk_dma = cmh_dma_map_single(pk_buf, pk_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(pk_dma)) { + kfree(pk_buf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + if (is_ed) + vcq_add_pke_eddsa_pubgen(&vcq[1], pke_cid, req.curve, + pke_eddsa_key_len(req.curve), + pk_dma, req.key_ref, + pke_swap_flags(req.curve)); + else + vcq_add_pke_ecdsa_pubgen(&vcq[1], pke_cid, req.curve, clen, + pk_dma, req.key_ref, + pke_swap_flags(req.curve)); + vcq_add_pke_flush(&vcq[2], pke_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + + cmh_dma_unmap_single(pk_dma, pk_len, DMA_FROM_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.public_key), + pk_buf, pk_len)) + ret = -EFAULT; + } + + kfree(pk_buf); + return ret; +} + +/** + * cmh_mgmt_pke_eddsa_keygen_sca() - Handle CMH_MGMT_IOC_PKE_EDDSA_KEYGEN_SCA ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_pke_eddsa_keygen_sca(void __user *argp) +{ + u32 pke_cid = cmh_core_default_id(CMH_CORE_PKE); + + struct cmh_ioctl_pke_eddsa_keygen_sca req; + /* header + SYS_NEW + EDDSA_KEYGEN_SCA + flush_pke + flush_sys */ + struct vcq_cmd vcq[5]; + u64 *ref_buf; + dma_addr_t ref_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + /* EdDSA SCA keygen is only supported for Ed448 */ + if (req.curve != PKE_CURVE_448) + return -EINVAL; + + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!ref_buf) + return -ENOMEM; + + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + kfree(ref_buf); + return -ENOMEM; + } + + vcq_set_header(&vcq[0], 5); + vcq_add_sys_new(&vcq[1], req.cid, ref_dma, PKE_ED448_SK_SCA_LEN); + vcq_add_pke_eddsa_keygen_sca(&vcq[2], pke_cid, req.curve, req.key_ref, + SYS_REF_LAST); + vcq_add_pke_flush(&vcq[3], pke_cid); + vcq_add_sys_flush(&vcq[4]); + + ret = cmh_tm_submit_sync_mbx(vcq, 5, 1, MGMT_MBX); + + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + + if (!ret) { + req.sca_ref = *ref_buf; + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + + kfree(ref_buf); + return ret; +} diff --git a/drivers/crypto/cmh/cmh_mgmt_pqc.c b/drivers/crypto/cmh/cmh_mgmt_pqc.c new file mode 100644 index 00000000000000..db479e80326bc3 --- /dev/null +++ b/drivers/crypto/cmh/cmh_mgmt_pqc.c @@ -0,0 +1,1279 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH -- PQC ioctl handlers for /dev/cmh_mgmt + * + * ML-KEM keygen/encapsulate/decapsulate, ML-DSA keygen/sign, + * SLH-DSA keygen/sign (pure + prehash). + * + * Split from cmh_mgmt.c for maintainability. + */ + +#include +#include +#include +#include + +#include "cmh_mgmt.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_key.h" +#include "cmh_dma.h" +#include "cmh_config.h" +#include "cmh_pqc.h" +#include "cmh_qse_abi.h" +#include "cmh_sys_abi.h" +#include + +#include + +/* -- PQC -- ML-KEM -- */ + +/** + * cmh_mgmt_ml_kem_keygen() - Handle CMH_MGMT_IOC_ML_KEM_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_ml_kem_keygen(void __user *argp) +{ + u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE); + + struct cmh_ioctl_ml_kem_keygen req; + struct vcq_cmd vcq[QSE_VCQ_CMDS_MAX]; + u32 ek_len, dk_len, seed_len, key_flags; + u32 qse_flags = 0; + bool masked, ds_ref, hw_rng; + u8 *seed_buf = NULL, *z_buf = NULL, *ek_buf, *dk_buf = NULL; + u64 *ref_buf = NULL; + dma_addr_t seed_dma = DMA_MAPPING_ERROR, z_dma = DMA_MAPPING_ERROR; + dma_addr_t ek_dma, dk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR; + int ret, idx; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (ml_kem_k_idx(req.k) < 0) + return -EINVAL; + if (req.flags & ~(CMH_QSE_FLAG_MASK | CMH_FLAG_MASK)) + return -EINVAL; + + masked = !!(req.flags & CMH_QSE_FLAG_MASKED); + ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF); + hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG); + + /* + * QSE keys only support PT storage -- the eSW dec/sign paths + * hardcode SYS_TYPE_FLAG_PT when reading the key back. + * QSE SCA protection uses masking (CMH_QSE_FLAG_MASKED), + * not the 2-share mechanism (CMH_FLAG_SCA). + */ + key_flags = req.flags & CMH_FLAG_MASK; + if (key_flags && key_flags != CMH_FLAG_PT) + return -EINVAL; + key_flags = CMH_FLAG_PT; + + /* Masked keygen must store dk in DS -- polynomial unmasking not supported */ + if (masked && !ds_ref) + return -EINVAL; + + ek_len = ML_KEM_EK_SIZE(req.k); + dk_len = masked ? ML_KEM_DK_SIZE_MASKED(req.k) + : ML_KEM_DK_SIZE(req.k); + seed_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN; + + if (hw_rng) + qse_flags |= QSE_FLAG_USE_RNG; + if (ds_ref) + qse_flags |= QSE_FLAG_USE_REF; + + ek_buf = kzalloc(ek_len, GFP_KERNEL); + if (!ek_buf) + return -ENOMEM; + + if (!hw_rng && req.seed && req.z) { + seed_buf = kmalloc(seed_len, GFP_KERNEL); + z_buf = kmalloc(seed_len, GFP_KERNEL); + if (!seed_buf || !z_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed), + seed_len) || + copy_from_user(z_buf, u64_to_user_ptr(req.z), seed_len)) { + ret = -EFAULT; + goto out_free; + } + } + + if (ds_ref) { + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!ref_buf) { + ret = -ENOMEM; + goto out_free; + } + } else { + dk_buf = kzalloc(dk_len, GFP_KERNEL); + if (!dk_buf) { + ret = -ENOMEM; + goto out_free; + } + } + + /* DMA map */ + ek_dma = cmh_dma_map_single(ek_buf, ek_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(ek_dma)) { + ret = -ENOMEM; + goto out_free; + } + + if (seed_buf) { + seed_dma = cmh_dma_map_single(seed_buf, seed_len, + DMA_TO_DEVICE); + z_dma = cmh_dma_map_single(z_buf, seed_len, DMA_TO_DEVICE); + if (cmh_dma_map_error(seed_dma) || cmh_dma_map_error(z_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + if (ds_ref) { + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } else { + dk_dma = cmh_dma_map_single(dk_buf, dk_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(dk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + idx = 0; + if (ds_ref) { + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MAX); + idx++; + vcq_add_sys_new(&vcq[idx++], req.dk_cid, ref_dma, dk_len); + vcq_add_qse_ml_kem_keygen(&vcq[idx++], qse_cid, req.k, qse_flags, + seed_dma, z_dma, + ek_dma, SYS_REF_LAST, + SYS_TYPE_SET(key_flags, + CORE_ID_QSE), + masked); + vcq_add_qse_flush(&vcq[idx++], qse_cid); + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MAX, + 1, MGMT_MBX); + } else { + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + idx++; + vcq_add_qse_ml_kem_keygen(&vcq[idx++], qse_cid, req.k, qse_flags, + seed_dma, z_dma, + ek_dma, dk_dma, 0, masked); + vcq_add_qse_flush(&vcq[idx++], qse_cid); + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, + 1, MGMT_MBX); + } + +out_unmap: + if (ds_ref && !cmh_dma_map_error(ref_dma)) + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!ds_ref && dk_buf && !cmh_dma_map_error(dk_dma)) + cmh_dma_unmap_single(dk_dma, dk_len, DMA_FROM_DEVICE); + if (z_buf && !cmh_dma_map_error(z_dma)) + cmh_dma_unmap_single(z_dma, seed_len, DMA_TO_DEVICE); + if (seed_buf && !cmh_dma_map_error(seed_dma)) + cmh_dma_unmap_single(seed_dma, seed_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(ek_dma)) + cmh_dma_unmap_single(ek_dma, ek_len, DMA_FROM_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.ek), ek_buf, ek_len)) { + ret = -EFAULT; + goto out_free; + } + if (ds_ref) { + req.dk_ref = *ref_buf; + } else { + if (copy_to_user(u64_to_user_ptr(req.dk), + dk_buf, dk_len)) { + ret = -EFAULT; + goto out_free; + } + } + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(dk_buf); + kfree(ref_buf); + kfree_sensitive(z_buf); + kfree_sensitive(seed_buf); + kfree(ek_buf); + return ret; +} + +/** + * cmh_mgmt_ml_kem_enc() - Handle CMH_MGMT_IOC_ML_KEM_ENC ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_ml_kem_enc(void __user *argp) +{ + u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE); + + struct cmh_ioctl_ml_kem_enc req; + struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN]; + u32 ek_len, ct_len, ss_out_len; + u32 qse_flags = 0; + bool masked, hw_rng; + u8 *ek_buf, *coin_buf = NULL, *ct_buf, *ss_buf; + dma_addr_t ek_dma, coin_dma = DMA_MAPPING_ERROR, ct_dma, ss_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved || req.__reserved2[0] || req.__reserved2[1]) + return -EINVAL; + if (ml_kem_k_idx(req.k) < 0) + return -EINVAL; + + masked = !!(req.flags & CMH_QSE_FLAG_MASKED); + hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG); + + ek_len = ML_KEM_EK_SIZE(req.k); + ct_len = ML_KEM_CT_SIZE(req.k); + ss_out_len = masked ? ML_KEM_SS_LEN_MASKED : ML_KEM_SS_LEN; + + if (hw_rng) + qse_flags |= QSE_FLAG_USE_RNG; + + ek_buf = kmalloc(ek_len, GFP_KERNEL); + ct_buf = kzalloc(ct_len, GFP_KERNEL); + ss_buf = kzalloc(ss_out_len, GFP_KERNEL); + if (!ek_buf || !ct_buf || !ss_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(ek_buf, u64_to_user_ptr(req.ek), ek_len)) { + ret = -EFAULT; + goto out_free; + } + + if (!hw_rng && req.coin) { + u32 coin_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN; + + coin_buf = kmalloc(coin_len, GFP_KERNEL); + if (!coin_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(coin_buf, u64_to_user_ptr(req.coin), + coin_len)) { + ret = -EFAULT; + goto out_free; + } + coin_dma = cmh_dma_map_single(coin_buf, coin_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(coin_dma)) { + ret = -ENOMEM; + goto out_free; + } + } + + ek_dma = cmh_dma_map_single(ek_buf, ek_len, DMA_TO_DEVICE); + ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_FROM_DEVICE); + ss_dma = cmh_dma_map_single(ss_buf, ss_out_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(ek_dma) || cmh_dma_map_error(ct_dma) || + cmh_dma_map_error(ss_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + vcq_add_qse_ml_kem_enc(&vcq[1], qse_cid, req.k, qse_flags, + coin_dma, ek_dma, ct_dma, ss_dma, 0, masked); + vcq_add_qse_flush(&vcq[2], qse_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(ss_dma)) + cmh_dma_unmap_single(ss_dma, ss_out_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ct_dma)) + cmh_dma_unmap_single(ct_dma, ct_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ek_dma)) + cmh_dma_unmap_single(ek_dma, ek_len, DMA_TO_DEVICE); + if (coin_buf && !cmh_dma_map_error(coin_dma)) + cmh_dma_unmap_single(coin_dma, + masked ? QSE_SEED_LEN_MASKED + : QSE_SEED_LEN, + DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.ct), ct_buf, ct_len)) { + ret = -EFAULT; + goto out_free; + } + /* Unmask ss if masked: ss = share0 ^ share1 */ + if (masked) { + crypto_xor(ss_buf, ss_buf + ML_KEM_SS_LEN, + ML_KEM_SS_LEN); + } + if (copy_to_user(u64_to_user_ptr(req.ss), ss_buf, + ML_KEM_SS_LEN)) { + ret = -EFAULT; + goto out_free; + } + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(ss_buf); + kfree(ct_buf); + kfree(coin_buf); + kfree(ek_buf); + return ret; +} + +/** + * cmh_mgmt_ml_kem_dec() - Handle CMH_MGMT_IOC_ML_KEM_DEC ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_ml_kem_dec(void __user *argp) +{ + u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE); + + struct cmh_ioctl_ml_kem_dec req; + struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN]; + u32 ct_len, dk_len, ss_out_len; + u32 qse_flags = 0; + bool masked, ds_ref; + u8 *ct_buf, *dk_buf = NULL, *ss_buf; + dma_addr_t ct_dma, dk_dma = DMA_MAPPING_ERROR, ss_dma; + u64 dk_ref; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved || req.__reserved2[0] || req.__reserved2[1]) + return -EINVAL; + if (ml_kem_k_idx(req.k) < 0) + return -EINVAL; + + masked = !!(req.flags & CMH_QSE_FLAG_MASKED); + ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF); + + ct_len = ML_KEM_CT_SIZE(req.k); + dk_len = masked ? ML_KEM_DK_SIZE_MASKED(req.k) + : ML_KEM_DK_SIZE(req.k); + ss_out_len = masked ? ML_KEM_SS_LEN_MASKED : ML_KEM_SS_LEN; + + ct_buf = kmalloc(ct_len, GFP_KERNEL); + ss_buf = kzalloc(ss_out_len, GFP_KERNEL); + if (!ct_buf || !ss_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(ct_buf, u64_to_user_ptr(req.ct), ct_len)) { + ret = -EFAULT; + goto out_free; + } + + ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_TO_DEVICE); + ss_dma = cmh_dma_map_single(ss_buf, ss_out_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(ss_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + /* + * dk: if DS_REF flag is set, req.dk is a DS reference. + * Otherwise, copy raw dk from user-space and use extmem DMA. + * Masked decaps requires DS ref (polynomial unmasking not supported). + */ + if (ds_ref) { + dk_ref = req.dk; + qse_flags |= QSE_FLAG_USE_REF; + } else { + if (masked) { + ret = -EINVAL; + goto out_unmap; + } + dk_buf = kmalloc(dk_len, GFP_KERNEL); + if (!dk_buf) { + ret = -ENOMEM; + goto out_unmap; + } + if (copy_from_user(dk_buf, u64_to_user_ptr(req.dk), dk_len)) { + ret = -EFAULT; + goto out_unmap; + } + dk_dma = cmh_dma_map_single(dk_buf, dk_len, DMA_TO_DEVICE); + if (cmh_dma_map_error(dk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + dk_ref = dk_dma; + } + + if (ds_ref) { + /* + * DS_REF decaps: CMH eSW resolves both dk and ss from DS. + * Phase 1: dec stores ss into SYS_REF_TEMP. + * Phase 2: sys_data reads ss from SYS_REF_TEMP to DMA. + */ + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + vcq_add_qse_ml_kem_dec(&vcq[1], qse_cid, req.k, qse_flags, + ct_dma, dk_ref, SYS_REF_TEMP, + SYS_TYPE_SET(SYS_TYPE_FLAG_PT, + CORE_ID_QSE), + masked); + vcq_add_qse_flush(&vcq[2], qse_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, + 1, MGMT_MBX); + if (ret) + goto out_unmap; + + /* Phase 2: extract ss from SYS_REF_TEMP */ + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + vcq_add_sys_data(&vcq[1], SYS_REF_TEMP, ss_dma, + ss_out_len); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, + 1, MGMT_MBX); + } else { + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + vcq_add_qse_ml_kem_dec(&vcq[1], qse_cid, req.k, qse_flags, + ct_dma, dk_ref, ss_dma, 0, masked); + vcq_add_qse_flush(&vcq[2], qse_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, + 1, MGMT_MBX); + } + +out_unmap: + if (dk_buf && !cmh_dma_map_error(dk_dma)) + cmh_dma_unmap_single(dk_dma, dk_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(ss_dma)) + cmh_dma_unmap_single(ss_dma, ss_out_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ct_dma)) + cmh_dma_unmap_single(ct_dma, ct_len, DMA_TO_DEVICE); + + if (!ret) { + if (masked) { + crypto_xor(ss_buf, ss_buf + ML_KEM_SS_LEN, + ML_KEM_SS_LEN); + } + if (copy_to_user(u64_to_user_ptr(req.ss), ss_buf, + ML_KEM_SS_LEN)) { + ret = -EFAULT; + goto out_free; + } + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(dk_buf); + kfree_sensitive(ss_buf); + kfree(ct_buf); + return ret; +} + +/* -- PQC -- ML-DSA -- */ + +/** + * cmh_mgmt_ml_dsa_keygen() - Handle CMH_MGMT_IOC_ML_DSA_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_ml_dsa_keygen(void __user *argp) +{ + u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE); + + struct cmh_ioctl_ml_dsa_keygen req; + struct vcq_cmd vcq[QSE_VCQ_CMDS_MAX]; + u32 pk_size, sk_size, seed_len, key_flags; + u32 qse_flags = 0; + bool masked, ds_ref, hw_rng; + u8 *seed_buf = NULL, *pk_buf, *sk_buf = NULL; + u64 *ref_buf = NULL; + dma_addr_t seed_dma = DMA_MAPPING_ERROR, pk_dma; + dma_addr_t sk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR; + int ret, idx, mi; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + mi = ml_dsa_mode_idx(req.mode); + if (mi < 0) + return -EINVAL; + if (req.flags & ~(CMH_QSE_FLAG_MASK | CMH_FLAG_MASK)) + return -EINVAL; + + masked = !!(req.flags & CMH_QSE_FLAG_MASKED); + ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF); + hw_rng = !!(req.flags & CMH_QSE_FLAG_HW_RNG); + + /* + * QSE keys only support PT storage -- the eSW sign path + * hardcodes SYS_TYPE_FLAG_PT when reading the key back. + * QSE SCA protection uses masking (CMH_QSE_FLAG_MASKED), + * not the 2-share mechanism (CMH_FLAG_SCA). + */ + key_flags = req.flags & CMH_FLAG_MASK; + if (key_flags && key_flags != CMH_FLAG_PT) + return -EINVAL; + key_flags = CMH_FLAG_PT; + + if (masked && !ds_ref) + return -EINVAL; + + pk_size = ml_dsa_pk_size[mi]; + sk_size = masked ? ml_dsa_sk_size_masked[mi] : ml_dsa_sk_size[mi]; + seed_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN; + + if (hw_rng) + qse_flags |= QSE_FLAG_USE_RNG; + if (ds_ref) + qse_flags |= QSE_FLAG_USE_REF; + + pk_buf = kzalloc(pk_size, GFP_KERNEL); + if (!pk_buf) + return -ENOMEM; + + if (!hw_rng && req.seed) { + seed_buf = kmalloc(seed_len, GFP_KERNEL); + if (!seed_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed), + seed_len)) { + ret = -EFAULT; + goto out_free; + } + } + + if (ds_ref) { + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!ref_buf) { + ret = -ENOMEM; + goto out_free; + } + } else { + sk_buf = kzalloc(sk_size, GFP_KERNEL); + if (!sk_buf) { + ret = -ENOMEM; + goto out_free; + } + } + + pk_dma = cmh_dma_map_single(pk_buf, pk_size, DMA_FROM_DEVICE); + if (cmh_dma_map_error(pk_dma)) { + ret = -ENOMEM; + goto out_free; + } + + if (seed_buf) { + seed_dma = cmh_dma_map_single(seed_buf, seed_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(seed_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + if (ds_ref) { + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), + DMA_FROM_DEVICE); + if (cmh_dma_map_error(ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } else { + sk_dma = cmh_dma_map_single(sk_buf, sk_size, DMA_FROM_DEVICE); + if (cmh_dma_map_error(sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + idx = 0; + if (ds_ref) { + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MAX); + idx++; + vcq_add_sys_new(&vcq[idx++], req.sk_cid, ref_dma, sk_size); + vcq_add_qse_ml_dsa_keygen(&vcq[idx++], qse_cid, req.mode, qse_flags, + seed_dma, pk_dma, + SYS_REF_LAST, + SYS_TYPE_SET(key_flags, + CORE_ID_QSE), + masked); + vcq_add_qse_flush(&vcq[idx++], qse_cid); + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MAX, + 1, MGMT_MBX); + } else { + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + idx++; + vcq_add_qse_ml_dsa_keygen(&vcq[idx++], qse_cid, req.mode, qse_flags, + seed_dma, pk_dma, + sk_dma, 0, masked); + vcq_add_qse_flush(&vcq[idx++], qse_cid); + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, + 1, MGMT_MBX); + } + +out_unmap: + if (ds_ref && !cmh_dma_map_error(ref_dma)) + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!ds_ref && sk_buf && !cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, sk_size, DMA_FROM_DEVICE); + if (seed_buf && !cmh_dma_map_error(seed_dma)) + cmh_dma_unmap_single(seed_dma, seed_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, pk_size, DMA_FROM_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.pk), pk_buf, pk_size)) { + ret = -EFAULT; + goto out_free; + } + if (ds_ref) { + req.sk_ref = *ref_buf; + } else { + if (copy_to_user(u64_to_user_ptr(req.sk), + sk_buf, sk_size)) { + ret = -EFAULT; + goto out_free; + } + } + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(sk_buf); + kfree(ref_buf); + kfree_sensitive(seed_buf); + kfree(pk_buf); + return ret; +} + +/** + * cmh_mgmt_ml_dsa_sign() - Handle CMH_MGMT_IOC_ML_DSA_SIGN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_ml_dsa_sign(void __user *argp) +{ + u32 qse_cid = cmh_core_default_id(CMH_CORE_QSE); + + struct cmh_ioctl_ml_dsa_sign req; + struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN]; + u32 sig_size, copy_len, rnd_len; + u32 qse_flags = 0; + bool masked; + u8 *m_buf, *sig_buf, *sk_buf = NULL, *rnd_buf = NULL; + dma_addr_t m_dma = DMA_MAPPING_ERROR, sig_dma; + dma_addr_t sk_dma = DMA_MAPPING_ERROR, rnd_dma = DMA_MAPPING_ERROR; + u64 sk_ref; + int mi, ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + mi = ml_dsa_mode_idx(req.mode); + if (mi < 0) + return -EINVAL; + if (req.mlen > ML_DSA_MAX_MLEN && req.mlen != ML_DSA_MLEN_EXTERNAL_MU) + return -EINVAL; + + masked = !!(req.flags & CMH_QSE_FLAG_MASKED); + rnd_len = masked ? QSE_SEED_LEN_MASKED : QSE_SEED_LEN; + sig_size = ml_dsa_sig_size[mi]; + copy_len = (req.mlen == ML_DSA_MLEN_EXTERNAL_MU) + ? ML_DSA_EXTMU_LEN : req.mlen; + + /* + * sk: if DS_REF, req.sk is a DS reference (masked sk lives in DS). + * Otherwise, copy raw sk from user-space. + * Masked sign requires DS ref (polynomial unmasking not supported). + */ + if (req.flags & CMH_QSE_FLAG_DS_REF) { + sk_ref = req.sk; + qse_flags |= QSE_FLAG_USE_REF; + } else { + u32 sk_size; + + if (masked) + return -EINVAL; + sk_size = ml_dsa_sk_size[mi]; + sk_buf = kmalloc(sk_size, GFP_KERNEL); + if (!sk_buf) + return -ENOMEM; + if (copy_from_user(sk_buf, u64_to_user_ptr(req.sk), sk_size)) { + kfree_sensitive(sk_buf); + return -EFAULT; + } + } + + m_buf = kmalloc(max_t(u32, copy_len, 1), GFP_KERNEL); + sig_buf = kzalloc(sig_size, GFP_KERNEL); + if (!m_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_len > 0 && + copy_from_user(m_buf, u64_to_user_ptr(req.m), copy_len)) { + ret = -EFAULT; + goto out_free; + } + + if (req.rnd) { + rnd_buf = kmalloc(rnd_len, GFP_KERNEL); + if (!rnd_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(rnd_buf, u64_to_user_ptr(req.rnd), + rnd_len)) { + ret = -EFAULT; + goto out_free; + } + } + + if (copy_len > 0) { + m_dma = cmh_dma_map_single(m_buf, copy_len, DMA_TO_DEVICE); + if (cmh_dma_map_error(m_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_FROM_DEVICE); + if (cmh_dma_map_error(sig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + if (sk_buf) { + sk_dma = cmh_dma_map_single(sk_buf, ml_dsa_sk_size[mi], + DMA_TO_DEVICE); + if (cmh_dma_map_error(sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + sk_ref = sk_dma; + } + + if (rnd_buf) { + rnd_dma = cmh_dma_map_single(rnd_buf, rnd_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rnd_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + vcq_add_qse_ml_dsa_sign(&vcq[1], qse_cid, req.mode, qse_flags, + rnd_dma, m_dma, sk_ref, sig_dma, + req.mlen, masked); + vcq_add_qse_flush(&vcq[2], qse_cid); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (rnd_buf && !cmh_dma_map_error(rnd_dma)) + cmh_dma_unmap_single(rnd_dma, rnd_len, DMA_TO_DEVICE); + if (sk_buf && !cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, ml_dsa_sk_size[mi], + DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_size, DMA_FROM_DEVICE); + if (copy_len > 0 && !cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, copy_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_size)) + ret = -EFAULT; + } + +out_free: + kfree(rnd_buf); + kfree(sig_buf); + kfree(m_buf); + kfree_sensitive(sk_buf); + return ret; +} + +/* -- PQC -- SLH-DSA -- */ + +/** + * cmh_mgmt_slhdsa_keygen() - Handle CMH_MGMT_IOC_SLHDSA_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_slhdsa_keygen(void __user *argp) +{ + u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ); + + struct cmh_ioctl_slhdsa_keygen req; + struct vcq_cmd vcq[HCQ_VCQ_CMDS_MAX]; + u32 pk_sz, sk_sz, seed_sz, sk_alloc, vcq_cnt, key_flags; + bool ds_ref; + u8 *seed_buf, *pk_buf, *sk_buf = NULL; + u64 *ref_buf = NULL; + dma_addr_t seed_dma, pk_dma, sk_dma = DMA_MAPPING_ERROR, ref_dma = DMA_MAPPING_ERROR; + int ret, idx; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX) + return -EINVAL; + if (req.flags & ~(CMH_QSE_FLAG_DS_REF | CMH_FLAG_MASK)) + return -EINVAL; + + ds_ref = !!(req.flags & CMH_QSE_FLAG_DS_REF); + + /* + * QSE keys only support PT storage -- the eSW sign path + * hardcodes SYS_TYPE_FLAG_PT when reading the key back. + * HCQ core sets key type internally during keygen. + */ + key_flags = req.flags & CMH_FLAG_MASK; + if (key_flags && key_flags != CMH_FLAG_PT) + return -EINVAL; + (void)key_flags; + + pk_sz = slhdsa_pk_size(req.parameter_set); + sk_sz = slhdsa_sk_size(req.parameter_set); + seed_sz = slhdsa_seed_size(req.parameter_set); + + seed_buf = kmalloc(seed_sz, GFP_KERNEL); + pk_buf = kzalloc(pk_sz, GFP_KERNEL); + if (!seed_buf || !pk_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(seed_buf, u64_to_user_ptr(req.seed), seed_sz)) { + ret = -EFAULT; + goto out_free; + } + + /* + * Both paths need ref_buf for sys_new output. Non-ds_ref also + * needs sk_buf (+16 for SYS header) to read back via sys_read. + */ + ref_buf = kzalloc(sizeof(u64), GFP_KERNEL); + if (!ref_buf) { + ret = -ENOMEM; + goto out_free; + } + if (!ds_ref) { + sk_alloc = sk_sz + SYS_WRAP_HDR_SIZE; + sk_buf = kzalloc(sk_alloc, GFP_KERNEL); + if (!sk_buf) { + ret = -ENOMEM; + goto out_free; + } + } + + seed_dma = cmh_dma_map_single(seed_buf, seed_sz, DMA_TO_DEVICE); + pk_dma = cmh_dma_map_single(pk_buf, pk_sz, DMA_FROM_DEVICE); + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE); + if (cmh_dma_map_error(seed_dma) || cmh_dma_map_error(pk_dma) || + cmh_dma_map_error(ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + if (!ds_ref) { + sk_dma = cmh_dma_map_single(sk_buf, sk_alloc, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + /* + * SLH-DSA keygen requires seed and sk as DS references. + * VCQ: hdr + sys_new(sk) + sys_write(seed->TEMP) + keygen + [sys_read] + flush + */ + idx = 0; + if (ds_ref) { + vcq_cnt = HCQ_VCQ_CMDS_MAX - 1; /* hdr+new+write+keygen+flush */ + vcq_set_header(&vcq[idx++], vcq_cnt); + vcq_add_sys_new(&vcq[idx++], req.sk_cid, ref_dma, + sk_sz); + } else { + vcq_cnt = HCQ_VCQ_CMDS_MAX; /* hdr+new+write+keygen+read+flush */ + vcq_set_header(&vcq[idx++], vcq_cnt); + vcq_add_sys_new(&vcq[idx++], SYS_CID_NONE, ref_dma, + sk_sz); + } + vcq_add_sys_write(&vcq[idx++], SYS_REF_TEMP, seed_dma, 0, + seed_sz, + SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_HCQ)); + vcq_add_hcq_slhdsa_keygen(&vcq[idx++], hcq_cid, req.parameter_set, + seed_sz, pk_sz, sk_sz, + SYS_REF_TEMP, pk_dma, SYS_REF_LAST); + if (!ds_ref) + vcq_add_sys_read(&vcq[idx++], SYS_REF_LAST, sk_dma, + 0, sk_sz + SYS_WRAP_HDR_SIZE); + vcq_add_hcq_flush(&vcq[idx++], hcq_cid); + + ret = cmh_tm_submit_sync_tmo(vcq, vcq_cnt, 1, MGMT_MBX, + cmh_tm_slow_op_timeout_jiffies()); + +out_unmap: + if (!ds_ref && sk_buf && !cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, sk_alloc, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ref_dma)) + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, pk_sz, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(seed_dma)) + cmh_dma_unmap_single(seed_dma, seed_sz, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.pk), pk_buf, pk_sz)) { + ret = -EFAULT; + goto out_free; + } + if (ds_ref) { + req.sk_ref = *ref_buf; + } else { + if (copy_to_user(u64_to_user_ptr(req.sk), + sk_buf + SYS_WRAP_HDR_SIZE, + sk_sz)) { + ret = -EFAULT; + goto out_free; + } + } + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(sk_buf); + kfree(ref_buf); + kfree(pk_buf); + kfree_sensitive(seed_buf); + return ret; +} + +/** + * cmh_mgmt_slhdsa_sign() - Handle CMH_MGMT_IOC_SLHDSA_SIGN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_slhdsa_sign(void __user *argp) +{ + u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ); + + struct cmh_ioctl_slhdsa_sign req; + struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN]; + u32 sig_sz, n_val; + u8 *msg_buf, *ctx_buf = NULL, *sig_buf, *rnd_buf = NULL; + dma_addr_t msg_dma = DMA_MAPPING_ERROR, ctx_dma = DMA_MAPPING_ERROR; + dma_addr_t sig_dma, rnd_dma = DMA_MAPPING_ERROR; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX) + return -EINVAL; + if (req.msg_len > SLHDSA_MAX_MSG_LEN) + return -EINVAL; + if (req.ctx_len > SLHDSA_MAX_CTX_LEN) + return -EINVAL; + + sig_sz = slhdsa_get_sig_size(req.parameter_set); + n_val = slhdsa_n[req.parameter_set - 1]; + + msg_buf = kmalloc(max_t(u32, req.msg_len, 1), GFP_KERNEL); + sig_buf = kzalloc(sig_sz, GFP_KERNEL); + if (!msg_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (req.msg_len > 0 && + copy_from_user(msg_buf, u64_to_user_ptr(req.msg), req.msg_len)) { + ret = -EFAULT; + goto out_free; + } + + if (req.ctx_len > 0 && req.ctx) { + ctx_buf = kmalloc(req.ctx_len, GFP_KERNEL); + if (!ctx_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(ctx_buf, u64_to_user_ptr(req.ctx), + req.ctx_len)) { + ret = -EFAULT; + goto out_free; + } + } + + if (req.add_random) { + rnd_buf = kmalloc(n_val, GFP_KERNEL); + if (!rnd_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(rnd_buf, u64_to_user_ptr(req.add_random), + n_val)) { + ret = -EFAULT; + goto out_free; + } + } + + sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE); + if (cmh_dma_map_error(sig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + if (req.msg_len > 0) { + msg_dma = cmh_dma_map_single(msg_buf, req.msg_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(msg_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + if (ctx_buf) { + ctx_dma = cmh_dma_map_single(ctx_buf, req.ctx_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(ctx_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + if (rnd_buf) { + rnd_dma = cmh_dma_map_single(rnd_buf, n_val, DMA_TO_DEVICE); + if (cmh_dma_map_error(rnd_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN); + vcq_add_hcq_slhdsa_sign(&vcq[1], hcq_cid, req.parameter_set, + req.msg_len, req.ctx_len, + rnd_dma, msg_dma, ctx_dma, + req.sk, sig_dma); + vcq_add_hcq_flush(&vcq[2], hcq_cid); + + ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, MGMT_MBX, + cmh_tm_slow_op_timeout_jiffies()); + +out_unmap: + if (rnd_buf && !cmh_dma_map_error(rnd_dma)) + cmh_dma_unmap_single(rnd_dma, n_val, DMA_TO_DEVICE); + if (ctx_buf && !cmh_dma_map_error(ctx_dma)) + cmh_dma_unmap_single(ctx_dma, req.ctx_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE); + if (req.msg_len > 0 && !cmh_dma_map_error(msg_dma)) + cmh_dma_unmap_single(msg_dma, req.msg_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_sz)) + ret = -EFAULT; + } + +out_free: + kfree(rnd_buf); + kfree(ctx_buf); + kfree(sig_buf); + kfree(msg_buf); + return ret; +} + +/* -- PQC -- SLH-DSA prehash -- */ + +/** + * cmh_mgmt_slhdsa_sign_prehash() - Handle CMH_MGMT_IOC_SLHDSA_SIGN_PREHASH ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_slhdsa_sign_prehash(void __user *argp) +{ + u32 hcq_cid = cmh_core_default_id(CMH_CORE_HCQ); + + struct cmh_ioctl_slhdsa_sign_prehash req; + struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN]; + u32 sig_sz, n_val, hcq_cmd; + u8 *msg_buf, *ctx_buf = NULL, *sig_buf, *rnd_buf = NULL; + dma_addr_t msg_dma = DMA_MAPPING_ERROR, ctx_dma = DMA_MAPPING_ERROR; + dma_addr_t sig_dma, rnd_dma = DMA_MAPPING_ERROR; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.parameter_set < 1 || req.parameter_set > HCQ_SLHDSA_PARAM_MAX) + return -EINVAL; + if (req.prehash_algo < 1 || req.prehash_algo > HCQ_SLHDSA_PREHASH_SHAKE256) + return -EINVAL; + if (req.msg_len > SLHDSA_MAX_MSG_LEN) + return -EINVAL; + if (req.ctx_len > SLHDSA_MAX_CTX_LEN) + return -EINVAL; + + hcq_cmd = req.digest ? HCQ_CMD_SLHDSA_SIGN_PREHASH_DIGEST + : HCQ_CMD_SLHDSA_SIGN_PREHASH; + + sig_sz = slhdsa_get_sig_size(req.parameter_set); + n_val = slhdsa_n[req.parameter_set - 1]; + + msg_buf = kmalloc(max_t(u32, req.msg_len, 1), GFP_KERNEL); + sig_buf = kzalloc(sig_sz, GFP_KERNEL); + if (!msg_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (req.msg_len > 0 && + copy_from_user(msg_buf, u64_to_user_ptr(req.msg), req.msg_len)) { + ret = -EFAULT; + goto out_free; + } + + if (req.ctx_len > 0 && req.ctx) { + ctx_buf = kmalloc(req.ctx_len, GFP_KERNEL); + if (!ctx_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(ctx_buf, u64_to_user_ptr(req.ctx), + req.ctx_len)) { + ret = -EFAULT; + goto out_free; + } + } + + if (req.add_random) { + rnd_buf = kmalloc(n_val, GFP_KERNEL); + if (!rnd_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(rnd_buf, u64_to_user_ptr(req.add_random), + n_val)) { + ret = -EFAULT; + goto out_free; + } + } + + sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE); + if (cmh_dma_map_error(sig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + if (req.msg_len > 0) { + msg_dma = cmh_dma_map_single(msg_buf, req.msg_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(msg_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + if (ctx_buf) { + ctx_dma = cmh_dma_map_single(ctx_buf, req.ctx_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(ctx_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + if (rnd_buf) { + rnd_dma = cmh_dma_map_single(rnd_buf, n_val, DMA_TO_DEVICE); + if (cmh_dma_map_error(rnd_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + } + + vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN); + vcq_add_hcq_slhdsa_sign_prehash(&vcq[1], hcq_cid, + hcq_cmd, req.parameter_set, + req.prehash_algo, + req.msg_len, req.ctx_len, + rnd_dma, msg_dma, ctx_dma, + req.sk, sig_dma); + vcq_add_hcq_flush(&vcq[2], hcq_cid); + + ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, MGMT_MBX, + cmh_tm_slow_op_timeout_jiffies()); + +out_unmap: + if (rnd_buf && !cmh_dma_map_error(rnd_dma)) + cmh_dma_unmap_single(rnd_dma, n_val, DMA_TO_DEVICE); + if (ctx_buf && !cmh_dma_map_error(ctx_dma)) + cmh_dma_unmap_single(ctx_dma, req.ctx_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE); + if (req.msg_len > 0 && !cmh_dma_map_error(msg_dma)) + cmh_dma_unmap_single(msg_dma, req.msg_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.sig), sig_buf, sig_sz)) + ret = -EFAULT; + } + +out_free: + kfree(rnd_buf); + kfree(ctx_buf); + kfree(sig_buf); + kfree(msg_buf); + return ret; +} + +/* -- EAC (Error and Alarm Controller) ---- */ + diff --git a/drivers/crypto/cmh/cmh_pke_sm2.c b/drivers/crypto/cmh/cmh_pke_sm2.c new file mode 100644 index 00000000000000..9a6e30c7f5e599 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pke_sm2.c @@ -0,0 +1,827 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM2 PKE Ioctl Handlers + * + * SM2 (GM/T 0003) is the Chinese national public-key standard over the + * sm2p256v1 curve (256-bit). It defines three protocols: + * + * - Signature: reuses ECDSA sign/verify with SM2_CURVE (0x18), handled + * by the existing cmh_mgmt_pke_ecdsa_{sign,verify}() paths. + * - Encryption: two-step (ENC_POINT + ENC_HASH / DEC_POINT + DEC_HASH). + * - Key Exchange: four-step (ECDH_KEYGEN + ID_DIGEST + ECDH + ECDH_HASH). + * + * This file implements the 8 SM2-specific ioctl handlers (0x16--0x1D). + * Sign/verify/keygen/pubgen use the existing ECDSA/EC paths unchanged. + * + * VCQ flag convention (from eSW API): + * - Most SM2 commands use flags=0 (no swap). + * - SM2_DEC_POINT and SM2_ECDH_HASH use PKE_SWAP_FLAGS on the + * PKE command itself. + * - SM2_ECDH and SM2_ECDH_HASH also apply PKE_SWAP_FLAGS on + * their sys_new/sys_data VCQ phases (Weierstrass DS format). + */ + +#include +#include + +#include "cmh_pke.h" +#include "cmh_pke_sm2.h" +#include "cmh_sys.h" +#include "cmh_dma.h" +#include "cmh_txn.h" +#include "cmh_mgmt.h" +#include "cmh_sys_abi.h" +#include + +/* SM2 fixed sizes (sm2p256v1: 256-bit curve) */ +#define SM2_CLEN 32U /* coordinate length */ +#define SM2_POINT_LEN 64U /* uncompressed EC point (x||y) */ +#define SM2_SHARED_KEY_LEN 16U /* ECDH shared key output */ +#define SM2_DIGEST_LEN 32U /* SM3 ZA digest */ +#define SM2_NONCE_LEN 32U /* nonce (when caller-provided) */ +/* + * SM2 enc_hash/dec_hash payload limit. + * + * The eSW PKE driver expands the GM/T 0003.4 KDF by issuing a single SM3 + * invocation per command (one 32-byte block of key stream). Messages + * longer than 32 bytes would require ceil(msg_len / 32) SM3 invocations + * with an incremented counter, which the eSW does not perform; longer + * inputs would silently produce incorrect ciphertext / plaintext. + * + * The eSW PKE SRAM can physically hold up to 4000 bytes of payload, but + * that capacity is unusable until a future eSW change implements the full + * KDF expansion. Until then we cap the LKM at the 32-byte limit + * documented in Documentation/ABI/testing/cmh-mgmt. + */ +#define SM2_MAX_MSG_LEN 32U /* max plaintext for encrypt/decrypt */ +#define SM2_MAX_ID_LEN 32U /* max identity string */ +#define SM2_CT_OVERHEAD 96U /* C1(64) + C3(32) */ +#define SM2_MAX_CT_LEN (SM2_CT_OVERHEAD + SM2_MAX_MSG_LEN) /* 128 */ + +/* -- SM2_ECDH_KEYGEN ------------------- */ + +/** + * cmh_mgmt_sm2_ecdh_keygen() - Handle CMH_MGMT_IOC_SM2_ECDH_KEYGEN ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_ecdh_keygen(void __user *argp) +{ + struct cmh_ioctl_sm2_ecdh_keygen req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 core_id = cmh_core_default_id(CMH_CORE_PKE); + u8 *nonce_buf, *sk_buf; + dma_addr_t nonce_dma, sk_dma; + int nonce_dir; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN) + return -EINVAL; + + sk_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL); + nonce_buf = kzalloc(SM2_NONCE_LEN, GFP_KERNEL); + if (!sk_buf || !nonce_buf) { + ret = -ENOMEM; + goto out_free; + } + + /* + * nonce_len=32: caller provides ephemeral scalar r (DMA_TO_DEVICE). + * nonce_len=0: HW generates r and writes it back (DMA_FROM_DEVICE). + * The caller MUST supply a valid nonce pointer in both cases. + */ + if (req.nonce_len) { + if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce), + SM2_NONCE_LEN)) { + ret = -EFAULT; + goto out_free; + } + nonce_dir = DMA_TO_DEVICE; + } else { + nonce_dir = DMA_FROM_DEVICE; + } + + sk_dma = cmh_dma_map_single(sk_buf, SM2_POINT_LEN, DMA_FROM_DEVICE); + nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, nonce_dir); + if (cmh_dma_map_error(sk_dma) || cmh_dma_map_error(nonce_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_sm2_ecdh_keygen(&vcq[1], core_id, nonce_dma, sk_dma, + req.nonce_len, 0); + vcq_add_pke_flush(&vcq[2], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(nonce_dma)) + cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, nonce_dir); + if (!cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, SM2_POINT_LEN, DMA_FROM_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.session_key), + sk_buf, SM2_POINT_LEN)) + ret = -EFAULT; + /* Write back HW-generated nonce when nonce_len=0 */ + if (!ret && !req.nonce_len) { + if (copy_to_user(u64_to_user_ptr(req.nonce), + nonce_buf, SM2_NONCE_LEN)) + ret = -EFAULT; + } + } + +out_free: + kfree_sensitive(nonce_buf); + kfree_sensitive(sk_buf); + return ret; +} + +/* -- SM2_ECDH -------------------------- */ + +/** + * cmh_mgmt_sm2_ecdh() - Handle CMH_MGMT_IOC_SM2_ECDH ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_ecdh(void __user *argp) +{ + struct cmh_ioctl_sm2_ecdh req; + /* Phase 1: hdr + sys_new + sm2_ecdh + pke_flush */ + struct vcq_cmd vcq[4]; + u32 sp_type, core_id; + u8 *nonce_buf, *peer_pk_buf, *peer_sk_buf, *sp_buf; + u64 *ref_buf; + dma_addr_t nonce_dma, peer_pk_dma, peer_sk_dma, sp_dma, ref_dma; + int nonce_dir, ret, idx; + bool keep_ds; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN) + return -EINVAL; + + keep_ds = (req.shared_point_ref != 0); + sp_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE); + core_id = cmh_core_default_id(CMH_CORE_PKE); + + peer_pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + peer_sk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + sp_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL); + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + nonce_buf = kzalloc(SM2_NONCE_LEN, GFP_KERNEL); + if (!peer_pk_buf || !peer_sk_buf || !sp_buf || !ref_buf || + !nonce_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(peer_pk_buf, u64_to_user_ptr(req.peer_public_key), + SM2_POINT_LEN) || + copy_from_user(peer_sk_buf, u64_to_user_ptr(req.peer_session_key), + SM2_POINT_LEN)) { + ret = -EFAULT; + goto out_free; + } + + if (req.nonce_len) { + if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce), + SM2_NONCE_LEN)) { + ret = -EFAULT; + goto out_free; + } + nonce_dir = DMA_TO_DEVICE; + } else { + nonce_dir = DMA_FROM_DEVICE; + } + + peer_pk_dma = cmh_dma_map_single(peer_pk_buf, SM2_POINT_LEN, + DMA_TO_DEVICE); + peer_sk_dma = cmh_dma_map_single(peer_sk_buf, SM2_POINT_LEN, + DMA_TO_DEVICE); + sp_dma = cmh_dma_map_single(sp_buf, SM2_POINT_LEN, DMA_FROM_DEVICE); + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE); + nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, nonce_dir); + + if (cmh_dma_map_error(peer_pk_dma) || cmh_dma_map_error(peer_sk_dma) || + cmh_dma_map_error(sp_dma) || cmh_dma_map_error(ref_dma) || + cmh_dma_map_error(nonce_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + /* Phase 1: sys_new(shared_point_ref) + SM2_ECDH(->SYS_REF_LAST) */ + idx = 0; + vcq_set_header(&vcq[idx++], 4); + vcq_add_sys_new(&vcq[idx], 0, ref_dma, SM2_POINT_LEN); + vcq[idx++].id |= PKE_SWAP_FLAGS; + vcq_add_pke_sm2_ecdh(&vcq[idx++], core_id, req.nonce_len, SM2_CLEN, + nonce_dma, peer_pk_dma, peer_sk_dma, + req.key_ref, SYS_REF_LAST, sp_type, 0); + vcq_add_pke_flush(&vcq[idx++], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX); + if (ret) + goto out_unmap; + + if (!keep_ds) { + /* Sync bounce buffer so CPU sees the DMA-written ref */ + cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + + /* Phase 2: read shared point from DS -> DMA, consuming the slot */ + vcq_set_header(&vcq[0], 3); + vcq_add_sys_data(&vcq[1], *ref_buf, sp_dma, SM2_POINT_LEN); + vcq[1].id |= PKE_SWAP_FLAGS; + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX); + } + +out_unmap: + if (!cmh_dma_map_error(nonce_dma)) + cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, nonce_dir); + if (!cmh_dma_map_error(ref_dma)) + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!cmh_dma_map_error(sp_dma)) + cmh_dma_unmap_single(sp_dma, SM2_POINT_LEN, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(peer_sk_dma)) + cmh_dma_unmap_single(peer_sk_dma, SM2_POINT_LEN, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(peer_pk_dma)) + cmh_dma_unmap_single(peer_pk_dma, SM2_POINT_LEN, + DMA_TO_DEVICE); + + if (!ret) { + if (!keep_ds) { + if (copy_to_user(u64_to_user_ptr(req.shared_point), + sp_buf, SM2_POINT_LEN)) + ret = -EFAULT; + } else { + /* Return DS ref for ECDH_HASH to consume */ + u64 __user *sp_refp = (__u64 __user *) + u64_to_user_ptr(req.shared_point_ref); + + if (put_user(*ref_buf, sp_refp)) { + /* + * Failed to deliver the DS ref to + * userspace. Logically delete the + * orphaned slot so it does not leak. + */ + vcq_set_header(&vcq[0], 3); + vcq_add_sys_grant(&vcq[1], *ref_buf, + 0, 0, 0); + vcq_add_sys_flush(&vcq[2]); + cmh_tm_submit_sync_mbx(vcq, 3, 1, + MGMT_MBX); + dev_warn(cmh_dev(), "SM2 ECDH put_user failed, DS slot cleaned up\n"); + ret = -EFAULT; + } + } + /* Write back HW-generated nonce when nonce_len=0 */ + if (!ret && !req.nonce_len) { + if (copy_to_user(u64_to_user_ptr(req.nonce), + nonce_buf, SM2_NONCE_LEN)) + ret = -EFAULT; + } + } + +out_free: + kfree_sensitive(nonce_buf); + kfree(ref_buf); + kfree_sensitive(sp_buf); + kfree(peer_sk_buf); + kfree(peer_pk_buf); + return ret; +} + +/* -- SM2_DEC_POINT --------------------- */ + +/** + * cmh_mgmt_sm2_dec_point() - Handle CMH_MGMT_IOC_SM2_DEC_POINT ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_dec_point(void __user *argp) +{ + struct cmh_ioctl_sm2_dec_point req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 core_id = cmh_core_default_id(CMH_CORE_PKE); + u8 *ct_buf, *dp_buf; + dma_addr_t ct_dma, dp_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.ciphertext_len <= SM2_CT_OVERHEAD || + req.ciphertext_len > SM2_MAX_CT_LEN) + return -EINVAL; + + /* Only need C1 (first 64 bytes) for the sidecar */ + ct_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + dp_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL); + if (!ct_buf || !dp_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(ct_buf, u64_to_user_ptr(req.ciphertext), + SM2_POINT_LEN)) { + ret = -EFAULT; + goto out_free; + } + + ct_dma = cmh_dma_map_single(ct_buf, SM2_POINT_LEN, DMA_TO_DEVICE); + dp_dma = cmh_dma_map_single(dp_buf, SM2_POINT_LEN, DMA_FROM_DEVICE); + if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(dp_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_sm2_dec_point(&vcq[1], core_id, req.ciphertext_len, SM2_CLEN, + ct_dma, dp_dma, req.key_ref, + PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[2], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(dp_dma)) + cmh_dma_unmap_single(dp_dma, SM2_POINT_LEN, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ct_dma)) + cmh_dma_unmap_single(ct_dma, SM2_POINT_LEN, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.dec_point), + dp_buf, SM2_POINT_LEN)) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(dp_buf); + kfree(ct_buf); + return ret; +} + +/* -- SM2_ENC_POINT --------------------- */ + +/** + * cmh_mgmt_sm2_enc_point() - Handle CMH_MGMT_IOC_SM2_ENC_POINT ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_enc_point(void __user *argp) +{ + struct cmh_ioctl_sm2_enc_point req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 core_id = cmh_core_default_id(CMH_CORE_PKE); + u8 *nonce_buf = NULL, *pk_buf, *ct_buf, *ep_buf; + dma_addr_t nonce_dma = DMA_MAPPING_ERROR, pk_dma, ct_dma, ep_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.nonce_len != 0 && req.nonce_len != SM2_NONCE_LEN) + return -EINVAL; + + pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + ct_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL); + ep_buf = kzalloc(SM2_POINT_LEN, GFP_KERNEL); + if (!pk_buf || !ct_buf || !ep_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(pk_buf, u64_to_user_ptr(req.public_key), + SM2_POINT_LEN)) { + ret = -EFAULT; + goto out_free; + } + + if (req.nonce_len) { + nonce_buf = kmalloc(SM2_NONCE_LEN, GFP_KERNEL); + if (!nonce_buf) { + ret = -ENOMEM; + goto out_free; + } + if (copy_from_user(nonce_buf, u64_to_user_ptr(req.nonce), + SM2_NONCE_LEN)) { + ret = -EFAULT; + goto out_free; + } + } + + pk_dma = cmh_dma_map_single(pk_buf, SM2_POINT_LEN, DMA_TO_DEVICE); + ct_dma = cmh_dma_map_single(ct_buf, SM2_POINT_LEN, DMA_FROM_DEVICE); + ep_dma = cmh_dma_map_single(ep_buf, SM2_POINT_LEN, DMA_FROM_DEVICE); + if (nonce_buf) + nonce_dma = cmh_dma_map_single(nonce_buf, SM2_NONCE_LEN, + DMA_TO_DEVICE); + if (cmh_dma_map_error(pk_dma) || cmh_dma_map_error(ct_dma) || + cmh_dma_map_error(ep_dma) || + (nonce_buf && cmh_dma_map_error(nonce_dma))) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_sm2_enc_point(&vcq[1], core_id, nonce_dma, pk_dma, ct_dma, + ep_dma, req.nonce_len, 0); + vcq_add_pke_flush(&vcq[2], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (nonce_buf && !cmh_dma_map_error(nonce_dma)) + cmh_dma_unmap_single(nonce_dma, SM2_NONCE_LEN, DMA_TO_DEVICE); + if (!cmh_dma_map_error(ep_dma)) + cmh_dma_unmap_single(ep_dma, SM2_POINT_LEN, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ct_dma)) + cmh_dma_unmap_single(ct_dma, SM2_POINT_LEN, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, SM2_POINT_LEN, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.ciphertext), + ct_buf, SM2_POINT_LEN) || + copy_to_user(u64_to_user_ptr(req.enc_point), + ep_buf, SM2_POINT_LEN)) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(nonce_buf); + kfree(ep_buf); + kfree(ct_buf); + kfree(pk_buf); + return ret; +} + +/* -- SM2_ID_DIGEST --------------------- */ + +/** + * cmh_mgmt_sm2_id_digest() - Handle CMH_MGMT_IOC_SM2_ID_DIGEST ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_id_digest(void __user *argp) +{ + struct cmh_ioctl_sm2_id_digest req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 core_id = cmh_core_default_id(CMH_CORE_PKE); + u8 *id_buf, *pk_buf, *dig_buf; + dma_addr_t id_dma, pk_dma, dig_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.id_len || req.id_len > SM2_MAX_ID_LEN) + return -EINVAL; + + id_buf = kmalloc(req.id_len, GFP_KERNEL); + pk_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + dig_buf = kzalloc(SM2_DIGEST_LEN, GFP_KERNEL); + if (!id_buf || !pk_buf || !dig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(id_buf, u64_to_user_ptr(req.id), req.id_len) || + copy_from_user(pk_buf, u64_to_user_ptr(req.public_key), + SM2_POINT_LEN)) { + ret = -EFAULT; + goto out_free; + } + + id_dma = cmh_dma_map_single(id_buf, req.id_len, DMA_TO_DEVICE); + pk_dma = cmh_dma_map_single(pk_buf, SM2_POINT_LEN, DMA_TO_DEVICE); + dig_dma = cmh_dma_map_single(dig_buf, SM2_DIGEST_LEN, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(id_dma) || cmh_dma_map_error(pk_dma) || + cmh_dma_map_error(dig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_sm2_id_digest(&vcq[1], core_id, id_dma, pk_dma, dig_dma, + req.id_len, 0); + vcq_add_pke_flush(&vcq[2], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(dig_dma)) + cmh_dma_unmap_single(dig_dma, SM2_DIGEST_LEN, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, SM2_POINT_LEN, DMA_TO_DEVICE); + if (!cmh_dma_map_error(id_dma)) + cmh_dma_unmap_single(id_dma, req.id_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.digest), + dig_buf, SM2_DIGEST_LEN)) + ret = -EFAULT; + } + +out_free: + kfree(dig_buf); + kfree(pk_buf); + kfree(id_buf); + return ret; +} + +/* -- SM2_ECDH_HASH --------------------- */ + +/** + * cmh_mgmt_sm2_ecdh_hash() - Handle CMH_MGMT_IOC_SM2_ECDH_HASH ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_ecdh_hash(void __user *argp) +{ + struct cmh_ioctl_sm2_ecdh_hash req; + /* Phase 1: hdr + sys_new + sm2_ecdh_hash + pke_flush; reused for Phase 2 */ + struct vcq_cmd vcq[4]; + u32 sk_type, core_id; + u8 *peer_dig_buf, *dig_buf, *sk_buf; + u64 *ref_buf; + dma_addr_t peer_dig_dma, dig_dma, sk_dma, ref_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.__reserved) + return -EINVAL; + + sk_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE); + core_id = cmh_core_default_id(CMH_CORE_PKE); + + peer_dig_buf = kmalloc(SM2_DIGEST_LEN, GFP_KERNEL); + dig_buf = kmalloc(SM2_DIGEST_LEN, GFP_KERNEL); + sk_buf = kzalloc(SM2_SHARED_KEY_LEN, GFP_KERNEL); + ref_buf = kzalloc_obj(u64, GFP_KERNEL); + if (!peer_dig_buf || !dig_buf || !sk_buf || !ref_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(peer_dig_buf, u64_to_user_ptr(req.peer_id_digest), + SM2_DIGEST_LEN) || + copy_from_user(dig_buf, u64_to_user_ptr(req.id_digest), + SM2_DIGEST_LEN)) { + ret = -EFAULT; + goto out_free; + } + + peer_dig_dma = cmh_dma_map_single(peer_dig_buf, SM2_DIGEST_LEN, + DMA_TO_DEVICE); + dig_dma = cmh_dma_map_single(dig_buf, SM2_DIGEST_LEN, DMA_TO_DEVICE); + sk_dma = cmh_dma_map_single(sk_buf, SM2_SHARED_KEY_LEN, + DMA_FROM_DEVICE); + ref_dma = cmh_dma_map_single(ref_buf, sizeof(u64), DMA_FROM_DEVICE); + if (cmh_dma_map_error(peer_dig_dma) || cmh_dma_map_error(dig_dma) || + cmh_dma_map_error(sk_dma) || cmh_dma_map_error(ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + /* + * Phase 1: sys_new(shared_key_ref) + SM2_ECDH_HASH + * The shared_point_ref from the ECDH step is passed directly + * as a DS reference -- the eSW hub reads it from DS. + */ + vcq_set_header(&vcq[0], 4); + vcq_add_sys_new(&vcq[1], 0, ref_dma, SM2_SHARED_KEY_LEN); + vcq[1].id |= PKE_SWAP_FLAGS; + vcq_add_pke_sm2_ecdh_hash(&vcq[2], core_id, peer_dig_dma, dig_dma, + req.shared_point_ref, SYS_REF_LAST, + sk_type, PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[3], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, 4, 1, MGMT_MBX); + if (ret) + goto out_unmap; + + /* Sync bounce buffer so CPU sees the DMA-written ref */ + cmh_dma_sync_for_cpu(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + + /* Phase 2: read shared key from DS -> DMA */ + vcq_set_header(&vcq[0], 3); + vcq_add_sys_data(&vcq[1], *ref_buf, sk_dma, SM2_SHARED_KEY_LEN); + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_sync_mbx(vcq, 3, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(ref_dma)) + cmh_dma_unmap_single(ref_dma, sizeof(u64), DMA_FROM_DEVICE); + if (!cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, SM2_SHARED_KEY_LEN, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(dig_dma)) + cmh_dma_unmap_single(dig_dma, SM2_DIGEST_LEN, DMA_TO_DEVICE); + if (!cmh_dma_map_error(peer_dig_dma)) + cmh_dma_unmap_single(peer_dig_dma, SM2_DIGEST_LEN, + DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.shared_key), + sk_buf, SM2_SHARED_KEY_LEN)) + ret = -EFAULT; + } + +out_free: + kfree(ref_buf); + kfree_sensitive(sk_buf); + kfree(dig_buf); + kfree(peer_dig_buf); + return ret; +} + +/* -- SM2_DEC_HASH ---------------------- */ + +/** + * cmh_mgmt_sm2_dec_hash() - Handle CMH_MGMT_IOC_SM2_DEC_HASH ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_dec_hash(void __user *argp) +{ + struct cmh_ioctl_sm2_dec_hash req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 msg_len, core_id; + u8 *ct_buf, *dp_buf, *pt_buf; + dma_addr_t ct_dma, dp_dma, pt_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (req.ciphertext_len <= SM2_CT_OVERHEAD || + req.ciphertext_len > SM2_MAX_CT_LEN) + return -EINVAL; + + msg_len = req.ciphertext_len - SM2_CT_OVERHEAD; + core_id = cmh_core_default_id(CMH_CORE_PKE); + + ct_buf = kmalloc(req.ciphertext_len, GFP_KERNEL); + dp_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + pt_buf = kzalloc(msg_len, GFP_KERNEL); + if (!ct_buf || !dp_buf || !pt_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(ct_buf, u64_to_user_ptr(req.ciphertext), + req.ciphertext_len) || + copy_from_user(dp_buf, u64_to_user_ptr(req.dec_point), + SM2_POINT_LEN)) { + ret = -EFAULT; + goto out_free; + } + + ct_dma = cmh_dma_map_single(ct_buf, req.ciphertext_len, + DMA_TO_DEVICE); + dp_dma = cmh_dma_map_single(dp_buf, SM2_POINT_LEN, DMA_TO_DEVICE); + pt_dma = cmh_dma_map_single(pt_buf, msg_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(ct_dma) || cmh_dma_map_error(dp_dma) || + cmh_dma_map_error(pt_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_sm2_dec_hash(&vcq[1], core_id, ct_dma, dp_dma, pt_dma, + req.ciphertext_len, 0); + vcq_add_pke_flush(&vcq[2], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(pt_dma)) + cmh_dma_unmap_single(pt_dma, msg_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(dp_dma)) + cmh_dma_unmap_single(dp_dma, SM2_POINT_LEN, DMA_TO_DEVICE); + if (!cmh_dma_map_error(ct_dma)) + cmh_dma_unmap_single(ct_dma, req.ciphertext_len, + DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.plaintext), + pt_buf, msg_len)) + ret = -EFAULT; + } + +out_free: + kfree_sensitive(pt_buf); + kfree_sensitive(dp_buf); + kfree(ct_buf); + return ret; +} + +/* -- SM2_ENC_HASH ---------------------- */ + +/** + * cmh_mgmt_sm2_enc_hash() - Handle CMH_MGMT_IOC_SM2_ENC_HASH ioctl + * @argp: User-space ioctl argument pointer + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_mgmt_sm2_enc_hash(void __user *argp) +{ + struct cmh_ioctl_sm2_enc_hash req; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u32 ct_len, core_id; + u8 *msg_buf, *ep_buf, *ct_buf; + dma_addr_t msg_dma, ep_dma, ct_dma; + int ret; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + if (req.version != CMH_MGMT_V1) + return -EINVAL; + if (!req.message_len || req.message_len > SM2_MAX_MSG_LEN) + return -EINVAL; + + ct_len = SM2_CT_OVERHEAD + req.message_len; + core_id = cmh_core_default_id(CMH_CORE_PKE); + + msg_buf = kmalloc(req.message_len, GFP_KERNEL); + ep_buf = kmalloc(SM2_POINT_LEN, GFP_KERNEL); + ct_buf = kzalloc(ct_len, GFP_KERNEL); + if (!msg_buf || !ep_buf || !ct_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (copy_from_user(msg_buf, u64_to_user_ptr(req.message), + req.message_len) || + copy_from_user(ep_buf, u64_to_user_ptr(req.enc_point), + SM2_POINT_LEN)) { + ret = -EFAULT; + goto out_free; + } + + msg_dma = cmh_dma_map_single(msg_buf, req.message_len, DMA_TO_DEVICE); + ep_dma = cmh_dma_map_single(ep_buf, SM2_POINT_LEN, DMA_TO_DEVICE); + ct_dma = cmh_dma_map_single(ct_buf, ct_len, DMA_FROM_DEVICE); + if (cmh_dma_map_error(msg_dma) || cmh_dma_map_error(ep_dma) || + cmh_dma_map_error(ct_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_sm2_enc_hash(&vcq[1], core_id, msg_dma, ep_dma, ct_dma, + req.message_len, 0); + vcq_add_pke_flush(&vcq[2], core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, MGMT_MBX); + +out_unmap: + if (!cmh_dma_map_error(ct_dma)) + cmh_dma_unmap_single(ct_dma, ct_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(ep_dma)) + cmh_dma_unmap_single(ep_dma, SM2_POINT_LEN, DMA_TO_DEVICE); + if (!cmh_dma_map_error(msg_dma)) + cmh_dma_unmap_single(msg_dma, req.message_len, DMA_TO_DEVICE); + + if (!ret) { + if (copy_to_user(u64_to_user_ptr(req.ciphertext), + ct_buf, ct_len)) + ret = -EFAULT; + } + +out_free: + kfree(ct_buf); + kfree(ep_buf); + kfree_sensitive(msg_buf); + return ret; +} diff --git a/drivers/crypto/cmh/cmh_sys.c b/drivers/crypto/cmh/cmh_sys.c new file mode 100644 index 00000000000000..b01d058e6d8908 --- /dev/null +++ b/drivers/crypto/cmh/cmh_sys.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SYS Core VCQ Builders + * + * VCQ builder functions for SYS core datastore commands. Each function + * populates a single vcq_cmd slot. Callers (cmh_mgmt.c, cmh_key.c) + * assemble complete VCQs by combining header + command(s) + flush, + * then submit via cmh_tm_submit_sync(). + * + * Hardware-required datastore semantics + * -------------------------------------- + * The commands below (NEW, WRITE, DATA, FIND, DELETE, FLUSH) are + * direct mappings of the eSW firmware SYS core command set. The + * eSW maintains per-mailbox datastore namespaces with two object + * classes: + * + * SYS_REF_TEMP -- Temporary objects. Lifetime is scoped to the + * current mailbox slot; reclaimed automatically + * when the slot is reused or on explicit FLUSH. + * Used for raw-key provisioning on every VCQ. + * + * SYS_REF_PERSIST -- Persistent objects. Survive across slots; + * require explicit DELETE to reclaim. Identified + * by a 64-bit Content ID (CID) and resolved to + * a per-MBX ref via SYS_CMD_FIND. + * + * These semantics are hardware requirements, not driver policy. + * The per-MBX temp-stack and per-MBX ref namespace are eSW firmware + * design constraints that cannot be changed by the kernel driver. + */ + +#include + +#include "cmh_sys.h" + +/** + * vcq_add_sys_flush() - Build a SYS_FLUSH VCQ command + * @slot: VCQ command slot to populate + */ +void vcq_add_sys_flush(struct vcq_cmd *slot) +{ + vcq_add_flush(slot, CORE_ID_SYS); +} + +/** + * vcq_add_sys_new() - Build a SYS_NEW VCQ command + * @slot: VCQ command slot to populate + * @cid: Content identifier for the new datastore object + * @ref_dma: DMA address of the object reference buffer + * @len: Length of the object data in bytes + */ +void vcq_add_sys_new(struct vcq_cmd *slot, u64 cid, u64 ref_dma, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_NEW); + slot->hwc.sys.cmd_new.cid = cid; + slot->hwc.sys.cmd_new.ref = ref_dma; + slot->hwc.sys.cmd_new.len = len; +} + +/** + * vcq_add_sys_write() - Build a SYS_WRITE VCQ command + * @slot: VCQ command slot to populate + * @ref: Datastore object reference handle + * @src_dma: DMA address of source data buffer + * @wrap_key: Wrapping key reference (0 if none) + * @len: Length of data to write in bytes + * @sys_type: Datastore object type identifier + */ +void vcq_add_sys_write(struct vcq_cmd *slot, u64 ref, u64 src_dma, + u64 wrap_key, u32 len, u32 sys_type) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_WRITE); + slot->hwc.sys.cmd_write.ref = ref; + slot->hwc.sys.cmd_write.src = src_dma; + slot->hwc.sys.cmd_write.key = wrap_key; + slot->hwc.sys.cmd_write.len = len; + slot->hwc.sys.cmd_write.type = sys_type; +} + +/** + * vcq_add_sys_read() - Build a SYS_READ VCQ command + * @slot: VCQ command slot to populate + * @ref: Datastore object reference handle + * @dst_dma: DMA address of destination buffer + * @wrap_key: Wrapping key reference (0 if none) + * @len: Length of data to read in bytes + */ +void vcq_add_sys_read(struct vcq_cmd *slot, u64 ref, u64 dst_dma, + u64 wrap_key, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_READ); + slot->hwc.sys.cmd_read.ref = ref; + slot->hwc.sys.cmd_read.dst = dst_dma; + slot->hwc.sys.cmd_read.key = wrap_key; + slot->hwc.sys.cmd_read.len = len; +} + +/** + * vcq_add_sys_data() - Build a SYS_DATA VCQ command + * @slot: VCQ command slot to populate + * @ref: Datastore object reference handle + * @dst_dma: DMA address of destination buffer + * @len: Length of data section to read in bytes + */ +void vcq_add_sys_data(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_DATA); + slot->hwc.sys.cmd_data.ref = ref; + slot->hwc.sys.cmd_data.dst = dst_dma; + slot->hwc.sys.cmd_data.len = len; +} + +/** + * vcq_add_sys_find() - Build a SYS_FIND VCQ command + * @slot: VCQ command slot to populate + * @cid: Content identifier to search for + * @dst_dma: DMA address of destination buffer for result + * @len: Length of destination buffer in bytes + */ +void vcq_add_sys_find(struct vcq_cmd *slot, u64 cid, u64 dst_dma, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_FIND); + slot->hwc.sys.cmd_find.cid = cid; + slot->hwc.sys.cmd_find.dst = dst_dma; + slot->hwc.sys.cmd_find.len = len; +} + +/** + * vcq_add_sys_list() - Build a SYS_LIST VCQ command + * @slot: VCQ command slot to populate + * @ref: Datastore object reference for enumeration start + * @dst_dma: DMA address of destination buffer for list + * @len: Length of destination buffer in bytes + */ +void vcq_add_sys_list(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_LIST); + slot->hwc.sys.cmd_list.ref = ref; + slot->hwc.sys.cmd_list.dst = dst_dma; + slot->hwc.sys.cmd_list.len = len; +} + +/** + * vcq_add_sys_grant() - Build a SYS_GRANT VCQ command + * @slot: VCQ command slot to populate + * @ref: Datastore object reference handle + * @read: Read permission bitmask + * @write: Write permission bitmask + * @execute: Execute permission bitmask + */ +void vcq_add_sys_grant(struct vcq_cmd *slot, u64 ref, u64 read, + u64 write, u64 execute) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_GRANT); + slot->hwc.sys.cmd_grant.ref = ref; + slot->hwc.sys.cmd_grant.read = read; + slot->hwc.sys.cmd_grant.write = write; + slot->hwc.sys.cmd_grant.execute = execute; +} + +/** + * vcq_add_sys_export() - Build a SYS_EXPORT VCQ command + * @slot: VCQ command slot to populate + * @cid: Content identifier of object to export + * @dst_dma: DMA address of destination buffer for wrapped blob + * @wrap_key: Wrapping key reference for export + * @len: Length of destination buffer in bytes + */ +void vcq_add_sys_export(struct vcq_cmd *slot, u64 cid, u64 dst_dma, + u64 wrap_key, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_EXPORT); + slot->hwc.sys.cmd_export.cid = cid; + slot->hwc.sys.cmd_export.dst = dst_dma; + slot->hwc.sys.cmd_export.key = wrap_key; + slot->hwc.sys.cmd_export.len = len; +} + +/** + * vcq_add_sys_import() - Build a SYS_IMPORT VCQ command + * @slot: VCQ command slot to populate + * @src_dma: DMA address of wrapped datastore blob to import + * @wrap_key: Wrapping key reference for unwrapping + * @len: Length of wrapped blob in bytes + */ +void vcq_add_sys_import(struct vcq_cmd *slot, u64 src_dma, + u64 wrap_key, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_SYS, 0, 1, SYS_CMD_IMPORT); + slot->hwc.sys.cmd_import.src = src_dma; + slot->hwc.sys.cmd_import.key = wrap_key; + slot->hwc.sys.cmd_import.len = len; +} + +/* -- KIC Core VCQ Builders --------------------- */ + +/** + * vcq_add_kic_hkdf1() - Build a KIC HKDF-Expand VCQ command + * @slot: VCQ command slot to populate + * @dst: Datastore reference for derived key output + * @base: Datastore reference for base key input + * @label_dma: DMA address of HKDF label/info buffer + * @key_len: Derived key length in bytes + * @label_len: Length of label buffer in bytes + * @type: Derived key datastore type + */ +void vcq_add_kic_hkdf1(struct vcq_cmd *slot, u64 dst, u64 base, + u64 label_dma, u32 key_len, u32 label_len, u32 type) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_HKDF1); + slot->hwc.kic.cmd_hkdf1.dst = dst; + slot->hwc.kic.cmd_hkdf1.base = base; + slot->hwc.kic.cmd_hkdf1.label = label_dma; + slot->hwc.kic.cmd_hkdf1.llen = label_len; + slot->hwc.kic.cmd_hkdf1.len = key_len; + slot->hwc.kic.cmd_hkdf1.type = type; +} + +/** + * vcq_add_kic_hkdf2() - Build a KIC HKDF-with-salt VCQ command + * @slot: VCQ command slot to populate + * @dst: Datastore reference for derived key output + * @base: Datastore reference for base key input + * @salt: Datastore reference for HKDF salt key + * @label_dma: DMA address of HKDF label/info buffer + * @key_len: Derived key length in bytes + * @label_len: Length of label buffer in bytes + * @type: Derived key datastore type + */ +void vcq_add_kic_hkdf2(struct vcq_cmd *slot, u64 dst, u64 base, u64 salt, + u64 label_dma, u32 key_len, u32 label_len, u32 type) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_HKDF2); + slot->hwc.kic.cmd_hkdf2.dst = dst; + slot->hwc.kic.cmd_hkdf2.base = base; + slot->hwc.kic.cmd_hkdf2.salt = salt; + slot->hwc.kic.cmd_hkdf2.label = label_dma; + slot->hwc.kic.cmd_hkdf2.llen = label_len; + slot->hwc.kic.cmd_hkdf2.len = key_len; + slot->hwc.kic.cmd_hkdf2.type = type; +} + +/** + * vcq_add_kic_aes_cmac_kdf() - Build a KIC AES-CMAC KDF VCQ command + * @slot: VCQ command slot to populate + * @out_key: Datastore reference for derived key output + * @base_key: Datastore reference for base key input + * @label_dma: DMA address of KDF label buffer + * @key_len: Derived key length in bytes + * @label_len: Length of label buffer in bytes + * @type: Derived key datastore type + */ +void vcq_add_kic_aes_cmac_kdf(struct vcq_cmd *slot, u64 out_key, u64 base_key, + u64 label_dma, u32 key_len, u32 label_len, + u32 type) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_AES_CMAC_KDF); + slot->hwc.kic.cmd_aes_cmac_kdf.base_key = base_key; + slot->hwc.kic.cmd_aes_cmac_kdf.out_key = out_key; + slot->hwc.kic.cmd_aes_cmac_kdf.label = label_dma; + slot->hwc.kic.cmd_aes_cmac_kdf.key_len = key_len; + slot->hwc.kic.cmd_aes_cmac_kdf.label_len = label_len; + slot->hwc.kic.cmd_aes_cmac_kdf.type = type; +} + +/** + * vcq_add_kic_dkek_derive() - Build a KIC DKEK derivation VCQ command + * @slot: VCQ command slot to populate + * @out_key: Datastore reference for derived DKEK output + * @base_key: Datastore reference for base key input + * @host_id: Host identifier for key binding + * @metadata_dma: DMA address of derivation metadata buffer + * @metadata_len: Length of metadata buffer in bytes + */ +void vcq_add_kic_dkek_derive(struct vcq_cmd *slot, u64 out_key, u64 base_key, + u32 host_id, u64 metadata_dma, u32 metadata_len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_KIC, 0, 1, KIC_CMD_DKEK_DERIVE); + slot->hwc.kic.cmd_dkek_derive.base_key = base_key; + slot->hwc.kic.cmd_dkek_derive.out_key = out_key; + slot->hwc.kic.cmd_dkek_derive.host_id = host_id; + slot->hwc.kic.cmd_dkek_derive.metadata = metadata_dma; + slot->hwc.kic.cmd_dkek_derive.metadata_len = metadata_len; +} + +/* -- DRBG Core VCQ Builders -------------------- */ + +/** + * vcq_add_drbg_reset() - Build a DRBG reset VCQ command + * @slot: VCQ command slot to populate + * + * Issues DRBG_CMD_RESET which clears the instantiated state, allowing + * a subsequent CONFIG to proceed without a double-instantiate error. + */ +void vcq_add_drbg_reset(struct vcq_cmd *slot) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_RESET); +} + +/** + * vcq_add_drbg_config() - Build a DRBG configuration VCQ command + * @slot: VCQ command slot to populate + * @ratio: Entropy-to-output ratio + * @strength: Security strength in bits + */ +void vcq_add_drbg_config(struct vcq_cmd *slot, u32 ratio, u32 strength) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_CONFIG); + slot->hwc.drbg.cmd_config.entropy_ratio = ratio; + slot->hwc.drbg.cmd_config.security_strength = strength; +} + +/** + * vcq_add_drbg_datastore() - Build a DRBG datastore setup VCQ command + * @slot: VCQ command slot to populate + * @ref: Datastore object reference handle + * @len: Length of datastore allocation in bytes + * @type: Datastore object type + */ +void vcq_add_drbg_datastore(struct vcq_cmd *slot, u64 ref, u32 len, u32 type) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_DATASTORE); + slot->hwc.drbg.cmd_datastore.ref = ref; + slot->hwc.drbg.cmd_datastore.len = len; + slot->hwc.drbg.cmd_datastore.type = type; +} + +/* -- EAC Core VCQ Builder ---------------------- */ + +/** + * vcq_add_eac_read() - Build an EAC read VCQ command + * @slot: VCQ command slot to populate + * @dst_dma: DMA address of destination buffer + * @len: Length of data to read in bytes + */ +void vcq_add_eac_read(struct vcq_cmd *slot, u64 dst_dma, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_EAC, 0, 1, EAC_CMD_READ); + slot->hwc.eac.cmd_read.dst = dst_dma; + slot->hwc.eac.cmd_read.len = len; +} diff --git a/drivers/crypto/cmh/include/cmh_key.h b/drivers/crypto/cmh/include/cmh_key.h new file mode 100644 index 00000000000000..bad69c92b89245 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_key.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Per-transform key context + * + * Per-transform key context used by all keyed crypto algorithms (AES, + * SM4, CCP, HMAC, KMAC). Stores raw key bytes supplied via the crypto + * API .setkey() callback: the key is DMA-mapped once at setkey time and + * written to SYS_REF_TEMP in every VCQ. + * + * Each keyed algorithm driver embeds a struct cmh_key_ctx in its + * per-transform context and calls cmh_key_setkey_raw() from its + * .setkey() callback. + * + * Raw-key atomicity (SYS_REF_TEMP) + * --------------------------------- + * SYS_CMD_WRITE to SYS_REF_TEMP is packed into the same VCQ as the + * algorithm commands (AES_CMD_INIT, HC_CMD_HMAC, etc.). SYS_REF_TEMP + * is per-MBX -- the CMH eSW allocates it in the tail of each mailbox's + * own VCQ buffer (mbx_alloc_temp), so concurrent raw-key requests on + * different MBXes do not interfere. + */ + +#ifndef CMH_KEY_H +#define CMH_KEY_H + +#include +#include "cmh_config.h" +#include "cmh_vcq.h" + +/* Key context mode */ +enum cmh_key_mode { + CMH_KEY_NONE = 0, /* no key set yet */ + CMH_KEY_RAW, /* raw key bytes in memory */ +}; + +/* Per-transform key context */ +struct cmh_key_ctx { + enum cmh_key_mode mode; + struct { + u8 *data; /* kmemdup'd raw key bytes */ + u32 len; /* key length in bytes */ + u32 sys_type; /* SYS_TYPE_SET(flags, core_id) */ + dma_addr_t dma; /* pre-mapped DMA addr (DMA_TO_DEVICE) */ + } raw; +}; + +/** + * cmh_key_setkey_raw() - Store raw key bytes in the transform context + * @ctx: Per-transform key context + * @key: Raw key bytes + * @keylen: Key length in bytes + * @core_id: Target algorithm core (e.g. CORE_ID_AES) + * + * SYS_TYPE_FLAG_PT is set so the written temp key + * can be read back as plaintext if needed. The actual SYS_CMD_WRITE + * to SYS_REF_TEMP is deferred to each encrypt/decrypt VCQ, where it + * is packed inline for atomicity. + * + * Return: 0 on success, -ENOMEM on allocation failure. + */ +int cmh_key_setkey_raw(struct cmh_key_ctx *ctx, const u8 *key, + u32 keylen, u32 core_id); + +/** + * cmh_key_destroy() - Free key resources + * @ctx: Per-transform key context + * + * Zeroises and frees the raw key buffer. + */ +void cmh_key_destroy(struct cmh_key_ctx *ctx); + +/** + * cmh_ds_type_to_core_id() - Map datastore key type to core ID + * @ds_type: CMH_DS_* key type constant + * + * Return: Corresponding CORE_ID_*, or CORE_ID_NUM (0x1F) on + * unrecognised type (caller should return -EINVAL). + */ +u32 cmh_ds_type_to_core_id(u32 ds_type); + +#endif /* CMH_KEY_H */ diff --git a/drivers/crypto/cmh/include/cmh_mgmt.h b/drivers/crypto/cmh/include/cmh_mgmt.h new file mode 100644 index 00000000000000..b211014bd71dd8 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_mgmt.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH -- Key Management misc_device (/dev/cmh_mgmt) + * + * ioctl interface for key CRUD + datastore export/import, + * PKE operations (RSA, ECDSA, ECDH, EdDSA), + * and PQC operations (ML-KEM, ML-DSA, SLH-DSA). + * + * Registered alongside crypto algorithms in module_init, + * unregistered before them in module_exit. + */ + +#ifndef CMH_MGMT_H +#define CMH_MGMT_H + +#ifdef CONFIG_CRYPTO_DEV_CMH_MGMT + +/* + * Pin all mgmt ioctls to MBX 0 for DS ownership and SYS_REF_TEMP scope. + * Shared by cmh_mgmt.c, cmh_mgmt_pke.c, cmh_mgmt_pqc.c, cmh_pke_sm2.c. + */ +#define MGMT_MBX 0 + +/* Maximum DMA buffer size for key data / datastore blobs */ +#define CMH_MGMT_MAX_DATA_LEN (256 * 1024) /* 256 KB */ + +int cmh_mgmt_register(void); +void cmh_mgmt_unregister(void); + +/* -- PKE ioctl handlers (cmh_mgmt_pke.c) -- */ +int cmh_mgmt_pke_rsa_enc(void __user *argp); +int cmh_mgmt_pke_rsa_dec(void __user *argp); +int cmh_mgmt_pke_rsa_crt_dec(void __user *argp); +int cmh_mgmt_pke_rsa_keygen(void __user *argp); +int cmh_mgmt_pke_ecdsa_sign(void __user *argp); +int cmh_mgmt_pke_ecdh(void __user *argp); +int cmh_mgmt_pke_ecdh_keygen(void __user *argp); +int cmh_mgmt_pke_eddsa_sign(void __user *argp); +int cmh_mgmt_pke_eddsa_verify(void __user *argp); +int cmh_mgmt_pke_ec_keygen(void __user *argp); +int cmh_mgmt_pke_ec_pubgen(void __user *argp); +int cmh_mgmt_pke_eddsa_keygen_sca(void __user *argp); + +/* -- PQC ioctl handlers (cmh_mgmt_pqc.c) -- */ +int cmh_mgmt_ml_kem_keygen(void __user *argp); +int cmh_mgmt_ml_kem_enc(void __user *argp); +int cmh_mgmt_ml_kem_dec(void __user *argp); +int cmh_mgmt_ml_dsa_keygen(void __user *argp); +int cmh_mgmt_ml_dsa_sign(void __user *argp); +int cmh_mgmt_slhdsa_keygen(void __user *argp); +int cmh_mgmt_slhdsa_sign(void __user *argp); +int cmh_mgmt_slhdsa_sign_prehash(void __user *argp); + +#else /* !CONFIG_CRYPTO_DEV_CMH_MGMT */ + +static inline int cmh_mgmt_register(void) { return 0; } +static inline void cmh_mgmt_unregister(void) { } + +#endif /* CONFIG_CRYPTO_DEV_CMH_MGMT */ + +#endif /* CMH_MGMT_H */ diff --git a/drivers/crypto/cmh/include/cmh_pke.h b/drivers/crypto/cmh/include/cmh_pke.h new file mode 100644 index 00000000000000..dcfdb3fc3cd617 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_pke.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- PKE Common Types and Helpers + * + * Shared definitions for RSA, ECDSA, ECDH, EdDSA, and SM2 drivers. + * Curve -> coordinate-length mapping, VCQ byte-swap flags, and + * common VCQ builder prototypes. + */ + +#ifndef CMH_PKE_H +#define CMH_PKE_H + +#include +#include "cmh_vcq.h" +#include "cmh_pke_abi.h" + +/* VCQ byte-swap flags for DMA transfers (per CMH VCQ ABI) */ +#define VCQ_FLAG_SWAP_BYTES 0x400000U +#define VCQ_FLAG_SWAP_WORDS 0x200000U + +/* VCQ byte-swap flags for PKE -- big-endian data on LE bus */ +#define PKE_SWAP_FLAGS (VCQ_FLAG_SWAP_BYTES | VCQ_FLAG_SWAP_WORDS) + +/* VCQ layout: header + [SYS_WRITE] + PKE_CMD + flush */ +#define PKE_VCQ_CMDS_MIN 3 /* header + cmd + flush */ +#define PKE_VCQ_CMDS_MAX 4 /* header + SYS_WRITE + cmd + flush */ + +/* Max RSA key size in bytes (4096 bits) */ +#define PKE_RSA_MAX_BYTES 512 +#define PKE_RSA_MIN_BITS 1024 +#define PKE_RSA_MAX_BITS 4096 + +/* EdDSA SCA: Ed448 blinded private key length (bytes) */ +#define PKE_ED448_SK_SCA_LEN 226 + +/** + * pke_curve_clen() - Get EC curve coordinate length in bytes + * @curve: PKE curve identifier (PKE_CURVE_*) + * + * Return: Coordinate length in bytes, or 0 for unknown curves. + */ +static inline u32 pke_curve_clen(u32 curve) +{ + switch (curve) { + case PKE_CURVE_P192: + case PKE_CURVE_BP192R1: + return 24; + case PKE_CURVE_P224: + case PKE_CURVE_BP224R1: + return 28; + case PKE_CURVE_P256: + case PKE_CURVE_SECP256K1: + case PKE_CURVE_BP256R1: + case PKE_CURVE_ANSSI_FRP256V1: + case PKE_CURVE_SM2: + case PKE_CURVE_25519: + return 32; + case PKE_CURVE_BP320R1: + return 40; + case PKE_CURVE_P384: + case PKE_CURVE_BP384R1: + return 48; + case PKE_CURVE_BP512R1: + return 64; + case PKE_CURVE_P521: + return 68; /* ceil(521/8) = 66, ABI uses ALIGN(66, 4) = 68 */ + case PKE_CURVE_448: + return 56; + default: + return 0; + } +} + +/** + * pke_curve_bits() - Get EC curve size in bits + * @curve: PKE curve identifier (PKE_CURVE_*) + * + * Return: Curve size in bits, or 0 for unknown curves. + */ +static inline u32 pke_curve_bits(u32 curve) +{ + switch (curve) { + case PKE_CURVE_P192: + case PKE_CURVE_BP192R1: + return 192; + case PKE_CURVE_P224: + case PKE_CURVE_BP224R1: + return 224; + case PKE_CURVE_P256: + case PKE_CURVE_SECP256K1: + case PKE_CURVE_BP256R1: + case PKE_CURVE_ANSSI_FRP256V1: + case PKE_CURVE_SM2: + case PKE_CURVE_25519: + return 256; + case PKE_CURVE_BP320R1: + return 320; + case PKE_CURVE_P384: + case PKE_CURVE_BP384R1: + return 384; + case PKE_CURVE_BP512R1: + return 512; + case PKE_CURVE_P521: + return 521; + case PKE_CURVE_448: + return 448; + default: + return 0; + } +} + +/** + * pke_eddsa_key_len() - Get EdDSA key/pubkey length + * @curve: PKE curve identifier (PKE_CURVE_25519 or PKE_CURVE_448) + * + * Ed25519 uses 32 bytes (== clen), Ed448 uses 57 bytes (clen + 1 + * flag byte per RFC 8032). Signature length is 2 * pke_eddsa_key_len(). + * + * Return: Key length in bytes. + */ +static inline u32 pke_eddsa_key_len(u32 curve) +{ + u32 clen = pke_curve_clen(curve); + + return (curve == PKE_CURVE_448) ? clen + 1 : clen; +} + +/** + * pke_curve_is_edwards() - Check if curve uses Edwards form + * @curve: PKE curve identifier (PKE_CURVE_*) + * + * Return: true for Curve25519 and Curve448, false otherwise. + */ +static inline bool pke_curve_is_edwards(u32 curve) +{ + return curve == PKE_CURVE_25519 || curve == PKE_CURVE_448; +} + +/** + * pke_swap_flags() - Get VCQ byte-swap flags for a given curve + * @curve: PKE curve identifier (PKE_CURVE_*) + * + * Weierstrass curves need byte+word swap; Edwards curves do not. + * + * Return: VCQ swap flags to OR into the command ID. + */ +static inline u32 pke_swap_flags(u32 curve) +{ + return pke_curve_is_edwards(curve) ? 0 : PKE_SWAP_FLAGS; +} + +/* Common VCQ builder prototypes */ + +void vcq_add_pke_flush(struct vcq_cmd *slot, u32 core_id); + +void vcq_add_pke_rsa_enc(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len, + u64 e_dma, u64 n_dma, u64 m_dma, u64 c_dma, + u32 flags); + +void vcq_add_pke_rsa_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len, + u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma, + u64 d_ref, u32 flags); + +void vcq_add_pke_rsa_crt_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len, + u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma, + u64 crt_ref, u32 flags); + +void vcq_add_pke_ecdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen, + u64 pk_dma, u64 dig_dma, u64 sig_dma, + u64 rp_dma, u32 flags); + +void vcq_add_pke_ecdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 dig_dma, u64 sig_dma, u64 sk_ref, + u32 dlen, u32 flags); + +void vcq_add_pke_ecdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 pk_dma, u64 sk_ref, u32 flags); + +void vcq_add_pke_ecdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 sk_ref, u32 sk_type, u32 flags); + +void vcq_add_pke_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 pkx_dma, u64 sk_ref, u32 flags); + +void vcq_add_pke_ecdh(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u32 sslen, u32 ss_type, u64 peer_dma, u64 sk_ref, + u64 ss_ref, u32 flags); + +void vcq_add_pke_eddsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen, + u64 pky_dma, u64 dig_dma, u64 sig_dma, + u64 rp_dma, u32 flags); + +void vcq_add_pke_eddsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 dig_dma, u64 sig_dma, u64 sk_ref, + u32 dlen, u32 flags); + +void vcq_add_pke_eddsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 pky_dma, u64 sk_ref, u32 flags); + +void vcq_add_pke_eddsa_keygen_sca(struct vcq_cmd *slot, u32 core_id, u32 curve, + u64 sk_ref, u64 sca_sk_ref); + +/* SM2 VCQ builders */ + +void vcq_add_pke_sm2_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma, + u64 session_key_dma, u32 nonce_len, u32 flags); + +void vcq_add_pke_sm2_ecdh(struct vcq_cmd *slot, u32 core_id, u32 nonce_len, + u32 private_key_len, u64 nonce_dma, + u64 peer_pk_dma, u64 peer_sk_dma, + u64 priv_ref, u64 sp_ref, u32 sp_type, u32 flags); + +void vcq_add_pke_sm2_dec_point(struct vcq_cmd *slot, u32 core_id, u32 ct_len, + u32 pk_len, u64 ct_dma, u64 dp_dma, + u64 priv_ref, u32 flags); + +void vcq_add_pke_sm2_enc_point(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma, + u64 pk_dma, u64 ct_dma, u64 ep_dma, + u32 nonce_len, u32 flags); + +void vcq_add_pke_sm2_id_digest(struct vcq_cmd *slot, u32 core_id, u64 id_dma, + u64 pk_dma, u64 dig_dma, u32 id_len, + u32 flags); + +void vcq_add_pke_sm2_ecdh_hash(struct vcq_cmd *slot, u32 core_id, u64 peer_dig_dma, + u64 dig_dma, u64 sp_ref, u64 sk_ref, + u32 sk_type, u32 flags); + +void vcq_add_pke_sm2_dec_hash(struct vcq_cmd *slot, u32 core_id, u64 ct_dma, + u64 dp_dma, u64 pt_dma, u32 ct_len, u32 flags); + +void vcq_add_pke_sm2_enc_hash(struct vcq_cmd *slot, u32 core_id, u64 msg_dma, + u64 ep_dma, u64 ct_dma, u32 msg_len, u32 flags); + +/* Registration */ + +int cmh_pke_rsa_register(void); +void cmh_pke_rsa_unregister(void); +int cmh_pke_ecdsa_register(void); +void cmh_pke_ecdsa_unregister(void); +int cmh_pke_ecdh_register(void); +void cmh_pke_ecdh_unregister(void); + +#endif /* CMH_PKE_H */ diff --git a/drivers/crypto/cmh/include/cmh_pke_sm2.h b/drivers/crypto/cmh/include/cmh_pke_sm2.h new file mode 100644 index 00000000000000..a2c7164b8d496e --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_pke_sm2.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM2 PKE Ioctl Handler Declarations + * + * SM2 signature (GM/T 0003.2) requires the caller to compute + * ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA) + * and pass SM3(ZA || M) as the digest to the sign/verify path. + * The CMH eSW does NOT compute ZA internally; the full + * identity pre-hash is the caller's responsibility. + * + * For the in-kernel akcipher "sm2" algorithm this means the + * caller (e.g. asymmetric_key subsystem) must pre-hash with ZA + * before invoking verify. The SM2_ID_DIGEST ioctl below can + * compute ZA for userspace callers of the misc-device path. + */ + +#ifndef CMH_PKE_SM2_H +#define CMH_PKE_SM2_H + +int cmh_mgmt_sm2_ecdh_keygen(void __user *argp); +int cmh_mgmt_sm2_ecdh(void __user *argp); +int cmh_mgmt_sm2_dec_point(void __user *argp); +int cmh_mgmt_sm2_enc_point(void __user *argp); +int cmh_mgmt_sm2_id_digest(void __user *argp); +int cmh_mgmt_sm2_ecdh_hash(void __user *argp); +int cmh_mgmt_sm2_dec_hash(void __user *argp); +int cmh_mgmt_sm2_enc_hash(void __user *argp); + +#endif /* CMH_PKE_SM2_H */ diff --git a/drivers/crypto/cmh/include/cmh_pqc.h b/drivers/crypto/cmh/include/cmh_pqc.h new file mode 100644 index 00000000000000..cd4761a0ce5c16 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_pqc.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- PQC Algorithm Registration + * + * Registration/unregistration functions for PQC akcipher algorithms: + * ML-DSA, SLH-DSA, LMS, XMSS. + */ + +#ifndef CMH_PQC_H +#define CMH_PQC_H + +int cmh_pqc_mldsa_register(void); +void cmh_pqc_mldsa_unregister(void); + +int cmh_pqc_slhdsa_register(void); +void cmh_pqc_slhdsa_unregister(void); + +int cmh_pqc_lms_register(void); +void cmh_pqc_lms_unregister(void); + +int cmh_pqc_xmss_register(void); +void cmh_pqc_xmss_unregister(void); + +#endif /* CMH_PQC_H */ diff --git a/drivers/crypto/cmh/include/cmh_sys.h b/drivers/crypto/cmh/include/cmh_sys.h new file mode 100644 index 00000000000000..dd336b67bd653a --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sys.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SYS Core VCQ Builders + * + * VCQ builder functions for SYS core commands (NEW, WRITE, READ, + * FIND, GRANT, DATA, EXPORT, IMPORT). Each builder populates one + * vcq_cmd slot with the appropriate magic, command ID, and payload. + * + * Callers combine these with vcq_set_header() + vcq_add_flush() + * and submit via cmh_tm_submit_sync(). + */ + +#ifndef CMH_SYS_H +#define CMH_SYS_H + +#include "cmh_vcq.h" + +void vcq_add_sys_new(struct vcq_cmd *slot, u64 cid, u64 ref_dma, u32 len); +void vcq_add_sys_write(struct vcq_cmd *slot, u64 ref, u64 src_dma, + u64 wrap_key, u32 len, u32 sys_type); +void vcq_add_sys_read(struct vcq_cmd *slot, u64 ref, u64 dst_dma, + u64 wrap_key, u32 len); +void vcq_add_sys_data(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len); +void vcq_add_sys_find(struct vcq_cmd *slot, u64 cid, u64 dst_dma, u32 len); +void vcq_add_sys_list(struct vcq_cmd *slot, u64 ref, u64 dst_dma, u32 len); +void vcq_add_sys_grant(struct vcq_cmd *slot, u64 ref, u64 read, + u64 write, u64 execute); +void vcq_add_sys_export(struct vcq_cmd *slot, u64 cid, u64 dst_dma, + u64 wrap_key, u32 len); +void vcq_add_sys_import(struct vcq_cmd *slot, u64 src_dma, + u64 wrap_key, u32 len); + +/* KIC core VCQ builders */ +void vcq_add_kic_hkdf1(struct vcq_cmd *slot, u64 dst, u64 base, + u64 label_dma, u32 key_len, u32 label_len, u32 type); +void vcq_add_kic_hkdf2(struct vcq_cmd *slot, u64 dst, u64 base, u64 salt, + u64 label_dma, u32 key_len, u32 label_len, u32 type); +void vcq_add_kic_aes_cmac_kdf(struct vcq_cmd *slot, u64 out_key, u64 base_key, + u64 label_dma, u32 key_len, u32 label_len, + u32 type); +void vcq_add_kic_dkek_derive(struct vcq_cmd *slot, u64 out_key, u64 base_key, + u32 host_id, u64 metadata_dma, u32 metadata_len); + +/* DRBG core VCQ builders */ +void vcq_add_drbg_reset(struct vcq_cmd *slot); +void vcq_add_drbg_config(struct vcq_cmd *slot, u32 ratio, u32 strength); +void vcq_add_drbg_datastore(struct vcq_cmd *slot, u64 ref, u32 len, u32 type); + +/* QSE core VCQ builders */ +void vcq_add_qse_flush(struct vcq_cmd *slot, u32 core_id); +void vcq_add_qse_ml_kem_keygen(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags, + u64 seed, u64 z, u64 ek, u64 dk, u32 dk_type, + bool masked); +void vcq_add_qse_ml_kem_enc(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags, + u64 coin, u64 ek, u64 ct, u64 ss, u32 ss_type, + bool masked); +void vcq_add_qse_ml_kem_dec(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags, + u64 ct, u64 dk, u64 ss, u32 ss_type, + bool masked); +void vcq_add_qse_ml_dsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags, + u64 seed, u64 pk, u64 sk, u32 sk_type, + bool masked); +void vcq_add_qse_ml_dsa_sign(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags, + u64 rnd, u64 m, u64 sk, u64 sig, u32 mlen, + bool masked); +void vcq_add_qse_ml_dsa_verify(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags, + u64 m, u64 pk, u64 sig, u32 mlen); + +/* HCQ core VCQ builders */ +void vcq_add_hcq_flush(struct vcq_cmd *slot, u32 core_id); +void vcq_add_hcq_slhdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 seed_len, u32 pk_len, u32 sk_len, + u64 seed, u64 pk, u64 sk); +void vcq_add_hcq_slhdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u32 ctx_len, + u64 add_random, u64 msg, u64 ctx, + u64 sk, u64 sig); +void vcq_add_hcq_slhdsa_sign_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u64 add_random, + u64 msg, u64 sk, u64 sig); +void vcq_add_hcq_slhdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u32 ctx_len, + u64 msg, u64 ctx, u64 pk, u64 sig); +void vcq_add_hcq_slhdsa_sign_prehash(struct vcq_cmd *slot, u32 core_id, + u32 cmd, u32 param_set, u32 prehash_algo, + u32 msg_len, u32 ctx_len, + u64 add_random, u64 msg, u64 ctx, + u64 sk, u64 sig); +void vcq_add_hcq_slhdsa_verify_prehash(struct vcq_cmd *slot, u32 core_id, + u32 cmd, u32 param_set, u32 prehash_algo, + u32 msg_len, u32 ctx_len, + u64 msg, u64 ctx, u64 pk, u64 sig); +void vcq_add_hcq_slhdsa_verify_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u64 msg, u64 pk, u64 sig); +void vcq_add_hcq_slhdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 sk_len, u64 sk, u64 pk); +void vcq_add_hcq_lms_verify(struct vcq_cmd *slot, u32 core_id, u32 lms_hss, + u32 pk_len, u32 sig_len, u32 dig_len, + u64 pk, u64 sig, u64 dig); +void vcq_add_hcq_xmss_verify(struct vcq_cmd *slot, u32 core_id, u32 xmss_mt, + u32 pk_len, u32 sig_len, u32 dig_len, + u64 pk, u64 sig, u64 dig); + +/* SYS core flush */ +void vcq_add_sys_flush(struct vcq_cmd *slot); + +/* EAC core VCQ builder */ +void vcq_add_eac_read(struct vcq_cmd *slot, u64 dst_dma, u32 len); + +#endif /* CMH_SYS_H */ diff --git a/include/uapi/linux/cmh_mgmt_ioctl.h b/include/uapi/linux/cmh_mgmt_ioctl.h new file mode 100644 index 00000000000000..a690454fae6909 --- /dev/null +++ b/include/uapi/linux/cmh_mgmt_ioctl.h @@ -0,0 +1,895 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Key Management ioctl Interface (User-Space API) + * + * ioctl commands for /dev/cmh_mgmt -- key CRUD, datastore + * export/import, KIC key derivation, PKE, SM2, and PQC operations. + * + * Relationship to the in-kernel crypto API + * ----------------------------------------- + * Most commands here have no crypto API representation (no transform + * type or verb exists): keystore CRUD, key generation, KIC key + * derivation, ML-KEM encapsulate/decapsulate, SM2 multi-step + * encrypt/decrypt/key-exchange, EdDSA, EAC, and DRBG configuration. + * For these the character device is the only available UAPI. + * + * A bounded subset names primitives the driver ALSO registers with + * the crypto API, and the overlap is intentional: + * - Hardware-held-key operations (RSA decrypt, ECDSA/ML-DSA/SLH-DSA + * sign, ECDH) reference a private key by datastore handle. The + * crypto API set_priv_key()/set_secret() take only raw key bytes + * and cannot name a key that never leaves the hardware; these + * ioctls keep the key hardware-resident. The registered + * transforms serve raw-key in-kernel users -- the paths are + * complementary. + * + * Multi-step protocol flows are documented above the PKE and SM2 + * struct sections. Single-command ioctls are self-documenting. + * + * Versioned structs: user space sets .version = CMH_MGMT_V1 so the + * driver can extend structs in the future without breaking ABI. + */ + +#ifndef _UAPI_CMH_MGMT_IOCTL_H +#define _UAPI_CMH_MGMT_IOCTL_H + +#include +#include +#include + +#define CMH_MGMT_V1 1 + +/* Special reference values */ +#define CMH_REF_NONE 0x0000000000000000ULL /* no key (plaintext) */ + +/* Flags for cmh_ioctl_key_new.flags / cmh_ioctl_key_write.flags */ +#define CMH_FLAG_PT _BITUL(16) /* key can be read as plaintext */ +#define CMH_FLAG_XC _BITUL(17) /* key can be exported over XC bus */ +#define CMH_FLAG_SCA _BITUL(18) /* SCA key stored in 2 shares */ +#define CMH_FLAG_MASK (CMH_FLAG_PT | CMH_FLAG_XC | CMH_FLAG_SCA) + +/* + * Datastore key types -- the LKM maps these to core IDs internally. + * User space passes these in cmh_ioctl_key_new.ds_type. + */ +#define CMH_DS_RAW_VALUE 1 +#define CMH_DS_AES_KEY 2 +#define CMH_DS_AES_XTS_KEY 3 +#define CMH_DS_HMAC_KEY 4 +#define CMH_DS_KMAC_KEY 5 +#define CMH_DS_SM4_KEY 6 +#define CMH_DS_CHACHA20_KEY 7 + +/* PKE key types -- all map to CORE_ID_PKE (0x0A) */ +#define CMH_DS_RSA_PRIV_KEY 10 +#define CMH_DS_RSA_PUB_KEY 11 +#define CMH_DS_RSA_CRT_KEY 12 +#define CMH_DS_ECDSA_PRIV_KEY 13 +#define CMH_DS_ECDSA_PUB_KEY 14 +#define CMH_DS_ECDH_PRIV_KEY 15 +#define CMH_DS_EDDSA_PRIV_KEY 16 +#define CMH_DS_SHARED_SECRET 17 +#define CMH_DS_SM2_PRIV_KEY 18 + +/* QSE key types -- map to CORE_ID_QSE (0x09) */ +#define CMH_DS_ML_KEM_DK 20 +#define CMH_DS_ML_DSA_SK 21 + +/* HCQ key types -- map to CORE_ID_HCQ (0x08) */ +#define CMH_DS_SLHDSA_SK 25 + +/* ioctl argument structures */ + +struct cmh_ioctl_key_new { + __u32 version; /* must be CMH_MGMT_V1 */ + __u32 ds_type; /* CMH_DS_* key type */ + __u32 len; /* key length in bytes */ + __u32 flags; /* CMH_FLAG_* (e.g. CMH_FLAG_PT) */ + __u64 cid; /* caller ID (name) for the key */ + __u64 ref; /* [out] CMH eSW returns key_ref here */ +}; + +struct cmh_ioctl_key_write { + __u32 version; + __u32 len; /* key data length */ + __u32 ds_type; /* CMH_DS_* key type */ + __u32 flags; /* CMH_FLAG_* (e.g. CMH_FLAG_PT) */ + __u64 ref; /* key reference from KEY_NEW */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to key material */ +}; + +struct cmh_ioctl_key_read { + __u32 version; + __u32 len; /* buffer length */ + __u64 ref; /* key reference */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to output buffer */ + __u32 out_len; /* [out] actual bytes written */ + __u32 __reserved; +}; + +struct cmh_ioctl_key_find { + __u32 version; + __u32 __reserved; + __u64 cid; /* caller ID to search for */ + __u64 ref; /* [out] resolved key reference */ + __u32 len; /* [out] key length */ + __u32 type; /* [out] key type */ +}; + +/* + * KEY_LIST -- iterate datastore objects. + * + * Pass start_ref=0 to begin from the first accessible object. + * On return, ref/cid/len/type describe that object. Pass the + * returned ref as start_ref in the next call to advance. Iteration + * ends when ref == 0 (no more objects). + */ +struct cmh_ioctl_key_list { + __u32 version; + __u32 __reserved; + __u64 start_ref; /* starting DS reference (0 = first) */ + __u64 ref; /* [out] object reference */ + __u64 cid; /* [out] caller ID */ + __u32 len; /* [out] object length */ + __u32 type; /* [out] object type */ +}; + +struct cmh_ioctl_key_grant { + __u32 version; + __u32 __reserved; + __u64 ref; /* key reference */ + __u64 read; /* per-MBX read permission bitfield */ + __u64 write; /* per-MBX write permission bitfield */ + __u64 execute; /* per-MBX execute permission bitfield */ +}; + +/* Export blob overhead beyond the raw object data (bytes) */ +#define CMH_DS_EXPORT_OVERHEAD_WRAPPED 48 /* 16B hdr + 16B nonce + 16B tag */ +#define CMH_DS_EXPORT_OVERHEAD_PLAIN 16 /* 16B hdr only */ + +/** + * struct cmh_ioctl_ds_export - Export a datastore object to a wrapped blob + * @version: protocol version (CMH_MGMT_V1) + * @len: DMA buffer size; must be >= export blob size: + * wrapped: CMH_DS_EXPORT_OVERHEAD_WRAPPED + object_len + * plaintext: CMH_DS_EXPORT_OVERHEAD_PLAIN + object_len + * object_len is known from KEY_NEW or KEY_FIND. + * If too small, the eSW rejects the command (-EIO). + * @cid: caller ID of the object to export + * @wrap_key: wrapping key ref (CMH_REF_NONE = plaintext export) + * @data: user-space pointer to output buffer (at least @len bytes) + * @out_len: [out] actual blob bytes written on success + * @__reserved: must be zero + */ +struct cmh_ioctl_ds_export { + __u32 version; + __u32 len; /* buffer length (see sizing rule above) */ + __u64 cid; /* caller ID for response tagging */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to output buffer */ + __u32 out_len; /* [out] actual bytes written */ + __u32 __reserved; +}; + +struct cmh_ioctl_ds_import { + __u32 version; + __u32 len; /* blob length */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to import blob */ +}; + +/* Flags for cmh_ioctl_kic_hkdf1.flags / cmh_ioctl_kic_hkdf2.flags */ +#define CMH_KIC_FLAG_TEMP 0x01 /* store result in TEMP (not persistent DS) */ + +/* + * KIC hardware base key references. + * + * Each CMH device has up to 8 hardware base keys provisioned in OTP/fuses. + * These values are passed in the base_key field of KIC ioctl structs. + * The key valid bitmask is visible via R_KIC_KEY_VALID (MMIO 0x100). + */ +#define CMH_KIC_KEY1 0x0000000100000001ULL +#define CMH_KIC_KEY2 0x0000000200000002ULL +#define CMH_KIC_KEY3 0x0000000300000003ULL +#define CMH_KIC_KEY4 0x0000000400000004ULL +#define CMH_KIC_KEY5 0x0000000500000005ULL +#define CMH_KIC_KEY6 0x0000000600000006ULL +#define CMH_KIC_KEY7 0x0000000700000007ULL +#define CMH_KIC_KEY8 0x0000000800000008ULL + +struct cmh_ioctl_kic_hkdf1 { + __u32 version; + __u32 key_len; /* output key length (e.g., 32) */ + __u64 base_key; /* KIC base key reference */ + __u64 cid; /* CID for the new DS entry (ignored if TEMP) */ + __u64 label; /* user-space pointer to label data */ + __u32 label_len; /* label length in bytes */ + __u32 flags; /* CMH_KIC_FLAG_* */ + __u64 ref; /* [out] derived key reference */ +}; + +struct cmh_ioctl_kic_hkdf2 { + __u32 version; + __u32 key_len; /* output key length (e.g., 32) */ + __u64 base_key; /* KIC base key reference */ + __u64 salt_key; /* salt key reference (CMH_REF_NONE = no salt) */ + __u64 cid; /* CID for the new DS entry (ignored if TEMP) */ + __u64 label; /* user-space pointer to label data */ + __u32 label_len; /* label length in bytes */ + __u32 flags; /* CMH_KIC_FLAG_* */ + __u64 ref; /* [out] derived key reference */ +}; + +struct cmh_ioctl_kic_aes_cmac_kdf { + __u32 version; + __u32 key_len; /* base & output key length (must be 32) */ + __u64 base_key; /* KIC base key or DS reference */ + __u64 cid; /* CID for the new DS entry (ignored if TEMP) */ + __u64 label; /* user-space pointer to label data */ + __u32 label_len; /* label length in bytes */ + __u32 flags; /* CMH_KIC_FLAG_* */ + __u64 ref; /* [out] derived key reference */ +}; + +#define KIC_DKEK_MAX_METADATA 64 /* max metadata length for DKEK */ + +struct cmh_ioctl_kic_dkek_derive { + __u32 version; + __u32 host_id; /* target host ID (0 = caller's own) */ + __u64 base_key; /* KIC base key reference */ + __u64 cid; /* CID for the new DS entry (ignored if TEMP) */ + __u64 metadata; /* user-space pointer to metadata */ + __u32 metadata_len; /* metadata length in bytes */ + __u32 flags; /* CMH_KIC_FLAG_* */ + __u64 ref; /* [out] derived KEK reference */ +}; + +/* -- PKE ioctl argument structures ----------- */ + +/* + * PKE multi-step protocol flows + * + * RSA encrypt/decrypt: + * 1. KEY_NEW(CMH_DS_RSA_PRIV_KEY) + KEY_WRITE -> priv_ref (or RSA_KEYGEN -> priv_ref) + * 2. RSA_ENC(e, n, plaintext) -> ciphertext (public key = raw e,n) + * 3. RSA_DEC(e, n, ciphertext, priv_ref) -> plaintext (or RSA_CRT_DEC) + * + * ECDSA sign: + * 1. EC_KEYGEN(curve) -> priv_ref (or KEY_NEW + KEY_WRITE) + * 2. EC_PUBGEN(priv_ref) -> public_key (raw x||y returned) + * 3. ECDSA_SIGN(digest, priv_ref) -> signature + * SM2 sign uses the same path with curve=CMH_CURVE_SM2. + * + * ECDH shared secret: + * 1. EC_KEYGEN(curve) -> priv_ref (or KEY_NEW + KEY_WRITE) + * 2. ECDH_KEYGEN(priv_ref) -> public_key_x (derive pub from priv) + * 3. Exchange public keys with peer + * 4. ECDH(peer_key_x, priv_ref) -> shared_secret (raw or DS ref via FLAG_DS_RESULT) + * + * EdDSA sign/verify: + * 1. EC_KEYGEN(CURVE_25519 or CURVE_448) -> priv_ref + * 2. EC_PUBGEN(priv_ref) -> public_key + * 3. EDDSA_SIGN(message, priv_ref) -> signature (pure EdDSA, not prehash) + * 4. EDDSA_VERIFY(message, signature, public_key_y) + * For Ed448 SCA: EDDSA_KEYGEN_SCA(priv_ref) -> sca_ref (2-share blinded key) + * + * SM2 encryption (GM/T 0003.4): + * 1. EC_KEYGEN(CMH_CURVE_SM2) -> priv_ref (or KEY_NEW + KEY_WRITE) + * 2. EC_PUBGEN(priv_ref) -> public_key + * 3. SM2_ENC_POINT(public_key) -> C1, enc_point (nonce_len=0: HW ephemeral) + * 4. SM2_ENC_HASH(enc_point, message) -> ciphertext (C1||C3||C2) + * Decrypt: + * 5. SM2_DEC_POINT(C1, priv_ref) -> dec_point + * 6. SM2_DEC_HASH(ciphertext, dec_point) -> plaintext + * enc_point and dec_point are raw DMA buffers (64B each), not DS refs. + * + * SM2 key exchange (GM/T 0003.3): + * 1. EC_KEYGEN(CMH_CURVE_SM2) -> priv_ref (long-lived, persistent DS) + * 2. EC_PUBGEN(priv_ref) -> public_key + * 3. SM2_ID_DIGEST(id, public_key) -> ZA (SM3-based identity digest) + * 4. SM2_ECDH_KEYGEN(nonce) -> session_key, r (ephemeral scalar r) + * - nonce_len=32: caller supplies r (deterministic) + * - nonce_len=0: HW generates r, writes it back to .nonce + * Exchange session_key with peer. + * 5. SM2_ECDH(r, priv_ref, peer_pub, peer_sess) -> shared_point + * - Must pass the same r from step 4 (nonce_len=32) + * - shared_point_ref=0: reads back raw shared_point, destroys DS slot + * - shared_point_ref=&ref: keeps DS slot alive, writes ref for ECDH_HASH + * 6. SM2_ECDH_HASH(shared_point_ref, ZA_self, ZA_peer) -> shared_key (16B) + * - shared_point_ref is a persistent DS reference from step 5 + * - The DS slot is consumed by the hub; caller should delete it afterward + * The nonce r is a raw 32-byte scalar in userspace memory between steps 4-5. + * The shared_point is a persistent DS ref between steps 5-6. + * The long-lived private key (priv_ref) persists independently. + */ + +/* PKE operation flags */ +#define CMH_PKE_FLAG_DS_RESULT _BITUL(0) /* store result in DS, return ref */ + +struct cmh_ioctl_pke_rsa_enc { + __u32 version; + __u32 bits; /* RSA key size in bits (512-4096) */ + __u64 e; /* user-space pointer to public exponent */ + __u32 e_len; /* exponent length in bytes */ + __u32 __reserved; + __u64 n; /* user-space pointer to modulus */ + __u64 input; /* user-space pointer to input data */ + __u64 output; /* user-space pointer to output buffer */ +}; + +struct cmh_ioctl_pke_rsa_dec { + __u32 version; + __u32 bits; + __u64 e; /* public exponent */ + __u32 e_len; + __u32 __reserved; + __u64 n; /* modulus */ + __u64 input; /* ciphertext */ + __u64 output; /* plaintext output */ + __u64 key_ref; /* private key DS reference */ +}; + +struct cmh_ioctl_pke_rsa_crt_dec { + __u32 version; + __u32 bits; + __u64 e; + __u32 e_len; + __u32 __reserved; + __u64 n; + __u64 input; + __u64 output; + __u64 crt_ref; /* CRT key DS reference */ +}; + +struct cmh_ioctl_pke_rsa_keygen { + __u32 version; + __u32 bits; /* key size in bits */ + __u64 e; /* user-space pointer to public exponent */ + __u32 e_len; + __u32 flags; /* CMH_FLAG_* */ + __u64 n; /* [out] user-space pointer to modulus buffer */ + __u64 d_cid; /* CID for private key DS entry */ + __u64 d_ref; /* [out] private key reference */ + __u64 crt_cid; /* CID for CRT key DS entry (0 = skip CRT) */ + __u64 crt_ref; /* [out] CRT key reference */ +}; + +struct cmh_ioctl_pke_ecdsa_sign { + __u32 version; + __u32 curve; /* ABI curve ID (e.g. 0x03 = P-256) */ + __u64 digest; /* user-space pointer to hash digest */ + __u32 digest_len; /* digest length in bytes */ + __u32 __reserved; + __u64 signature; /* [out] user-space pointer to (r,s) */ + __u64 key_ref; /* private key DS reference */ +}; + +struct cmh_ioctl_pke_ecdh { + __u32 version; + __u32 curve; + __u64 peer_key_x; /* user-space pointer to peer public key X */ + __u64 key_ref; /* private key DS reference */ + __u32 flags; /* CMH_PKE_FLAG_DS_RESULT */ + __u32 __reserved; + __u64 result_cid; /* CID for DS result (if FLAG_DS_RESULT) */ + __u64 output; /* [out] raw shared secret or DS ref */ +}; + +struct cmh_ioctl_pke_ecdh_keygen { + __u32 version; + __u32 curve; + __u64 key_ref; /* private key DS reference */ + __u64 public_key_x; /* [out] user-space pointer to public key X */ +}; + +struct cmh_ioctl_pke_eddsa_sign { + __u32 version; + __u32 curve; /* CURVE_25519 or CURVE_448 */ + __u64 digest; /* user-space ptr to message (not digest) */ + __u32 digest_len; + __u32 __reserved; + __u64 signature; /* [out] user-space pointer to signature */ + __u64 key_ref; /* private key DS reference */ +}; + +struct cmh_ioctl_pke_eddsa_verify { + __u32 version; + __u32 curve; + __u64 digest; + __u32 digest_len; + __u32 __reserved; + __u64 signature; + __u64 public_key_y; /* user-space pointer to public key Y */ +}; + +struct cmh_ioctl_pke_ec_keygen { + __u32 version; + __u32 curve; + __u32 flags; /* CMH_FLAG_* */ + __u32 __reserved; + __u64 cid; /* CID for the new key DS entry */ + __u64 ref; /* [out] private key reference */ +}; + +struct cmh_ioctl_pke_ec_pubgen { + __u32 version; + __u32 curve; + __u64 key_ref; /* private key DS reference */ + __u64 public_key; /* [out] user-space pointer to public key */ +}; + +struct cmh_ioctl_pke_eddsa_keygen_sca { + __u32 version; + __u32 curve; /* must be CURVE_448 */ + __u64 key_ref; /* input: normal Ed448 private key DS ref */ + __u64 cid; /* CID for the new SCA key DS entry */ + __u64 sca_ref; /* [out] SCA private key reference */ +}; + +/* + * ioctl numbers -- type 'J', sequential. + * 'C' conflicts with OSS sound, CAPI/ISDN, and COSA WAN drivers; + * 'J' is unregistered in Documentation/userspace-api/ioctl/ioctl-number.rst. + */ + +#define CMH_MGMT_IOC_MAGIC 'J' + +#define CMH_IOCTL_KEY_NEW _IOWR(CMH_MGMT_IOC_MAGIC, 0x01, struct cmh_ioctl_key_new) +#define CMH_IOCTL_KEY_WRITE _IOW(CMH_MGMT_IOC_MAGIC, 0x02, struct cmh_ioctl_key_write) +#define CMH_IOCTL_KEY_READ _IOWR(CMH_MGMT_IOC_MAGIC, 0x03, struct cmh_ioctl_key_read) +#define CMH_IOCTL_KEY_FIND _IOWR(CMH_MGMT_IOC_MAGIC, 0x04, struct cmh_ioctl_key_find) +#define CMH_IOCTL_KEY_GRANT _IOW(CMH_MGMT_IOC_MAGIC, 0x05, struct cmh_ioctl_key_grant) +#define CMH_IOCTL_KEY_DELETE _IOW(CMH_MGMT_IOC_MAGIC, 0x06, struct cmh_ioctl_key_grant) +#define CMH_IOCTL_DS_EXPORT _IOWR(CMH_MGMT_IOC_MAGIC, 0x07, struct cmh_ioctl_ds_export) +#define CMH_IOCTL_DS_IMPORT _IOW(CMH_MGMT_IOC_MAGIC, 0x08, struct cmh_ioctl_ds_import) +#define CMH_IOCTL_KIC_HKDF1 _IOWR(CMH_MGMT_IOC_MAGIC, 0x09, struct cmh_ioctl_kic_hkdf1) +#define CMH_IOCTL_KIC_HKDF2 _IOWR(CMH_MGMT_IOC_MAGIC, 0x0A, struct cmh_ioctl_kic_hkdf2) +#define CMH_IOCTL_KEY_NEW_RANDOM _IOWR(CMH_MGMT_IOC_MAGIC, 0x0B, struct cmh_ioctl_key_new) +#define CMH_IOCTL_KIC_AES_CMAC_KDF _IOWR(CMH_MGMT_IOC_MAGIC, 0x0C, \ + struct cmh_ioctl_kic_aes_cmac_kdf) +#define CMH_IOCTL_KIC_DKEK_DERIVE _IOWR(CMH_MGMT_IOC_MAGIC, 0x0D, \ + struct cmh_ioctl_kic_dkek_derive) +#define CMH_IOCTL_KEY_LIST _IOWR(CMH_MGMT_IOC_MAGIC, 0x0E, struct cmh_ioctl_key_list) + +/* PKE operation ioctls */ +#define CMH_IOCTL_PKE_RSA_ENC _IOWR(CMH_MGMT_IOC_MAGIC, 0x10, \ + struct cmh_ioctl_pke_rsa_enc) +#define CMH_IOCTL_PKE_RSA_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x11, \ + struct cmh_ioctl_pke_rsa_dec) +#define CMH_IOCTL_PKE_RSA_CRT_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x12, \ + struct cmh_ioctl_pke_rsa_crt_dec) +#define CMH_IOCTL_PKE_RSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x13, \ + struct cmh_ioctl_pke_rsa_keygen) +#define CMH_IOCTL_PKE_ECDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x14, \ + struct cmh_ioctl_pke_ecdsa_sign) +#define CMH_IOCTL_PKE_ECDH _IOWR(CMH_MGMT_IOC_MAGIC, 0x16, \ + struct cmh_ioctl_pke_ecdh) +#define CMH_IOCTL_PKE_ECDH_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x17, \ + struct cmh_ioctl_pke_ecdh_keygen) +#define CMH_IOCTL_PKE_EDDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x18, \ + struct cmh_ioctl_pke_eddsa_sign) +#define CMH_IOCTL_PKE_EDDSA_VERIFY _IOW(CMH_MGMT_IOC_MAGIC, 0x19, \ + struct cmh_ioctl_pke_eddsa_verify) +#define CMH_IOCTL_PKE_EC_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x1A, \ + struct cmh_ioctl_pke_ec_keygen) +#define CMH_IOCTL_PKE_EC_PUBGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x1B, \ + struct cmh_ioctl_pke_ec_pubgen) +#define CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA _IOWR(CMH_MGMT_IOC_MAGIC, 0x1C, \ + struct cmh_ioctl_pke_eddsa_keygen_sca) + +/* -- PQC ioctl argument structures ----------- */ + +/* + * PQC operation flags (bits [2:0]). + * PQC keygen ioctls accept CMH_FLAG_PT in bits [18:16] to explicitly + * set the DS key storage attribute when CMH_QSE_FLAG_DS_REF is set. + * CMH_FLAG_SCA and CMH_FLAG_XC are rejected -- QSE SCA protection uses + * polynomial masking (CMH_QSE_FLAG_MASKED), not 2-share storage, + * and the eSW dec/sign paths hardcode SYS_TYPE_FLAG_PT. + * If no CMH_FLAG_* bits are set, DS keys default to CMH_FLAG_PT. + */ +#define CMH_QSE_FLAG_MASKED _BITUL(0) /* use masked (SCA-resistant) HW commands */ +#define CMH_QSE_FLAG_DS_REF _BITUL(1) /* store key output in DS, return ref */ +#define CMH_QSE_FLAG_HW_RNG _BITUL(2) /* use HW RNG for seed/randomness */ +#define CMH_QSE_FLAG_MASK (_BITUL(0) | _BITUL(1) | _BITUL(2)) + +/* -- SYS wrap header size -------------------- */ +/* sys_read prepends a 16-byte header even for plaintext reads */ +#define CMH_SYS_WRAP_HDR_SIZE 16 + +/* -- Seed / randomness lengths --------------- */ + +#define CMH_QSE_SEED_LEN 32 /* ML-KEM/ML-DSA seed size */ +#define CMH_QSE_SEED_LEN_MASKED 64 /* seed size for masked mode */ + +/* -- ML-DSA ExternalMu sentinel -------------- */ +/* Pass this as mlen to use 64-byte pre-hashed mu instead of raw message */ +#define CMH_ML_DSA_MLEN_EXTERNAL_MU 0xFFFFFFFFU + +/* -- ML-KEM size macros ---------------------- */ + +#define CMH_ML_KEM_EK_SIZE(k) (384U * (k) + 32U) +#define CMH_ML_KEM_DK_SIZE(k) (768U * (k) + 96U) +/* CT sizes: k=2 -> 768, k=3 -> 1088, k=4 -> 1568 */ +#define CMH_ML_KEM_CT_SIZE_512 768U +#define CMH_ML_KEM_CT_SIZE_768 1088U +#define CMH_ML_KEM_CT_SIZE_1024 1568U +#define CMH_ML_KEM_SS_LEN 32U + +/* -- ML-DSA size macros ---------------------- */ +/* Indexed by mode: [0]=44 (mode=2), [1]=65 (mode=3), [2]=87 (mode=5) */ + +#define CMH_ML_DSA_44_PK_SIZE 1312U +#define CMH_ML_DSA_44_SK_SIZE 2560U +#define CMH_ML_DSA_44_SIG_SIZE 2420U +#define CMH_ML_DSA_65_PK_SIZE 1952U +#define CMH_ML_DSA_65_SK_SIZE 4032U +#define CMH_ML_DSA_65_SIG_SIZE 3309U +#define CMH_ML_DSA_87_PK_SIZE 2592U +#define CMH_ML_DSA_87_SK_SIZE 4896U +#define CMH_ML_DSA_87_SIG_SIZE 4627U + +/* -- SLH-DSA parameter set IDs --------------- */ + +#define CMH_SLHDSA_SHAKE_128S 1U +#define CMH_SLHDSA_SHAKE_128F 2U +#define CMH_SLHDSA_SHAKE_192S 3U +#define CMH_SLHDSA_SHAKE_192F 4U +#define CMH_SLHDSA_SHAKE_256S 5U +#define CMH_SLHDSA_SHAKE_256F 6U +#define CMH_SLHDSA_SHA2_128S 7U +#define CMH_SLHDSA_SHA2_128F 8U +#define CMH_SLHDSA_SHA2_192S 9U +#define CMH_SLHDSA_SHA2_192F 10U +#define CMH_SLHDSA_SHA2_256S 11U +#define CMH_SLHDSA_SHA2_256F 12U +#define CMH_SLHDSA_PARAM_MAX 12U + +/* SLH-DSA prehash algorithm IDs */ +#define CMH_SLHDSA_PREHASH_SHA256 1U +#define CMH_SLHDSA_PREHASH_SHA512 2U +#define CMH_SLHDSA_PREHASH_SHAKE128 3U +#define CMH_SLHDSA_PREHASH_SHAKE256 4U +#define CMH_SLHDSA_PREHASH_MAX 4U + +/* SLH-DSA n-value table indexed by (param_set - 1) */ +#define CMH_SLHDSA_N_128 16U +#define CMH_SLHDSA_N_192 24U +#define CMH_SLHDSA_N_256 32U + +/* SLH-DSA key sizes: pk = 2*n, sk = 4*n, seed = 3*n */ +#define CMH_SLHDSA_PK_SIZE(n) (2U * (n)) +#define CMH_SLHDSA_SK_SIZE(n) (4U * (n)) +#define CMH_SLHDSA_SEED_SIZE(n) (3U * (n)) + +/* SLH-DSA signature sizes indexed by (param_set - 1) */ +#define CMH_SLHDSA_SIG_SIZE_SHAKE_128S 7856U +#define CMH_SLHDSA_SIG_SIZE_SHAKE_128F 17088U +#define CMH_SLHDSA_SIG_SIZE_SHAKE_192S 16224U +#define CMH_SLHDSA_SIG_SIZE_SHAKE_192F 35664U +#define CMH_SLHDSA_SIG_SIZE_SHAKE_256S 29792U +#define CMH_SLHDSA_SIG_SIZE_SHAKE_256F 49856U +#define CMH_SLHDSA_SIG_SIZE_SHA2_128S 7856U +#define CMH_SLHDSA_SIG_SIZE_SHA2_128F 17088U +#define CMH_SLHDSA_SIG_SIZE_SHA2_192S 16224U +#define CMH_SLHDSA_SIG_SIZE_SHA2_192F 35664U +#define CMH_SLHDSA_SIG_SIZE_SHA2_256S 29792U +#define CMH_SLHDSA_SIG_SIZE_SHA2_256F 49856U + +/* -- PKE curve IDs -------------- */ + +#define CMH_CURVE_P192 0x01U +#define CMH_CURVE_P224 0x02U +#define CMH_CURVE_P256 0x03U +#define CMH_CURVE_P384 0x04U +#define CMH_CURVE_P521 0x05U +#define CMH_CURVE_SECP256K1 0x07U +#define CMH_CURVE_BP192R1 0x11U +#define CMH_CURVE_BP224R1 0x12U +#define CMH_CURVE_BP256R1 0x13U +#define CMH_CURVE_BP320R1 0x14U +#define CMH_CURVE_BP384R1 0x15U +#define CMH_CURVE_BP512R1 0x16U +#define CMH_CURVE_SM2 0x18U +#define CMH_CURVE_25519 0x21U +#define CMH_CURVE_448 0x22U + +/* ML-KEM */ + +struct cmh_ioctl_ml_kem_keygen { + __u32 version; + __u32 k; /* security parameter: 2/3/4 */ + __u32 flags; /* CMH_QSE_FLAG_* */ + __u32 __reserved; + __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */ + __u64 z; /* user-space pointer to z (or 0 for HW RNG) */ + __u64 ek; /* [out] user-space pointer to encapsulation key */ + __u64 dk; /* [out] user-space pointer to decapsulation key + * or [out] DS ref if CMH_QSE_FLAG_DS_REF + */ + __u64 dk_cid; /* CID for DS entry (if DS_REF) */ + __u64 dk_ref; /* [out] dk DS reference (if DS_REF) */ +}; + +struct cmh_ioctl_ml_kem_enc { + __u32 version; + __u32 k; + __u32 flags; /* CMH_QSE_FLAG_* */ + __u32 __reserved; + __u64 coin; /* user-space pointer to random coin (or 0) */ + __u64 ek; /* user-space pointer to encapsulation key */ + __u64 ct; /* [out] user-space pointer to ciphertext */ + __u64 ss; /* [out] user-space pointer to shared secret */ + __u64 __reserved2[2]; /* reserved for future use */ +}; + +struct cmh_ioctl_ml_kem_dec { + __u32 version; + __u32 k; + __u32 flags; /* CMH_QSE_FLAG_* */ + __u32 __reserved; + __u64 ct; /* user-space pointer to ciphertext */ + __u64 dk; /* user-space pointer to dk or DS ref */ + __u64 ss; /* [out] user-space pointer to shared secret */ + __u64 __reserved2[2]; /* reserved for future use */ +}; + +/* ML-DSA */ + +struct cmh_ioctl_ml_dsa_keygen { + __u32 version; + __u32 mode; /* security parameter: 2/3/5 */ + __u32 flags; /* CMH_QSE_FLAG_* */ + __u32 __reserved; + __u64 seed; /* user-space pointer to seed (or 0 for HW RNG) */ + __u64 pk; /* [out] user-space pointer to public key */ + __u64 sk; /* [out] user-space pointer to secret key + * or [out] DS ref if CMH_QSE_FLAG_DS_REF + */ + __u64 sk_cid; /* CID for DS entry (if DS_REF) */ + __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */ +}; + +struct cmh_ioctl_ml_dsa_sign { + __u32 version; + __u32 mode; + __u32 flags; /* CMH_QSE_FLAG_* */ + __u32 mlen; /* message length in bytes */ + __u64 m; /* user-space pointer to message */ + __u64 sk; /* user-space pointer to sk or DS ref */ + __u64 sig; /* [out] user-space pointer to signature */ + __u64 rnd; /* user-space pointer to randomness (or 0) */ +}; + +/* SLH-DSA */ + +struct cmh_ioctl_slhdsa_keygen { + __u32 version; + __u32 parameter_set; /* HCQ_SLHDSA_SHAKE_128S .. SHA2_256F */ + __u32 flags; /* CMH_QSE_FLAG_DS_REF */ + __u32 __reserved; + __u64 seed; /* user-space pointer to seed */ + __u64 pk; /* [out] user-space pointer to public key */ + __u64 sk; /* [out] user-space pointer to secret key + * or [out] DS ref if CMH_QSE_FLAG_DS_REF + */ + __u64 sk_cid; /* CID for DS entry (if DS_REF) */ + __u64 sk_ref; /* [out] sk DS reference (if DS_REF) */ +}; + +struct cmh_ioctl_slhdsa_sign { + __u32 version; + __u32 parameter_set; + __u32 msg_len; + __u32 ctx_len; + __u64 msg; /* user-space pointer to message */ + __u64 ctx; /* user-space pointer to context (or 0) */ + __u64 sk; /* DS ref for secret key */ + __u64 sig; /* [out] user-space pointer to signature */ + __u64 add_random; /* user-space pointer to addl. randomness (or 0) */ +}; + +struct cmh_ioctl_slhdsa_sign_prehash { + __u32 version; + __u32 parameter_set; + __u32 prehash_algo; /* CMH_SLHDSA_PREHASH_* */ + __u32 digest; /* 0 = raw msg (eSW hashes), 1 = pre-computed digest */ + __u32 msg_len; + __u32 ctx_len; + __u64 msg; /* user-space pointer to message/digest */ + __u64 ctx; /* user-space pointer to context (or 0) */ + __u64 sk; /* DS ref for secret key */ + __u64 sig; /* [out] user-space pointer to signature */ + __u64 add_random; /* user-space pointer to addl. randomness (or 0) */ +}; + +/* -- SM2 ioctl argument structures ----------- */ + +/* SM2 fixed key sizes (sm2p256v1 curve, 256-bit) */ +#define CMH_SM2_CLEN 32U /* coordinate length */ +#define CMH_SM2_PUBKEY_LEN 64U /* uncompressed (x||y) */ +#define CMH_SM2_POINT_LEN 64U /* EC point (x||y) */ +#define CMH_SM2_SHARED_KEY_LEN 16U /* ECDH shared key */ +#define CMH_SM2_DIGEST_LEN 32U /* SM3 digest (ZA) */ +/* + * SM2 enc_hash/dec_hash payload limit. + * + * The eSW PKE driver implements only a single-block GM/T 0003.4 KDF + * (one SM3 invocation, 32 bytes of key stream). Longer messages would + * silently produce incorrect ciphertext / plaintext, so the driver caps + * the payload at 32 bytes. See Documentation/ABI/testing/cmh-mgmt. + */ +#define CMH_SM2_MAX_MSG_LEN 32U /* encrypt/decrypt */ +#define CMH_SM2_MAX_ID_LEN 32U /* identity string */ +#define CMH_SM2_CT_OVERHEAD 96U /* C1(64) + C3(32) */ +#define CMH_SM2_MAX_CT_LEN 128U /* 96 + max_msg = 128 */ + +struct cmh_ioctl_sm2_ecdh_keygen { + __u32 version; + __u32 nonce_len; /* 0 = HW generates r (written back), 32 = caller provides r */ + __u64 nonce; /* [in/out] user-space pointer to nonce buffer (32B) */ + __u64 session_key; /* [out] user-space pointer to session key R=r*G (64B) */ +}; + +struct cmh_ioctl_sm2_ecdh { + __u32 version; + __u32 nonce_len; /* 0 = HW generates (written back), 32 = caller provides */ + __u64 nonce; /* [in/out] user-space pointer to nonce r (32B) */ + __u64 peer_public_key; /* user-space pointer to peer pub key (64B) */ + __u64 peer_session_key; /* user-space pointer to peer session key (64B) */ + __u64 key_ref; /* private key DS reference */ + __u64 shared_point; /* [out] user-space pointer to shared point (64B) */ + __u64 shared_point_ref; /* [in/out] 0 = read-back mode; &ref = keep DS, write ref */ +}; + +struct cmh_ioctl_sm2_dec_point { + __u32 version; + __u32 ciphertext_len; /* total ciphertext length (97..128) */ + __u64 ciphertext; /* user-space pointer to ciphertext (64B: C1) */ + __u64 dec_point; /* [out] user-space pointer to dec point (64B) */ + __u64 key_ref; /* private key DS reference */ +}; + +struct cmh_ioctl_sm2_enc_point { + __u32 version; + __u32 nonce_len; /* 0 = HW generates, 32 = caller provides */ + __u64 nonce; /* user-space pointer to nonce (or 0) */ + __u64 public_key; /* user-space pointer to public key (64B) */ + __u64 ciphertext; /* [out] user-space pointer to C1 (64B) */ + __u64 enc_point; /* [out] user-space pointer to enc point (64B) */ +}; + +struct cmh_ioctl_sm2_id_digest { + __u32 version; + __u32 id_len; /* identity length in bytes (<=32) */ + __u64 id; /* user-space pointer to identity string */ + __u64 public_key; /* user-space pointer to public key (64B) */ + __u64 digest; /* [out] user-space pointer to ZA digest (32B) */ +}; + +/* + * SM2 ECDH_HASH -- derive shared key from shared point + ZA digests. + * + * IMPORTANT: The digest fields use ABSOLUTE ordering per GM/T 0003.3, + * NOT relative own/peer ordering. Both parties must pass: + * peer_id_digest = Z_A (initiator's digest) -- hashed FIRST + * id_digest = Z_B (responder's digest) -- hashed SECOND + * The eSW computes: KDF(shared_point || peer_id_digest || id_digest). + */ +struct cmh_ioctl_sm2_ecdh_hash { + __u32 version; + __u32 __reserved; + __u64 peer_id_digest; /* ptr to Z_A -- initiator's digest (32B) */ + __u64 id_digest; /* ptr to Z_B -- responder's digest (32B) */ + __u64 shared_point_ref; /* DS reference from SM2_ECDH */ + __u64 shared_key; /* [out] ptr to shared key (16B) */ +}; + +struct cmh_ioctl_sm2_dec_hash { + __u32 version; + __u32 ciphertext_len; /* ciphertext length (97..128) */ + __u64 ciphertext; /* user-space pointer to full ciphertext */ + __u64 dec_point; /* user-space pointer to dec point (64B) */ + __u64 plaintext; /* [out] user-space pointer to plaintext */ +}; + +struct cmh_ioctl_sm2_enc_hash { + __u32 version; + __u32 message_len; /* message length (1..32) */ + __u64 message; /* user-space pointer to plaintext */ + __u64 enc_point; /* user-space pointer to enc point (64B) */ + __u64 ciphertext; /* [out] user-space pointer to ciphertext */ +}; + +/* PQC ioctl numbers */ +#define CMH_IOCTL_ML_KEM_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x20, \ + struct cmh_ioctl_ml_kem_keygen) +#define CMH_IOCTL_ML_KEM_ENC _IOWR(CMH_MGMT_IOC_MAGIC, 0x21, \ + struct cmh_ioctl_ml_kem_enc) +#define CMH_IOCTL_ML_KEM_DEC _IOWR(CMH_MGMT_IOC_MAGIC, 0x22, \ + struct cmh_ioctl_ml_kem_dec) +#define CMH_IOCTL_ML_DSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x23, \ + struct cmh_ioctl_ml_dsa_keygen) +#define CMH_IOCTL_ML_DSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x24, \ + struct cmh_ioctl_ml_dsa_sign) +#define CMH_IOCTL_SLHDSA_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x28, \ + struct cmh_ioctl_slhdsa_keygen) +#define CMH_IOCTL_SLHDSA_SIGN _IOWR(CMH_MGMT_IOC_MAGIC, 0x29, \ + struct cmh_ioctl_slhdsa_sign) +#define CMH_IOCTL_SLHDSA_SIGN_PREHASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x2D, \ + struct cmh_ioctl_slhdsa_sign_prehash) + +/* SM2 operation ioctls */ +#define CMH_IOCTL_SM2_ECDH_KEYGEN _IOWR(CMH_MGMT_IOC_MAGIC, 0x30, \ + struct cmh_ioctl_sm2_ecdh_keygen) +#define CMH_IOCTL_SM2_ECDH _IOWR(CMH_MGMT_IOC_MAGIC, 0x31, \ + struct cmh_ioctl_sm2_ecdh) +#define CMH_IOCTL_SM2_DEC_POINT _IOWR(CMH_MGMT_IOC_MAGIC, 0x32, \ + struct cmh_ioctl_sm2_dec_point) +#define CMH_IOCTL_SM2_ENC_POINT _IOWR(CMH_MGMT_IOC_MAGIC, 0x33, \ + struct cmh_ioctl_sm2_enc_point) +#define CMH_IOCTL_SM2_ID_DIGEST _IOWR(CMH_MGMT_IOC_MAGIC, 0x34, \ + struct cmh_ioctl_sm2_id_digest) +#define CMH_IOCTL_SM2_ECDH_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x35, \ + struct cmh_ioctl_sm2_ecdh_hash) +#define CMH_IOCTL_SM2_DEC_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x36, \ + struct cmh_ioctl_sm2_dec_hash) +#define CMH_IOCTL_SM2_ENC_HASH _IOWR(CMH_MGMT_IOC_MAGIC, 0x37, \ + struct cmh_ioctl_sm2_enc_hash) + +/* + * EAC (Error and Alarm Controller) -- read and clear error registers. + * + * Returns a snapshot of all hardware error/safety/notification registers. + * The eSW atomically reads and clears the registers on each call, so + * successive reads show only new events. + */ +struct cmh_ioctl_eac_read { + __u32 version; /* must be CMH_MGMT_V1 */ + __u32 __reserved; + __u64 mailbox_notification; /* [out] MBX safety notification bitmask */ + __u32 hw_error; /* [out] HWC error bitmask */ + __u32 hw_nmi; /* [out] HWC NMI bitmask */ + __u32 hw_panic; /* [out] HWC panic bitmask */ + __u32 safety_fatal; /* [out] HWC fatal safety bitmask */ + __u32 safety_notification; /* [out] HWC safety notification bitmask */ + __u32 sw_info0; /* [out] eSW tracing info */ + __u32 sw_info1; /* [out] eSW tracing info */ + __u32 sram_bank_errors[4]; /* [out] correctable ECC error counts */ + __u32 __pad; /* explicit tail padding (prevent info leak) */ +}; + +/* + * DRBG CONFIG -- configure the hardware DRBG before first use. + * + * This is a management operation normally performed once at system + * start-up. Must be called before any hwrng reads or DRBG GENERATE + * operations. + */ +#define CMH_DRBG_RATIO_ONE 0 /* 1:1 entropy ratio */ +#define CMH_DRBG_RATIO_ONE_HALF 1 /* 1:2 */ +#define CMH_DRBG_RATIO_ONE_THIRD 2 /* 1:3 */ +#define CMH_DRBG_RATIO_ONE_FOURTH 3 /* 1:4 */ + +#define CMH_DRBG_STRENGTH_128 0x00 /* 128-bit security */ +#define CMH_DRBG_STRENGTH_256 0x10 /* 256-bit security */ + +struct cmh_ioctl_drbg_config { + __u32 version; /* must be CMH_MGMT_V1 */ + __u32 entropy_ratio; /* CMH_DRBG_RATIO_* */ + __u32 security_strength; /* CMH_DRBG_STRENGTH_* */ + __u32 __reserved; +}; + +/* EAC ioctl number */ +#define CMH_IOCTL_EAC_READ _IOWR(CMH_MGMT_IOC_MAGIC, 0x0F, \ + struct cmh_ioctl_eac_read) + +/* DRBG management ioctl number */ +#define CMH_IOCTL_DRBG_CONFIG _IOW(CMH_MGMT_IOC_MAGIC, 0x40, \ + struct cmh_ioctl_drbg_config) + +#endif /* _UAPI_CMH_MGMT_IOCTL_H */ From 581caa969a04a02e91abf3bd70a15e308ea150dd Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:22 -0700 Subject: [PATCH 06/21] crypto: cmh - add SHA-2/SHA-3/SHAKE ahash Register ahash algorithms for SHA-224, SHA-256, SHA-384, SHA-512, SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128, and SHAKE256 using the CMH hash core (core ID 0x02). Supports incremental update/finup/final, init/export/import for request cloning, and the CRYPTO_AHASH_REQ_VIRT flag for zero-copy from virtual buffers. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 3 +- drivers/crypto/cmh/cmh_hash.c | 860 ++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/include/cmh_hash.h | 26 + 4 files changed, 897 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_hash.c create mode 100644 drivers/crypto/cmh/include/cmh_hash.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 1492e575598cab..c0531f41622953 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -14,7 +14,8 @@ cmh-y := \ cmh_dma.o \ cmh_sysfs.o \ cmh_key.o \ - cmh_sys.o + cmh_sys.o \ + cmh_hash.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_hash.c b/drivers/crypto/cmh/cmh_hash.c new file mode 100644 index 00000000000000..2256bf4314c33f --- /dev/null +++ b/drivers/crypto/cmh/cmh_hash.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API Hash Driver + * + * Registers asynchronous hash (ahash) algorithms with the Linux crypto + * subsystem. Implements SHA-2 (224/256/384/512), SHA-3 + * (224/256/384/512), and SHAKE (128/256) families using the CMH Hash + * Core (HC). + * + * Incremental HW update model -- each .update() with enough data for + * at least one full block submits a self-contained VCQ transaction: + * + * .init() -> software-only: zero per-request context + * .update() -> buffer data in holdback; when >= block_size bytes: + * INIT [+ RESTORE] + UPDATE(full blocks) + SAVE + FLUSH + * -> return -EINPROGRESS (else return 0, data in holdback) + * .final() -> INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH + * .finup() -> linearise holdback + new data, then final path + * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot, zero-copy) + * .export() -> software-only: copy checkpoint + holdback to out + * .import() -> software-only: restore checkpoint + holdback from in + * + * The FLUSH after each .update() releases the HC core, so no lockout. + * Two hash sessions interleave fine on the same MBX -- each saves its + * own state via SAVE and restores via RESTORE on the next call. + * + * Export/import is purely software (no HW interaction), enabling + * crypto API transform clone for all plain-hash algorithms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_hash.h" +#include "cmh_vcq.h" +#include "cmh_txn.h" +#include "cmh_dma.h" + +/* Algorithm Table */ + +struct cmh_hash_alg_info { + u32 hc_algo; /* HC_ALGO_* (SHA2, SHA3, SHAKE) */ + u32 digest_size; /* bytes */ + u32 block_size; /* cra_blocksize for Linux crypto API */ + const char *alg_name; /* Linux crypto name: "sha256" */ + const char *drv_name; /* driver name: "cri-cmh-sha256" */ +}; + +static const struct cmh_hash_alg_info cmh_hash_algs_info[] = { + /* SHA-2 family */ + { + .hc_algo = HC_ALGO_SHA2_224, + .digest_size = CMH_SHA224_DIGEST_SIZE, + .block_size = 64, + .alg_name = "sha224", + .drv_name = "cri-cmh-sha224", + }, + { + .hc_algo = HC_ALGO_SHA2_256, + .digest_size = CMH_SHA256_DIGEST_SIZE, + .block_size = 64, + .alg_name = "sha256", + .drv_name = "cri-cmh-sha256", + }, + { + .hc_algo = HC_ALGO_SHA2_384, + .digest_size = CMH_SHA384_DIGEST_SIZE, + .block_size = 128, + .alg_name = "sha384", + .drv_name = "cri-cmh-sha384", + }, + { + .hc_algo = HC_ALGO_SHA2_512, + .digest_size = CMH_SHA512_DIGEST_SIZE, + .block_size = 128, + .alg_name = "sha512", + .drv_name = "cri-cmh-sha512", + }, + /* SHA-3 family */ + { + .hc_algo = HC_ALGO_SHA3_224, + .digest_size = CMH_SHA3_224_DIGEST_SIZE, + .block_size = 144, /* rate = 1600/8 - 2*224/8 = 144 */ + .alg_name = "sha3-224", + .drv_name = "cri-cmh-sha3-224", + }, + { + .hc_algo = HC_ALGO_SHA3_256, + .digest_size = CMH_SHA3_256_DIGEST_SIZE, + .block_size = 136, /* rate = 1600/8 - 2*256/8 = 136 */ + .alg_name = "sha3-256", + .drv_name = "cri-cmh-sha3-256", + }, + { + .hc_algo = HC_ALGO_SHA3_384, + .digest_size = CMH_SHA3_384_DIGEST_SIZE, + .block_size = 104, /* rate = 1600/8 - 2*384/8 = 104 */ + .alg_name = "sha3-384", + .drv_name = "cri-cmh-sha3-384", + }, + { + .hc_algo = HC_ALGO_SHA3_512, + .digest_size = CMH_SHA3_512_DIGEST_SIZE, + .block_size = 72, /* rate = 1600/8 - 2*512/8 = 72 */ + .alg_name = "sha3-512", + .drv_name = "cri-cmh-sha3-512", + }, + /* + * SHAKE (XOF) family -- fixed-output ahash registration. + * + * cra_blocksize = 1: SHAKE is a sponge/XOF, not Merkle-Damgaard. + * The Keccak rate (168 for SHAKE-128, 136 for SHAKE-256) exceeds + * MAX_ALGAPI_BLOCKSIZE (160) on Linux <=6.7. Using 1 signals + * "byte-oriented" which is correct for XOF consumers. The kernel + * raised the limit to 208 in 6.8 (commit 2f3a22704889). + */ + { + .hc_algo = HC_ALGO_SHAKE128, + .digest_size = CMH_SHAKE128_DIGEST_SIZE, + .block_size = 1, /* XOF: no meaningful block for crypto API */ + .alg_name = "shake128", + .drv_name = "cri-cmh-shake128", + }, + { + .hc_algo = HC_ALGO_SHAKE256, + .digest_size = CMH_SHAKE256_DIGEST_SIZE, + .block_size = 1, /* XOF: no meaningful block for crypto API */ + .alg_name = "shake256", + .drv_name = "cri-cmh-shake256", + }, +}; + +#define CMH_HASH_ALG_COUNT ARRAY_SIZE(cmh_hash_algs_info) + +/* Per-Request State */ + +/* Maximum cra_blocksize across all registered algorithms (SHA3-224) */ +#define CMH_HASH_MAX_BLOCK 144 + +/* + * Exported hash state -- serialised by .export(), deserialised by + * .import(). This is what statesize advertises to the crypto subsystem. + */ +struct cmh_hash_export_state { + u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */ + u8 buf[CMH_HASH_MAX_BLOCK]; /* holdback buffer */ + u32 buf_len; /* valid bytes in buf[] */ + u32 hw_started; /* non-zero if checkpoint valid */ +}; + +/* + * Maximum payload commands any hash transaction can produce: + * INIT + RESTORE + UPDATE + SAVE/FINAL + FLUSH = 5 + * Worst-case packed output (stride=7, 1 payload per VCQ): + * 5 VCQs x 2 entries = 10 + */ +#define CMH_HASH_MAX_PAYLOAD 5 +#define CMH_HASH_MAX_PACKED (CMH_HASH_MAX_PAYLOAD * 2) + +/* + * Stored in ahash_request_ctx(). Tracks the algorithm, a holdback + * buffer for partial blocks, an HC context checkpoint from the last + * SAVE, and DMA state for the current in-flight async operation. + * + * The checkpoint is embedded inline rather than heap-allocated because + * the kernel ahash API has no per-request destructor. If a request is + * abandoned without .final() (e.g. transform freed early), a heap + * checkpoint would leak unconditionally. + */ +struct cmh_hash_reqctx { + const struct cmh_hash_alg_info *info; + int error; + u32 hw_started; /* non-zero after first HW submission */ + u32 buf_len; /* bytes in holdback buf[] */ + u32 has_checkpoint; /* non-zero if checkpoint[] valid */ + /* DMA state for current async operation */ + dma_addr_t ckpt_dma; /* RESTORE input */ + dma_addr_t save_dma; /* SAVE output (update only) */ + dma_addr_t data_dma; /* UPDATE input */ + dma_addr_t digest_dma; /* FINAL output (final/digest only) */ + u8 *save_buf; /* SAVE output buffer */ + u8 *data_buf; /* linearised data for DMA */ + u32 data_len; /* bytes in data_buf */ + u8 *digest_buf; /* digest output buffer */ + u8 buf[CMH_HASH_MAX_BLOCK]; /* holdback for partial block */ + u8 checkpoint[HC_CONTEXT_SIZE]; /* HC context from last SAVE */ + struct vcq_cmd packed[CMH_HASH_MAX_PACKED]; +}; + +/* VCQ Builders (HC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */ + +/* Add an HC_CMD_UPDATE entry */ +static void vcq_add_hc_update(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_UPDATE); + slot->hwc.hc.cmd_update.input = input_phys; + slot->hwc.hc.cmd_update.inlen = len; +} + +/* Add an HC_CMD_SAVE entry */ +static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id, u64 output_phys, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE); + slot->hwc.hc.cmd_save.output = output_phys; + slot->hwc.hc.cmd_save.outlen = outlen; +} + +/* Add an HC_CMD_RESTORE entry */ +static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 inlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE); + slot->hwc.hc.cmd_restore.input = input_phys; + slot->hwc.hc.cmd_restore.inlen = inlen; +} + +/* Request Context Cleanup */ + +static void cmh_hash_free_reqctx(struct cmh_hash_reqctx *rctx) +{ + rctx->has_checkpoint = 0; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +/* + * Wrapper struct: embeds ahash_alg + a pointer to our alg_info table + * entry so we can recover it in the tfm callbacks. + */ +struct cmh_hash_alg_drv { + struct ahash_alg alg; + const struct cmh_hash_alg_info *info; +}; + +/* + * Find the cmh_hash_alg_info from the crypto_ahash (embedded in our + * registered template). We stash the info pointer in the algorithm's + * driver-private area at registration time (see cmh_hash_register). + */ +static const struct cmh_hash_alg_info * +cmh_hash_get_info(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + return container_of(alg, struct cmh_hash_alg_drv, alg)->info; +} + +static int cmh_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + rctx->info = cmh_hash_get_info(tfm); + return 0; +} + +/* + * Update completion -- called from threaded IRQ after SAVE completes. + * Takes ownership of save_buf as the new checkpoint. + */ +static void cmh_hash_update_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* Unmap DMA buffers */ + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->data_dma, rctx->data_len, + DMA_TO_DEVICE); + + if (!error) { + memcpy(rctx->checkpoint, rctx->save_buf, HC_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + kfree(rctx->save_buf); + rctx->save_buf = NULL; + rctx->hw_started = 1; + } else { + kfree(rctx->save_buf); + rctx->save_buf = NULL; + rctx->error = error; + } + + kfree(rctx->data_buf); + rctx->data_buf = NULL; + rctx->data_len = 0; + + cmh_complete(&req->base, error); +} + +/* + * .update -- buffer incoming data, submit full blocks to HW. + * + * Maintains a partial-block holdback buffer in rctx->buf[]. When + * enough data is available for at least one full block, the full + * blocks are linearised and submitted as: + * INIT [+ RESTORE] + UPDATE(full_blocks) + SAVE + FLUSH + * + * The tail (< block_size) stays in the holdback for the next call. + * Returns -EINPROGRESS on HW submission, 0 if only buffering. + */ +static int cmh_hash_update(struct ahash_request *req) +{ + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_hash_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_HASH_MAX_PAYLOAD]; + struct core_dispatch d; + u32 block_size = info->block_size; + u32 total_avail, full_len, tail_len, from_src; + u32 idx; + int ret; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + total_avail = rctx->buf_len + req->nbytes; + + /* Not enough for a full block -- just buffer */ + if (total_avail < block_size) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->buf + rctx->buf_len, + req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(rctx->buf + rctx->buf_len, + req->src, 0, + req->nbytes, 0); + rctx->buf_len = total_avail; + return 0; + } + + /* Have at least one full block -- submit to HW */ + full_len = total_avail - total_avail % block_size; + tail_len = total_avail - full_len; + from_src = full_len - rctx->buf_len; + + /* Linearise: holdback prefix + full blocks from scatterlist */ + rctx->data_buf = kmalloc(full_len, gfp); + if (!rctx->data_buf) + return -ENOMEM; + + if (rctx->buf_len > 0) + memcpy(rctx->data_buf, rctx->buf, rctx->buf_len); + + if (from_src > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->data_buf + rctx->buf_len, + req->svirt, from_src); + else + scatterwalk_map_and_copy(rctx->data_buf + rctx->buf_len, + req->src, 0, + from_src, 0); + } + + /* Move tail to holdback */ + if (tail_len > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->buf, req->svirt + from_src, + tail_len); + else + scatterwalk_map_and_copy(rctx->buf, req->src, + from_src, tail_len, + 0); + } + rctx->buf_len = tail_len; + rctx->data_len = full_len; + + /* Allocate SAVE output buffer */ + rctx->save_buf = kzalloc(HC_CONTEXT_SIZE, gfp); + if (!rctx->save_buf) { + ret = -ENOMEM; + goto err_free; + } + + /* DMA map data, save output, and checkpoint */ + rctx->data_dma = cmh_dma_map_single(rctx->data_buf, full_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->data_dma)) { + ret = -ENOMEM; + goto err_free; + } + + rctx->save_dma = cmh_dma_map_single(rctx->save_buf, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->save_dma)) { + ret = -ENOMEM; + goto err_unmap_data; + } + + rctx->ckpt_dma = DMA_MAPPING_ERROR; + if (rctx->has_checkpoint) { + rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->ckpt_dma)) { + ret = -ENOMEM; + goto err_unmap_save; + } + } + + /* Build VCQ: INIT [+ RESTORE] + UPDATE + SAVE + FLUSH */ + d = cmh_core_select_instance(CMH_CORE_HC); + idx = 0; + + vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo); + + if (rctx->has_checkpoint) + vcq_add_hc_restore(&cmds[idx++], d.core_id, + (u64)rctx->ckpt_dma, HC_CONTEXT_SIZE); + + vcq_add_hc_update(&cmds[idx++], d.core_id, + (u64)rctx->data_dma, full_len); + + vcq_add_hc_save(&cmds[idx++], d.core_id, + (u64)rctx->save_dma, HC_CONTEXT_SIZE); + + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_HASH_MAX_PACKED, + d.mbx_idx, + cmh_hash_update_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto err_unmap_ckpt; + + return -EINPROGRESS; + +err_unmap_ckpt: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); +err_unmap_save: + cmh_dma_unmap_single(rctx->save_dma, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); +err_unmap_data: + cmh_dma_unmap_single(rctx->data_dma, full_len, DMA_TO_DEVICE); +err_free: + kfree(rctx->save_buf); + rctx->save_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + rctx->data_len = 0; + return ret; +} + +/* + * Final completion -- unmap all DMA, copy digest, signal done. + */ +static void cmh_hash_final_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (rctx->data_buf) + cmh_dma_unmap_single(rctx->data_dma, rctx->data_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, + rctx->info->digest_size); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + cmh_hash_free_reqctx(rctx); + cmh_complete(&req->base, error); +} + +/* + * Submit the final VCQ transaction: + * INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH + * + * @data_buf: linearised residual data, or NULL for empty-hash. + * Ownership transferred -- callback frees it. + * @data_len: bytes in data_buf. + */ +static int cmh_hash_submit_final(struct ahash_request *req, + u8 *data_buf, u32 data_len) +{ + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_hash_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_HASH_MAX_PAYLOAD]; + struct core_dispatch d; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + rctx->data_buf = data_buf; + rctx->data_len = data_len; + + /* Allocate digest output buffer */ + rctx->digest_buf = kzalloc(info->digest_size, gfp); + if (!rctx->digest_buf) { + ret = -ENOMEM; + goto err_free_data; + } + + rctx->digest_dma = cmh_dma_map_single(rctx->digest_buf, + info->digest_size, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->digest_dma)) { + ret = -ENOMEM; + goto err_free_digest; + } + + /* Map residual data for UPDATE */ + rctx->data_dma = DMA_MAPPING_ERROR; + if (data_buf && data_len > 0) { + rctx->data_dma = cmh_dma_map_single(data_buf, data_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->data_dma)) { + ret = -ENOMEM; + goto err_unmap_digest; + } + } + + /* Map checkpoint for RESTORE */ + rctx->ckpt_dma = DMA_MAPPING_ERROR; + if (rctx->has_checkpoint) { + rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->ckpt_dma)) { + ret = -ENOMEM; + goto err_unmap_data; + } + } + + /* Build VCQ: INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH */ + d = cmh_core_select_instance(CMH_CORE_HC); + idx = 0; + + vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo); + + if (rctx->has_checkpoint) + vcq_add_hc_restore(&cmds[idx++], d.core_id, + (u64)rctx->ckpt_dma, HC_CONTEXT_SIZE); + + if (data_buf && data_len > 0) + vcq_add_hc_update(&cmds[idx++], d.core_id, + (u64)rctx->data_dma, data_len); + + vcq_add_hc_final(&cmds[idx++], d.core_id, + (u64)rctx->digest_dma, info->digest_size); + + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_HASH_MAX_PACKED, + d.mbx_idx, + cmh_hash_final_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto err_unmap_ckpt; + + return -EINPROGRESS; + +err_unmap_ckpt: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); +err_unmap_data: + if (data_buf && data_len > 0) + cmh_dma_unmap_single(rctx->data_dma, data_len, + DMA_TO_DEVICE); +err_unmap_digest: + cmh_dma_unmap_single(rctx->digest_dma, info->digest_size, + DMA_FROM_DEVICE); +err_free_digest: + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; +err_free_data: + kfree(data_buf); + rctx->data_buf = NULL; + cmh_hash_free_reqctx(rctx); + return ret; +} + +static int cmh_hash_final(struct ahash_request *req) +{ + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + u8 *data_buf = NULL; + u32 data_len = 0; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (rctx->buf_len > 0) { + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + data_buf = kmalloc(rctx->buf_len, gfp); + if (!data_buf) + return -ENOMEM; + memcpy(data_buf, rctx->buf, rctx->buf_len); + data_len = rctx->buf_len; + rctx->buf_len = 0; + } + + return cmh_hash_submit_final(req, data_buf, data_len); +} + +static int cmh_hash_finup(struct ahash_request *req); + +/* + * One-shot digest -- delegates to init + finup so that all data is + * linearised and mapped through cmh_dma_map_single(), which is the + * only DMA mapping path aware of all supported DMA backends. + */ +static int cmh_hash_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_hash_init(req); + if (ret) + return ret; + return cmh_hash_finup(req); +} + +/* + * .finup -- update + final combined into a single transaction. + * + * Linearises the holdback buffer + new data and submits everything + * through the final path. Avoids the kernel's ahash_def_finup() + * which would allocate a subrequest and clone via export/import. + */ +static int cmh_hash_finup(struct ahash_request *req) +{ + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + u32 data_len; + u8 *data_buf; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + data_len = rctx->buf_len + req->nbytes; + + if (data_len == 0) + return cmh_hash_submit_final(req, NULL, 0); + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + data_buf = kmalloc(data_len, gfp); + if (!data_buf) + return -ENOMEM; + + if (rctx->buf_len > 0) + memcpy(data_buf, rctx->buf, rctx->buf_len); + + if (req->nbytes > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(data_buf + rctx->buf_len, + req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(data_buf + rctx->buf_len, + req->src, 0, + req->nbytes, 0); + } + + rctx->buf_len = 0; + return cmh_hash_submit_final(req, data_buf, data_len); +} + +/* + * Export -- purely software. + * + * Serialise the HC checkpoint (if any) and holdback buffer into the + * export state structure. No HW interaction needed because the + * incremental model keeps checkpoint up-to-date after each .update(). + */ +static int cmh_hash_export(struct ahash_request *req, void *out) +{ + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + struct cmh_hash_export_state *state = out; + + if (rctx->hw_started && rctx->has_checkpoint) + memcpy(state->checkpoint, rctx->checkpoint, HC_CONTEXT_SIZE); + else + memset(state->checkpoint, 0, HC_CONTEXT_SIZE); + + if (rctx->buf_len > 0) + memcpy(state->buf, rctx->buf, rctx->buf_len); + + state->buf_len = rctx->buf_len; + state->hw_started = rctx->hw_started; + + return 0; +} + +/* + * Import -- purely software. + * + * Restore checkpoint and holdback from a previously exported state. + * The next .update() or .final() will RESTORE the checkpoint into HW. + */ +static int cmh_hash_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hash_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_hash_export_state *state = in; + + memset(rctx, 0, sizeof(*rctx)); + rctx->info = cmh_hash_get_info(tfm); + + if (state->buf_len > CMH_HASH_MAX_BLOCK) + return -EINVAL; + + rctx->hw_started = state->hw_started; + rctx->buf_len = state->buf_len; + memcpy(rctx->buf, state->buf, state->buf_len); + + if (state->hw_started) { + memcpy(rctx->checkpoint, state->checkpoint, HC_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + } + + return 0; +} + +/* Transform init (cra_init) -- set per-request context size */ + +static int cmh_hash_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_hash_reqctx)); + return 0; +} + +/* Registration */ + +static struct cmh_hash_alg_drv cmh_hash_drvs[CMH_HASH_ALG_COUNT]; + +/** + * cmh_hash_register() - Register SHA-256/384/512/3-256/3-384/3-512 hash algorithms + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_hash_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < CMH_HASH_ALG_COUNT; i++) { + const struct cmh_hash_alg_info *info = &cmh_hash_algs_info[i]; + struct cmh_hash_alg_drv *drv = &cmh_hash_drvs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + alg->init = cmh_hash_init; + alg->update = cmh_hash_update; + alg->final = cmh_hash_final; + alg->finup = cmh_hash_finup; + alg->digest = cmh_hash_digest; + alg->export = cmh_hash_export; + alg->import = cmh_hash_import; + + alg->halg.digestsize = info->digest_size; + alg->halg.statesize = sizeof(struct cmh_hash_export_state); + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = info->block_size; + alg->halg.base.cra_ctxsize = 0; + alg->halg.base.cra_init = cmh_hash_cra_init; + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "hash: failed to register %s (rc=%d)\n", + info->drv_name, ret); + /* Unregister any already-registered algorithms */ + while (i--) + crypto_unregister_ahash(&cmh_hash_drvs[i].alg); + return ret; + } + + dev_dbg(cmh_dev(), "hash: registered %s (priority 300)\n", + info->drv_name); + } + + dev_info(cmh_dev(), "hash: %zu algorithm(s) registered\n", + CMH_HASH_ALG_COUNT); + return 0; +} + +/** + * cmh_hash_unregister() - Unregister SHA hash algorithms from the crypto framework + */ +void cmh_hash_unregister(void) +{ + unsigned int i; + + for (i = 0; i < CMH_HASH_ALG_COUNT; i++) { + crypto_unregister_ahash(&cmh_hash_drvs[i].alg); + dev_dbg(cmh_dev(), "hash: unregistered %s\n", + cmh_hash_algs_info[i].drv_name); + } + + dev_info(cmh_dev(), "hash: cleaned up\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 7673ed3e586189..dea019346567b7 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -29,6 +29,7 @@ #include "cmh_mqi.h" #include "cmh_txn.h" #include "cmh_rh.h" +#include "cmh_hash.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -197,6 +198,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_rh_init; + /* Register hash algorithms with the kernel crypto API */ + ret = cmh_hash_register(); + if (ret) + goto err_hash_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -209,6 +215,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_hash_unregister(); +err_hash_register: cmh_rh_cleanup(cfg); err_rh_init: cmh_tm_cleanup(); @@ -235,6 +243,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_hash_unregister(); cmh_rh_cleanup(cfg); cmh_tm_cleanup(); cmh_mqi_cleanup(cfg); diff --git a/drivers/crypto/cmh/include/cmh_hash.h b/drivers/crypto/cmh/include/cmh_hash.h new file mode 100644 index 00000000000000..bf17d3af7787ab --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_hash.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API Hash Driver + * + * Registers ahash algorithms (SHA-2, SHA-3, and SHAKE families) with the + * Linux crypto subsystem. Uses an incremental HW update model: + * + * .init() -> software-only: zero per-request context + * .update() -> holdback partial blocks; submit full blocks via + * INIT [+ RESTORE] + UPDATE + SAVE + FLUSH + * .final() -> INIT [+ RESTORE] [+ UPDATE(residual)] + FINAL + FLUSH + * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot) + * .export() -> software-only: copy checkpoint + holdback + * .import() -> software-only: restore checkpoint + holdback + */ + +#ifndef CMH_HASH_H +#define CMH_HASH_H + +#include "cmh_config.h" + +int cmh_hash_register(void); +void cmh_hash_unregister(void); + +#endif /* CMH_HASH_H */ From 667a287ae2b1f7bd6bfeec190a870603883fb471 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:23 -0700 Subject: [PATCH 07/21] crypto: cmh - add HMAC ahash Register ahash algorithms for HMAC-SHA-224, HMAC-SHA-256, HMAC-SHA-384, HMAC-SHA-512, HMAC-SHA3-224, HMAC-SHA3-256, HMAC-SHA3-384, and HMAC-SHA3-512 using the CMH hash core. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 3 +- drivers/crypto/cmh/cmh_hmac.c | 684 ++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/include/cmh_hmac.h | 16 + 4 files changed, 711 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_hmac.c create mode 100644 drivers/crypto/cmh/include/cmh_hmac.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index c0531f41622953..1f760c0214ef7b 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -15,7 +15,8 @@ cmh-y := \ cmh_sysfs.o \ cmh_key.o \ cmh_sys.o \ - cmh_hash.o + cmh_hash.o \ + cmh_hmac.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_hmac.c b/drivers/crypto/cmh/cmh_hmac.c new file mode 100644 index 00000000000000..1f536088eabf2c --- /dev/null +++ b/drivers/crypto/cmh/cmh_hmac.c @@ -0,0 +1,684 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API HMAC Driver + * + * Registers HMAC ahash algorithms with the Linux crypto subsystem. + * Supports HMAC-SHA-2 (224/256/384/512) and HMAC-SHA-3 (224/256/384/512) + * using the CMH Hash Core (HC) via HC_CMD_HMAC. + * + * Uses the same self-contained transaction model as cmh_hash.c: + * .setkey() -> store raw key bytes + * .init() -> software-only: initialize per-request context + * .update() -> software-only: copy SG data into per-call chunk + * .final() -> [SYS_CMD_WRITE] + HC_CMD_HMAC + [GATHER] + FINAL + FLUSH + * + * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into + * the same VCQ as HC_CMD_HMAC (see cmh_key.h for details). + * + * ahash .export()/.import() (state cloning): supported at the + * software accumulation level only. The HW hash core does NOT + * support save/restore of intermediate HMAC state (SHA3 sponge + * invertibility, SHA2 blocked for consistency). Since this driver + * accumulates all input data in kernel memory before submitting + * atomically in .final(), export/import simply serializes the + * input queue -- no keying material or HW state is exposed. + * + * All HMAC data is accumulated in kernel memory and capped at + * HMAC_MAX_DATA (64 KB). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_hmac.h" +#include "cmh_vcq.h" +#include "cmh_hc_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * Maximum data that can be accumulated across .update() calls. + * HMAC save/restore is intentionally unsupported (see file header), + * so all data must be buffered in kernel memory and submitted + * atomically in .final(). This cap prevents unbounded allocation. + */ +#define HMAC_MAX_DATA (64 * 1024) + +/* Algorithm Table */ + +struct cmh_hmac_alg_info { + u32 hc_algo; /* HC_ALGO_* */ + u32 digest_size; /* bytes */ + u32 block_size; /* cra_blocksize */ + const char *alg_name; /* Linux crypto name: "hmac(sha256)" */ + const char *drv_name; /* driver name: "cri-cmh-hmac-sha256" */ +}; + +static const struct cmh_hmac_alg_info cmh_hmac_algs_info[] = { + /* HMAC-SHA-2 family */ + { + .hc_algo = HC_ALGO_SHA2_224, + .digest_size = CMH_SHA224_DIGEST_SIZE, + .block_size = 64, + .alg_name = "hmac(sha224)", + .drv_name = "cri-cmh-hmac-sha224", + }, + { + .hc_algo = HC_ALGO_SHA2_256, + .digest_size = CMH_SHA256_DIGEST_SIZE, + .block_size = 64, + .alg_name = "hmac(sha256)", + .drv_name = "cri-cmh-hmac-sha256", + }, + { + .hc_algo = HC_ALGO_SHA2_384, + .digest_size = CMH_SHA384_DIGEST_SIZE, + .block_size = 128, + .alg_name = "hmac(sha384)", + .drv_name = "cri-cmh-hmac-sha384", + }, + { + .hc_algo = HC_ALGO_SHA2_512, + .digest_size = CMH_SHA512_DIGEST_SIZE, + .block_size = 128, + .alg_name = "hmac(sha512)", + .drv_name = "cri-cmh-hmac-sha512", + }, + /* HMAC-SHA-3 family */ + { + .hc_algo = HC_ALGO_SHA3_224, + .digest_size = CMH_SHA3_224_DIGEST_SIZE, + .block_size = 144, + .alg_name = "hmac(sha3-224)", + .drv_name = "cri-cmh-hmac-sha3-224", + }, + { + .hc_algo = HC_ALGO_SHA3_256, + .digest_size = CMH_SHA3_256_DIGEST_SIZE, + .block_size = 136, + .alg_name = "hmac(sha3-256)", + .drv_name = "cri-cmh-hmac-sha3-256", + }, + { + .hc_algo = HC_ALGO_SHA3_384, + .digest_size = CMH_SHA3_384_DIGEST_SIZE, + .block_size = 104, + .alg_name = "hmac(sha3-384)", + .drv_name = "cri-cmh-hmac-sha3-384", + }, + { + .hc_algo = HC_ALGO_SHA3_512, + .digest_size = CMH_SHA3_512_DIGEST_SIZE, + .block_size = 72, + .alg_name = "hmac(sha3-512)", + .drv_name = "cri-cmh-hmac-sha3-512", + }, +}; + +#define CMH_HMAC_ALG_COUNT ARRAY_SIZE(cmh_hmac_algs_info) + +/* Per-Request State */ + +struct cmh_hmac_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* + * Maximum payload commands any HMAC transaction can produce: + * [SYS_CMD_WRITE] + HC_CMD_HMAC + [GATHER] + FINAL + FLUSH = 5 + * Worst-case packed output (stride=7, 1 payload per VCQ): + * 5 VCQs x 2 entries = 10 + */ +#define CMH_HMAC_MAX_PAYLOAD 5 +#define CMH_HMAC_MAX_PACKED (CMH_HMAC_MAX_PAYLOAD * 2) + +struct cmh_hmac_reqctx { + const struct cmh_hmac_alg_info *info; + int error; + struct list_head chunks; + u32 num_chunks; + u32 total_len; + /* DMA state for async final */ + dma_addr_t digest_dma; + dma_addr_t key_dma; + u8 *digest_buf; + struct cmh_sg_map *sgm; + u32 keylen; + struct vcq_cmd packed[CMH_HMAC_MAX_PACKED]; +}; + +/* Flat state for export/import -- holds accumulated input data only */ +struct cmh_hmac_export_state { + u32 total_len; + u8 data[]; +}; + +/* + * Flat state buffer for export/import. The CMH hash core does not + * support save/restore of intermediate HMAC state, so this driver + * accumulates input in SW and serialises the buffer on export. + * + * PAGE_SIZE (4096) caps the exportable accumulated-data window. + * Full-range export (up to HMAC_MAX_DATA = 64 KB) is not feasible + * because the crypto subsystem pre-allocates statesize bytes per + * request. Export returns -EINVAL if the caller has accumulated + * more than CMH_HMAC_EXPORT_MAX. + */ +#define CMH_HMAC_STATE_SIZE 4096 +#define CMH_HMAC_EXPORT_MAX (CMH_HMAC_STATE_SIZE - sizeof(struct cmh_hmac_export_state)) + +/* Per-Transform State (carries key across requests) */ + +struct cmh_hmac_tfm_ctx { + struct cmh_key_ctx key; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* VCQ Builders (HMAC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */ + +/* Add an HC_CMD_HMAC entry */ +static void vcq_add_hc_hmac(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u32 keylen, u32 algo) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_HMAC); + slot->hwc.hc.cmd_hmac.key = key_ref; + slot->hwc.hc.cmd_hmac.keylen = keylen; + slot->hwc.hc.cmd_hmac.algo = algo; +} + +/* Request Context Cleanup */ + +static void cmh_hmac_free_chunks(struct cmh_hmac_reqctx *rctx, + struct cmh_hmac_tfm_ctx *tctx) +{ + struct cmh_hmac_chunk *chunk, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) { + list_del(&chunk->list); + list_del(&chunk->tfm_node); + kfree_sensitive(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks = 0; + rctx->total_len = 0; +} + +/* + * Build a DMA-mapped CMH eSW scatter-gather chain from accumulated chunks. + */ +static struct cmh_sg_map * +cmh_hmac_build_sg(struct cmh_hmac_reqctx *rctx, gfp_t gfp) +{ + struct cmh_dma_buf *bufs; + struct cmh_hmac_chunk *chunk; + struct cmh_sg_map *sgm; + u32 i; + + bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp); + if (!bufs) + return NULL; + + i = 0; + list_for_each_entry(chunk, &rctx->chunks, list) { + bufs[i].data = chunk->data; + bufs[i].len = chunk->len; + i++; + } + + sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp); + kfree(bufs); + return sgm; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +struct cmh_hmac_alg_drv { + struct ahash_alg alg; + const struct cmh_hmac_alg_info *info; +}; + +static const struct cmh_hmac_alg_info * +cmh_hmac_get_info(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + return container_of(alg, struct cmh_hmac_alg_drv, alg)->info; +} + +static int cmh_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_HC); +} + +static int cmh_hmac_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->info = cmh_hmac_get_info(tfm); + rctx->error = 0; + INIT_LIST_HEAD(&rctx->chunks); + rctx->num_chunks = 0; + rctx->total_len = 0; + + return 0; +} + +static int cmh_hmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_hmac_chunk *chunk; + int nents; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + if (req->nbytes > HMAC_MAX_DATA - rctx->total_len) { + rctx->error = -EINVAL; + goto err_free_chunks; + } + + chunk = kmalloc(sizeof(*chunk) + req->nbytes, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC); + if (!chunk) { + rctx->error = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) { + memcpy(chunk->data, req->svirt, req->nbytes); + } else { + nents = sg_nents_for_len(req->src, req->nbytes); + if (nents < 0 || + sg_copy_to_buffer(req->src, nents, + chunk->data, req->nbytes) != req->nbytes) { + kfree_sensitive(chunk); + rctx->error = -EINVAL; + goto err_free_chunks; + } + } + + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks++; + rctx->total_len += req->nbytes; + + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * The crypto API hash path does not call .final() + * on error, and hash_sock_destruct has no per-request + * destructor, so chunks would be orphaned otherwise. + */ + cmh_hmac_free_chunks(rctx, tctx); + return rctx->error; +} + +static void cmh_hmac_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, + rctx->info->digest_size); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + cmh_dma_free_sg(rctx->sgm); + rctx->sgm = NULL; + cmh_hmac_free_chunks(rctx, tctx); + cmh_complete(&req->base, error); +} + +static int cmh_hmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_hmac_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_HMAC_MAX_PAYLOAD]; + struct cmh_sg_map *sgm = NULL; + dma_addr_t digest_dma = DMA_MAPPING_ERROR, key_dma = DMA_MAPPING_ERROR; + u8 *digest_buf; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + if (rctx->error) { + ret = rctx->error; + goto out_free; + } + + if (tctx->key.mode == CMH_KEY_NONE) { + ret = -ENOKEY; + goto out_free; + } + + if (rctx->num_chunks > 0) { + sgm = cmh_hmac_build_sg(rctx, gfp); + if (!sgm) { + ret = -ENOMEM; + goto out_free; + } + } + + digest_buf = kzalloc(info->digest_size, gfp); + if (!digest_buf) { + ret = -ENOMEM; + goto out_free_sg; + } + digest_dma = cmh_dma_map_single(digest_buf, info->digest_size, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(digest_dma)) { + ret = -ENOMEM; + goto out_free_digest; + } + + /* Resolve key reference */ + idx = 0; + + /* + * Raw key: pack SYS_CMD_WRITE(SYS_REF_TEMP) into the + * same VCQ so the key write + HMAC are atomic. + */ + key_dma = tctx->key.raw.dma; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, (u64)key_dma, + SYS_REF_NONE, tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_HC); + + target_mbx = d.mbx_idx; + + core_id = d.core_id; + + vcq_add_hc_hmac(&cmds[idx++], core_id, key_ref, keylen, info->hc_algo); + + if (sgm) + vcq_add_hc_gather(&cmds[idx++], core_id, (u64)sgm->items_dma, + HC_CMD_UPDATE); + + vcq_add_hc_final(&cmds[idx++], core_id, (u64)digest_dma, info->digest_size); + vcq_add_flush(&cmds[idx++], core_id); + + rctx->digest_buf = digest_buf; + rctx->digest_dma = digest_dma; + rctx->sgm = sgm; + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_HMAC_MAX_PACKED, + target_mbx, + cmh_hmac_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(digest_dma, info->digest_size, + DMA_FROM_DEVICE); +out_free_digest: + kfree(digest_buf); + +out_free_sg: + cmh_dma_free_sg(sgm); + +out_free: + cmh_hmac_free_chunks(rctx, tctx); + return ret; +} + +static int cmh_hmac_finup(struct ahash_request *req) +{ + int ret; + + ret = cmh_hmac_update(req); + if (ret) + return ret; + + return cmh_hmac_final(req); +} + +static int cmh_hmac_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_hmac_init(req); + if (ret) + return ret; + + return cmh_hmac_finup(req); +} + +/* + * ahash .export()/.import(): serialize/deserialize the software + * accumulation buffer. No HW state is involved. + */ + +static int cmh_hmac_export(struct ahash_request *req, void *out) +{ + struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_hmac_export_state *state = out; + struct cmh_hmac_chunk *chunk; + u32 offset = 0; + + if (rctx->total_len > CMH_HMAC_EXPORT_MAX) + return -ENOSPC; + + state->total_len = rctx->total_len; + list_for_each_entry(chunk, &rctx->chunks, list) { + memcpy(state->data + offset, chunk->data, chunk->len); + offset += chunk->len; + } + return 0; +} + +static int cmh_hmac_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_hmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_hmac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_hmac_export_state *state = in; + struct cmh_hmac_chunk *chunk; + + /* + * Do NOT call free_chunks() here: the crypto API does not + * guarantee the request context is in a valid state before + * import(), so the list pointers may be stale or invalid. + * Re-initialize from scratch instead. Any pre-existing chunks + * are tracked on tctx->all_chunks and freed in cra_exit. + */ + rctx->info = cmh_hmac_get_info(tfm); + rctx->error = 0; + INIT_LIST_HEAD(&rctx->chunks); + rctx->num_chunks = 0; + rctx->total_len = 0; + + if (state->total_len > CMH_HMAC_EXPORT_MAX) + return -EINVAL; + + if (state->total_len) { + chunk = kmalloc(sizeof(*chunk) + state->total_len, GFP_KERNEL); + if (!chunk) + return -ENOMEM; + chunk->len = state->total_len; + memcpy(chunk->data, state->data, state->total_len); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks = 1; + rctx->total_len = state->total_len; + } + return 0; +} + +/* Transform init/exit (cra_init/cra_exit) */ + +static int cmh_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct cmh_hmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + tctx->key.mode = CMH_KEY_NONE; + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_hmac_reqctx)); + return 0; +} + +static void cmh_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct cmh_hmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + struct cmh_hmac_chunk *chunk, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) { + list_del(&chunk->tfm_node); + kfree_sensitive(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + + cmh_key_destroy(&tctx->key); +} + +/* Registration */ + +static struct cmh_hmac_alg_drv cmh_hmac_drvs[CMH_HMAC_ALG_COUNT]; + +/** + * cmh_hmac_register() - Register HMAC-SHA hash algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_hmac_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < CMH_HMAC_ALG_COUNT; i++) { + const struct cmh_hmac_alg_info *info = &cmh_hmac_algs_info[i]; + struct cmh_hmac_alg_drv *drv = &cmh_hmac_drvs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + alg->init = cmh_hmac_init; + alg->update = cmh_hmac_update; + alg->final = cmh_hmac_final; + alg->finup = cmh_hmac_finup; + alg->digest = cmh_hmac_digest; + alg->export = cmh_hmac_export; + alg->import = cmh_hmac_import; + alg->setkey = cmh_hmac_setkey; + + alg->halg.digestsize = info->digest_size; + alg->halg.statesize = CMH_HMAC_STATE_SIZE; + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = info->block_size; + alg->halg.base.cra_ctxsize = sizeof(struct cmh_hmac_tfm_ctx); + alg->halg.base.cra_init = cmh_hmac_cra_init; + alg->halg.base.cra_exit = cmh_hmac_cra_exit; + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "hmac: failed to register %s (rc=%d)\n", + info->drv_name, ret); + while (i--) + crypto_unregister_ahash(&cmh_hmac_drvs[i].alg); + return ret; + } + + dev_dbg(cmh_dev(), "hmac: registered %s (priority 300)\n", + info->drv_name); + } + + dev_info(cmh_dev(), "hmac: %zu algorithm(s) registered\n", + CMH_HMAC_ALG_COUNT); + return 0; +} + +/** + * cmh_hmac_unregister() - Unregister HMAC-SHA hash algorithms from the crypto framework + */ +void cmh_hmac_unregister(void) +{ + unsigned int i; + + for (i = 0; i < CMH_HMAC_ALG_COUNT; i++) { + crypto_unregister_ahash(&cmh_hmac_drvs[i].alg); + dev_dbg(cmh_dev(), "hmac: unregistered %s\n", + cmh_hmac_algs_info[i].drv_name); + } + + dev_info(cmh_dev(), "hmac: cleaned up\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index dea019346567b7..618c981e3997b9 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -30,6 +30,7 @@ #include "cmh_txn.h" #include "cmh_rh.h" #include "cmh_hash.h" +#include "cmh_hmac.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -203,6 +204,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_hash_register; + /* Register HMAC hash algorithms */ + ret = cmh_hmac_register(); + if (ret) + goto err_hmac_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -215,6 +221,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_hmac_unregister(); +err_hmac_register: cmh_hash_unregister(); err_hash_register: cmh_rh_cleanup(cfg); @@ -243,6 +251,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_hmac_unregister(); cmh_hash_unregister(); cmh_rh_cleanup(cfg); cmh_tm_cleanup(); diff --git a/drivers/crypto/cmh/include/cmh_hmac.h b/drivers/crypto/cmh/include/cmh_hmac.h new file mode 100644 index 00000000000000..fb1a11fb76ebe9 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_hmac.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API HMAC Driver + * + * Registers HMAC ahash algorithms (HMAC-SHA-2, HMAC-SHA-3) with the + * Linux crypto subsystem using HC_CMD_HMAC. + */ + +#ifndef CMH_HMAC_H +#define CMH_HMAC_H + +int cmh_hmac_register(void); +void cmh_hmac_unregister(void); + +#endif /* CMH_HMAC_H */ From 37f4c2975a5d8ac99546109a76175c70352dd25d Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:24 -0700 Subject: [PATCH 08/21] crypto: cmh - add CSHAKE/KMAC ahash Register ahash algorithms for cSHAKE128, cSHAKE256, KMAC128, and KMAC256 using the CMH hash core. cSHAKE supports incremental update and export/import. KMAC has a 64KB data cap imposed by the hardware. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 4 +- drivers/crypto/cmh/cmh_cshake.c | 808 ++++++++++++++++++++++++ drivers/crypto/cmh/cmh_kmac.c | 630 ++++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 18 + drivers/crypto/cmh/include/cmh_cshake.h | 16 + drivers/crypto/cmh/include/cmh_kmac.h | 16 + 6 files changed, 1491 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_cshake.c create mode 100644 drivers/crypto/cmh/cmh_kmac.c create mode 100644 drivers/crypto/cmh/include/cmh_cshake.h create mode 100644 drivers/crypto/cmh/include/cmh_kmac.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 1f760c0214ef7b..2bb240b97f31ac 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -16,7 +16,9 @@ cmh-y := \ cmh_key.o \ cmh_sys.o \ cmh_hash.o \ - cmh_hmac.o + cmh_hmac.o \ + cmh_cshake.o \ + cmh_kmac.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_cshake.c b/drivers/crypto/cmh/cmh_cshake.c new file mode 100644 index 00000000000000..02f9b853dd3384 --- /dev/null +++ b/drivers/crypto/cmh/cmh_cshake.c @@ -0,0 +1,808 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API CSHAKE Driver + * + * Registers cSHAKE-128 and cSHAKE-256 as ahash algorithms using the + * CMH Hash Core (HC) via HC_CMD_CSHAKE. + * + * CSHAKE (NIST SP 800-185) extends SHAKE with two domain separation + * parameters: function name N and customization string S. When both + * are empty, cSHAKE reduces to plain SHAKE -- the driver falls back to + * HC_CMD_INIT in that case (per SP 800-185 S6.2). + * + * N and S are set via .setkey() using a self-describing binary header + * (matching the upstream authenc precedent): + * + * struct cshake_cfg { __be32 n_len; __be32 s_len; }; + * setkey blob: cshake_cfg || N[n_len] || S[s_len] + * + * If .setkey() is never called, the driver defaults to plain SHAKE + * (N="" S=""). .setkey() is per-tfm, not per-request. + * + * N is embedded inline in the HC_CMD_CSHAKE struct (max 36 bytes). + * S is passed as VCQ inline data following the command slot (multi-span). + * + * Uses the same self-contained transaction model as cmh_hash.c: + * .init() -> software-only + * .update() -> software-only (accumulate chunks) + * .final() -> CSHAKE [+ inline S] [+ RESTORE] [+ GATHER] + FINAL + FLUSH + * .export() -> CSHAKE [+ inline S] [+ RESTORE] [+ GATHER] + SAVE + FLUSH + * .import() -> restore HC context checkpoint (software-only) + * + * The HC core supports HC_CMD_SAVE / HC_CMD_RESTORE for cSHAKE mode. + * The cSHAKE domain-separation prefix (function name N, customization + * string S) is absorbed into the Keccak sponge state by HC_CMD_CSHAKE + * on the first submission, and preserved through save/restore. + * Export/import enables crypto API transform cloning. + * + * .setkey() here configures public domain-separation parameters (N, S), + * not a secret key. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_cshake.h" +#include "cmh_vcq.h" +#include "cmh_hc_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" + +/* Algorithm Table */ + +struct cmh_cshake_alg_info { + u32 hc_algo; + u32 digest_size; + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_cshake_alg_info cmh_cshake_algs_info[] = { + { + .hc_algo = HC_ALGO_SHAKE128, + .digest_size = CMH_SHAKE128_DIGEST_SIZE, + .alg_name = "cshake128", + .drv_name = "cri-cmh-cshake128", + }, + { + .hc_algo = HC_ALGO_SHAKE256, + .digest_size = CMH_SHAKE256_DIGEST_SIZE, + .alg_name = "cshake256", + .drv_name = "cri-cmh-cshake256", + }, +}; + +#define CMH_CSHAKE_ALG_COUNT ARRAY_SIZE(cmh_cshake_algs_info) + +/* Per-Request State */ + +struct cmh_cshake_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* + * Max payload slots for CSHAKE: + * CSHAKE (1) + inline S (ceil(S_len/64)) + GATHER (1) + FINAL (1) + FLUSH (1) + * S can be up to SHAKE-128 block (168 bytes) = 3 inline slots. + * Conservative: 1 + 3 + 1 + 1 + 1 = 7, plus headers. + * + * Or INIT + GATHER + FINAL + FLUSH = 4 (plain SHAKE fallback). + */ +#define CMH_CSHAKE_MAX_PAYLOAD 8 +#define CMH_CSHAKE_MAX_PACKED (CMH_CSHAKE_MAX_PAYLOAD * 2) + +/* + * Checkpoint embedded inline: the kernel ahash API has no per-request + * destructor, so a heap-allocated checkpoint leaks if a request is + * abandoned without .final(). + */ +struct cmh_cshake_reqctx { + const struct cmh_cshake_alg_info *info; + int error; + struct list_head chunks; + u32 num_chunks; + u32 total_len; + u32 has_checkpoint; + u8 checkpoint[HC_CONTEXT_SIZE]; + /* DMA state for async final */ + dma_addr_t digest_dma; + dma_addr_t ckpt_dma; + u8 *digest_buf; + struct cmh_sg_map *sgm; + struct vcq_cmd packed[CMH_CSHAKE_MAX_PACKED]; +}; + +/* Per-Transform State (carries N and S across requests) */ + +struct cmh_cshake_tfm_ctx { + u8 *func_name; /* N (function name), NULL if empty */ + u32 func_name_len; + u8 *custom; /* S (customization string), NULL if empty */ + u32 custom_len; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* VCQ Builders */ + +/* VCQ Builders (cSHAKE-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */ + +static void vcq_add_hc_save(struct vcq_cmd *slot, u32 core_id, + u64 output_phys, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_SAVE); + slot->hwc.hc.cmd_save.output = output_phys; + slot->hwc.hc.cmd_save.outlen = outlen; +} + +static void vcq_add_hc_restore(struct vcq_cmd *slot, u32 core_id, + u64 input_phys, u32 inlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_RESTORE); + slot->hwc.hc.cmd_restore.input = input_phys; + slot->hwc.hc.cmd_restore.inlen = inlen; +} + +static void vcq_add_hc_cshake(struct vcq_cmd *slot, u32 core_id, u32 algo, + const u8 *name, u32 namelen, + u32 customlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_CSHAKE); + slot->hwc.hc.cmd_cshake.custom = 0; /* inline -- CMH eSW reads from next slot(s) */ + slot->hwc.hc.cmd_cshake.customlen = customlen; + slot->hwc.hc.cmd_cshake.algo = algo; + slot->hwc.hc.cmd_cshake.namelen = namelen; + if (namelen > 0 && name) + memcpy(slot->hwc.hc.cmd_cshake.name, name, + min_t(u32, namelen, HC_CSHAKE_MAX_NAMELEN)); +} + +/* Request Context Cleanup */ + +static void cmh_cshake_free_chunks(struct cmh_cshake_reqctx *rctx, + struct cmh_cshake_tfm_ctx *tctx) +{ + struct cmh_cshake_chunk *chunk, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) { + list_del(&chunk->list); + list_del(&chunk->tfm_node); + kfree(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks = 0; + rctx->total_len = 0; +} + +static void cmh_cshake_free_reqctx(struct cmh_cshake_reqctx *rctx, + struct cmh_cshake_tfm_ctx *tctx) +{ + cmh_cshake_free_chunks(rctx, tctx); + rctx->has_checkpoint = 0; +} + +static struct cmh_sg_map * +cmh_cshake_build_sg(struct cmh_cshake_reqctx *rctx, gfp_t gfp) +{ + struct cmh_dma_buf *bufs; + struct cmh_cshake_chunk *chunk; + struct cmh_sg_map *sgm; + u32 i; + + bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp); + if (!bufs) + return NULL; + + i = 0; + list_for_each_entry(chunk, &rctx->chunks, list) { + bufs[i].data = chunk->data; + bufs[i].len = chunk->len; + i++; + } + + sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp); + kfree(bufs); + return sgm; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +struct cmh_cshake_alg_drv { + struct ahash_alg alg; + const struct cmh_cshake_alg_info *info; +}; + +static const struct cmh_cshake_alg_info * +cmh_cshake_get_info(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + return container_of(alg, struct cmh_cshake_alg_drv, alg)->info; +} + +/* + * .setkey() -- parse N and S from the self-describing cshake_cfg header. + * + * Blob format: cshake_cfg { __be32 n_len; __be32 s_len; } || N || S + * If never called, the driver defaults to plain SHAKE (N="" S=""). + */ +struct cshake_cfg { + __be32 n_len; + __be32 s_len; +}; + +static int cmh_cshake_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cshake_cfg cfg; + u32 n_len, s_len; + const u8 *ptr; + + if (keylen < sizeof(cfg)) + return -EINVAL; + + memcpy(&cfg, key, sizeof(cfg)); + n_len = be32_to_cpu(cfg.n_len); + s_len = be32_to_cpu(cfg.s_len); + + if (keylen != sizeof(cfg) + n_len + s_len) + return -EINVAL; + + if (n_len > HC_CSHAKE_MAX_NAMELEN) + return -EINVAL; + + if (s_len > HC_CSHAKE_MAX_CUSTOMLEN) + return -EINVAL; + + /* Free previous N and S */ + kfree(tctx->func_name); + kfree(tctx->custom); + tctx->func_name = NULL; + tctx->func_name_len = 0; + tctx->custom = NULL; + tctx->custom_len = 0; + + ptr = key + sizeof(cfg); + + if (n_len > 0) { + tctx->func_name = kmemdup(ptr, n_len, GFP_KERNEL); + if (!tctx->func_name) + return -ENOMEM; + tctx->func_name_len = n_len; + ptr += n_len; + } + + if (s_len > 0) { + tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL); + if (!tctx->custom) { + kfree(tctx->func_name); + tctx->func_name = NULL; + tctx->func_name_len = 0; + return -ENOMEM; + } + tctx->custom_len = s_len; + } + + return 0; +} + +static int cmh_cshake_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + + rctx->info = cmh_cshake_get_info(tfm); + rctx->error = 0; + INIT_LIST_HEAD(&rctx->chunks); + rctx->num_chunks = 0; + rctx->total_len = 0; + rctx->has_checkpoint = 0; + + return 0; +} + +static int cmh_cshake_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + struct cmh_cshake_chunk *chunk; + int nents; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + chunk = kmalloc(sizeof(*chunk) + req->nbytes, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC); + if (!chunk) { + rctx->error = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) { + memcpy(chunk->data, req->svirt, req->nbytes); + } else { + nents = sg_nents_for_len(req->src, req->nbytes); + if (nents < 0 || + sg_copy_to_buffer(req->src, nents, + chunk->data, req->nbytes) != req->nbytes) { + kfree(chunk); + rctx->error = -EINVAL; + goto err_free_chunks; + } + } + + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks++; + rctx->total_len += req->nbytes; + + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * The crypto API hash path does not call .final() on error, + * so chunks would be orphaned otherwise. + */ + cmh_cshake_free_chunks(rctx, tctx); + return rctx->error; +} + +static void cmh_cshake_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, + rctx->info->digest_size); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + cmh_dma_free_sg(rctx->sgm); + rctx->sgm = NULL; + cmh_cshake_free_reqctx(rctx, tctx); + cmh_complete(&req->base, error); +} + +static int cmh_cshake_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_cshake_alg_info *info = rctx->info; + struct core_dispatch d; + struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD]; + struct cmh_sg_map *sgm = NULL; + dma_addr_t digest_dma = DMA_MAPPING_ERROR; + dma_addr_t ckpt_dma = DMA_MAPPING_ERROR; + u8 *digest_buf; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + if (rctx->error) { + ret = rctx->error; + goto out_free; + } + + if (rctx->num_chunks > 0) { + sgm = cmh_cshake_build_sg(rctx, gfp); + if (!sgm) { + ret = -ENOMEM; + goto out_free; + } + } + + digest_buf = kzalloc(info->digest_size, gfp); + if (!digest_buf) { + ret = -ENOMEM; + goto out_free_sg; + } + digest_dma = cmh_dma_map_single(digest_buf, info->digest_size, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(digest_dma)) { + ret = -ENOMEM; + goto out_free_digest; + } + + /* Map checkpoint buffer if present (CMH eSW reads it) */ + if (rctx->has_checkpoint) { + ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + HC_CONTEXT_SIZE, DMA_TO_DEVICE); + if (cmh_dma_map_error(ckpt_dma)) { + ret = -ENOMEM; + goto out_unmap_digest; + } + } + + d = cmh_core_select_instance(CMH_CORE_HC); + idx = 0; + + if (rctx->has_checkpoint) { + /* + * Resuming from a saved checkpoint (after export/import): + * INIT + RESTORE [+ GATHER] + FINAL + FLUSH + * The cSHAKE prefix (N,S) is already absorbed in the + * saved Keccak state -- no need to replay HC_CMD_CSHAKE. + */ + vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo); + vcq_add_hc_restore(&cmds[idx++], d.core_id, (u64)ckpt_dma, + HC_CONTEXT_SIZE); + } else { + bool use_cshake = (tctx->func_name_len > 0 || + tctx->custom_len > 0); + + if (use_cshake) { + u32 span; + + vcq_add_hc_cshake(&cmds[idx], d.core_id, + info->hc_algo, + tctx->func_name, + tctx->func_name_len, + tctx->custom_len); + span = vcq_add_inline_data(&cmds[idx], + tctx->custom, + tctx->custom_len); + idx += span; + } else { + vcq_add_hc_init(&cmds[idx++], d.core_id, + info->hc_algo); + } + } + + if (sgm) + vcq_add_hc_gather(&cmds[idx++], d.core_id, (u64)sgm->items_dma, + HC_CMD_UPDATE); + + vcq_add_hc_final(&cmds[idx++], d.core_id, (u64)digest_dma, info->digest_size); + vcq_add_flush(&cmds[idx++], d.core_id); + + rctx->digest_buf = digest_buf; + rctx->digest_dma = digest_dma; + rctx->ckpt_dma = ckpt_dma; + rctx->sgm = sgm; + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CSHAKE_MAX_PACKED, + d.mbx_idx, + cmh_cshake_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(ckpt_dma, HC_CONTEXT_SIZE, + DMA_TO_DEVICE); +out_unmap_digest: + cmh_dma_unmap_single(digest_dma, info->digest_size, + DMA_FROM_DEVICE); +out_free_digest: + kfree(digest_buf); + +out_free_sg: + cmh_dma_free_sg(sgm); + +out_free: + cmh_cshake_free_reqctx(rctx, tctx); + return ret; +} + +static int cmh_cshake_finup(struct ahash_request *req) +{ + int ret; + + ret = cmh_cshake_update(req); + if (ret) + return ret; + + return cmh_cshake_final(req); +} + +static int cmh_cshake_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_cshake_init(req); + if (ret) + return ret; + + return cmh_cshake_finup(req); +} + +static int cmh_cshake_export(struct ahash_request *req, void *out) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_cshake_alg_info *info = rctx->info; + struct core_dispatch d; + struct vcq_cmd cmds[CMH_CSHAKE_MAX_PAYLOAD]; + struct cmh_sg_map *sgm = NULL; + dma_addr_t save_dma = DMA_MAPPING_ERROR; + dma_addr_t ckpt_dma = DMA_MAPPING_ERROR; + u8 *save_buf; + u32 idx; + int ret; + + if (rctx->num_chunks > 0) { + sgm = cmh_cshake_build_sg(rctx, GFP_KERNEL); + if (!sgm) + return -ENOMEM; + } + + save_buf = kzalloc(HC_CONTEXT_SIZE, GFP_KERNEL); + if (!save_buf) { + cmh_dma_free_sg(sgm); + return -ENOMEM; + } + save_dma = cmh_dma_map_single(save_buf, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(save_dma)) { + kfree(save_buf); + cmh_dma_free_sg(sgm); + return -ENOMEM; + } + + /* Map checkpoint buffer if present (CMH eSW reads it) */ + if (rctx->has_checkpoint) { + ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + HC_CONTEXT_SIZE, DMA_TO_DEVICE); + if (cmh_dma_map_error(ckpt_dma)) { + cmh_dma_unmap_single(save_dma, HC_CONTEXT_SIZE, + DMA_FROM_DEVICE); + kfree(save_buf); + cmh_dma_free_sg(sgm); + return -ENOMEM; + } + } + + d = cmh_core_select_instance(CMH_CORE_HC); + idx = 0; + + if (rctx->has_checkpoint) { + /* + * Resuming from a saved checkpoint: + * INIT + RESTORE [+ GATHER] + SAVE + FLUSH + */ + vcq_add_hc_init(&cmds[idx++], d.core_id, info->hc_algo); + vcq_add_hc_restore(&cmds[idx++], d.core_id, (u64)ckpt_dma, + HC_CONTEXT_SIZE); + } else { + bool use_cshake = (tctx->func_name_len > 0 || + tctx->custom_len > 0); + + if (use_cshake) { + u32 span; + + vcq_add_hc_cshake(&cmds[idx], d.core_id, + info->hc_algo, + tctx->func_name, + tctx->func_name_len, + tctx->custom_len); + span = vcq_add_inline_data(&cmds[idx], + tctx->custom, + tctx->custom_len); + idx += span; + } else { + vcq_add_hc_init(&cmds[idx++], d.core_id, + info->hc_algo); + } + } + + if (sgm) + vcq_add_hc_gather(&cmds[idx++], d.core_id, (u64)sgm->items_dma, + HC_CMD_UPDATE); + + vcq_add_hc_save(&cmds[idx++], d.core_id, (u64)save_dma, + HC_CONTEXT_SIZE); + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit(cmds, idx, rctx->packed, CMH_CSHAKE_MAX_PACKED, + d.mbx_idx); + + /* Unmap before CPU read */ + if (rctx->has_checkpoint) + cmh_dma_unmap_single(ckpt_dma, HC_CONTEXT_SIZE, DMA_TO_DEVICE); + cmh_dma_unmap_single(save_dma, HC_CONTEXT_SIZE, DMA_FROM_DEVICE); + + if (!ret) { + memcpy(out, save_buf, HC_CONTEXT_SIZE); + /* Checkpoint now represents all accumulated state */ + memcpy(rctx->checkpoint, save_buf, HC_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + /* Accumulated chunks are now captured in checkpoint */ + cmh_cshake_free_chunks(rctx, tctx); + } + + kfree(save_buf); + cmh_dma_free_sg(sgm); + return ret; +} + +static int cmh_cshake_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_cshake_reqctx *rctx = ahash_request_ctx(req); + + rctx->info = cmh_cshake_get_info(tfm); + rctx->error = 0; + INIT_LIST_HEAD(&rctx->chunks); + rctx->num_chunks = 0; + rctx->total_len = 0; + + memcpy(rctx->checkpoint, in, HC_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + + return 0; +} + +/* Transform init/exit */ + +static int cmh_cshake_cra_init(struct crypto_tfm *tfm) +{ + struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->func_name = NULL; + tctx->func_name_len = 0; + tctx->custom = NULL; + tctx->custom_len = 0; + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_cshake_reqctx)); + return 0; +} + +static void cmh_cshake_cra_exit(struct crypto_tfm *tfm) +{ + struct cmh_cshake_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + struct cmh_cshake_chunk *chunk, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) { + list_del(&chunk->tfm_node); + kfree(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + + kfree(tctx->func_name); + kfree(tctx->custom); + tctx->func_name = NULL; + tctx->custom = NULL; +} + +/* Registration */ + +static struct cmh_cshake_alg_drv cmh_cshake_drvs[CMH_CSHAKE_ALG_COUNT]; + +/** + * cmh_cshake_register() - Register cSHAKE-128/256 hash algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_cshake_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) { + const struct cmh_cshake_alg_info *info = + &cmh_cshake_algs_info[i]; + struct cmh_cshake_alg_drv *drv = &cmh_cshake_drvs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + alg->init = cmh_cshake_init; + alg->update = cmh_cshake_update; + alg->final = cmh_cshake_final; + alg->finup = cmh_cshake_finup; + alg->digest = cmh_cshake_digest; + alg->export = cmh_cshake_export; + alg->import = cmh_cshake_import; + alg->setkey = cmh_cshake_setkey; + + alg->halg.digestsize = info->digest_size; + alg->halg.statesize = HC_CONTEXT_SIZE; + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_OPTIONAL_KEY | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = 1; /* XOF */ + alg->halg.base.cra_ctxsize = sizeof(struct cmh_cshake_tfm_ctx); + alg->halg.base.cra_init = cmh_cshake_cra_init; + alg->halg.base.cra_exit = cmh_cshake_cra_exit; + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "cshake: failed to register %s (rc=%d)\n", + info->drv_name, ret); + while (i--) + crypto_unregister_ahash(&cmh_cshake_drvs[i].alg); + return ret; + } + + dev_dbg(cmh_dev(), "cshake: registered %s (priority 300)\n", + info->drv_name); + } + + dev_info(cmh_dev(), "cshake: %zu algorithm(s) registered\n", + CMH_CSHAKE_ALG_COUNT); + return 0; +} + +/** + * cmh_cshake_unregister() - Unregister cSHAKE hash algorithms from the crypto framework + */ +void cmh_cshake_unregister(void) +{ + unsigned int i; + + for (i = 0; i < CMH_CSHAKE_ALG_COUNT; i++) { + crypto_unregister_ahash(&cmh_cshake_drvs[i].alg); + dev_dbg(cmh_dev(), "cshake: unregistered %s\n", + cmh_cshake_algs_info[i].drv_name); + } + + dev_info(cmh_dev(), "cshake: cleaned up\n"); +} diff --git a/drivers/crypto/cmh/cmh_kmac.c b/drivers/crypto/cmh/cmh_kmac.c new file mode 100644 index 00000000000000..7177a2558e9773 --- /dev/null +++ b/drivers/crypto/cmh/cmh_kmac.c @@ -0,0 +1,630 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API KMAC Driver + * + * Registers KMAC-128 and KMAC-256 as keyed ahash algorithms using the + * CMH Hash Core (HC) via HC_CMD_KMAC. + * + * KMAC (NIST SP 800-185) is a keyed variant of cSHAKE. The function + * name N is always "KMAC" (hardcoded by the CMH eSW). The user sets: + * - A key via .setkey() (raw bytes + optional S) + * - An optional customization string S via the setkey blob + * + * setkey blob format: + * struct kmac_key_param { __be32 keylen; __be32 s_len; }; + * blob: kmac_key_param || key[keylen] || S[s_len] + * + * Uses the same self-contained transaction model as cmh_hmac.c: + * .setkey() -> store raw key (+ S) + * .init() -> software-only + * .update() -> software-only (accumulate chunks) + * .final() -> [SYS_CMD_WRITE] + HC_CMD_KMAC [+ inline S] + + * [GATHER] + FINAL + FLUSH + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_kmac.h" +#include "cmh_vcq.h" +#include "cmh_hc_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * Maximum data that can be accumulated across .update() calls. + * The CMH eSW rejects HC_CMD_SAVE when ctx->outlen != 0, which is + * always the case for KMAC (eip59_hc_kmac() sets ctx->outlen for + * right_encode(outlen) at finalization). All data must be buffered + * in kernel memory and submitted atomically in .final(). + * + * The CMH eSW does not serialize outlen into the external save + * context, so HC_CMD_SAVE fails for KMAC mode. + */ +#define KMAC_MAX_DATA (64 * 1024) + +/* Algorithm Table */ + +struct cmh_kmac_alg_info { + u32 hc_algo; + u32 digest_size; + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_kmac_alg_info cmh_kmac_algs_info[] = { + { + .hc_algo = HC_ALGO_SHAKE128, + .digest_size = CMH_SHAKE128_DIGEST_SIZE, + .alg_name = "kmac128", + .drv_name = "cri-cmh-kmac128", + }, + { + .hc_algo = HC_ALGO_SHAKE256, + .digest_size = CMH_SHAKE256_DIGEST_SIZE, + .alg_name = "kmac256", + .drv_name = "cri-cmh-kmac256", + }, +}; + +#define CMH_KMAC_ALG_COUNT ARRAY_SIZE(cmh_kmac_algs_info) + +/* Per-Request State */ + +struct cmh_kmac_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* + * Max payload slots for KMAC: + * SYS_CMD_WRITE (1) + KMAC (1) + inline S (3 max) + GATHER (1) + + * FINAL (1) + FLUSH (1) = 8 + */ +#define CMH_KMAC_MAX_PAYLOAD 9 +#define CMH_KMAC_MAX_PACKED (CMH_KMAC_MAX_PAYLOAD * 2) + +struct cmh_kmac_reqctx { + const struct cmh_kmac_alg_info *info; + int error; + struct list_head chunks; + u32 num_chunks; + u32 total_len; + /* DMA state for async final */ + dma_addr_t digest_dma; + dma_addr_t key_dma; + u8 *digest_buf; + struct cmh_sg_map *sgm; + u32 keylen; + struct vcq_cmd packed[CMH_KMAC_MAX_PACKED]; +}; + +/* Per-Transform State (carries key + S across requests) */ + +struct cmh_kmac_tfm_ctx { + struct cmh_key_ctx key; + u8 *custom; /* S (customization string), NULL if empty */ + u32 custom_len; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* VCQ Builders (KMAC-specific; shared builders in cmh_hc_abi.h / cmh_vcq.h) */ + +static void vcq_add_hc_kmac(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u32 keylen, + u32 customlen, u32 algo, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HC_CMD_KMAC); + slot->hwc.hc.cmd_kmac.key = key_ref; + slot->hwc.hc.cmd_kmac.custom = 0; /* inline */ + slot->hwc.hc.cmd_kmac.keylen = keylen; + slot->hwc.hc.cmd_kmac.customlen = customlen; + slot->hwc.hc.cmd_kmac.algo = algo; + slot->hwc.hc.cmd_kmac.outlen = outlen; +} + +/* Request Context Cleanup */ + +static void cmh_kmac_free_chunks(struct cmh_kmac_reqctx *rctx, + struct cmh_kmac_tfm_ctx *tctx) +{ + struct cmh_kmac_chunk *chunk, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &rctx->chunks, list) { + list_del(&chunk->list); + list_del(&chunk->tfm_node); + kfree(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks = 0; + rctx->total_len = 0; +} + +static struct cmh_sg_map * +cmh_kmac_build_sg(struct cmh_kmac_reqctx *rctx, gfp_t gfp) +{ + struct cmh_dma_buf *bufs; + struct cmh_kmac_chunk *chunk; + struct cmh_sg_map *sgm; + u32 i; + + bufs = kcalloc(rctx->num_chunks, sizeof(*bufs), gfp); + if (!bufs) + return NULL; + + i = 0; + list_for_each_entry(chunk, &rctx->chunks, list) { + bufs[i].data = chunk->data; + bufs[i].len = chunk->len; + i++; + } + + sgm = cmh_dma_build_sg(bufs, rctx->num_chunks, gfp); + kfree(bufs); + return sgm; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +struct cmh_kmac_alg_drv { + struct ahash_alg alg; + const struct cmh_kmac_alg_info *info; +}; + +static const struct cmh_kmac_alg_info * +cmh_kmac_get_info(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + return container_of(alg, struct cmh_kmac_alg_drv, alg)->info; +} + +/* + * setkey blob for KMAC (raw key path): + * struct kmac_key_param { __be32 keylen; __be32 s_len; }; + * blob: kmac_key_param || key[keylen] || S[s_len] + */ +struct kmac_key_param { + __be32 keylen; + __be32 s_len; +}; + +static int cmh_kmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + /* raw key bytes with optional S */ + { + struct kmac_key_param hdr; + u32 raw_keylen, s_len; + const u8 *ptr; + + if (keylen < sizeof(hdr)) + return -EINVAL; + + memcpy(&hdr, key, sizeof(hdr)); + raw_keylen = be32_to_cpu(hdr.keylen); + s_len = be32_to_cpu(hdr.s_len); + + if (keylen != sizeof(hdr) + raw_keylen + s_len) + return -EINVAL; + + if (raw_keylen == 0) + return -EINVAL; + + if (s_len > HC_CSHAKE_MAX_CUSTOMLEN) + return -EINVAL; + + ptr = key + sizeof(hdr); + + /* Store raw key */ + { + int ret = cmh_key_setkey_raw(&tctx->key, ptr, + raw_keylen, CORE_ID_HC); + if (ret) + return ret; + } + ptr += raw_keylen; + + /* Store S */ + kfree(tctx->custom); + tctx->custom = NULL; + tctx->custom_len = 0; + + if (s_len > 0) { + tctx->custom = kmemdup(ptr, s_len, GFP_KERNEL); + if (!tctx->custom) { + cmh_key_destroy(&tctx->key); + return -ENOMEM; + } + tctx->custom_len = s_len; + } + + return 0; + } +} + +static int cmh_kmac_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->info = cmh_kmac_get_info(tfm); + rctx->error = 0; + INIT_LIST_HEAD(&rctx->chunks); + rctx->num_chunks = 0; + rctx->total_len = 0; + + return 0; +} + +static int cmh_kmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_kmac_chunk *chunk; + int nents; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + if (req->nbytes > KMAC_MAX_DATA - rctx->total_len) { + rctx->error = -EINVAL; + goto err_free_chunks; + } + + chunk = kmalloc(sizeof(*chunk) + req->nbytes, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC); + if (!chunk) { + rctx->error = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) { + memcpy(chunk->data, req->svirt, req->nbytes); + } else { + nents = sg_nents_for_len(req->src, req->nbytes); + if (nents < 0 || + sg_copy_to_buffer(req->src, nents, + chunk->data, req->nbytes) != req->nbytes) { + kfree(chunk); + rctx->error = -EINVAL; + goto err_free_chunks; + } + } + + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->num_chunks++; + rctx->total_len += req->nbytes; + + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * The crypto API hash path does not call .final() on error, + * so chunks would be orphaned otherwise. + */ + cmh_kmac_free_chunks(rctx, tctx); + return rctx->error; +} + +static void cmh_kmac_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_dma_unmap_single(rctx->digest_dma, rctx->info->digest_size, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, + rctx->info->digest_size); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + cmh_dma_free_sg(rctx->sgm); + rctx->sgm = NULL; + cmh_kmac_free_chunks(rctx, tctx); + cmh_complete(&req->base, error); +} + +static int cmh_kmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_kmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_kmac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_kmac_alg_info *info = rctx->info; + struct vcq_cmd cmds[CMH_KMAC_MAX_PAYLOAD]; + struct cmh_sg_map *sgm = NULL; + dma_addr_t digest_dma = DMA_MAPPING_ERROR, key_dma = DMA_MAPPING_ERROR; + u8 *digest_buf; + u64 key_ref; + u32 key_len; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + if (rctx->error) { + ret = rctx->error; + goto out_free; + } + + if (tctx->key.mode == CMH_KEY_NONE) { + ret = -ENOKEY; + goto out_free; + } + + if (rctx->num_chunks > 0) { + sgm = cmh_kmac_build_sg(rctx, gfp); + if (!sgm) { + ret = -ENOMEM; + goto out_free; + } + } + + digest_buf = kzalloc(info->digest_size, gfp); + if (!digest_buf) { + ret = -ENOMEM; + goto out_free_sg; + } + digest_dma = cmh_dma_map_single(digest_buf, info->digest_size, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(digest_dma)) { + ret = -ENOMEM; + goto out_free_digest; + } + + /* Resolve key reference */ + idx = 0; + + key_dma = tctx->key.raw.dma; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, (u64)key_dma, + SYS_REF_NONE, tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + key_len = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_HC); + + target_mbx = d.mbx_idx; + + core_id = d.core_id; + + { + u32 span; + + vcq_add_hc_kmac(&cmds[idx], core_id, key_ref, key_len, + tctx->custom_len, info->hc_algo, + info->digest_size); + + /* Add inline S data after the KMAC slot */ + span = vcq_add_inline_data(&cmds[idx], tctx->custom, + tctx->custom_len); + idx += span; + } + + if (sgm) + vcq_add_hc_gather(&cmds[idx++], core_id, (u64)sgm->items_dma, + HC_CMD_UPDATE); + + vcq_add_hc_final(&cmds[idx++], core_id, (u64)digest_dma, info->digest_size); + vcq_add_flush(&cmds[idx++], core_id); + + rctx->digest_buf = digest_buf; + rctx->digest_dma = digest_dma; + rctx->sgm = sgm; + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_KMAC_MAX_PACKED, + target_mbx, + cmh_kmac_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(digest_dma, info->digest_size, + DMA_FROM_DEVICE); +out_free_digest: + kfree(digest_buf); + +out_free_sg: + cmh_dma_free_sg(sgm); + +out_free: + cmh_kmac_free_chunks(rctx, tctx); + return ret; +} + +static int cmh_kmac_finup(struct ahash_request *req) +{ + int ret; + + ret = cmh_kmac_update(req); + if (ret) + return ret; + + return cmh_kmac_final(req); +} + +static int cmh_kmac_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_kmac_init(req); + if (ret) + return ret; + + return cmh_kmac_finup(req); +} + +static int cmh_kmac_export(struct ahash_request *req, void *out) +{ + return -EOPNOTSUPP; +} + +static int cmh_kmac_import(struct ahash_request *req, const void *in) +{ + return -EOPNOTSUPP; +} + +/* Transform init/exit */ + +static int cmh_kmac_cra_init(struct crypto_tfm *tfm) +{ + struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->key.mode = CMH_KEY_NONE; + tctx->custom = NULL; + tctx->custom_len = 0; + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_kmac_reqctx)); + return 0; +} + +static void cmh_kmac_cra_exit(struct crypto_tfm *tfm) +{ + struct cmh_kmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + struct cmh_kmac_chunk *chunk, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(chunk, tmp, &tctx->all_chunks, tfm_node) { + list_del(&chunk->tfm_node); + kfree(chunk); + } + spin_unlock_bh(&tctx->chunk_lock); + + cmh_key_destroy(&tctx->key); + kfree(tctx->custom); + tctx->custom = NULL; +} + +/* Registration */ + +static struct cmh_kmac_alg_drv cmh_kmac_drvs[CMH_KMAC_ALG_COUNT]; + +/** + * cmh_kmac_register() - Register KMAC-128/256 hash algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_kmac_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) { + const struct cmh_kmac_alg_info *info = + &cmh_kmac_algs_info[i]; + struct cmh_kmac_alg_drv *drv = &cmh_kmac_drvs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + alg->init = cmh_kmac_init; + alg->update = cmh_kmac_update; + alg->final = cmh_kmac_final; + alg->finup = cmh_kmac_finup; + alg->digest = cmh_kmac_digest; + alg->export = cmh_kmac_export; + alg->import = cmh_kmac_import; + alg->setkey = cmh_kmac_setkey; + + alg->halg.digestsize = info->digest_size; + alg->halg.statesize = sizeof(struct cmh_kmac_reqctx); + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = 1; /* XOF/keyed XOF */ + alg->halg.base.cra_ctxsize = sizeof(struct cmh_kmac_tfm_ctx); + alg->halg.base.cra_init = cmh_kmac_cra_init; + alg->halg.base.cra_exit = cmh_kmac_cra_exit; + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "kmac: failed to register %s (rc=%d)\n", + info->drv_name, ret); + while (i--) + crypto_unregister_ahash(&cmh_kmac_drvs[i].alg); + return ret; + } + + dev_dbg(cmh_dev(), "kmac: registered %s (priority 300)\n", + info->drv_name); + } + + dev_info(cmh_dev(), "kmac: %zu algorithm(s) registered\n", + CMH_KMAC_ALG_COUNT); + return 0; +} + +/** + * cmh_kmac_unregister() - Unregister KMAC hash algorithms from the crypto framework + */ +void cmh_kmac_unregister(void) +{ + unsigned int i; + + for (i = 0; i < CMH_KMAC_ALG_COUNT; i++) { + crypto_unregister_ahash(&cmh_kmac_drvs[i].alg); + dev_dbg(cmh_dev(), "kmac: unregistered %s\n", + cmh_kmac_algs_info[i].drv_name); + } + + dev_info(cmh_dev(), "kmac: cleaned up\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 618c981e3997b9..08d14aae912d14 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -31,6 +31,8 @@ #include "cmh_rh.h" #include "cmh_hash.h" #include "cmh_hmac.h" +#include "cmh_cshake.h" +#include "cmh_kmac.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -209,6 +211,16 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_hmac_register; + /* Register CSHAKE hash algorithms */ + ret = cmh_cshake_register(); + if (ret) + goto err_cshake_register; + + /* Register KMAC hash algorithms */ + ret = cmh_kmac_register(); + if (ret) + goto err_kmac_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -221,6 +233,10 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_kmac_unregister(); +err_kmac_register: + cmh_cshake_unregister(); +err_cshake_register: cmh_hmac_unregister(); err_hmac_register: cmh_hash_unregister(); @@ -251,6 +267,8 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_kmac_unregister(); + cmh_cshake_unregister(); cmh_hmac_unregister(); cmh_hash_unregister(); cmh_rh_cleanup(cfg); diff --git a/drivers/crypto/cmh/include/cmh_cshake.h b/drivers/crypto/cmh/include/cmh_cshake.h new file mode 100644 index 00000000000000..9bafe0baf52fa1 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_cshake.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API CSHAKE Driver + * + * Registers cSHAKE-128 and cSHAKE-256 ahash algorithms using + * HC_CMD_CSHAKE with inline customization string S. + */ + +#ifndef CMH_CSHAKE_H +#define CMH_CSHAKE_H + +int cmh_cshake_register(void); +void cmh_cshake_unregister(void); + +#endif /* CMH_CSHAKE_H */ diff --git a/drivers/crypto/cmh/include/cmh_kmac.h b/drivers/crypto/cmh/include/cmh_kmac.h new file mode 100644 index 00000000000000..b3c92d71a0b660 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_kmac.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API KMAC Driver + * + * Registers KMAC-128 and KMAC-256 ahash algorithms using + * HC_CMD_KMAC with inline customization string S. + */ + +#ifndef CMH_KMAC_H +#define CMH_KMAC_H + +int cmh_kmac_register(void); +void cmh_kmac_unregister(void); + +#endif /* CMH_KMAC_H */ From caa47f7aa11d690952f67d1ca70df3af359996c1 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:25 -0700 Subject: [PATCH 09/21] crypto: cmh - add SM3 ahash Register the SM3 ahash algorithm using the CMH SM3 core (core ID 0x05). Supports incremental update/finup/final and export/import. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 3 +- drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/cmh_sm3.c | 651 +++++++++++++++++++++++++++ drivers/crypto/cmh/include/cmh_sm3.h | 27 ++ 4 files changed, 689 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_sm3.c create mode 100644 drivers/crypto/cmh/include/cmh_sm3.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 2bb240b97f31ac..b3018fbcf21184 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -18,7 +18,8 @@ cmh-y := \ cmh_hash.o \ cmh_hmac.o \ cmh_cshake.o \ - cmh_kmac.o + cmh_kmac.o \ + cmh_sm3.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 08d14aae912d14..bc326a9d7626fd 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -33,6 +33,7 @@ #include "cmh_hmac.h" #include "cmh_cshake.h" #include "cmh_kmac.h" +#include "cmh_sm3.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -221,6 +222,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_kmac_register; + /* Register SM3 hash algorithm */ + ret = cmh_sm3_register(); + if (ret) + goto err_sm3_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -233,6 +239,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_sm3_unregister(); +err_sm3_register: cmh_kmac_unregister(); err_kmac_register: cmh_cshake_unregister(); @@ -267,6 +275,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_sm3_unregister(); cmh_kmac_unregister(); cmh_cshake_unregister(); cmh_hmac_unregister(); diff --git a/drivers/crypto/cmh/cmh_sm3.c b/drivers/crypto/cmh/cmh_sm3.c new file mode 100644 index 00000000000000..156f93da70af76 --- /dev/null +++ b/drivers/crypto/cmh/cmh_sm3.c @@ -0,0 +1,651 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM3 Hash Driver (CORE_ID_SM3) + * + * Registers an asynchronous hash (ahash) algorithm for SM3 + * (GB/T 32905-2016) using the CMH SM3 core. This is a standalone + * driver separate from cmh_hash.c (which handles HC-based SHA-2/3/SHAKE) + * because SM3 runs on a different hardware core with its own command + * IDs and context layout. + * + * Incremental HW update model (same pattern as cmh_hash.c): + * + * .init() -> software-only: zero per-request context + * .update() -> buffer data in holdback; when >= block_size bytes: + * SM3_CMD_INIT [+ RESTORE] + UPDATE + SAVE + FLUSH + * -> return -EINPROGRESS (else return 0) + * .final() -> SM3_CMD_INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH + * .finup() -> linearise holdback + new data, then final path + * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot, zero-copy) + * .export() -> software-only: copy checkpoint + holdback to out + * .import() -> software-only: restore checkpoint + holdback from in + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_sm3.h" +#include "cmh_vcq.h" +#include "cmh_txn.h" +#include "cmh_dma.h" + +/* Per-Request State */ + +/* + * Exported SM3 state -- serialised by .export(), deserialised by + * .import(). This is what statesize advertises to the crypto subsystem. + */ +struct cmh_sm3_export_state { + u8 checkpoint[SM3_CONTEXT_SIZE]; /* SM3 context from last SAVE */ + u8 buf[CMH_SM3_BLOCK_SIZE]; /* holdback buffer */ + u32 buf_len; /* valid bytes in buf[] */ + u32 hw_started; /* non-zero if checkpoint valid */ +}; + +#define CMH_SM3_MAX_PAYLOAD 5 /* INIT + RESTORE + UPDATE + FINAL/SAVE + FLUSH */ +#define CMH_SM3_MAX_PACKED (CMH_SM3_MAX_PAYLOAD * 2) + +/* + * Checkpoint embedded inline: the kernel ahash API has no per-request + * destructor, so a heap-allocated checkpoint leaks if a request is + * abandoned without .final(). + */ +struct cmh_sm3_reqctx { + int error; + u32 hw_started; + u32 buf_len; + u32 has_checkpoint; + u8 checkpoint[SM3_CONTEXT_SIZE]; /* SM3 context from last SAVE */ + /* DMA state for current async operation */ + dma_addr_t ckpt_dma; + dma_addr_t save_dma; + dma_addr_t data_dma; + dma_addr_t digest_dma; + u8 *save_buf; + u8 *data_buf; + u32 data_len; + u8 *digest_buf; + u8 buf[CMH_SM3_BLOCK_SIZE]; /* holdback for partial block */ + struct vcq_cmd packed[CMH_SM3_MAX_PACKED]; +}; + +/* VCQ Builders -- SM3 core (CORE_ID_SM3); generic flush from cmh_vcq.h */ + +static void vcq_add_sm3_init(struct vcq_cmd *slot, u32 core_id) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_INIT); + /* SM3 has a single algorithm -- no algo selector field */ +} + +static void vcq_add_sm3_update(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_UPDATE); + slot->hwc.sm3.cmd_update.input = input_phys; + slot->hwc.sm3.cmd_update.inlen = len; +} + +static void vcq_add_sm3_final(struct vcq_cmd *slot, u32 core_id, u64 digest_phys, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_FINAL); + slot->hwc.sm3.cmd_final.digest = digest_phys; + slot->hwc.sm3.cmd_final.outlen = outlen; +} + +static void vcq_add_sm3_save(struct vcq_cmd *slot, u32 core_id, u64 output_phys, u32 outlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_SAVE); + slot->hwc.sm3.cmd_save.output = output_phys; + slot->hwc.sm3.cmd_save.outlen = outlen; +} + +static void vcq_add_sm3_restore(struct vcq_cmd *slot, u32 core_id, u64 input_phys, u32 inlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM3_CMD_RESTORE); + slot->hwc.sm3.cmd_restore.input = input_phys; + slot->hwc.sm3.cmd_restore.inlen = inlen; +} + +/* Request Context Cleanup */ + +static void cmh_sm3_free_reqctx(struct cmh_sm3_reqctx *rctx) +{ + rctx->has_checkpoint = 0; +} + +/* VCQ Packing + Submit */ + +/* ahash Operations */ + +static int cmh_sm3_init(struct ahash_request *req) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + return 0; +} + +/* + * Update completion -- takes ownership of save_buf as new checkpoint. + */ +static void cmh_sm3_update_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->save_dma, SM3_CONTEXT_SIZE, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->data_dma, rctx->data_len, + DMA_TO_DEVICE); + + if (!error) { + memcpy(rctx->checkpoint, rctx->save_buf, SM3_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + kfree(rctx->save_buf); + rctx->save_buf = NULL; + rctx->hw_started = 1; + } else { + kfree(rctx->save_buf); + rctx->save_buf = NULL; + rctx->error = error; + } + + kfree(rctx->data_buf); + rctx->data_buf = NULL; + rctx->data_len = 0; + + cmh_complete(&req->base, error); +} + +static int cmh_sm3_update(struct ahash_request *req) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_SM3_MAX_PAYLOAD]; + struct core_dispatch d; + u32 total_avail, full_len, tail_len, from_src; + u32 idx; + int ret; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (!req->nbytes) + return 0; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + total_avail = rctx->buf_len + req->nbytes; + + if (total_avail < CMH_SM3_BLOCK_SIZE) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->buf + rctx->buf_len, + req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(rctx->buf + rctx->buf_len, + req->src, 0, + req->nbytes, 0); + rctx->buf_len = total_avail; + return 0; + } + + full_len = total_avail - total_avail % CMH_SM3_BLOCK_SIZE; + tail_len = total_avail - full_len; + from_src = full_len - rctx->buf_len; + + rctx->data_buf = kmalloc(full_len, gfp); + if (!rctx->data_buf) + return -ENOMEM; + + if (rctx->buf_len > 0) + memcpy(rctx->data_buf, rctx->buf, rctx->buf_len); + + if (from_src > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->data_buf + rctx->buf_len, + req->svirt, from_src); + else + scatterwalk_map_and_copy(rctx->data_buf + rctx->buf_len, + req->src, 0, + from_src, 0); + } + + if (tail_len > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(rctx->buf, req->svirt + from_src, + tail_len); + else + scatterwalk_map_and_copy(rctx->buf, req->src, + from_src, tail_len, + 0); + } + rctx->buf_len = tail_len; + rctx->data_len = full_len; + + rctx->save_buf = kzalloc(SM3_CONTEXT_SIZE, gfp); + if (!rctx->save_buf) { + ret = -ENOMEM; + goto err_free; + } + + rctx->data_dma = cmh_dma_map_single(rctx->data_buf, full_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->data_dma)) { + ret = -ENOMEM; + goto err_free; + } + + rctx->save_dma = cmh_dma_map_single(rctx->save_buf, SM3_CONTEXT_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->save_dma)) { + ret = -ENOMEM; + goto err_unmap_data; + } + + rctx->ckpt_dma = DMA_MAPPING_ERROR; + if (rctx->has_checkpoint) { + rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + SM3_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->ckpt_dma)) { + ret = -ENOMEM; + goto err_unmap_save; + } + } + + d = cmh_core_select_instance(CMH_CORE_SM3); + idx = 0; + + vcq_add_sm3_init(&cmds[idx++], d.core_id); + + if (rctx->has_checkpoint) + vcq_add_sm3_restore(&cmds[idx++], d.core_id, + (u64)rctx->ckpt_dma, SM3_CONTEXT_SIZE); + + vcq_add_sm3_update(&cmds[idx++], d.core_id, + (u64)rctx->data_dma, full_len); + + vcq_add_sm3_save(&cmds[idx++], d.core_id, + (u64)rctx->save_dma, SM3_CONTEXT_SIZE); + + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM3_MAX_PACKED, + d.mbx_idx, + cmh_sm3_update_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto err_unmap_ckpt; + + return -EINPROGRESS; + +err_unmap_ckpt: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE, + DMA_TO_DEVICE); +err_unmap_save: + cmh_dma_unmap_single(rctx->save_dma, SM3_CONTEXT_SIZE, + DMA_FROM_DEVICE); +err_unmap_data: + cmh_dma_unmap_single(rctx->data_dma, full_len, DMA_TO_DEVICE); +err_free: + kfree(rctx->save_buf); + rctx->save_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + rctx->data_len = 0; + return ret; +} + +static void cmh_sm3_final_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (rctx->data_buf) + cmh_dma_unmap_single(rctx->data_dma, rctx->data_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->digest_dma, CMH_SM3_DIGEST_SIZE, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->digest_buf, CMH_SM3_DIGEST_SIZE); + + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; + kfree(rctx->data_buf); + rctx->data_buf = NULL; + cmh_sm3_free_reqctx(rctx); + cmh_complete(&req->base, error); +} + +static int cmh_sm3_submit_final(struct ahash_request *req, + u8 *data_buf, u32 data_len) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_SM3_MAX_PAYLOAD]; + struct core_dispatch d; + u32 idx; + int ret; + gfp_t gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + rctx->data_buf = data_buf; + rctx->data_len = data_len; + + rctx->digest_buf = kzalloc(CMH_SM3_DIGEST_SIZE, gfp); + if (!rctx->digest_buf) { + ret = -ENOMEM; + goto err_free_data; + } + + rctx->digest_dma = cmh_dma_map_single(rctx->digest_buf, + CMH_SM3_DIGEST_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->digest_dma)) { + ret = -ENOMEM; + goto err_free_digest; + } + + rctx->data_dma = DMA_MAPPING_ERROR; + if (data_buf && data_len > 0) { + rctx->data_dma = cmh_dma_map_single(data_buf, data_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->data_dma)) { + ret = -ENOMEM; + goto err_unmap_digest; + } + } + + rctx->ckpt_dma = DMA_MAPPING_ERROR; + if (rctx->has_checkpoint) { + rctx->ckpt_dma = cmh_dma_map_single(rctx->checkpoint, + SM3_CONTEXT_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->ckpt_dma)) { + ret = -ENOMEM; + goto err_unmap_data; + } + } + + d = cmh_core_select_instance(CMH_CORE_SM3); + idx = 0; + + vcq_add_sm3_init(&cmds[idx++], d.core_id); + + if (rctx->has_checkpoint) + vcq_add_sm3_restore(&cmds[idx++], d.core_id, + (u64)rctx->ckpt_dma, SM3_CONTEXT_SIZE); + + if (data_buf && data_len > 0) + vcq_add_sm3_update(&cmds[idx++], d.core_id, + (u64)rctx->data_dma, data_len); + + vcq_add_sm3_final(&cmds[idx++], d.core_id, + (u64)rctx->digest_dma, CMH_SM3_DIGEST_SIZE); + + vcq_add_flush(&cmds[idx++], d.core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM3_MAX_PACKED, + d.mbx_idx, + cmh_sm3_final_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto err_unmap_ckpt; + + return -EINPROGRESS; + +err_unmap_ckpt: + if (rctx->has_checkpoint) + cmh_dma_unmap_single(rctx->ckpt_dma, SM3_CONTEXT_SIZE, + DMA_TO_DEVICE); +err_unmap_data: + if (data_buf && data_len > 0) + cmh_dma_unmap_single(rctx->data_dma, data_len, + DMA_TO_DEVICE); +err_unmap_digest: + cmh_dma_unmap_single(rctx->digest_dma, CMH_SM3_DIGEST_SIZE, + DMA_FROM_DEVICE); +err_free_digest: + kfree(rctx->digest_buf); + rctx->digest_buf = NULL; +err_free_data: + kfree(data_buf); + rctx->data_buf = NULL; + cmh_sm3_free_reqctx(rctx); + return ret; +} + +static int cmh_sm3_final(struct ahash_request *req) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + u8 *data_buf = NULL; + u32 data_len = 0; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + if (rctx->buf_len > 0) { + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + data_buf = kmalloc(rctx->buf_len, gfp); + if (!data_buf) + return -ENOMEM; + memcpy(data_buf, rctx->buf, rctx->buf_len); + data_len = rctx->buf_len; + rctx->buf_len = 0; + } + + return cmh_sm3_submit_final(req, data_buf, data_len); +} + +static int cmh_sm3_finup(struct ahash_request *req); + +/* + * One-shot digest -- delegates to init + finup so that all data is + * linearised and mapped through cmh_dma_map_single(), which is the + * only DMA mapping path aware of all supported DMA backends. + */ +static int cmh_sm3_digest(struct ahash_request *req) +{ + int ret; + + ret = cmh_sm3_init(req); + if (ret) + return ret; + return cmh_sm3_finup(req); +} + +static int cmh_sm3_finup(struct ahash_request *req) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + u32 data_len; + u8 *data_buf; + gfp_t gfp; + + if (rctx->error) + return rctx->error; + + data_len = rctx->buf_len + req->nbytes; + + if (data_len == 0) + return cmh_sm3_submit_final(req, NULL, 0); + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + data_buf = kmalloc(data_len, gfp); + if (!data_buf) + return -ENOMEM; + + if (rctx->buf_len > 0) + memcpy(data_buf, rctx->buf, rctx->buf_len); + + if (req->nbytes > 0) { + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(data_buf + rctx->buf_len, + req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(data_buf + rctx->buf_len, + req->src, 0, + req->nbytes, 0); + } + + rctx->buf_len = 0; + return cmh_sm3_submit_final(req, data_buf, data_len); +} + +static int cmh_sm3_export(struct ahash_request *req, void *out) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + struct cmh_sm3_export_state *state = out; + + if (rctx->hw_started && rctx->has_checkpoint) + memcpy(state->checkpoint, rctx->checkpoint, SM3_CONTEXT_SIZE); + else + memset(state->checkpoint, 0, SM3_CONTEXT_SIZE); + + if (rctx->buf_len > 0) + memcpy(state->buf, rctx->buf, rctx->buf_len); + + state->buf_len = rctx->buf_len; + state->hw_started = rctx->hw_started; + + return 0; +} + +static int cmh_sm3_import(struct ahash_request *req, const void *in) +{ + struct cmh_sm3_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_sm3_export_state *state = in; + + memset(rctx, 0, sizeof(*rctx)); + + if (state->buf_len > CMH_SM3_BLOCK_SIZE) + return -EINVAL; + + rctx->hw_started = state->hw_started; + rctx->buf_len = state->buf_len; + memcpy(rctx->buf, state->buf, state->buf_len); + + if (state->hw_started) { + memcpy(rctx->checkpoint, state->checkpoint, SM3_CONTEXT_SIZE); + rctx->has_checkpoint = 1; + } + + return 0; +} + +/* Transform init (cra_init) */ + +static int cmh_sm3_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cmh_sm3_reqctx)); + return 0; +} + +/* Registration */ + +static struct ahash_alg cmh_sm3_ahash_alg = { + .init = cmh_sm3_init, + .update = cmh_sm3_update, + .final = cmh_sm3_final, + .finup = cmh_sm3_finup, + .digest = cmh_sm3_digest, + .export = cmh_sm3_export, + .import = cmh_sm3_import, + + .halg = { + .digestsize = CMH_SM3_DIGEST_SIZE, + .statesize = sizeof(struct cmh_sm3_export_state), + .base = { + .cra_name = "sm3", + .cra_driver_name = "cri-cmh-sm3", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT, + .cra_blocksize = CMH_SM3_BLOCK_SIZE, + .cra_ctxsize = 0, + .cra_init = cmh_sm3_cra_init, + .cra_module = THIS_MODULE, + }, + }, +}; + +/** + * cmh_sm3_register() - Register SM3 hash algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_sm3_register(void) +{ + int ret; + + ret = crypto_register_ahash(&cmh_sm3_ahash_alg); + if (ret) { + dev_err(cmh_dev(), "sm3: failed to register cmh-sm3 (rc=%d)\n", + ret); + return ret; + } + + dev_info(cmh_dev(), "sm3: registered cri-cmh-sm3 (priority 300)\n"); + dev_info(cmh_dev(), "sm3: 1 algorithm(s) registered\n"); + return 0; +} + +/** + * cmh_sm3_unregister() - Unregister SM3 hash algorithm from the crypto framework + */ +void cmh_sm3_unregister(void) +{ + crypto_unregister_ahash(&cmh_sm3_ahash_alg); + dev_info(cmh_dev(), "sm3: unregistered cri-cmh-sm3\n"); + dev_info(cmh_dev(), "sm3: cleaned up\n"); +} diff --git a/drivers/crypto/cmh/include/cmh_sm3.h b/drivers/crypto/cmh/include/cmh_sm3.h new file mode 100644 index 00000000000000..2f73537f9c87dc --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sm3.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM3 Hash Driver + * + * Registers an ahash algorithm for SM3 (GB/T 32905-2016) with the + * Linux crypto subsystem using the CMH SM3 core (CORE_ID_SM3). + * Uses the same incremental HW update model as cmh_hash.c: + * + * .init() -> software-only: zero per-request context + * .update() -> holdback partial blocks; submit full blocks via + * SM3_CMD_INIT [+ RESTORE] + UPDATE + SAVE + FLUSH + * .final() -> SM3_CMD_INIT [+ RESTORE] [+ UPDATE] + FINAL + FLUSH + * .digest() -> INIT + UPDATE + FINAL + FLUSH (single-shot) + * .export() -> software-only: copy checkpoint + holdback + * .import() -> software-only: restore checkpoint + holdback + */ + +#ifndef CMH_SM3_H +#define CMH_SM3_H + +#include "cmh_config.h" + +int cmh_sm3_register(void); +void cmh_sm3_unregister(void); + +#endif /* CMH_SM3_H */ From 88ae3e1a3ba4d817367308218bd9fedfeccdb6e7 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:26 -0700 Subject: [PATCH 10/21] crypto: cmh - add AES skcipher/aead/cmac Register AES algorithms using the CMH AES core (core ID 0x03): - skcipher: AES-ECB, AES-CBC, AES-CTR, AES-XTS, AES-CFB - aead: AES-GCM, AES-CCM - ahash: AES-CMAC Supports 128, 192, and 256-bit keys. AEAD algorithms handle associated data, payload, and authentication tag with correct encrypt/decrypt separation. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 5 +- drivers/crypto/cmh/cmh_aes.c | 736 ++++++++++++++++++++ drivers/crypto/cmh/cmh_aes_aead.c | 987 +++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_aes_cmac.c | 537 +++++++++++++++ drivers/crypto/cmh/cmh_main.c | 25 + drivers/crypto/cmh/include/cmh_aes.h | 24 + 6 files changed, 2313 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_aes.c create mode 100644 drivers/crypto/cmh/cmh_aes_aead.c create mode 100644 drivers/crypto/cmh/cmh_aes_cmac.c create mode 100644 drivers/crypto/cmh/include/cmh_aes.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index b3018fbcf21184..ced8d1748e6ce0 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -19,7 +19,10 @@ cmh-y := \ cmh_hmac.o \ cmh_cshake.o \ cmh_kmac.o \ - cmh_sm3.o + cmh_sm3.o \ + cmh_aes.o \ + cmh_aes_aead.o \ + cmh_aes_cmac.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_aes.c b/drivers/crypto/cmh/cmh_aes.c new file mode 100644 index 00000000000000..b36295763e338d --- /dev/null +++ b/drivers/crypto/cmh/cmh_aes.c @@ -0,0 +1,736 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API AES (skcipher) Driver + * + * Registers skcipher algorithms with the Linux crypto subsystem: + * ecb(aes), cbc(aes), ctr(aes), cfb(aes), xts(aes) + * + * Uses the CMH AES Core via VCQ commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + [AES_CMD_UPDATE] + AES_CMD_FINAL + * + VCQ_CMD_FLUSH + * + * The AES core requires bidirectional DMA -- both input and output + * buffers are mapped and passed in a single AES_CMD_FINAL command. + * + * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into + * the same VCQ as AES commands (see cmh_key.h for details). + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_aes.h" +#include "cmh_vcq.h" +#include "cmh_aes_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* Algorithm Table */ + +struct cmh_aes_alg_info { + u32 aes_mode; /* AES_MODE_* */ + u32 ivsize; /* bytes (0 for ECB) */ + u32 min_keysize; /* minimum key bytes */ + u32 max_keysize; /* maximum key bytes */ + const char *alg_name; /* Linux crypto name: "ecb(aes)" */ + const char *drv_name; /* driver name: "cri-cmh-ecb-aes" */ +}; + +static const struct cmh_aes_alg_info aes_algs[] = { + { AES_MODE_ECB, 0, AES_KEYSIZE_128, AES_KEYSIZE_256, + "ecb(aes)", "cri-cmh-ecb-aes" }, + { AES_MODE_CBC, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256, + "cbc(aes)", "cri-cmh-cbc-aes" }, + { AES_MODE_CTR, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256, + "ctr(aes)", "cri-cmh-ctr-aes" }, + { AES_MODE_CFB, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256, + "cfb(aes)", "cri-cmh-cfb-aes" }, + { AES_MODE_XTS, CMH_AES_IV_SIZE, 2 * AES_KEYSIZE_128, 2 * AES_KEYSIZE_256, + "xts(aes)", "cri-cmh-xts-aes" }, +}; + +/* Per-transform context (allocated by crypto framework) */ + +struct cmh_aes_tfm_ctx { + struct cmh_key_ctx key; +}; + +/* Per-request context (lives in skcipher_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + [AES_CMD_UPDATE] + AES_CMD_FINAL + * + VCQ_CMD_FLUSH = 5 + * UPDATE is used for XTS data > 2 blocks (see cmh_aes_crypt). + */ +#define CMH_AES_MAX_PAYLOAD 5 +#define CMH_AES_MAX_PACKED (CMH_AES_MAX_PAYLOAD * 2) + +struct cmh_aes_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t iv2_dma; + dma_addr_t key_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *iv2_buf; + u32 cryptlen; + u32 ivsize; + u32 keylen; + u32 aes_mode; + u32 aes_op; + /* CTR counter-wrap split state */ + u32 ctr_chunk1_len; + u32 core_id; + s32 target_mbx; + u64 key_ref; + struct vcq_cmd packed[CMH_AES_MAX_PACKED]; +}; + +/* VCQ Builders -- AES-specific */ + +static void vcq_add_aes_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u64 iv_dma, + u32 keylen, u32 ivlen, u32 mode, u32 op, + u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = iv_dma; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = ivlen; + slot->hwc.aes.cmd_init.mode = mode; + slot->hwc.aes.cmd_init.op = op; + slot->hwc.aes.cmd_init.aadlen = 0; + slot->hwc.aes.cmd_init.iolen = iolen; + slot->hwc.aes.cmd_init.taglen = 0; +} + +static void vcq_add_aes_update(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_UPDATE); + slot->hwc.aes.cmd_update.input = input_dma; + slot->hwc.aes.cmd_update.output = output_dma; + slot->hwc.aes.cmd_update.iolen = iolen; +} + +static void vcq_add_aes_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_FINAL); + slot->hwc.aes.cmd_final.input = input_dma; + slot->hwc.aes.cmd_final.output = output_dma; + slot->hwc.aes.cmd_final.iolen = iolen; + slot->hwc.aes.cmd_final.tag = 0; + slot->hwc.aes.cmd_final.taglen = 0; +} + +/* + * We wrap each skcipher_alg with its info pointer in a compound struct, + * then use container_of() in cmh_aes_get_info() to recover it. + * This is the same pattern used by hash, hmac, cshake, kmac. + */ +struct cmh_aes_alg_drv { + struct skcipher_alg alg; + const struct cmh_aes_alg_info *info; +}; + +static bool aes_is_stream_mode(u32 mode) +{ + return mode == AES_MODE_CTR || mode == AES_MODE_CFB; +} + +/* + * Update req->iv after a successful encrypt/decrypt. + * + * The Linux skcipher API contract requires that req->iv is updated to + * reflect the state needed to continue processing in a chained call: + * CBC encrypt: IV <- last ciphertext block + * CBC decrypt: IV <- last ciphertext block of the *input* + * CTR: IV <- counter incremented by ceil(cryptlen / blocksize) + * CFB: IV <- last ciphertext block + */ +static void cmh_aes_update_iv(struct skcipher_request *req, u32 mode, + u32 op, const u8 *in_buf, const u8 *out_buf) +{ + u32 bs = CMH_AES_BLOCK_SIZE; + u32 nblocks; + + switch (mode) { + case AES_MODE_CBC: + if (op == AES_OP_ENCRYPT) + memcpy(req->iv, out_buf + req->cryptlen - bs, bs); + else + memcpy(req->iv, in_buf + req->cryptlen - bs, bs); + break; + case AES_MODE_CTR: + /* + * Arithmetic big-endian 128-bit counter increment. + * Process from the least-significant byte (index 15) + * upward, carrying as needed. + */ + nblocks = DIV_ROUND_UP(req->cryptlen, bs); + { + u8 *iv = req->iv; + int i; + + for (i = bs - 1; i >= 0 && nblocks; i--) { + u32 sum = (u32)iv[i] + (nblocks & 0xff); + + iv[i] = (u8)sum; + nblocks = (nblocks >> 8) + (sum >> 8); + } + } + break; + case AES_MODE_CFB: + /* + * CFB-128 chains on the last ciphertext block. On encrypt, + * that is out_buf; on decrypt, it is in_buf. + * + * For sub-block requests (cryptlen < 16), there is no + * complete ciphertext block to chain, so the IV is left + * unchanged -- CFB-128 has no defined chaining semantic + * for partial blocks (shift-register CFB-n is a different + * mode). Without this guard the pointer arithmetic + * underflows and reads before the buffer. + */ + if (req->cryptlen >= bs) { + if (op == AES_OP_ENCRYPT) + memcpy(req->iv, out_buf + req->cryptlen - bs, + bs); + else + memcpy(req->iv, in_buf + req->cryptlen - bs, + bs); + } + break; + default: + break; + } +} + +/* skcipher Operations */ + +static const struct cmh_aes_alg_info * +cmh_aes_get_info(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + return container_of(alg, struct cmh_aes_alg_drv, alg)->info; +} + +static int cmh_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + const struct cmh_aes_alg_info *info = cmh_aes_get_info(tfm); + + if (info->aes_mode == AES_MODE_XTS) { + int err; + + /* XTS: double key (32, 48, or 64 bytes) */ + if (keylen != 2 * AES_KEYSIZE_128 && + keylen != 2 * AES_KEYSIZE_192 && + keylen != 2 * AES_KEYSIZE_256) + return -EINVAL; + err = xts_verify_key(tfm, key, keylen); + if (err) + return err; + } else { + /* Standard: 16, 24, or 32 bytes */ + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + } + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES); +} + +static int cmh_aes_init_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_aes_reqctx)); + return 0; +} + +static void cmh_aes_exit_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +#define CMH_AES_MAX_CRYPTLEN SZ_32M + +/* DMA unmap helper */ +static void cmh_aes_unmap_dma(struct cmh_aes_reqctx *rctx) +{ + if (rctx->iv2_buf) + cmh_dma_unmap_single(rctx->iv2_dma, rctx->ivsize, + DMA_TO_DEVICE); + if (rctx->ivsize > 0) + cmh_dma_unmap_single(rctx->iv_dma, rctx->ivsize, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE); +} + +static void cmh_aes_free_bufs(struct cmh_aes_reqctx *rctx) +{ + kfree(rctx->iv2_buf); + rctx->iv2_buf = NULL; + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; +} + +/* + * Submit the second CTR chunk after the first completes. + * Called from cmh_aes_complete when ctr_chunk1_len > 0. + */ +static void cmh_aes_complete(void *data, int error); + +static int cmh_aes_ctr_submit_chunk2(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_MAX_PAYLOAD]; + u32 chunk1 = rctx->ctr_chunk1_len; + u32 chunk2 = rctx->cryptlen - chunk1; + u64 key_ref; + u32 keylen; + u32 idx = 0; + + /* Clear split flag so next completion is final */ + rctx->ctr_chunk1_len = 0; + + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + + vcq_add_aes_init(&cmds[idx++], rctx->core_id, key_ref, + (u64)rctx->iv2_dma, keylen, rctx->ivsize, + rctx->aes_mode, rctx->aes_op, 0); + vcq_add_aes_final(&cmds[idx++], rctx->core_id, + (u64)(rctx->in_dma + chunk1), + (u64)(rctx->out_dma + chunk1), chunk2); + vcq_add_flush(&cmds[idx++], rctx->core_id); + + return cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_MAX_PACKED, + rctx->target_mbx, + cmh_aes_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); +} + +/* + * Async completion callback -- fires from RH threaded IRQ context. + * + * Unmaps DMA buffers, copies output to req->dst scatterlist, + * updates the IV state, frees temporaries, and completes the request. + * + * For CTR counter-wrap splits, the first chunk completion chains + * into a second VCQ submission rather than finalizing immediately. + */ +static void cmh_aes_complete(void *data, int error) +{ + struct skcipher_request *req = data; + struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* + * CTR counter-wrap: first chunk completed, submit second. + * DMA mappings remain valid (they cover the full buffer). + * + * Recursion depth bounded: chunk2 clears ctr_chunk1_len before + * submission, so the second cmh_aes_complete invocation sees 0 + * and finalizes (max depth = 2). + */ + if (rctx->ctr_chunk1_len && !error) { + int ret; + + ret = cmh_aes_ctr_submit_chunk2(req); + + if (!ret || ret == -EBUSY) + return; + /* Submission failed; clean up below */ + error = ret; + } + + cmh_aes_unmap_dma(rctx); + + if (!error) { + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + 0, rctx->cryptlen, 1); + cmh_aes_update_iv(req, rctx->aes_mode, rctx->aes_op, + rctx->in_buf, rctx->out_buf); + } + + cmh_aes_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * Core encrypt/decrypt -- builds a VCQ transaction and submits async. + * + * Returns -EINPROGRESS on successful submission (completion callback + * will fire later). Returns 0 for trivial cases (zero-length). + * Returns negative errno on pre-submission errors. + */ +static int cmh_aes_crypt(struct skcipher_request *req, u32 aes_op) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + const struct cmh_aes_alg_info *info = cmh_aes_get_info(tfm); + struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + if (!req->cryptlen) + return 0; + + if (req->cryptlen > CMH_AES_MAX_CRYPTLEN) + return -EINVAL; + + switch (info->aes_mode) { + case AES_MODE_CTR: + case AES_MODE_CFB: + break; + case AES_MODE_XTS: + if (req->cryptlen < CMH_AES_BLOCK_SIZE) + return -EINVAL; + break; + default: + if (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1)) + return -EINVAL; + break; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Initialise reqctx */ + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = req->cryptlen; + rctx->ivsize = info->ivsize; + rctx->aes_mode = info->aes_mode; + rctx->aes_op = aes_op; + rctx->iv2_buf = NULL; + + /* Linearise input from scatterlist */ + rctx->in_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->in_buf) + return -ENOMEM; + + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0); + + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + + /* Allocate and map output buffer */ + rctx->out_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + + /* Map IV if required */ + if (info->ivsize > 0) { + rctx->iv_buf = kmemdup(req->iv, info->ivsize, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, info->ivsize, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* + * iolen in INIT: XTS needs total length upfront for tweak + * computation; all other modes use 0 (streaming). + */ + vcq_add_aes_init(&cmds[idx++], core_id, key_ref, (u64)rctx->iv_dma, + keylen, info->ivsize, info->aes_mode, aes_op, + info->aes_mode == AES_MODE_XTS ? + req->cryptlen : 0); + + if (info->aes_mode == AES_MODE_XTS && + req->cryptlen > 2 * CMH_AES_BLOCK_SIZE) { + u32 final_len, update_len; + + if (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1)) + final_len = CMH_AES_BLOCK_SIZE + + (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1)); + else + final_len = 2 * CMH_AES_BLOCK_SIZE; + + update_len = req->cryptlen - final_len; + + vcq_add_aes_update(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, update_len); + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)(rctx->in_dma + update_len), + (u64)(rctx->out_dma + update_len), + final_len); + } else if (info->aes_mode == AES_MODE_CTR) { + /* + * CTR counter-wrap workaround: + * The AES-SCA hardware uses a 64-bit block counter. + * If the lower 64 bits of the IV would wrap during + * this operation, split into two separate VCQ + * transactions -- the completion callback for the + * first chunk submits the second. + */ + u64 lower64 = get_unaligned_be64(rctx->iv_buf + 8); + u32 nblocks = DIV_ROUND_UP(req->cryptlen, + CMH_AES_BLOCK_SIZE); + u64 bwrap = lower64 ? (~lower64 + 1ULL) : U64_MAX; + + if (nblocks > bwrap) { + u32 chunk1 = (u32)bwrap * CMH_AES_BLOCK_SIZE; + u64 upper64; + + /* Prepare second IV for chained submission */ + rctx->iv2_buf = kmalloc(info->ivsize, gfp); + if (!rctx->iv2_buf) { + ret = -ENOMEM; + goto out_unmap_iv; + } + upper64 = get_unaligned_be64(rctx->iv_buf); + put_unaligned_be64(upper64 + 1, rctx->iv2_buf); + put_unaligned_be64(0, rctx->iv2_buf + 8); + + rctx->iv2_dma = + cmh_dma_map_single(rctx->iv2_buf, + info->ivsize, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv2_dma)) { + ret = -ENOMEM; + goto out_free_iv2; + } + + /* Store state for the chained second submission */ + rctx->ctr_chunk1_len = chunk1; + rctx->core_id = core_id; + rctx->target_mbx = target_mbx; + rctx->key_ref = key_ref; + + /* First transaction: only chunk1 */ + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, chunk1); + } else { + /* No wrap: single FINAL with all data */ + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, + req->cryptlen); + } + } else { + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, req->cryptlen); + } + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_MAX_PACKED, target_mbx, + cmh_aes_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->iv2_buf) { + cmh_dma_unmap_single(rctx->iv2_dma, info->ivsize, + DMA_TO_DEVICE); + } +out_free_iv2: + kfree(rctx->iv2_buf); +out_unmap_iv: + if (info->ivsize > 0) + cmh_dma_unmap_single(rctx->iv_dma, info->ivsize, + DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_out: + cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); + return ret; +} + +static int cmh_aes_encrypt(struct skcipher_request *req) +{ + return cmh_aes_crypt(req, AES_OP_ENCRYPT); +} + +static int cmh_aes_decrypt(struct skcipher_request *req) +{ + return cmh_aes_crypt(req, AES_OP_DECRYPT); +} + +/* Registration */ + +static struct cmh_aes_alg_drv aes_drv_algs[ARRAY_SIZE(aes_algs)]; + +/** + * cmh_aes_register() - Register AES-CBC/CTR/ECB/XTS skcipher algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_aes_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + const struct cmh_aes_alg_info *info = &aes_algs[i]; + struct cmh_aes_alg_drv *drv = &aes_drv_algs[i]; + struct skcipher_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->setkey = cmh_aes_setkey; + alg->encrypt = cmh_aes_encrypt; + alg->decrypt = cmh_aes_decrypt; + alg->init = cmh_aes_init_tfm; + alg->exit = cmh_aes_exit_tfm; + alg->min_keysize = info->min_keysize; + alg->max_keysize = info->max_keysize; + alg->ivsize = info->ivsize; + + strscpy(alg->base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + alg->base.cra_blocksize = aes_is_stream_mode(info->aes_mode) + ? 1 : CMH_AES_BLOCK_SIZE; + alg->base.cra_ctxsize = sizeof(struct cmh_aes_tfm_ctx); + alg->base.cra_module = THIS_MODULE; + + ret = crypto_register_skcipher(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_aes: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_aes: registered %s\n", info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_skcipher(&aes_drv_algs[i].alg); + return ret; +} + +/** + * cmh_aes_unregister() - Unregister AES skcipher algorithms from the crypto framework + */ +void cmh_aes_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + crypto_unregister_skcipher(&aes_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_aes: unregistered %s\n", aes_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/cmh_aes_aead.c b/drivers/crypto/cmh/cmh_aes_aead.c new file mode 100644 index 00000000000000..0b59c5f7d47407 --- /dev/null +++ b/drivers/crypto/cmh/cmh_aes_aead.c @@ -0,0 +1,987 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API AES AEAD Driver (GCM/CCM) + * + * Registers AEAD algorithms with the Linux crypto subsystem: + * gcm(aes), ccm(aes) + * + * GCM: AES_CMD_INIT(mode=GCM) + [AAD_FINAL] + AES_CMD_FINAL + FLUSH + * - Standard 12-byte IV (nonce), 16-byte tag + * - AES_CMD_INIT carries aadlen/iolen/taglen + * - AES_CMD_FINAL carries tag DMA for encrypt (produce) / decrypt (verify) + * + * CCM: AES_CMD_CCM_INIT + [AAD_FINAL] + AES_CMD_FINAL + FLUSH + * - Variable nonce (7--13 bytes), variable tag (4--16 bytes) + * - Uses AES_CMD_CCM_INIT (0x0A) with aes_cmd_init struct + * - Nonce passed via IV field, taglen in init + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_aes.h" +#include "cmh_vcq.h" +#include "cmh_aes_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * GCM IV contract: + * + * The AES core requires exactly 16 bytes loaded into its IV register. + * For standard 96-bit nonce GCM, the driver passes: + * + * IV[0..11] = user-supplied 12-byte nonce + * IV[12..15] = 0x00000000 + * + * The hardware internally sets the last 32 bits to the big-endian + * counter value 1 (forming J0 = nonce || 0x00000001) before + * processing AAD. The driver must NOT pre-set the counter. + * + * If the IV format is incorrect, GCM authentication will fail + * (encrypt produces wrong ciphertext/tag, decrypt rejects). + */ +#define AES_GCM_IV_SIZE 12U /* GCM nonce size (standard) */ +#define AES_GCM_HW_IV_SIZE 16U /* HW requires 16-byte IV buffer */ +#define AES_GCM_TAG_SIZE 16U + +/* CCM: callers pass a 16-byte IV in RFC 3610 format: + * iv[0] = L-1, iv[1..14-iv[0]] = nonce, rest = counter (zeroed). + * Nonce length = 14 - iv[0], range 7..13. + */ +#define AES_CCM_IV_SIZE 16U + +enum cmh_aes_aead_type { + CMH_AES_AEAD_GCM, + CMH_AES_AEAD_CCM, +}; + +struct cmh_aes_aead_info { + enum cmh_aes_aead_type type; + u32 aes_mode; /* AES_MODE_GCM or AES_MODE_CCM */ + u32 ivsize; + u32 maxauthsize; + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_aes_aead_info aes_aead_algs[] = { + { CMH_AES_AEAD_GCM, AES_MODE_GCM, AES_GCM_IV_SIZE, + AES_GCM_TAG_SIZE, "gcm(aes)", "cri-cmh-gcm-aes" }, + { CMH_AES_AEAD_CCM, AES_MODE_CCM, AES_CCM_IV_SIZE, + AES_GCM_TAG_SIZE, "ccm(aes)", "cri-cmh-ccm-aes" }, +}; + +struct cmh_aes_aead_tfm_ctx { + struct cmh_key_ctx key; + u32 authsize; /* tag length set by setauthsize */ + struct crypto_cipher *sw_cipher; /* CCM empty-input fallback */ + struct crypto_aead *fallback; /* CCM authsize=10 fallback */ +}; + +/* Per-request context (lives in aead_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + AAD_FINAL + AES_CMD_FINAL + FLUSH = 5 + */ +#define CMH_AES_AEAD_MAX_PAYLOAD 5 +#define CMH_AES_AEAD_MAX_PACKED (CMH_AES_AEAD_MAX_PAYLOAD * 2) + +struct cmh_aes_aead_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + dma_addr_t aad_dma; + dma_addr_t tag_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *aad_buf; + u8 *tag_buf; + u32 cryptlen; + u32 assoclen; + u32 authsize; + u32 iv_map_len; + u32 keylen; + bool encrypting; + bool empty_gcm_fallback; + struct vcq_cmd packed[CMH_AES_AEAD_MAX_PACKED]; +}; + +struct cmh_aes_aead_drv { + struct aead_alg alg; + const struct cmh_aes_aead_info *info; +}; + +static const struct cmh_aes_aead_info * +cmh_aes_aead_get_info(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + + return container_of(alg, struct cmh_aes_aead_drv, alg)->info; +} + +/* VCQ Builders -- AEAD-specific */ + +static void vcq_add_aes_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 iv_dma, u32 keylen, u32 ivlen, + u32 mode, u32 op, u32 aadlen, u32 iolen, + u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = iv_dma; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = ivlen; + slot->hwc.aes.cmd_init.mode = mode; + slot->hwc.aes.cmd_init.op = op; + slot->hwc.aes.cmd_init.aadlen = aadlen; + slot->hwc.aes.cmd_init.iolen = iolen; + slot->hwc.aes.cmd_init.taglen = taglen; +} + +static void vcq_add_aes_ccm_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 nonce_dma, u32 keylen, u32 noncelen, + u32 op, u32 aadlen, u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_CCM_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = nonce_dma; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = noncelen; + slot->hwc.aes.cmd_init.mode = AES_MODE_CCM; + slot->hwc.aes.cmd_init.op = op; + slot->hwc.aes.cmd_init.aadlen = aadlen; + slot->hwc.aes.cmd_init.iolen = iolen; + slot->hwc.aes.cmd_init.taglen = taglen; +} + +static void vcq_add_aes_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma, + u32 aadlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_AAD_FINAL); + slot->hwc.aes.cmd_aad_final.data = aad_dma; + slot->hwc.aes.cmd_aad_final.datalen = aadlen; +} + +static void vcq_add_aes_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_FINAL); + slot->hwc.aes.cmd_final.input = input_dma; + slot->hwc.aes.cmd_final.output = output_dma; + slot->hwc.aes.cmd_final.tag = tag_dma; + slot->hwc.aes.cmd_final.iolen = iolen; + slot->hwc.aes.cmd_final.taglen = taglen; +} + +/* setkey */ +static int cmh_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + int ret; + + if (keylen != 16 && keylen != 24 && keylen != 32) + return -EINVAL; + + /* Keep SW fallback ciphers in sync for CCM edge cases */ + if (tctx->sw_cipher) { + ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen); + if (ret) + return ret; + } + if (tctx->fallback) { + ret = crypto_aead_setkey(tctx->fallback, key, keylen); + if (ret) + return ret; + } + + ret = cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES); + + return ret; +} + +static int cmh_aes_aead_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm); + int ret; + + if (info->type == CMH_AES_AEAD_GCM) { + /* GCM: accept 4, 8, 12, 13, 14, 15, 16 per NIST SP 800-38D */ + if (authsize < 4 || authsize > 16 || + (authsize > 4 && authsize < 8) || + (authsize > 8 && authsize < 12)) + return -EINVAL; + } else { + /* CCM: accept all RFC 3610 values {4,6,8,10,12,14,16} */ + if (authsize < 4 || authsize > 16 || (authsize & 1)) + return -EINVAL; + /* Forward to SW fallback for authsize=10 (HW unsupported) */ + if (tctx->fallback) { + ret = crypto_aead_setauthsize(tctx->fallback, + authsize); + if (ret) + return ret; + } + } + + tctx->authsize = authsize; + return 0; +} + +static int cmh_aes_aead_init_tfm(struct crypto_aead *tfm) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm); + + memset(tctx, 0, sizeof(*tctx)); + tctx->authsize = info->maxauthsize; + + if (info->type == CMH_AES_AEAD_CCM) { + struct crypto_aead *fb; + struct crypto_cipher *ci; + + ci = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ci)) + return PTR_ERR(ci); + tctx->sw_cipher = ci; + + fb = crypto_alloc_aead("ccm(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fb)) { + crypto_free_cipher(ci); + tctx->sw_cipher = NULL; + return PTR_ERR(fb); + } + tctx->fallback = fb; + + /* + * Subreq lives at (rctx + 1). Alignment is guaranteed + * by the crypto framework's __ctx ALIGN mechanism. + */ + crypto_aead_set_reqsize(tfm, + sizeof(struct cmh_aes_aead_reqctx) + + sizeof(struct aead_request) + + crypto_aead_reqsize(fb)); + } else { + crypto_aead_set_reqsize(tfm, + sizeof(struct cmh_aes_aead_reqctx)); + } + + return 0; +} + +static void cmh_aes_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + if (tctx->fallback) + crypto_free_aead(tctx->fallback); + if (tctx->sw_cipher) + crypto_free_cipher(tctx->sw_cipher); + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_aes_aead_unmap_dma(struct cmh_aes_aead_reqctx *rctx) +{ + u32 tag_map_len; + + cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE); + /* + * The empty-GCM fallback maps a full AES block (16 bytes) for the + * ECB output regardless of authsize, so unmap with the mapped size. + */ + tag_map_len = rctx->empty_gcm_fallback ? + AES_GCM_HW_IV_SIZE : rctx->authsize; + cmh_dma_unmap_single(rctx->tag_dma, tag_map_len, + (rctx->encrypting || rctx->empty_gcm_fallback) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (rctx->cryptlen > 0) { + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, + DMA_TO_DEVICE); + } + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +} + +static void cmh_aes_aead_free_bufs(struct cmh_aes_aead_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; + kfree(rctx->aad_buf); + rctx->aad_buf = NULL; +} + +static void cmh_aes_aead_complete(void *data, int error) +{ + struct aead_request *req = data; + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_aes_aead_unmap_dma(rctx); + + /* + * Map HW error on decrypt to -EBADMSG. The eSW AES core uses a + * single error code (-EIO) for both authentication failures and + * other core errors (e.g. DMA timeout), so we cannot distinguish + * them from the MBX_STATUS alone. In practice the only error + * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA + * timeout would indicate a fatal HW problem where -EBADMSG vs + * -EIO is moot. The kernel crypto API requires -EBADMSG for + * AEAD authentication failures. + */ + if (error == -EIO && !rctx->encrypting) + error = -EBADMSG; + + if (!error) { + /* GCM empty-input decrypt: compare computed tag with expected */ + if (rctx->empty_gcm_fallback && !rctx->encrypting) { + if (crypto_memneq(rctx->tag_buf, rctx->in_buf, + rctx->authsize)) + error = -EBADMSG; + } + if (!error && rctx->cryptlen > 0) + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + req->assoclen, + rctx->cryptlen, 1); + if (!error && rctx->encrypting) + scatterwalk_map_and_copy(rctx->tag_buf, req->dst, + req->assoclen + + rctx->cryptlen, + rctx->authsize, 1); + } + + cmh_aes_aead_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * GCM empty-input fallback. + * + * When both AAD and plaintext are empty, GCM reduces to: + * tag = E(K, J0) where J0 = nonce || 0x00000001 + * + * The eSW GCM engine rejects this degenerate case, so we compute it + * via a single ECB block encryption of J0. + * + * VCQ: [SYS_CMD_WRITE] + AES_CMD_INIT(ECB) + AES_CMD_FINAL + FLUSH + */ +static int cmh_aes_gcm_empty(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + authsize = tctx->authsize; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = 0; + rctx->assoclen = 0; + rctx->authsize = authsize; + rctx->encrypting = (aes_op == AES_OP_ENCRYPT); + rctx->empty_gcm_fallback = true; + + /* Build J0 = nonce || 0x00000001 in iv_buf */ + rctx->iv_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp); + if (!rctx->iv_buf) + return -ENOMEM; + memcpy(rctx->iv_buf, req->iv, AES_GCM_IV_SIZE); + rctx->iv_buf[15] = 0x01; /* big-endian counter = 1 */ + rctx->iv_map_len = AES_GCM_HW_IV_SIZE; + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, AES_GCM_HW_IV_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Tag buffer -- receives E(K, J0) output */ + rctx->tag_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_iv; + } + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, AES_GCM_HW_IV_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* For decrypt: read expected tag from request for later comparison */ + if (!rctx->encrypting) { + rctx->in_buf = kmalloc(authsize, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, + authsize, 0); + } + + /* Resolve key */ + idx = 0; + rctx->key_dma = tctx->key.raw.dma; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* ECB INIT: single block encryption of J0 */ + vcq_add_aes_aead_init(&cmds[idx++], core_id, key_ref, + 0, keylen, 0, AES_MODE_ECB, AES_OP_ENCRYPT, + 0, AES_GCM_HW_IV_SIZE, 0); + + /* FINAL: J0 in, E(K,J0) out */ + vcq_add_aes_aead_final(&cmds[idx++], core_id, + (u64)rctx->iv_dma, (u64)rctx->tag_dma, + 0, AES_GCM_HW_IV_SIZE, 0); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_AEAD_MAX_PACKED, + target_mbx, + cmh_aes_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_free_in; + + return -EINPROGRESS; + +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, AES_GCM_HW_IV_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_iv: + cmh_dma_unmap_single(rctx->iv_dma, AES_GCM_HW_IV_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); + return ret; +} + +/* + * CCM empty-input fallback. + * + * When both AAD and plaintext are empty, CCM reduces to: + * T = E(K, B0) -- CBC-MAC of the single formatting block + * S0 = E(K, A0) -- CTR block zero + * tag = (T XOR S0)[0..authsize-1] + * + * The eSW rejects this degenerate case, so the driver computes it + * synchronously via two crypto_cipher single-block encryptions. + */ +static int cmh_aes_ccm_empty(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + u32 authsize = tctx->authsize; + u8 b0[CMH_AES_BLOCK_SIZE], a0[CMH_AES_BLOCK_SIZE]; + u8 t[CMH_AES_BLOCK_SIZE], s0[CMH_AES_BLOCK_SIZE]; + u8 tag[CMH_AES_BLOCK_SIZE]; + u8 L; + u32 i; + + /* Defense-in-depth: iv[0] = L-1, valid L is 2..8 per RFC 3610 S2.1 */ + if (WARN_ON_ONCE(req->iv[0] < 1 || req->iv[0] > 7)) + return -EINVAL; + + L = req->iv[0] + 1; + + if (tctx->key.mode != CMH_KEY_RAW) + return -EOPNOTSUPP; + + /* B0: flags || nonce || Q(=0). Adata=0, t=authsize, q=L. */ + memset(b0, 0, CMH_AES_BLOCK_SIZE); + b0[0] = (u8)(8 * ((authsize - 2) / 2) + (L - 1)); + memcpy(&b0[1], &req->iv[1], 15 - L); + + /* A0: (L-1) || nonce || counter(=0) */ + memset(a0, 0, CMH_AES_BLOCK_SIZE); + a0[0] = (u8)(L - 1); + memcpy(&a0[1], &req->iv[1], 15 - L); + + crypto_cipher_encrypt_one(tctx->sw_cipher, t, b0); + crypto_cipher_encrypt_one(tctx->sw_cipher, s0, a0); + + for (i = 0; i < authsize; i++) + tag[i] = t[i] ^ s0[i]; + + if (aes_op == AES_OP_ENCRYPT) { + scatterwalk_map_and_copy(tag, req->dst, + req->assoclen, authsize, 1); + } else { + u8 expected[CMH_AES_BLOCK_SIZE]; + + scatterwalk_map_and_copy(expected, req->src, + req->assoclen, authsize, 0); + if (crypto_memneq(tag, expected, authsize)) + return -EBADMSG; + } + + return 0; +} + +/* + * CCM authsize=10 fallback. + * + * The eSW AES CCM core does not support authsize=10 (valid per RFC 3610). + * Forward the entire request to the generic CCM implementation. + */ +static void cmh_aes_ccm_fb_done(void *data, int err) +{ + struct aead_request *req = data; + + cmh_complete(&req->base, err); +} + +static int cmh_aes_ccm_fallback(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + struct aead_request *subreq = (void *)(rctx + 1); + + aead_request_set_tfm(subreq, tctx->fallback); + aead_request_set_callback(subreq, req->base.flags, + cmh_aes_ccm_fb_done, req); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (aes_op == AES_OP_ENCRYPT) ? + crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); +} + +/* + * Core AEAD encrypt/decrypt -- async path. + * + * Encrypt: plaintext -> ciphertext + tag appended + * Decrypt: ciphertext + tag -> plaintext (tag verified by eSW) + * + * VCQ: [SYS_CMD_WRITE] + INIT/CCM_INIT + [AAD_FINAL] + FINAL + FLUSH + */ +static int cmh_aes_aead_crypt(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm); + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize, cryptlen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + authsize = tctx->authsize; + + if (aes_op == AES_OP_ENCRYPT) { + cryptlen = req->cryptlen; + } else { + if (req->cryptlen < authsize) + return -EINVAL; + cryptlen = req->cryptlen - authsize; + } + + /* + * Validate CCM IV format early -- the empty-input fallback and + * nonce extraction both depend on iv[0] being in range [1,7]. + */ + if (info->type == CMH_AES_AEAD_CCM) { + if (req->iv[0] < 1 || req->iv[0] > 7) + return -EINVAL; + } + + /* + * The CMH eSW rejects GCM/CCM when both aadlen and iolen are zero. + * For GCM, the tag is simply E(K, J0) -- handle with ECB fallback. + * For CCM, compute tag = E(K,B0) XOR E(K,A0) in software. + */ + if (cryptlen == 0 && req->assoclen == 0) { + if (info->type == CMH_AES_AEAD_GCM) + return cmh_aes_gcm_empty(req, aes_op); + return cmh_aes_ccm_empty(req, aes_op); + } + + /* + * HW does not support authsize=10 for CCM. Forward the entire + * request to the generic CCM implementation. + */ + if (info->type == CMH_AES_AEAD_CCM && authsize == 10) + return cmh_aes_ccm_fallback(req, aes_op); + + /* + * HW uses a proprietary LLI scatter-gather format that is + * incompatible with struct scatterlist, so the payload is + * linearised into contiguous buffers for DMA. Cap total + * size to prevent excessive memory consumption. + */ + if ((u64)cryptlen + req->assoclen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = cryptlen; + rctx->assoclen = req->assoclen; + rctx->authsize = authsize; + rctx->encrypting = (aes_op == AES_OP_ENCRYPT); + + /* Linearise AAD */ + if (req->assoclen > 0) { + rctx->aad_buf = kmalloc(req->assoclen, gfp); + if (!rctx->aad_buf) + return -ENOMEM; + scatterwalk_map_and_copy(rctx->aad_buf, req->src, + 0, req->assoclen, 0); + rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf, + req->assoclen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->aad_dma)) { + ret = -ENOMEM; + goto out_free_aad; + } + } + + /* Linearise input */ + if (cryptlen > 0) { + rctx->in_buf = kmalloc(cryptlen, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_aad; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, + req->assoclen, cryptlen, 0); + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + } + + /* Allocate output buffer */ + if (cryptlen > 0) { + rctx->out_buf = kmalloc(cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + } + + /* Tag buffer */ + rctx->tag_buf = kmalloc(authsize, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + if (!rctx->encrypting) { + scatterwalk_map_and_copy(rctx->tag_buf, req->src, + req->assoclen + cryptlen, + authsize, 0); + } else { + memset(rctx->tag_buf, 0, authsize); + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize, + rctx->encrypting ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map IV/nonce */ + if (info->type == CMH_AES_AEAD_GCM) { + rctx->iv_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + memcpy(rctx->iv_buf, req->iv, AES_GCM_IV_SIZE); + rctx->iv_map_len = AES_GCM_HW_IV_SIZE; + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, + rctx->iv_map_len, + DMA_TO_DEVICE); + } else { + u32 noncelen; + + if (req->iv[0] < 1 || req->iv[0] > 7) { + ret = -EINVAL; + goto out_unmap_tag; + } + noncelen = 14 - req->iv[0]; + + rctx->iv_buf = kmemdup(req->iv + 1, noncelen, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + rctx->iv_map_len = noncelen; + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, + rctx->iv_map_len, + DMA_TO_DEVICE); + } + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* Build INIT command */ + if (info->type == CMH_AES_AEAD_CCM) { + vcq_add_aes_ccm_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, + rctx->iv_map_len, aes_op, + req->assoclen, cryptlen, authsize); + } else { + vcq_add_aes_aead_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, + AES_GCM_HW_IV_SIZE, info->aes_mode, + aes_op, req->assoclen, cryptlen, + authsize); + } + + if (req->assoclen > 0) + vcq_add_aes_aad_final(&cmds[idx++], core_id, + (u64)rctx->aad_dma, req->assoclen); + + vcq_add_aes_aead_final(&cmds[idx++], core_id, + cryptlen > 0 ? (u64)rctx->in_dma : 0, + cryptlen > 0 ? (u64)rctx->out_dma : 0, + (u64)rctx->tag_dma, cryptlen, authsize); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_AEAD_MAX_PACKED, + target_mbx, + cmh_aes_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_out: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_aad: + if (req->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, req->assoclen, + DMA_TO_DEVICE); +out_free_aad: + kfree(rctx->aad_buf); + return ret; +} + +static int cmh_aes_aead_encrypt(struct aead_request *req) +{ + return cmh_aes_aead_crypt(req, AES_OP_ENCRYPT); +} + +static int cmh_aes_aead_decrypt(struct aead_request *req) +{ + return cmh_aes_aead_crypt(req, AES_OP_DECRYPT); +} + +/* Registration */ + +static struct cmh_aes_aead_drv aes_aead_drv_algs[ARRAY_SIZE(aes_aead_algs)]; + +/** + * cmh_aes_aead_register() - Register AES-GCM/CCM AEAD algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_aes_aead_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) { + const struct cmh_aes_aead_info *info = &aes_aead_algs[i]; + struct cmh_aes_aead_drv *drv = &aes_aead_drv_algs[i]; + struct aead_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->setkey = cmh_aes_aead_setkey; + alg->setauthsize = cmh_aes_aead_setauthsize; + alg->encrypt = cmh_aes_aead_encrypt; + alg->decrypt = cmh_aes_aead_decrypt; + alg->init = cmh_aes_aead_init_tfm; + alg->exit = cmh_aes_aead_exit_tfm; + alg->ivsize = info->ivsize; + alg->maxauthsize = info->maxauthsize; + + strscpy(alg->base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + if (info->type == CMH_AES_AEAD_CCM) { + alg->base.cra_flags |= CRYPTO_ALG_NEED_FALLBACK; + /* + * Bump priority above 300 so we beat the generic + * ccm_base template instance. That template inherits + * priority (ctr + cbcmac) / 2 = 300 when both + * constituents are at 300, and list ordering would + * otherwise let it shadow our driver. + */ + alg->base.cra_priority = 301; + } + alg->base.cra_blocksize = 1; + alg->base.cra_ctxsize = sizeof(struct cmh_aes_aead_tfm_ctx); + alg->base.cra_module = THIS_MODULE; + + ret = crypto_register_aead(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_aes_aead: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_aes_aead: registered %s\n", info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_aead(&aes_aead_drv_algs[i].alg); + return ret; +} + +/** + * cmh_aes_aead_unregister() - Unregister AES AEAD algorithms from the crypto framework + */ +void cmh_aes_aead_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) { + crypto_unregister_aead(&aes_aead_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_aes_aead: unregistered %s\n", + aes_aead_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/cmh_aes_cmac.c b/drivers/crypto/cmh/cmh_aes_cmac.c new file mode 100644 index 00000000000000..a711c575398d88 --- /dev/null +++ b/drivers/crypto/cmh/cmh_aes_cmac.c @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API AES-CMAC (ahash) Driver + * + * Registers cmac(aes) as an ahash algorithm. + * + * CMAC produces a 16-byte tag (MAC) from a key and message. + * VCQ sequence: [SYS_CMD_WRITE] + AES_CMD_INIT(CMAC) + + * AES_CMD_AAD_FINAL_AUTH + FLUSH + * + * The ahash interface accumulates data in a kernel buffer via .update(), + * then .final() builds and submits the VCQ asynchronously. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_aes.h" +#include "cmh_vcq.h" +#include "cmh_aes_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define AES_CMAC_DIGEST_SIZE 16U +#define AES_CMAC_BLOCK_SIZE 16U + +/* + * Maximum accumulated data for CMAC -- driver-imposed, not HW. + * + * The AES core does not expose external save/restore VCQ commands, + * so the driver must accumulate all data in kernel memory via + * .update() and submit it atomically in .final(). This cap limits + * the per-request kernel allocation. + */ +#define AES_CMAC_MAX_DATA (64 * 1024) + +/* Per-transform context */ +struct cmh_aes_cmac_tfm_ctx { + struct cmh_key_ctx key; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* One chunk per .update() call -- data is embedded via flexible array */ +struct cmh_aes_cmac_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* Per-request context (lives in ahash_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + AES_CMD_AAD_FINAL_AUTH + FLUSH = 4 + */ +#define CMH_AES_CMAC_MAX_PAYLOAD 4 +#define CMH_AES_CMAC_MAX_PACKED (CMH_AES_CMAC_MAX_PAYLOAD * 2) + +struct cmh_aes_cmac_reqctx { + struct list_head chunks; + u32 total_len; + u8 *buf; /* linearised in final() for DMA */ + /* DMA state for async final */ + dma_addr_t key_dma; + dma_addr_t in_dma; + dma_addr_t tag_dma; + u8 *tag_buf; + u32 keylen; + struct vcq_cmd packed[CMH_AES_CMAC_MAX_PACKED]; +}; + +/* Flat state for export/import -- holds accumulated input data only */ +struct cmh_aes_cmac_export_state { + u32 total_len; + u8 data[]; +}; + +/* + * Flat state buffer for export/import. The CMH AES core does not + * support save/restore of intermediate CMAC state, so this driver + * accumulates input in SW and serialises the buffer on export. + * + * PAGE_SIZE (4096) caps the exportable accumulated-data window. + * Full-range export is not feasible because the crypto subsystem + * pre-allocates statesize bytes per request. Export returns -EINVAL + * if the caller has accumulated more than CMH_AES_CMAC_EXPORT_MAX. + */ +#define CMH_AES_CMAC_STATE_SIZE 4096 +#define CMH_AES_CMAC_EXPORT_MAX \ + (CMH_AES_CMAC_STATE_SIZE - sizeof(struct cmh_aes_cmac_export_state)) + +/* + * Export/import: not supported. + * + * The AES core lacks external save/restore VCQ commands, so there is + * no way to checkpoint intermediate CMAC state to host memory. + * Pending eSW ABI extension to add save/restore for the AES core. + */ + +static int cmh_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + if (keylen != 16 && keylen != 24 && keylen != 32) + return -EINVAL; + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES); +} + +static void cmh_aes_cmac_free_chunks(struct cmh_aes_cmac_reqctx *rctx, + struct cmh_aes_cmac_tfm_ctx *tctx) +{ + struct cmh_aes_cmac_chunk *c, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &rctx->chunks, list) { + list_del(&c->list); + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len = 0; +} + +static int cmh_aes_cmac_init(struct ahash_request *req) +{ + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + return 0; +} + +static int cmh_aes_cmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_aes_cmac_chunk *chunk; + gfp_t gfp; + int ret; + + if (!req->nbytes) + return 0; + + if (req->nbytes > AES_CMAC_MAX_DATA - rctx->total_len) { + ret = -EINVAL; + goto err_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp); + if (!chunk) { + ret = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(chunk->data, req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(chunk->data, req->src, + 0, req->nbytes, 0); + + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len += req->nbytes; + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * callers may not call .final() on error, so they would leak. + */ + cmh_aes_cmac_free_chunks(rctx, tctx); + return ret; +} + +static void cmh_aes_cmac_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* Unmap DMA */ + if (rctx->total_len > 0) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, AES_CMAC_DIGEST_SIZE, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->tag_buf, AES_CMAC_DIGEST_SIZE); + + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + cmh_aes_cmac_free_chunks(rctx, tctx); + cmh_complete(&req->base, error); +} + +static int cmh_aes_cmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_CMAC_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) { + ret = -ENOKEY; + goto out_free_buf; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Linearise accumulated chunks into a contiguous buffer for DMA */ + if (rctx->total_len > 0) { + struct cmh_aes_cmac_chunk *c; + u32 off = 0; + + rctx->buf = kmalloc(rctx->total_len, gfp); + if (!rctx->buf) { + ret = -ENOMEM; + goto out_free_chunks; + } + list_for_each_entry(c, &rctx->chunks, list) { + memcpy(rctx->buf + off, c->data, c->len); + off += c->len; + } + } + + /* Tag output buffer */ + rctx->tag_buf = kzalloc(AES_CMAC_DIGEST_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_free_buf; + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, + AES_CMAC_DIGEST_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map input data (may be zero-length for empty CMAC) */ + if (rctx->total_len > 0) { + rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_unmap_tag; + } + } + + /* Resolve key */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* + * INIT: mode=CMAC, op=ENCRYPT (CMAC always "encrypts") + * CMAC data goes through the AAD path: + * aadlen = total data length, iolen = 0 + */ + { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = 0; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = 0; + slot->hwc.aes.cmd_init.mode = AES_MODE_CMAC; + slot->hwc.aes.cmd_init.op = AES_OP_ENCRYPT; + slot->hwc.aes.cmd_init.aadlen = rctx->total_len; + slot->hwc.aes.cmd_init.iolen = 0; + slot->hwc.aes.cmd_init.taglen = AES_CMAC_DIGEST_SIZE; + } + + /* AAD_FINAL_AUTH: final AAD + tag extraction in one atomic step */ + { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_AAD_FINAL_AUTH); + slot->hwc.aes.cmd_aad_final_auth.data = + rctx->total_len > 0 ? (u64)rctx->in_dma : 0; + slot->hwc.aes.cmd_aad_final_auth.datalen = rctx->total_len; + slot->hwc.aes.cmd_aad_final_auth.tag = (u64)rctx->tag_dma; + slot->hwc.aes.cmd_aad_final_auth.taglen = AES_CMAC_DIGEST_SIZE; + } + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_CMAC_MAX_PACKED, + target_mbx, + cmh_aes_cmac_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + /* -EBUSY = backlogged; ownership transferred to callback. */ + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->total_len > 0 && !cmh_dma_map_error(rctx->in_dma)) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, AES_CMAC_DIGEST_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_free_buf: +out_free_chunks: + cmh_aes_cmac_free_chunks(rctx, tctx); + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + rctx->total_len = 0; + return ret; +} + +/* + * ahash .export()/.import(): serialize/deserialize the software + * accumulation buffer. No HW state is involved -- the AES core + * does not support save/restore, but we only export the input queue. + */ + +static int cmh_aes_cmac_export(struct ahash_request *req, void *out) +{ + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_aes_cmac_export_state *state = out; + struct cmh_aes_cmac_chunk *chunk; + u32 offset = 0; + + if (rctx->total_len > CMH_AES_CMAC_EXPORT_MAX) + return -ENOSPC; + + state->total_len = rctx->total_len; + list_for_each_entry(chunk, &rctx->chunks, list) { + memcpy(state->data + offset, chunk->data, chunk->len); + offset += chunk->len; + } + return 0; +} + +static int cmh_aes_cmac_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_aes_cmac_export_state *state = in; + struct cmh_aes_cmac_chunk *chunk; + + /* + * Do NOT call free_chunks() here: the crypto API does not + * guarantee the request context is in a valid state before + * import(), so the list pointers may be stale or invalid. + * Re-initialize from scratch instead. Any pre-existing chunks + * are tracked on tctx->all_chunks and freed in exit_tfm. + */ + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + + if (state->total_len > CMH_AES_CMAC_EXPORT_MAX) + return -EINVAL; + + if (state->total_len) { + chunk = kmalloc(sizeof(*chunk) + state->total_len, GFP_KERNEL); + if (!chunk) + return -ENOMEM; + chunk->len = state->total_len; + memcpy(chunk->data, state->data, state->total_len); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len = state->total_len; + } + return 0; +} + +static int cmh_aes_cmac_finup(struct ahash_request *req) +{ + int err; + + err = cmh_aes_cmac_update(req); + if (err) + return err; + return cmh_aes_cmac_final(req); +} + +static int cmh_aes_cmac_digest(struct ahash_request *req) +{ + int err; + + err = cmh_aes_cmac_init(req); + if (err) + return err; + return cmh_aes_cmac_finup(req); +} + +static int cmh_aes_cmac_init_tfm(struct crypto_ahash *tfm) +{ + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_aes_cmac_reqctx)); + return 0; +} + +static void cmh_aes_cmac_exit_tfm(struct crypto_ahash *tfm) +{ + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_chunk *c, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) { + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + + cmh_key_destroy(&tctx->key); +} + +static struct ahash_alg cmh_aes_cmac_alg = { + .init = cmh_aes_cmac_init, + .update = cmh_aes_cmac_update, + .final = cmh_aes_cmac_final, + .finup = cmh_aes_cmac_finup, + .digest = cmh_aes_cmac_digest, + .export = cmh_aes_cmac_export, + .import = cmh_aes_cmac_import, + .setkey = cmh_aes_cmac_setkey, + .init_tfm = cmh_aes_cmac_init_tfm, + .exit_tfm = cmh_aes_cmac_exit_tfm, + .halg = { + .digestsize = AES_CMAC_DIGEST_SIZE, + .statesize = CMH_AES_CMAC_STATE_SIZE, + .base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "cri-cmh-cmac-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT, + .cra_blocksize = AES_CMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cmh_aes_cmac_tfm_ctx), + .cra_module = THIS_MODULE, + }, + }, +}; + +/** + * cmh_aes_cmac_register() - Register AES-CMAC hash algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_aes_cmac_register(void) +{ + int ret; + + ret = crypto_register_ahash(&cmh_aes_cmac_alg); + if (ret) + dev_err(cmh_dev(), "cmh_aes_cmac: failed to register cmac(aes) (rc=%d)\n", + ret); + else + dev_dbg(cmh_dev(), "cmh_aes_cmac: registered cmac(aes)\n"); + + return ret; +} + +/** + * cmh_aes_cmac_unregister() - Unregister AES-CMAC hash algorithm from the crypto framework + */ +void cmh_aes_cmac_unregister(void) +{ + crypto_unregister_ahash(&cmh_aes_cmac_alg); + dev_dbg(cmh_dev(), "cmh_aes_cmac: unregistered cmac(aes)\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index bc326a9d7626fd..535a6ff551a22e 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -34,6 +34,7 @@ #include "cmh_cshake.h" #include "cmh_kmac.h" #include "cmh_sm3.h" +#include "cmh_aes.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -227,6 +228,21 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_sm3_register; + /* Register AES skcipher algorithms */ + ret = cmh_aes_register(); + if (ret) + goto err_aes_register; + + /* Register AES AEAD algorithms (GCM, CCM) */ + ret = cmh_aes_aead_register(); + if (ret) + goto err_aes_aead_register; + + /* Register AES CMAC algorithm */ + ret = cmh_aes_cmac_register(); + if (ret) + goto err_aes_cmac_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -239,6 +255,12 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_aes_cmac_unregister(); +err_aes_cmac_register: + cmh_aes_aead_unregister(); +err_aes_aead_register: + cmh_aes_unregister(); +err_aes_register: cmh_sm3_unregister(); err_sm3_register: cmh_kmac_unregister(); @@ -275,6 +297,9 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_aes_cmac_unregister(); + cmh_aes_aead_unregister(); + cmh_aes_unregister(); cmh_sm3_unregister(); cmh_kmac_unregister(); cmh_cshake_unregister(); diff --git a/drivers/crypto/cmh/include/cmh_aes.h b/drivers/crypto/cmh/include/cmh_aes.h new file mode 100644 index 00000000000000..591afaa36f8592 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_aes.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- AES Crypto API Drivers + * + * Registers AES algorithms with the Linux crypto subsystem: + * skcipher: ecb/cbc/ctr/cfb/xts(aes) + * aead: gcm/ccm(aes) + * shash: cmac(aes) + */ + +#ifndef CMH_AES_H +#define CMH_AES_H + +int cmh_aes_register(void); +void cmh_aes_unregister(void); + +int cmh_aes_aead_register(void); +void cmh_aes_aead_unregister(void); + +int cmh_aes_cmac_register(void); +void cmh_aes_cmac_unregister(void); + +#endif /* CMH_AES_H */ From f521557fc1b54954d67bf615c8510e39d633089a Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:27 -0700 Subject: [PATCH 11/21] crypto: cmh - add SM4 skcipher/aead/cmac/xcbc Register SM4 algorithms using the CMH SM4 core (core ID 0x04): - skcipher: SM4-ECB, SM4-CBC, SM4-CTR, SM4-XTS, SM4-CFB - aead: SM4-GCM, SM4-CCM - ahash: SM4-CMAC, SM4-XCBC Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 5 +- drivers/crypto/cmh/cmh_main.c | 25 + drivers/crypto/cmh/cmh_sm4_aead.c | 870 ++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_sm4_cmac.c | 754 ++++++++++++++++++++++ drivers/crypto/cmh/cmh_sm4_skcipher.c | 690 ++++++++++++++++++++ drivers/crypto/cmh/include/cmh_sm4.h | 24 + 6 files changed, 2367 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_sm4_aead.c create mode 100644 drivers/crypto/cmh/cmh_sm4_cmac.c create mode 100644 drivers/crypto/cmh/cmh_sm4_skcipher.c create mode 100644 drivers/crypto/cmh/include/cmh_sm4.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index ced8d1748e6ce0..1f36cd9c0b98ad 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -22,7 +22,10 @@ cmh-y := \ cmh_sm3.o \ cmh_aes.o \ cmh_aes_aead.o \ - cmh_aes_cmac.o + cmh_aes_cmac.o \ + cmh_sm4_skcipher.o \ + cmh_sm4_aead.o \ + cmh_sm4_cmac.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 535a6ff551a22e..0a73964b978587 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -35,6 +35,7 @@ #include "cmh_kmac.h" #include "cmh_sm3.h" #include "cmh_aes.h" +#include "cmh_sm4.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -243,6 +244,21 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_aes_cmac_register; + /* Register SM4 skcipher algorithms */ + ret = cmh_sm4_register(); + if (ret) + goto err_sm4_register; + + /* Register SM4 AEAD algorithms (GCM, CCM) */ + ret = cmh_sm4_aead_register(); + if (ret) + goto err_sm4_aead_register; + + /* Register SM4 CMAC/XCBC algorithms */ + ret = cmh_sm4_cmac_register(); + if (ret) + goto err_sm4_cmac_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -255,6 +271,12 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_sm4_cmac_unregister(); +err_sm4_cmac_register: + cmh_sm4_aead_unregister(); +err_sm4_aead_register: + cmh_sm4_unregister(); +err_sm4_register: cmh_aes_cmac_unregister(); err_aes_cmac_register: cmh_aes_aead_unregister(); @@ -297,6 +319,9 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_sm4_cmac_unregister(); + cmh_sm4_aead_unregister(); + cmh_sm4_unregister(); cmh_aes_cmac_unregister(); cmh_aes_aead_unregister(); cmh_aes_unregister(); diff --git a/drivers/crypto/cmh/cmh_sm4_aead.c b/drivers/crypto/cmh/cmh_sm4_aead.c new file mode 100644 index 00000000000000..478119bb9c088f --- /dev/null +++ b/drivers/crypto/cmh/cmh_sm4_aead.c @@ -0,0 +1,870 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API SM4 AEAD Driver (GCM/CCM) + * + * Registers AEAD algorithms with the Linux crypto subsystem: + * gcm(sm4), ccm(sm4) + * + * GCM: SM4_CMD_INIT(mode=GCM) + [AAD_FINAL] + SM4_CMD_FINAL + FLUSH + * CCM: SM4_CMD_CCM_INIT + [AAD_FINAL] + SM4_CMD_FINAL + FLUSH + * - SM4 CCM uses a distinct sm4_cmd_ccm_init struct + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_sm4.h" +#include "cmh_vcq.h" +#include "cmh_sm4_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * GCM IV contract: + * + * The SM4 core requires exactly 16 bytes loaded into its IV register. + * For standard 96-bit nonce GCM, the driver passes: + * + * IV[0..11] = user-supplied 12-byte nonce + * IV[12..15] = 0x00000000 + * + * The hardware internally sets the last 32 bits to the big-endian + * counter value 1 (forming J0 = nonce || 0x00000001) before + * processing AAD. The driver must NOT pre-set the counter. + * + * If the IV format is incorrect, GCM authentication will fail + * (encrypt produces wrong ciphertext/tag, decrypt rejects). + */ +#define SM4_GCM_IV_SIZE 12U /* GCM nonce size (standard) */ +#define SM4_GCM_HW_IV_SIZE 16U /* HW requires 16-byte IV buffer */ +#define SM4_GCM_TAG_SIZE 16U + +/* CCM: callers pass a 16-byte IV in RFC 3610 format: + * iv[0] = L-1, iv[1..14-iv[0]] = nonce, rest = counter (zeroed). + * Nonce length = 14 - iv[0], range 7..13. + */ +#define SM4_CCM_IV_SIZE 16U + +enum cmh_sm4_aead_type { + CMH_SM4_AEAD_GCM, + CMH_SM4_AEAD_CCM, +}; + +struct cmh_sm4_aead_info { + enum cmh_sm4_aead_type type; + u32 sm4_mode; + u32 ivsize; + u32 maxauthsize; + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_sm4_aead_info sm4_aead_algs[] = { + { CMH_SM4_AEAD_GCM, SM4_MODE_GCM, SM4_GCM_IV_SIZE, + SM4_GCM_TAG_SIZE, "gcm(sm4)", "cri-cmh-gcm-sm4" }, + { CMH_SM4_AEAD_CCM, SM4_MODE_CCM, SM4_CCM_IV_SIZE, + SM4_GCM_TAG_SIZE, "ccm(sm4)", "cri-cmh-ccm-sm4" }, +}; + +struct cmh_sm4_aead_tfm_ctx { + struct cmh_key_ctx key; + u32 authsize; + struct crypto_cipher *sw_cipher; /* CCM empty-input fallback */ +}; + +/* Per-request context (lives in aead_request::__ctx) */ + +#define CMH_SM4_AEAD_MAX_PAYLOAD 5 +#define CMH_SM4_AEAD_MAX_PACKED (CMH_SM4_AEAD_MAX_PAYLOAD * 2) + +struct cmh_sm4_aead_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + dma_addr_t aad_dma; + dma_addr_t tag_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *aad_buf; + u8 *tag_buf; + u32 cryptlen; + u32 assoclen; + u32 authsize; + u32 iv_map_len; + u32 keylen; + bool encrypting; + bool empty_gcm_fallback; + struct vcq_cmd packed[CMH_SM4_AEAD_MAX_PACKED]; +}; + +struct cmh_sm4_aead_drv { + struct aead_alg alg; + const struct cmh_sm4_aead_info *info; +}; + +static const struct cmh_sm4_aead_info * +cmh_sm4_aead_get_info(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + + return container_of(alg, struct cmh_sm4_aead_drv, alg)->info; +} + +/* VCQ Builders -- SM4 AEAD-specific */ + +static void vcq_add_sm4_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 iv_dma, u32 keylen, u32 ivlen, + u32 mode, u32 op, u32 aadlen, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_INIT); + slot->hwc.sm4.cmd_init.key = key_ref; + slot->hwc.sm4.cmd_init.iv = iv_dma; + slot->hwc.sm4.cmd_init.keylen = keylen; + slot->hwc.sm4.cmd_init.ivlen = ivlen; + slot->hwc.sm4.cmd_init.mode = mode; + slot->hwc.sm4.cmd_init.op = op; + slot->hwc.sm4.cmd_init.aadlen = aadlen; + slot->hwc.sm4.cmd_init.iolen = iolen; +} + +static void vcq_add_sm4_ccm_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 nonce_dma, u32 keylen, u32 noncelen, + u32 op, u32 aadlen, u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_CCM_INIT); + slot->hwc.sm4.cmd_ccm_init.key = key_ref; + slot->hwc.sm4.cmd_ccm_init.nonce = nonce_dma; + slot->hwc.sm4.cmd_ccm_init.keylen = keylen; + slot->hwc.sm4.cmd_ccm_init.noncelen = noncelen; + slot->hwc.sm4.cmd_ccm_init.op = op; + slot->hwc.sm4.cmd_ccm_init.aadlen = aadlen; + slot->hwc.sm4.cmd_ccm_init.iolen = iolen; + slot->hwc.sm4.cmd_ccm_init.taglen = taglen; +} + +static void vcq_add_sm4_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma, + u32 aadlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_AAD_FINAL); + slot->hwc.sm4.cmd_aad_final.data = aad_dma; + slot->hwc.sm4.cmd_aad_final.datalen = aadlen; +} + +static void vcq_add_sm4_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_FINAL); + slot->hwc.sm4.cmd_final.input = input_dma; + slot->hwc.sm4.cmd_final.output = output_dma; + slot->hwc.sm4.cmd_final.tag = tag_dma; + slot->hwc.sm4.cmd_final.iolen = iolen; + slot->hwc.sm4.cmd_final.taglen = taglen; +} + +/* setkey */ +static int cmh_sm4_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + /* SM4 always uses 128-bit keys */ + if (keylen != CMH_SM4_KEY_SIZE) + return -EINVAL; + + if (tctx->sw_cipher) { + int ret; + + ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen); + if (ret) + return ret; + } + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_SM4); +} + +static int cmh_sm4_aead_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_sm4_aead_info *info = cmh_sm4_aead_get_info(tfm); + + if (info->type == CMH_SM4_AEAD_GCM) { + /* eSW enforces taglen == 16 for SM4 GCM (EIP40_SM4_TAG_SIZE) */ + if (authsize != 16) + return -EINVAL; + } else { + /* CCM: accept 4, 6, 8, 10, 12, 14, 16 per RFC 3610 */ + if (authsize < 4 || authsize > 16 || (authsize & 1)) + return -EINVAL; + } + + tctx->authsize = authsize; + return 0; +} + +static int cmh_sm4_aead_init_tfm(struct crypto_aead *tfm) +{ + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_sm4_aead_info *info = cmh_sm4_aead_get_info(tfm); + + memset(tctx, 0, sizeof(*tctx)); + tctx->authsize = info->maxauthsize; + + if (info->type == CMH_SM4_AEAD_CCM) { + struct crypto_cipher *ci; + + ci = crypto_alloc_cipher("sm4", 0, 0); + if (IS_ERR(ci)) + return PTR_ERR(ci); + tctx->sw_cipher = ci; + } + + crypto_aead_set_reqsize(tfm, sizeof(struct cmh_sm4_aead_reqctx)); + return 0; +} + +static void cmh_sm4_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + if (tctx->sw_cipher) + crypto_free_cipher(tctx->sw_cipher); + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_sm4_aead_unmap_dma(struct cmh_sm4_aead_reqctx *rctx) +{ + u32 tag_map_len; + + cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE); + tag_map_len = rctx->empty_gcm_fallback ? + SM4_GCM_HW_IV_SIZE : rctx->authsize; + cmh_dma_unmap_single(rctx->tag_dma, tag_map_len, + (rctx->encrypting || rctx->empty_gcm_fallback) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (rctx->cryptlen > 0) { + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, + DMA_TO_DEVICE); + } + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +} + +static void cmh_sm4_aead_free_bufs(struct cmh_sm4_aead_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; + kfree(rctx->aad_buf); + rctx->aad_buf = NULL; +} + +static void cmh_sm4_aead_complete(void *data, int error) +{ + struct aead_request *req = data; + struct cmh_sm4_aead_reqctx *rctx = aead_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_sm4_aead_unmap_dma(rctx); + + /* + * Map HW error on decrypt to -EBADMSG. The eSW SM4 core uses a + * single error code (-EIO) for both authentication failures and + * other core errors (e.g. DMA timeout), so we cannot distinguish + * them from the MBX_STATUS alone. In practice the only error + * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA + * timeout would indicate a fatal HW problem where -EBADMSG vs + * -EIO is moot. The kernel crypto API requires -EBADMSG for + * AEAD authentication failures. + */ + if (error == -EIO && !rctx->encrypting) + error = -EBADMSG; + + if (!error) { + if (rctx->empty_gcm_fallback && !rctx->encrypting) { + if (crypto_memneq(rctx->tag_buf, rctx->in_buf, + rctx->authsize)) + error = -EBADMSG; + } + if (!error && rctx->cryptlen > 0) + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + req->assoclen, + rctx->cryptlen, 1); + if (!error && rctx->encrypting) + scatterwalk_map_and_copy(rctx->tag_buf, req->dst, + req->assoclen + + rctx->cryptlen, + rctx->authsize, 1); + } + + cmh_sm4_aead_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * GCM empty-input fallback (SM4). + * + * When both AAD and plaintext are empty, GCM reduces to: + * tag = E(K, J0) where J0 = nonce || 0x00000001 + * + * The eSW GCM engine rejects this degenerate case, so we compute it + * via a single ECB block encryption of J0. + * + * VCQ: [SYS_CMD_WRITE] + SM4_CMD_INIT(ECB) + SM4_CMD_FINAL + FLUSH + */ +static int cmh_sm4_gcm_empty(struct aead_request *req, u32 sm4_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_sm4_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_SM4_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + authsize = tctx->authsize; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = 0; + rctx->assoclen = 0; + rctx->authsize = authsize; + rctx->encrypting = (sm4_op == SM4_OP_ENCRYPT); + rctx->empty_gcm_fallback = true; + + /* Build J0 = nonce || 0x00000001 in iv_buf */ + rctx->iv_buf = kzalloc(SM4_GCM_HW_IV_SIZE, gfp); + if (!rctx->iv_buf) + return -ENOMEM; + memcpy(rctx->iv_buf, req->iv, SM4_GCM_IV_SIZE); + rctx->iv_buf[15] = 0x01; + rctx->iv_map_len = SM4_GCM_HW_IV_SIZE; + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, SM4_GCM_HW_IV_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Tag buffer -- receives E(K, J0) output */ + rctx->tag_buf = kzalloc(SM4_GCM_HW_IV_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_iv; + } + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, SM4_GCM_HW_IV_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* For decrypt: read expected tag from request */ + if (!rctx->encrypting) { + rctx->in_buf = kmalloc(authsize, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, + authsize, 0); + } + + /* Resolve key */ + idx = 0; + rctx->key_dma = tctx->key.raw.dma; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_SM4); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* ECB INIT: single block encryption of J0 */ + vcq_add_sm4_aead_init(&cmds[idx++], core_id, key_ref, + 0, keylen, 0, SM4_MODE_ECB, SM4_OP_ENCRYPT, + 0, SM4_GCM_HW_IV_SIZE); + + /* FINAL: J0 in, E(K,J0) out */ + vcq_add_sm4_aead_final(&cmds[idx++], core_id, + (u64)rctx->iv_dma, (u64)rctx->tag_dma, + 0, SM4_GCM_HW_IV_SIZE, 0); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM4_AEAD_MAX_PACKED, + target_mbx, + cmh_sm4_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_free_in; + + return -EINPROGRESS; + +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, SM4_GCM_HW_IV_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_iv: + cmh_dma_unmap_single(rctx->iv_dma, SM4_GCM_HW_IV_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); + return ret; +} + +/* + * CCM empty-input fallback (SM4). + * + * When both AAD and plaintext are empty, CCM reduces to: + * T = E(K, B0) -- CBC-MAC of the single formatting block + * S0 = E(K, A0) -- CTR block zero + * tag = (T XOR S0)[0..authsize-1] + * + * The eSW rejects this degenerate case, so the driver computes it + * synchronously via two crypto_cipher single-block encryptions. + */ +static int cmh_sm4_ccm_empty(struct aead_request *req, u32 sm4_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + u32 authsize = tctx->authsize; + u8 b0[CMH_SM4_BLOCK_SIZE], a0[CMH_SM4_BLOCK_SIZE]; + u8 t[CMH_SM4_BLOCK_SIZE], s0[CMH_SM4_BLOCK_SIZE]; + u8 tag[CMH_SM4_BLOCK_SIZE]; + u8 L; + u32 i; + + /* Defense-in-depth: iv[0] = L-1, valid L is 2..8 per RFC 3610 S2.1 */ + if (WARN_ON_ONCE(req->iv[0] < 1 || req->iv[0] > 7)) + return -EINVAL; + + L = req->iv[0] + 1; + + if (tctx->key.mode != CMH_KEY_RAW) + return -EOPNOTSUPP; + + /* B0: flags || nonce || Q(=0). Adata=0, t=authsize, q=L. */ + memset(b0, 0, CMH_SM4_BLOCK_SIZE); + b0[0] = (u8)(8 * ((authsize - 2) / 2) + (L - 1)); + memcpy(&b0[1], &req->iv[1], 15 - L); + + /* A0: (L-1) || nonce || counter(=0) */ + memset(a0, 0, CMH_SM4_BLOCK_SIZE); + a0[0] = (u8)(L - 1); + memcpy(&a0[1], &req->iv[1], 15 - L); + + crypto_cipher_encrypt_one(tctx->sw_cipher, t, b0); + crypto_cipher_encrypt_one(tctx->sw_cipher, s0, a0); + + for (i = 0; i < authsize; i++) + tag[i] = t[i] ^ s0[i]; + + if (sm4_op == SM4_OP_ENCRYPT) { + scatterwalk_map_and_copy(tag, req->dst, + req->assoclen, authsize, 1); + } else { + u8 expected[CMH_SM4_BLOCK_SIZE]; + + scatterwalk_map_and_copy(expected, req->src, + req->assoclen, authsize, 0); + if (crypto_memneq(tag, expected, authsize)) + return -EBADMSG; + } + + return 0; +} + +static int cmh_sm4_aead_crypt(struct aead_request *req, u32 sm4_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_sm4_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_sm4_aead_info *info = cmh_sm4_aead_get_info(tfm); + struct cmh_sm4_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_SM4_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize, cryptlen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + authsize = tctx->authsize; + + if (sm4_op == SM4_OP_ENCRYPT) { + cryptlen = req->cryptlen; + } else { + if (req->cryptlen < authsize) + return -EINVAL; + cryptlen = req->cryptlen - authsize; + } + + /* + * Validate CCM IV format early -- the empty-input fallback and + * nonce extraction both depend on iv[0] being in range [1,7]. + */ + if (info->type == CMH_SM4_AEAD_CCM) { + if (req->iv[0] < 1 || req->iv[0] > 7) + return -EINVAL; + } + + /* + * The CMH eSW rejects SM4 GCM/CCM when both aadlen and iolen + * are zero. For GCM, the tag is simply E(K, J0) -- use ECB + * fallback. For CCM, compute tag = E(K,B0) XOR E(K,A0) in SW. + */ + if (cryptlen == 0 && req->assoclen == 0) { + if (info->type == CMH_SM4_AEAD_GCM) + return cmh_sm4_gcm_empty(req, sm4_op); + return cmh_sm4_ccm_empty(req, sm4_op); + } + + /* + * HW uses a proprietary LLI scatter-gather format that is + * incompatible with struct scatterlist, so the payload is + * linearised into contiguous buffers for DMA. Cap total + * size to prevent excessive memory consumption. + */ + if ((u64)cryptlen + req->assoclen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = cryptlen; + rctx->assoclen = req->assoclen; + rctx->authsize = authsize; + rctx->encrypting = (sm4_op == SM4_OP_ENCRYPT); + + /* Linearise AAD */ + if (req->assoclen > 0) { + rctx->aad_buf = kmalloc(req->assoclen, gfp); + if (!rctx->aad_buf) + return -ENOMEM; + scatterwalk_map_and_copy(rctx->aad_buf, req->src, + 0, req->assoclen, 0); + rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf, + req->assoclen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->aad_dma)) { + ret = -ENOMEM; + goto out_free_aad; + } + } + + /* Linearise input */ + if (cryptlen > 0) { + rctx->in_buf = kmalloc(cryptlen, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_aad; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, + req->assoclen, cryptlen, 0); + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + } + + /* Allocate output buffer */ + if (cryptlen > 0) { + rctx->out_buf = kmalloc(cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + } + + /* Tag buffer */ + rctx->tag_buf = kmalloc(authsize, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + if (!rctx->encrypting) { + scatterwalk_map_and_copy(rctx->tag_buf, req->src, + req->assoclen + cryptlen, + authsize, 0); + } else { + memset(rctx->tag_buf, 0, authsize); + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize, + rctx->encrypting ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map IV/nonce */ + if (info->type == CMH_SM4_AEAD_GCM) { + rctx->iv_buf = kzalloc(SM4_GCM_HW_IV_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + memcpy(rctx->iv_buf, req->iv, SM4_GCM_IV_SIZE); + rctx->iv_map_len = SM4_GCM_HW_IV_SIZE; + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, + rctx->iv_map_len, + DMA_TO_DEVICE); + } else { + u32 noncelen; + + if (req->iv[0] < 1 || req->iv[0] > 7) { + ret = -EINVAL; + goto out_unmap_tag; + } + noncelen = 14 - req->iv[0]; + + rctx->iv_buf = kmemdup(req->iv + 1, noncelen, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + rctx->iv_map_len = noncelen; + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, + rctx->iv_map_len, + DMA_TO_DEVICE); + } + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_SM4); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* Build INIT command */ + if (info->type == CMH_SM4_AEAD_CCM) { + vcq_add_sm4_ccm_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, + rctx->iv_map_len, sm4_op, + req->assoclen, cryptlen, authsize); + } else { + vcq_add_sm4_aead_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, + SM4_GCM_HW_IV_SIZE, info->sm4_mode, + sm4_op, req->assoclen, cryptlen); + } + + if (req->assoclen > 0) + vcq_add_sm4_aad_final(&cmds[idx++], core_id, + (u64)rctx->aad_dma, req->assoclen); + + vcq_add_sm4_aead_final(&cmds[idx++], core_id, + cryptlen > 0 ? (u64)rctx->in_dma : 0, + cryptlen > 0 ? (u64)rctx->out_dma : 0, + (u64)rctx->tag_dma, cryptlen, authsize); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM4_AEAD_MAX_PACKED, + target_mbx, + cmh_sm4_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_out: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_aad: + if (req->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, req->assoclen, + DMA_TO_DEVICE); +out_free_aad: + kfree(rctx->aad_buf); + return ret; +} + +static int cmh_sm4_aead_encrypt(struct aead_request *req) +{ + return cmh_sm4_aead_crypt(req, SM4_OP_ENCRYPT); +} + +static int cmh_sm4_aead_decrypt(struct aead_request *req) +{ + return cmh_sm4_aead_crypt(req, SM4_OP_DECRYPT); +} + +/* Registration */ + +static struct cmh_sm4_aead_drv sm4_aead_drv_algs[ARRAY_SIZE(sm4_aead_algs)]; + +/** + * cmh_sm4_aead_register() - Register SM4-GCM/CCM AEAD algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_sm4_aead_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(sm4_aead_algs); i++) { + const struct cmh_sm4_aead_info *info = &sm4_aead_algs[i]; + struct cmh_sm4_aead_drv *drv = &sm4_aead_drv_algs[i]; + struct aead_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->setkey = cmh_sm4_aead_setkey; + alg->setauthsize = cmh_sm4_aead_setauthsize; + alg->encrypt = cmh_sm4_aead_encrypt; + alg->decrypt = cmh_sm4_aead_decrypt; + alg->init = cmh_sm4_aead_init_tfm; + alg->exit = cmh_sm4_aead_exit_tfm; + alg->ivsize = info->ivsize; + alg->maxauthsize = info->maxauthsize; + + strscpy(alg->base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + alg->base.cra_blocksize = 1; + alg->base.cra_ctxsize = sizeof(struct cmh_sm4_aead_tfm_ctx); + alg->base.cra_module = THIS_MODULE; + + ret = crypto_register_aead(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_sm4_aead: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_sm4_aead: registered %s\n", info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_aead(&sm4_aead_drv_algs[i].alg); + return ret; +} + +/** + * cmh_sm4_aead_unregister() - Unregister SM4 AEAD algorithms from the crypto framework + */ +void cmh_sm4_aead_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sm4_aead_algs); i++) { + crypto_unregister_aead(&sm4_aead_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_sm4_aead: unregistered %s\n", + sm4_aead_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/cmh_sm4_cmac.c b/drivers/crypto/cmh/cmh_sm4_cmac.c new file mode 100644 index 00000000000000..4ebeea445eb6c0 --- /dev/null +++ b/drivers/crypto/cmh/cmh_sm4_cmac.c @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API SM4-CMAC / SM4-XCBC (ahash) Driver + * + * Registers cmac(sm4) and xcbc(sm4) as ahash algorithms. + * + * Both produce a 16-byte tag (MAC) from a key and message. + * VCQ sequence: [SYS_CMD_WRITE] + SM4_CMD_INIT(CMAC/XCBC) + + * SM4_CMD_AAD_FINAL + SM4_CMD_FINAL + FLUSH + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_sm4.h" +#include "cmh_vcq.h" +#include "cmh_sm4_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define SM4_MAC_DIGEST_SIZE 16U +#define SM4_MAC_BLOCK_SIZE 16U +/* + * Maximum accumulated data for SM4 MAC -- driver-imposed, not HW. + * + * The SM4 core does not expose external save/restore VCQ commands, + * so the driver must accumulate all data in kernel memory via + * .update() and submit it atomically in .final(). This cap limits + * the per-request kernel allocation. + */ +#define SM4_MAC_MAX_DATA (64 * 1024) + +struct cmh_sm4_mac_alg_info { + u32 sm4_mode; /* SM4_MODE_CMAC or SM4_MODE_XCBC */ + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_sm4_mac_alg_info sm4_mac_algs[] = { + { SM4_MODE_CMAC, "cmac(sm4)", "cri-cmh-cmac-sm4" }, + { SM4_MODE_XCBC, "xcbc(sm4)", "cri-cmh-xcbc-sm4" }, +}; + +struct cmh_sm4_mac_tfm_ctx { + struct cmh_key_ctx key; + u32 sm4_mode; + struct crypto_cipher *sw_cipher; /* empty-input fallback (CMAC/XCBC) */ + /* Cached subkeys (derived at setkey time for concurrency safety) */ + u8 xcbc_k1[CMH_SM4_BLOCK_SIZE]; /* K1 = E(K, 0x01..01) */ + u8 xcbc_k3[CMH_SM4_BLOCK_SIZE]; /* K3 = E(K, 0x03..03) */ + u8 cmac_k2[CMH_SM4_BLOCK_SIZE]; /* K2 = dbl(dbl(E(K, 0))) */ + bool subkeys_valid; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* Per-request context (lives in ahash_request::__ctx) */ +/* Chunk node for O(1) update() appends */ +struct cmh_sm4_mac_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* Per-request context (lives in ahash_request::__ctx) */ + +#define CMH_SM4_MAC_MAX_PAYLOAD 5 +#define CMH_SM4_MAC_MAX_PACKED (CMH_SM4_MAC_MAX_PAYLOAD * 2) + +struct cmh_sm4_mac_reqctx { + struct list_head chunks; + u32 total_len; + u8 *buf; /* linearised in final() */ + /* DMA state for async final */ + dma_addr_t key_dma; + dma_addr_t in_dma; + dma_addr_t tag_dma; + u8 *tag_buf; + u32 keylen; + struct vcq_cmd packed[CMH_SM4_MAC_MAX_PACKED]; +}; + +/* Flat state for export/import -- holds accumulated input data only */ +struct cmh_sm4_mac_export_state { + u32 total_len; + u8 data[]; +}; + +/* + * Flat state buffer for export/import. The CMH SM4 core does not + * support save/restore of intermediate MAC state, so this driver + * accumulates input in SW and serialises the buffer on export. + * + * PAGE_SIZE (4096) caps the exportable accumulated-data window. + * Full-range export is not feasible because the crypto subsystem + * pre-allocates statesize bytes per request. Export returns -EINVAL + * if the caller has accumulated more than CMH_SM4_MAC_EXPORT_MAX. + */ +#define CMH_SM4_MAC_STATE_SIZE 4096 +#define CMH_SM4_MAC_EXPORT_MAX \ + (CMH_SM4_MAC_STATE_SIZE - sizeof(struct cmh_sm4_mac_export_state)) + +struct cmh_sm4_mac_drv { + struct ahash_alg alg; + const struct cmh_sm4_mac_alg_info *info; +}; + +/* + * GF(2^128) doubling used to derive the CMAC subkeys (NIST SP 800-38B). + * Shift the 128-bit big-endian value left by one bit and, if the top bit + * was set, reduce with Rb = 0x87. + */ +static void cmh_sm4_cmac_dbl(u8 out[CMH_SM4_BLOCK_SIZE], + const u8 in[CMH_SM4_BLOCK_SIZE]) +{ + u8 carry = in[0] >> 7; + unsigned int i; + + for (i = 0; i < CMH_SM4_BLOCK_SIZE - 1; i++) + out[i] = (in[i] << 1) | (in[i + 1] >> 7); + out[CMH_SM4_BLOCK_SIZE - 1] = (in[CMH_SM4_BLOCK_SIZE - 1] << 1) ^ + (carry ? 0x87 : 0x00); +} + +static int cmh_sm4_mac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + int ret; + + if (keylen != CMH_SM4_KEY_SIZE) + return -EINVAL; + + if (tctx->sw_cipher && tctx->sm4_mode == SM4_MODE_XCBC) { + u8 const1[CMH_SM4_BLOCK_SIZE], const3[CMH_SM4_BLOCK_SIZE]; + + ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen); + if (ret) + return ret; + + /* Pre-derive XCBC subkeys for concurrent-safe final() */ + memset(const1, 0x01, CMH_SM4_BLOCK_SIZE); + memset(const3, 0x03, CMH_SM4_BLOCK_SIZE); + crypto_cipher_encrypt_one(tctx->sw_cipher, tctx->xcbc_k1, + const1); + crypto_cipher_encrypt_one(tctx->sw_cipher, tctx->xcbc_k3, + const3); + + /* + * Leave sw_cipher keyed with K1 permanently. + * final() only needs E(K1, block) and never touches the + * original key again, so no re-keying in the hot path + * eliminates the per-tfm concurrency race entirely. + */ + ret = crypto_cipher_setkey(tctx->sw_cipher, tctx->xcbc_k1, + CMH_SM4_BLOCK_SIZE); + if (ret) + return ret; + } else if (tctx->sw_cipher && tctx->sm4_mode == SM4_MODE_CMAC) { + u8 zero[CMH_SM4_BLOCK_SIZE] = { 0 }; + u8 l[CMH_SM4_BLOCK_SIZE], k1[CMH_SM4_BLOCK_SIZE]; + + ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen); + if (ret) + return ret; + + /* + * Pre-derive the CMAC subkey K2 for the empty-message + * fallback (NIST SP 800-38B): + * L = E(K, 0^128); K1 = dbl(L); K2 = dbl(K1) + * sw_cipher is left keyed with the original K, so final() + * computes E(K, K2 ^ pad) with no hot-path re-keying. + */ + crypto_cipher_encrypt_one(tctx->sw_cipher, l, zero); + cmh_sm4_cmac_dbl(k1, l); + cmh_sm4_cmac_dbl(tctx->cmac_k2, k1); + memzero_explicit(l, sizeof(l)); + memzero_explicit(k1, sizeof(k1)); + } + + ret = cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_SM4); + if (ret) + return ret; + + if (tctx->sw_cipher) + tctx->subkeys_valid = true; + + return 0; +} + +static void cmh_sm4_mac_free_chunks(struct cmh_sm4_mac_reqctx *rctx, + struct cmh_sm4_mac_tfm_ctx *tctx) +{ + struct cmh_sm4_mac_chunk *c, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &rctx->chunks, list) { + list_del(&c->list); + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); +} + +static int cmh_sm4_mac_init(struct ahash_request *req) +{ + struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + return 0; +} + +static int cmh_sm4_mac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_sm4_mac_chunk *chunk; + gfp_t gfp; + int ret; + + if (!req->nbytes) + return 0; + + if (req->nbytes > SM4_MAC_MAX_DATA - rctx->total_len) { + ret = -EINVAL; + goto err_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp); + if (!chunk) { + ret = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(chunk->data, req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(chunk->data, req->src, + 0, req->nbytes, 0); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len += req->nbytes; + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * callers may not call .final() on error, so they would leak. + */ + cmh_sm4_mac_free_chunks(rctx, tctx); + return ret; +} + +static void cmh_sm4_mac_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->total_len > 0) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, SM4_MAC_DIGEST_SIZE, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->tag_buf, SM4_MAC_DIGEST_SIZE); + + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + cmh_sm4_mac_free_chunks(rctx, tctx); + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + rctx->total_len = 0; + cmh_complete(&req->base, error); +} + +static int cmh_sm4_mac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_SM4_MAC_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) { + ret = -ENOKEY; + goto out_free_chunks; + } + + /* + * XCBC empty-input SW fallback (RFC 3566). + * + * For a zero-length message: + * K1 = E(K, 0x01010101...) -- encryption subkey + * K3 = E(K, 0x03030303...) -- incomplete-block subkey + * pad = 0x80 00...00 -- single 1 bit + 127 zero bits + * tag = E(K1, pad XOR K3) + * + * The eSW produces incorrect output for this case, so the driver + * computes it synchronously using crypto_cipher. + * + * For DS keys we cannot derive subkeys (no raw key material), + * and the HW also cannot handle empty XCBC correctly, so + * return -EOPNOTSUPP. + */ + if (rctx->total_len == 0 && tctx->sm4_mode == SM4_MODE_XCBC) { + u8 block[CMH_SM4_BLOCK_SIZE]; + u32 i; + + if (tctx->key.mode != CMH_KEY_RAW || + !tctx->subkeys_valid) { + cmh_sm4_mac_free_chunks(rctx, tctx); + return -EOPNOTSUPP; + } + + /* block = pad XOR K3 */ + memset(block, 0, CMH_SM4_BLOCK_SIZE); + block[0] = 0x80; + for (i = 0; i < CMH_SM4_BLOCK_SIZE; i++) + block[i] ^= tctx->xcbc_k3[i]; + + /* + * tag = E(K1, block) + * + * sw_cipher is permanently keyed with K1 (set at setkey + * time), so this is safe for concurrent requests sharing + * the same tfm -- no re-keying, no race. + */ + crypto_cipher_encrypt_one(tctx->sw_cipher, req->result, + block); + + cmh_sm4_mac_free_chunks(rctx, tctx); + return 0; + } + + /* + * CMAC empty-input SW fallback (NIST SP 800-38B). + * + * For a zero-length message the sole block is incomplete, so the + * K2 subkey is used: + * pad = 0x80 00...00 -- single 1 bit + 127 zero bits + * tag = E(K, pad XOR K2) + * + * The eSW produces incorrect output for this case, so the driver + * computes it synchronously using crypto_cipher. + * + * For DS keys we cannot derive subkeys (no raw key material), + * and the HW also cannot handle empty CMAC correctly, so + * return -EOPNOTSUPP. + */ + if (rctx->total_len == 0 && tctx->sm4_mode == SM4_MODE_CMAC) { + u8 block[CMH_SM4_BLOCK_SIZE]; + u32 i; + + if (tctx->key.mode != CMH_KEY_RAW || !tctx->subkeys_valid) { + cmh_sm4_mac_free_chunks(rctx, tctx); + return -EOPNOTSUPP; + } + + /* block = pad XOR K2 */ + memset(block, 0, CMH_SM4_BLOCK_SIZE); + block[0] = 0x80; + for (i = 0; i < CMH_SM4_BLOCK_SIZE; i++) + block[i] ^= tctx->cmac_k2[i]; + + /* + * tag = E(K, block). sw_cipher is keyed with the original + * key K (set at setkey time, never re-keyed), so this is + * safe for concurrent requests sharing the same tfm. + */ + crypto_cipher_encrypt_one(tctx->sw_cipher, req->result, + block); + + cmh_sm4_mac_free_chunks(rctx, tctx); + return 0; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Linearise chunks into a single contiguous buffer for DMA */ + if (rctx->total_len > 0) { + struct cmh_sm4_mac_chunk *c; + u32 off = 0; + + rctx->buf = kmalloc(rctx->total_len, gfp); + if (!rctx->buf) { + ret = -ENOMEM; + goto out_free_chunks; + } + list_for_each_entry(c, &rctx->chunks, list) { + memcpy(rctx->buf + off, c->data, c->len); + off += c->len; + } + } + + rctx->tag_buf = kzalloc(SM4_MAC_DIGEST_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_free_buf; + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, + SM4_MAC_DIGEST_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + if (rctx->total_len > 0) { + rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_unmap_tag; + } + } + + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_SM4); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* + * INIT: mode=CMAC or XCBC + * CMAC/XCBC data goes through the AAD path: + * aadlen = total data length, iolen = 0 + */ + { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_INIT); + slot->hwc.sm4.cmd_init.key = key_ref; + slot->hwc.sm4.cmd_init.iv = 0; + slot->hwc.sm4.cmd_init.keylen = keylen; + slot->hwc.sm4.cmd_init.ivlen = 0; + slot->hwc.sm4.cmd_init.mode = tctx->sm4_mode; + slot->hwc.sm4.cmd_init.op = SM4_OP_ENCRYPT; + slot->hwc.sm4.cmd_init.aadlen = rctx->total_len; + slot->hwc.sm4.cmd_init.iolen = 0; + } + + /* AAD_FINAL: send data through the AAD path */ + if (rctx->total_len > 0) { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_AAD_FINAL); + slot->hwc.sm4.cmd_aad_final.data = (u64)rctx->in_dma; + slot->hwc.sm4.cmd_aad_final.datalen = rctx->total_len; + } + + /* FINAL: tag extraction only (no data) */ + { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_FINAL); + slot->hwc.sm4.cmd_final.input = 0; + slot->hwc.sm4.cmd_final.output = 0; + slot->hwc.sm4.cmd_final.tag = (u64)rctx->tag_dma; + slot->hwc.sm4.cmd_final.iolen = 0; + slot->hwc.sm4.cmd_final.taglen = SM4_MAC_DIGEST_SIZE; + } + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM4_MAC_MAX_PACKED, + target_mbx, + cmh_sm4_mac_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->total_len > 0 && !cmh_dma_map_error(rctx->in_dma)) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, SM4_MAC_DIGEST_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_free_buf: + kfree_sensitive(rctx->buf); + rctx->buf = NULL; +out_free_chunks: + cmh_sm4_mac_free_chunks(rctx, tctx); + rctx->total_len = 0; + return ret; +} + +/* + * ahash .export()/.import(): serialize/deserialize the software + * accumulation buffer. No HW state is involved. + */ + +static int cmh_sm4_mac_export(struct ahash_request *req, void *out) +{ + struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_sm4_mac_export_state *state = out; + struct cmh_sm4_mac_chunk *chunk; + u32 offset = 0; + + if (rctx->total_len > CMH_SM4_MAC_EXPORT_MAX) + return -ENOSPC; + + state->total_len = rctx->total_len; + list_for_each_entry(chunk, &rctx->chunks, list) { + memcpy(state->data + offset, chunk->data, chunk->len); + offset += chunk->len; + } + return 0; +} + +static int cmh_sm4_mac_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_sm4_mac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_sm4_mac_export_state *state = in; + struct cmh_sm4_mac_chunk *chunk; + + /* + * Do NOT call free_chunks() here: the crypto API does not + * guarantee the request context is in a valid state before + * import(), so the list pointers may be stale or invalid. + * Re-initialize from scratch instead. Any pre-existing chunks + * are tracked on tctx->all_chunks and freed in exit_tfm. + */ + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + + if (state->total_len > CMH_SM4_MAC_EXPORT_MAX) + return -EINVAL; + + if (state->total_len) { + chunk = kmalloc(sizeof(*chunk) + state->total_len, GFP_KERNEL); + if (!chunk) + return -ENOMEM; + chunk->len = state->total_len; + memcpy(chunk->data, state->data, state->total_len); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len = state->total_len; + } + return 0; +} + +static int cmh_sm4_mac_finup(struct ahash_request *req) +{ + int err; + + err = cmh_sm4_mac_update(req); + if (err) + return err; + return cmh_sm4_mac_final(req); +} + +static int cmh_sm4_mac_digest(struct ahash_request *req) +{ + int err; + + err = cmh_sm4_mac_init(req); + if (err) + return err; + return cmh_sm4_mac_finup(req); +} + +/* Registration */ + +static struct cmh_sm4_mac_drv sm4_mac_drv_algs[ARRAY_SIZE(sm4_mac_algs)]; + +static int cmh_sm4_mac_init_tfm(struct crypto_ahash *tfm) +{ + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct cmh_sm4_mac_drv *drv = + container_of(alg, struct cmh_sm4_mac_drv, alg); + + memset(tctx, 0, sizeof(*tctx)); + tctx->sm4_mode = drv->info->sm4_mode; + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + + /* Allocate SW cipher for the CMAC/XCBC empty-input fallback */ + if (tctx->sm4_mode == SM4_MODE_XCBC || + tctx->sm4_mode == SM4_MODE_CMAC) { + struct crypto_cipher *ci; + + ci = crypto_alloc_cipher("sm4", 0, 0); + if (IS_ERR(ci)) + return PTR_ERR(ci); + tctx->sw_cipher = ci; + } + + crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_sm4_mac_reqctx)); + return 0; +} + +static void cmh_sm4_mac_exit_tfm(struct crypto_ahash *tfm) +{ + struct cmh_sm4_mac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_sm4_mac_chunk *c, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) { + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + + if (tctx->sw_cipher) + crypto_free_cipher(tctx->sw_cipher); + memzero_explicit(tctx->xcbc_k1, sizeof(tctx->xcbc_k1)); + memzero_explicit(tctx->xcbc_k3, sizeof(tctx->xcbc_k3)); + memzero_explicit(tctx->cmac_k2, sizeof(tctx->cmac_k2)); + cmh_key_destroy(&tctx->key); +} + +/** + * cmh_sm4_cmac_register() - Register SM4-CMAC/XCBC hash algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_sm4_cmac_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(sm4_mac_algs); i++) { + const struct cmh_sm4_mac_alg_info *info = &sm4_mac_algs[i]; + struct cmh_sm4_mac_drv *drv = &sm4_mac_drv_algs[i]; + struct ahash_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->init = cmh_sm4_mac_init; + alg->update = cmh_sm4_mac_update; + alg->final = cmh_sm4_mac_final; + alg->finup = cmh_sm4_mac_finup; + alg->digest = cmh_sm4_mac_digest; + alg->export = cmh_sm4_mac_export; + alg->import = cmh_sm4_mac_import; + alg->setkey = cmh_sm4_mac_setkey; + alg->init_tfm = cmh_sm4_mac_init_tfm; + alg->exit_tfm = cmh_sm4_mac_exit_tfm; + + alg->halg.digestsize = SM4_MAC_DIGEST_SIZE; + alg->halg.statesize = CMH_SM4_MAC_STATE_SIZE; + + strscpy(alg->halg.base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->halg.base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->halg.base.cra_priority = 300; + alg->halg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT; + alg->halg.base.cra_blocksize = SM4_MAC_BLOCK_SIZE; + alg->halg.base.cra_ctxsize = sizeof(struct cmh_sm4_mac_tfm_ctx); + alg->halg.base.cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_sm4_mac: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_sm4_mac: registered %s\n", + info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_ahash(&sm4_mac_drv_algs[i].alg); + return ret; +} + +/** + * cmh_sm4_cmac_unregister() - Unregister SM4 MAC hash algorithms from the crypto framework + */ +void cmh_sm4_cmac_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sm4_mac_algs); i++) { + crypto_unregister_ahash(&sm4_mac_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_sm4_mac: unregistered %s\n", + sm4_mac_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/cmh_sm4_skcipher.c b/drivers/crypto/cmh/cmh_sm4_skcipher.c new file mode 100644 index 00000000000000..8cd76cba92359a --- /dev/null +++ b/drivers/crypto/cmh/cmh_sm4_skcipher.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API SM4 (skcipher) Driver + * + * Registers skcipher algorithms with the Linux crypto subsystem: + * ecb(sm4), cbc(sm4), ctr(sm4), cfb(sm4), xts(sm4) + * + * Uses the CMH SM4 Core via VCQ commands: + * [SYS_CMD_WRITE] + SM4_CMD_INIT + SM4_CMD_FINAL + VCQ_CMD_FLUSH + * + * The SM4 core requires bidirectional DMA -- both input and output + * buffers are mapped and passed in a single SM4_CMD_FINAL command. + * + * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into + * the same VCQ as SM4 commands (see cmh_key.h for details). + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_sm4.h" +#include "cmh_vcq.h" +#include "cmh_sm4_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* Algorithm Table */ + +struct cmh_sm4_alg_info { + u32 sm4_mode; /* SM4_MODE_* */ + u32 ivsize; /* bytes (0 for ECB) */ + u32 min_keysize; + u32 max_keysize; + const char *alg_name; /* Linux crypto name: "ecb(sm4)" */ + const char *drv_name; /* driver name: "cri-cmh-ecb-sm4" */ +}; + +static const struct cmh_sm4_alg_info sm4_algs[] = { + { SM4_MODE_ECB, 0, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE, + "ecb(sm4)", "cri-cmh-ecb-sm4" }, + { SM4_MODE_CBC, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE, + "cbc(sm4)", "cri-cmh-cbc-sm4" }, + { SM4_MODE_CTR, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE, + "ctr(sm4)", "cri-cmh-ctr-sm4" }, + { SM4_MODE_CFB, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE, CMH_SM4_KEY_SIZE, + "cfb(sm4)", "cri-cmh-cfb-sm4" }, + { SM4_MODE_XTS, CMH_SM4_IV_SIZE, CMH_SM4_KEY_SIZE * 2, + CMH_SM4_KEY_SIZE * 2, + "xts(sm4)", "cri-cmh-xts-sm4" }, +}; + +/* Per-transform context (allocated by crypto framework) */ + +struct cmh_sm4_tfm_ctx { + struct cmh_key_ctx key; +}; + +/* Per-request context (lives in skcipher_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + SM4_CMD_INIT + [SM4_CMD_UPDATE] + SM4_CMD_FINAL + * + VCQ_CMD_FLUSH = 5 + * UPDATE is used for XTS data > 2 blocks (see cmh_sm4_crypt). + */ +#define CMH_SM4_MAX_PAYLOAD 5 +#define CMH_SM4_MAX_PACKED (CMH_SM4_MAX_PAYLOAD * 2) + +struct cmh_sm4_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t iv2_dma; + dma_addr_t key_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *iv2_buf; + u32 cryptlen; + u32 ivsize; + u32 keylen; + u32 sm4_mode; + u32 sm4_op; + /* CTR counter-wrap split state */ + u32 ctr_chunk1_len; + u32 core_id; + s32 target_mbx; + u64 key_ref; + struct vcq_cmd packed[CMH_SM4_MAX_PACKED]; +}; + +/* VCQ Builders -- SM4-specific */ + +static void vcq_add_sm4_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u64 iv_dma, + u32 keylen, u32 ivlen, u32 mode, u32 op, + u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_INIT); + slot->hwc.sm4.cmd_init.key = key_ref; + slot->hwc.sm4.cmd_init.iv = iv_dma; + slot->hwc.sm4.cmd_init.keylen = keylen; + slot->hwc.sm4.cmd_init.ivlen = ivlen; + slot->hwc.sm4.cmd_init.mode = mode; + slot->hwc.sm4.cmd_init.op = op; + slot->hwc.sm4.cmd_init.aadlen = 0; + slot->hwc.sm4.cmd_init.iolen = iolen; +} + +static void vcq_add_sm4_update(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_UPDATE); + slot->hwc.sm4.cmd_update.input = input_dma; + slot->hwc.sm4.cmd_update.output = output_dma; + slot->hwc.sm4.cmd_update.iolen = iolen; +} + +static void vcq_add_sm4_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, SM4_CMD_FINAL); + slot->hwc.sm4.cmd_final.input = input_dma; + slot->hwc.sm4.cmd_final.output = output_dma; + slot->hwc.sm4.cmd_final.iolen = iolen; + slot->hwc.sm4.cmd_final.tag = 0; + slot->hwc.sm4.cmd_final.taglen = 0; +} + +/* + * We wrap each skcipher_alg with its info pointer in a compound struct, + * then use container_of() in cmh_sm4_get_info() to recover it. + */ +struct cmh_sm4_alg_drv { + struct skcipher_alg alg; + const struct cmh_sm4_alg_info *info; +}; + +static bool sm4_is_stream_mode(u32 mode) +{ + return mode == SM4_MODE_CTR || mode == SM4_MODE_CFB; +} + +/* + * Update req->iv after a successful encrypt/decrypt. + * Same semantics as cmh_aes_update_iv -- see cmh_aes.c. + */ +static void cmh_sm4_update_iv(struct skcipher_request *req, u32 mode, + u32 op, const u8 *in_buf, const u8 *out_buf) +{ + u32 bs = CMH_SM4_BLOCK_SIZE; + u32 nblocks; + + switch (mode) { + case SM4_MODE_CBC: + if (op == SM4_OP_ENCRYPT) + memcpy(req->iv, out_buf + req->cryptlen - bs, bs); + else + memcpy(req->iv, in_buf + req->cryptlen - bs, bs); + break; + case SM4_MODE_CTR: + /* Arithmetic big-endian 128-bit counter increment */ + nblocks = DIV_ROUND_UP(req->cryptlen, bs); + { + u8 *iv = req->iv; + int i; + + for (i = bs - 1; i >= 0 && nblocks; i--) { + u32 sum = (u32)iv[i] + (nblocks & 0xff); + + iv[i] = (u8)sum; + nblocks = (nblocks >> 8) + (sum >> 8); + } + } + break; + case SM4_MODE_CFB: + /* + * For sub-block requests (cryptlen < 16), there is no + * complete ciphertext block to chain, so the IV is left + * unchanged -- CFB-128 has no defined chaining semantic + * for partial blocks (shift-register CFB-n is a different + * mode). Without this guard the pointer arithmetic + * underflows and reads before the buffer. + */ + if (req->cryptlen >= bs) { + if (op == SM4_OP_ENCRYPT) + memcpy(req->iv, out_buf + req->cryptlen - bs, + bs); + else + memcpy(req->iv, in_buf + req->cryptlen - bs, + bs); + } + break; + default: + break; + } +} + +/* skcipher Operations */ + +static const struct cmh_sm4_alg_info * +cmh_sm4_get_info(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + return container_of(alg, struct cmh_sm4_alg_drv, alg)->info; +} + +static int cmh_sm4_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + const struct cmh_sm4_alg_info *info = cmh_sm4_get_info(tfm); + + if (info->sm4_mode == SM4_MODE_XTS) { + int err; + + /* XTS: double key (32 bytes) */ + if (keylen != CMH_SM4_KEY_SIZE * 2) + return -EINVAL; + err = xts_verify_key(tfm, key, keylen); + if (err) + return err; + } else { + /* SM4 always uses 128-bit (16-byte) keys */ + if (keylen != CMH_SM4_KEY_SIZE) + return -EINVAL; + } + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_SM4); +} + +static int cmh_sm4_init_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_sm4_reqctx)); + return 0; +} + +static void cmh_sm4_exit_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +#define CMH_SM4_MAX_CRYPTLEN SZ_32M + +/* DMA unmap helper */ +static void cmh_sm4_unmap_dma(struct cmh_sm4_reqctx *rctx) +{ + if (rctx->iv2_buf) + cmh_dma_unmap_single(rctx->iv2_dma, rctx->ivsize, + DMA_TO_DEVICE); + if (rctx->ivsize > 0) + cmh_dma_unmap_single(rctx->iv_dma, rctx->ivsize, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE); +} + +static void cmh_sm4_free_bufs(struct cmh_sm4_reqctx *rctx) +{ + kfree(rctx->iv2_buf); + rctx->iv2_buf = NULL; + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; +} + +/* + * Submit the second CTR chunk after the first completes. + * Called from cmh_sm4_complete when ctr_chunk1_len > 0. + */ +static int cmh_sm4_ctr_submit_chunk2(struct skcipher_request *req); + +static void cmh_sm4_complete(void *data, int error) +{ + struct skcipher_request *req = data; + struct cmh_sm4_reqctx *rctx = skcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* + * CTR counter-wrap: first chunk completed, submit second. + * DMA mappings remain valid (they cover the full buffer). + * + * Recursion depth bounded: chunk2 clears ctr_chunk1_len before + * submission, so the second cmh_sm4_complete invocation sees 0 + * and finalizes (max depth = 2). + */ + if (rctx->ctr_chunk1_len && !error) { + int ret = cmh_sm4_ctr_submit_chunk2(req); + + if (!ret || ret == -EBUSY) + return; + /* Submission failed; clean up below */ + error = ret; + } + + cmh_sm4_unmap_dma(rctx); + + if (!error) { + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + 0, rctx->cryptlen, 1); + cmh_sm4_update_iv(req, rctx->sm4_mode, rctx->sm4_op, + rctx->in_buf, rctx->out_buf); + } + + cmh_sm4_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +static int cmh_sm4_ctr_submit_chunk2(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct cmh_sm4_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_SM4_MAX_PAYLOAD]; + u32 chunk1 = rctx->ctr_chunk1_len; + u32 chunk2 = rctx->cryptlen - chunk1; + u64 key_ref; + u32 keylen; + u32 idx = 0; + + /* Clear split flag so next completion is final */ + rctx->ctr_chunk1_len = 0; + + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + + vcq_add_sm4_init(&cmds[idx++], rctx->core_id, key_ref, + (u64)rctx->iv2_dma, keylen, rctx->ivsize, + rctx->sm4_mode, rctx->sm4_op, chunk2); + vcq_add_sm4_final(&cmds[idx++], rctx->core_id, + (u64)(rctx->in_dma + chunk1), + (u64)(rctx->out_dma + chunk1), chunk2); + vcq_add_flush(&cmds[idx++], rctx->core_id); + + return cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM4_MAX_PACKED, + rctx->target_mbx, + cmh_sm4_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); +} + +static int cmh_sm4_crypt(struct skcipher_request *req, u32 sm4_op) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_sm4_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + const struct cmh_sm4_alg_info *info = cmh_sm4_get_info(tfm); + struct cmh_sm4_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_SM4_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + if (!req->cryptlen) + return 0; + + if (req->cryptlen > CMH_SM4_MAX_CRYPTLEN) + return -EINVAL; + + switch (info->sm4_mode) { + case SM4_MODE_CTR: + case SM4_MODE_CFB: + break; + case SM4_MODE_XTS: + if (req->cryptlen < CMH_SM4_BLOCK_SIZE) + return -EINVAL; + break; + default: + if (req->cryptlen & (CMH_SM4_BLOCK_SIZE - 1)) + return -EINVAL; + break; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = req->cryptlen; + rctx->ivsize = info->ivsize; + rctx->sm4_mode = info->sm4_mode; + rctx->sm4_op = sm4_op; + rctx->iv2_buf = NULL; + + rctx->in_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->in_buf) + return -ENOMEM; + + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0); + + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + + rctx->out_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + + if (info->ivsize > 0) { + rctx->iv_buf = kmemdup(req->iv, info->ivsize, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, info->ivsize, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + } + + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_SM4); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* + * iolen in INIT: passed for all modes. The EIP-40 eSW ignores + * it for CTR (stream cipher), but uses it for XTS/CBC/ECB to + * know the total data length. Pass cryptlen unconditionally. + */ + vcq_add_sm4_init(&cmds[idx++], core_id, key_ref, (u64)rctx->iv_dma, + keylen, info->ivsize, info->sm4_mode, sm4_op, + req->cryptlen); + + if (info->sm4_mode == SM4_MODE_XTS && + req->cryptlen > 2 * CMH_SM4_BLOCK_SIZE) { + u32 final_len, update_len; + + if (req->cryptlen & (CMH_SM4_BLOCK_SIZE - 1)) + final_len = CMH_SM4_BLOCK_SIZE + + (req->cryptlen & (CMH_SM4_BLOCK_SIZE - 1)); + else + final_len = 2 * CMH_SM4_BLOCK_SIZE; + + update_len = req->cryptlen - final_len; + + vcq_add_sm4_update(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, update_len); + vcq_add_sm4_final(&cmds[idx++], core_id, + (u64)(rctx->in_dma + update_len), + (u64)(rctx->out_dma + update_len), + final_len); + } else if (info->sm4_mode == SM4_MODE_CTR) { + /* + * CTR counter-wrap: split at the 64-bit boundary, + * consistent with the AES-SCA driver. The completion + * callback submits chunk2 with IV = {upper64+1, 0}. + */ + u64 lower64 = get_unaligned_be64(rctx->iv_buf + 8); + u32 nblocks = DIV_ROUND_UP(req->cryptlen, + CMH_SM4_BLOCK_SIZE); + u64 bwrap = lower64 ? (~lower64 + 1ULL) : U64_MAX; + + if (nblocks > bwrap) { + u32 chunk1 = (u32)bwrap * CMH_SM4_BLOCK_SIZE; + u64 upper64; + + /* Prepare second IV for chained submission */ + rctx->iv2_buf = kmalloc(info->ivsize, gfp); + if (!rctx->iv2_buf) { + ret = -ENOMEM; + goto out_unmap_iv; + } + upper64 = get_unaligned_be64(rctx->iv_buf); + put_unaligned_be64(upper64 + 1, rctx->iv2_buf); + put_unaligned_be64(0, rctx->iv2_buf + 8); + + rctx->iv2_dma = + cmh_dma_map_single(rctx->iv2_buf, + info->ivsize, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv2_dma)) { + ret = -ENOMEM; + goto out_free_iv2; + } + + /* Store state for the chained second submission */ + rctx->ctr_chunk1_len = chunk1; + rctx->core_id = core_id; + rctx->target_mbx = target_mbx; + rctx->key_ref = key_ref; + + /* First transaction: only chunk1 */ + vcq_add_sm4_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, chunk1); + } else { + /* No wrap: single FINAL with all data */ + vcq_add_sm4_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, + req->cryptlen); + } + } else { + vcq_add_sm4_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, req->cryptlen); + } + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_SM4_MAX_PACKED, target_mbx, + cmh_sm4_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->iv2_buf) { + cmh_dma_unmap_single(rctx->iv2_dma, info->ivsize, + DMA_TO_DEVICE); + } +out_free_iv2: + kfree(rctx->iv2_buf); +out_unmap_iv: + if (info->ivsize > 0) + cmh_dma_unmap_single(rctx->iv_dma, info->ivsize, + DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_out: + cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); + return ret; +} + +static int cmh_sm4_encrypt(struct skcipher_request *req) +{ + return cmh_sm4_crypt(req, SM4_OP_ENCRYPT); +} + +static int cmh_sm4_decrypt(struct skcipher_request *req) +{ + return cmh_sm4_crypt(req, SM4_OP_DECRYPT); +} + +/* Registration */ + +static struct cmh_sm4_alg_drv sm4_drv_algs[ARRAY_SIZE(sm4_algs)]; + +/** + * cmh_sm4_register() - Register SM4-CBC/CTR/ECB/XTS skcipher algorithms + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_sm4_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(sm4_algs); i++) { + const struct cmh_sm4_alg_info *info = &sm4_algs[i]; + struct cmh_sm4_alg_drv *drv = &sm4_drv_algs[i]; + struct skcipher_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->setkey = cmh_sm4_setkey; + alg->encrypt = cmh_sm4_encrypt; + alg->decrypt = cmh_sm4_decrypt; + alg->init = cmh_sm4_init_tfm; + alg->exit = cmh_sm4_exit_tfm; + alg->min_keysize = info->min_keysize; + alg->max_keysize = info->max_keysize; + alg->ivsize = info->ivsize; + + strscpy(alg->base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + alg->base.cra_blocksize = sm4_is_stream_mode(info->sm4_mode) + ? 1 : CMH_SM4_BLOCK_SIZE; + alg->base.cra_ctxsize = sizeof(struct cmh_sm4_tfm_ctx); + alg->base.cra_module = THIS_MODULE; + + ret = crypto_register_skcipher(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_sm4: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_sm4: registered %s\n", info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_skcipher(&sm4_drv_algs[i].alg); + return ret; +} + +/** + * cmh_sm4_unregister() - Unregister SM4 skcipher algorithms from the crypto framework + */ +void cmh_sm4_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sm4_algs); i++) { + crypto_unregister_skcipher(&sm4_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_sm4: unregistered %s\n", sm4_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/include/cmh_sm4.h b/drivers/crypto/cmh/include/cmh_sm4.h new file mode 100644 index 00000000000000..9f4b0fb918dbc7 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_sm4.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SM4 Crypto API Drivers + * + * Registers SM4 algorithms with the Linux crypto subsystem: + * skcipher: ecb/cbc/ctr/cfb/xts(sm4) + * aead: gcm/ccm(sm4) + * shash: cmac/xcbc(sm4) + */ + +#ifndef CMH_SM4_H +#define CMH_SM4_H + +int cmh_sm4_register(void); +void cmh_sm4_unregister(void); + +int cmh_sm4_aead_register(void); +void cmh_sm4_aead_unregister(void); + +int cmh_sm4_cmac_register(void); +void cmh_sm4_cmac_unregister(void); + +#endif /* CMH_SM4_H */ From 7ecaf84814eb1c5b02c5da092604201dc2e47b65 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:28 -0700 Subject: [PATCH 12/21] crypto: cmh - add ChaCha20-Poly1305 Register ChaCha20-Poly1305 AEAD and ChaCha20 skcipher algorithms using the CMH CCP core (core ID 0x18). Also registers the Poly1305 ahash for standalone use. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 5 +- drivers/crypto/cmh/cmh_ccp.c | 364 +++++++++++++++++ drivers/crypto/cmh/cmh_ccp_aead.c | 583 +++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_ccp_poly.c | 528 ++++++++++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 25 ++ drivers/crypto/cmh/include/cmh_ccp.h | 24 ++ 6 files changed, 1528 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_ccp.c create mode 100644 drivers/crypto/cmh/cmh_ccp_aead.c create mode 100644 drivers/crypto/cmh/cmh_ccp_poly.c create mode 100644 drivers/crypto/cmh/include/cmh_ccp.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 1f36cd9c0b98ad..4ebd0e1d10bcde 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -25,7 +25,10 @@ cmh-y := \ cmh_aes_cmac.o \ cmh_sm4_skcipher.o \ cmh_sm4_aead.o \ - cmh_sm4_cmac.o + cmh_sm4_cmac.o \ + cmh_ccp.o \ + cmh_ccp_aead.o \ + cmh_ccp_poly.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_ccp.c b/drivers/crypto/cmh/cmh_ccp.c new file mode 100644 index 00000000000000..deb1db9200f859 --- /dev/null +++ b/drivers/crypto/cmh/cmh_ccp.c @@ -0,0 +1,364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API ChaCha20 (skcipher) Driver + * + * Registers the "chacha20" skcipher algorithm with the Linux crypto + * subsystem, backed by the CMH CCP core. + * + * VCQ sequence: + * [SYS_CMD_WRITE] + CCP_CMD_CHACHA20_INIT + CCP_CMD_FINAL + CCP_CMD_FLUSH + * + * The CCP core expects a 16-byte counter+nonce (ctrnonce): + * bytes [0..3] = 32-bit LE counter + * bytes [4..15] = 12-byte nonce + * + * The Linux chacha20 skcipher interface passes a 16-byte IV in the + * same format, so we forward it directly. + * + * ChaCha20 is a stream cipher -- arbitrary plaintext lengths are + * supported (no block-alignment requirement). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_ccp.h" +#include "cmh_vcq.h" +#include "cmh_ccp_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* Per-transform context */ + +struct cmh_ccp_tfm_ctx { + struct cmh_key_ctx key; +}; + +/* Per-request context (lives in skcipher_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + CCP_CMD_CHACHA20_INIT + CCP_CMD_FINAL + FLUSH = 4 + */ +#define CMH_CCP_MAX_PAYLOAD 4 +#define CMH_CCP_MAX_PACKED (CMH_CCP_MAX_PAYLOAD * 2) + +struct cmh_ccp_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u32 cryptlen; + u32 keylen; + struct vcq_cmd packed[CMH_CCP_MAX_PACKED]; +}; + +/* VCQ Builders -- ChaCha20-specific */ + +static void vcq_add_ccp_chacha_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 ctrnonce_dma, u32 keylen, u32 op) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_CHACHA20_INIT); + slot->hwc.ccp.cmd_chacha.key = key_ref; + slot->hwc.ccp.cmd_chacha.ctrnonce = ctrnonce_dma; + slot->hwc.ccp.cmd_chacha.keylen = keylen; + slot->hwc.ccp.cmd_chacha.ctrnoncelen = CCP_CTRNONCE_SIZE; + slot->hwc.ccp.cmd_chacha.ctrlen = CCP_CHACHA_CTR_LEN; + slot->hwc.ccp.cmd_chacha.op = op; +} + +static void vcq_add_ccp_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL); + slot->hwc.ccp.cmd_final.input = input_dma; + slot->hwc.ccp.cmd_final.output = output_dma; + slot->hwc.ccp.cmd_final.tag = 0; + slot->hwc.ccp.cmd_final.iolen = iolen; + slot->hwc.ccp.cmd_final.taglen = 0; +} + +/* skcipher Operations */ +static int cmh_ccp_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + /* ChaCha20 requires 32-byte key per RFC 8439 */ + if (keylen != 32) + return -EINVAL; + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_CCP); +} + +static int cmh_ccp_init_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_ccp_reqctx)); + return 0; +} + +static void cmh_ccp_exit_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_ccp_unmap_dma(struct cmh_ccp_reqctx *rctx) +{ + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE); +} + +static void cmh_ccp_free_bufs(struct cmh_ccp_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; +} + +static void cmh_ccp_complete(void *data, int error) +{ + struct skcipher_request *req = data; + struct cmh_ccp_reqctx *rctx = skcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_ccp_unmap_dma(rctx); + + if (!error) { + u32 counter, nblocks; + + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + 0, rctx->cryptlen, 1); + + /* + * Update the 32-bit LE block counter at IV[0..3]. + * ChaCha20 processes 64-byte blocks; the nonce at + * IV[4..15] is unchanged. + */ + counter = get_unaligned_le32(req->iv); + nblocks = DIV_ROUND_UP(rctx->cryptlen, 64); + put_unaligned_le32(counter + nblocks, req->iv); + } + + cmh_ccp_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * Core encrypt/decrypt -- builds a VCQ transaction and submits async. + * + * ChaCha20 is a stream cipher: encrypt and decrypt use the same + * underlying XOR operation. + */ +static int cmh_ccp_crypt(struct skcipher_request *req, u32 ccp_op) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct cmh_ccp_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_CCP_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + if (!req->cryptlen) + return 0; + + /* Limit linearisation buffers to avoid large allocations. */ + if (req->cryptlen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = req->cryptlen; + + /* Linearise input from scatterlist */ + rctx->in_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->in_buf) + return -ENOMEM; + + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0); + + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + + rctx->out_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + + rctx->iv_buf = kmemdup(req->iv, CCP_CTRNONCE_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, CCP_CTRNONCE_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_CCP); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + vcq_add_ccp_chacha_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, ccp_op); + + vcq_add_ccp_final(&cmds[idx++], core_id, (u64)rctx->in_dma, + (u64)rctx->out_dma, req->cryptlen); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CCP_MAX_PACKED, target_mbx, + cmh_ccp_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + /* -EBUSY = backlogged; ownership transferred to callback. */ + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_out: + cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); + return ret; +} + +static int cmh_ccp_encrypt(struct skcipher_request *req) +{ + return cmh_ccp_crypt(req, CCP_OP_ENCRYPT); +} + +static int cmh_ccp_decrypt(struct skcipher_request *req) +{ + return cmh_ccp_crypt(req, CCP_OP_DECRYPT); +} + +/* Registration */ + +static struct skcipher_alg cmh_chacha20_alg = { + .setkey = cmh_ccp_setkey, + .encrypt = cmh_ccp_encrypt, + .decrypt = cmh_ccp_decrypt, + .init = cmh_ccp_init_tfm, + .exit = cmh_ccp_exit_tfm, + .min_keysize = 32, + .max_keysize = 32, + .ivsize = CCP_CTRNONCE_SIZE, + .base = { + .cra_name = "chacha20", + .cra_driver_name = "cri-cmh-chacha20", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, /* stream cipher */ + .cra_ctxsize = sizeof(struct cmh_ccp_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +/** + * cmh_ccp_register() - Register ChaCha20 skcipher algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_ccp_register(void) +{ + int ret; + + ret = crypto_register_skcipher(&cmh_chacha20_alg); + if (ret) + dev_err(cmh_dev(), "cmh_ccp: failed to register chacha20 (rc=%d)\n", ret); + else + dev_dbg(cmh_dev(), "cmh_ccp: registered chacha20\n"); + + return ret; +} + +/** + * cmh_ccp_unregister() - Unregister ChaCha20 skcipher algorithm from the crypto framework + */ +void cmh_ccp_unregister(void) +{ + crypto_unregister_skcipher(&cmh_chacha20_alg); + dev_dbg(cmh_dev(), "cmh_ccp: unregistered chacha20\n"); +} diff --git a/drivers/crypto/cmh/cmh_ccp_aead.c b/drivers/crypto/cmh/cmh_ccp_aead.c new file mode 100644 index 00000000000000..20b6f9d1746ab1 --- /dev/null +++ b/drivers/crypto/cmh/cmh_ccp_aead.c @@ -0,0 +1,583 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API ChaCha20-Poly1305 AEAD Driver (RFC 7539) + * + * Registers "rfc7539(chacha20,poly1305)" as an AEAD algorithm with the + * Linux crypto subsystem, backed by the CMH CCP core. + * + * VCQ sequence: + * [SYS_CMD_WRITE] + CCP_CMD_AEAD_INIT + [CCP_CMD_AAD_FINAL] + * + CCP_CMD_FINAL + CCP_CMD_FLUSH + * + * The RFC 7539 AEAD interface passes a 12-byte nonce via req->iv. + * The CCP core expects a 16-byte ctrnonce (4-byte LE counter + 12-byte + * nonce). We prepend a zero counter (per RFC 7539 S2.8: counter 0 + * generates the Poly1305 key, counter 1 starts encryption -- the + * CMH eSW handles this internally from the initial counter value of 0). + * + * Tag is always 16 bytes (Poly1305 authenticator). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_ccp.h" +#include "cmh_vcq.h" +#include "cmh_ccp_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define CCP_AEAD_IV_SIZE 12U /* RFC 7539 nonce */ +#define CCP_ESP_IV_SIZE 8U /* RFC 7539 ESP nonce (4-byte salt at setkey) */ +#define CCP_ESP_SALT_SIZE 4U +#define CCP_AEAD_TAG_SIZE 16U /* Poly1305 tag */ + +struct cmh_ccp_aead_tfm_ctx { + struct cmh_key_ctx key; + u32 authsize; + u8 salt[CCP_ESP_SALT_SIZE]; /* ESP salt (unused for rfc7539) */ +}; + +/* Per-request context (lives in aead_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + CCP_CMD_AEAD_INIT + CCP_CMD_AAD_FINAL + * + CCP_CMD_FINAL + FLUSH = 5 + */ +#define CMH_CCP_AEAD_MAX_PAYLOAD 5 +#define CMH_CCP_AEAD_MAX_PACKED (CMH_CCP_AEAD_MAX_PAYLOAD * 2) + +struct cmh_ccp_aead_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + dma_addr_t aad_dma; + dma_addr_t tag_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *aad_buf; + u8 *tag_buf; + u32 cryptlen; + u32 assoclen; + u32 authsize; + u32 keylen; + bool encrypting; + struct vcq_cmd packed[CMH_CCP_AEAD_MAX_PACKED]; +}; + +/* VCQ Builders -- CCP AEAD-specific */ + +static void vcq_add_ccp_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 ctrnonce_dma, u32 keylen, u32 op) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_AEAD_INIT); + slot->hwc.ccp.cmd_aead.key = key_ref; + slot->hwc.ccp.cmd_aead.ctrnonce = ctrnonce_dma; + slot->hwc.ccp.cmd_aead.keylen = keylen; + slot->hwc.ccp.cmd_aead.ctrnoncelen = CCP_CTRNONCE_SIZE; + slot->hwc.ccp.cmd_aead.op = op; +} + +static void vcq_add_ccp_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma, + u32 aadlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_AAD_FINAL); + slot->hwc.ccp.cmd_aad_final.aad = aad_dma; + slot->hwc.ccp.cmd_aad_final.aadlen = aadlen; +} + +static void vcq_add_ccp_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL); + slot->hwc.ccp.cmd_final.input = input_dma; + slot->hwc.ccp.cmd_final.output = output_dma; + slot->hwc.ccp.cmd_final.tag = tag_dma; + slot->hwc.ccp.cmd_final.iolen = iolen; + slot->hwc.ccp.cmd_final.taglen = taglen; +} + +/* setkey */ +static int cmh_ccp_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + /* RFC 7539 AEAD requires 32-byte key */ + if (keylen != CHACHA_KEY_SIZE) + return -EINVAL; + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_CCP); +} + +static int cmh_ccp_aead_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + /* Poly1305 tag is always 16 bytes */ + if (authsize != CCP_AEAD_TAG_SIZE) + return -EINVAL; + + tctx->authsize = authsize; + return 0; +} + +static int cmh_ccp_aead_init_tfm(struct crypto_aead *tfm) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + tctx->authsize = CCP_AEAD_TAG_SIZE; + crypto_aead_set_reqsize(tfm, sizeof(struct cmh_ccp_aead_reqctx)); + return 0; +} + +static void cmh_ccp_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_ccp_aead_unmap_dma(struct cmh_ccp_aead_reqctx *rctx) +{ + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, rctx->authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); + if (rctx->cryptlen > 0) { + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, + DMA_TO_DEVICE); + } + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +} + +static void cmh_ccp_aead_free_bufs(struct cmh_ccp_aead_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; + kfree(rctx->aad_buf); + rctx->aad_buf = NULL; +} + +static void cmh_ccp_aead_complete(void *data, int error) +{ + struct aead_request *req = data; + struct cmh_ccp_aead_reqctx *rctx = aead_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_ccp_aead_unmap_dma(rctx); + + /* + * Map HW error on decrypt to -EBADMSG. The eSW CCP core uses a + * single error code (-EIO) for both authentication failures and + * other core errors (e.g. DMA timeout), so we cannot distinguish + * them from the MBX_STATUS alone. In practice the only error + * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA + * timeout would indicate a fatal HW problem where -EBADMSG vs + * -EIO is moot. The kernel crypto API requires -EBADMSG for + * AEAD authentication failures. + */ + if (error == -EIO && !rctx->encrypting) + error = -EBADMSG; + + if (!error) { + if (rctx->cryptlen > 0) + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + req->assoclen, + rctx->cryptlen, 1); + if (rctx->encrypting) + scatterwalk_map_and_copy(rctx->tag_buf, req->dst, + req->assoclen + + rctx->cryptlen, + rctx->authsize, 1); + } + + cmh_ccp_aead_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * Core AEAD encrypt/decrypt -- async path. + * + * Encrypt: plaintext -> ciphertext + 16-byte tag + * Decrypt: ciphertext + tag -> plaintext (tag verified by CMH eSW) + * + * VCQ: [SYS_CMD_WRITE] + AEAD_INIT + [AAD_FINAL] + FINAL + FLUSH + */ +static int cmh_ccp_aead_crypt(struct aead_request *req, u32 ccp_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_ccp_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_CCP_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize, cryptlen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + authsize = tctx->authsize; + + if (ccp_op == CCP_OP_ENCRYPT) { + cryptlen = req->cryptlen; + } else { + if (req->cryptlen < authsize) + return -EINVAL; + cryptlen = req->cryptlen - authsize; + } + + /* + * HW uses a proprietary LLI scatter-gather format that is + * incompatible with struct scatterlist, so the payload is + * linearised into contiguous buffers for DMA. Cap total + * size to prevent excessive memory consumption. + */ + if ((u64)cryptlen + req->assoclen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = cryptlen; + rctx->assoclen = req->assoclen; + rctx->authsize = authsize; + rctx->encrypting = (ccp_op == CCP_OP_ENCRYPT); + + /* + * rfc7539esp: the last ivsize (8) bytes of the AAD region are the + * IV/nonce, not actual associated data. Subtract them so HW only + * authenticates the real AAD. + */ + if (crypto_aead_ivsize(tfm) == CCP_ESP_IV_SIZE) { + if (rctx->assoclen < CCP_ESP_IV_SIZE) + return -EINVAL; + rctx->assoclen -= CCP_ESP_IV_SIZE; + } + + /* Linearise AAD */ + if (rctx->assoclen > 0) { + rctx->aad_buf = kmalloc(rctx->assoclen, gfp); + if (!rctx->aad_buf) + return -ENOMEM; + scatterwalk_map_and_copy(rctx->aad_buf, req->src, + 0, rctx->assoclen, 0); + rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf, + rctx->assoclen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->aad_dma)) { + ret = -ENOMEM; + goto out_free_aad; + } + } + + /* Linearise input */ + if (cryptlen > 0) { + rctx->in_buf = kmalloc(cryptlen, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_aad; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, + req->assoclen, cryptlen, 0); + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + } + + /* Allocate output buffer */ + if (cryptlen > 0) { + rctx->out_buf = kmalloc(cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + } + + /* Tag buffer */ + rctx->tag_buf = kmalloc(authsize, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + if (!rctx->encrypting) { + scatterwalk_map_and_copy(rctx->tag_buf, req->src, + req->assoclen + cryptlen, + authsize, 0); + } else { + memset(rctx->tag_buf, 0, authsize); + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize, + rctx->encrypting ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Build 16-byte ctrnonce: 4-byte zero counter + 12-byte nonce. + * rfc7539: counter(4) | req->iv(12) + * rfc7539esp: counter(4) | salt(4) | req->iv(8) + */ + rctx->iv_buf = kzalloc(CCP_CTRNONCE_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + if (crypto_aead_ivsize(tfm) == CCP_ESP_IV_SIZE) { + memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN, + tctx->salt, CCP_ESP_SALT_SIZE); + memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN + CCP_ESP_SALT_SIZE, + req->iv, CCP_ESP_IV_SIZE); + } else { + memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN, + req->iv, CCP_AEAD_IV_SIZE); + } + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, CCP_CTRNONCE_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_CCP); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* AEAD_INIT */ + vcq_add_ccp_aead_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, ccp_op); + + /* AAD_FINAL if we have associated data */ + if (rctx->assoclen > 0) + vcq_add_ccp_aad_final(&cmds[idx++], core_id, + (u64)rctx->aad_dma, rctx->assoclen); + + /* FINAL with tag */ + vcq_add_ccp_aead_final(&cmds[idx++], core_id, + cryptlen > 0 ? (u64)rctx->in_dma : 0, + cryptlen > 0 ? (u64)rctx->out_dma : 0, + (u64)rctx->tag_dma, cryptlen, authsize); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CCP_AEAD_MAX_PACKED, + target_mbx, + cmh_ccp_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_out: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_aad: + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +out_free_aad: + kfree(rctx->aad_buf); + return ret; +} + +static int cmh_ccp_aead_encrypt(struct aead_request *req) +{ + return cmh_ccp_aead_crypt(req, CCP_OP_ENCRYPT); +} + +static int cmh_ccp_aead_decrypt(struct aead_request *req) +{ + return cmh_ccp_aead_crypt(req, CCP_OP_DECRYPT); +} + +/* -- rfc7539esp: ESP variant with 4-byte salt + 8-byte IV --------------- */ + +/* + * ESP setkey: 36 bytes = 32-byte ChaCha20 key + 4-byte salt. + * The salt is prepended to the 8-byte per-packet IV from the ESP header + * to form the 12-byte RFC 7539 nonce. + */ +static int cmh_ccp_esp_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + if (keylen != CHACHA_KEY_SIZE + CCP_ESP_SALT_SIZE) + return -EINVAL; + + memcpy(tctx->salt, key + CHACHA_KEY_SIZE, CCP_ESP_SALT_SIZE); + return cmh_key_setkey_raw(&tctx->key, key, CHACHA_KEY_SIZE, CORE_ID_CCP); +} + +/* Registration */ + +static struct aead_alg cmh_rfc7539_alg = { + .setkey = cmh_ccp_aead_setkey, + .setauthsize = cmh_ccp_aead_setauthsize, + .encrypt = cmh_ccp_aead_encrypt, + .decrypt = cmh_ccp_aead_decrypt, + .init = cmh_ccp_aead_init_tfm, + .exit = cmh_ccp_aead_exit_tfm, + .ivsize = CCP_AEAD_IV_SIZE, + .maxauthsize = CCP_AEAD_TAG_SIZE, + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "cri-cmh-rfc7539-chacha20-poly1305", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct cmh_ccp_aead_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +static struct aead_alg cmh_rfc7539esp_alg = { + .setkey = cmh_ccp_esp_setkey, + .setauthsize = cmh_ccp_aead_setauthsize, + .encrypt = cmh_ccp_aead_encrypt, + .decrypt = cmh_ccp_aead_decrypt, + .init = cmh_ccp_aead_init_tfm, + .exit = cmh_ccp_aead_exit_tfm, + .ivsize = CCP_ESP_IV_SIZE, + .maxauthsize = CCP_AEAD_TAG_SIZE, + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "cri-cmh-rfc7539esp-chacha20-poly1305", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct cmh_ccp_aead_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +/** + * cmh_ccp_aead_register() - Register ChaCha20-Poly1305 AEAD algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_ccp_aead_register(void) +{ + int ret; + + ret = crypto_register_aead(&cmh_rfc7539_alg); + if (ret) { + dev_err(cmh_dev(), "cmh_ccp_aead: failed to register rfc7539 (rc=%d)\n", + ret); + return ret; + } + dev_dbg(cmh_dev(), "cmh_ccp_aead: registered rfc7539(chacha20,poly1305)\n"); + + ret = crypto_register_aead(&cmh_rfc7539esp_alg); + if (ret) { + dev_err(cmh_dev(), "cmh_ccp_aead: failed to register rfc7539esp (rc=%d)\n", + ret); + crypto_unregister_aead(&cmh_rfc7539_alg); + return ret; + } + dev_dbg(cmh_dev(), "cmh_ccp_aead: registered rfc7539esp(chacha20,poly1305)\n"); + + return 0; +} + +/** + * cmh_ccp_aead_unregister() - Unregister ChaCha20-Poly1305 AEAD algorithms + */ +void cmh_ccp_aead_unregister(void) +{ + crypto_unregister_aead(&cmh_rfc7539esp_alg); + crypto_unregister_aead(&cmh_rfc7539_alg); + dev_dbg(cmh_dev(), "cmh_ccp_aead: unregistered rfc7539/rfc7539esp\n"); +} diff --git a/drivers/crypto/cmh/cmh_ccp_poly.c b/drivers/crypto/cmh/cmh_ccp_poly.c new file mode 100644 index 00000000000000..020a98fbe6070e --- /dev/null +++ b/drivers/crypto/cmh/cmh_ccp_poly.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API Poly1305 (ahash) Driver + * + * Registers "poly1305" as an ahash algorithm with the Linux crypto + * subsystem, backed by the CMH CCP core. + * + * Poly1305 is a one-time authenticator that produces a 16-byte MAC. + * It requires two 16-byte keys: r (clamped multiplier) and s (nonce). + * + * Key format: 32 bytes = r_key[0..15] || s_key[16..31] + * This matches the Poly1305 key layout in RFC 7539 S2.5. + * + * VCQ sequence: + * SYS_CMD_WRITE(s_key) + SYS_CMD_WRITE(r_key) + * + CCP_CMD_POLY1305_INIT + CCP_CMD_FINAL + CCP_CMD_FLUSH + * + * Both keys are written to SYS_REF_TEMP; the CMH eSW stacks them + * so that POLY1305_INIT finds r_key (most recent) as rkey and + * s_key (previous) as skey. + * + * The ahash interface accumulates data via .update() and submits the + * full VCQ asynchronously in .final(). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_ccp.h" +#include "cmh_vcq.h" +#include "cmh_ccp_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define POLY1305_DIGEST_SIZE 16U +#define POLY1305_BLOCK_SIZE 16U +#define POLY1305_KEY_SIZE 32U /* r(16) + s(16) */ + +/* + * Maximum accumulated data for Poly1305 -- driver-imposed, not HW. + * + * The CCP core does not expose external save/restore VCQ commands, + * so the driver must accumulate all data in kernel memory via + * .update() and submit it atomically in .final(). This cap limits + * the per-request kernel allocation. + */ +#define POLY_MAX_DATA (64 * 1024) + +/* + * Per-transform context -- stores the raw 32-byte key (r || s). + * + * Only the raw-key path is supported for standalone Poly1305. + */ +struct cmh_poly_tfm_ctx { + u8 key[POLY1305_KEY_SIZE]; + dma_addr_t rkey_dma; + dma_addr_t skey_dma; + u32 keylen; + bool has_key; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* Chunk node for O(1) update() appends */ +struct cmh_poly_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* Per-request context (lives in ahash_request::__ctx) */ + +/* + * Maximum payload commands: + * SYS_CMD_WRITE(s) + SYS_CMD_WRITE(r) + POLY1305_INIT + * + CCP_CMD_FINAL + FLUSH = 5 + */ +#define CMH_POLY_MAX_PAYLOAD 5 +#define CMH_POLY_MAX_PACKED (CMH_POLY_MAX_PAYLOAD * 2) + +struct cmh_poly_reqctx { + struct list_head chunks; + u32 total_len; + u8 *buf; /* linearised in final() */ + /* DMA state for async final */ + dma_addr_t in_dma; + dma_addr_t tag_dma; + u8 *tag_buf; + struct vcq_cmd packed[CMH_POLY_MAX_PACKED]; +}; + +/* + * Export/import: not supported. + * + * The CCP core lacks external save/restore VCQ commands, so there is + * no way to checkpoint intermediate Poly1305 state to host memory. + * Pending eSW ABI extension to add save/restore for the CCP core. + */ + +static void vcq_add_ccp_poly_init(struct vcq_cmd *slot, u32 core_id, + u64 rkey_ref, u32 rkeylen, + u64 skey_ref, u32 skeylen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_POLY1305_INIT); + slot->hwc.ccp.cmd_poly.rkey = rkey_ref; + slot->hwc.ccp.cmd_poly.rkeylen = rkeylen; + slot->hwc.ccp.cmd_poly.skey = skey_ref; + slot->hwc.ccp.cmd_poly.skeylen = skeylen; +} + +static void vcq_add_ccp_poly_final(struct vcq_cmd *slot, u32 core_id, + u64 input_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL); + slot->hwc.ccp.cmd_final.input = input_dma; + slot->hwc.ccp.cmd_final.output = 0; + slot->hwc.ccp.cmd_final.tag = tag_dma; + slot->hwc.ccp.cmd_final.iolen = iolen; + slot->hwc.ccp.cmd_final.taglen = taglen; +} + +static int cmh_poly_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + /* Poly1305: exactly 32 bytes (r[16] + s[16]) */ + if (keylen != POLY1305_KEY_SIZE) + return -EINVAL; + + /* Unmap old key DMA if re-keying */ + if (tctx->has_key) { + cmh_dma_unmap_single(tctx->rkey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + } + + memcpy(tctx->key, key, POLY1305_KEY_SIZE); + tctx->keylen = POLY1305_KEY_SIZE; + + /* + * Pre-map both key halves for DMA. The key buffer lives in + * the tfm context and is stable until exit_tfm() or re-setkey. + */ + tctx->skey_dma = cmh_dma_map_single(tctx->key + CCP_POLY_KEY_SIZE, + CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(tctx->skey_dma)) { + tctx->has_key = false; + return -ENOMEM; + } + + tctx->rkey_dma = cmh_dma_map_single(tctx->key, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(tctx->rkey_dma)) { + cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + tctx->has_key = false; + return -ENOMEM; + } + + tctx->has_key = true; + return 0; +} + +static void cmh_poly_free_chunks(struct cmh_poly_reqctx *rctx, + struct cmh_poly_tfm_ctx *tctx) +{ + struct cmh_poly_chunk *c, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &rctx->chunks, list) { + list_del(&c->list); + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); +} + +static int cmh_poly_init(struct ahash_request *req) +{ + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + return 0; +} + +static int cmh_poly_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + struct cmh_poly_chunk *chunk; + gfp_t gfp; + int ret; + + if (!req->nbytes) + return 0; + + if (req->nbytes > POLY_MAX_DATA - rctx->total_len) { + ret = -EINVAL; + goto err_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp); + if (!chunk) { + ret = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(chunk->data, req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(chunk->data, req->src, + 0, req->nbytes, 0); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len += req->nbytes; + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * Callers may not call .final() on error, so they would leak. + */ + cmh_poly_free_chunks(rctx, tctx); + return ret; +} + +static void cmh_poly_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->total_len > 0) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, POLY1305_DIGEST_SIZE, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->tag_buf, POLY1305_DIGEST_SIZE); + + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + cmh_poly_free_chunks(rctx, tctx); + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + rctx->total_len = 0; + cmh_complete(&req->base, error); +} + +static int cmh_poly_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_POLY_MAX_PAYLOAD]; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (!tctx->has_key) { + ret = -ENOKEY; + goto out_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Linearise chunks into a single contiguous buffer for DMA */ + if (rctx->total_len > 0) { + struct cmh_poly_chunk *c; + u32 off = 0; + + rctx->buf = kmalloc(rctx->total_len, gfp); + if (!rctx->buf) { + ret = -ENOMEM; + goto out_free_chunks; + } + list_for_each_entry(c, &rctx->chunks, list) { + memcpy(rctx->buf + off, c->data, c->len); + off += c->len; + } + } + + /* Tag output buffer */ + rctx->tag_buf = kzalloc(POLY1305_DIGEST_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_free_buf; + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, + POLY1305_DIGEST_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map input data */ + if (rctx->total_len > 0) { + rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_unmap_tag; + } + } + + /* + * Key DMA handles are pre-mapped in setkey() and live in + * the tfm context. Use them directly for the VCQ writes. + */ + + d = cmh_core_select_instance(CMH_CORE_CCP); + target_mbx = d.mbx_idx; + core_id = d.core_id; + idx = 0; + + /* Write s_key to SYS_REF_TEMP first (bottom of stack) */ + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)tctx->skey_dma, SYS_REF_NONE, + CCP_POLY_KEY_SIZE, + SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_CCP)); + + /* Write r_key to SYS_REF_TEMP second (top of stack) */ + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)tctx->rkey_dma, SYS_REF_NONE, + CCP_POLY_KEY_SIZE, + SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_CCP)); + + /* POLY1305_INIT: rkey=TEMP (top), skey=TEMP (next) */ + vcq_add_ccp_poly_init(&cmds[idx++], core_id, SYS_REF_TEMP, + CCP_POLY_KEY_SIZE, SYS_REF_TEMP, + CCP_POLY_KEY_SIZE); + + /* FINAL: data -> tag */ + vcq_add_ccp_poly_final(&cmds[idx++], core_id, + rctx->total_len > 0 ? (u64)rctx->in_dma : 0, + (u64)rctx->tag_dma, rctx->total_len, + POLY1305_DIGEST_SIZE); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_POLY_MAX_PACKED, target_mbx, + cmh_poly_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_unmap_in; + + return -EINPROGRESS; + +out_unmap_in: + if (rctx->total_len > 0 && rctx->in_dma) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, POLY1305_DIGEST_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_free_buf: + kfree_sensitive(rctx->buf); + rctx->buf = NULL; +out_free_chunks: + cmh_poly_free_chunks(rctx, tctx); + rctx->total_len = 0; + return ret; +} + +static int cmh_poly_export(struct ahash_request *req, void *out) +{ + return -EOPNOTSUPP; +} + +static int cmh_poly_import(struct ahash_request *req, const void *in) +{ + return -EOPNOTSUPP; +} + +static int cmh_poly_finup(struct ahash_request *req) +{ + int err; + + err = cmh_poly_update(req); + if (err) + return err; + return cmh_poly_final(req); +} + +static int cmh_poly_digest(struct ahash_request *req) +{ + int err; + + err = cmh_poly_init(req); + if (err) + return err; + return cmh_poly_finup(req); +} + +static int cmh_poly_init_tfm(struct crypto_ahash *tfm) +{ + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_poly_reqctx)); + return 0; +} + +static void cmh_poly_exit_tfm(struct crypto_ahash *tfm) +{ + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_chunk *c, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) { + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + + if (tctx->has_key) { + cmh_dma_unmap_single(tctx->rkey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + } + memzero_explicit(tctx->key, POLY1305_KEY_SIZE); +} + +static struct ahash_alg cmh_poly1305_alg = { + .init = cmh_poly_init, + .update = cmh_poly_update, + .final = cmh_poly_final, + .finup = cmh_poly_finup, + .digest = cmh_poly_digest, + .export = cmh_poly_export, + .import = cmh_poly_import, + .setkey = cmh_poly_setkey, + .init_tfm = cmh_poly_init_tfm, + .exit_tfm = cmh_poly_exit_tfm, + .halg = { + .digestsize = POLY1305_DIGEST_SIZE, + .statesize = sizeof(struct cmh_poly_reqctx), + .base = { + .cra_name = "poly1305", + .cra_driver_name = "cri-cmh-poly1305", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT, + .cra_blocksize = POLY1305_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cmh_poly_tfm_ctx), + .cra_module = THIS_MODULE, + }, + }, +}; + +/** + * cmh_ccp_poly_register() - Register Poly1305 hash algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_ccp_poly_register(void) +{ + int ret; + + ret = crypto_register_ahash(&cmh_poly1305_alg); + if (ret) + dev_err(cmh_dev(), "cmh_ccp_poly: failed to register poly1305 (rc=%d)\n", + ret); + else + dev_dbg(cmh_dev(), "cmh_ccp_poly: registered poly1305\n"); + + return ret; +} + +/** + * cmh_ccp_poly_unregister() - Unregister Poly1305 hash algorithm from the crypto framework + */ +void cmh_ccp_poly_unregister(void) +{ + crypto_unregister_ahash(&cmh_poly1305_alg); + dev_dbg(cmh_dev(), "cmh_ccp_poly: unregistered poly1305\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 0a73964b978587..e5b2a86c3149b3 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -36,6 +36,7 @@ #include "cmh_sm3.h" #include "cmh_aes.h" #include "cmh_sm4.h" +#include "cmh_ccp.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -259,6 +260,21 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_sm4_cmac_register; + /* Register CCP ChaCha20 skcipher algorithm */ + ret = cmh_ccp_register(); + if (ret) + goto err_ccp_register; + + /* Register CCP ChaCha20-Poly1305 AEAD (RFC 7539) */ + ret = cmh_ccp_aead_register(); + if (ret) + goto err_ccp_aead_register; + + /* Register CCP Poly1305 shash algorithm */ + ret = cmh_ccp_poly_register(); + if (ret) + goto err_ccp_poly_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -271,6 +287,12 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_ccp_poly_unregister(); +err_ccp_poly_register: + cmh_ccp_aead_unregister(); +err_ccp_aead_register: + cmh_ccp_unregister(); +err_ccp_register: cmh_sm4_cmac_unregister(); err_sm4_cmac_register: cmh_sm4_aead_unregister(); @@ -319,6 +341,9 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_ccp_poly_unregister(); + cmh_ccp_aead_unregister(); + cmh_ccp_unregister(); cmh_sm4_cmac_unregister(); cmh_sm4_aead_unregister(); cmh_sm4_unregister(); diff --git a/drivers/crypto/cmh/include/cmh_ccp.h b/drivers/crypto/cmh/include/cmh_ccp.h new file mode 100644 index 00000000000000..363d208cbcebc3 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_ccp.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- CCP Crypto API Drivers + * + * Registers CCP algorithms with the Linux crypto subsystem: + * skcipher: chacha20 + * shash: poly1305 + * aead: rfc7539(chacha20poly1305) + */ + +#ifndef CMH_CCP_H +#define CMH_CCP_H + +int cmh_ccp_register(void); +void cmh_ccp_unregister(void); + +int cmh_ccp_aead_register(void); +void cmh_ccp_aead_unregister(void); + +int cmh_ccp_poly_register(void); +void cmh_ccp_poly_unregister(void); + +#endif /* CMH_CCP_H */ From 23c4d23abd14fdb7141dff9373e68e6fd738b0fe Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:29 -0700 Subject: [PATCH 13/21] crypto: cmh - add DRBG hwrng Register the CMH DRBG core (core ID 0x0f) as an hwrng provider. The hardware implements a NIST SP 800-90A compliant DRBG with automatic self-seeding. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 3 +- drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/cmh_rng.c | 316 ++++++++++++++++++++++++++++++++++ 3 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_rng.c diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 4ebd0e1d10bcde..1c4cb817424c47 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -28,7 +28,8 @@ cmh-y := \ cmh_sm4_cmac.o \ cmh_ccp.o \ cmh_ccp_aead.o \ - cmh_ccp_poly.o + cmh_ccp_poly.o \ + cmh_rng.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index e5b2a86c3149b3..40440e2b73b432 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -34,6 +34,7 @@ #include "cmh_cshake.h" #include "cmh_kmac.h" #include "cmh_sm3.h" +#include "cmh_rng.h" #include "cmh_aes.h" #include "cmh_sm4.h" #include "cmh_ccp.h" @@ -230,6 +231,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_sm3_register; + /* Register hwrng backed by DRBG core */ + ret = cmh_rng_register(pdev); + if (ret) + goto err_rng_register; + /* Register AES skcipher algorithms */ ret = cmh_aes_register(); if (ret) @@ -305,6 +311,8 @@ static int cmh_probe(struct platform_device *pdev) err_aes_aead_register: cmh_aes_unregister(); err_aes_register: + cmh_rng_unregister(); +err_rng_register: cmh_sm3_unregister(); err_sm3_register: cmh_kmac_unregister(); @@ -350,6 +358,7 @@ static void cmh_remove(struct platform_device *pdev) cmh_aes_cmac_unregister(); cmh_aes_aead_unregister(); cmh_aes_unregister(); + cmh_rng_unregister(); cmh_sm3_unregister(); cmh_kmac_unregister(); cmh_cshake_unregister(); diff --git a/drivers/crypto/cmh/cmh_rng.c b/drivers/crypto/cmh/cmh_rng.c new file mode 100644 index 00000000000000..c9693f6cc3609c --- /dev/null +++ b/drivers/crypto/cmh/cmh_rng.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Hardware RNG (DRBG) Driver + * + * Implements a Linux hwrng backed by the CMH DRBG core. Each .read() + * builds a 3-entry VCQ (header + GENERATE + FLUSH) and submits it + * synchronously through the Transaction Manager. + * + * DRBG configuration (CONFIG) is a management-host operation in the + * CMH security model. The driver's behaviour is controlled by the + * drbg_config setting (debug-only module parameter): + * + * "auto" (default) -- attempt CONFIG at probe with the hardcoded + * ratio/strength defaults. Succeeds in stateless mode (any host may + * CONFIG) or when this host is the management host in stateful + * mode. On -EPERM the driver logs a notice and continues -- + * GENERATE will work once the management host configures the DRBG. + * + * "skip" -- do not issue CONFIG; assume an external management host + * will configure the DRBG. hwrng is still registered; .read() + * returns -EAGAIN until GENERATE succeeds. + * + * The management host (or any privileged user-space process) can also + * reconfigure the DRBG at runtime via CMH_IOCTL_DRBG_CONFIG. + */ + +#include +#include +#include +#include +#include + +#include "cmh_rng.h" +#include "cmh_vcq.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_sys.h" +#include "cmh_config.h" + +/* VCQ layout for .read(): header + GENERATE + FLUSH = 3 entries. */ +#define DRBG_READ_VCQ_CMDS 3 + +/* VCQ layout for CONFIG: header + RESET + CONFIG + FLUSH = 4 entries. */ +#define DRBG_CONFIG_VCQ_CMDS 4 + +/* + * Linux hwrng quality is expressed in bits of entropy per 1024 bits of + * input. The kernel clamps to this maximum; mirror it here so our + * MODULE_PARM_DESC and clamp logic stay in sync. + */ +#define CMH_HWRNG_QUALITY_MAX 1024 + +/* Module parameters */ + +static int hwrng_quality; +module_param(hwrng_quality, int, 0444); +MODULE_PARM_DESC(hwrng_quality, + "hwrng quality (0=no CRNG seeding, 1-1024=enable; default: 0)"); + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +static char *drbg_config = "auto"; +module_param(drbg_config, charp, 0444); +MODULE_PARM_DESC(drbg_config, + "[debug] DRBG config at probe: \"auto\"=attempt CONFIG, \"skip\"=assume external (default: auto)"); +#else +static const char *drbg_config = "auto"; +#endif + +/* + * DRBG parameters -- hardcoded to production defaults. + * Entropy ratio 0 = 1:1 (full entropy), security strength 0x10 = 256-bit. + */ +#define CMH_DRBG_ENTROPY_RATIO 0 +#define CMH_DRBG_SECURITY_STRENGTH 0x10 + +static unsigned int drbg_timeout_ms = 500; + +/* VCQ Builders */ + +static void vcq_add_drbg_generate(struct vcq_cmd *slot, u64 dst_phys, u32 len) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(CORE_ID_DRBG, 0, 1, DRBG_CMD_GENERATE); + slot->hwc.drbg.cmd_generate.dst = dst_phys; + slot->hwc.drbg.cmd_generate.len = len; +} + +/* + * Maximum bytes per DRBG GENERATE request. The kernel calls .read() + * repeatedly to fill larger requests, so capping here is safe. + * 32 bytes matches the 256-bit security strength natural output size. + */ +#define CMH_DRBG_MAX_GENERATE 32U + +/* hwrng .read() callback */ + +static int cmh_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct cmh_dma_orphan *orphan; + struct vcq_cmd vcq[DRBG_READ_VCQ_CMDS]; + dma_addr_t dma_addr; + void *dmabuf; + size_t nbytes; + int ret; + + if (max == 0) + return 0; + + /* + * Our path uses GFP_KERNEL allocations and synchronous VCQ + * submission -- both may sleep. When the caller indicates + * non-blocking context (!wait), return 0 ("no data yet") so + * the hwrng core retries later. + */ + if (!wait) + return 0; + + nbytes = min_t(size_t, max, CMH_DRBG_MAX_GENERATE); + + orphan = kmalloc_obj(*orphan, GFP_KERNEL); + if (!orphan) + return -ENOMEM; + + dmabuf = kmalloc(nbytes, GFP_KERNEL); + if (!dmabuf) { + kfree(orphan); + return -ENOMEM; + } + + dma_addr = cmh_dma_map_single(dmabuf, nbytes, DMA_FROM_DEVICE); + if (cmh_dma_map_error(dma_addr)) { + kfree(dmabuf); + kfree(orphan); + return -ENOMEM; + } + + orphan->buf = dmabuf; + orphan->addr = dma_addr; + orphan->len = nbytes; + orphan->dir = DMA_FROM_DEVICE; + + vcq_set_header(&vcq[0], DRBG_READ_VCQ_CMDS); + vcq_add_drbg_generate(&vcq[1], dma_addr, nbytes); + vcq_add_flush(&vcq[2], CORE_ID_DRBG); + + /* + * Use the noabort variant: if the MBX is occupied by a slow + * operation (e.g. SLH-DSA sign at 120 s), we must not issue + * MBX_COMMAND_ABORT -- that would kill the unrelated in-flight + * VCQ. On timeout with an in-flight VCQ (-EINPROGRESS), the + * orphan callback defers DMA cleanup until the RH fires. + */ + ret = cmh_tm_submit_sync_noabort(vcq, DRBG_READ_VCQ_CMDS, 1, + msecs_to_jiffies(drbg_timeout_ms), + cmh_dma_orphan_free, orphan); + if (ret == -EINPROGRESS) { + /* Orphan callback owns dmabuf -- will free on VCQ completion */ + return -EAGAIN; + } + + /* Normal path or cancelled-from-queue: caller owns DMA */ + cmh_dma_unmap_single(dma_addr, nbytes, DMA_FROM_DEVICE); + kfree(orphan); + + if (ret) { + /* + * Only translate known transient conditions to -EAGAIN + * so the hwrng subsystem retries later. Propagate + * unexpected failures unchanged to avoid masking real + * faults and causing indefinite retry loops. + */ + switch (ret) { + case -EAGAIN: + case -EBUSY: + case -ETIMEDOUT: + case -EIO: + /* + * -ENODEV: the TM is not running -- occurs when the + * hwrng kthread (PF_NOFREEZE, not frozen during + * suspend) calls .read() while the device is suspended. + * Treat as transient: the TM restarts on resume. + */ + case -ENODEV: + dev_dbg_ratelimited(cmh_dev(), + "rng: transient DRBG failure (rc=%d)\n", + ret); + kfree(dmabuf); + return -EAGAIN; + default: + dev_err_ratelimited(cmh_dev(), + "rng: DRBG generate failed (rc=%d)\n", + ret); + kfree(dmabuf); + return ret; + } + } + + memcpy(data, dmabuf, nbytes); + kfree(dmabuf); + + return nbytes; +} + +/* Registration */ + +static bool cmh_rng_registered; + +static struct hwrng cmh_hwrng = { + .name = "cri-cmh-drbg", + .read = cmh_rng_read, +}; + +/** + * cmh_rng_register() - Register the CMH hardware RNG device + * @pdev: Platform device for the CMH accelerator + * + * Reads hwrng quality from device tree and module parameters, validates + * DRBG configuration, optionally sends a DRBG CONFIG VCQ to firmware, + * and registers the hwrng device with the kernel hwrng framework. + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_rng_register(struct platform_device *pdev) +{ + int ret; + + cmh_hwrng.quality = hwrng_quality; + + if (cmh_hwrng.quality > CMH_HWRNG_QUALITY_MAX) + cmh_hwrng.quality = CMH_HWRNG_QUALITY_MAX; + + /* + * DRBG CONFIG is a management-host operation. In "auto" mode, + * attempt it -- this succeeds in stateless mode (any host) or + * when we are the management host in stateful mode. On -EPERM + * (not management host) we continue without error -- GENERATE + * will work once the management host configures the DRBG. + * + * In "skip" mode, do not issue CONFIG -- assume the management + * host has already configured (or will configure) the DRBG. + */ + if (strcmp(drbg_config, "skip") != 0) { + struct vcq_cmd cfg_vcq[DRBG_CONFIG_VCQ_CMDS]; + + if (strcmp(drbg_config, "auto") != 0) + dev_warn(&pdev->dev, + "rng: unrecognized drbg_config=\"%s\", treating as \"auto\"\n", + drbg_config); + + vcq_set_header(&cfg_vcq[0], DRBG_CONFIG_VCQ_CMDS); + vcq_add_drbg_reset(&cfg_vcq[1]); + vcq_add_drbg_config(&cfg_vcq[2], CMH_DRBG_ENTROPY_RATIO, + CMH_DRBG_SECURITY_STRENGTH); + vcq_add_flush(&cfg_vcq[3], CORE_ID_DRBG); + ret = cmh_tm_submit_sync(cfg_vcq, DRBG_CONFIG_VCQ_CMDS, 1); + if (ret == -EPERM) + dev_notice(&pdev->dev, + "rng: DRBG config not permitted (not management host); assuming external configuration\n"); + else if (ret) + dev_warn(&pdev->dev, + "rng: DRBG config failed (rc=%d)\n", ret); + else + dev_info(&pdev->dev, + "rng: DRBG configured (ratio=%u strength=0x%02x)\n", + CMH_DRBG_ENTROPY_RATIO, + CMH_DRBG_SECURITY_STRENGTH); + } else { + dev_info(&pdev->dev, + "rng: DRBG config skipped (drbg_config=skip); assuming external configuration\n"); + } + + ret = hwrng_register(&cmh_hwrng); + if (ret) { + dev_err(&pdev->dev, "rng: hwrng_register failed (rc=%d)\n", + ret); + return ret; + } + + dev_info(&pdev->dev, + "rng: registered cri-cmh-drbg (quality=%d timeout=%ums)\n", + cmh_hwrng.quality, drbg_timeout_ms); + + cmh_rng_registered = true; + return 0; +} + +/** + * cmh_rng_unregister() - Unregister the CMH hardware RNG device + * + * Unregisters the hwrng device from the kernel hwrng framework if it + * was previously registered. + */ +void cmh_rng_unregister(void) +{ + if (!cmh_rng_registered) + return; + hwrng_unregister(&cmh_hwrng); + cmh_rng_registered = false; + dev_info(cmh_dev(), "rng: unregistered cri-cmh-drbg\n"); +} + +/* -- debugfs timeout accessor ------------------------------------------ */ + +#ifdef CONFIG_CRYPTO_DEV_CMH_DEBUG +/** + * cmh_rng_timeout_drbg_ptr() - Return pointer to drbg_timeout_ms for debugfs + * + * Exposes the DRBG operation timeout for runtime tuning via debugfs + * config/ directory. + * + * Return: pointer to the static drbg_timeout_ms variable. + */ +unsigned int *cmh_rng_timeout_drbg_ptr(void) { return &drbg_timeout_ms; } +#endif From 777ad722a1da5c16319f96e80535541cc3d5c79a Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:30 -0700 Subject: [PATCH 14/21] crypto: cmh - add RSA akcipher Register the RSA akcipher algorithm using the CMH PKE core (core ID 0x0a). Supports encrypt, decrypt, sign, and verify operations with 2048, 3072, and 4096-bit keys. 512- and 1024-bit keys are also accepted for legacy/test interoperability. Includes common PKE helpers shared by subsequent ECDSA and ECDH patches. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 4 +- drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/cmh_pke_common.c | 578 +++++++++++++++++++++++++ drivers/crypto/cmh/cmh_pke_rsa.c | 642 ++++++++++++++++++++++++++++ 4 files changed, 1232 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_pke_common.c create mode 100644 drivers/crypto/cmh/cmh_pke_rsa.c diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 1c4cb817424c47..7afd9852c33753 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -29,7 +29,9 @@ cmh-y := \ cmh_ccp.o \ cmh_ccp_aead.o \ cmh_ccp_poly.o \ - cmh_rng.o + cmh_rng.o \ + cmh_pke_common.o \ + cmh_pke_rsa.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 40440e2b73b432..07f26b0dd2ef54 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -38,6 +38,7 @@ #include "cmh_aes.h" #include "cmh_sm4.h" #include "cmh_ccp.h" +#include "cmh_pke.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -281,6 +282,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_ccp_poly_register; + /* Register PKE RSA akcipher */ + ret = cmh_pke_rsa_register(); + if (ret) + goto err_pke_rsa_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -293,6 +299,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_pke_rsa_unregister(); +err_pke_rsa_register: cmh_ccp_poly_unregister(); err_ccp_poly_register: cmh_ccp_aead_unregister(); @@ -349,6 +357,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_pke_rsa_unregister(); cmh_ccp_poly_unregister(); cmh_ccp_aead_unregister(); cmh_ccp_unregister(); diff --git a/drivers/crypto/cmh/cmh_pke_common.c b/drivers/crypto/cmh/cmh_pke_common.c new file mode 100644 index 00000000000000..ab3e2eb7d3f88d --- /dev/null +++ b/drivers/crypto/cmh/cmh_pke_common.c @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- PKE Common VCQ Builders + * + * VCQ builder functions for all PKE core commands. Each builder + * populates a single vcq_cmd slot with the appropriate magic, + * command ID, byte-swap flags, and command-specific payload. + * + * RSA commands always use PKE_SWAP_FLAGS (VCQ_FLAG_SWAP_BYTES | + * VCQ_FLAG_SWAP_WORDS). EC Weierstrass curves (NIST P-*, Brainpool, + * secp256k1, SM2) use PKE_SWAP_FLAGS; Edwards curves (25519, 448) + * use no swap flags. SM2 commands use per-command flags documented + * in the eSW ABI. + * + * Callers combine these with vcq_set_header() + vcq_add_flush() + * and submit via cmh_tm_submit_sync(). + */ + +#include + +#include "cmh_pke.h" + +/** + * vcq_add_pke_flush() - Add a PKE flush command to a VCQ slot + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * + * Populates @slot with a flush command for the specified PKE core. + */ +void vcq_add_pke_flush(struct vcq_cmd *slot, u32 core_id) +{ + vcq_add_flush(slot, core_id); +} + +/* RSA */ + +/** + * vcq_add_pke_rsa_enc() - Build a VCQ command for RSA public-key encryption + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @bits: RSA key size in bits + * @e_len: Length of the public exponent in bytes + * @e_dma: DMA address of public exponent buffer + * @n_dma: DMA address of modulus buffer + * @m_dma: DMA address of plaintext message buffer + * @c_dma: DMA address of ciphertext output buffer + * @flags: VCQ command flags + */ +void vcq_add_pke_rsa_enc(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len, + u64 e_dma, u64 n_dma, u64 m_dma, u64 c_dma, + u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_ENC); + slot->hwc.pke.cmd_rsa_enc.bits = bits; + slot->hwc.pke.cmd_rsa_enc.e_len = e_len; + slot->hwc.pke.cmd_rsa_enc.e = e_dma; + slot->hwc.pke.cmd_rsa_enc.n = n_dma; + slot->hwc.pke.cmd_rsa_enc.m = m_dma; + slot->hwc.pke.cmd_rsa_enc.c = c_dma; +} + +/** + * vcq_add_pke_rsa_dec() - Build a VCQ command for RSA private-key decryption + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @bits: RSA key size in bits + * @e_len: Length of the public exponent in bytes + * @e_dma: DMA address of public exponent buffer + * @n_dma: DMA address of modulus buffer + * @c_dma: DMA address of ciphertext input buffer + * @m_dma: DMA address of plaintext output buffer + * @d_ref: Datastore reference for the private exponent + * @flags: VCQ command flags + */ +void vcq_add_pke_rsa_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len, + u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma, + u64 d_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_DEC); + slot->hwc.pke.cmd_rsa_dec.bits = bits; + slot->hwc.pke.cmd_rsa_dec.e_len = e_len; + slot->hwc.pke.cmd_rsa_dec.e = e_dma; + slot->hwc.pke.cmd_rsa_dec.n = n_dma; + slot->hwc.pke.cmd_rsa_dec.c = c_dma; + slot->hwc.pke.cmd_rsa_dec.m = m_dma; + slot->hwc.pke.cmd_rsa_dec.d = d_ref; +} + +/** + * vcq_add_pke_rsa_crt_dec() - Build a VCQ command for RSA-CRT decryption + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @bits: RSA key size in bits + * @e_len: Length of the public exponent in bytes + * @e_dma: DMA address of public exponent buffer + * @n_dma: DMA address of modulus buffer + * @c_dma: DMA address of ciphertext input buffer + * @m_dma: DMA address of plaintext output buffer + * @crt_ref: Datastore reference for CRT private key components + * @flags: VCQ command flags + */ +void vcq_add_pke_rsa_crt_dec(struct vcq_cmd *slot, u32 core_id, u32 bits, u32 e_len, + u64 e_dma, u64 n_dma, u64 c_dma, u64 m_dma, + u64 crt_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_RSA_CRT_DEC); + slot->hwc.pke.cmd_rsa_crt_dec.bits = bits; + slot->hwc.pke.cmd_rsa_crt_dec.e_len = e_len; + slot->hwc.pke.cmd_rsa_crt_dec.e = e_dma; + slot->hwc.pke.cmd_rsa_crt_dec.n = n_dma; + slot->hwc.pke.cmd_rsa_crt_dec.c = c_dma; + slot->hwc.pke.cmd_rsa_crt_dec.m = m_dma; + slot->hwc.pke.cmd_rsa_crt_dec.crt = crt_ref; +} + +/* ECDSA */ + +/** + * vcq_add_pke_ecdsa_verify() - Build a VCQ command for ECDSA signature verification + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521) + * @dlen: Digest length in bytes + * @pk_dma: DMA address of public key buffer + * @dig_dma: DMA address of digest buffer + * @sig_dma: DMA address of signature buffer + * @rp_dma: DMA address of r-prime verification output buffer + * @flags: VCQ command flags + */ +void vcq_add_pke_ecdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen, + u64 pk_dma, u64 dig_dma, u64 sig_dma, + u64 rp_dma, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_VERIFY); + slot->hwc.pke.cmd_ecdsa_verify.curve = curve; + slot->hwc.pke.cmd_ecdsa_verify.digest_len = dlen; + slot->hwc.pke.cmd_ecdsa_verify.public_key = pk_dma; + slot->hwc.pke.cmd_ecdsa_verify.digest = dig_dma; + slot->hwc.pke.cmd_ecdsa_verify.signature = sig_dma; + slot->hwc.pke.cmd_ecdsa_verify.rprime = rp_dma; +} + +/** + * vcq_add_pke_ecdsa_sign() - Build a VCQ command for ECDSA signing + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521) + * @sklen: Secret key length in bytes + * @dig_dma: DMA address of digest buffer + * @sig_dma: DMA address of signature output buffer + * @sk_ref: Datastore reference for the secret key + * @dlen: Digest length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_ecdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 dig_dma, u64 sig_dma, u64 sk_ref, + u32 dlen, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_SIGN); + slot->hwc.pke.cmd_ecdsa_sign.curve = curve; + slot->hwc.pke.cmd_ecdsa_sign.secret_key_len = sklen; + slot->hwc.pke.cmd_ecdsa_sign.digest = dig_dma; + slot->hwc.pke.cmd_ecdsa_sign.signature = sig_dma; + slot->hwc.pke.cmd_ecdsa_sign.secret_key = sk_ref; + slot->hwc.pke.cmd_ecdsa_sign.digest_len = dlen; +} + +/** + * vcq_add_pke_ecdsa_pubgen() - Build a VCQ command for ECDSA public key generation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521) + * @sklen: Secret key length in bytes + * @pk_dma: DMA address of public key output buffer + * @sk_ref: Datastore reference for the secret key + * @flags: VCQ command flags + * + * Generates the public key from an existing private key stored in the + * datastore. + */ +void vcq_add_pke_ecdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 pk_dma, u64 sk_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_PUBGEN); + slot->hwc.pke.cmd_ecdsa_pubgen.curve = curve; + slot->hwc.pke.cmd_ecdsa_pubgen.secret_key_len = sklen; + slot->hwc.pke.cmd_ecdsa_pubgen.public_key = pk_dma; + slot->hwc.pke.cmd_ecdsa_pubgen.secret_key = sk_ref; +} + +/** + * vcq_add_pke_ecdsa_keygen() - Build a VCQ command for ECDSA key pair generation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521) + * @sklen: Secret key length in bytes + * @sk_ref: Datastore reference for the generated secret key + * @sk_type: Datastore type for the secret key object + * @flags: VCQ command flags + */ +void vcq_add_pke_ecdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 sk_ref, u32 sk_type, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDSA_KEYGEN); + slot->hwc.pke.cmd_ecdsa_keygen.curve = curve; + slot->hwc.pke.cmd_ecdsa_keygen.secret_key_len = sklen; + slot->hwc.pke.cmd_ecdsa_keygen.secret_key = sk_ref; + slot->hwc.pke.cmd_ecdsa_keygen.secret_key_type = sk_type; +} + +/* ECDH */ + +/** + * vcq_add_pke_ecdh_keygen() - Build a VCQ command for ECDH key pair generation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521, X25519, X448) + * @sklen: Secret key length in bytes + * @pkx_dma: DMA address of public key X-coordinate output buffer + * @sk_ref: Datastore reference for the generated secret key + * @flags: VCQ command flags + */ +void vcq_add_pke_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 pkx_dma, u64 sk_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDH_KEYGEN); + slot->hwc.pke.cmd_ecdh_keygen.curve = curve; + slot->hwc.pke.cmd_ecdh_keygen.secret_key_len = sklen; + slot->hwc.pke.cmd_ecdh_keygen.public_key_x = pkx_dma; + slot->hwc.pke.cmd_ecdh_keygen.secret_key = sk_ref; +} + +/** + * vcq_add_pke_ecdh() - Build a VCQ command for ECDH shared secret computation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (e.g. NIST P-256, P-384, P-521, X25519, X448) + * @sklen: Secret key length in bytes + * @sslen: Shared secret length in bytes + * @ss_type: Datastore type for the shared secret object + * @peer_dma: DMA address of peer public key buffer + * @sk_ref: Datastore reference for the local secret key + * @ss_ref: Datastore reference for the computed shared secret + * @flags: VCQ command flags + */ +void vcq_add_pke_ecdh(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u32 sslen, u32 ss_type, u64 peer_dma, u64 sk_ref, + u64 ss_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_ECDH); + slot->hwc.pke.cmd_ecdh.curve = curve; + slot->hwc.pke.cmd_ecdh.secret_key_len = sklen; + slot->hwc.pke.cmd_ecdh.shared_secret_len = sslen; + slot->hwc.pke.cmd_ecdh.shared_secret_type = ss_type; + slot->hwc.pke.cmd_ecdh.peer_key_x = peer_dma; + slot->hwc.pke.cmd_ecdh.secret_key = sk_ref; + slot->hwc.pke.cmd_ecdh.shared_secret = ss_ref; +} + +/* EdDSA */ + +/** + * vcq_add_pke_eddsa_verify() - Build a VCQ command for EdDSA signature verification + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (Ed25519 or Ed448) + * @dlen: Digest (message) length in bytes + * @pky_dma: DMA address of public key Y-coordinate buffer + * @dig_dma: DMA address of digest buffer + * @sig_dma: DMA address of signature buffer + * @rp_dma: DMA address of r-prime verification output buffer + * @flags: VCQ command flags + */ +void vcq_add_pke_eddsa_verify(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 dlen, + u64 pky_dma, u64 dig_dma, u64 sig_dma, + u64 rp_dma, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_VERIFY); + slot->hwc.pke.cmd_eddsa_verify.curve = curve; + slot->hwc.pke.cmd_eddsa_verify.digest_len = dlen; + slot->hwc.pke.cmd_eddsa_verify.public_key_y = pky_dma; + slot->hwc.pke.cmd_eddsa_verify.digest = dig_dma; + slot->hwc.pke.cmd_eddsa_verify.signature = sig_dma; + slot->hwc.pke.cmd_eddsa_verify.rprime = rp_dma; +} + +/** + * vcq_add_pke_eddsa_sign() - Build a VCQ command for EdDSA signing + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (Ed25519 or Ed448) + * @sklen: Secret key length in bytes + * @dig_dma: DMA address of digest (message) buffer + * @sig_dma: DMA address of signature output buffer + * @sk_ref: Datastore reference for the secret key + * @dlen: Digest (message) length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_eddsa_sign(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 dig_dma, u64 sig_dma, u64 sk_ref, + u32 dlen, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_SIGN); + slot->hwc.pke.cmd_eddsa_sign.curve = curve; + slot->hwc.pke.cmd_eddsa_sign.secret_key_len = sklen; + slot->hwc.pke.cmd_eddsa_sign.digest = dig_dma; + slot->hwc.pke.cmd_eddsa_sign.signature = sig_dma; + slot->hwc.pke.cmd_eddsa_sign.secret_key = sk_ref; + slot->hwc.pke.cmd_eddsa_sign.digest_len = dlen; +} + +/** + * vcq_add_pke_eddsa_pubgen() - Build a VCQ command for EdDSA public key generation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (Ed25519 or Ed448) + * @sklen: Secret key length in bytes + * @pky_dma: DMA address of public key Y-coordinate output buffer + * @sk_ref: Datastore reference for the secret key + * @flags: VCQ command flags + * + * Generates the public key from an existing private key stored in the + * datastore. + */ +void vcq_add_pke_eddsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 curve, u32 sklen, + u64 pky_dma, u64 sk_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_EDDSA_PUBGEN); + slot->hwc.pke.cmd_eddsa_pubgen.curve = curve; + slot->hwc.pke.cmd_eddsa_pubgen.secret_key_len = sklen; + slot->hwc.pke.cmd_eddsa_pubgen.public_key_y = pky_dma; + slot->hwc.pke.cmd_eddsa_pubgen.secret_key = sk_ref; +} + +/** + * vcq_add_pke_eddsa_keygen_sca() - Build a VCQ command for EdDSA SCA key generation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @curve: Curve identifier (Ed448) + * @sk_ref: Datastore reference for the input secret key + * @sca_sk_ref: Datastore reference for the SCA-masked output key + * + * Blinds an Ed448 private key into a side-channel-protected masked + * form. No byte-swap flags are used (CRI reference uses flags=0). + */ +void vcq_add_pke_eddsa_keygen_sca(struct vcq_cmd *slot, u32 core_id, u32 curve, + u64 sk_ref, u64 sca_sk_ref) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, + PKE_CMD_EDDSA_PRIV_KEYGEN_SCA); + slot->hwc.pke.cmd_eddsa_keygen_sca.curve = curve; + slot->hwc.pke.cmd_eddsa_keygen_sca.secret_key = sk_ref; + slot->hwc.pke.cmd_eddsa_keygen_sca.sca_secret_key = sca_sk_ref; +} + +/* SM2 */ + +/** + * vcq_add_pke_sm2_ecdh_keygen() - Build a VCQ command for SM2 ECDH ephemeral key generation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @nonce_dma: DMA address of nonce input buffer + * @session_key_dma: DMA address of session key output buffer + * @nonce_len: Nonce length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_ecdh_keygen(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma, + u64 session_key_dma, u32 nonce_len, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, + PKE_CMD_SM2_ECDH_KEYGEN); + slot->hwc.pke.cmd_sm2_ecdh_keygen.nonce = nonce_dma; + slot->hwc.pke.cmd_sm2_ecdh_keygen.session_key = session_key_dma; + slot->hwc.pke.cmd_sm2_ecdh_keygen.nonce_len = nonce_len; +} + +/** + * vcq_add_pke_sm2_ecdh() - Build a VCQ command for SM2 ECDH shared secret computation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @nonce_len: Nonce length in bytes + * @private_key_len: Private key length in bytes + * @nonce_dma: DMA address of nonce buffer + * @peer_pk_dma: DMA address of peer public key buffer + * @peer_sk_dma: DMA address of peer session key buffer + * @priv_ref: Datastore reference for the local private key + * @sp_ref: Datastore reference for the shared point output + * @sp_type: Datastore type for the shared point object + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_ecdh(struct vcq_cmd *slot, u32 core_id, u32 nonce_len, + u32 private_key_len, u64 nonce_dma, + u64 peer_pk_dma, u64 peer_sk_dma, + u64 priv_ref, u64 sp_ref, u32 sp_type, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ECDH); + slot->hwc.pke.cmd_sm2_ecdh.nonce_len = nonce_len; + slot->hwc.pke.cmd_sm2_ecdh.private_key_len = private_key_len; + slot->hwc.pke.cmd_sm2_ecdh.nonce = nonce_dma; + slot->hwc.pke.cmd_sm2_ecdh.peer_public_key = peer_pk_dma; + slot->hwc.pke.cmd_sm2_ecdh.peer_session_key = peer_sk_dma; + slot->hwc.pke.cmd_sm2_ecdh.private_key = priv_ref; + slot->hwc.pke.cmd_sm2_ecdh.shared_point = sp_ref; + slot->hwc.pke.cmd_sm2_ecdh.shared_point_type = sp_type; +} + +/** + * vcq_add_pke_sm2_dec_point() - Build a VCQ command for SM2 decryption point multiplication + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @ct_len: Ciphertext length in bytes + * @pk_len: Private key length in bytes + * @ct_dma: DMA address of ciphertext input buffer + * @dp_dma: DMA address of decryption point output buffer + * @priv_ref: Datastore reference for the private key + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_dec_point(struct vcq_cmd *slot, u32 core_id, u32 ct_len, + u32 pk_len, u64 ct_dma, u64 dp_dma, + u64 priv_ref, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_DEC_POINT); + slot->hwc.pke.cmd_sm2_dec_point.ciphertext_len = ct_len; + slot->hwc.pke.cmd_sm2_dec_point.private_key_len = pk_len; + slot->hwc.pke.cmd_sm2_dec_point.ciphertext = ct_dma; + slot->hwc.pke.cmd_sm2_dec_point.dec_point = dp_dma; + slot->hwc.pke.cmd_sm2_dec_point.private_key = priv_ref; +} + +/** + * vcq_add_pke_sm2_enc_point() - Build a VCQ command for SM2 encryption point multiplication + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @nonce_dma: DMA address of nonce buffer + * @pk_dma: DMA address of public key buffer + * @ct_dma: DMA address of ciphertext header output buffer + * @ep_dma: DMA address of encryption point output buffer + * @nonce_len: Nonce length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_enc_point(struct vcq_cmd *slot, u32 core_id, u64 nonce_dma, + u64 pk_dma, u64 ct_dma, u64 ep_dma, + u32 nonce_len, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ENC_POINT); + slot->hwc.pke.cmd_sm2_enc_point.nonce = nonce_dma; + slot->hwc.pke.cmd_sm2_enc_point.public_key = pk_dma; + slot->hwc.pke.cmd_sm2_enc_point.ciphertext = ct_dma; + slot->hwc.pke.cmd_sm2_enc_point.enc_point = ep_dma; + slot->hwc.pke.cmd_sm2_enc_point.nonce_len = nonce_len; +} + +/** + * vcq_add_pke_sm2_id_digest() - Build a VCQ command for SM2 identity digest computation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @id_dma: DMA address of identity string buffer + * @pk_dma: DMA address of public key buffer + * @dig_dma: DMA address of digest output buffer + * @id_len: Identity string length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_id_digest(struct vcq_cmd *slot, u32 core_id, u64 id_dma, + u64 pk_dma, u64 dig_dma, u32 id_len, + u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ID_DIGEST); + slot->hwc.pke.cmd_sm2_id_digest.id = id_dma; + slot->hwc.pke.cmd_sm2_id_digest.public_key = pk_dma; + slot->hwc.pke.cmd_sm2_id_digest.digest = dig_dma; + slot->hwc.pke.cmd_sm2_id_digest.id_len = id_len; +} + +/** + * vcq_add_pke_sm2_ecdh_hash() - Build a VCQ command for SM2 ECDH key derivation hash + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @peer_dig_dma: DMA address of peer identity digest buffer + * @dig_dma: DMA address of local identity digest buffer + * @sp_ref: Datastore reference for the shared point + * @sk_ref: Datastore reference for the derived shared key output + * @sk_type: Datastore type for the shared key object + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_ecdh_hash(struct vcq_cmd *slot, u32 core_id, u64 peer_dig_dma, + u64 dig_dma, u64 sp_ref, u64 sk_ref, + u32 sk_type, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ECDH_HASH); + slot->hwc.pke.cmd_sm2_ecdh_hash.peer_id_digest = peer_dig_dma; + slot->hwc.pke.cmd_sm2_ecdh_hash.id_digest = dig_dma; + slot->hwc.pke.cmd_sm2_ecdh_hash.shared_point = sp_ref; + slot->hwc.pke.cmd_sm2_ecdh_hash.shared_key = sk_ref; + slot->hwc.pke.cmd_sm2_ecdh_hash.shared_key_type = sk_type; +} + +/** + * vcq_add_pke_sm2_dec_hash() - Build a VCQ command for SM2 decryption hash verification + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @ct_dma: DMA address of ciphertext input buffer + * @dp_dma: DMA address of decryption point buffer + * @pt_dma: DMA address of plaintext output buffer + * @ct_len: Ciphertext length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_dec_hash(struct vcq_cmd *slot, u32 core_id, u64 ct_dma, + u64 dp_dma, u64 pt_dma, u32 ct_len, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_DEC_HASH); + slot->hwc.pke.cmd_sm2_dec_hash.ciphertext = ct_dma; + slot->hwc.pke.cmd_sm2_dec_hash.dec_point = dp_dma; + slot->hwc.pke.cmd_sm2_dec_hash.plaintext = pt_dma; + slot->hwc.pke.cmd_sm2_dec_hash.ciphertext_len = ct_len; +} + +/** + * vcq_add_pke_sm2_enc_hash() - Build a VCQ command for SM2 encryption hash computation + * @slot: VCQ command slot to populate + * @core_id: PKE hardware core ID + * @msg_dma: DMA address of plaintext message buffer + * @ep_dma: DMA address of encryption point buffer + * @ct_dma: DMA address of ciphertext output buffer + * @msg_len: Message length in bytes + * @flags: VCQ command flags + */ +void vcq_add_pke_sm2_enc_hash(struct vcq_cmd *slot, u32 core_id, u64 msg_dma, + u64 ep_dma, u64 ct_dma, u32 msg_len, u32 flags) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, flags, 1, PKE_CMD_SM2_ENC_HASH); + slot->hwc.pke.cmd_sm2_enc_hash.message = msg_dma; + slot->hwc.pke.cmd_sm2_enc_hash.enc_point = ep_dma; + slot->hwc.pke.cmd_sm2_enc_hash.ciphertext = ct_dma; + slot->hwc.pke.cmd_sm2_enc_hash.message_len = msg_len; +} diff --git a/drivers/crypto/cmh/cmh_pke_rsa.c b/drivers/crypto/cmh/cmh_pke_rsa.c new file mode 100644 index 00000000000000..010f8bd98f0dc0 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pke_rsa.c @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- RSA akcipher Driver + * + * Registers "rsa" akcipher algorithm with the Linux crypto subsystem + * (priority 300, overrides software rsa-generic at 100). + * + * Raw RSA operations only (m^e mod n / c^d mod n). The kernel's + * pkcs1pad() template wraps this for PKCS#1 v1.5 / PSS / OAEP. + * + * Key format: DER-encoded ASN.1, parsed by kernel rsa_parse_pub_key() + * / rsa_parse_priv_key() helpers. + * + * Private key via cmh_key_ctx: raw keys written via SYS_REF_TEMP. + * Datastore-referenced keys are only reachable through the ioctl + * path (cmh_mgmt.c). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_pke.h" +#include "cmh_sys.h" +#include "cmh_sys_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +struct cmh_rsa_tfm_ctx { + struct cmh_key_ctx key; /* private key (raw d only) */ + u8 *n; /* modulus (big-endian) */ + u8 *e; /* public exponent (big-endian) */ + size_t n_sz; + size_t e_sz; + u32 bits; /* key size in bits */ +}; + +static inline struct cmh_rsa_tfm_ctx *cmh_rsa_ctx(struct crypto_akcipher *tfm) +{ + return akcipher_tfm_ctx(tfm); +} + +struct cmh_rsa_reqctx { + u8 *e_buf; + u8 *n_buf; + u8 *m_buf; + u8 *c_buf; + u8 *d_buf; /* dec only: private key copy */ + dma_addr_t e_dma; + dma_addr_t n_dma; + dma_addr_t m_dma; + dma_addr_t c_dma; + dma_addr_t d_dma; + u32 key_bytes; + u32 e_padded; + u32 n_sz; + u32 d_len; /* dec only */ +}; + +static u32 cmh_rsa_key_bits(size_t n_sz) +{ + /* + * Only accept exact modulus sizes supported by the hardware. + * The programmed RSA width must match the actual modulus buffer + * length; rounding a shorter modulus up to the next size would + * let the device read past the end of the DMA buffer. + */ + switch (n_sz) { + case 64: + return 512; + case 128: + return 1024; + case 256: + return 2048; + case 384: + return 3072; + case 512: + return 4096; + default: + return 0; + } +} + +static void cmh_rsa_enc_complete(void *data, int error) +{ + struct akcipher_request *req = data; + struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (!cmh_dma_map_error(rctx->c_dma)) + cmh_dma_unmap_single(rctx->c_dma, rctx->key_bytes, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->m_dma)) + cmh_dma_unmap_single(rctx->m_dma, rctx->key_bytes, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->n_dma)) + cmh_dma_unmap_single(rctx->n_dma, rctx->n_sz, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->e_dma)) + cmh_dma_unmap_single(rctx->e_dma, rctx->e_padded, + DMA_TO_DEVICE); + + if (!error) { + int nents; + + nents = sg_nents_for_len(req->dst, rctx->key_bytes); + if (nents < 0 || + sg_copy_from_buffer(req->dst, nents, + rctx->c_buf, + rctx->key_bytes) != rctx->key_bytes) + error = -EINVAL; + else + req->dst_len = rctx->key_bytes; + } + + kfree(rctx->c_buf); + rctx->c_buf = NULL; + kfree_sensitive(rctx->m_buf); + rctx->m_buf = NULL; + kfree(rctx->n_buf); + rctx->n_buf = NULL; + kfree(rctx->e_buf); + rctx->e_buf = NULL; + cmh_complete(&req->base, error); +} + +/* + * RSA encrypt: c = m^e mod n (public key operation) + * Also used for signature verification (verify = encrypt for raw RSA). + */ +static int cmh_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req); + u32 key_bytes = ctx->bits / 8; + u32 e_padded = ALIGN(ctx->e_sz, 4); + struct core_dispatch d = cmh_core_select_instance(CMH_CORE_PKE); + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + int ret, nents; + gfp_t gfp; + + if (!ctx->n || !ctx->e) + return -EINVAL; + if (req->src_len > key_bytes || req->dst_len < key_bytes) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->key_bytes = key_bytes; + rctx->e_padded = e_padded; + rctx->n_sz = ctx->n_sz; + rctx->e_dma = DMA_MAPPING_ERROR; + rctx->n_dma = DMA_MAPPING_ERROR; + rctx->m_dma = DMA_MAPPING_ERROR; + rctx->c_dma = DMA_MAPPING_ERROR; + + rctx->e_buf = kzalloc(e_padded, gfp); + rctx->n_buf = kmemdup(ctx->n, ctx->n_sz, gfp); + rctx->m_buf = kzalloc(key_bytes, gfp); + rctx->c_buf = kzalloc(key_bytes, gfp); + if (!rctx->e_buf || !rctx->n_buf || !rctx->m_buf || !rctx->c_buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->e_buf + e_padded - ctx->e_sz, ctx->e, ctx->e_sz); + + nents = sg_nents_for_len(req->src, req->src_len); + if (nents < 0 || + sg_pcopy_to_buffer(req->src, nents, + rctx->m_buf + key_bytes - req->src_len, + req->src_len, 0) != req->src_len) { + ret = -EINVAL; + goto out_free; + } + + rctx->e_dma = cmh_dma_map_single(rctx->e_buf, e_padded, + DMA_TO_DEVICE); + rctx->n_dma = cmh_dma_map_single(rctx->n_buf, ctx->n_sz, + DMA_TO_DEVICE); + rctx->m_dma = cmh_dma_map_single(rctx->m_buf, key_bytes, + DMA_TO_DEVICE); + rctx->c_dma = cmh_dma_map_single(rctx->c_buf, key_bytes, + DMA_FROM_DEVICE); + + if (cmh_dma_map_error(rctx->e_dma) || + cmh_dma_map_error(rctx->n_dma) || + cmh_dma_map_error(rctx->m_dma) || + cmh_dma_map_error(rctx->c_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_rsa_enc(&vcq[1], d.core_id, ctx->bits, e_padded, + rctx->e_dma, rctx->n_dma, rctx->m_dma, + rctx->c_dma, PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[2], d.core_id); + + ret = cmh_tm_submit_async(vcq, PKE_VCQ_CMDS_MIN, 1, d.mbx_idx, + cmh_rsa_enc_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), 0); + if (ret == -EBUSY) + return -EBUSY; + if (!ret) + return -EINPROGRESS; + +out_unmap: + if (!cmh_dma_map_error(rctx->c_dma)) + cmh_dma_unmap_single(rctx->c_dma, key_bytes, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->m_dma)) + cmh_dma_unmap_single(rctx->m_dma, key_bytes, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->n_dma)) + cmh_dma_unmap_single(rctx->n_dma, ctx->n_sz, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->e_dma)) + cmh_dma_unmap_single(rctx->e_dma, e_padded, + DMA_TO_DEVICE); + +out_free: + kfree(rctx->c_buf); + kfree_sensitive(rctx->m_buf); + kfree(rctx->n_buf); + kfree(rctx->e_buf); + return ret; +} + +static void cmh_rsa_dec_complete(void *data, int error) +{ + struct akcipher_request *req = data; + struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (!cmh_dma_map_error(rctx->d_dma)) + cmh_dma_unmap_single(rctx->d_dma, rctx->d_len, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->m_dma)) + cmh_dma_unmap_single(rctx->m_dma, rctx->key_bytes, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->c_dma)) + cmh_dma_unmap_single(rctx->c_dma, rctx->key_bytes, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->n_dma)) + cmh_dma_unmap_single(rctx->n_dma, rctx->n_sz, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->e_dma)) + cmh_dma_unmap_single(rctx->e_dma, rctx->e_padded, + DMA_TO_DEVICE); + + if (!error) { + int nents; + + nents = sg_nents_for_len(req->dst, rctx->key_bytes); + if (nents < 0 || + sg_copy_from_buffer(req->dst, nents, + rctx->m_buf, + rctx->key_bytes) != rctx->key_bytes) + error = -EINVAL; + else + req->dst_len = rctx->key_bytes; + } + + kfree_sensitive(rctx->d_buf); + rctx->d_buf = NULL; + kfree_sensitive(rctx->m_buf); + rctx->m_buf = NULL; + kfree(rctx->c_buf); + rctx->c_buf = NULL; + kfree(rctx->n_buf); + rctx->n_buf = NULL; + kfree(rctx->e_buf); + rctx->e_buf = NULL; + cmh_complete(&req->base, error); +} + +/* + * RSA decrypt: m = c^d mod n (private key operation) + * Also used for signing (sign = decrypt for raw RSA). + * + * Private key 'd' is written via SYS_REF_TEMP inline. + */ +static int cmh_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + struct cmh_rsa_reqctx *rctx = akcipher_request_ctx(req); + u32 key_bytes = ctx->bits / 8; + u32 e_padded = ALIGN(ctx->e_sz, 4); + struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX]; + struct core_dispatch dd; + int ret, idx, nents; + gfp_t gfp; + + if (ctx->key.mode != CMH_KEY_RAW) + return -EINVAL; + if (!ctx->n || !ctx->e) + return -EINVAL; + if (req->src_len > key_bytes || req->dst_len < key_bytes) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->key_bytes = key_bytes; + rctx->e_padded = e_padded; + rctx->n_sz = ctx->n_sz; + rctx->e_dma = DMA_MAPPING_ERROR; + rctx->n_dma = DMA_MAPPING_ERROR; + rctx->m_dma = DMA_MAPPING_ERROR; + rctx->c_dma = DMA_MAPPING_ERROR; + rctx->d_dma = DMA_MAPPING_ERROR; + + rctx->e_buf = kzalloc(e_padded, gfp); + rctx->n_buf = kmemdup(ctx->n, ctx->n_sz, gfp); + rctx->c_buf = kzalloc(key_bytes, gfp); + rctx->m_buf = kzalloc(key_bytes, gfp); + if (!rctx->e_buf || !rctx->n_buf || !rctx->c_buf || !rctx->m_buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->e_buf + e_padded - ctx->e_sz, ctx->e, ctx->e_sz); + + nents = sg_nents_for_len(req->src, req->src_len); + if (nents < 0 || + sg_pcopy_to_buffer(req->src, nents, + rctx->c_buf + key_bytes - req->src_len, + req->src_len, 0) != req->src_len) { + ret = -EINVAL; + goto out_free; + } + + rctx->e_dma = cmh_dma_map_single(rctx->e_buf, e_padded, + DMA_TO_DEVICE); + rctx->n_dma = cmh_dma_map_single(rctx->n_buf, ctx->n_sz, + DMA_TO_DEVICE); + rctx->c_dma = cmh_dma_map_single(rctx->c_buf, key_bytes, + DMA_TO_DEVICE); + rctx->m_dma = cmh_dma_map_single(rctx->m_buf, key_bytes, + DMA_FROM_DEVICE); + + if (cmh_dma_map_error(rctx->e_dma) || + cmh_dma_map_error(rctx->n_dma) || + cmh_dma_map_error(rctx->c_dma) || + cmh_dma_map_error(rctx->m_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + dd = cmh_core_select_instance(CMH_CORE_PKE); + + rctx->d_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, gfp); + if (!rctx->d_buf) { + ret = -ENOMEM; + goto out_unmap; + } + rctx->d_len = ctx->key.raw.len; + + rctx->d_dma = cmh_dma_map_single(rctx->d_buf, ctx->key.raw.len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->d_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + idx = 1; + vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, rctx->d_dma, + SYS_REF_NONE, ctx->key.raw.len, + ctx->key.raw.sys_type); + vcq[idx].id |= PKE_SWAP_FLAGS; + idx++; + vcq_add_pke_rsa_dec(&vcq[idx++], dd.core_id, ctx->bits, e_padded, + rctx->e_dma, rctx->n_dma, rctx->c_dma, + rctx->m_dma, SYS_REF_TEMP, PKE_SWAP_FLAGS); + vcq_add_pke_flush(&vcq[idx++], dd.core_id); + vcq_set_header(&vcq[0], idx); + + ret = cmh_tm_submit_async(vcq, idx, 1, dd.mbx_idx, + cmh_rsa_dec_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), 0); + if (ret == -EBUSY) + return -EBUSY; + if (!ret) + return -EINPROGRESS; + +out_unmap: + if (!cmh_dma_map_error(rctx->d_dma)) + cmh_dma_unmap_single(rctx->d_dma, rctx->d_len, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->m_dma)) + cmh_dma_unmap_single(rctx->m_dma, key_bytes, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->c_dma)) + cmh_dma_unmap_single(rctx->c_dma, key_bytes, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->n_dma)) + cmh_dma_unmap_single(rctx->n_dma, ctx->n_sz, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->e_dma)) + cmh_dma_unmap_single(rctx->e_dma, e_padded, + DMA_TO_DEVICE); + +out_free: + kfree_sensitive(rctx->d_buf); + kfree_sensitive(rctx->m_buf); + kfree(rctx->c_buf); + kfree(rctx->n_buf); + kfree(rctx->e_buf); + return ret; +} + +static int cmh_rsa_set_pub_key(struct crypto_akcipher *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + struct rsa_key rsa = {}; + int ret; + + ret = rsa_parse_pub_key(&rsa, key, keylen); + if (ret) + return ret; + + /* Strip ASN.1 leading zero padding from modulus */ + while (rsa.n_sz > 0 && rsa.n[0] == 0) { + rsa.n++; + rsa.n_sz--; + } + + ctx->bits = cmh_rsa_key_bits(rsa.n_sz); + if (!ctx->bits) + return -EINVAL; + + kfree(ctx->n); + kfree(ctx->e); + ctx->n = NULL; + ctx->e = NULL; + ctx->n_sz = 0; + ctx->e_sz = 0; + + ctx->n = kmemdup(rsa.n, rsa.n_sz, GFP_KERNEL); + ctx->e = kmemdup(rsa.e, rsa.e_sz, GFP_KERNEL); + if (!ctx->n || !ctx->e) { + kfree(ctx->n); + kfree(ctx->e); + ctx->n = NULL; + ctx->e = NULL; + return -ENOMEM; + } + + ctx->n_sz = rsa.n_sz; + ctx->e_sz = rsa.e_sz; + + return 0; +} + +static int cmh_rsa_set_priv_key(struct crypto_akcipher *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + struct rsa_key rsa = {}; + u32 key_bytes; + u8 *d_padded; + int ret; + + ret = rsa_parse_priv_key(&rsa, key, keylen); + if (ret) + return ret; + + /* Strip ASN.1 leading zero padding from modulus */ + while (rsa.n_sz > 0 && rsa.n[0] == 0) { + rsa.n++; + rsa.n_sz--; + } + + ctx->bits = cmh_rsa_key_bits(rsa.n_sz); + if (!ctx->bits || !rsa.d_sz) + return -EINVAL; + + key_bytes = ctx->bits / 8; + + /* Strip ASN.1 leading zero padding from private exponent */ + while (rsa.d_sz > 0 && rsa.d[0] == 0) { + rsa.d++; + rsa.d_sz--; + } + + if (!rsa.d_sz || rsa.d_sz > key_bytes) + return -EINVAL; + + kfree(ctx->n); + kfree(ctx->e); + ctx->n = NULL; + ctx->e = NULL; + ctx->n_sz = 0; + ctx->e_sz = 0; + + ctx->n = kmemdup(rsa.n, rsa.n_sz, GFP_KERNEL); + ctx->e = kmemdup(rsa.e, rsa.e_sz, GFP_KERNEL); + if (!ctx->n || !ctx->e) { + ret = -ENOMEM; + goto err; + } + + ctx->n_sz = rsa.n_sz; + ctx->e_sz = rsa.e_sz; + + /* + * Left-pad d to key_bytes (big-endian alignment). + * The CMH eSW resolves SYS_REF_TEMP by checking + * hdr->len >= key_bytes, so the written buffer must + * be at least key_bytes wide. + */ + d_padded = kzalloc(key_bytes, GFP_KERNEL); + if (!d_padded) { + ret = -ENOMEM; + goto err; + } + memcpy(d_padded + key_bytes - rsa.d_sz, rsa.d, rsa.d_sz); + + ret = cmh_key_setkey_raw(&ctx->key, d_padded, key_bytes, + CORE_ID_PKE); + kfree_sensitive(d_padded); + if (ret) + goto err; + + return 0; +err: + kfree(ctx->n); + kfree(ctx->e); + ctx->n = NULL; + ctx->e = NULL; + ctx->n_sz = 0; + ctx->e_sz = 0; + ctx->bits = 0; + return ret; +} + +static unsigned int cmh_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + + return ctx->n_sz; +} + +static int cmh_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + tfm->reqsize = sizeof(struct cmh_rsa_reqctx); + return 0; +} + +static void cmh_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct cmh_rsa_tfm_ctx *ctx = cmh_rsa_ctx(tfm); + + cmh_key_destroy(&ctx->key); + kfree(ctx->n); + kfree(ctx->e); + ctx->n = NULL; + ctx->e = NULL; +} + +/* + * Raw RSA stays as akcipher (encrypt/decrypt only). The kernel's + * rsassa-pkcs1 sig template wraps our akcipher for sign/verify, + * matching the upstream split (rsa.c = akcipher, + * rsassa-pkcs1.c = sig template). + */ +static struct akcipher_alg cmh_rsa_alg = { + .encrypt = cmh_rsa_enc, + .decrypt = cmh_rsa_dec, + .set_pub_key = cmh_rsa_set_pub_key, + .set_priv_key = cmh_rsa_set_priv_key, + .max_size = cmh_rsa_max_size, + .init = cmh_rsa_init_tfm, + .exit = cmh_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "cri-cmh-rsa", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_rsa_tfm_ctx), + }, +}; + +static bool cmh_rsa_registered; + +/** + * cmh_pke_rsa_register() - Register RSA akcipher algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pke_rsa_register(void) +{ + int ret; + + ret = crypto_register_akcipher(&cmh_rsa_alg); + if (ret) { + dev_err(cmh_dev(), + "cmh: failed to register rsa akcipher (%d)\n", + ret); + return ret; + } + + cmh_rsa_registered = true; + return 0; +} + +/** + * cmh_pke_rsa_unregister() - Unregister RSA akcipher algorithm from the crypto framework + */ +void cmh_pke_rsa_unregister(void) +{ + if (cmh_rsa_registered) + crypto_unregister_akcipher(&cmh_rsa_alg); + cmh_rsa_registered = false; +} From a20bac6acc9cee1fbf49ec6225ef22828e4893a0 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:31 -0700 Subject: [PATCH 15/21] crypto: cmh - add ECDSA/SM2 sig Register ECDSA and SM2 sig algorithms using the CMH PKE core. Supports P-256, P-384, P-521, and SM2 curves for sign and verify operations. SM2 is registered as verify-only via the crypto API; full SM2 operations (encrypt, decrypt, key exchange) are available through the /dev/cmh_mgmt ioctl interface. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 3 +- drivers/crypto/cmh/cmh_main.c | 8 + drivers/crypto/cmh/cmh_pke_ecdsa.c | 575 +++++++++++++++++++++++++++++ 3 files changed, 585 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_pke_ecdsa.c diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 7afd9852c33753..fdbf66b13628ae 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -31,7 +31,8 @@ cmh-y := \ cmh_ccp_poly.o \ cmh_rng.o \ cmh_pke_common.o \ - cmh_pke_rsa.o + cmh_pke_rsa.o \ + cmh_pke_ecdsa.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 07f26b0dd2ef54..2191682f3d5436 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -287,6 +287,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_pke_rsa_register; + /* Register PKE ECDSA/SM2 sig */ + ret = cmh_pke_ecdsa_register(); + if (ret) + goto err_pke_ecdsa_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -299,6 +304,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_pke_ecdsa_unregister(); +err_pke_ecdsa_register: cmh_pke_rsa_unregister(); err_pke_rsa_register: cmh_ccp_poly_unregister(); @@ -357,6 +364,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_pke_ecdsa_unregister(); cmh_pke_rsa_unregister(); cmh_ccp_poly_unregister(); cmh_ccp_aead_unregister(); diff --git a/drivers/crypto/cmh/cmh_pke_ecdsa.c b/drivers/crypto/cmh/cmh_pke_ecdsa.c new file mode 100644 index 00000000000000..6b65f7fb72cc9d --- /dev/null +++ b/drivers/crypto/cmh/cmh_pke_ecdsa.c @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- ECDSA / SM2 Signature Driver (sig_alg, synchronous) + * + * Registers "ecdsa-nist-p256", "ecdsa-nist-p384", and "ecdsa-nist-p521" + * sig algorithms with sign, verify, set_pub_key, and set_priv_key callbacks. + * Registers "sm2" as verify-only (set_pub_key + verify); SM2 sign is + * provided via the cmh_mgmt ioctl path in cmh_pke_sm2.c. + * + * In-kernel consumers typically use verify-only (module signatures, IMA), + * but we provide sign as well for completeness -- matching the CMH eSW + * capability. + * + * Key format: Public key = raw 04 || X || Y (uncompressed). + * Signature format: struct ecdsa_raw_sig (two u64[ECC_MAX_DIGITS] arrays + * in VLI format -- native byte order, LE digit order) for both sign + * output and verify input. This matches the kernel crypto sig API. + * + * Private key via cmh_key_ctx: raw keys written via SYS_REF_TEMP. + * Datastore-referenced keys are only reachable through the ioctl + * path (cmh_mgmt.c). + * + * SM2 note: The SM2 sig entry is verify-only (no sign/set_priv_key). + * SM2 signature verification requires the digest to be SM3(ZA || M) + * where ZA = SM3(ENTLA || IDA || a || b || xG || yG || xA || yA). + * The ZA identity pre-hash is the caller's responsibility; the driver + * passes the digest directly to the CMH eSW SM2 verify engine. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_pke.h" +#include "cmh_sys.h" +#include "cmh_sys_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * Number of ECC digits needed for a given coordinate byte length. + * P-256: 4, P-384: 6, P-521/SM2(clen=68): 9. + */ +static inline unsigned int clen_to_ndigits(u32 clen) +{ + return DIV_ROUND_UP(clen, sizeof(u64)); +} + +struct cmh_ecdsa_tfm_ctx { + struct cmh_key_ctx key; /* private key (raw only) */ + u8 *pub_key; /* uncompressed (x, y) without 04 prefix */ + u32 pub_key_len; + u32 curve; /* PKE_CURVE_* */ + u32 clen; /* coordinate length in bytes */ +}; + +static inline struct cmh_ecdsa_tfm_ctx *cmh_ecdsa_ctx(struct crypto_sig *tfm) +{ + return crypto_sig_ctx(tfm); +} + +/* + * Convert one VLI component (u64 array, LE digit order, native byte order) + * to big-endian byte array of @out_len bytes. The VLI value is right-aligned + * in the output (leading zero bytes if ndigits*8 > out_len are discarded; + * leading zero padding added if ndigits*8 < out_len). + */ +static void ecdsa_vli_to_be(const u64 *vli, unsigned int ndigits, + u8 *out, unsigned int out_len) +{ + unsigned int full_len = ndigits * sizeof(u64); + unsigned int i, skip; + + memset(out, 0, out_len); + + if (full_len <= out_len) { + /* VLI fits entirely -- write at right end of out */ + u8 *dst = out + (out_len - full_len); + + for (i = 0; i < ndigits; i++) + put_unaligned_be64(vli[ndigits - 1 - i], + &dst[i * sizeof(u64)]); + } else { + /* VLI wider than out -- skip leading (zero) bytes */ + u8 tmp[ECC_MAX_BYTES]; + + for (i = 0; i < ndigits; i++) + put_unaligned_be64(vli[ndigits - 1 - i], + &tmp[i * sizeof(u64)]); + skip = full_len - out_len; + WARN_ON_ONCE(memchr_inv(tmp, 0, skip)); + memcpy(out, tmp + skip, out_len); + } +} + +/* + * Convert big-endian byte array to VLI (u64 array, LE digit order). + * Output is zero-filled to @max_digits entries. + */ +static void ecdsa_be_to_vli(const u8 *in, unsigned int in_len, + u64 *vli, unsigned int max_digits) +{ + unsigned int full_len = max_digits * sizeof(u64); + u8 tmp[ECC_MAX_BYTES]; + unsigned int i; + + if (WARN_ON_ONCE(max_digits > ECC_MAX_DIGITS)) + max_digits = ECC_MAX_DIGITS; + + memset(tmp, 0, full_len); + if (in_len <= full_len) + memcpy(tmp + (full_len - in_len), in, in_len); + else + memcpy(tmp, in + (in_len - full_len), full_len); + + for (i = 0; i < max_digits; i++) { + unsigned int off = (max_digits - 1 - i) * sizeof(u64); + + vli[i] = get_unaligned_be64(&tmp[off]); + } +} + +/* + * Extract raw (r || s) big-endian byte arrays from struct ecdsa_raw_sig. + * Each component is written as @clen bytes into @raw_rs. + */ +static int ecdsa_sig_to_raw(const void *src, unsigned int slen, + u8 *raw_rs, u32 clen) +{ + const struct ecdsa_raw_sig *sig = src; + unsigned int ndigits = clen_to_ndigits(clen); + + if (slen != sizeof(struct ecdsa_raw_sig)) + return -EINVAL; + + ecdsa_vli_to_be(sig->r, ndigits, raw_rs, clen); + ecdsa_vli_to_be(sig->s, ndigits, raw_rs + clen, clen); + return 0; +} + +/* + * Encode raw (r || s) big-endian byte arrays into struct ecdsa_raw_sig. + * Returns sizeof(struct ecdsa_raw_sig) on success. + */ +static int ecdsa_raw_to_sig(const u8 *raw_rs, u32 clen, + void *dst, unsigned int dlen) +{ + struct ecdsa_raw_sig *sig = dst; + + if (dlen < sizeof(struct ecdsa_raw_sig)) + return -ENOSPC; + + memset(sig, 0, sizeof(*sig)); + ecdsa_be_to_vli(raw_rs, clen, sig->r, ECC_MAX_DIGITS); + ecdsa_be_to_vli(raw_rs + clen, clen, sig->s, ECC_MAX_DIGITS); + return sizeof(struct ecdsa_raw_sig); +} + +/* + * ECDSA verify (synchronous sig_alg) + * + * @src: struct ecdsa_raw_sig (VLI format) + * @slen: signature length (must be sizeof(struct ecdsa_raw_sig)) + * @digest: hash digest + * @dlen: digest length + * + * Returns 0 on successful verification, negative errno on failure. + */ +static int cmh_ecdsa_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + u32 clen = ctx->clen; + u32 sig_raw_len = 2 * clen; + u32 copy_len = min_t(u32, dlen, clen); + struct core_dispatch d = cmh_core_select_instance(CMH_CORE_PKE); + struct vcq_cmd vcq[PKE_VCQ_CMDS_MIN]; + u8 *sig_raw = NULL, *dig_buf = NULL, *pk_buf = NULL, *rp_buf = NULL; + dma_addr_t pk_dma, dig_dma, sig_dma, rp_dma; + int ret; + + if (!ctx->pub_key) + return -EINVAL; + + sig_raw = kzalloc(sig_raw_len, GFP_KERNEL); + dig_buf = kzalloc(clen, GFP_KERNEL); + pk_buf = kmemdup(ctx->pub_key, ctx->pub_key_len, GFP_KERNEL); + rp_buf = kzalloc(clen, GFP_KERNEL); + if (!sig_raw || !dig_buf || !pk_buf || !rp_buf) { + ret = -ENOMEM; + goto out_free; + } + + /* Extract raw (r, s) big-endian from VLI signature */ + ret = ecdsa_sig_to_raw(src, slen, sig_raw, clen); + if (ret) + goto out_free; + + /* + * Truncate or zero-pad digest to clen bytes, right-aligned. + * Matches ECDSA bits2int: use leftmost min(dlen, clen) bytes, + * zero-pad on the left when dlen < clen. + */ + memcpy(dig_buf + (clen - copy_len), digest, copy_len); + + pk_dma = cmh_dma_map_single(pk_buf, ctx->pub_key_len, DMA_TO_DEVICE); + dig_dma = cmh_dma_map_single(dig_buf, clen, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_raw, sig_raw_len, DMA_TO_DEVICE); + rp_dma = cmh_dma_map_single(rp_buf, clen, DMA_FROM_DEVICE); + + if (cmh_dma_map_error(pk_dma) || cmh_dma_map_error(dig_dma) || + cmh_dma_map_error(sig_dma) || cmh_dma_map_error(rp_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MIN); + vcq_add_pke_ecdsa_verify(&vcq[1], d.core_id, ctx->curve, clen, + pk_dma, dig_dma, sig_dma, rp_dma, + pke_swap_flags(ctx->curve)); + vcq_add_pke_flush(&vcq[2], d.core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, PKE_VCQ_CMDS_MIN, 1, d.mbx_idx); + +out_unmap: + if (!cmh_dma_map_error(rp_dma)) + cmh_dma_unmap_single(rp_dma, clen, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_raw_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(dig_dma)) + cmh_dma_unmap_single(dig_dma, clen, DMA_TO_DEVICE); + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, ctx->pub_key_len, DMA_TO_DEVICE); + +out_free: + kfree(rp_buf); + kfree(pk_buf); + kfree(sig_raw); + kfree(dig_buf); + return ret; +} + +/* + * ECDSA sign (synchronous sig_alg) + * + * @src: hash digest + * @slen: digest length + * @dst: output buffer for struct ecdsa_raw_sig (VLI format) + * @dlen: output buffer length + * + * Returns sizeof(struct ecdsa_raw_sig) on success, negative errno on failure. + */ +static int cmh_ecdsa_sign(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + u32 clen = ctx->clen; + u32 sig_raw_len = 2 * clen; + u32 copy_len = min_t(u32, slen, clen); + struct core_dispatch dd; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX]; + u8 *dig_buf = NULL, *sig_buf = NULL, *sk_buf = NULL; + dma_addr_t dig_dma, sig_dma, sk_dma; + int ret, idx; + + if (ctx->key.mode != CMH_KEY_RAW) + return -EINVAL; + if (dlen < sizeof(struct ecdsa_raw_sig)) + return -EINVAL; + + dig_buf = kzalloc(clen, GFP_KERNEL); + sig_buf = kzalloc(sig_raw_len, GFP_KERNEL); + sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, GFP_KERNEL); + if (!dig_buf || !sig_buf || !sk_buf) { + ret = -ENOMEM; + goto out_free; + } + + /* + * Truncate or zero-pad digest to clen bytes, right-aligned. + * Matches ECDSA bits2int: use leftmost min(slen, clen) bytes, + * zero-pad on the left when slen < clen. + */ + memcpy(dig_buf + (clen - copy_len), src, copy_len); + + dig_dma = cmh_dma_map_single(dig_buf, clen, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_buf, sig_raw_len, DMA_FROM_DEVICE); + sk_dma = cmh_dma_map_single(sk_buf, ctx->key.raw.len, DMA_TO_DEVICE); + + if (cmh_dma_map_error(dig_dma) || cmh_dma_map_error(sig_dma) || + cmh_dma_map_error(sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + dd = cmh_core_select_instance(CMH_CORE_PKE); + + idx = 1; + vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, sk_dma, + SYS_REF_NONE, ctx->key.raw.len, + ctx->key.raw.sys_type); + vcq[idx].id |= pke_swap_flags(ctx->curve); + idx++; + vcq_add_pke_ecdsa_sign(&vcq[idx++], dd.core_id, ctx->curve, clen, + dig_dma, sig_dma, SYS_REF_TEMP, + clen, pke_swap_flags(ctx->curve)); + vcq_add_pke_flush(&vcq[idx++], dd.core_id); + vcq_set_header(&vcq[0], idx); + + ret = cmh_tm_submit_sync_mbx(vcq, idx, 1, dd.mbx_idx); + if (!ret) { + /* Sync bounce buffer so CPU sees the DMA-written signature */ + cmh_dma_sync_for_cpu(sig_dma, sig_raw_len, DMA_FROM_DEVICE); + + /* Encode raw (r||s) into VLI ecdsa_raw_sig for kernel API */ + ret = ecdsa_raw_to_sig(sig_buf, clen, dst, dlen); + } + +out_unmap: + if (!cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, ctx->key.raw.len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_raw_len, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(dig_dma)) + cmh_dma_unmap_single(dig_dma, clen, DMA_TO_DEVICE); + +out_free: + kfree_sensitive(sk_buf); + kfree(sig_buf); + kfree(dig_buf); + return ret; +} + +static int cmh_ecdsa_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + const u8 *d = key; + u32 clen = ctx->clen; + u32 raw_clen; + + /* Accept 04 || X || Y (uncompressed point) */ + if (keylen < 1 || d[0] != 0x04) + return -EINVAL; + d++; + keylen--; + + if (keylen & 1) + return -EINVAL; + raw_clen = keylen / 2; + + /* + * Kernel passes ceil(bits/8) per coordinate (e.g. 66 for P-521), + * but our HW ABI uses clen (ALIGN(66,4)=68 for P-521). + * Accept raw_clen <= clen and zero-pad on the left. + */ + if (raw_clen > clen || raw_clen == 0) + return -EINVAL; + + kfree(ctx->pub_key); + ctx->pub_key = NULL; + ctx->pub_key_len = 0; + + ctx->pub_key = kzalloc(2 * clen, GFP_KERNEL); + if (!ctx->pub_key) + return -ENOMEM; + + /* Right-align each coordinate to clen bytes */ + memcpy(ctx->pub_key + (clen - raw_clen), d, raw_clen); + memcpy(ctx->pub_key + clen + (clen - raw_clen), d + raw_clen, + raw_clen); + ctx->pub_key_len = 2 * clen; + return 0; +} + +static int cmh_ecdsa_set_priv_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + if (keylen != ctx->clen) + return -EINVAL; + + return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_PKE); +} + +static unsigned int cmh_ecdsa_key_size(struct crypto_sig *tfm) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + /* crypto_sig_keysize() returns bits, not bytes */ + return pke_curve_bits(ctx->curve); +} + +static unsigned int cmh_ecdsa_max_size(struct crypto_sig *tfm) +{ + return sizeof(struct ecdsa_raw_sig); +} + +static unsigned int cmh_ecdsa_digest_size(struct crypto_sig *tfm) +{ + /* + * Accept digests up to SHA-512 (64 bytes). Digests longer + * than the curve order are truncated per ECDSA bits2int. + * Matches kernel ecdsa_digest_size(). + */ + return SHA512_DIGEST_SIZE; +} + +static int cmh_ecdsa_p256_init(struct crypto_sig *tfm) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_P256; + ctx->clen = pke_curve_clen(PKE_CURVE_P256); + return 0; +} + +static int cmh_ecdsa_p384_init(struct crypto_sig *tfm) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_P384; + ctx->clen = pke_curve_clen(PKE_CURVE_P384); + return 0; +} + +static int cmh_ecdsa_p521_init(struct crypto_sig *tfm) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_P521; + ctx->clen = pke_curve_clen(PKE_CURVE_P521); + return 0; +} + +static int cmh_sm2_init(struct crypto_sig *tfm) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_SM2; + ctx->clen = pke_curve_clen(PKE_CURVE_SM2); + return 0; +} + +static void cmh_ecdsa_exit(struct crypto_sig *tfm) +{ + struct cmh_ecdsa_tfm_ctx *ctx = cmh_ecdsa_ctx(tfm); + + cmh_key_destroy(&ctx->key); + kfree(ctx->pub_key); + ctx->pub_key = NULL; +} + +static struct sig_alg cmh_ecdsa_algs[] = { + { + .sign = cmh_ecdsa_sign, + .verify = cmh_ecdsa_verify, + .set_pub_key = cmh_ecdsa_set_pub_key, + .set_priv_key = cmh_ecdsa_set_priv_key, + .key_size = cmh_ecdsa_key_size, + .max_size = cmh_ecdsa_max_size, + .digest_size = cmh_ecdsa_digest_size, + .init = cmh_ecdsa_p256_init, + .exit = cmh_ecdsa_exit, + .base = { + .cra_name = "ecdsa-nist-p256", + .cra_driver_name = "cri-cmh-ecdsa-nist-p256", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx), + }, + }, + { + .sign = cmh_ecdsa_sign, + .verify = cmh_ecdsa_verify, + .set_pub_key = cmh_ecdsa_set_pub_key, + .set_priv_key = cmh_ecdsa_set_priv_key, + .key_size = cmh_ecdsa_key_size, + .max_size = cmh_ecdsa_max_size, + .digest_size = cmh_ecdsa_digest_size, + .init = cmh_ecdsa_p384_init, + .exit = cmh_ecdsa_exit, + .base = { + .cra_name = "ecdsa-nist-p384", + .cra_driver_name = "cri-cmh-ecdsa-nist-p384", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx), + }, + }, + { + .sign = cmh_ecdsa_sign, + .verify = cmh_ecdsa_verify, + .set_pub_key = cmh_ecdsa_set_pub_key, + .set_priv_key = cmh_ecdsa_set_priv_key, + .key_size = cmh_ecdsa_key_size, + .max_size = cmh_ecdsa_max_size, + .digest_size = cmh_ecdsa_digest_size, + .init = cmh_ecdsa_p521_init, + .exit = cmh_ecdsa_exit, + .base = { + .cra_name = "ecdsa-nist-p521", + .cra_driver_name = "cri-cmh-ecdsa-nist-p521", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx), + }, + }, + { + .verify = cmh_ecdsa_verify, + .set_pub_key = cmh_ecdsa_set_pub_key, + .key_size = cmh_ecdsa_key_size, + .max_size = cmh_ecdsa_max_size, + .digest_size = cmh_ecdsa_digest_size, + .init = cmh_sm2_init, + .exit = cmh_ecdsa_exit, + .base = { + .cra_name = "sm2", + .cra_driver_name = "cri-cmh-sm2", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdsa_tfm_ctx), + }, + }, +}; + +/** + * cmh_pke_ecdsa_register() - Register ECDSA/SM2 sig algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pke_ecdsa_register(void) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(cmh_ecdsa_algs); i++) { + ret = crypto_register_sig(&cmh_ecdsa_algs[i]); + if (ret) { + dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n", + cmh_ecdsa_algs[i].base.cra_name, ret); + goto err_unregister; + } + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_sig(&cmh_ecdsa_algs[i]); + return ret; +} + +/** + * cmh_pke_ecdsa_unregister() - Unregister ECDSA/SM2 sig algorithms from the crypto framework + */ +void cmh_pke_ecdsa_unregister(void) +{ + int i = ARRAY_SIZE(cmh_ecdsa_algs); + + while (i--) + crypto_unregister_sig(&cmh_ecdsa_algs[i]); +} From b71251daa1fc60966fa3ecb9b484c5a3f686f962 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:32 -0700 Subject: [PATCH 16/21] crypto: cmh - add ECDH/X25519 kpp Register ECDH and X25519 kpp algorithms using the CMH PKE core. Supports P-256, P-384, and Curve25519 for key agreement. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 3 +- drivers/crypto/cmh/cmh_main.c | 8 + drivers/crypto/cmh/cmh_pke_ecdh.c | 698 ++++++++++++++++++++++++++++++ 3 files changed, 708 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_pke_ecdh.c diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index fdbf66b13628ae..a4cea0a56fc17f 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -32,7 +32,8 @@ cmh-y := \ cmh_rng.o \ cmh_pke_common.o \ cmh_pke_rsa.o \ - cmh_pke_ecdsa.o + cmh_pke_ecdsa.o \ + cmh_pke_ecdh.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 2191682f3d5436..dd4e8812c457eb 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -292,6 +292,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_pke_ecdsa_register; + /* Register PKE ECDH/X25519 kpp */ + ret = cmh_pke_ecdh_register(); + if (ret) + goto err_pke_ecdh_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -304,6 +309,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_pke_ecdh_unregister(); +err_pke_ecdh_register: cmh_pke_ecdsa_unregister(); err_pke_ecdsa_register: cmh_pke_rsa_unregister(); @@ -364,6 +371,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_pke_ecdh_unregister(); cmh_pke_ecdsa_unregister(); cmh_pke_rsa_unregister(); cmh_ccp_poly_unregister(); diff --git a/drivers/crypto/cmh/cmh_pke_ecdh.c b/drivers/crypto/cmh/cmh_pke_ecdh.c new file mode 100644 index 00000000000000..d8b821cc4217c1 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pke_ecdh.c @@ -0,0 +1,698 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- ECDH / X25519 kpp Driver + * + * Registers "ecdh-nist-p256", "ecdh-nist-p384", and "curve25519" + * kpp algorithms with priority 300. + * + * - set_secret: decodes private key from kpp_secret + ecdh struct + * (NIST curves) or raw 32-byte scalar (Curve25519). + * Stores in cmh_key_ctx: raw keys written via SYS_REF_TEMP. + * Datastore-referenced keys are only reachable through the ioctl + * path (cmh_mgmt.c). + * + * - generate_public_key: PKE_CMD_ECDH_KEYGEN -> outputs X coordinate + * (NIST Weierstrass) or full public key (Edwards/Montgomery). + * For NIST curves, we generate X||Y by calling ECDSA_PUBGEN instead, + * matching the kernel ecdh.c pattern that outputs uncompressed X||Y. + * + * - compute_shared_secret: PKE_CMD_ECDH -> shared secret X coordinate. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmh_pke.h" +#include "cmh_sys.h" +#include "cmh_sys_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * ECDH key format: kpp_secret header + key_size(u16) + key data. + * We decode this inline to avoid depending on CONFIG_CRYPTO_ECDH. + */ +#define ECDH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + sizeof(unsigned short)) + +struct cmh_ecdh_tfm_ctx { + struct cmh_key_ctx key; + u32 curve; /* PKE_CURVE_* */ + u32 clen; /* coordinate length in bytes */ +}; + +static inline struct cmh_ecdh_tfm_ctx *cmh_ecdh_ctx(struct crypto_kpp *tfm) +{ + return kpp_tfm_ctx(tfm); +} + +/* + * Per-request context for ECDH/X25519 operations. + * + * generate_public_key: single-phase async VCQ. + * compute_shared_secret: 2-phase async VCQ with callback chaining. + * Phase 1: sys_write(sk) + sys_new(ref) + ecdh(peer) + pflush + * -> phase1 callback reads ref, submits Phase 2. + * Phase 2: sys_data(ref, ss_dma) + sys_flush + * -> phase2 callback extracts shared secret, completes req. + * + * Both phases target the same mbx_idx so the DS reference remains + * valid, since DS objects are MBX-scoped. + */ +struct cmh_ecdh_reqctx { + /* Buffers */ + u8 *pk_buf; /* keygen: output public key */ + u8 *sk_buf; /* private key copy */ + u8 *peer_buf; /* compute: peer public key */ + u8 *ss_buf; /* compute: shared secret output */ + u64 *ref_buf; /* compute: DS ref from Phase 1 */ + /* DMA handles */ + dma_addr_t pk_dma; + dma_addr_t sk_dma; + dma_addr_t peer_dma; + dma_addr_t ss_dma; + dma_addr_t ref_dma; + /* Sizes and params for Phase 2 re-submit */ + u32 out_len; /* keygen: public key size */ + u32 clen; + u32 peer_len; + u32 sk_len; + u32 dma_swap; + int mbx_idx; /* pinned MBX for Phase 2 */ +}; + +/* + * set_secret: NIST curves decode kpp_secret + u16 key_size + raw scalar. + * Curve25519 uses raw 32-byte scalar directly. + */ +static int cmh_ecdh_set_secret_nist(struct crypto_kpp *tfm, + const void *buf, unsigned int len) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + const u8 *ptr = buf; + struct kpp_secret secret; + unsigned short key_size; + int ret; + + if (!buf || len < ECDH_KPP_SECRET_MIN_SIZE) + return -EINVAL; + + memcpy(&secret, ptr, sizeof(secret)); + ptr += sizeof(secret); + + if (secret.type != CRYPTO_KPP_SECRET_TYPE_ECDH) + return -EINVAL; + if (len < secret.len) + return -EINVAL; + + memcpy(&key_size, ptr, sizeof(key_size)); + ptr += sizeof(key_size); + + if (key_size == 0) { + /* + * key_size == 0: generate a validated random private key. + * Uses the kernel ECC library (FIPS 186-5 A.2.2) to ensure + * the scalar is in the valid range [2, n-3] for the curve. + */ + u64 priv[ECC_MAX_DIGITS]; + unsigned int ndigits = ctx->clen / sizeof(u64); + unsigned int curve_id; + u8 *rnd; + + if (secret.len != ECDH_KPP_SECRET_MIN_SIZE) + return -EINVAL; + if (ndigits > ECC_MAX_DIGITS) + return -EINVAL; + /* Reject non-limb-aligned clen to prevent ndigits truncation */ + if (ctx->clen % sizeof(u64)) + return -EINVAL; + + if (ctx->curve == PKE_CURVE_P256) + curve_id = ECC_CURVE_NIST_P256; + else if (ctx->curve == PKE_CURVE_P384) + curve_id = ECC_CURVE_NIST_P384; + else + return -EINVAL; + + ret = ecc_gen_privkey(curve_id, ndigits, priv); + if (ret) { + memzero_explicit(priv, sizeof(priv)); + return ret; + } + + rnd = kmalloc(ctx->clen, GFP_KERNEL); + if (!rnd) { + memzero_explicit(priv, sizeof(priv)); + return -ENOMEM; + } + + /* Convert VLI (native LE-digit-order) to big-endian bytes */ + ecc_swap_digits(priv, (u64 *)rnd, ndigits); + memzero_explicit(priv, sizeof(priv)); + + ret = cmh_key_setkey_raw(&ctx->key, rnd, ctx->clen, + CORE_ID_PKE); + kfree_sensitive(rnd); + return ret; + } + + if (key_size != ctx->clen) + return -EINVAL; + + if (secret.len != ECDH_KPP_SECRET_MIN_SIZE + key_size) + return -EINVAL; + + return cmh_key_setkey_raw(&ctx->key, ptr, key_size, CORE_ID_PKE); +} + +static int cmh_ecdh_set_secret_x25519(struct crypto_kpp *tfm, + const void *buf, unsigned int len) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + + if (len != pke_curve_clen(PKE_CURVE_25519)) + return -EINVAL; + + return cmh_key_setkey_raw(&ctx->key, buf, len, CORE_ID_PKE); +} + +static void cmh_ecdh_keygen_complete(void *data, int error) +{ + struct kpp_request *req = data; + struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (!cmh_dma_map_error(rctx->sk_dma)) + cmh_dma_unmap_single(rctx->sk_dma, rctx->sk_len, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->pk_dma)) + cmh_dma_unmap_single(rctx->pk_dma, rctx->out_len, + DMA_FROM_DEVICE); + + if (!error) { + int nents; + + nents = sg_nents_for_len(req->dst, rctx->out_len); + if (nents < 0 || + sg_copy_from_buffer(req->dst, nents, + rctx->pk_buf, + rctx->out_len) != rctx->out_len) + error = -EINVAL; + else + req->dst_len = rctx->out_len; + } + + kfree_sensitive(rctx->sk_buf); + rctx->sk_buf = NULL; + kfree(rctx->pk_buf); + rctx->pk_buf = NULL; + cmh_complete(&req->base, error); +} + +/* + * generate_public_key: For NIST ECDH, use ECDH_KEYGEN which outputs + * the public key X-coordinate. But the kernel kpp interface expects + * uncompressed X||Y, so we use ECDSA_PUBGEN which gives us (X,Y). + * For Curve25519, ECDH_KEYGEN gives us the Montgomery u-coordinate + * which is the full public key. + */ +static int cmh_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req); + u32 clen = ctx->clen; + bool is_25519 = (ctx->curve == PKE_CURVE_25519); + u32 out_len = is_25519 ? clen : 2 * clen; + struct vcq_cmd vcq[PKE_VCQ_CMDS_MAX]; + struct core_dispatch dd; + u32 swap, dma_swap; + int ret, idx; + gfp_t gfp; + + if (ctx->key.mode != CMH_KEY_RAW) + return -EINVAL; + if (req->dst_len < out_len) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->out_len = out_len; + rctx->sk_len = ctx->key.raw.len; + rctx->pk_dma = DMA_MAPPING_ERROR; + rctx->sk_dma = DMA_MAPPING_ERROR; + + rctx->pk_buf = kzalloc(out_len, gfp); + if (!rctx->pk_buf) + return -ENOMEM; + + rctx->pk_dma = cmh_dma_map_single(rctx->pk_buf, out_len, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->pk_dma)) { + ret = -ENOMEM; + goto out_free; + } + + swap = PKE_SWAP_FLAGS; + dma_swap = pke_swap_flags(ctx->curve); + + dd = cmh_core_select_instance(CMH_CORE_PKE); + + rctx->sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, gfp); + if (!rctx->sk_buf) { + ret = -ENOMEM; + goto out_unmap; + } + rctx->sk_dma = cmh_dma_map_single(rctx->sk_buf, ctx->key.raw.len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], PKE_VCQ_CMDS_MAX); + idx = 1; + vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, rctx->sk_dma, + SYS_REF_NONE, ctx->key.raw.len, + ctx->key.raw.sys_type); + vcq[idx].id |= dma_swap; + idx++; + if (is_25519) + vcq_add_pke_ecdh_keygen(&vcq[idx++], dd.core_id, ctx->curve, + clen, rctx->pk_dma, SYS_REF_TEMP, + swap); + else + vcq_add_pke_ecdsa_pubgen(&vcq[idx++], dd.core_id, + ctx->curve, clen, rctx->pk_dma, + SYS_REF_TEMP, swap); + vcq_add_pke_flush(&vcq[idx++], dd.core_id); + + ret = cmh_tm_submit_async(vcq, PKE_VCQ_CMDS_MAX, 1, dd.mbx_idx, + cmh_ecdh_keygen_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), 0); + if (ret == -EBUSY) + return -EBUSY; + if (!ret) + return -EINPROGRESS; + +out_unmap: + if (!cmh_dma_map_error(rctx->sk_dma)) + cmh_dma_unmap_single(rctx->sk_dma, ctx->key.raw.len, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->pk_dma)) + cmh_dma_unmap_single(rctx->pk_dma, out_len, + DMA_FROM_DEVICE); + +out_free: + kfree_sensitive(rctx->sk_buf); + kfree(rctx->pk_buf); + return ret; +} + +static void cmh_ecdh_ss_phase2_complete(void *data, int error) +{ + struct kpp_request *req = data; + struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (!cmh_dma_map_error(rctx->ss_dma)) + cmh_dma_unmap_single(rctx->ss_dma, rctx->clen, + DMA_FROM_DEVICE); + + if (!error) { + int nents; + + nents = sg_nents_for_len(req->dst, rctx->clen); + if (nents < 0 || + sg_copy_from_buffer(req->dst, nents, + rctx->ss_buf, + rctx->clen) != rctx->clen) + error = -EINVAL; + else + req->dst_len = rctx->clen; + } + + kfree(rctx->ref_buf); + rctx->ref_buf = NULL; + kfree_sensitive(rctx->ss_buf); + rctx->ss_buf = NULL; + cmh_complete(&req->base, error); +} + +static void cmh_ecdh_ss_phase1_complete(void *data, int error) +{ + struct kpp_request *req = data; + struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req); + struct vcq_cmd vcq[3]; + int ret; + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* Phase 1-only resources: sk, peer -- always clean up */ + if (!cmh_dma_map_error(rctx->sk_dma)) + cmh_dma_unmap_single(rctx->sk_dma, rctx->sk_len, + DMA_TO_DEVICE); + kfree_sensitive(rctx->sk_buf); + rctx->sk_buf = NULL; + + if (!cmh_dma_map_error(rctx->peer_dma)) + cmh_dma_unmap_single(rctx->peer_dma, rctx->peer_len, + DMA_TO_DEVICE); + kfree(rctx->peer_buf); + rctx->peer_buf = NULL; + + if (error) + goto out_cleanup; + + /* Read the DS reference written by Phase 1 */ + cmh_dma_sync_for_cpu(rctx->ref_dma, sizeof(u64), DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->ref_dma, sizeof(u64), DMA_FROM_DEVICE); + rctx->ref_dma = DMA_MAPPING_ERROR; + + /* Phase 2: extract shared secret from DS */ + vcq_set_header(&vcq[0], 3); + vcq_add_sys_data(&vcq[1], *rctx->ref_buf, rctx->ss_dma, + rctx->clen); + vcq[1].id |= rctx->dma_swap; + vcq_add_sys_flush(&vcq[2]); + + ret = cmh_tm_submit_async(vcq, 3, 1, rctx->mbx_idx, + cmh_ecdh_ss_phase2_complete, req, + true, 0); + if (ret == -EBUSY || !ret) + return; + + error = ret; + +out_cleanup: + if (!cmh_dma_map_error(rctx->ref_dma)) + cmh_dma_unmap_single(rctx->ref_dma, sizeof(u64), + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->ss_dma)) + cmh_dma_unmap_single(rctx->ss_dma, rctx->clen, + DMA_FROM_DEVICE); + kfree(rctx->ref_buf); + rctx->ref_buf = NULL; + kfree_sensitive(rctx->ss_buf); + rctx->ss_buf = NULL; + cmh_complete(&req->base, error); +} + +/* + * compute_shared_secret: PKE_CMD_ECDH. + * + * req->src = peer public key (X||Y for NIST, raw 32B for Curve25519). + * Output = shared secret X coordinate (clen bytes). + * + * The CMH ECDH command stores the shared secret in a DS object, + * not directly to DMA. We create a DS slot with SYS_CMD_NEW, + * reference it via SYS_REF_LAST, then extract the result with a + * second VCQ submission using SYS_CMD_DATA with the actual ref. + */ +static int cmh_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + struct cmh_ecdh_reqctx *rctx = kpp_request_ctx(req); + u32 clen = ctx->clen; + bool is_25519 = (ctx->curve == PKE_CURVE_25519); + u32 peer_len = is_25519 ? clen : 2 * clen; + u32 ss_type = SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_PKE); + struct vcq_cmd vcq[5]; + struct core_dispatch dd; + u32 swap, dma_swap; + int ret, idx, nents; + gfp_t gfp; + + if (ctx->key.mode != CMH_KEY_RAW) + return -EINVAL; + if (req->src_len < peer_len || req->dst_len < clen) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->clen = clen; + rctx->peer_len = peer_len; + rctx->sk_len = ctx->key.raw.len; + rctx->pk_dma = DMA_MAPPING_ERROR; + rctx->sk_dma = DMA_MAPPING_ERROR; + rctx->peer_dma = DMA_MAPPING_ERROR; + rctx->ss_dma = DMA_MAPPING_ERROR; + rctx->ref_dma = DMA_MAPPING_ERROR; + + rctx->peer_buf = kmalloc(peer_len, gfp); + rctx->ss_buf = kzalloc(clen, gfp); + rctx->ref_buf = kzalloc_obj(u64, gfp); + if (!rctx->peer_buf || !rctx->ss_buf || !rctx->ref_buf) { + ret = -ENOMEM; + goto out_free; + } + + nents = sg_nents_for_len(req->src, peer_len); + if (nents < 0 || + sg_pcopy_to_buffer(req->src, nents, rctx->peer_buf, + peer_len, 0) != peer_len) { + ret = -EINVAL; + goto out_free; + } + + rctx->peer_dma = cmh_dma_map_single(rctx->peer_buf, peer_len, + DMA_TO_DEVICE); + rctx->ss_dma = cmh_dma_map_single(rctx->ss_buf, clen, + DMA_FROM_DEVICE); + rctx->ref_dma = cmh_dma_map_single(rctx->ref_buf, sizeof(u64), + DMA_FROM_DEVICE); + + if (cmh_dma_map_error(rctx->peer_dma) || + cmh_dma_map_error(rctx->ss_dma) || + cmh_dma_map_error(rctx->ref_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + swap = PKE_SWAP_FLAGS; + dma_swap = pke_swap_flags(ctx->curve); + rctx->dma_swap = dma_swap; + + dd = cmh_core_select_instance(CMH_CORE_PKE); + rctx->mbx_idx = dd.mbx_idx; + + rctx->sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, gfp); + if (!rctx->sk_buf) { + ret = -ENOMEM; + goto out_unmap; + } + rctx->sk_dma = cmh_dma_map_single(rctx->sk_buf, ctx->key.raw.len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], 5); + idx = 1; + vcq_add_sys_write(&vcq[idx], SYS_REF_TEMP, rctx->sk_dma, + SYS_REF_NONE, ctx->key.raw.len, + ctx->key.raw.sys_type); + vcq[idx].id |= dma_swap; + idx++; + vcq_add_sys_new(&vcq[idx++], 0, rctx->ref_dma, clen); + vcq_add_pke_ecdh(&vcq[idx++], dd.core_id, ctx->curve, clen, + clen, ss_type, rctx->peer_dma, + SYS_REF_TEMP, SYS_REF_LAST, swap); + vcq_add_pke_flush(&vcq[idx++], dd.core_id); + + ret = cmh_tm_submit_async(vcq, 5, 1, dd.mbx_idx, + cmh_ecdh_ss_phase1_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), 0); + if (ret == -EBUSY) + return -EBUSY; + if (!ret) + return -EINPROGRESS; + +out_unmap: + if (!cmh_dma_map_error(rctx->sk_dma)) + cmh_dma_unmap_single(rctx->sk_dma, rctx->sk_len, + DMA_TO_DEVICE); + if (!cmh_dma_map_error(rctx->ss_dma)) + cmh_dma_unmap_single(rctx->ss_dma, clen, + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->ref_dma)) + cmh_dma_unmap_single(rctx->ref_dma, sizeof(u64), + DMA_FROM_DEVICE); + if (!cmh_dma_map_error(rctx->peer_dma)) + cmh_dma_unmap_single(rctx->peer_dma, peer_len, + DMA_TO_DEVICE); + +out_free: + kfree_sensitive(rctx->sk_buf); + kfree(rctx->ref_buf); + kfree_sensitive(rctx->ss_buf); + kfree(rctx->peer_buf); + return ret; +} + +static unsigned int cmh_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + + /* Max output = X||Y for generate_public_key (NIST) */ + return 2 * ctx->clen; +} + +static unsigned int cmh_x25519_max_size(struct crypto_kpp *tfm) +{ + return pke_curve_clen(PKE_CURVE_25519); /* single coordinate */ +} + +static int cmh_ecdh_p256_init(struct crypto_kpp *tfm) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_P256; + ctx->clen = pke_curve_clen(PKE_CURVE_P256); + tfm->reqsize = sizeof(struct cmh_ecdh_reqctx); + return 0; +} + +static int cmh_ecdh_p384_init(struct crypto_kpp *tfm) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_P384; + ctx->clen = pke_curve_clen(PKE_CURVE_P384); + tfm->reqsize = sizeof(struct cmh_ecdh_reqctx); + return 0; +} + +static int cmh_x25519_init(struct crypto_kpp *tfm) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->curve = PKE_CURVE_25519; + ctx->clen = pke_curve_clen(PKE_CURVE_25519); + tfm->reqsize = sizeof(struct cmh_ecdh_reqctx); + return 0; +} + +static void cmh_ecdh_exit(struct crypto_kpp *tfm) +{ + struct cmh_ecdh_tfm_ctx *ctx = cmh_ecdh_ctx(tfm); + + cmh_key_destroy(&ctx->key); +} + +static struct kpp_alg cmh_ecdh_algs[] = { + { + .set_secret = cmh_ecdh_set_secret_nist, + .generate_public_key = cmh_ecdh_generate_public_key, + .compute_shared_secret = cmh_ecdh_compute_shared_secret, + .max_size = cmh_ecdh_max_size, + .init = cmh_ecdh_p256_init, + .exit = cmh_ecdh_exit, + .base = { + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "cri-cmh-ecdh-nist-p256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdh_tfm_ctx), + }, + }, + { + .set_secret = cmh_ecdh_set_secret_nist, + .generate_public_key = cmh_ecdh_generate_public_key, + .compute_shared_secret = cmh_ecdh_compute_shared_secret, + .max_size = cmh_ecdh_max_size, + .init = cmh_ecdh_p384_init, + .exit = cmh_ecdh_exit, + .base = { + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "cri-cmh-ecdh-nist-p384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdh_tfm_ctx), + }, + }, + { + .set_secret = cmh_ecdh_set_secret_x25519, + .generate_public_key = cmh_ecdh_generate_public_key, + .compute_shared_secret = cmh_ecdh_compute_shared_secret, + .max_size = cmh_x25519_max_size, + .init = cmh_x25519_init, + .exit = cmh_ecdh_exit, + .base = { + .cra_name = "curve25519", + .cra_driver_name = "cri-cmh-curve25519", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_ecdh_tfm_ctx), + }, + }, +}; + +/** + * cmh_pke_ecdh_register() - Register ECDH kpp algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pke_ecdh_register(void) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(cmh_ecdh_algs); i++) { + ret = crypto_register_kpp(&cmh_ecdh_algs[i]); + if (ret) { + dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n", + cmh_ecdh_algs[i].base.cra_name, ret); + goto err_unregister; + } + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_kpp(&cmh_ecdh_algs[i]); + return ret; +} + +/** + * cmh_pke_ecdh_unregister() - Unregister ECDH kpp algorithms from the crypto framework + */ +void cmh_pke_ecdh_unregister(void) +{ + int i = ARRAY_SIZE(cmh_ecdh_algs); + + while (i--) + crypto_unregister_kpp(&cmh_ecdh_algs[i]); +} From 49d5c457b317c571ebe34ddd59d109ef0e1904a4 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:33 -0700 Subject: [PATCH 17/21] crypto: cmh - add ML-KEM/ML-DSA (QSE) Register ML-KEM (Kyber) and ML-DSA (Dilithium) algorithms using the CMH QSE core (core ID 0x09). ML-KEM is ioctl-only (keygen, encaps, decaps). ML-DSA is registered as a sig algorithm with priority 5001 to override the kernel's verify-only mldsa implementation at priority 5000. This follows the established pattern where hardware drivers override software-only fallbacks (e.g. ccp at 300 over generic AES at 100, qat similarly). The CMH driver provides full HW-accelerated sign + verify vs the kernel's verify-only software implementation. Includes cmh_pqc_sizes.c with compile-time tables of PQC key and signature sizes for all supported parameter sets. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 5 +- drivers/crypto/cmh/cmh_main.c | 9 + drivers/crypto/cmh/cmh_pqc_mldsa.c | 394 +++++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_pqc_sizes.c | 39 +++ drivers/crypto/cmh/cmh_qse.c | 211 +++++++++++++++ 5 files changed, 657 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_pqc_mldsa.c create mode 100644 drivers/crypto/cmh/cmh_pqc_sizes.c create mode 100644 drivers/crypto/cmh/cmh_qse.c diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index a4cea0a56fc17f..3425eb65d653ac 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -33,7 +33,10 @@ cmh-y := \ cmh_pke_common.o \ cmh_pke_rsa.o \ cmh_pke_ecdsa.o \ - cmh_pke_ecdh.o + cmh_pke_ecdh.o \ + cmh_qse.o \ + cmh_pqc_mldsa.o \ + cmh_pqc_sizes.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index dd4e8812c457eb..bb81e2767974a1 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -39,6 +39,7 @@ #include "cmh_sm4.h" #include "cmh_ccp.h" #include "cmh_pke.h" +#include "cmh_pqc.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -297,6 +298,11 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_pke_ecdh_register; + /* Register PQC ML-KEM/ML-DSA */ + ret = cmh_pqc_mldsa_register(); + if (ret) + goto err_pqc_mldsa_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -309,6 +315,8 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_pqc_mldsa_unregister(); +err_pqc_mldsa_register: cmh_pke_ecdh_unregister(); err_pke_ecdh_register: cmh_pke_ecdsa_unregister(); @@ -371,6 +379,7 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_pqc_mldsa_unregister(); cmh_pke_ecdh_unregister(); cmh_pke_ecdsa_unregister(); cmh_pke_rsa_unregister(); diff --git a/drivers/crypto/cmh/cmh_pqc_mldsa.c b/drivers/crypto/cmh/cmh_pqc_mldsa.c new file mode 100644 index 00000000000000..cbe63c34a1c80d --- /dev/null +++ b/drivers/crypto/cmh/cmh_pqc_mldsa.c @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- ML-DSA Signature Driver (sig_alg, synchronous) + * + * Registers "mldsa44", "mldsa65", "mldsa87" sig algorithms + * with sign, verify, set_pub_key, and set_priv_key callbacks. + * + * Key format: + * Public key = raw pk bytes (1312 / 1952 / 2592 bytes) + * Private key = raw sk bytes (2560 / 4032 / 4896 bytes) + * + * Sign: src = message bytes (up to 10240 bytes), dst = raw signature + * Verify: src = raw signature, digest = message bytes + * + * Non-masked mode only for sig_alg API. + * Masked mode available through /dev/cmh_mgmt ioctl. + */ + +#include +#include +#include +#include +#include + +#include "cmh_sys.h" +#include "cmh_qse_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" +#include "cmh_pqc.h" + +struct cmh_mldsa_tfm_ctx { + struct cmh_key_ctx key; /* private key (raw only) */ + u8 *pub_key; + u32 pub_key_len; + u32 mode; /* ML_DSA_MODE_44/65/87 */ + int mode_idx; /* index into size tables */ +}; + +static inline struct cmh_mldsa_tfm_ctx *cmh_mldsa_ctx(struct crypto_sig *tfm) +{ + return crypto_sig_ctx(tfm); +} + +/* + * ML-DSA sign (synchronous sig_alg) + * + * @src: message bytes + * @slen: message length + * @dst: signature output buffer + * @dlen: output buffer length + * + * Returns signature length on success, negative errno on failure. + */ +static int cmh_mldsa_sign(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + int mi = ctx->mode_idx; + u32 sig_size = ml_dsa_sig_size[mi]; + u32 sk_size = ml_dsa_sk_size[mi]; + struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN]; + struct core_dispatch dd; + u8 *m_buf = NULL, *sig_buf = NULL, *sk_buf = NULL; + dma_addr_t m_dma = DMA_MAPPING_ERROR; + dma_addr_t sig_dma = DMA_MAPPING_ERROR; + dma_addr_t sk_dma = DMA_MAPPING_ERROR; + int ret, idx; + + if (ctx->key.mode != CMH_KEY_RAW) + return -EINVAL; + if (dlen < sig_size) + return -EINVAL; + if (!slen || slen > ML_DSA_MAX_MLEN) + return -EINVAL; + + m_buf = kmemdup(src, slen, GFP_KERNEL); + sig_buf = kzalloc(sig_size, GFP_KERNEL); + if (!m_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + if (ctx->key.raw.len != sk_size) { + ret = -EINVAL; + goto out_free; + } + + sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, GFP_KERNEL); + if (!sk_buf) { + ret = -ENOMEM; + goto out_free; + } + + m_dma = cmh_dma_map_single(m_buf, slen, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_FROM_DEVICE); + sk_dma = cmh_dma_map_single(sk_buf, sk_size, DMA_TO_DEVICE); + + if (cmh_dma_map_error(m_dma) || cmh_dma_map_error(sig_dma) || + cmh_dma_map_error(sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + dd = cmh_core_select_instance(CMH_CORE_QSE); + + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + idx = 1; + vcq_add_qse_ml_dsa_sign(&vcq[idx++], dd.core_id, ctx->mode, + QSE_FLAG_USE_RNG, + 0, m_dma, sk_dma, sig_dma, slen, false); + vcq_add_qse_flush(&vcq[idx++], dd.core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, + dd.mbx_idx); + if (!ret) { + /* Sync bounce buffer so CPU sees the DMA-written signature */ + cmh_dma_sync_for_cpu(sig_dma, sig_size, DMA_FROM_DEVICE); + memcpy(dst, sig_buf, sig_size); + ret = sig_size; + } + +out_unmap: + if (!cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, sk_size, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_size, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, slen, DMA_TO_DEVICE); + +out_free: + kfree_sensitive(sk_buf); + kfree(sig_buf); + kfree(m_buf); + return ret; +} + +/* + * ML-DSA verify (synchronous sig_alg) + * + * @src: raw signature + * @slen: signature length + * @digest: message bytes + * @dlen: message length + * + * Returns 0 on successful verification, negative errno on failure. + */ +static int cmh_mldsa_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + int mi = ctx->mode_idx; + u32 sig_size = ml_dsa_sig_size[mi]; + u32 pk_size = ml_dsa_pk_size[mi]; + struct core_dispatch d = cmh_core_select_instance(CMH_CORE_QSE); + struct vcq_cmd vcq[QSE_VCQ_CMDS_MIN]; + u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL; + dma_addr_t sig_dma = DMA_MAPPING_ERROR; + dma_addr_t m_dma = DMA_MAPPING_ERROR; + dma_addr_t pk_dma = DMA_MAPPING_ERROR; + int ret; + + if (!ctx->pub_key) + return -EINVAL; + if (slen != sig_size) + return -EINVAL; + if (!dlen || dlen > ML_DSA_MAX_MLEN) + return -EINVAL; + + sig_buf = kmemdup(src, slen, GFP_KERNEL); + m_buf = kmemdup(digest, dlen, GFP_KERNEL); + pk_buf = kmemdup(ctx->pub_key, pk_size, GFP_KERNEL); + if (!sig_buf || !m_buf || !pk_buf) { + ret = -ENOMEM; + goto out_free; + } + + sig_dma = cmh_dma_map_single(sig_buf, sig_size, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE); + pk_dma = cmh_dma_map_single(pk_buf, pk_size, DMA_TO_DEVICE); + + if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) || + cmh_dma_map_error(pk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], QSE_VCQ_CMDS_MIN); + vcq_add_qse_ml_dsa_verify(&vcq[1], d.core_id, ctx->mode, 0, + m_dma, pk_dma, sig_dma, dlen); + vcq_add_qse_flush(&vcq[2], d.core_id); + + ret = cmh_tm_submit_sync_mbx(vcq, QSE_VCQ_CMDS_MIN, 1, d.mbx_idx); + +out_unmap: + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, pk_size, DMA_TO_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_size, DMA_TO_DEVICE); + +out_free: + kfree(pk_buf); + kfree(m_buf); + kfree(sig_buf); + return ret; +} + +static int cmh_mldsa_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + u32 expected = ml_dsa_pk_size[ctx->mode_idx]; + + if (keylen != expected) + return -EINVAL; + + kfree(ctx->pub_key); + ctx->pub_key = NULL; + ctx->pub_key_len = 0; + + ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->pub_key) + return -ENOMEM; + + ctx->pub_key_len = keylen; + return 0; +} + +static int cmh_mldsa_set_priv_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + u32 expected = ml_dsa_sk_size[ctx->mode_idx]; + + if (keylen != expected) + return -EINVAL; + + return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_QSE); +} + +static unsigned int cmh_mldsa_key_size(struct crypto_sig *tfm) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + + /* crypto_sig_keysize() returns bits, not bytes */ + return ml_dsa_pk_size[ctx->mode_idx] * 8; +} + +static unsigned int cmh_mldsa_max_size(struct crypto_sig *tfm) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + + return ml_dsa_sig_size[ctx->mode_idx]; +} + +static int cmh_mldsa_44_init(struct crypto_sig *tfm) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->mode = ML_DSA_MODE_44; + ctx->mode_idx = 0; + return 0; +} + +static int cmh_mldsa_65_init(struct crypto_sig *tfm) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->mode = ML_DSA_MODE_65; + ctx->mode_idx = 1; + return 0; +} + +static int cmh_mldsa_87_init(struct crypto_sig *tfm) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->mode = ML_DSA_MODE_87; + ctx->mode_idx = 2; + return 0; +} + +static void cmh_mldsa_exit(struct crypto_sig *tfm) +{ + struct cmh_mldsa_tfm_ctx *ctx = cmh_mldsa_ctx(tfm); + + cmh_key_destroy(&ctx->key); + kfree(ctx->pub_key); + ctx->pub_key = NULL; +} + +/* + * Priority 5001: the kernel's software ML-DSA (crypto/mldsa.c) registers + * at priority 5000 but only implements verify -- sign returns -EOPNOTSUPP. + * We provide full HW-accelerated sign + verify, so we must override. + */ +static struct sig_alg cmh_mldsa_algs[] = { + { + .sign = cmh_mldsa_sign, + .verify = cmh_mldsa_verify, + .set_pub_key = cmh_mldsa_set_pub_key, + .set_priv_key = cmh_mldsa_set_priv_key, + .key_size = cmh_mldsa_key_size, + .max_size = cmh_mldsa_max_size, + .init = cmh_mldsa_44_init, + .exit = cmh_mldsa_exit, + .base = { + .cra_name = "mldsa44", + .cra_driver_name = "cri-cmh-mldsa44", + .cra_priority = 5001, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx), + }, + }, + { + .sign = cmh_mldsa_sign, + .verify = cmh_mldsa_verify, + .set_pub_key = cmh_mldsa_set_pub_key, + .set_priv_key = cmh_mldsa_set_priv_key, + .key_size = cmh_mldsa_key_size, + .max_size = cmh_mldsa_max_size, + .init = cmh_mldsa_65_init, + .exit = cmh_mldsa_exit, + .base = { + .cra_name = "mldsa65", + .cra_driver_name = "cri-cmh-mldsa65", + .cra_priority = 5001, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx), + }, + }, + { + .sign = cmh_mldsa_sign, + .verify = cmh_mldsa_verify, + .set_pub_key = cmh_mldsa_set_pub_key, + .set_priv_key = cmh_mldsa_set_priv_key, + .key_size = cmh_mldsa_key_size, + .max_size = cmh_mldsa_max_size, + .init = cmh_mldsa_87_init, + .exit = cmh_mldsa_exit, + .base = { + .cra_name = "mldsa87", + .cra_driver_name = "cri-cmh-mldsa87", + .cra_priority = 5001, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_mldsa_tfm_ctx), + }, + }, +}; + +/** + * cmh_pqc_mldsa_register() - Register ML-DSA akcipher algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pqc_mldsa_register(void) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(cmh_mldsa_algs); i++) { + ret = crypto_register_sig(&cmh_mldsa_algs[i]); + if (ret) { + dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n", + cmh_mldsa_algs[i].base.cra_name, ret); + goto err_unregister; + } + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_sig(&cmh_mldsa_algs[i]); + return ret; +} + +/** + * cmh_pqc_mldsa_unregister() - Unregister ML-DSA akcipher algorithms from the crypto framework + */ +void cmh_pqc_mldsa_unregister(void) +{ + int i = ARRAY_SIZE(cmh_mldsa_algs); + + while (i--) + crypto_unregister_sig(&cmh_mldsa_algs[i]); +} diff --git a/drivers/crypto/cmh/cmh_pqc_sizes.c b/drivers/crypto/cmh/cmh_pqc_sizes.c new file mode 100644 index 00000000000000..39e3d56f4312f4 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pqc_sizes.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- PQC Algorithm Size Tables + * + * Centralised ML-DSA and SLH-DSA parameter-size arrays. Declared + * extern in cmh_qse_abi.h / cmh_hcq_abi.h, defined here once to + * avoid per-TU duplication. + */ + +#include +#include +#include + +#include "cmh_qse_abi.h" +#include "cmh_hcq_abi.h" + +/* ML-DSA size tables (indexed by ml_dsa_mode_idx()) */ +const u32 ml_dsa_pk_size[3] = { 1312U, 1952U, 2592U }; +const u32 ml_dsa_sk_size[3] = { 2560U, 4032U, 4896U }; +const u32 ml_dsa_sk_size_masked[3] = { 3360U, 5472U, 6368U }; +const u32 ml_dsa_sig_size[3] = { 2420U, 3309U, 4627U }; + +static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sk_size)); +static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sk_size_masked)); +static_assert(ARRAY_SIZE(ml_dsa_pk_size) == ARRAY_SIZE(ml_dsa_sig_size)); + +/* SLH-DSA n-values and signature sizes (indexed by param_set - 1) */ +const u32 slhdsa_n[12] = { + 16, 16, 24, 24, 32, 32, /* SHAKE 128s/f, 192s/f, 256s/f */ + 16, 16, 24, 24, 32, 32, /* SHA2 128s/f, 192s/f, 256s/f */ +}; + +const u32 slhdsa_sig_size[12] = { + 7856, 17088, 16224, 35664, 29792, 49856, /* SHAKE */ + 7856, 17088, 16224, 35664, 29792, 49856, /* SHA2 */ +}; + +static_assert(ARRAY_SIZE(slhdsa_n) == ARRAY_SIZE(slhdsa_sig_size)); diff --git a/drivers/crypto/cmh/cmh_qse.c b/drivers/crypto/cmh/cmh_qse.c new file mode 100644 index 00000000000000..257dc3ee29a8d1 --- /dev/null +++ b/drivers/crypto/cmh/cmh_qse.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- QSE Core VCQ Builders + * + * VCQ builder functions for ML-KEM and ML-DSA commands (plain and masked). + * Each function populates a single vcq_cmd slot. Callers assemble + * complete VCQs with header + command(s) + flush, then submit via + * cmh_tm_submit_sync(). + */ + +#include + +#include "cmh_sys.h" + +/* -- QSE flush -- */ + +/** + * vcq_add_qse_flush() - Build a QSE flush VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + */ +void vcq_add_qse_flush(struct vcq_cmd *slot, u32 core_id) +{ + vcq_add_flush(slot, core_id); +} + +/* -- ML-KEM -- */ + +/** + * vcq_add_qse_ml_kem_keygen() - Build an ML-KEM key generation VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @k: ML-KEM security parameter (k = 2, 3, or 4) + * @flags: Command flags + * @seed: DMA address of seed input buffer + * @z: DMA address of implicit rejection value buffer + * @ek: DMA address of encapsulation key output buffer + * @dk: DMA address of decapsulation key output buffer + * @dk_type: Decapsulation key datastore type + * @masked: Use masked (side-channel protected) variant + */ +void vcq_add_qse_ml_kem_keygen(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags, + u64 seed, u64 z, u64 ek, u64 dk, u32 dk_type, + bool masked) +{ + u32 cmd_id = masked ? QSE_CMD_ML_KEM_KEYGEN_MASKED + : QSE_CMD_ML_KEM_KEYGEN; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id); + slot->hwc.qse.cmd_ml_kem_keygen.k = k; + slot->hwc.qse.cmd_ml_kem_keygen.flags = flags; + slot->hwc.qse.cmd_ml_kem_keygen.seed = seed; + slot->hwc.qse.cmd_ml_kem_keygen.z = z; + slot->hwc.qse.cmd_ml_kem_keygen.ek = ek; + slot->hwc.qse.cmd_ml_kem_keygen.dk = dk; + slot->hwc.qse.cmd_ml_kem_keygen.dk_type = dk_type; +} + +/** + * vcq_add_qse_ml_kem_enc() - Build an ML-KEM encapsulation VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @k: ML-KEM security parameter (k = 2, 3, or 4) + * @flags: Command flags + * @coin: DMA address of encapsulation coin/randomness buffer + * @ek: DMA address of encapsulation key input buffer + * @ct: DMA address of ciphertext output buffer + * @ss: DMA address of shared secret output buffer + * @ss_type: Shared secret datastore type + * @masked: Use masked (side-channel protected) variant + */ +void vcq_add_qse_ml_kem_enc(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags, + u64 coin, u64 ek, u64 ct, u64 ss, u32 ss_type, + bool masked) +{ + u32 cmd_id = masked ? QSE_CMD_ML_KEM_ENC_MASKED + : QSE_CMD_ML_KEM_ENC; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id); + slot->hwc.qse.cmd_ml_kem_enc.k = k; + slot->hwc.qse.cmd_ml_kem_enc.flags = flags; + slot->hwc.qse.cmd_ml_kem_enc.coin = coin; + slot->hwc.qse.cmd_ml_kem_enc.ek = ek; + slot->hwc.qse.cmd_ml_kem_enc.ct = ct; + slot->hwc.qse.cmd_ml_kem_enc.ss = ss; + slot->hwc.qse.cmd_ml_kem_enc.ss_type = ss_type; +} + +/** + * vcq_add_qse_ml_kem_dec() - Build an ML-KEM decapsulation VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @k: ML-KEM security parameter (k = 2, 3, or 4) + * @flags: Command flags + * @ct: DMA address of ciphertext input buffer + * @dk: DMA address of decapsulation key input buffer + * @ss: DMA address of shared secret output buffer + * @ss_type: Shared secret datastore type + * @masked: Use masked (side-channel protected) variant + */ +void vcq_add_qse_ml_kem_dec(struct vcq_cmd *slot, u32 core_id, u32 k, u32 flags, + u64 ct, u64 dk, u64 ss, u32 ss_type, + bool masked) +{ + u32 cmd_id = masked ? QSE_CMD_ML_KEM_DEC_MASKED + : QSE_CMD_ML_KEM_DEC; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id); + slot->hwc.qse.cmd_ml_kem_dec.k = k; + slot->hwc.qse.cmd_ml_kem_dec.flags = flags; + slot->hwc.qse.cmd_ml_kem_dec.ct = ct; + slot->hwc.qse.cmd_ml_kem_dec.dk = dk; + slot->hwc.qse.cmd_ml_kem_dec.ss = ss; + slot->hwc.qse.cmd_ml_kem_dec.ss_type = ss_type; +} + +/* -- ML-DSA -- */ + +/** + * vcq_add_qse_ml_dsa_keygen() - Build an ML-DSA key generation VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @mode: ML-DSA mode (44, 65, or 87) + * @flags: Command flags + * @seed: DMA address of seed input buffer + * @pk: DMA address of public key output buffer + * @sk: DMA address of secret key output buffer + * @sk_type: Secret key datastore type + * @masked: Use masked (side-channel protected) variant + */ +void vcq_add_qse_ml_dsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags, + u64 seed, u64 pk, u64 sk, u32 sk_type, + bool masked) +{ + u32 cmd_id = masked ? QSE_CMD_ML_DSA_KEYGEN_MASKED + : QSE_CMD_ML_DSA_KEYGEN; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id); + slot->hwc.qse.cmd_ml_dsa_keygen.mode = mode; + slot->hwc.qse.cmd_ml_dsa_keygen.flags = flags; + slot->hwc.qse.cmd_ml_dsa_keygen.seed = seed; + slot->hwc.qse.cmd_ml_dsa_keygen.pk = pk; + slot->hwc.qse.cmd_ml_dsa_keygen.sk = sk; + slot->hwc.qse.cmd_ml_dsa_keygen.sk_type = sk_type; +} + +/** + * vcq_add_qse_ml_dsa_sign() - Build an ML-DSA signing VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @mode: ML-DSA mode (44, 65, or 87) + * @flags: Command flags + * @rnd: DMA address of signing randomness buffer + * @m: DMA address of message buffer + * @sk: DMA address of secret key buffer + * @sig: DMA address of signature output buffer + * @mlen: Length of message in bytes + * @masked: Use masked (side-channel protected) variant + */ +void vcq_add_qse_ml_dsa_sign(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags, + u64 rnd, u64 m, u64 sk, u64 sig, u32 mlen, + bool masked) +{ + u32 cmd_id = masked ? QSE_CMD_ML_DSA_SIGN_MASKED + : QSE_CMD_ML_DSA_SIGN; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd_id); + slot->hwc.qse.cmd_ml_dsa_sign.mode = mode; + slot->hwc.qse.cmd_ml_dsa_sign.flags = flags; + slot->hwc.qse.cmd_ml_dsa_sign.rnd = rnd; + slot->hwc.qse.cmd_ml_dsa_sign.m = m; + slot->hwc.qse.cmd_ml_dsa_sign.sk = sk; + slot->hwc.qse.cmd_ml_dsa_sign.sig = sig; + slot->hwc.qse.cmd_ml_dsa_sign.mlen = mlen; +} + +/** + * vcq_add_qse_ml_dsa_verify() - Build an ML-DSA signature verify VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @mode: ML-DSA mode (44, 65, or 87) + * @flags: Command flags + * @m: DMA address of message buffer + * @pk: DMA address of public key buffer + * @sig: DMA address of signature buffer to verify + * @mlen: Length of message in bytes + */ +void vcq_add_qse_ml_dsa_verify(struct vcq_cmd *slot, u32 core_id, u32 mode, u32 flags, + u64 m, u64 pk, u64 sig, u32 mlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, QSE_CMD_ML_DSA_VERIFY); + slot->hwc.qse.cmd_ml_dsa_verify.mode = mode; + slot->hwc.qse.cmd_ml_dsa_verify.flags = flags; + slot->hwc.qse.cmd_ml_dsa_verify.m = m; + slot->hwc.qse.cmd_ml_dsa_verify.pk = pk; + slot->hwc.qse.cmd_ml_dsa_verify.sig = sig; + slot->hwc.qse.cmd_ml_dsa_verify.mlen = mlen; +} From 7708c2182bfcdeaf2c5afc5228c118145cb4b466 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:34 -0700 Subject: [PATCH 18/21] crypto: cmh - add SLH-DSA/LMS/XMSS (HCQ) Register SLH-DSA, LMS, LMS-HSS, XMSS, and XMSS-MT algorithms using the CMH HCQ core (core ID 0x08). SLH-DSA is registered as a sig algorithm with sign and verify support. LMS, LMS-HSS, XMSS, and XMSS-MT are registered as verify-only sig algorithms: their stateful signing semantics (one-time-key private state the signer must track) are not modeled by the kernel crypto API, so only verification is exposed. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- drivers/crypto/cmh/Makefile | 6 +- drivers/crypto/cmh/cmh_hcq.c | 313 +++++++++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 24 ++ drivers/crypto/cmh/cmh_pqc_lms.c | 230 +++++++++++++++++ drivers/crypto/cmh/cmh_pqc_slhdsa.c | 377 ++++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_pqc_xmss.c | 230 +++++++++++++++++ 6 files changed, 1179 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_hcq.c create mode 100644 drivers/crypto/cmh/cmh_pqc_lms.c create mode 100644 drivers/crypto/cmh/cmh_pqc_slhdsa.c create mode 100644 drivers/crypto/cmh/cmh_pqc_xmss.c diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 3425eb65d653ac..c3332804a9d7c6 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -36,7 +36,11 @@ cmh-y := \ cmh_pke_ecdh.o \ cmh_qse.o \ cmh_pqc_mldsa.o \ - cmh_pqc_sizes.o + cmh_pqc_sizes.o \ + cmh_hcq.o \ + cmh_pqc_slhdsa.o \ + cmh_pqc_lms.o \ + cmh_pqc_xmss.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_hcq.c b/drivers/crypto/cmh/cmh_hcq.c new file mode 100644 index 00000000000000..8fc3a5cb0f9fed --- /dev/null +++ b/drivers/crypto/cmh/cmh_hcq.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- HCQ Core VCQ Builders + * + * VCQ builder functions for SLH-DSA, LMS, and XMSS commands. + * Each function populates a single vcq_cmd slot. Callers assemble + * complete VCQs with header + command(s) + flush, then submit via + * cmh_tm_submit_sync(). + */ + +#include + +#include "cmh_sys.h" + +/* -- HCQ flush -- */ + +/** + * vcq_add_hcq_flush() - Build an HCQ flush VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + */ +void vcq_add_hcq_flush(struct vcq_cmd *slot, u32 core_id) +{ + vcq_add_flush(slot, core_id); +} + +/* -- SLH-DSA -- */ + +/** + * vcq_add_hcq_slhdsa_keygen() - Build an SLH-DSA key generation VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @param_set: SLH-DSA parameter set identifier + * @seed_len: Length of seed buffer in bytes + * @pk_len: Length of public key buffer in bytes + * @sk_len: Length of secret key buffer in bytes + * @seed: DMA address of seed input buffer + * @pk: DMA address of public key output buffer + * @sk: DMA address of secret key output buffer + */ +void vcq_add_hcq_slhdsa_keygen(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 seed_len, u32 pk_len, u32 sk_len, + u64 seed, u64 pk, u64 sk) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_KEYGEN); + slot->hwc.hcq.cmd_slhdsa_keygen.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_keygen.seed_len = seed_len; + slot->hwc.hcq.cmd_slhdsa_keygen.pk_len = pk_len; + slot->hwc.hcq.cmd_slhdsa_keygen.sk_len = sk_len; + slot->hwc.hcq.cmd_slhdsa_keygen.seed = seed; + slot->hwc.hcq.cmd_slhdsa_keygen.pk = pk; + slot->hwc.hcq.cmd_slhdsa_keygen.sk = sk; +} + +/** + * vcq_add_hcq_slhdsa_sign() - Build an SLH-DSA signing VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @param_set: SLH-DSA parameter set identifier + * @msg_len: Length of message buffer in bytes + * @ctx_len: Length of context string in bytes + * @add_random: DMA address of additional randomness buffer + * @msg: DMA address of message buffer + * @ctx: DMA address of context string buffer + * @sk: DMA address of secret key buffer + * @sig: DMA address of signature output buffer + */ +void vcq_add_hcq_slhdsa_sign(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u32 ctx_len, + u64 add_random, u64 msg, u64 ctx, + u64 sk, u64 sig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_SIGN); + slot->hwc.hcq.cmd_slhdsa_sign.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_sign.message_len = msg_len; + slot->hwc.hcq.cmd_slhdsa_sign.add_random = add_random; + slot->hwc.hcq.cmd_slhdsa_sign.message = msg; + slot->hwc.hcq.cmd_slhdsa_sign.context = ctx; + slot->hwc.hcq.cmd_slhdsa_sign.sk = sk; + slot->hwc.hcq.cmd_slhdsa_sign.sig = sig; + slot->hwc.hcq.cmd_slhdsa_sign.context_len = ctx_len; +} + +/** + * vcq_add_hcq_slhdsa_sign_internal() - Build an SLH-DSA internal signing VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @param_set: SLH-DSA parameter set identifier + * @msg_len: Length of message buffer in bytes + * @add_random: DMA address of additional randomness buffer + * @msg: DMA address of message buffer + * @sk: DMA address of secret key buffer + * @sig: DMA address of signature output buffer + */ +void vcq_add_hcq_slhdsa_sign_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u64 add_random, + u64 msg, u64 sk, u64 sig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_SIGN_INTERNAL); + slot->hwc.hcq.cmd_slhdsa_sign_internal.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_sign_internal.message_len = msg_len; + slot->hwc.hcq.cmd_slhdsa_sign_internal.add_random = add_random; + slot->hwc.hcq.cmd_slhdsa_sign_internal.message = msg; + slot->hwc.hcq.cmd_slhdsa_sign_internal.sk = sk; + slot->hwc.hcq.cmd_slhdsa_sign_internal.sig = sig; +} + +/** + * vcq_add_hcq_slhdsa_verify() - Build an SLH-DSA verification VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @param_set: SLH-DSA parameter set identifier + * @msg_len: Length of message buffer in bytes + * @ctx_len: Length of context string in bytes + * @msg: DMA address of message buffer + * @ctx: DMA address of context string buffer + * @pk: DMA address of public key buffer + * @sig: DMA address of signature buffer to verify + */ +void vcq_add_hcq_slhdsa_verify(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u32 ctx_len, + u64 msg, u64 ctx, u64 pk, u64 sig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_VERIFY); + slot->hwc.hcq.cmd_slhdsa_verify.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_verify.message_len = msg_len; + slot->hwc.hcq.cmd_slhdsa_verify.message = msg; + slot->hwc.hcq.cmd_slhdsa_verify.context = ctx; + slot->hwc.hcq.cmd_slhdsa_verify.pk = pk; + slot->hwc.hcq.cmd_slhdsa_verify.sig = sig; + slot->hwc.hcq.cmd_slhdsa_verify.context_len = ctx_len; +} + +/** + * vcq_add_hcq_slhdsa_sign_prehash() - Build an SLH-DSA prehash signing VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @cmd: VCQ command ID (sign-prehash variant) + * @param_set: SLH-DSA parameter set identifier + * @prehash_algo: Prehash algorithm identifier + * @msg_len: Length of message buffer in bytes + * @ctx_len: Length of context string in bytes + * @add_random: DMA address of additional randomness buffer + * @msg: DMA address of message buffer + * @ctx: DMA address of context string buffer + * @sk: DMA address of secret key buffer + * @sig: DMA address of signature output buffer + */ +void vcq_add_hcq_slhdsa_sign_prehash(struct vcq_cmd *slot, u32 core_id, + u32 cmd, u32 param_set, u32 prehash_algo, + u32 msg_len, u32 ctx_len, + u64 add_random, u64 msg, u64 ctx, + u64 sk, u64 sig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd); + slot->hwc.hcq.cmd_slhdsa_sign_prehash.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.prehash_algo = prehash_algo; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.message_len = msg_len; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.context_len = ctx_len; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.add_random = add_random; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.message = msg; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.context = ctx; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.sk = sk; + slot->hwc.hcq.cmd_slhdsa_sign_prehash.sig = sig; +} + +/** + * vcq_add_hcq_slhdsa_verify_prehash() - Build an SLH-DSA prehash verify VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @cmd: VCQ command ID (verify-prehash variant) + * @param_set: SLH-DSA parameter set identifier + * @prehash_algo: Prehash algorithm identifier + * @msg_len: Length of message buffer in bytes + * @ctx_len: Length of context string in bytes + * @msg: DMA address of message buffer + * @ctx: DMA address of context string buffer + * @pk: DMA address of public key buffer + * @sig: DMA address of signature buffer to verify + */ +void vcq_add_hcq_slhdsa_verify_prehash(struct vcq_cmd *slot, u32 core_id, + u32 cmd, u32 param_set, u32 prehash_algo, + u32 msg_len, u32 ctx_len, + u64 msg, u64 ctx, u64 pk, u64 sig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, cmd); + slot->hwc.hcq.cmd_slhdsa_verify_prehash.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.prehash_algo = prehash_algo; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.message_len = msg_len; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.context_len = ctx_len; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.message = msg; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.context = ctx; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.pk = pk; + slot->hwc.hcq.cmd_slhdsa_verify_prehash.sig = sig; +} + +/** + * vcq_add_hcq_slhdsa_verify_internal() - Build an SLH-DSA internal verify VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @param_set: SLH-DSA parameter set identifier + * @msg_len: Length of message buffer in bytes + * @msg: DMA address of message buffer + * @pk: DMA address of public key buffer + * @sig: DMA address of signature buffer to verify + */ +void vcq_add_hcq_slhdsa_verify_internal(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 msg_len, u64 msg, u64 pk, u64 sig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, + HCQ_CMD_SLHDSA_VERIFY_INTERNAL); + slot->hwc.hcq.cmd_slhdsa_verify_internal.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_verify_internal.message_len = msg_len; + slot->hwc.hcq.cmd_slhdsa_verify_internal.message = msg; + slot->hwc.hcq.cmd_slhdsa_verify_internal.pk = pk; + slot->hwc.hcq.cmd_slhdsa_verify_internal.sig = sig; +} + +/** + * vcq_add_hcq_slhdsa_pubgen() - Build an SLH-DSA public key generation VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @param_set: SLH-DSA parameter set identifier + * @sk_len: Length of secret key buffer in bytes + * @sk: DMA address of secret key input buffer + * @pk: DMA address of public key output buffer + */ +void vcq_add_hcq_slhdsa_pubgen(struct vcq_cmd *slot, u32 core_id, u32 param_set, + u32 sk_len, u64 sk, u64 pk) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_SLHDSA_PUBGEN); + slot->hwc.hcq.cmd_slhdsa_pubgen.parameter_set = param_set; + slot->hwc.hcq.cmd_slhdsa_pubgen.sk_len = sk_len; + slot->hwc.hcq.cmd_slhdsa_pubgen.sk = sk; + slot->hwc.hcq.cmd_slhdsa_pubgen.pk = pk; +} + +/* -- LMS -- */ + +/** + * vcq_add_hcq_lms_verify() - Build an LMS/HSS signature verify VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @lms_hss: LMS/HSS mode flag (0 = LMS, 1 = HSS) + * @pk_len: Length of public key buffer in bytes + * @sig_len: Length of signature buffer in bytes + * @dig_len: Length of digest buffer in bytes + * @pk: DMA address of public key buffer + * @sig: DMA address of signature buffer + * @dig: DMA address of digest buffer + */ +void vcq_add_hcq_lms_verify(struct vcq_cmd *slot, u32 core_id, u32 lms_hss, + u32 pk_len, u32 sig_len, u32 dig_len, + u64 pk, u64 sig, u64 dig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_LMS_VERIFY); + slot->hwc.hcq.cmd_lms_verify.lms_hss = lms_hss; + slot->hwc.hcq.cmd_lms_verify.pk_len = pk_len; + slot->hwc.hcq.cmd_lms_verify.sig_len = sig_len; + slot->hwc.hcq.cmd_lms_verify.dig_len = dig_len; + slot->hwc.hcq.cmd_lms_verify.pk = pk; + slot->hwc.hcq.cmd_lms_verify.sig = sig; + slot->hwc.hcq.cmd_lms_verify.dig = dig; +} + +/* -- XMSS -- */ + +/** + * vcq_add_hcq_xmss_verify() - Build an XMSS/XMSS^MT signature verify VCQ command + * @slot: VCQ command slot to populate + * @core_id: Hardware core ID for dispatch + * @xmss_mt: XMSS/XMSS^MT mode flag (0 = XMSS, 1 = XMSS^MT) + * @pk_len: Length of public key buffer in bytes + * @sig_len: Length of signature buffer in bytes + * @dig_len: Length of digest buffer in bytes + * @pk: DMA address of public key buffer + * @sig: DMA address of signature buffer + * @dig: DMA address of digest buffer + */ +void vcq_add_hcq_xmss_verify(struct vcq_cmd *slot, u32 core_id, u32 xmss_mt, + u32 pk_len, u32 sig_len, u32 dig_len, + u64 pk, u64 sig, u64 dig) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, HCQ_CMD_XMSS_VERIFY); + slot->hwc.hcq.cmd_xmss_verify.xmss_mt = xmss_mt; + slot->hwc.hcq.cmd_xmss_verify.pk_len = pk_len; + slot->hwc.hcq.cmd_xmss_verify.sig_len = sig_len; + slot->hwc.hcq.cmd_xmss_verify.dig_len = dig_len; + slot->hwc.hcq.cmd_xmss_verify.pk = pk; + slot->hwc.hcq.cmd_xmss_verify.sig = sig; + slot->hwc.hcq.cmd_xmss_verify.dig = dig; +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index bb81e2767974a1..0b5d22daec671a 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -303,6 +303,21 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_pqc_mldsa_register; + /* Register PQC SLH-DSA */ + ret = cmh_pqc_slhdsa_register(); + if (ret) + goto err_pqc_slhdsa_register; + + /* Register PQC LMS */ + ret = cmh_pqc_lms_register(); + if (ret) + goto err_pqc_lms_register; + + /* Register PQC XMSS */ + ret = cmh_pqc_xmss_register(); + if (ret) + goto err_pqc_xmss_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -315,6 +330,12 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_pqc_xmss_unregister(); +err_pqc_xmss_register: + cmh_pqc_lms_unregister(); +err_pqc_lms_register: + cmh_pqc_slhdsa_unregister(); +err_pqc_slhdsa_register: cmh_pqc_mldsa_unregister(); err_pqc_mldsa_register: cmh_pke_ecdh_unregister(); @@ -379,6 +400,9 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_pqc_xmss_unregister(); + cmh_pqc_lms_unregister(); + cmh_pqc_slhdsa_unregister(); cmh_pqc_mldsa_unregister(); cmh_pke_ecdh_unregister(); cmh_pke_ecdsa_unregister(); diff --git a/drivers/crypto/cmh/cmh_pqc_lms.c b/drivers/crypto/cmh/cmh_pqc_lms.c new file mode 100644 index 00000000000000..13b2e26aa7bda9 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pqc_lms.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- LMS/HSS Signature Driver (verify-only, sig_alg, synchronous) + * + * Registers "lms" and "lms-hss" sig algorithms with verify-only + * callbacks. Sign is not supported (stateful signature -- key + * management must happen externally). + * + * Verify: src = raw signature, digest = message bytes + * Public key: raw pk bytes (variable length, set via set_pub_key) + */ + +#include +#include +#include +#include +#include + +#include "cmh_sys.h" +#include "cmh_hcq_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_pqc.h" + +#define LMS_VCQ_CMDS 3 /* header + cmd + flush */ + +struct cmh_lms_tfm_ctx { + u8 *pub_key; + u32 pub_key_len; + u32 lms_hss; /* 0 = LMS, 1 = LMS-HSS */ +}; + +static inline struct cmh_lms_tfm_ctx *cmh_lms_ctx(struct crypto_sig *tfm) +{ + return crypto_sig_ctx(tfm); +} + +/* + * LMS/HSS verify (synchronous sig_alg) + * + * @src: raw signature + * @slen: signature length + * @digest: message bytes + * @dlen: message length + * + * Returns 0 on successful verification, negative errno on failure. + */ +static int cmh_lms_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm); + struct core_dispatch d = cmh_core_select_instance(CMH_CORE_HCQ); + struct vcq_cmd vcq[LMS_VCQ_CMDS]; + u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL; + dma_addr_t sig_dma = DMA_MAPPING_ERROR; + dma_addr_t m_dma = DMA_MAPPING_ERROR; + dma_addr_t pk_dma = DMA_MAPPING_ERROR; + int ret; + + if (!ctx->pub_key) + return -EINVAL; + if (!slen || slen > LMS_MAX_SIG_LEN) + return -EINVAL; + if (!dlen || dlen > LMS_MAX_MSG_LEN) + return -EINVAL; + + sig_buf = kmemdup(src, slen, GFP_KERNEL); + m_buf = kmemdup(digest, dlen, GFP_KERNEL); + pk_buf = kmemdup(ctx->pub_key, ctx->pub_key_len, GFP_KERNEL); + if (!sig_buf || !m_buf || !pk_buf) { + ret = -ENOMEM; + goto out_free; + } + + sig_dma = cmh_dma_map_single(sig_buf, slen, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE); + pk_dma = cmh_dma_map_single(pk_buf, ctx->pub_key_len, DMA_TO_DEVICE); + + if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) || + cmh_dma_map_error(pk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], LMS_VCQ_CMDS); + vcq_add_hcq_lms_verify(&vcq[1], d.core_id, ctx->lms_hss, + ctx->pub_key_len, slen, dlen, + pk_dma, sig_dma, m_dma); + vcq_add_hcq_flush(&vcq[2], d.core_id); + + /* LMS verify traverses Merkle hash chains -- inherently slow */ + ret = cmh_tm_submit_sync_tmo(vcq, LMS_VCQ_CMDS, 1, d.mbx_idx, + cmh_tm_slow_op_timeout_jiffies()); + +out_unmap: + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, ctx->pub_key_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, slen, DMA_TO_DEVICE); + +out_free: + kfree(pk_buf); + kfree(m_buf); + kfree(sig_buf); + return ret; +} + +static int cmh_lms_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm); + + if (!keylen || keylen > LMS_MAX_PK_LEN) + return -EINVAL; + + kfree(ctx->pub_key); + ctx->pub_key = NULL; + ctx->pub_key_len = 0; + + ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->pub_key) + return -ENOMEM; + + ctx->pub_key_len = keylen; + return 0; +} + +static unsigned int cmh_lms_key_size(struct crypto_sig *tfm) +{ + struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm); + + return ctx->pub_key_len * 8; +} + +static int cmh_lms_init(struct crypto_sig *tfm) +{ + struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + return 0; +} + +static int cmh_lms_hss_init(struct crypto_sig *tfm) +{ + struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->lms_hss = 1; + return 0; +} + +static void cmh_lms_exit(struct crypto_sig *tfm) +{ + struct cmh_lms_tfm_ctx *ctx = cmh_lms_ctx(tfm); + + kfree(ctx->pub_key); + ctx->pub_key = NULL; +} + +static struct sig_alg cmh_lms_algs[] = { + { + .verify = cmh_lms_verify, + .set_pub_key = cmh_lms_set_pub_key, + .key_size = cmh_lms_key_size, + .init = cmh_lms_init, + .exit = cmh_lms_exit, + .base = { + .cra_name = "lms", + .cra_driver_name = "cri-cmh-lms", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_lms_tfm_ctx), + }, + }, + { + .verify = cmh_lms_verify, + .set_pub_key = cmh_lms_set_pub_key, + .key_size = cmh_lms_key_size, + .init = cmh_lms_hss_init, + .exit = cmh_lms_exit, + .base = { + .cra_name = "lms-hss", + .cra_driver_name = "cri-cmh-lms-hss", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_lms_tfm_ctx), + }, + }, +}; + +/** + * cmh_pqc_lms_register() - Register LMS/LMS-HSS sig algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pqc_lms_register(void) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(cmh_lms_algs); i++) { + ret = crypto_register_sig(&cmh_lms_algs[i]); + if (ret) { + dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n", + cmh_lms_algs[i].base.cra_name, ret); + goto err_unregister; + } + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_sig(&cmh_lms_algs[i]); + return ret; +} + +/** + * cmh_pqc_lms_unregister() - Unregister LMS/LMS-HSS sig algorithms from the crypto framework + */ +void cmh_pqc_lms_unregister(void) +{ + int i = ARRAY_SIZE(cmh_lms_algs); + + while (i--) + crypto_unregister_sig(&cmh_lms_algs[i]); +} diff --git a/drivers/crypto/cmh/cmh_pqc_slhdsa.c b/drivers/crypto/cmh/cmh_pqc_slhdsa.c new file mode 100644 index 00000000000000..9cc8cdb442b267 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pqc_slhdsa.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- SLH-DSA Signature Driver (sig_alg, synchronous) + * + * Registers SLH-DSA sig algorithms for all 12 parameter sets + * (SHAKE/SHA2 x 128/192/256 x s/f) with sign and verify callbacks. + * + * Key format: + * Public key = raw pk bytes (2*n bytes) + * Private key = raw sk bytes (4*n) + * + * Sign: src = message, dst = raw signature + * Verify: src = raw signature, digest = message bytes + * + * Private keys are raw (written to SYS_REF_TEMP per-operation). + */ + +#include +#include +#include +#include +#include +#include + +#include "cmh_sys.h" +#include "cmh_qse_abi.h" +#include "cmh_hcq_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" +#include "cmh_pqc.h" + +struct cmh_slhdsa_tfm_ctx { + struct cmh_key_ctx key; /* private key (raw sk bytes) */ + u8 *pub_key; + u32 pub_key_len; + u32 param_set; /* HCQ_SLHDSA_SHAKE_128S .. SHA2_256F */ +}; + +static inline struct cmh_slhdsa_tfm_ctx *cmh_slhdsa_ctx(struct crypto_sig *tfm) +{ + return crypto_sig_ctx(tfm); +} + +/* + * SLH-DSA sign (synchronous sig_alg) + * + * @src: message bytes + * @slen: message length + * @dst: signature output buffer + * @dlen: output buffer length + * + * Returns signature length on success, negative errno on failure. + * Uses raw private keys written to SYS_REF_TEMP per-operation. + */ +static int cmh_slhdsa_sign(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + u32 sig_sz = slhdsa_get_sig_size(ctx->param_set); + u32 sk_sz = slhdsa_sk_size(ctx->param_set); + struct vcq_cmd vcq[HCQ_VCQ_CMDS_MAX]; /* raw: hdr+write+sign+flush */ + u32 vcq_count; + u8 *m_buf = NULL, *sig_buf = NULL, *sk_buf = NULL; + dma_addr_t m_dma = DMA_MAPPING_ERROR; + dma_addr_t sig_dma = DMA_MAPPING_ERROR; + dma_addr_t sk_dma = DMA_MAPPING_ERROR; + int ret, idx; + + if (ctx->key.mode == CMH_KEY_NONE) + return -EINVAL; + if (dlen < sig_sz) + return -EINVAL; + if (!slen || slen > SLHDSA_MAX_MSG_LEN) + return -EINVAL; + + m_buf = kmemdup(src, slen, GFP_KERNEL); + sig_buf = kzalloc(sig_sz, GFP_KERNEL); + if (!m_buf || !sig_buf) { + ret = -ENOMEM; + goto out_free; + } + + m_dma = cmh_dma_map_single(m_buf, slen, DMA_TO_DEVICE); + sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_FROM_DEVICE); + if (cmh_dma_map_error(m_dma) || cmh_dma_map_error(sig_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + sk_dma = DMA_MAPPING_ERROR; + idx = 0; + + struct core_dispatch d; + + d = cmh_core_select_instance(CMH_CORE_HCQ); + + if (ctx->key.raw.len != sk_sz) { + ret = -EINVAL; + goto out_unmap; + } + sk_buf = kmemdup(ctx->key.raw.data, ctx->key.raw.len, + GFP_KERNEL); + if (!sk_buf) { + ret = -ENOMEM; + goto out_unmap; + } + sk_dma = cmh_dma_map_single(sk_buf, sk_sz, DMA_TO_DEVICE); + if (cmh_dma_map_error(sk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_count = HCQ_VCQ_CMDS_MIN + 1; + vcq_set_header(&vcq[idx++], vcq_count); + vcq_add_sys_write(&vcq[idx++], SYS_REF_TEMP, sk_dma, + SYS_REF_NONE, sk_sz, + ctx->key.raw.sys_type); + vcq_add_hcq_slhdsa_sign_internal(&vcq[idx++], d.core_id, + ctx->param_set, + slen, 0, + m_dma, SYS_REF_TEMP, + sig_dma); + vcq_add_hcq_flush(&vcq[idx++], d.core_id); + + ret = cmh_tm_submit_sync_tmo(vcq, vcq_count, 1, d.mbx_idx, + cmh_tm_slow_op_timeout_jiffies()); + + if (!ret) { + /* Sync bounce buffer so CPU sees the DMA-written signature */ + cmh_dma_sync_for_cpu(sig_dma, sig_sz, DMA_FROM_DEVICE); + memcpy(dst, sig_buf, sig_sz); + ret = sig_sz; + } + +out_unmap: + if (sk_buf) { + if (!cmh_dma_map_error(sk_dma)) + cmh_dma_unmap_single(sk_dma, sk_sz, DMA_TO_DEVICE); + kfree_sensitive(sk_buf); + } + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_sz, DMA_FROM_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, slen, DMA_TO_DEVICE); + +out_free: + kfree(sig_buf); + kfree(m_buf); + return ret; +} + +/* + * SLH-DSA verify (synchronous sig_alg) + * + * @src: raw signature + * @slen: signature length + * @digest: message bytes + * @dlen: message length + * + * Returns 0 on successful verification, negative errno on failure. + */ +static int cmh_slhdsa_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + u32 sig_sz = slhdsa_get_sig_size(ctx->param_set); + u32 pk_sz = slhdsa_pk_size(ctx->param_set); + struct core_dispatch d = cmh_core_select_instance(CMH_CORE_HCQ); + struct vcq_cmd vcq[HCQ_VCQ_CMDS_MIN]; + u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL; + dma_addr_t sig_dma = DMA_MAPPING_ERROR; + dma_addr_t m_dma = DMA_MAPPING_ERROR; + dma_addr_t pk_dma = DMA_MAPPING_ERROR; + int ret; + + if (!ctx->pub_key) + return -EINVAL; + if (slen != sig_sz) + return -EINVAL; + if (!dlen || dlen > SLHDSA_MAX_MSG_LEN) + return -EINVAL; + + sig_buf = kmemdup(src, slen, GFP_KERNEL); + m_buf = kmemdup(digest, dlen, GFP_KERNEL); + pk_buf = kmemdup(ctx->pub_key, pk_sz, GFP_KERNEL); + if (!sig_buf || !m_buf || !pk_buf) { + ret = -ENOMEM; + goto out_free; + } + + sig_dma = cmh_dma_map_single(sig_buf, sig_sz, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE); + pk_dma = cmh_dma_map_single(pk_buf, pk_sz, DMA_TO_DEVICE); + + if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) || + cmh_dma_map_error(pk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], HCQ_VCQ_CMDS_MIN); + vcq_add_hcq_slhdsa_verify_internal(&vcq[1], d.core_id, ctx->param_set, + dlen, m_dma, pk_dma, sig_dma); + vcq_add_hcq_flush(&vcq[2], d.core_id); + + /* SLH-DSA verify recomputes hyper-tree hashes -- inherently slow */ + ret = cmh_tm_submit_sync_tmo(vcq, HCQ_VCQ_CMDS_MIN, 1, d.mbx_idx, + cmh_tm_slow_op_timeout_jiffies()); + +out_unmap: + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, pk_sz, DMA_TO_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, sig_sz, DMA_TO_DEVICE); + +out_free: + kfree(pk_buf); + kfree(m_buf); + kfree(sig_buf); + return ret; +} + +static int cmh_slhdsa_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + u32 expected = slhdsa_pk_size(ctx->param_set); + + if (keylen != expected) + return -EINVAL; + + kfree(ctx->pub_key); + ctx->pub_key = NULL; + ctx->pub_key_len = 0; + + ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->pub_key) + return -ENOMEM; + + ctx->pub_key_len = keylen; + return 0; +} + +static int cmh_slhdsa_set_priv_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + + /* Raw sk (4*n bytes) */ + if (keylen != slhdsa_sk_size(ctx->param_set)) + return -EINVAL; + + return cmh_key_setkey_raw(&ctx->key, key, keylen, CORE_ID_HCQ); +} + +static unsigned int cmh_slhdsa_key_size(struct crypto_sig *tfm) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + + /* crypto_sig_keysize() returns bits, not bytes */ + return slhdsa_pk_size(ctx->param_set) * 8; +} + +static unsigned int cmh_slhdsa_max_size(struct crypto_sig *tfm) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + + return slhdsa_get_sig_size(ctx->param_set); +} + +static void cmh_slhdsa_exit(struct crypto_sig *tfm) +{ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); + + cmh_key_destroy(&ctx->key); + kfree(ctx->pub_key); + ctx->pub_key = NULL; +} + +/* Generate init functions for all 12 parameter sets */ +#define SLHDSA_INIT(ps_val) \ + static int cmh_slhdsa_init_##ps_val(struct crypto_sig *tfm) \ + { \ + struct cmh_slhdsa_tfm_ctx *ctx = cmh_slhdsa_ctx(tfm); \ + memset(ctx, 0, sizeof(*ctx)); \ + ctx->param_set = ps_val; \ + return 0; \ + } + +SLHDSA_INIT(HCQ_SLHDSA_SHAKE_128S) +SLHDSA_INIT(HCQ_SLHDSA_SHAKE_128F) +SLHDSA_INIT(HCQ_SLHDSA_SHAKE_192S) +SLHDSA_INIT(HCQ_SLHDSA_SHAKE_192F) +SLHDSA_INIT(HCQ_SLHDSA_SHAKE_256S) +SLHDSA_INIT(HCQ_SLHDSA_SHAKE_256F) +SLHDSA_INIT(HCQ_SLHDSA_SHA2_128S) +SLHDSA_INIT(HCQ_SLHDSA_SHA2_128F) +SLHDSA_INIT(HCQ_SLHDSA_SHA2_192S) +SLHDSA_INIT(HCQ_SLHDSA_SHA2_192F) +SLHDSA_INIT(HCQ_SLHDSA_SHA2_256S) +SLHDSA_INIT(HCQ_SLHDSA_SHA2_256F) + +#define SLHDSA_ALG(name, drv, ps_val) { \ + .sign = cmh_slhdsa_sign, \ + .verify = cmh_slhdsa_verify, \ + .set_pub_key = cmh_slhdsa_set_pub_key, \ + .set_priv_key = cmh_slhdsa_set_priv_key, \ + .key_size = cmh_slhdsa_key_size, \ + .max_size = cmh_slhdsa_max_size, \ + .init = cmh_slhdsa_init_##ps_val, \ + .exit = cmh_slhdsa_exit, \ + .base = { \ + .cra_name = name, \ + .cra_driver_name = drv, \ + .cra_priority = 300, \ + .cra_module = THIS_MODULE, \ + .cra_ctxsize = sizeof(struct cmh_slhdsa_tfm_ctx), \ + }, \ + } + +static struct sig_alg cmh_slhdsa_algs[] = { + SLHDSA_ALG("slh-dsa-shake-128s", "cri-cmh-slh-dsa-shake-128s", HCQ_SLHDSA_SHAKE_128S), + SLHDSA_ALG("slh-dsa-shake-128f", "cri-cmh-slh-dsa-shake-128f", HCQ_SLHDSA_SHAKE_128F), + SLHDSA_ALG("slh-dsa-shake-192s", "cri-cmh-slh-dsa-shake-192s", HCQ_SLHDSA_SHAKE_192S), + SLHDSA_ALG("slh-dsa-shake-192f", "cri-cmh-slh-dsa-shake-192f", HCQ_SLHDSA_SHAKE_192F), + SLHDSA_ALG("slh-dsa-shake-256s", "cri-cmh-slh-dsa-shake-256s", HCQ_SLHDSA_SHAKE_256S), + SLHDSA_ALG("slh-dsa-shake-256f", "cri-cmh-slh-dsa-shake-256f", HCQ_SLHDSA_SHAKE_256F), + SLHDSA_ALG("slh-dsa-sha2-128s", "cri-cmh-slh-dsa-sha2-128s", HCQ_SLHDSA_SHA2_128S), + SLHDSA_ALG("slh-dsa-sha2-128f", "cri-cmh-slh-dsa-sha2-128f", HCQ_SLHDSA_SHA2_128F), + SLHDSA_ALG("slh-dsa-sha2-192s", "cri-cmh-slh-dsa-sha2-192s", HCQ_SLHDSA_SHA2_192S), + SLHDSA_ALG("slh-dsa-sha2-192f", "cri-cmh-slh-dsa-sha2-192f", HCQ_SLHDSA_SHA2_192F), + SLHDSA_ALG("slh-dsa-sha2-256s", "cri-cmh-slh-dsa-sha2-256s", HCQ_SLHDSA_SHA2_256S), + SLHDSA_ALG("slh-dsa-sha2-256f", "cri-cmh-slh-dsa-sha2-256f", HCQ_SLHDSA_SHA2_256F), +}; + +/** + * cmh_pqc_slhdsa_register() - Register SLH-DSA akcipher algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pqc_slhdsa_register(void) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(cmh_slhdsa_algs); i++) { + ret = crypto_register_sig(&cmh_slhdsa_algs[i]); + if (ret) { + dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n", + cmh_slhdsa_algs[i].base.cra_name, ret); + goto err_unregister; + } + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_sig(&cmh_slhdsa_algs[i]); + return ret; +} + +/** + * cmh_pqc_slhdsa_unregister() - Unregister SLH-DSA akcipher algorithms from the crypto framework + */ +void cmh_pqc_slhdsa_unregister(void) +{ + int i = ARRAY_SIZE(cmh_slhdsa_algs); + + while (i--) + crypto_unregister_sig(&cmh_slhdsa_algs[i]); +} diff --git a/drivers/crypto/cmh/cmh_pqc_xmss.c b/drivers/crypto/cmh/cmh_pqc_xmss.c new file mode 100644 index 00000000000000..433ffcd6463de5 --- /dev/null +++ b/drivers/crypto/cmh/cmh_pqc_xmss.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- XMSS/XMSS-MT Signature Driver (verify-only, sig_alg, synchronous) + * + * Registers "xmss" and "xmss-mt" sig algorithms with verify-only + * callbacks. Sign is not supported (stateful signature -- key + * management must happen externally). + * + * Verify: src = raw signature, digest = message bytes + * Public key: raw pk bytes (variable length, set via set_pub_key) + */ + +#include +#include +#include +#include +#include + +#include "cmh_sys.h" +#include "cmh_hcq_abi.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_pqc.h" + +#define XMSS_VCQ_CMDS 3 /* header + cmd + flush */ + +struct cmh_xmss_tfm_ctx { + u8 *pub_key; + u32 pub_key_len; + u32 xmss_mt; /* 0 = XMSS, 1 = XMSS-MT */ +}; + +static inline struct cmh_xmss_tfm_ctx *cmh_xmss_ctx(struct crypto_sig *tfm) +{ + return crypto_sig_ctx(tfm); +} + +/* + * XMSS/XMSS-MT verify (synchronous sig_alg) + * + * @src: raw signature + * @slen: signature length + * @digest: message bytes + * @dlen: message length + * + * Returns 0 on successful verification, negative errno on failure. + */ +static int cmh_xmss_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm); + struct core_dispatch d = cmh_core_select_instance(CMH_CORE_HCQ); + struct vcq_cmd vcq[XMSS_VCQ_CMDS]; + u8 *sig_buf = NULL, *m_buf = NULL, *pk_buf = NULL; + dma_addr_t sig_dma = DMA_MAPPING_ERROR; + dma_addr_t m_dma = DMA_MAPPING_ERROR; + dma_addr_t pk_dma = DMA_MAPPING_ERROR; + int ret; + + if (!ctx->pub_key) + return -EINVAL; + if (!slen || slen > XMSS_MAX_SIG_LEN) + return -EINVAL; + if (!dlen || dlen > XMSS_MAX_MSG_LEN) + return -EINVAL; + + sig_buf = kmemdup(src, slen, GFP_KERNEL); + m_buf = kmemdup(digest, dlen, GFP_KERNEL); + pk_buf = kmemdup(ctx->pub_key, ctx->pub_key_len, GFP_KERNEL); + if (!sig_buf || !m_buf || !pk_buf) { + ret = -ENOMEM; + goto out_free; + } + + sig_dma = cmh_dma_map_single(sig_buf, slen, DMA_TO_DEVICE); + m_dma = cmh_dma_map_single(m_buf, dlen, DMA_TO_DEVICE); + pk_dma = cmh_dma_map_single(pk_buf, ctx->pub_key_len, DMA_TO_DEVICE); + + if (cmh_dma_map_error(sig_dma) || cmh_dma_map_error(m_dma) || + cmh_dma_map_error(pk_dma)) { + ret = -ENOMEM; + goto out_unmap; + } + + vcq_set_header(&vcq[0], XMSS_VCQ_CMDS); + vcq_add_hcq_xmss_verify(&vcq[1], d.core_id, ctx->xmss_mt, + ctx->pub_key_len, slen, dlen, + pk_dma, sig_dma, m_dma); + vcq_add_hcq_flush(&vcq[2], d.core_id); + + /* XMSS verify traverses Merkle hash chains -- inherently slow */ + ret = cmh_tm_submit_sync_tmo(vcq, XMSS_VCQ_CMDS, 1, d.mbx_idx, + cmh_tm_slow_op_timeout_jiffies()); + +out_unmap: + if (!cmh_dma_map_error(pk_dma)) + cmh_dma_unmap_single(pk_dma, ctx->pub_key_len, DMA_TO_DEVICE); + if (!cmh_dma_map_error(m_dma)) + cmh_dma_unmap_single(m_dma, dlen, DMA_TO_DEVICE); + if (!cmh_dma_map_error(sig_dma)) + cmh_dma_unmap_single(sig_dma, slen, DMA_TO_DEVICE); + +out_free: + kfree(pk_buf); + kfree(m_buf); + kfree(sig_buf); + return ret; +} + +static int cmh_xmss_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm); + + if (!keylen || keylen > XMSS_MAX_PK_LEN) + return -EINVAL; + + kfree(ctx->pub_key); + ctx->pub_key = NULL; + ctx->pub_key_len = 0; + + ctx->pub_key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->pub_key) + return -ENOMEM; + + ctx->pub_key_len = keylen; + return 0; +} + +static unsigned int cmh_xmss_key_size(struct crypto_sig *tfm) +{ + struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm); + + return ctx->pub_key_len * 8; +} + +static int cmh_xmss_init(struct crypto_sig *tfm) +{ + struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + return 0; +} + +static int cmh_xmss_mt_init(struct crypto_sig *tfm) +{ + struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + ctx->xmss_mt = 1; + return 0; +} + +static void cmh_xmss_exit(struct crypto_sig *tfm) +{ + struct cmh_xmss_tfm_ctx *ctx = cmh_xmss_ctx(tfm); + + kfree(ctx->pub_key); + ctx->pub_key = NULL; +} + +static struct sig_alg cmh_xmss_algs[] = { + { + .verify = cmh_xmss_verify, + .set_pub_key = cmh_xmss_set_pub_key, + .key_size = cmh_xmss_key_size, + .init = cmh_xmss_init, + .exit = cmh_xmss_exit, + .base = { + .cra_name = "xmss", + .cra_driver_name = "cri-cmh-xmss", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_xmss_tfm_ctx), + }, + }, + { + .verify = cmh_xmss_verify, + .set_pub_key = cmh_xmss_set_pub_key, + .key_size = cmh_xmss_key_size, + .init = cmh_xmss_mt_init, + .exit = cmh_xmss_exit, + .base = { + .cra_name = "xmss-mt", + .cra_driver_name = "cri-cmh-xmss-mt", + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct cmh_xmss_tfm_ctx), + }, + }, +}; + +/** + * cmh_pqc_xmss_register() - Register XMSS/XMSS-MT sig algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_pqc_xmss_register(void) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(cmh_xmss_algs); i++) { + ret = crypto_register_sig(&cmh_xmss_algs[i]); + if (ret) { + dev_err(cmh_dev(), "cmh: failed to register %s (%d)\n", + cmh_xmss_algs[i].base.cra_name, ret); + goto err_unregister; + } + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_sig(&cmh_xmss_algs[i]); + return ret; +} + +/** + * cmh_pqc_xmss_unregister() - Unregister XMSS/XMSS-MT sig algorithms from the crypto framework + */ +void cmh_pqc_xmss_unregister(void) +{ + int i = ARRAY_SIZE(cmh_xmss_algs); + + while (i--) + crypto_unregister_sig(&cmh_xmss_algs[i]); +} From 8dec60a6fd08939629e24c15f2818c9fead13de7 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:35 -0700 Subject: [PATCH 19/21] Documentation: ioctl: add CMH ioctl documentation and register 'J' Add Documentation/userspace-api/ioctl/cmh_mgmt.rst documenting the ioctl commands on the /dev/cmh_mgmt misc device for the CRI CryptoManager Hub (CMH) hardware crypto accelerator driver. Covers key management, KIC key derivation, PKE (RSA, ECDSA, ECDH, EdDSA), PQC (ML-KEM, ML-DSA, SLH-DSA), SM2, EAC, and DRBG. Link the page into the userspace-api/ioctl index toctree. Register ioctl magic number 'J' (0x4A) in ioctl-number.rst. The driver uses ioctls 0x01-0x40. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- .../userspace-api/ioctl/cmh_mgmt.rst | 941 ++++++++++++++++++ Documentation/userspace-api/ioctl/index.rst | 1 + .../userspace-api/ioctl/ioctl-number.rst | 1 + 3 files changed, 943 insertions(+) create mode 100644 Documentation/userspace-api/ioctl/cmh_mgmt.rst diff --git a/Documentation/userspace-api/ioctl/cmh_mgmt.rst b/Documentation/userspace-api/ioctl/cmh_mgmt.rst new file mode 100644 index 00000000000000..b0968ba6b15336 --- /dev/null +++ b/Documentation/userspace-api/ioctl/cmh_mgmt.rst @@ -0,0 +1,941 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================================= +CMH Key Management ioctl Interface (cmh_mgmt) +============================================= + +:Author: Cryptography Research, Inc. (CRI) +:Maintainer: linux-crypto@vger.kernel.org + +Introduction +============ + +The ``/dev/cmh_mgmt`` character device provides user-space access to key +management, key derivation, public-key, and post-quantum cryptographic +operations on the CryptoManager Hub (CMH) hardware accelerator. + +The device is created by the ``cmh`` kernel module as a ``misc_device``. +All operations are synchronous -- the ioctl blocks until the hardware +completes. Opening the device requires ``CAP_SYS_ADMIN``. + +All ioctl argument structures are versioned: user space sets the +``version`` field to ``CMH_MGMT_V1`` (currently 1). This allows the +driver to extend structures in the future without breaking the ABI. + +Data types and ioctl numbers are defined in +````. The ioctl type letter is ``'J'`` +(0x4A). + +Error Handling +============== + +Unless otherwise noted, all ioctls return 0 on success and a negative +errno on failure. Common error codes: + +========== ============================================================= +``EINVAL`` Invalid ``version`` field, unsupported parameter, or + out-of-range length. +``EFAULT`` Failed to copy data to/from user space. +``ENOMEM`` Kernel memory allocation failed. +``EIO`` Hardware returned an error (eSW command failure). +``ENOENT`` Key not found (``KEY_FIND``, ``KEY_LIST``). +========== ============================================================= + +Datastore Concepts +================== + +The CMH hardware maintains an embedded datastore managed by the eSW +firmware. Objects in the datastore are identified by a 64-bit reference +(``ref``) and optionally by a 64-bit Content ID (``cid``). + +Two storage classes exist: + +**Temporary (SYS_REF_TEMP)** + Lifetime is scoped to a single mailbox slot. The eSW firmware + reclaims the object when the slot is reused. Used for raw-key + provisioning via ``KEY_NEW`` + ``KEY_WRITE``. + +**Persistent (SYS_REF_PERSIST)** + Survives across mailbox slots. Requires explicit deletion via + ``KEY_DELETE``. Identified by CID; resolved to a per-mailbox ref + via ``KEY_FIND``. + +Mailbox Dispatch +================ + +All ``/dev/cmh_mgmt`` ioctls are submitted on a single management +mailbox. This is a structural requirement of the eSW datastore model, +not a tunable: + +* Datastore access control is **per-mailbox**. ``KEY_NEW`` grants the + creating mailbox read/write/execute access; other mailboxes have none + until granted. The returned 64-bit ``ref`` encodes a randomised + offset and does **not** carry the owning mailbox, so an operation that + receives only a ``ref`` (``KEY_GRANT``, ``KEY_READ``, ``KEY_DELETE``, + ``DS_EXPORT``) cannot itself determine which mailbox owns the object. + Using one fixed management mailbox guarantees that a key's create, + modify, grant, read and hardware-held-key compute steps all share the + mailbox that holds its access rights, without exposing mailbox + identity in the UABI. User space may still widen a key's access to + additional mailboxes via ``KEY_GRANT``. + +* The eSW ``SYS_REF_TEMP`` scratch store is per-mailbox and persists + across ioctl calls, so a multi-step flow that derives into + ``SYS_REF_TEMP`` (for example a ``KIC_*`` derivation) and later + consumes it (``DS_EXPORT`` with ``wrap_key = SYS_REF_TEMP``) requires + both calls to use the same mailbox. + +Per-core ``cri,mbx`` device-tree affinity applies to the *stateless* +in-kernel crypto API path, which carries no datastore state between +requests and is balanced across mailboxes by the driver. + +Key Types +========= + +The ``ds_type`` field in ``KEY_NEW`` and ``KEY_WRITE`` selects the +datastore object type. Values are defined as ``CMH_DS_*`` constants: + +================================= ===== ============================== +Constant Value Description +================================= ===== ============================== +``CMH_DS_RAW_VALUE`` 1 Raw byte array +``CMH_DS_AES_KEY`` 2 AES key (128/192/256-bit) +``CMH_DS_AES_XTS_KEY`` 3 AES-XTS key (256/512-bit) +``CMH_DS_HMAC_KEY`` 4 HMAC key +``CMH_DS_KMAC_KEY`` 5 KMAC key +``CMH_DS_SM4_KEY`` 6 SM4 key (128-bit) +``CMH_DS_CHACHA20_KEY`` 7 ChaCha20 key (256-bit) +``CMH_DS_RSA_PRIV_KEY`` 10 RSA private key +``CMH_DS_RSA_PUB_KEY`` 11 RSA public key +``CMH_DS_RSA_CRT_KEY`` 12 RSA CRT private key +``CMH_DS_ECDSA_PRIV_KEY`` 13 ECDSA private key +``CMH_DS_ECDSA_PUB_KEY`` 14 ECDSA public key +``CMH_DS_ECDH_PRIV_KEY`` 15 ECDH private key +``CMH_DS_EDDSA_PRIV_KEY`` 16 EdDSA private key +``CMH_DS_SHARED_SECRET`` 17 Shared secret +``CMH_DS_SM2_PRIV_KEY`` 18 SM2 private key +``CMH_DS_ML_KEM_DK`` 20 ML-KEM decapsulation key +``CMH_DS_ML_DSA_SK`` 21 ML-DSA secret key +``CMH_DS_SLHDSA_SK`` 25 SLH-DSA secret key +================================= ===== ============================== + +Key Flags +========= + +The ``flags`` field in ``KEY_NEW`` and ``KEY_WRITE`` is a bitmask: + +================== =========== ======================================== +Flag Bit Description +================== =========== ======================================== +``CMH_FLAG_PT`` 16 Key can be read as plaintext +``CMH_FLAG_XC`` 17 Key can be exported over XC bus +``CMH_FLAG_SCA`` 18 SCA key stored in 2 shares +================== =========== ======================================== + +Elliptic Curve IDs +================== + +Curve identifiers for PKE operations (``curve`` field): + +========================== ===== +Constant Value +========================== ===== +``CMH_CURVE_P192`` 0x01 +``CMH_CURVE_P224`` 0x02 +``CMH_CURVE_P256`` 0x03 +``CMH_CURVE_P384`` 0x04 +``CMH_CURVE_P521`` 0x05 +``CMH_CURVE_SECP256K1`` 0x07 +``CMH_CURVE_BP192R1`` 0x11 +``CMH_CURVE_BP224R1`` 0x12 +``CMH_CURVE_BP256R1`` 0x13 +``CMH_CURVE_BP320R1`` 0x14 +``CMH_CURVE_BP384R1`` 0x15 +``CMH_CURVE_BP512R1`` 0x16 +``CMH_CURVE_SM2`` 0x18 +``CMH_CURVE_25519`` 0x21 +``CMH_CURVE_448`` 0x22 +========================== ===== + +Key Management ioctls +===================== + +CMH_IOCTL_KEY_NEW +----------------- + +Create a new empty datastore object. + +:Direction: ``_IOWR`` +:Number: 0x01 +:Argument: ``struct cmh_ioctl_key_new`` + +:: + + struct cmh_ioctl_key_new { + __u32 version; /* must be CMH_MGMT_V1 */ + __u32 ds_type; /* CMH_DS_* key type */ + __u32 len; /* key length in bytes */ + __u32 flags; /* CMH_FLAG_* */ + __u64 cid; /* caller ID (name) for the key */ + __u64 ref; /* [out] key reference */ + }; + +The returned ``ref`` is used in subsequent ``KEY_WRITE``, ``KEY_READ``, +and crypto operation ioctls. + +CMH_IOCTL_KEY_NEW_RANDOM +------------------------ + +Create a new datastore object filled with hardware-generated random data. + +:Direction: ``_IOWR`` +:Number: 0x0B +:Argument: ``struct cmh_ioctl_key_new`` + +Same structure as ``KEY_NEW``. The hardware DRBG fills the object with +``len`` random bytes. + +CMH_IOCTL_KEY_WRITE +------------------- + +Write key material into a previously created datastore object. + +:Direction: ``_IOW`` +:Number: 0x02 +:Argument: ``struct cmh_ioctl_key_write`` + +:: + + struct cmh_ioctl_key_write { + __u32 version; + __u32 len; /* key data length */ + __u32 ds_type; /* CMH_DS_* key type */ + __u32 flags; /* CMH_FLAG_* */ + __u64 ref; /* key reference from KEY_NEW */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to key material */ + }; + +If ``wrap_key`` is ``CMH_REF_NONE`` (0), key material is written in +plaintext. Otherwise, the data is unwrapped using the specified +wrapping key. + +CMH_IOCTL_KEY_READ +------------------ + +Read key material from a datastore object. + +:Direction: ``_IOWR`` +:Number: 0x03 +:Argument: ``struct cmh_ioctl_key_read`` + +:: + + struct cmh_ioctl_key_read { + __u32 version; + __u32 len; /* buffer length */ + __u64 ref; /* key reference */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to output buffer */ + __u32 out_len; /* [out] actual bytes written */ + __u32 __reserved; + }; + +Plaintext reads require the ``CMH_FLAG_PT`` attribute on the key. +The eSW prepends a 16-byte header (``CMH_SYS_WRAP_HDR_SIZE``) even +for plaintext reads; the output buffer must accommodate this. The +output overhead is ``CMH_DS_EXPORT_OVERHEAD_PLAIN`` (16 bytes) for +plaintext reads and ``CMH_DS_EXPORT_OVERHEAD_WRAPPED`` (48 bytes: +16-byte header + 16-byte nonce + 16-byte tag) for wrapped reads. + +CMH_IOCTL_KEY_FIND +------------------ + +Resolve a Content ID to a datastore reference. + +:Direction: ``_IOWR`` +:Number: 0x04 +:Argument: ``struct cmh_ioctl_key_find`` + +:: + + struct cmh_ioctl_key_find { + __u32 version; + __u32 __reserved; + __u64 cid; /* caller ID to search for */ + __u64 ref; /* [out] resolved key reference */ + __u32 len; /* [out] key length */ + __u32 type; /* [out] key type */ + }; + +Returns ``-ENOENT`` if no object with the given CID exists. + +CMH_IOCTL_KEY_LIST +------------------ + +Iterate datastore objects. + +:Direction: ``_IOWR`` +:Number: 0x0E +:Argument: ``struct cmh_ioctl_key_list`` + +:: + + struct cmh_ioctl_key_list { + __u32 version; + __u32 __reserved; + __u64 start_ref; /* starting DS reference (0 = first) */ + __u64 ref; /* [out] object reference */ + __u64 cid; /* [out] caller ID */ + __u32 len; /* [out] object length */ + __u32 type; /* [out] object type */ + }; + +Pass ``start_ref=0`` to begin from the first object. On return, pass +the returned ``ref`` as ``start_ref`` in the next call. Iteration ends +when ``ref == 0``. + +CMH_IOCTL_KEY_GRANT +------------------- + +Set per-mailbox access permissions on a datastore object. + +:Direction: ``_IOW`` +:Number: 0x05 +:Argument: ``struct cmh_ioctl_key_grant`` + +:: + + struct cmh_ioctl_key_grant { + __u32 version; + __u32 __reserved; + __u64 ref; /* key reference */ + __u64 read; /* per-MBX read permission bitfield */ + __u64 write; /* per-MBX write permission bitfield */ + __u64 execute; /* per-MBX execute permission bitfield */ + }; + +CMH_IOCTL_KEY_DELETE +-------------------- + +Delete a datastore object (persistent keys only). + +:Direction: ``_IOW`` +:Number: 0x06 +:Argument: ``struct cmh_ioctl_key_grant`` + +Uses the same structure as ``KEY_GRANT``; only the ``ref`` field is +used. + +Datastore Export/Import ioctls +============================== + +CMH_IOCTL_DS_EXPORT +------------------- + +Export the entire datastore as an encrypted blob. + +:Direction: ``_IOWR`` +:Number: 0x07 +:Argument: ``struct cmh_ioctl_ds_export`` + +:: + + struct cmh_ioctl_ds_export { + __u32 version; + __u32 len; /* buffer length */ + __u64 cid; /* caller ID for response tagging */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to output buffer */ + __u32 out_len; /* [out] actual bytes written */ + __u32 __reserved; + }; + +CMH_IOCTL_DS_IMPORT +------------------- + +Import a previously exported datastore blob. + +:Direction: ``_IOW`` +:Number: 0x08 +:Argument: ``struct cmh_ioctl_ds_import`` + +:: + + struct cmh_ioctl_ds_import { + __u32 version; + __u32 len; /* blob length */ + __u64 wrap_key; /* wrapping key ref (CMH_REF_NONE = plaintext) */ + __u64 data; /* user-space pointer to import blob */ + }; + +Key Derivation ioctls (KIC) +=========================== + +The Key Initialization Core (KIC) provides hardware key derivation from +OTP-provisioned base keys. Up to 8 base keys are available +(``CMH_KIC_KEY1`` through ``CMH_KIC_KEY8``). + +CMH_IOCTL_KIC_HKDF1 +-------------------- + +HKDF-based key derivation (single-step, label only). + +:Direction: ``_IOWR`` +:Number: 0x09 +:Argument: ``struct cmh_ioctl_kic_hkdf1`` + +:: + + struct cmh_ioctl_kic_hkdf1 { + __u32 version; + __u32 key_len; /* output key length */ + __u64 base_key; /* KIC base key reference */ + __u64 cid; /* CID for the new DS entry */ + __u64 label; /* user-space pointer to label data */ + __u32 label_len; /* label length in bytes */ + __u32 flags; /* CMH_KIC_FLAG_* */ + __u64 ref; /* [out] derived key reference */ + }; + +If ``CMH_KIC_FLAG_TEMP`` is set, the result is stored in the temporary +datastore (not persistent). + +CMH_IOCTL_KIC_HKDF2 +-------------------- + +HKDF-based key derivation (two-step, with salt key). + +:Direction: ``_IOWR`` +:Number: 0x0A +:Argument: ``struct cmh_ioctl_kic_hkdf2`` + +:: + + struct cmh_ioctl_kic_hkdf2 { + __u32 version; + __u32 key_len; + __u64 base_key; + __u64 salt_key; /* salt key reference (CMH_REF_NONE = no salt) */ + __u64 cid; + __u64 label; + __u32 label_len; + __u32 flags; + __u64 ref; /* [out] derived key reference */ + }; + +CMH_IOCTL_KIC_AES_CMAC_KDF +--------------------------- + +AES-CMAC-based key derivation (NIST SP 800-108). + +:Direction: ``_IOWR`` +:Number: 0x0C +:Argument: ``struct cmh_ioctl_kic_aes_cmac_kdf`` + +:: + + struct cmh_ioctl_kic_aes_cmac_kdf { + __u32 version; + __u32 key_len; /* base & output key length (must be 32) */ + __u64 base_key; + __u64 cid; + __u64 label; + __u32 label_len; + __u32 flags; + __u64 ref; /* [out] derived key reference */ + }; + +CMH_IOCTL_KIC_DKEK_DERIVE +-------------------------- + +Derive a Device Key Encryption Key (DKEK) for secure key export. + +:Direction: ``_IOWR`` +:Number: 0x0D +:Argument: ``struct cmh_ioctl_kic_dkek_derive`` + +:: + + struct cmh_ioctl_kic_dkek_derive { + __u32 version; + __u32 host_id; /* target host ID (0 = caller's own) */ + __u64 base_key; + __u64 cid; + __u64 metadata; /* user-space pointer to metadata */ + __u32 metadata_len; + __u32 flags; + __u64 ref; /* [out] derived KEK reference */ + }; + +PKE (Public Key Engine) ioctls +============================== + +RSA Operations +-------------- + +CMH_IOCTL_PKE_RSA_ENC +~~~~~~~~~~~~~~~~~~~~~~ + +RSA public-key encryption. + +:Direction: ``_IOWR`` +:Number: 0x10 +:Argument: ``struct cmh_ioctl_pke_rsa_enc`` + +The public key (e, n) is passed as raw user-space buffers. + +CMH_IOCTL_PKE_RSA_DEC +~~~~~~~~~~~~~~~~~~~~~~ + +RSA private-key decryption using a datastore key reference. + +:Direction: ``_IOWR`` +:Number: 0x11 +:Argument: ``struct cmh_ioctl_pke_rsa_dec`` + +CMH_IOCTL_PKE_RSA_CRT_DEC +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RSA CRT private-key decryption (faster, uses CRT key format). + +:Direction: ``_IOWR`` +:Number: 0x12 +:Argument: ``struct cmh_ioctl_pke_rsa_crt_dec`` + +CMH_IOCTL_PKE_RSA_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate an RSA key pair in hardware. + +:Direction: ``_IOWR`` +:Number: 0x13 +:Argument: ``struct cmh_ioctl_pke_rsa_keygen`` + +Returns private key and optional CRT key as datastore references. +The modulus is written back to user space. + +ECDSA Operations +---------------- + +CMH_IOCTL_PKE_ECDSA_SIGN +~~~~~~~~~~~~~~~~~~~~~~~~~ + +ECDSA signature generation using a datastore private key. + +:Direction: ``_IOWR`` +:Number: 0x14 +:Argument: ``struct cmh_ioctl_pke_ecdsa_sign`` + +CMH_IOCTL_PKE_ECDH +~~~~~~~~~~~~~~~~~~~ + +Compute ECDH shared secret from a peer public key and a datastore +private key. + +:Direction: ``_IOWR`` +:Number: 0x16 +:Argument: ``struct cmh_ioctl_pke_ecdh`` + +If ``CMH_PKE_FLAG_DS_RESULT`` is set, the shared secret is stored in +the datastore and a reference is returned instead of raw bytes. + +CMH_IOCTL_PKE_ECDH_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Derive a public key from a datastore private key. + +:Direction: ``_IOWR`` +:Number: 0x17 +:Argument: ``struct cmh_ioctl_pke_ecdh_keygen`` + +EdDSA Operations +---------------- + +CMH_IOCTL_PKE_EDDSA_SIGN +~~~~~~~~~~~~~~~~~~~~~~~~~ + +EdDSA (Ed25519/Ed448) signature generation. + +:Direction: ``_IOWR`` +:Number: 0x18 +:Argument: ``struct cmh_ioctl_pke_eddsa_sign`` + +Note: the ``digest`` field is the full message (pure EdDSA), not a +pre-computed hash. + +CMH_IOCTL_PKE_EDDSA_VERIFY +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +EdDSA signature verification. + +:Direction: ``_IOW`` +:Number: 0x19 +:Argument: ``struct cmh_ioctl_pke_eddsa_verify`` + +EC Key Management +----------------- + +CMH_IOCTL_PKE_EC_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate an EC private key in the hardware datastore. + +:Direction: ``_IOWR`` +:Number: 0x1A +:Argument: ``struct cmh_ioctl_pke_ec_keygen`` + +CMH_IOCTL_PKE_EC_PUBGEN +~~~~~~~~~~~~~~~~~~~~~~~~ + +Derive the public key from a datastore private key. + +:Direction: ``_IOWR`` +:Number: 0x1B +:Argument: ``struct cmh_ioctl_pke_ec_pubgen`` + +CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate a 2-share SCA-protected Ed448 private key. + +:Direction: ``_IOWR`` +:Number: 0x1C +:Argument: ``struct cmh_ioctl_pke_eddsa_keygen_sca`` + +Post-Quantum Cryptography (PQC) ioctls +======================================= + +PQC operations support the following flags in the ``flags`` field: + +============================ ==== ==================================== +Flag Bit Description +============================ ==== ==================================== +``CMH_QSE_FLAG_MASKED`` 0 Use masked (SCA-resistant) HW path +``CMH_QSE_FLAG_DS_REF`` 1 Store key output in DS, return ref +``CMH_QSE_FLAG_HW_RNG`` 2 Use HW RNG for seed/randomness +============================ ==== ==================================== + +ML-KEM (FIPS 203) +----------------- + +CMH_IOCTL_ML_KEM_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate an ML-KEM key pair. + +:Direction: ``_IOWR`` +:Number: 0x20 +:Argument: ``struct cmh_ioctl_ml_kem_keygen`` + +Security parameter ``k`` selects the strength: 2 (ML-KEM-512), +3 (ML-KEM-768), or 4 (ML-KEM-1024). + +CMH_IOCTL_ML_KEM_ENC +~~~~~~~~~~~~~~~~~~~~~ + +ML-KEM encapsulation. Produces ciphertext and shared secret. + +:Direction: ``_IOWR`` +:Number: 0x21 +:Argument: ``struct cmh_ioctl_ml_kem_enc`` + +CMH_IOCTL_ML_KEM_DEC +~~~~~~~~~~~~~~~~~~~~~ + +ML-KEM decapsulation. Recovers shared secret from ciphertext. + +:Direction: ``_IOWR`` +:Number: 0x22 +:Argument: ``struct cmh_ioctl_ml_kem_dec`` + +ML-DSA (FIPS 204) +----------------- + +CMH_IOCTL_ML_DSA_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate an ML-DSA key pair. + +:Direction: ``_IOWR`` +:Number: 0x23 +:Argument: ``struct cmh_ioctl_ml_dsa_keygen`` + +Security parameter ``mode`` selects the strength: 2 (ML-DSA-44), +3 (ML-DSA-65), or 5 (ML-DSA-87). + +.. note:: + + When ``CMH_QSE_FLAG_DS_REF`` keeps the secret key in the datastore, + the public key returned in ``pk`` is the only copy: there is no + operation to derive the public key from the secret-key reference + for ML-DSA. User space must persist ``pk`` at keygen time. + +CMH_IOCTL_ML_DSA_SIGN +~~~~~~~~~~~~~~~~~~~~~~ + +ML-DSA signature generation. + +:Direction: ``_IOWR`` +:Number: 0x24 +:Argument: ``struct cmh_ioctl_ml_dsa_sign`` + +If ``mlen`` is set to ``CMH_ML_DSA_MLEN_EXTERNAL_MU`` (0xFFFFFFFF), +the ``m`` pointer is interpreted as a 64-byte pre-hashed mu value +(ExternalMu mode). + +CMH_IOCTL_SLHDSA_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate an SLH-DSA key pair. + +:Direction: ``_IOWR`` +:Number: 0x28 +:Argument: ``struct cmh_ioctl_slhdsa_keygen`` + +.. note:: + + When ``CMH_QSE_FLAG_DS_REF`` keeps the secret key in the datastore, + the public key returned in ``pk`` is the only copy: there is no + operation to derive the public key from the secret-key reference + for SLH-DSA. User space must persist ``pk`` at keygen time. + +CMH_IOCTL_SLHDSA_SIGN +~~~~~~~~~~~~~~~~~~~~~~ + +SLH-DSA signature generation (pure mode). + +:Direction: ``_IOWR`` +:Number: 0x29 +:Argument: ``struct cmh_ioctl_slhdsa_sign`` + +CMH_IOCTL_SLHDSA_SIGN_PREHASH +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SLH-DSA pre-hash signature generation. + +:Direction: ``_IOWR`` +:Number: 0x2D +:Argument: ``struct cmh_ioctl_slhdsa_sign_prehash`` + +The ``prehash_algo`` field selects the hash algorithm +(``CMH_SLHDSA_PREHASH_SHA256``, etc.). + +CMH_IOCTL_SM2_ENC_POINT +~~~~~~~~~~~~~~~~~~~~~~~~ + +:Direction: ``_IOWR`` +:Number: 0x33 +:Argument: ``struct cmh_ioctl_sm2_enc_point`` + +CMH_IOCTL_SM2_ENC_HASH +~~~~~~~~~~~~~~~~~~~~~~~ + +:Direction: ``_IOWR`` +:Number: 0x37 +:Argument: ``struct cmh_ioctl_sm2_enc_hash`` + +CMH_IOCTL_SM2_DEC_POINT +~~~~~~~~~~~~~~~~~~~~~~~~ + +:Direction: ``_IOWR`` +:Number: 0x32 +:Argument: ``struct cmh_ioctl_sm2_dec_point`` + +CMH_IOCTL_SM2_DEC_HASH +~~~~~~~~~~~~~~~~~~~~~~~ + +:Direction: ``_IOWR`` +:Number: 0x36 +:Argument: ``struct cmh_ioctl_sm2_dec_hash`` + +SM2 Key Exchange (GM/T 0003.3) +------------------------------ + +The key exchange protocol is a multi-step flow: + +1. ``EC_KEYGEN(CMH_CURVE_SM2)`` -- generate a long-lived private key. +2. ``EC_PUBGEN`` -- derive the public key. +3. ``SM2_ID_DIGEST`` -- compute the SM3 identity digest (ZA). +4. ``SM2_ECDH_KEYGEN`` -- generate an ephemeral session key. +5. Exchange session keys with the peer. +6. ``SM2_ECDH`` -- compute the shared point. +7. ``SM2_ECDH_HASH`` -- derive the shared key from the shared point + and both parties' ZA digests. + +CMH_IOCTL_SM2_ECDH_KEYGEN +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Direction: ``_IOWR`` +:Number: 0x30 +:Argument: ``struct cmh_ioctl_sm2_ecdh_keygen`` + +``nonce_len`` must be 0 or 32. If ``nonce_len=0``, the hardware +generates the ephemeral scalar and writes it back to the ``nonce`` +buffer. + +CMH_IOCTL_SM2_ECDH +~~~~~~~~~~~~~~~~~~~ + +:Direction: ``_IOWR`` +:Number: 0x31 +:Argument: ``struct cmh_ioctl_sm2_ecdh`` + +If ``shared_point_ref`` points to a non-zero value, the shared point +is kept in the datastore for use by ``SM2_ECDH_HASH``. + +CMH_IOCTL_SM2_ID_DIGEST +~~~~~~~~~~~~~~~~~~~~~~~~ + +Compute the SM3 identity digest (ZA) for a public key and identity +string. + +:Direction: ``_IOWR`` +:Number: 0x34 +:Argument: ``struct cmh_ioctl_sm2_id_digest`` + +CMH_IOCTL_SM2_ECDH_HASH +~~~~~~~~~~~~~~~~~~~~~~~~ + +Derive the shared key from the shared point and ZA digests. + +:Direction: ``_IOWR`` +:Number: 0x35 +:Argument: ``struct cmh_ioctl_sm2_ecdh_hash`` + +.. important:: + + The digest fields use **absolute** ordering per GM/T 0003.3, not + relative own/peer ordering. Both parties must pass: + + - ``peer_id_digest`` = Z_A (initiator's digest) -- hashed first + - ``id_digest`` = Z_B (responder's digest) -- hashed second + +Hardware Management ioctls +========================== + +CMH_IOCTL_EAC_READ +------------------- + +Read and clear the hardware Error and Alarm Controller registers. + +:Direction: ``_IOWR`` +:Number: 0x0F +:Argument: ``struct cmh_ioctl_eac_read`` + +:: + + struct cmh_ioctl_eac_read { + __u32 version; + __u32 __reserved; + __u64 mailbox_notification; + __u32 hw_error; + __u32 hw_nmi; + __u32 hw_panic; + __u32 safety_fatal; + __u32 safety_notification; + __u32 sw_info0; + __u32 sw_info1; + __u32 sram_bank_errors[4]; + __u32 __pad; + }; + +The eSW atomically reads and clears the registers on each call. +Successive reads show only new events since the last read. + +CMH_IOCTL_DRBG_CONFIG +---------------------- + +Configure the hardware DRBG before first use. + +:Direction: ``_IOW`` +:Number: 0x40 +:Argument: ``struct cmh_ioctl_drbg_config`` + +:: + + struct cmh_ioctl_drbg_config { + __u32 version; + __u32 entropy_ratio; /* CMH_DRBG_RATIO_* */ + __u32 security_strength; /* CMH_DRBG_STRENGTH_* */ + __u32 __reserved; + }; + +This is a management operation normally performed once at system +startup. Must be called before any ``hwrng`` reads or DRBG generate +operations. + +ioctl Number Summary +==================== + +====================================== ==== ==== ========================================= +ioctl Dir Seq Argument +====================================== ==== ==== ========================================= +``CMH_IOCTL_KEY_NEW`` IOWR 0x01 ``cmh_ioctl_key_new`` +``CMH_IOCTL_KEY_WRITE`` IOW 0x02 ``cmh_ioctl_key_write`` +``CMH_IOCTL_KEY_READ`` IOWR 0x03 ``cmh_ioctl_key_read`` +``CMH_IOCTL_KEY_FIND`` IOWR 0x04 ``cmh_ioctl_key_find`` +``CMH_IOCTL_KEY_GRANT`` IOW 0x05 ``cmh_ioctl_key_grant`` +``CMH_IOCTL_KEY_DELETE`` IOW 0x06 ``cmh_ioctl_key_grant`` +``CMH_IOCTL_DS_EXPORT`` IOWR 0x07 ``cmh_ioctl_ds_export`` +``CMH_IOCTL_DS_IMPORT`` IOW 0x08 ``cmh_ioctl_ds_import`` +``CMH_IOCTL_KIC_HKDF1`` IOWR 0x09 ``cmh_ioctl_kic_hkdf1`` +``CMH_IOCTL_KIC_HKDF2`` IOWR 0x0A ``cmh_ioctl_kic_hkdf2`` +``CMH_IOCTL_KEY_NEW_RANDOM`` IOWR 0x0B ``cmh_ioctl_key_new`` +``CMH_IOCTL_KIC_AES_CMAC_KDF`` IOWR 0x0C ``cmh_ioctl_kic_aes_cmac_kdf`` +``CMH_IOCTL_KIC_DKEK_DERIVE`` IOWR 0x0D ``cmh_ioctl_kic_dkek_derive`` +``CMH_IOCTL_KEY_LIST`` IOWR 0x0E ``cmh_ioctl_key_list`` +``CMH_IOCTL_EAC_READ`` IOWR 0x0F ``cmh_ioctl_eac_read`` +``CMH_IOCTL_PKE_RSA_ENC`` IOWR 0x10 ``cmh_ioctl_pke_rsa_enc`` +``CMH_IOCTL_PKE_RSA_DEC`` IOWR 0x11 ``cmh_ioctl_pke_rsa_dec`` +``CMH_IOCTL_PKE_RSA_CRT_DEC`` IOWR 0x12 ``cmh_ioctl_pke_rsa_crt_dec`` +``CMH_IOCTL_PKE_RSA_KEYGEN`` IOWR 0x13 ``cmh_ioctl_pke_rsa_keygen`` +``CMH_IOCTL_PKE_ECDSA_SIGN`` IOWR 0x14 ``cmh_ioctl_pke_ecdsa_sign`` +``CMH_IOCTL_PKE_ECDH`` IOWR 0x16 ``cmh_ioctl_pke_ecdh`` +``CMH_IOCTL_PKE_ECDH_KEYGEN`` IOWR 0x17 ``cmh_ioctl_pke_ecdh_keygen`` +``CMH_IOCTL_PKE_EDDSA_SIGN`` IOWR 0x18 ``cmh_ioctl_pke_eddsa_sign`` +``CMH_IOCTL_PKE_EDDSA_VERIFY`` IOW 0x19 ``cmh_ioctl_pke_eddsa_verify`` +``CMH_IOCTL_PKE_EC_KEYGEN`` IOWR 0x1A ``cmh_ioctl_pke_ec_keygen`` +``CMH_IOCTL_PKE_EC_PUBGEN`` IOWR 0x1B ``cmh_ioctl_pke_ec_pubgen`` +``CMH_IOCTL_PKE_EDDSA_KEYGEN_SCA`` IOWR 0x1C ``cmh_ioctl_pke_eddsa_keygen_sca`` +``CMH_IOCTL_ML_KEM_KEYGEN`` IOWR 0x20 ``cmh_ioctl_ml_kem_keygen`` +``CMH_IOCTL_ML_KEM_ENC`` IOWR 0x21 ``cmh_ioctl_ml_kem_enc`` +``CMH_IOCTL_ML_KEM_DEC`` IOWR 0x22 ``cmh_ioctl_ml_kem_dec`` +``CMH_IOCTL_ML_DSA_KEYGEN`` IOWR 0x23 ``cmh_ioctl_ml_dsa_keygen`` +``CMH_IOCTL_ML_DSA_SIGN`` IOWR 0x24 ``cmh_ioctl_ml_dsa_sign`` +``CMH_IOCTL_SLHDSA_KEYGEN`` IOWR 0x28 ``cmh_ioctl_slhdsa_keygen`` +``CMH_IOCTL_SLHDSA_SIGN`` IOWR 0x29 ``cmh_ioctl_slhdsa_sign`` +``CMH_IOCTL_SLHDSA_SIGN_PREHASH`` IOWR 0x2D ``cmh_ioctl_slhdsa_sign_prehash`` +``CMH_IOCTL_SM2_ECDH_KEYGEN`` IOWR 0x30 ``cmh_ioctl_sm2_ecdh_keygen`` +``CMH_IOCTL_SM2_ECDH`` IOWR 0x31 ``cmh_ioctl_sm2_ecdh`` +``CMH_IOCTL_SM2_DEC_POINT`` IOWR 0x32 ``cmh_ioctl_sm2_dec_point`` +``CMH_IOCTL_SM2_ENC_POINT`` IOWR 0x33 ``cmh_ioctl_sm2_enc_point`` +``CMH_IOCTL_SM2_ID_DIGEST`` IOWR 0x34 ``cmh_ioctl_sm2_id_digest`` +``CMH_IOCTL_SM2_ECDH_HASH`` IOWR 0x35 ``cmh_ioctl_sm2_ecdh_hash`` +``CMH_IOCTL_SM2_DEC_HASH`` IOWR 0x36 ``cmh_ioctl_sm2_dec_hash`` +``CMH_IOCTL_SM2_ENC_HASH`` IOWR 0x37 ``cmh_ioctl_sm2_enc_hash`` +``CMH_IOCTL_DRBG_CONFIG`` IOW 0x40 ``cmh_ioctl_drbg_config`` +====================================== ==== ==== ========================================= + +Migration Plan +============== + +Several ioctl commands provide operations that may gain dedicated kernel +crypto API bindings in the future. When those APIs land, the driver will +register through them and the corresponding ioctls will be deprecated +(retained for backward compatibility but no longer the primary interface): + +- **EdDSA** (``CMH_IOCTL_PKE_EDDSA_*``): will migrate to the kernel ``sig`` + API once ed25519/ed448 algorithm types are accepted upstream. + +- **ML-KEM** (``CMH_IOCTL_ML_KEM_*``): will migrate to the kernel KEM API + once the in-flight KEM subsystem series lands. + +- **Key lifecycle** (``CMH_IOCTL_KEY_*``): will evaluate integration with + the kernel KEYS subsystem (trusted-keys / encrypted-keys) as a follow-up + series. + +Operations that are inherently vendor-specific (EAC Chip Authentication, +KIC key derivation, SM2 key exchange, DRBG configuration, datastore +export/import) will remain as ioctls permanently -- they have no +corresponding kernel abstraction and are not expected to gain one. diff --git a/Documentation/userspace-api/ioctl/index.rst b/Documentation/userspace-api/ioctl/index.rst index 475675eae08658..bf88bb6b9a6f6f 100644 --- a/Documentation/userspace-api/ioctl/index.rst +++ b/Documentation/userspace-api/ioctl/index.rst @@ -12,4 +12,5 @@ IOCTLs ioctl-decoding cdrom + cmh_mgmt hdio diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 3f0ef1e27eb07d..220417eb32a9ac 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -170,6 +170,7 @@ Code Seq# Include File Comments 'I' all linux/isdn.h conflict! 'I' 00-0F drivers/isdn/divert/isdn_divert.h conflict! 'I' 40-4F linux/mISDNif.h conflict! +'J' 01-40 uapi/linux/cmh_mgmt_ioctl.h CRI CryptoManager Hub (CMH) 'K' all linux/kd.h 'L' 00-1F linux/loop.h conflict! 'L' 10-1F drivers/scsi/mpt3sas/mpt3sas_ctl.h conflict! From 690395eef6c048addf900699666352f0a6bb0623 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:36 -0700 Subject: [PATCH 20/21] selftests: crypto: cmh - add kselftest for management ioctl Add a minimal kselftest exercising the /dev/cmh_mgmt ioctl interface: - open/close the device node - invalid ioctl returns -ENOTTY - bad version field returns -EINVAL - KEY_NEW + KEY_DELETE lifecycle - KIC HKDF1 key derivation - ML-KEM-768 keygen via hardware RNG Tests use the kselftest_harness.h fixture framework and output TAP. Tests that require hardware features not present on the device under test are gracefully skipped (SKIP). Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- .../selftests/drivers/crypto/cmh/Makefile | 6 + .../drivers/crypto/cmh/cmh_mgmt_test.c | 183 ++++++++++++++++++ .../selftests/drivers/crypto/cmh/config | 1 + 3 files changed, 190 insertions(+) create mode 100644 tools/testing/selftests/drivers/crypto/cmh/Makefile create mode 100644 tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c create mode 100644 tools/testing/selftests/drivers/crypto/cmh/config diff --git a/tools/testing/selftests/drivers/crypto/cmh/Makefile b/tools/testing/selftests/drivers/crypto/cmh/Makefile new file mode 100644 index 00000000000000..86cb63839b27b6 --- /dev/null +++ b/tools/testing/selftests/drivers/crypto/cmh/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_GEN_PROGS := cmh_mgmt_test + +CFLAGS += -Wall -Wno-misleading-indentation -O2 $(KHDR_INCLUDES) + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c b/tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c new file mode 100644 index 00000000000000..4514b5a1349ae4 --- /dev/null +++ b/tools/testing/selftests/drivers/crypto/cmh/cmh_mgmt_test.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Kselftest for /dev/cmh_mgmt ioctl interface. + * + * Tests basic ioctl operations on the CRI CryptoManager Hub management + * device. Requires the cmh module loaded on real or emulated hardware. + * + * Run: ./cmh_mgmt_test + * Output: TAP format (compatible with kselftest harness) + */ + +#include +#include +#include +#include +#include +#include + +#include "kselftest_harness.h" +#include + +#define CMH_DEV "/dev/cmh_mgmt" + +FIXTURE(cmh_mgmt) +{ + int fd; +}; + +FIXTURE_SETUP(cmh_mgmt) +{ + self->fd = open(CMH_DEV, O_RDWR); + if (self->fd < 0 && errno == ENOENT) + SKIP(return, "Device " CMH_DEV " not present (module not loaded?)"); + if (self->fd < 0 && errno == EACCES) + SKIP(return, "Permission denied -- run as root or with CAP_SYS_ADMIN"); + ASSERT_GE(self->fd, 0); +} + +FIXTURE_TEARDOWN(cmh_mgmt) +{ + if (self->fd >= 0) + close(self->fd); +} + +/* + * Test 1: open and close succeed. + * If we get here, FIXTURE_SETUP already validated the open. + */ +TEST_F(cmh_mgmt, open_close) +{ + ASSERT_GE(self->fd, 0); +} + +/* + * Test 2: invalid ioctl number returns -ENOTTY. + */ +TEST_F(cmh_mgmt, invalid_ioctl) +{ + int ret; + unsigned long bogus_cmd = _IOC(_IOC_READ, 'J', 0xFF, 4); + + ret = ioctl(self->fd, bogus_cmd, NULL); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, ENOTTY); +} + +/* + * Test 3: KEY_NEW with bad version field returns -EINVAL. + */ +TEST_F(cmh_mgmt, bad_version) +{ + struct cmh_ioctl_key_new req; + int ret; + + memset(&req, 0, sizeof(req)); + req.version = 0; /* invalid */ + req.ds_type = CMH_DS_AES_KEY; + req.len = 32; + req.flags = CMH_FLAG_PT; + req.cid = 0xDEAD; + + ret = ioctl(self->fd, CMH_IOCTL_KEY_NEW, &req); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, EINVAL); +} + +/* + * Test 4: KEY_NEW creates a key, KEY_DELETE destroys it. + */ +TEST_F(cmh_mgmt, key_new_delete) +{ + struct cmh_ioctl_key_new new_req; + struct cmh_ioctl_key_grant del_req; + int ret; + + memset(&new_req, 0, sizeof(new_req)); + new_req.version = CMH_MGMT_V1; + new_req.ds_type = CMH_DS_AES_KEY; + new_req.len = 32; + new_req.flags = CMH_FLAG_PT; + new_req.cid = 0x5E1F7E57ULL; /* "SELFTEST" */ + + ret = ioctl(self->fd, CMH_IOCTL_KEY_NEW, &new_req); + ASSERT_EQ(ret, 0); + ASSERT_NE(new_req.ref, (uint64_t)0); + + /* Delete the key */ + memset(&del_req, 0, sizeof(del_req)); + del_req.version = CMH_MGMT_V1; + del_req.ref = new_req.ref; + + ret = ioctl(self->fd, CMH_IOCTL_KEY_DELETE, &del_req); + ASSERT_EQ(ret, 0); +} + +/* + * Test 5: KIC HKDF1 key derivation from hardware base key. + * Requires at least one KIC base key provisioned (KIC_KEY1). + */ +TEST_F(cmh_mgmt, kic_hkdf1) +{ + struct cmh_ioctl_kic_hkdf1 req; + static const char label[] = "kselftest-label"; + int ret; + + memset(&req, 0, sizeof(req)); + req.version = CMH_MGMT_V1; + req.key_len = 32; + req.base_key = CMH_KIC_KEY1; + req.cid = 0x4B534C46ULL; /* "KSLF" */ + req.label = (uint64_t)(uintptr_t)label; + req.label_len = sizeof(label) - 1; + req.flags = CMH_KIC_FLAG_TEMP; + + ret = ioctl(self->fd, CMH_IOCTL_KIC_HKDF1, &req); + if (ret < 0 && errno == EIO) + SKIP(return, "KIC base key 1 not provisioned on this device"); + ASSERT_EQ(ret, 0); + ASSERT_NE(req.ref, (uint64_t)0); +} + +/* + * Test 6: ML-KEM-768 keygen using hardware RNG. + * Verifies the PQC keygen path end-to-end. + */ +TEST_F(cmh_mgmt, ml_kem_keygen) +{ + struct cmh_ioctl_ml_kem_keygen req; + /* ML-KEM-768: ek = 384*3+32 = 1184, dk = 768*3+96 = 2400 */ + uint8_t ek[1184]; + uint8_t dk[2400]; + int ret; + + memset(&req, 0, sizeof(req)); + req.version = CMH_MGMT_V1; + req.k = 3; /* ML-KEM-768 */ + req.flags = CMH_QSE_FLAG_HW_RNG; + req.seed = 0; /* HW RNG */ + req.z = 0; /* HW RNG */ + req.ek = (uint64_t)(uintptr_t)ek; + req.dk = (uint64_t)(uintptr_t)dk; + req.dk_cid = 0; + req.dk_ref = 0; + + memset(ek, 0, sizeof(ek)); + memset(dk, 0, sizeof(dk)); + + ret = ioctl(self->fd, CMH_IOCTL_ML_KEM_KEYGEN, &req); + if (ret < 0 && errno == ENODEV) + SKIP(return, "QSE core not available on this hardware"); + ASSERT_EQ(ret, 0); + + /* Verify output is non-zero (extremely unlikely for random keys) */ + { + int i, nonzero = 0; + + for (i = 0; i < 64; i++) + nonzero += (ek[i] != 0); + ASSERT_GT(nonzero, 0); + } +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/drivers/crypto/cmh/config b/tools/testing/selftests/drivers/crypto/cmh/config new file mode 100644 index 00000000000000..063c1dd0e23b33 --- /dev/null +++ b/tools/testing/selftests/drivers/crypto/cmh/config @@ -0,0 +1 @@ +CONFIG_CRYPTO_DEV_CMH=m From 219cd56032454c06e72800c52b3af785437f8408 Mon Sep 17 00:00:00 2001 From: Alex Ousherovitch Date: Thu, 9 Jul 2026 13:30:37 -0700 Subject: [PATCH 21/21] MAINTAINERS: add Rambus CryptoManager Hub (CMH) Add MAINTAINERS entry for the CRI CryptoManager Hub (CMH) hardware crypto accelerator driver under drivers/crypto/cmh/. Co-developed-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Saravanakrishnan Krishnamoorthy Signed-off-by: Alex Ousherovitch Reviewed-by: Joel Wittenauer Reviewed-by: Thi Nguyen Signed-off-by: Linux RISC-V bot --- MAINTAINERS | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 4a8b0fd665ce24..4eac8e69af2337 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6891,6 +6891,23 @@ F: kernel/cred.c F: rust/kernel/cred.rs F: Documentation/security/credentials.rst +CRI CRYPTOMANAGER HUB (CMH) HARDWARE CRYPTO ACCELERATOR +M: Alex Ousherovitch +M: Saravanakrishnan Krishnamoorthy +R: Joel Wittenauer +R: Thi Nguyen +L: linux-crypto@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/cmh-mgmt +F: Documentation/ABI/testing/debugfs-driver-cmh +F: Documentation/ABI/testing/sysfs-driver-cmh +F: Documentation/crypto/device_drivers/cmh.rst +F: Documentation/devicetree/bindings/crypto/cri,cmh.yaml +F: Documentation/userspace-api/ioctl/cmh_mgmt.rst +F: drivers/crypto/cmh/ +F: include/uapi/linux/cmh_mgmt_ioctl.h +F: tools/testing/selftests/drivers/crypto/cmh/ + INTEL CRPS COMMON REDUNDANT PSU DRIVER M: Ninad Palsule L: linux-hwmon@vger.kernel.org