From 370ae03a578a6c278e7df077caff2e91b9909b45 Mon Sep 17 00:00:00 2001 From: "Matthias J. Kannwischer" Date: Fri, 6 Nov 2026 21:34:40 +0800 Subject: [PATCH] riscv64: Add RVV assembly arithmetic backend and Zbb Keccak Signed-off-by: Matthias J. Kannwischer --- .github/workflows/bench.yml | 4 +- BIBLIOGRAPHY.md | 10 + BIBLIOGRAPHY.yml | 6 + dev/fips202/riscv64/auto.h | 14 + .../riscv64/src/fips202_native_riscv64.h | 28 + .../src/keccak_f1600_x1_zbb_riscv64_asm.S | 345 ++++++++++++ .../riscv64/src/keccakf1600_round_constants.c | 37 ++ dev/fips202/riscv64/x1_zbb.h | 25 + dev/riscv64/README.md | 21 + dev/riscv64/meta.h | 270 +++++++++ dev/riscv64/src/arith_native_riscv64.h | 343 ++++++++++++ .../src/mldsa_intt_vlen128_riscv64_asm.S | 88 +++ .../src/mldsa_intt_vlen256_riscv64_asm.S | 88 +++ .../src/mldsa_ntt_vlen128_riscv64_asm.S | 88 +++ .../src/mldsa_ntt_vlen256_riscv64_asm.S | 88 +++ .../src/mldsa_poly_caddq_riscv64_asm.S | 87 +++ .../src/mldsa_poly_chknorm_riscv64_asm.S | 108 ++++ .../src/mldsa_poly_decompose_32_riscv64_asm.S | 116 ++++ .../src/mldsa_poly_decompose_88_riscv64_asm.S | 116 ++++ ...sa_poly_pointwise_montgomery_riscv64_asm.S | 113 ++++ .../src/mldsa_poly_use_hint_32_riscv64_asm.S | 135 +++++ .../src/mldsa_poly_use_hint_88_riscv64_asm.S | 144 +++++ ..._pointwise_acc_montgomery_l4_riscv64_asm.S | 59 ++ ..._pointwise_acc_montgomery_l5_riscv64_asm.S | 59 ++ ..._pointwise_acc_montgomery_l7_riscv64_asm.S | 59 ++ .../src/mldsa_polyz_unpack_17_riscv64_asm.S | 123 +++++ .../src/mldsa_polyz_unpack_19_riscv64_asm.S | 124 +++++ .../src/mldsa_rej_uniform_eta2_riscv64_asm.S | 170 ++++++ .../src/mldsa_rej_uniform_eta4_riscv64_asm.S | 158 ++++++ .../src/mldsa_rej_uniform_riscv64_asm.S | 171 ++++++ dev/riscv64/src/rv64v_decompose.inc | 62 +++ dev/riscv64/src/rv64v_montgomery.inc | 37 ++ dev/riscv64/src/rv64v_ntt.inc | 404 ++++++++++++++ dev/riscv64/src/rv64v_pointwise_acc.inc | 80 +++ dev/riscv64/src/rv64v_unpack.inc | 61 +++ dev/riscv64/src/rv64v_zetas.c | 245 +++++++++ flake.nix | 2 +- mldsa/mldsa_native.c | 69 +++ mldsa/mldsa_native_asm.S | 87 +++ mldsa/src/fips202/native/auto.h | 4 + mldsa/src/fips202/native/riscv64/auto.h | 14 + .../riscv64/src/fips202_native_riscv64.h | 28 + .../src/keccak_f1600_x1_zbb_riscv64_asm.S | 422 +++++++++++++++ .../riscv64/src/keccakf1600_round_constants.c | 37 ++ mldsa/src/fips202/native/riscv64/x1_zbb.h | 25 + mldsa/src/native/meta.h | 4 + mldsa/src/native/riscv64/meta.h | 270 +++++++++ .../native/riscv64/src/arith_native_riscv64.h | 343 ++++++++++++ .../src/mldsa_intt_vlen128_riscv64_asm.S | 511 ++++++++++++++++++ .../src/mldsa_intt_vlen256_riscv64_asm.S | 511 ++++++++++++++++++ .../src/mldsa_ntt_vlen128_riscv64_asm.S | 506 +++++++++++++++++ .../src/mldsa_ntt_vlen256_riscv64_asm.S | 506 +++++++++++++++++ .../src/mldsa_poly_caddq_riscv64_asm.S | 64 +++ .../src/mldsa_poly_chknorm_riscv64_asm.S | 72 +++ .../src/mldsa_poly_decompose_32_riscv64_asm.S | 90 +++ .../src/mldsa_poly_decompose_88_riscv64_asm.S | 95 ++++ ...sa_poly_pointwise_montgomery_riscv64_asm.S | 80 +++ .../src/mldsa_poly_use_hint_32_riscv64_asm.S | 96 ++++ .../src/mldsa_poly_use_hint_88_riscv64_asm.S | 108 ++++ ..._pointwise_acc_montgomery_l4_riscv64_asm.S | 105 ++++ ..._pointwise_acc_montgomery_l5_riscv64_asm.S | 110 ++++ ..._pointwise_acc_montgomery_l7_riscv64_asm.S | 120 ++++ .../src/mldsa_polyz_unpack_17_riscv64_asm.S | 98 ++++ .../src/mldsa_polyz_unpack_19_riscv64_asm.S | 99 ++++ .../src/mldsa_rej_uniform_eta2_riscv64_asm.S | 114 ++++ .../src/mldsa_rej_uniform_eta4_riscv64_asm.S | 109 ++++ .../src/mldsa_rej_uniform_riscv64_asm.S | 127 +++++ mldsa/src/native/riscv64/src/rv64v_zetas.c | 245 +++++++++ proofs/cbmc/intt_native_riscv64/Makefile | 61 +++ .../intt_native_riscv64_harness.c | 16 + .../keccak_f1600_x1_native_riscv64/Makefile | 57 ++ .../keccak_f1600_x1_native_riscv64_harness.c | 13 + proofs/cbmc/ntt_native_riscv64/Makefile | 61 +++ .../ntt_native_riscv64_harness.c | 16 + .../pointwise_acc_native_riscv64/Makefile | 54 ++ .../pointwise_acc_native_riscv64_harness.c | 37 ++ proofs/cbmc/pointwise_native_riscv64/Makefile | 46 ++ .../pointwise_native_riscv64_harness.c | 17 + .../cbmc/poly_caddq_native_riscv64/Makefile | 59 ++ .../poly_caddq_native_riscv64_harness.c | 15 + .../cbmc/poly_chknorm_native_riscv64/Makefile | 59 ++ .../poly_chknorm_native_riscv64_harness.c | 16 + .../poly_decompose_32_native_riscv64/Makefile | 68 +++ ...poly_decompose_32_native_riscv64_harness.c | 19 + .../poly_decompose_88_native_riscv64/Makefile | 65 +++ ...poly_decompose_88_native_riscv64_harness.c | 19 + .../poly_use_hint_native_riscv64/Makefile | 54 ++ .../poly_use_hint_native_riscv64_harness.c | 24 + .../polyz_unpack_17_native_riscv64/Makefile | 65 +++ .../polyz_unpack_17_native_riscv64_harness.c | 19 + .../polyz_unpack_19_native_riscv64/Makefile | 68 +++ .../polyz_unpack_19_native_riscv64_harness.c | 19 + .../rej_uniform_eta_native_riscv64/Makefile | 55 ++ .../rej_uniform_eta_native_riscv64_harness.c | 29 + .../cbmc/rej_uniform_native_riscv64/Makefile | 59 ++ .../rej_uniform_native_riscv64_harness.c | 20 + proofs/hol_light/riscv64/README.md | 69 +++ .../mldsa/keccak_f1600_x1_zbb_riscv64_asm.S | 418 ++++++++++++++ .../mldsa/mldsa_intt_vlen128_riscv64_asm.S | 508 +++++++++++++++++ .../mldsa/mldsa_intt_vlen256_riscv64_asm.S | 508 +++++++++++++++++ .../mldsa/mldsa_ntt_vlen128_riscv64_asm.S | 503 +++++++++++++++++ .../mldsa/mldsa_ntt_vlen256_riscv64_asm.S | 503 +++++++++++++++++ .../mldsa/mldsa_poly_caddq_riscv64_asm.S | 61 +++ .../mldsa/mldsa_poly_chknorm_riscv64_asm.S | 69 +++ .../mldsa_poly_decompose_32_riscv64_asm.S | 83 +++ .../mldsa_poly_decompose_88_riscv64_asm.S | 88 +++ ...sa_poly_pointwise_montgomery_riscv64_asm.S | 73 +++ .../mldsa_poly_use_hint_32_riscv64_asm.S | 89 +++ .../mldsa_poly_use_hint_88_riscv64_asm.S | 101 ++++ ..._pointwise_acc_montgomery_l4_riscv64_asm.S | 100 ++++ ..._pointwise_acc_montgomery_l5_riscv64_asm.S | 105 ++++ ..._pointwise_acc_montgomery_l7_riscv64_asm.S | 115 ++++ .../mldsa/mldsa_polyz_unpack_17_riscv64_asm.S | 91 ++++ .../mldsa/mldsa_polyz_unpack_19_riscv64_asm.S | 91 ++++ .../mldsa_rej_uniform_eta2_riscv64_asm.S | 108 ++++ .../mldsa_rej_uniform_eta4_riscv64_asm.S | 103 ++++ .../mldsa/mldsa_rej_uniform_riscv64_asm.S | 124 +++++ scripts/autogen | 298 +++++++++- scripts/check-contracts | 4 +- scripts/simpasm | 15 +- test/abicheck/abicheck.c | 2 + test/abicheck/abicheck.mk | 1 + test/abicheck/riscv64/abicheck_riscv64.c | 77 +++ test/abicheck/riscv64/abicheck_riscv64.h | 31 ++ test/abicheck/riscv64/abicheck_riscv64.mk | 48 ++ test/abicheck/riscv64/callstub_riscv64.S | 151 ++++++ .../checks/check_intt_vlen128_riscv64_asm.c | 65 +++ .../checks/check_intt_vlen256_riscv64_asm.c | 65 +++ .../check_keccak_f1600_x1_zbb_riscv64_asm.c | 66 +++ .../checks/check_ntt_vlen128_riscv64_asm.c | 65 +++ .../checks/check_ntt_vlen256_riscv64_asm.c | 65 +++ .../checks/check_poly_caddq_riscv64_asm.c | 62 +++ .../checks/check_poly_chknorm_riscv64_asm.c | 63 +++ .../check_poly_decompose_32_riscv64_asm.c | 66 +++ .../check_poly_decompose_88_riscv64_asm.c | 66 +++ ...ck_poly_pointwise_montgomery_riscv64_asm.c | 67 +++ .../check_poly_use_hint_32_riscv64_asm.c | 65 +++ .../check_poly_use_hint_88_riscv64_asm.c | 65 +++ ..._pointwise_acc_montgomery_l4_riscv64_asm.c | 71 +++ ..._pointwise_acc_montgomery_l5_riscv64_asm.c | 71 +++ ..._pointwise_acc_montgomery_l7_riscv64_asm.c | 71 +++ .../check_polyz_unpack_17_riscv64_asm.c | 65 +++ .../check_polyz_unpack_19_riscv64_asm.c | 65 +++ .../check_rej_uniform_eta2_riscv64_asm.c | 68 +++ .../check_rej_uniform_eta4_riscv64_asm.c | 68 +++ .../checks/check_rej_uniform_riscv64_asm.c | 68 +++ test/abicheck/riscv64/checks_riscv64_all.h | 80 +++ test/abicheck/riscv64/selftest_riscv64.S | 74 +++ test/abicheck/selftest.c | 75 ++- test/mk/auto.mk | 20 +- test/mk/components.mk | 11 +- 151 files changed, 16370 insertions(+), 21 deletions(-) create mode 100644 dev/fips202/riscv64/auto.h create mode 100644 dev/fips202/riscv64/src/fips202_native_riscv64.h create mode 100644 dev/fips202/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S create mode 100644 dev/fips202/riscv64/src/keccakf1600_round_constants.c create mode 100644 dev/fips202/riscv64/x1_zbb.h create mode 100644 dev/riscv64/README.md create mode 100644 dev/riscv64/meta.h create mode 100644 dev/riscv64/src/arith_native_riscv64.h create mode 100644 dev/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_caddq_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S create mode 100644 dev/riscv64/src/mldsa_rej_uniform_riscv64_asm.S create mode 100644 dev/riscv64/src/rv64v_decompose.inc create mode 100644 dev/riscv64/src/rv64v_montgomery.inc create mode 100644 dev/riscv64/src/rv64v_ntt.inc create mode 100644 dev/riscv64/src/rv64v_pointwise_acc.inc create mode 100644 dev/riscv64/src/rv64v_unpack.inc create mode 100644 dev/riscv64/src/rv64v_zetas.c create mode 100644 mldsa/src/fips202/native/riscv64/auto.h create mode 100644 mldsa/src/fips202/native/riscv64/src/fips202_native_riscv64.h create mode 100644 mldsa/src/fips202/native/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S create mode 100644 mldsa/src/fips202/native/riscv64/src/keccakf1600_round_constants.c create mode 100644 mldsa/src/fips202/native/riscv64/x1_zbb.h create mode 100644 mldsa/src/native/riscv64/meta.h create mode 100644 mldsa/src/native/riscv64/src/arith_native_riscv64.h create mode 100644 mldsa/src/native/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_caddq_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/mldsa_rej_uniform_riscv64_asm.S create mode 100644 mldsa/src/native/riscv64/src/rv64v_zetas.c create mode 100644 proofs/cbmc/intt_native_riscv64/Makefile create mode 100644 proofs/cbmc/intt_native_riscv64/intt_native_riscv64_harness.c create mode 100644 proofs/cbmc/keccak_f1600_x1_native_riscv64/Makefile create mode 100644 proofs/cbmc/keccak_f1600_x1_native_riscv64/keccak_f1600_x1_native_riscv64_harness.c create mode 100644 proofs/cbmc/ntt_native_riscv64/Makefile create mode 100644 proofs/cbmc/ntt_native_riscv64/ntt_native_riscv64_harness.c create mode 100644 proofs/cbmc/pointwise_acc_native_riscv64/Makefile create mode 100644 proofs/cbmc/pointwise_acc_native_riscv64/pointwise_acc_native_riscv64_harness.c create mode 100644 proofs/cbmc/pointwise_native_riscv64/Makefile create mode 100644 proofs/cbmc/pointwise_native_riscv64/pointwise_native_riscv64_harness.c create mode 100644 proofs/cbmc/poly_caddq_native_riscv64/Makefile create mode 100644 proofs/cbmc/poly_caddq_native_riscv64/poly_caddq_native_riscv64_harness.c create mode 100644 proofs/cbmc/poly_chknorm_native_riscv64/Makefile create mode 100644 proofs/cbmc/poly_chknorm_native_riscv64/poly_chknorm_native_riscv64_harness.c create mode 100644 proofs/cbmc/poly_decompose_32_native_riscv64/Makefile create mode 100644 proofs/cbmc/poly_decompose_32_native_riscv64/poly_decompose_32_native_riscv64_harness.c create mode 100644 proofs/cbmc/poly_decompose_88_native_riscv64/Makefile create mode 100644 proofs/cbmc/poly_decompose_88_native_riscv64/poly_decompose_88_native_riscv64_harness.c create mode 100644 proofs/cbmc/poly_use_hint_native_riscv64/Makefile create mode 100644 proofs/cbmc/poly_use_hint_native_riscv64/poly_use_hint_native_riscv64_harness.c create mode 100644 proofs/cbmc/polyz_unpack_17_native_riscv64/Makefile create mode 100644 proofs/cbmc/polyz_unpack_17_native_riscv64/polyz_unpack_17_native_riscv64_harness.c create mode 100644 proofs/cbmc/polyz_unpack_19_native_riscv64/Makefile create mode 100644 proofs/cbmc/polyz_unpack_19_native_riscv64/polyz_unpack_19_native_riscv64_harness.c create mode 100644 proofs/cbmc/rej_uniform_eta_native_riscv64/Makefile create mode 100644 proofs/cbmc/rej_uniform_eta_native_riscv64/rej_uniform_eta_native_riscv64_harness.c create mode 100644 proofs/cbmc/rej_uniform_native_riscv64/Makefile create mode 100644 proofs/cbmc/rej_uniform_native_riscv64/rej_uniform_native_riscv64_harness.c create mode 100644 proofs/hol_light/riscv64/README.md create mode 100644 proofs/hol_light/riscv64/mldsa/keccak_f1600_x1_zbb_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen128_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen256_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen128_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen256_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_caddq_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_chknorm_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_32_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_88_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_pointwise_montgomery_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_32_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_88_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_17_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_19_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta2_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta4_riscv64_asm.S create mode 100644 proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_riscv64_asm.S create mode 100644 test/abicheck/riscv64/abicheck_riscv64.c create mode 100644 test/abicheck/riscv64/abicheck_riscv64.h create mode 100644 test/abicheck/riscv64/abicheck_riscv64.mk create mode 100644 test/abicheck/riscv64/callstub_riscv64.S create mode 100644 test/abicheck/riscv64/checks/check_intt_vlen128_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_intt_vlen256_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_keccak_f1600_x1_zbb_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_ntt_vlen128_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_ntt_vlen256_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_caddq_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_chknorm_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_decompose_32_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_decompose_88_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_pointwise_montgomery_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_use_hint_32_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_poly_use_hint_88_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_polyz_unpack_17_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_polyz_unpack_19_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_rej_uniform_eta2_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_rej_uniform_eta4_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks/check_rej_uniform_riscv64_asm.c create mode 100644 test/abicheck/riscv64/checks_riscv64_all.h create mode 100644 test/abicheck/riscv64/selftest_riscv64.S diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 89c27b247c..d9b26f51c0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -63,13 +63,13 @@ jobs: - system: bpi name: SpacemiT K1 8 (Banana Pi F3) benchmarks bench_pmu: PERF - archflags: "-march=rv64imafdcv_zicsr_zifencei" + archflags: "-march=rv64imafdcv_zicsr_zifencei_zbb" cflags: "" ldflags: "-static" bench_extra_args: -w exec-on-bpi cross_prefix: riscv64-unknown-linux-gnu- nix_shell: cross-riscv64 - only_no_opt: true + only_no_opt: false zephyr_target: "" - system: m1-mac-mini name: Mac Mini (M1, 2020) benchmarks diff --git a/BIBLIOGRAPHY.md b/BIBLIOGRAPHY.md index f397dff750..697139cd83 100644 --- a/BIBLIOGRAPHY.md +++ b/BIBLIOGRAPHY.md @@ -297,6 +297,7 @@ source code and documentation. - [dev/aarch64_opt/README.md](dev/aarch64_opt/README.md) - [dev/aarch64_opt/src/mldsa_intt_aarch64_asm.S](dev/aarch64_opt/src/mldsa_intt_aarch64_asm.S) - [dev/aarch64_opt/src/mldsa_ntt_aarch64_asm.S](dev/aarch64_opt/src/mldsa_ntt_aarch64_asm.S) + - [dev/riscv64/README.md](dev/riscv64/README.md) - [mldsa/src/native/aarch64/src/mldsa_intt_aarch64_asm.S](mldsa/src/native/aarch64/src/mldsa_intt_aarch64_asm.S) - [mldsa/src/native/aarch64/src/mldsa_ntt_aarch64_asm.S](mldsa/src/native/aarch64/src/mldsa_ntt_aarch64_asm.S) - [proofs/hol_light/aarch64/mldsa/mldsa_intt_aarch64_asm.S](proofs/hol_light/aarch64/mldsa/mldsa_intt_aarch64_asm.S) @@ -420,6 +421,15 @@ source code and documentation. - [proofs/hol_light/x86_64/mldsa/mldsa_rej_uniform_eta2_avx2_asm.S](proofs/hol_light/x86_64/mldsa/mldsa_rej_uniform_eta2_avx2_asm.S) - [proofs/hol_light/x86_64/mldsa/mldsa_rej_uniform_eta4_avx2_asm.S](proofs/hol_light/x86_64/mldsa/mldsa_rej_uniform_eta4_avx2_asm.S) +### `RVPSABI` + +* RISC-V ABIs Specification +* Author(s): + - RISC-V International +* URL: https://github.com/riscv-non-isa/riscv-elf-psabi-doc +* Referenced from: + - [test/abicheck/riscv64/abicheck_riscv64.c](test/abicheck/riscv64/abicheck_riscv64.c) + ### `Round3_Spec` * CRYSTALS-Dilithium Algorithm Specifications and Supporting Documentation (Version 3.1) diff --git a/BIBLIOGRAPHY.yml b/BIBLIOGRAPHY.yml index 2c749b44c4..c80abcc5a9 100644 --- a/BIBLIOGRAPHY.yml +++ b/BIBLIOGRAPHY.yml @@ -241,6 +241,12 @@ - Mitchell, Mark url: https://gitlab.com/x86-psABIs/x86-64-ABI +- id: RVPSABI + short: RISC-V psABI + name: "RISC-V ABIs Specification" + author: RISC-V International + url: https://github.com/riscv-non-isa/riscv-elf-psabi-doc + - id: ArmARMv8M short: Armv8-M ARM name: "Armv8-M Architecture Reference Manual (DDI 0553)" diff --git a/dev/fips202/riscv64/auto.h b/dev/fips202/riscv64/auto.h new file mode 100644 index 0000000000..0f1f6871b5 --- /dev/null +++ b/dev/fips202/riscv64/auto.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DEV_FIPS202_RISCV64_AUTO_H +#define MLD_DEV_FIPS202_RISCV64_AUTO_H +/* Default FIPS202 assembly profile for RISC-V 64 systems */ + +#if defined(__riscv_zbb) +#include "x1_zbb.h" +#endif + +#endif /* !MLD_DEV_FIPS202_RISCV64_AUTO_H */ diff --git a/dev/fips202/riscv64/src/fips202_native_riscv64.h b/dev/fips202/riscv64/src/fips202_native_riscv64.h new file mode 100644 index 0000000000..cfc571a582 --- /dev/null +++ b/dev/fips202/riscv64/src/fips202_native_riscv64.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_DEV_FIPS202_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H +#define MLD_DEV_FIPS202_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +#define mld_keccakf1600_round_constants \ + MLD_NAMESPACE(keccakf1600_round_constants) +MLD_INTERNAL_DATA_DECLARATION const uint64_t + mld_keccakf1600_round_constants[24]; + +#define mld_keccak_f1600_x1_zbb_riscv64_asm \ + MLD_NAMESPACE(keccak_f1600_x1_zbb_riscv64_asm) +void mld_keccak_f1600_x1_zbb_riscv64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/keccak_f1600_x1_zbb_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mld_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#endif /* !MLD_DEV_FIPS202_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H */ diff --git a/dev/fips202/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S b/dev/fips202/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S new file mode 100644 index 0000000000..f74bacdb3c --- /dev/null +++ b/dev/fips202/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S @@ -0,0 +1,345 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: keccak_f1600_x1_zbb_riscv64_asm + Description: RV64 scalar implementation of the Keccak-f[1600] permutation for a single state, using Zbb + Signature: void mld_keccak_f1600_x1_zbb_riscv64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [ZBB] + a0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + a1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 144 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_SYS_RISCV64) && defined(__riscv_zbb) && \ + defined(MLD_FIPS202_RISCV64_NEED_X1_ZBB) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * The 25 lanes of the state are held in registers throughout, lane + * A[x,y] in register Axy with x in {a,e,i,o,u} and y in {b,g,k,m,s}. This + * leaves three temporaries. Each round is computed in place: + * + * - theta: the column parities C[0] and C[2] are kept on the stack while + * the others are computed. D[x] = C[x-1] ^ ROL(C[x+1], 1) is applied to + * column x only after C[x] has been computed. A parity is rotated in + * place once its unrotated value is no longer needed. + * - rho and pi: pi permutes the 24 lanes other than A[0,0] in a single + * cycle. Following the cycle backwards, each lane is overwritten by the + * rotated lane it receives, which has not been overwritten yet. The + * start of the cycle, A[1,0], is left in a temporary by theta. + * - chi: per plane, the two terms that need the original A[0] and A[1] + * are computed first; the plane is then overwritten in place. + * - iota: the round constants are loaded via a pointer kept on the stack. + * The loop body computes two rounds. + * + * Zbb provides the rotation (rori) and AND-NOT (andn) instructions. + */ + +#define Aba a0 +#define Abe a1 +#define Abi a2 +#define Abo a3 +#define Abu a4 +#define Aga a5 +#define Age a6 +#define Agi a7 +#define Ago s2 +#define Agu s3 +#define Aka s4 +#define Ake s5 +#define Aki s6 +#define Ako s7 +#define Aku s8 +#define Ama s9 +#define Ame s10 +#define Ami s11 +#define Amo s0 +#define Amu s1 +#define Asa t3 +#define Ase t4 +#define Asi t5 +#define Aso t6 +#define Asu ra + +#define tmp0 t0 +#define tmp1 t1 +#define tmp2 t2 + +/* Stack layout */ +#define STACK_SIZE 144 +#define STACK_GPRS 0 /* ra, s0-s11 */ +#define STACK_STATE 104 /* state pointer */ +#define STACK_RC 112 /* pointer to the next round constant */ +#define STACK_RC_END 120 /* end of the round constants */ +#define STACK_C0 128 /* column parity C[0] */ +#define STACK_C2 136 /* column parity C[2] */ + +.macro save_gprs + sd ra, (STACK_GPRS + 0)(sp) + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + sd s\i, (STACK_GPRS + 8 + 8*\i)(sp) + .endr +.endm + +.macro restore_gprs + ld ra, (STACK_GPRS + 0)(sp) + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + ld s\i, (STACK_GPRS + 8 + 8*\i)(sp) + .endr +.endm + +/* \d = \a ^ \b ^ \c ^ \e ^ \f */ +.macro xor5 d, a, b, c, e, f + xor \d, \a, \b + xor \d, \d, \c + xor \d, \d, \e + xor \d, \d, \f +.endm + +/* Column x ^= \d, for the lanes \a, \b, \c, \e, \f of column x */ +.macro xorcol d, a, b, c, e, f + xor \a, \a, \d + xor \b, \b, \d + xor \c, \c, \d + xor \e, \e, \d + xor \f, \f, \d +.endm + +.macro theta + /* Park C[0] and C[2] on the stack */ + xor5 tmp0, Aba, Aga, Aka, Ama, Asa + sd tmp0, STACK_C0(sp) + xor5 tmp0, Abi, Agi, Aki, Ami, Asi + sd tmp0, STACK_C2(sp) + + /* tmp0 = C[1], tmp1 = C[4] */ + xor5 tmp0, Abe, Age, Ake, Ame, Ase + xor5 tmp1, Abu, Agu, Aku, Amu, Asu + + /* D[0] = C[4] ^ ROL(C[1], 1). Afterwards, C[4] is only needed + * rotated, so rotate it in place. */ + rori tmp2, tmp0, 63 + xor tmp2, tmp2, tmp1 + rori tmp1, tmp1, 63 + xorcol tmp2, Aba, Aga, Aka, Ama, Asa + + /* D[3] = C[2] ^ ROL(C[4], 1) in tmp1; then tmp2 = C[3], computed + * before column 3 changes */ + ld tmp2, STACK_C2(sp) + xor tmp1, tmp1, tmp2 + xor5 tmp2, Abo, Ago, Ako, Amo, Aso + xorcol tmp1, Abo, Ago, Ako, Amo, Aso + + /* D[2] = C[1] ^ ROL(C[3], 1) in tmp0; tmp1 = C[0] */ + rori tmp1, tmp2, 63 + xor tmp0, tmp0, tmp1 + ld tmp1, STACK_C0(sp) + xorcol tmp0, Abi, Agi, Aki, Ami, Asi + + /* D[4] = C[3] ^ ROL(C[0], 1) in tmp2; tmp0 = C[2] */ + rori tmp0, tmp1, 63 + xor tmp2, tmp2, tmp0 + ld tmp0, STACK_C2(sp) + xorcol tmp2, Abu, Agu, Aku, Amu, Asu + + /* D[1] = C[0] ^ ROL(C[2], 1); the new A[1,0] goes to tmp2 for + * rho_pi. */ + rori tmp0, tmp0, 63 + xor tmp0, tmp0, tmp1 + xor tmp2, Abe, tmp0 + xor Age, Age, tmp0 + xor Ake, Ake, tmp0 + xor Ame, Ame, tmp0 + xor Ase, Ase, tmp0 +.endm + +/* Lane A[x,y] moves to A[y,2x+3y] and is rotated left by r[x,y]. + * Expects A[1,0] in tmp2. */ +.macro rho_pi + rori Abe, Age, 20 // Abe = ROL(Age, 44) + rori Age, Agu, 44 // Age = ROL(Agu, 20) + rori Agu, Asi, 3 // Agu = ROL(Asi, 61) + rori Asi, Aku, 25 // Asi = ROL(Aku, 39) + rori Aku, Asa, 46 // Aku = ROL(Asa, 18) + rori Asa, Abi, 2 // Asa = ROL(Abi, 62) + rori Abi, Aki, 21 // Abi = ROL(Aki, 43) + rori Aki, Ako, 39 // Aki = ROL(Ako, 25) + rori Ako, Amu, 56 // Ako = ROL(Amu, 8) + rori Amu, Aso, 8 // Amu = ROL(Aso, 56) + rori Aso, Ama, 23 // Aso = ROL(Ama, 41) + rori Ama, Abu, 37 // Ama = ROL(Abu, 27) + rori Abu, Asu, 50 // Abu = ROL(Asu, 14) + rori Asu, Ase, 62 // Asu = ROL(Ase, 2) + rori Ase, Ago, 9 // Ase = ROL(Ago, 55) + rori Ago, Ame, 19 // Ago = ROL(Ame, 45) + rori Ame, Aga, 28 // Ame = ROL(Aga, 36) + rori Aga, Abo, 36 // Aga = ROL(Abo, 28) + rori Abo, Amo, 43 // Abo = ROL(Amo, 21) + rori Amo, Ami, 49 // Amo = ROL(Ami, 15) + rori Ami, Ake, 54 // Ami = ROL(Ake, 10) + rori Ake, Agi, 58 // Ake = ROL(Agi, 6) + rori Agi, Aka, 61 // Agi = ROL(Aka, 3) + rori Aka, tmp2, 63 // Aka = ROL(Abe, 1) +.endm + +/* A[x] ^= ~A[x+1] & A[x+2] for the lanes \a0..\a4 of one plane. The terms + * for A[3] and A[4], which need the original A[0] and A[1], are computed + * first; A[0], A[1], A[2] are then updated in place. */ +.macro chi_plane a0, a1, a2, a3, a4 + andn tmp0, \a0, \a4 + andn tmp1, \a1, \a0 + andn tmp2, \a2, \a1 + xor \a0, \a0, tmp2 + andn tmp2, \a3, \a2 + xor \a1, \a1, tmp2 + andn tmp2, \a4, \a3 + xor \a2, \a2, tmp2 + xor \a3, \a3, tmp0 + xor \a4, \a4, tmp1 +.endm + +.macro chi + chi_plane Aba, Abe, Abi, Abo, Abu + chi_plane Aga, Age, Agi, Ago, Agu + chi_plane Aka, Ake, Aki, Ako, Aku + chi_plane Ama, Ame, Ami, Amo, Amu + chi_plane Asa, Ase, Asi, Aso, Asu +.endm + +/* Load or store the state from/to the address in \base. The lane Aba is + * also the state pointer register, so it is accessed last. */ +.macro access_state op, base + \op Abe, 8(\base) + \op Abi, 16(\base) + \op Abo, 24(\base) + \op Abu, 32(\base) + \op Aga, 40(\base) + \op Age, 48(\base) + \op Agi, 56(\base) + \op Ago, 64(\base) + \op Agu, 72(\base) + \op Aka, 80(\base) + \op Ake, 88(\base) + \op Aki, 96(\base) + \op Ako, 104(\base) + \op Aku, 112(\base) + \op Ama, 120(\base) + \op Ame, 128(\base) + \op Ami, 136(\base) + \op Amo, 144(\base) + \op Amu, 152(\base) + \op Asa, 160(\base) + \op Ase, 168(\base) + \op Asi, 176(\base) + \op Aso, 184(\base) + \op Asu, 192(\base) + \op Aba, 0(\base) +.endm + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(keccak_f1600_x1_zbb_riscv64_asm) +MLD_ASM_FN_SYMBOL(keccak_f1600_x1_zbb_riscv64_asm) + + addi sp, sp, -STACK_SIZE + save_gprs + sd a0, STACK_STATE(sp) + sd a1, STACK_RC(sp) + addi tmp0, a1, (24 * 8) + sd tmp0, STACK_RC_END(sp) + access_state ld, a0 + +mld_keccak_f1600_x1_zbb_riscv64_loop: + theta + rho_pi + chi + /* iota */ + ld tmp0, STACK_RC(sp) + ld tmp1, 0(tmp0) + xor Aba, Aba, tmp1 + + theta + rho_pi + chi + /* iota */ + ld tmp0, STACK_RC(sp) + ld tmp1, 8(tmp0) + xor Aba, Aba, tmp1 + addi tmp0, tmp0, 16 + sd tmp0, STACK_RC(sp) + ld tmp1, STACK_RC_END(sp) + bne tmp0, tmp1, mld_keccak_f1600_x1_zbb_riscv64_loop + + ld tmp0, STACK_STATE(sp) + access_state sd, tmp0 + restore_gprs + addi sp, sp, STACK_SIZE + ret +.option pop + +MLD_ASM_FN_SIZE(keccak_f1600_x1_zbb_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef Aba +#undef Abe +#undef Abi +#undef Abo +#undef Abu +#undef Aga +#undef Age +#undef Agi +#undef Ago +#undef Agu +#undef Aka +#undef Ake +#undef Aki +#undef Ako +#undef Aku +#undef Ama +#undef Ame +#undef Ami +#undef Amo +#undef Amu +#undef Asa +#undef Ase +#undef Asi +#undef Aso +#undef Asu +#undef tmp0 +#undef tmp1 +#undef tmp2 +#undef STACK_SIZE +#undef STACK_GPRS +#undef STACK_STATE +#undef STACK_RC +#undef STACK_RC_END +#undef STACK_C0 +#undef STACK_C2 + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64 && __riscv_zbb && MLD_FIPS202_RISCV64_NEED_X1_ZBB && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/fips202/riscv64/src/keccakf1600_round_constants.c b/dev/fips202/riscv64/src/keccakf1600_round_constants.c new file mode 100644 index 0000000000..065203eea5 --- /dev/null +++ b/dev/fips202/riscv64/src/keccakf1600_round_constants.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" + +#if defined(MLD_FIPS202_RISCV64_NEED_X1_ZBB) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_riscv64.h" + +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint64_t + mld_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +#else /* MLD_FIPS202_RISCV64_NEED_X1_ZBB && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLD_EMPTY_CU(fips202_riscv64_round_constants) + +#endif /* !(MLD_FIPS202_RISCV64_NEED_X1_ZBB && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/dev/fips202/riscv64/x1_zbb.h b/dev/fips202/riscv64/x1_zbb.h new file mode 100644 index 0000000000..8a6ab27e3e --- /dev/null +++ b/dev/fips202/riscv64/x1_zbb.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DEV_FIPS202_RISCV64_X1_ZBB_H +#define MLD_DEV_FIPS202_RISCV64_X1_ZBB_H + +/* Part of backend API */ +#define MLD_USE_NATIVE_FIPS202_X1 +/* Guard for assembly file */ +#define MLD_FIPS202_RISCV64_NEED_X1_ZBB + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_riscv64.h" +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x1_native(uint64_t *state) +{ + mld_keccak_f1600_x1_zbb_riscv64_asm(state, mld_keccakf1600_round_constants); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_DEV_FIPS202_RISCV64_X1_ZBB_H */ diff --git a/dev/riscv64/README.md b/dev/riscv64/README.md new file mode 100644 index 0000000000..749d763b79 --- /dev/null +++ b/dev/riscv64/README.md @@ -0,0 +1,21 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# RISC-V Vector Extension Backend + +This is an arithmetic backend for CPUs implementing the RISC-V Vector Extension. The backend is functional for all physical `VLEN`, but the NTT and inverse NTT are so far only implemented for VLEN=128 and VLEN=256, falling back to the default C implementations for other VLENs. + +## Requirements + +- RISC-V 64-bit architecture +- Vector extension (RVV) version 1.0, or Zve64x with VLEN=64 +- Standard "gc" extensions (integer and compressed instructions) + +## Implementation notes + +- All routines are written in assembly in `dev/riscv64/src` and imported into `mldsa/src/native/riscv64/src` by `scripts/autogen` via `scripts/simpasm`. Compressed instructions and linker relaxation are disabled (`.option norvc`, `.option norelax`), so that the object code is fixed, as needed for formal verification. +- Multiplications by known constants (twiddles, scaling factors) use Barrett multiplication[^NeonNTT], based on `vsmul`. The routines set the fixed-point rounding mode `vxrm`, which is not preserved across calls. Multiplications of two variables use Montgomery multiplication, matching the C reference exactly. +- The NTT makes two passes over memory. Layers 1-4 load the coefficients with unit stride and use scalar twiddles on register groups of 8, 4, 2 and 1. Layers 5-8 load them with a stride of 16 coefficients, so that the butterfly partners of these layers are in different registers, and use per-lane twiddles. This avoids permutations within registers, which are costly on the SpacemiT X60 for register groups. +- The register group sizes (LMUL) were chosen by benchmarks on the SpacemiT X60 (VLEN=256). The pointwise multiplication, `poly_decompose` and `poly_use_hint` use LMUL=8, about 7% faster than LMUL=4; `poly_caddq` and `poly_chknorm` use LMUL=4, as LMUL=8 is slower. The pointwise multiplication with accumulation uses LMUL=4 for the inputs and LMUL=8 for the 64-bit accumulator, about 12% faster than LMUL=2. The NTT layers 5-8 use register groups of 4, 4, 2 and 1 with replicated twiddles, about 7% faster than LMUL=1. The eta rejection samplers use LMUL=2 for the nibbles, 16-24% faster than LMUL=1. Rejection sampling and `polyz_unpack` use LMUL=2: LMUL=4 is 50-65% slower because of the cost of `vrgatherei16` on large register groups, and LMUL=1 is slower for rejection sampling and does not divide into whole bytes for `polyz_unpack_17` at VLEN=64. Other microarchitectures may favor different choices. + + +[^NeonNTT]: Becker, Hwang, Kannwischer, Yang, Yang: Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1, [https://eprint.iacr.org/2021/986](https://eprint.iacr.org/2021/986) diff --git a/dev/riscv64/meta.h b/dev/riscv64/meta.h new file mode 100644 index 0000000000..24a424da02 --- /dev/null +++ b/dev/riscv64/meta.h @@ -0,0 +1,270 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_DEV_RISCV64_META_H +#define MLD_DEV_RISCV64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLD_ARITH_BACKEND_RISCV64 + +/* Set of primitives that this backend replaces */ +#define MLD_USE_NATIVE_NTT +#define MLD_USE_NATIVE_INTT +#define MLD_USE_NATIVE_REJ_UNIFORM +#define MLD_USE_NATIVE_REJ_UNIFORM_ETA2 +#define MLD_USE_NATIVE_REJ_UNIFORM_ETA4 +#define MLD_USE_NATIVE_POLY_DECOMPOSE_32 +#define MLD_USE_NATIVE_POLY_DECOMPOSE_88 +#define MLD_USE_NATIVE_POLY_CADDQ +#define MLD_USE_NATIVE_POLY_USE_HINT_32 +#define MLD_USE_NATIVE_POLY_USE_HINT_88 +#define MLD_USE_NATIVE_POLY_CHKNORM +#define MLD_USE_NATIVE_POLYZ_UNPACK_17 +#define MLD_USE_NATIVE_POLYZ_UNPACK_19 +#define MLD_USE_NATIVE_POINTWISE_MONTGOMERY +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4 +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5 +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7 + +#if !defined(__ASSEMBLER__) +#if !defined(CBMC) +#include +#endif + +#include "../../common.h" +#include "../api.h" +#include "src/arith_native_riscv64.h" + +/* Returns the vector register length VLEN in bits. + * CBMC proofs use the contract instead of the body. */ +static MLD_INLINE int mld_rv64v_vlen(void) +__contract__( + ensures(return_value >= 0) +) +{ +#if !defined(CBMC) + return (int)(32 * __riscv_vsetvlmax_e32m1()); +#else + return 0; +#endif +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N]) +{ + /* VLEN = 128 and VLEN = 256 only for now */ + const int vlen = mld_rv64v_vlen(); + if (vlen == 256) + { + mld_ntt_vlen256_riscv64_asm(data, mld_rv64v_ntt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + if (vlen == 128) + { + mld_ntt_vlen128_riscv64_asm(data, mld_rv64v_ntt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + return MLD_NATIVE_FUNC_FALLBACK; +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N]) +{ + /* VLEN = 128 and VLEN = 256 only for now */ + const int vlen = mld_rv64v_vlen(); + if (vlen == 256) + { + mld_intt_vlen256_riscv64_asm(data, mld_rv64v_intt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + if (vlen == 128) + { + mld_intt_vlen128_riscv64_asm(data, mld_rv64v_intt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + return MLD_NATIVE_FUNC_FALLBACK; +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + /* Safety: The return value is at most len <= MLDSA_N, hence the cast + * is safe. */ + return (int)mld_rej_uniform_riscv64_asm(r, len, buf, buflen); +} + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_eta2_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + uint64_t outlen; + /* Constant time: Inputs and outputs to this function are secret. + * It is safe to leak which coefficients are accepted/rejected. + * The assembly implementation must not leak any other information about the + * accepted coefficients. Constant-time testing cannot cover this, and we + * hence have to manually verify the assembly. + * We declassify prior the input data and mark the outputs as secret. + */ + MLD_CT_TESTING_DECLASSIFY(buf, buflen); + outlen = mld_rej_uniform_eta2_riscv64_asm(r, len, buf, buflen); + MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen); + /* Safety: outlen is at most len <= MLDSA_N, hence the cast is safe. */ + return (int)outlen; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + uint64_t outlen; + /* Constant time: see mld_rej_uniform_eta2_native(). */ + MLD_CT_TESTING_DECLASSIFY(buf, buflen); + outlen = mld_rej_uniform_eta4_riscv64_asm(r, len, buf, buflen); + MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen); + /* Safety: outlen is at most len <= MLDSA_N, hence the cast is safe. */ + return (int)outlen; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4 */ +#endif /* !MLD_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0) +{ + mld_poly_decompose_32_riscv64_asm(a1, a0); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0) +{ + mld_poly_decompose_88_riscv64_asm(a1, a0); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_SIGN_API */ + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N]) +{ + mld_poly_caddq_riscv64_asm(a); + return MLD_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h) +{ + mld_poly_use_hint_32_riscv64_asm(a, h); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h) +{ + mld_poly_use_hint_88_riscv64_asm(a, h); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_VERIFY_API */ + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B) +{ + return mld_poly_chknorm_riscv64_asm(a, B); +} + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *buf) +{ + mld_polyz_unpack_17_riscv64_asm(r, buf); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *buf) +{ + mld_polyz_unpack_19_riscv64_asm(r, buf); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_pointwise_montgomery_native( + int32_t a[MLDSA_N], const int32_t b[MLDSA_N]) +{ + mld_poly_pointwise_montgomery_riscv64_asm(a, b); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native( + int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], + const int32_t v[4][MLDSA_N]) +{ + mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(w, u, v); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native( + int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], + const int32_t v[5][MLDSA_N]) +{ + mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(w, u, v); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native( + int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], + const int32_t v[7][MLDSA_N]) +{ + mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(w, u, v); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_DEV_RISCV64_META_H */ diff --git a/dev/riscv64/src/arith_native_riscv64.h b/dev/riscv64/src/arith_native_riscv64.h new file mode 100644 index 0000000000..9628007ded --- /dev/null +++ b/dev/riscv64/src/arith_native_riscv64.h @@ -0,0 +1,343 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_DEV_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#define MLD_DEV_RISCV64_SRC_ARITH_NATIVE_RISCV64_H + +#include "../../../cbmc.h" +#include "../../../common.h" + +#define mld_rv64v_ntt_zetas MLD_NAMESPACE(rv64v_ntt_zetas) +#define mld_rv64v_intt_zetas MLD_NAMESPACE(rv64v_intt_zetas) + +MLD_INTERNAL_DATA_DECLARATION const int32_t mld_rv64v_ntt_zetas[512]; +MLD_INTERNAL_DATA_DECLARATION const int32_t mld_rv64v_intt_zetas[512]; + +#define mld_ntt_vlen256_riscv64_asm MLD_NAMESPACE(ntt_vlen256_riscv64_asm) +void mld_ntt_vlen256_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_ntt_vlen256_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_ntt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + ensures(array_abs_bound(p, 0, MLDSA_N, 94279698)) +); + +#define mld_ntt_vlen128_riscv64_asm MLD_NAMESPACE(ntt_vlen128_riscv64_asm) +void mld_ntt_vlen128_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_ntt_vlen128_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_ntt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + ensures(array_abs_bound(p, 0, MLDSA_N, 94279698)) +); + +#define mld_intt_vlen256_riscv64_asm MLD_NAMESPACE(intt_vlen256_riscv64_asm) +void mld_intt_vlen256_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_intt_vlen256_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_intt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) +); + +#define mld_intt_vlen128_riscv64_asm MLD_NAMESPACE(intt_vlen128_riscv64_asm) +void mld_intt_vlen128_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_intt_vlen128_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_intt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) +); + +#define mld_rej_uniform_riscv64_asm MLD_NAMESPACE(rej_uniform_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +uint64_t mld_rej_uniform_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_rej_uniform_riscv64_asm.ml */ +__contract__( + requires(len <= MLDSA_N && buflen <= 840 && buflen % 3 == 0) + requires(memory_no_alias(buf, buflen)) + requires(memory_no_alias(r, sizeof(int32_t) * len)) + assigns(memory_slice(r, sizeof(int32_t) * len)) + ensures(return_value <= len) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLDSA_Q)) +); + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2 +#define mld_rej_uniform_eta2_riscv64_asm \ + MLD_NAMESPACE(rej_uniform_eta2_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +uint64_t mld_rej_uniform_eta2_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_rej_uniform_eta2_riscv64_asm.ml */ +__contract__( + requires(len <= MLDSA_N && buflen <= 272) + requires(memory_no_alias(buf, buflen)) + requires(memory_no_alias(r, sizeof(int32_t) * len)) + assigns(memory_slice(r, sizeof(int32_t) * len)) + ensures(return_value <= len) + ensures(array_abs_bound(r, 0, (unsigned) return_value, 3)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4 +#define mld_rej_uniform_eta4_riscv64_asm \ + MLD_NAMESPACE(rej_uniform_eta4_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +uint64_t mld_rej_uniform_eta4_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_rej_uniform_eta4_riscv64_asm.ml */ +__contract__( + requires(len <= MLDSA_N && buflen <= 272) + requires(memory_no_alias(buf, buflen)) + requires(memory_no_alias(r, sizeof(int32_t) * len)) + assigns(memory_slice(r, sizeof(int32_t) * len)) + ensures(return_value <= len) + ensures(array_abs_bound(r, 0, (unsigned) return_value, 5)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4 */ +#endif /* !MLD_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +#define mld_poly_decompose_32_riscv64_asm \ + MLD_NAMESPACE(poly_decompose_32_riscv64_asm) +void mld_poly_decompose_32_riscv64_asm(int32_t a1[MLDSA_N], int32_t a0[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_decompose_32_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a1, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a0, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a0, 0, MLDSA_N, 0, MLDSA_Q)) + assigns(memory_slice(a1, sizeof(int32_t) * MLDSA_N)) + assigns(memory_slice(a0, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 16 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 32)) */ + ensures(array_bound(a1, 0, MLDSA_N, 0, 16)) + /* check-magic: 261889 == (MLDSA_Q - 1) / 32 + 1 */ + ensures(array_abs_bound(a0, 0, MLDSA_N, 261889)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +#define mld_poly_decompose_88_riscv64_asm \ + MLD_NAMESPACE(poly_decompose_88_riscv64_asm) +void mld_poly_decompose_88_riscv64_asm(int32_t a1[MLDSA_N], int32_t a0[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_decompose_88_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a1, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a0, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a0, 0, MLDSA_N, 0, MLDSA_Q)) + assigns(memory_slice(a1, sizeof(int32_t) * MLDSA_N)) + assigns(memory_slice(a0, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 44 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 88)) */ + ensures(array_bound(a1, 0, MLDSA_N, 0, 44)) + /* check-magic: 95233 == (MLDSA_Q - 1) / 88 + 1 */ + ensures(array_abs_bound(a0, 0, MLDSA_N, 95233)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_SIGN_API */ + +#define mld_poly_caddq_riscv64_asm MLD_NAMESPACE(poly_caddq_riscv64_asm) +void mld_poly_caddq_riscv64_asm(int32_t a[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_caddq_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(a, 0, MLDSA_N, MLDSA_Q)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) +); + +#if !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +#define mld_poly_use_hint_32_riscv64_asm \ + MLD_NAMESPACE(poly_use_hint_32_riscv64_asm) +void mld_poly_use_hint_32_riscv64_asm(int32_t a[MLDSA_N], + const int32_t h[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_use_hint_32_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(h, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) + requires(array_bound(h, 0, MLDSA_N, 0, 2)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(a, 0, MLDSA_N, 0, 16)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +#define mld_poly_use_hint_88_riscv64_asm \ + MLD_NAMESPACE(poly_use_hint_88_riscv64_asm) +void mld_poly_use_hint_88_riscv64_asm(int32_t a[MLDSA_N], + const int32_t h[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_use_hint_88_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(h, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) + requires(array_bound(h, 0, MLDSA_N, 0, 2)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(a, 0, MLDSA_N, 0, 44)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_VERIFY_API */ + +#define mld_poly_chknorm_riscv64_asm MLD_NAMESPACE(poly_chknorm_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +int mld_poly_chknorm_riscv64_asm(const int32_t a[MLDSA_N], int64_t B) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_chknorm_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(forall(k0, 0, MLDSA_N, a[k0] > INT32_MIN)) + requires(0 <= B && B <= INT32_MAX) + ensures(return_value == 0 || return_value == 1) + ensures((return_value == 0) == array_abs_bound(a, 0, MLDSA_N, B)) +); + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +#define mld_polyz_unpack_17_riscv64_asm \ + MLD_NAMESPACE(polyz_unpack_17_riscv64_asm) +void mld_polyz_unpack_17_riscv64_asm(int32_t r[MLDSA_N], const uint8_t buf[576]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_polyz_unpack_17_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, 576)) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(r, 0, MLDSA_N, -((1 << 17) - 1), (1 << 17) + 1)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +#define mld_polyz_unpack_19_riscv64_asm \ + MLD_NAMESPACE(polyz_unpack_19_riscv64_asm) +void mld_polyz_unpack_19_riscv64_asm(int32_t r[MLDSA_N], const uint8_t buf[640]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_polyz_unpack_19_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, 640)) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(r, 0, MLDSA_N, -((1 << 19) - 1), (1 << 19) + 1)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +#define mld_poly_pointwise_montgomery_riscv64_asm \ + MLD_NAMESPACE(poly_pointwise_montgomery_riscv64_asm) +void mld_poly_pointwise_montgomery_riscv64_asm(int32_t a[MLDSA_N], + const int32_t b[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_poly_pointwise_montgomery_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(b, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(array_abs_bound(a, 0, MLDSA_N, 94279698)) + requires(array_abs_bound(b, 0, MLDSA_N, 94279698)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(a, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4 +#define mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm \ + MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) +void mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm( + int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], + const int32_t v[4][MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(w, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(u, sizeof(int32_t) * 4 * MLDSA_N)) + requires(memory_no_alias(v, sizeof(int32_t) * 4 * MLDSA_N)) + requires(forall(l0, 0, 4, array_bound(u[l0], 0, MLDSA_N, 0, MLDSA_Q))) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(forall(l1, 0, 4, array_abs_bound(v[l1], 0, MLDSA_N, 94279698))) + assigns(memory_slice(w, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(w, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5 +#define mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm \ + MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) +void mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm( + int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], + const int32_t v[5][MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(w, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(u, sizeof(int32_t) * 5 * MLDSA_N)) + requires(memory_no_alias(v, sizeof(int32_t) * 5 * MLDSA_N)) + requires(forall(l0, 0, 5, array_bound(u[l0], 0, MLDSA_N, 0, MLDSA_Q))) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(forall(l1, 0, 5, array_abs_bound(v[l1], 0, MLDSA_N, 94279698))) + assigns(memory_slice(w, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(w, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7 +#define mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm \ + MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) +void mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm( + int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], + const int32_t v[7][MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(w, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(u, sizeof(int32_t) * 7 * MLDSA_N)) + requires(memory_no_alias(v, sizeof(int32_t) * 7 * MLDSA_N)) + requires(forall(l0, 0, 7, array_bound(u[l0], 0, MLDSA_N, 0, MLDSA_Q))) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(forall(l1, 0, 7, array_abs_bound(v[l1], 0, MLDSA_N, 94279698))) + assigns(memory_slice(w, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(w, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7 */ + +#endif /* !MLD_DEV_RISCV64_SRC_ARITH_NATIVE_RISCV64_H */ diff --git a/dev/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S b/dev/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S new file mode 100644 index 0000000000..ed4206c064 --- /dev/null +++ b/dev/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S @@ -0,0 +1,88 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: intt_vlen128_riscv64_asm + Description: Inverse negacyclic NTT of a polynomial in place, input in bitreversed order, scaled by 2^32 (VLEN=128 only) + Signature: void mld_intt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_intt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_ntt.inc" + +/* + * Inverse NTT with 4 lanes per register, see rv64v_ntt.inc; the reverse + * of the forward NTT. Pass B and pass A each run in 4 batches. + * + * Requires VLEN = 128: the register groups of layers 1-4 assume that each + * register holds exactly the VL lanes in use. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(intt_vlen128_riscv64_asm) +MLD_ASM_FN_SYMBOL(intt_vlen128_riscv64_asm) + + ntt_setup 4 + + /* Pass B: layers 8-5 */ + mv ptr, data + addi lane_ptr, zetas, 128 + li batch, 4 +mld_intt_vlen128_riscv64_pass_b: + vsetvl_m1 + access_pass_b vlse32.v + intt_layers8765 + vsetvl_m1 + access_pass_b vsse32.v + addi ptr, ptr, (64 * 4) + addi lane_ptr, lane_ptr, (4 * 4) + addi batch, batch, -1 + bnez batch, mld_intt_vlen128_riscv64_pass_b + + /* Pass A: layers 4-1 */ + mv ptr, data + li batch, 4 +mld_intt_vlen128_riscv64_pass_a: + vsetvl_m1 + access_pass_a vle32.v + intt_layers4321 + vsetvl_m1 + access_pass_a vse32.v + addi ptr, ptr, (4 * 4) + addi batch, batch, -1 + bnez batch, mld_intt_vlen128_riscv64_pass_a + + ret +.option pop + +MLD_ASM_FN_SIZE(intt_vlen128_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S b/dev/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S new file mode 100644 index 0000000000..35c87774c0 --- /dev/null +++ b/dev/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S @@ -0,0 +1,88 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: intt_vlen256_riscv64_asm + Description: Inverse negacyclic NTT of a polynomial in place, input in bitreversed order, scaled by 2^32 (VLEN=256 only) + Signature: void mld_intt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_intt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_ntt.inc" + +/* + * Inverse NTT with 8 lanes per register, see rv64v_ntt.inc; the reverse + * of the forward NTT. Pass B and pass A each run in 2 batches. + * + * Requires VLEN = 256: the register groups of layers 1-4 assume that each + * register holds exactly the VL lanes in use. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(intt_vlen256_riscv64_asm) +MLD_ASM_FN_SYMBOL(intt_vlen256_riscv64_asm) + + ntt_setup 8 + + /* Pass B: layers 8-5 */ + mv ptr, data + addi lane_ptr, zetas, 128 + li batch, 2 +mld_intt_vlen256_riscv64_pass_b: + vsetvl_m1 + access_pass_b vlse32.v + intt_layers8765 + vsetvl_m1 + access_pass_b vsse32.v + addi ptr, ptr, (64 * 8) + addi lane_ptr, lane_ptr, (4 * 8) + addi batch, batch, -1 + bnez batch, mld_intt_vlen256_riscv64_pass_b + + /* Pass A: layers 4-1 */ + mv ptr, data + li batch, 2 +mld_intt_vlen256_riscv64_pass_a: + vsetvl_m1 + access_pass_a vle32.v + intt_layers4321 + vsetvl_m1 + access_pass_a vse32.v + addi ptr, ptr, (4 * 8) + addi batch, batch, -1 + bnez batch, mld_intt_vlen256_riscv64_pass_a + + ret +.option pop + +MLD_ASM_FN_SIZE(intt_vlen256_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S b/dev/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S new file mode 100644 index 0000000000..6e2a0fee3a --- /dev/null +++ b/dev/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S @@ -0,0 +1,88 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: ntt_vlen128_riscv64_asm + Description: Forward negacyclic NTT of a polynomial in place, output in bitreversed order (VLEN=128 only) + Signature: void mld_ntt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_ntt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_ntt.inc" + +/* + * Forward NTT with 4 lanes per register, see rv64v_ntt.inc. Pass A and + * pass B each run in 4 batches. + * + * Requires VLEN = 128: the register groups of layers 1-4 assume that each + * register holds exactly the VL lanes in use. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(ntt_vlen128_riscv64_asm) +MLD_ASM_FN_SYMBOL(ntt_vlen128_riscv64_asm) + + ntt_setup 4 + + /* Pass A: layers 1-4 */ + mv ptr, data + li batch, 4 +mld_ntt_vlen128_riscv64_pass_a: + vsetvl_m1 + access_pass_a vle32.v + ntt_layers1234 + vsetvl_m1 + access_pass_a vse32.v + addi ptr, ptr, (4 * 4) + addi batch, batch, -1 + bnez batch, mld_ntt_vlen128_riscv64_pass_a + + /* Pass B: layers 5-8 */ + mv ptr, data + addi lane_ptr, zetas, 128 + li batch, 4 +mld_ntt_vlen128_riscv64_pass_b: + vsetvl_m1 + access_pass_b vlse32.v + ntt_layers5678 + vsetvl_m1 + access_pass_b vsse32.v + addi ptr, ptr, (64 * 4) + addi lane_ptr, lane_ptr, (4 * 4) + addi batch, batch, -1 + bnez batch, mld_ntt_vlen128_riscv64_pass_b + + ret +.option pop + +MLD_ASM_FN_SIZE(ntt_vlen128_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S b/dev/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S new file mode 100644 index 0000000000..bd4eafe743 --- /dev/null +++ b/dev/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S @@ -0,0 +1,88 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: ntt_vlen256_riscv64_asm + Description: Forward negacyclic NTT of a polynomial in place, output in bitreversed order (VLEN=256 only) + Signature: void mld_ntt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_ntt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_ntt.inc" + +/* + * Forward NTT with 8 lanes per register, see rv64v_ntt.inc. Pass A and + * pass B each run in 2 batches. + * + * Requires VLEN = 256: the register groups of layers 1-4 assume that each + * register holds exactly the VL lanes in use. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(ntt_vlen256_riscv64_asm) +MLD_ASM_FN_SYMBOL(ntt_vlen256_riscv64_asm) + + ntt_setup 8 + + /* Pass A: layers 1-4 */ + mv ptr, data + li batch, 2 +mld_ntt_vlen256_riscv64_pass_a: + vsetvl_m1 + access_pass_a vle32.v + ntt_layers1234 + vsetvl_m1 + access_pass_a vse32.v + addi ptr, ptr, (4 * 8) + addi batch, batch, -1 + bnez batch, mld_ntt_vlen256_riscv64_pass_a + + /* Pass B: layers 5-8 */ + mv ptr, data + addi lane_ptr, zetas, 128 + li batch, 2 +mld_ntt_vlen256_riscv64_pass_b: + vsetvl_m1 + access_pass_b vlse32.v + ntt_layers5678 + vsetvl_m1 + access_pass_b vsse32.v + addi ptr, ptr, (64 * 8) + addi lane_ptr, lane_ptr, (4 * 8) + addi batch, batch, -1 + bnez batch, mld_ntt_vlen256_riscv64_pass_b + + ret +.option pop + +MLD_ASM_FN_SIZE(ntt_vlen256_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_poly_caddq_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_caddq_riscv64_asm.S new file mode 100644 index 0000000000..d74e283f5f --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_caddq_riscv64_asm.S @@ -0,0 +1,87 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_caddq_riscv64_asm + Description: Add q to all negative coefficients of a polynomial + Signature: void mld_poly_caddq_riscv64_asm(int32_t a[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes a[i] += q if a[i] < 0. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 4 + * (LMUL=4). VLMAX for e32,m4 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + */ + +#define ptr a0 +#define count a1 +#define vl t0 +#define modulus t1 +#define stride t2 + +#define data v0 +#define tmp v4 + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_caddq_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_caddq_riscv64_asm) + + li modulus, MLDSA_Q + li count, MLDSA_N + + vsetvli vl, count, e32, m4, ta, ma + slli stride, vl, 2 + +mld_poly_caddq_riscv64_loop: + vle32.v data, (ptr) + vsra.vi tmp, data, 31 + vand.vx tmp, tmp, modulus + vadd.vv data, data, tmp + vse32.v data, (ptr) + + add ptr, ptr, stride + sub count, count, vl + bnez count, mld_poly_caddq_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(poly_caddq_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef ptr +#undef count +#undef vl +#undef modulus +#undef stride +#undef data +#undef tmp + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S new file mode 100644 index 0000000000..48c9199f4b --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S @@ -0,0 +1,108 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_chknorm_riscv64_asm + Description: Infinity-norm bound check on polynomial coefficients + Signature: int mld_poly_chknorm_riscv64_asm(const int32_t a[256], int64_t B) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t a[256] + description: Input polynomial + a1: + type: scalar + c_parameter: int64_t B + description: Norm bound + test_with: 131072 # representative non-negative bound (1 << 17) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Returns 0 if |a[i]| < B for all i, and 1 otherwise. Requires + * a[i] != INT32_MIN and 0 <= B <= INT32_MAX. + * + * Computes the maximum of |a[i]| over all coefficients with fixed control + * flow, and compares it with B once at the end. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 4 + * (LMUL=4). VLMAX for e32,m4 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + */ + +#define ptr a0 +#define bound a1 +#define count a2 +#define vl t0 +#define stride t1 +#define max_val t2 + +#define data v0 +#define neg v4 +#define max_vec v8 +#define red v12 + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_chknorm_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_chknorm_riscv64_asm) + + li count, MLDSA_N + + vsetvli vl, count, e32, m4, ta, ma + slli stride, vl, 2 + vmv.v.i max_vec, 0 + +mld_poly_chknorm_riscv64_loop: + vle32.v data, (ptr) + vrsub.vi neg, data, 0 + vmax.vv data, data, neg + vmaxu.vv max_vec, max_vec, data + + add ptr, ptr, stride + sub count, count, vl + bnez count, mld_poly_chknorm_riscv64_loop + + vmv.s.x red, zero + vredmaxu.vs red, max_vec, red + vmv.x.s max_val, red + + /* Return (max_val >= B) */ + slt a0, max_val, bound + xori a0, a0, 1 + ret +.option pop + +MLD_ASM_FN_SIZE(poly_chknorm_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef ptr +#undef bound +#undef count +#undef vl +#undef stride +#undef max_val +#undef data +#undef neg +#undef max_vec +#undef red + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S new file mode 100644 index 0000000000..2383702ca1 --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S @@ -0,0 +1,116 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_decompose_32_riscv64_asm + Description: Decompose polynomial coefficients into high and low bits for GAMMA2 = (q-1)/32 + Signature: void mld_poly_decompose_32_riscv64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output polynomial with high bits + a1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial, output polynomial with low bits +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_SIGN_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes a1[i], a0[i] = decompose(a0[i]); see rv64v_decompose.inc. + * Requires 0 <= a0[i] < q. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 8 + * (LMUL=8). VLMAX for e32,m8 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + */ + +#define a1_ptr a0 +#define a0_ptr a1 +#define count a2 +#define vl t0 +#define stride t1 +#define modulus t2 +#define c127 t3 +#define half_q t4 +#define two_gamma2 t5 +#define dec_mul t6 +#define dec_round a3 +#define c43 a4 + +#define va1 v0 +#define va0 v8 +#define tmp v16 + +#include "rv64v_decompose.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_decompose_32_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_decompose_32_riscv64_asm) + + decompose_setup 32 + li count, MLDSA_N + + vsetvli vl, count, e32, m8, ta, ma + slli stride, vl, 2 + +mld_poly_decompose_32_riscv64_loop: + vle32.v va0, (a0_ptr) + decompose 32, va1, va0, tmp + vse32.v va1, (a1_ptr) + vse32.v va0, (a0_ptr) + + add a1_ptr, a1_ptr, stride + add a0_ptr, a0_ptr, stride + sub count, count, vl + bnez count, mld_poly_decompose_32_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(poly_decompose_32_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef a1_ptr +#undef a0_ptr +#undef count +#undef vl +#undef stride +#undef modulus +#undef c127 +#undef half_q +#undef two_gamma2 +#undef dec_mul +#undef dec_round +#undef c43 +#undef va1 +#undef va0 +#undef tmp + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_SIGN_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S new file mode 100644 index 0000000000..b354480e3f --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S @@ -0,0 +1,116 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_decompose_88_riscv64_asm + Description: Decompose polynomial coefficients into high and low bits for GAMMA2 = (q-1)/88 + Signature: void mld_poly_decompose_88_riscv64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output polynomial with high bits + a1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial, output polynomial with low bits +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_SIGN_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes a1[i], a0[i] = decompose(a0[i]); see rv64v_decompose.inc. + * Requires 0 <= a0[i] < q. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 8 + * (LMUL=8). VLMAX for e32,m8 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + */ + +#define a1_ptr a0 +#define a0_ptr a1 +#define count a2 +#define vl t0 +#define stride t1 +#define modulus t2 +#define c127 t3 +#define half_q t4 +#define two_gamma2 t5 +#define dec_mul t6 +#define dec_round a3 +#define c43 a4 + +#define va1 v0 +#define va0 v8 +#define tmp v16 + +#include "rv64v_decompose.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_decompose_88_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_decompose_88_riscv64_asm) + + decompose_setup 88 + li count, MLDSA_N + + vsetvli vl, count, e32, m8, ta, ma + slli stride, vl, 2 + +mld_poly_decompose_88_riscv64_loop: + vle32.v va0, (a0_ptr) + decompose 88, va1, va0, tmp + vse32.v va1, (a1_ptr) + vse32.v va0, (a0_ptr) + + add a1_ptr, a1_ptr, stride + add a0_ptr, a0_ptr, stride + sub count, count, vl + bnez count, mld_poly_decompose_88_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(poly_decompose_88_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef a1_ptr +#undef a0_ptr +#undef count +#undef vl +#undef stride +#undef modulus +#undef c127 +#undef half_q +#undef two_gamma2 +#undef dec_mul +#undef dec_round +#undef c43 +#undef va1 +#undef va0 +#undef tmp + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_SIGN_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 44) && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ diff --git a/dev/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S new file mode 100644 index 0000000000..30aac7ae76 --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S @@ -0,0 +1,113 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_pointwise_montgomery_riscv64_asm + Description: Pointwise Montgomery multiplication of two polynomials + Signature: void mld_poly_pointwise_montgomery_riscv64_asm(int32_t a[256], const int32_t b[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t b[256] + description: Input polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes a[i] = a[i] * b[i] * 2^-32 mod q with signed Montgomery + * reduction, matching the C reference exactly; see rv64v_montgomery.inc. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 8 + * (LMUL=8). VLMAX for e32,m8 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + * + * Bounds: |a|, |b| < MLD_NTT_BOUND, so |a * b| < 2^31 * q, and the result + * is bounded by q in absolute value. + */ + +#define a_ptr a0 +#define b_ptr a1 +#define count a2 +#define vl t0 +#define modulus t1 +#define qinv t2 +#define stride t3 + +#define va v0 +#define vb v8 +#define prod_lo v16 +#define prod_hi v24 + +#include "rv64v_montgomery.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_pointwise_montgomery_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_pointwise_montgomery_riscv64_asm) + + montgomery_setup + li count, MLDSA_N + + vsetvli vl, count, e32, m8, ta, ma + slli stride, vl, 2 + +mld_poly_pointwise_montgomery_riscv64_loop: + vle32.v va, (a_ptr) + vle32.v vb, (b_ptr) + vmul.vv prod_lo, va, vb + vmulh.vv prod_hi, va, vb + montgomery_reduce va, prod_lo, prod_hi + vse32.v va, (a_ptr) + + add a_ptr, a_ptr, stride + add b_ptr, b_ptr, stride + sub count, count, vl + bnez count, mld_poly_pointwise_montgomery_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(poly_pointwise_montgomery_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef a_ptr +#undef b_ptr +#undef count +#undef vl +#undef modulus +#undef qinv +#undef stride +#undef va +#undef vb +#undef prod_lo +#undef prod_hi + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (!MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S new file mode 100644 index 0000000000..7cb004b17b --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S @@ -0,0 +1,135 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_use_hint_32_riscv64_asm + Description: Use hint polynomial to correct the high bits of a polynomial for GAMMA2 = (q-1)/32 + Signature: void mld_poly_use_hint_32_riscv64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_VERIFY_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes a[i] = use_hint(a[i], h[i]). Requires 0 <= a[i] < q and + * h[i] in {0, 1}. With a1, a0 = decompose(a); see rv64v_decompose.inc, + * and d = h if a0 > 0 and d = -h otherwise: + * + * r = (a1 + d) mod 16 + * + * For h = 0, this is a1. The sign of d is computed without branches as + * d = (h ^ n) - n with n = (a0 - 1) >> 31, which is -1 if a0 <= 0 and 0 + * otherwise. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 8 + * (LMUL=8). VLMAX for e32,m8 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + */ + +#define a_ptr a0 +#define h_ptr a1 +#define count a2 +#define vl t0 +#define stride t1 +#define modulus t2 +#define c127 t3 +#define half_q t4 +#define two_gamma2 t5 +#define dec_mul t6 +#define dec_round a3 +#define c43 a4 + +#define va1 v0 +#define va0 v8 +#define tmp v16 +#define vh v24 + +#include "rv64v_decompose.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_use_hint_32_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_use_hint_32_riscv64_asm) + + decompose_setup 32 + li count, MLDSA_N + + vsetvli vl, count, e32, m8, ta, ma + slli stride, vl, 2 + +mld_poly_use_hint_32_riscv64_loop: + vle32.v va0, (a_ptr) + vle32.v vh, (h_ptr) + decompose 32, va1, va0, tmp + + /* d = (h ^ n) - n with n = (a0 - 1) >> 31 */ + vadd.vi va0, va0, -1 + vsra.vi va0, va0, 31 + vxor.vv tmp, vh, va0 + vsub.vv tmp, tmp, va0 + + /* a1 = (a1 + d) & 15 */ + vadd.vv va1, va1, tmp + vand.vi va1, va1, 15 + vse32.v va1, (a_ptr) + + add a_ptr, a_ptr, stride + add h_ptr, h_ptr, stride + sub count, count, vl + bnez count, mld_poly_use_hint_32_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(poly_use_hint_32_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef a_ptr +#undef h_ptr +#undef count +#undef vl +#undef stride +#undef modulus +#undef c127 +#undef half_q +#undef two_gamma2 +#undef dec_mul +#undef dec_round +#undef c43 +#undef va1 +#undef va0 +#undef tmp +#undef vh + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_VERIFY_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S b/dev/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S new file mode 100644 index 0000000000..cc132c17ce --- /dev/null +++ b/dev/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S @@ -0,0 +1,144 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_use_hint_88_riscv64_asm + Description: Use hint polynomial to correct the high bits of a polynomial for GAMMA2 = (q-1)/88 + Signature: void mld_poly_use_hint_88_riscv64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_VERIFY_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes a[i] = use_hint(a[i], h[i]). Requires 0 <= a[i] < q and + * h[i] in {0, 1}. With a1, a0 = decompose(a); see rv64v_decompose.inc, + * and d = h if a0 > 0 and d = -h otherwise: + * + * r = a1 + d, with -1 wrapped to 43 and 44 to 0 + * + * For h = 0, this is a1. The sign of d is computed without branches as + * d = (h ^ n) - n with n = (a0 - 1) >> 31, which is -1 if a0 <= 0 and 0 + * otherwise. + * + * Vector-length agnostic: works for any VLEN. Uses register groups of 8 + * (LMUL=8). VLMAX for e32,m8 is a power of 2, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration. + */ + +#define a_ptr a0 +#define h_ptr a1 +#define count a2 +#define vl t0 +#define stride t1 +#define modulus t2 +#define c127 t3 +#define half_q t4 +#define two_gamma2 t5 +#define dec_mul t6 +#define dec_round a3 +#define c43 a4 +#define c44 a5 + +#define va1 v0 +#define va0 v8 +#define tmp v16 +#define vh v24 + +#include "rv64v_decompose.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_use_hint_88_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_use_hint_88_riscv64_asm) + + decompose_setup 88 + li c44, 44 + li count, MLDSA_N + + vsetvli vl, count, e32, m8, ta, ma + slli stride, vl, 2 + +mld_poly_use_hint_88_riscv64_loop: + vle32.v va0, (a_ptr) + vle32.v vh, (h_ptr) + decompose 88, va1, va0, tmp + + /* d = (h ^ n) - n with n = (a0 - 1) >> 31 */ + vadd.vi va0, va0, -1 + vsra.vi va0, va0, 31 + vxor.vv tmp, vh, va0 + vsub.vv tmp, tmp, va0 + + /* a1 = a1 + d, wrapping -1 to 43 and 44 to 0 */ + vadd.vv va1, va1, tmp + vsra.vi tmp, va1, 31 + vand.vx tmp, tmp, c44 + vadd.vv va1, va1, tmp + vrsub.vx tmp, va1, c43 + vsra.vi tmp, tmp, 31 + vand.vv tmp, tmp, va1 + vxor.vv va1, va1, tmp + vse32.v va1, (a_ptr) + + add a_ptr, a_ptr, stride + add h_ptr, h_ptr, stride + sub count, count, vl + bnez count, mld_poly_use_hint_88_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(poly_use_hint_88_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef a_ptr +#undef h_ptr +#undef count +#undef vl +#undef stride +#undef modulus +#undef c127 +#undef half_q +#undef two_gamma2 +#undef dec_mul +#undef dec_round +#undef c43 +#undef c44 +#undef va1 +#undef va0 +#undef tmp +#undef vh + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_VERIFY_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 44) && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ diff --git a/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S b/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S new file mode 100644 index 0000000000..34d3d3a99a --- /dev/null +++ b/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S @@ -0,0 +1,59 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l4_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 4 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(int32_t w[256], const int32_t u[4][256], const int32_t v[4][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t u[4][256] + description: First input vector + a2: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t v[4][256] + description: Second input vector +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_pointwise_acc.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) + + pointwise_acc_montgomery 4, mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_loop + ret +.option pop + +MLD_ASM_FN_SIZE(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S b/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S new file mode 100644 index 0000000000..2d24972680 --- /dev/null +++ b/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S @@ -0,0 +1,59 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l5_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 5 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(int32_t w[256], const int32_t u[5][256], const int32_t v[5][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t u[5][256] + description: First input vector + a2: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t v[5][256] + description: Second input vector +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_pointwise_acc.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) + + pointwise_acc_montgomery 5, mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_loop + ret +.option pop + +MLD_ASM_FN_SIZE(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S b/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S new file mode 100644 index 0000000000..5044c50799 --- /dev/null +++ b/dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S @@ -0,0 +1,59 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l7_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 7 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(int32_t w[256], const int32_t u[7][256], const int32_t v[7][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t u[7][256] + description: First input vector + a2: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t v[7][256] + description: Second input vector +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +#include "rv64v_pointwise_acc.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) + + pointwise_acc_montgomery 7, mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_loop + ret +.option pop + +MLD_ASM_FN_SIZE(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S b/dev/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S new file mode 100644 index 0000000000..8ab6bb8134 --- /dev/null +++ b/dev/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S @@ -0,0 +1,123 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyz_unpack_17_riscv64_asm + Description: Unpack polynomial z with 18-bit packed coefficients for GAMMA1 = 2^17 + Signature: void mld_polyz_unpack_17_riscv64_asm(int32_t r[256], const uint8_t buf[576]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 576 + permissions: read-only + c_parameter: const uint8_t buf[576] + description: Packed input bytes +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes r[i] = 2^17 - x[i], where x[i] is the i-th 18-bit value of the + * packed input; see rv64v_unpack.inc. + * + * Vector-length agnostic: works for any VLEN of at least 64. VLMAX for + * e32,m2 is a power of 2 and at least 4, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration, and 18*VL/8 is a whole number of + * bytes. + */ + +#define out a0 +#define buf a1 +#define count a2 +#define nbytes a3 +#define vl4 a4 +#define two a5 +#define gamma1 a6 +#define vl t0 +#define mask t1 +#define tmp t2 +#define stride t3 + +#define shifts v4 /* register group of 2 */ +#define gidx v8 /* register group of 4 */ +#define gtmp v12 /* register group of 4 */ +#define src v16 /* register group of 2 */ +#define coeffs v20 /* register group of 2 */ + +#include "rv64v_unpack.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyz_unpack_17_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyz_unpack_17_riscv64_asm) + + li count, MLDSA_N + li two, 2 + li gamma1, (1 << 17) + vsetvli vl, count, e32, m2, ta, ma + slli stride, vl, 2 + /* nbytes = 18 * vl / 8 = 9 * vl / 4 */ + slli nbytes, vl, 3 + add nbytes, nbytes, vl + srli nbytes, nbytes, 2 + unpack_setup 18 + +mld_polyz_unpack_17_riscv64_loop: + unpack 18 + vrsub.vx coeffs, coeffs, gamma1 + vse32.v coeffs, (out) + + add buf, buf, nbytes + add out, out, stride + sub count, count, vl + bnez count, mld_polyz_unpack_17_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(polyz_unpack_17_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef out +#undef buf +#undef count +#undef nbytes +#undef vl4 +#undef two +#undef gamma1 +#undef vl +#undef mask +#undef tmp +#undef stride +#undef shifts +#undef gidx +#undef gtmp +#undef src +#undef coeffs + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (!MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API) && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44) \ + && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S b/dev/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S new file mode 100644 index 0000000000..4eba3ed5a4 --- /dev/null +++ b/dev/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyz_unpack_19_riscv64_asm + Description: Unpack polynomial z with 20-bit packed coefficients for GAMMA1 = 2^19 + Signature: void mld_polyz_unpack_19_riscv64_asm(int32_t r[256], const uint8_t buf[640]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 640 + permissions: read-only + c_parameter: const uint8_t buf[640] + description: Packed input bytes +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Computes r[i] = 2^19 - x[i], where x[i] is the i-th 20-bit value of the + * packed input; see rv64v_unpack.inc. + * + * Vector-length agnostic: works for any VLEN of at least 64. VLMAX for + * e32,m2 is a power of 2 and at least 4, so it either divides 256 or + * exceeds it. Hence a single vsetvli with AVL = 256 yields a VL that is + * used unchanged for every iteration, and 20*VL/8 is a whole number of + * bytes. + */ + +#define out a0 +#define buf a1 +#define count a2 +#define nbytes a3 +#define vl4 a4 +#define two a5 +#define gamma1 a6 +#define vl t0 +#define mask t1 +#define tmp t2 +#define stride t3 + +#define shifts v4 /* register group of 2 */ +#define gidx v8 /* register group of 4 */ +#define gtmp v12 /* register group of 4 */ +#define src v16 /* register group of 2 */ +#define coeffs v20 /* register group of 2 */ + +#include "rv64v_unpack.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyz_unpack_19_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyz_unpack_19_riscv64_asm) + + li count, MLDSA_N + li two, 2 + li gamma1, (1 << 19) + vsetvli vl, count, e32, m2, ta, ma + slli stride, vl, 2 + /* nbytes = 20 * vl / 8 = 5 * vl / 2 */ + slli nbytes, vl, 2 + add nbytes, nbytes, vl + srli nbytes, nbytes, 1 + unpack_setup 20 + +mld_polyz_unpack_19_riscv64_loop: + unpack 20 + vrsub.vx coeffs, coeffs, gamma1 + vse32.v coeffs, (out) + + add buf, buf, nbytes + add out, out, stride + sub count, count, vl + bnez count, mld_polyz_unpack_19_riscv64_loop + + ret +.option pop + +MLD_ASM_FN_SIZE(polyz_unpack_19_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef out +#undef buf +#undef count +#undef nbytes +#undef vl4 +#undef two +#undef gamma1 +#undef vl +#undef mask +#undef tmp +#undef stride +#undef shifts +#undef gidx +#undef gtmp +#undef src +#undef coeffs + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (!MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API) && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S b/dev/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S new file mode 100644 index 0000000000..fc42cdc6dd --- /dev/null +++ b/dev/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S @@ -0,0 +1,170 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_eta2_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers in [-2, +2] + Signature: uint64_t mld_rej_uniform_eta2_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer + test_with: 272 # 2 * SHAKE256_RATE +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_KEYPAIR_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Splits each input byte into its low and high nibble (in this order), + * accepts nibbles t < 15 and maps them to 2 - (t mod 5). + * + * Vector-length agnostic: works for any VLEN of at least 64. + * + * Each iteration loads up to VLEN/8 bytes and zero-extends them to 16-bit + * lanes (LMUL=2). Moving the high nibble to the upper byte of each lane + * interleaves the nibbles as 8-bit lanes, which are mapped, compressed and + * sign-extended to 32 bits (LMUL=8) for the store. The number of bytes per + * iteration is min(VLEN/8, remaining), which vsetvli returns unchanged. + */ + +#define out a0 /* output pointer (advancing) */ +#define len a1 +#define buf a2 /* input pointer (advancing) */ +#define remaining a3 /* remaining input bytes */ +#define ctr a4 /* number of sampled coefficients */ +#define chunk a5 /* maximum bytes per iteration (VLEN/8) */ +#define nbytes a6 /* bytes in this iteration */ +#define room a7 /* len - ctr */ +#define nnibbles t0 /* 2 * nbytes */ +#define accepted t1 /* nibbles accepted in this iteration */ +#define c52 t2 +#define c5 t3 + +#define bytes v1 +#define nibbles v2 /* register group of 2 */ +#define hi v4 /* register group of 2 */ +#define accept_mask v6 +#define quot v12 /* register group of 2 */ +#define compressed v14 /* register group of 2 */ +#define wide v16 /* register group of 8 */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(rej_uniform_eta2_riscv64_asm) +MLD_ASM_FN_SYMBOL(rej_uniform_eta2_riscv64_asm) + + li ctr, 0 + li c52, 52 + li c5, 5 + vsetvli chunk, zero, e16, m2, ta, ma + +mld_rej_uniform_eta2_riscv64_loop: + bgeu ctr, len, mld_rej_uniform_eta2_riscv64_done + beqz remaining, mld_rej_uniform_eta2_riscv64_done + + /* nbytes = min(chunk, remaining) */ + mv nbytes, chunk + bgeu remaining, chunk, mld_rej_uniform_eta2_riscv64_load + mv nbytes, remaining +mld_rej_uniform_eta2_riscv64_load: + vsetvli zero, nbytes, e16, m2, ta, ma + vle8.v bytes, (buf) + add buf, buf, nbytes + sub remaining, remaining, nbytes + + /* 16-bit lane i = (byte_i >> 4) << 8 | (byte_i & 15) */ + vzext.vf2 nibbles, bytes + vsrl.vi hi, nibbles, 4 + vsll.vi hi, hi, 8 + vand.vi nibbles, nibbles, 15 + vor.vv nibbles, nibbles, hi + + slli nnibbles, nbytes, 1 + vsetvli zero, nnibbles, e8, m2, ta, ma + /* r = 2 - (t mod 5), with t mod 5 = t - 5 * ((t * 52) >> 8) for + * t < 15 */ + vmsleu.vi accept_mask, nibbles, 14 + vmulhu.vx quot, nibbles, c52 + vmul.vx quot, quot, c5 + vsub.vv nibbles, nibbles, quot + vrsub.vi nibbles, nibbles, 2 + vcpop.m accepted, accept_mask + vcompress.vm compressed, nibbles, accept_mask + + /* accepted = min(accepted, len - ctr) */ + sub room, len, ctr + bgeu room, accepted, mld_rej_uniform_eta2_riscv64_store + mv accepted, room +mld_rej_uniform_eta2_riscv64_store: + vsetvli zero, accepted, e32, m8, ta, ma + vsext.vf4 wide, compressed + vse32.v wide, (out) + add ctr, ctr, accepted + slli accepted, accepted, 2 // in bytes + add out, out, accepted + j mld_rej_uniform_eta2_riscv64_loop + +mld_rej_uniform_eta2_riscv64_done: + mv a0, ctr // return value + ret +.option pop + +MLD_ASM_FN_SIZE(rej_uniform_eta2_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef out +#undef len +#undef buf +#undef remaining +#undef ctr +#undef chunk +#undef nbytes +#undef room +#undef nnibbles +#undef accepted +#undef c52 +#undef c5 +#undef bytes +#undef nibbles +#undef hi +#undef accept_mask +#undef quot +#undef compressed +#undef wide + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_KEYPAIR_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLDSA_ETA == 2) && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S b/dev/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S new file mode 100644 index 0000000000..dbb2dc6508 --- /dev/null +++ b/dev/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S @@ -0,0 +1,158 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_eta4_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers in [-4, +4] + Signature: uint64_t mld_rej_uniform_eta4_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer + test_with: 272 # 2 * SHAKE256_RATE +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_KEYPAIR_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Splits each input byte into its low and high nibble (in this order), + * accepts nibbles t < 9 and maps them to 4 - t. + * + * Vector-length agnostic: works for any VLEN of at least 64. + * + * Each iteration loads up to VLEN/8 bytes and zero-extends them to 16-bit + * lanes (LMUL=2). Moving the high nibble to the upper byte of each lane + * interleaves the nibbles as 8-bit lanes, which are mapped, compressed and + * sign-extended to 32 bits (LMUL=8) for the store. The number of bytes per + * iteration is min(VLEN/8, remaining), which vsetvli returns unchanged. + */ + +#define out a0 /* output pointer (advancing) */ +#define len a1 +#define buf a2 /* input pointer (advancing) */ +#define remaining a3 /* remaining input bytes */ +#define ctr a4 /* number of sampled coefficients */ +#define chunk a5 /* maximum bytes per iteration (VLEN/8) */ +#define nbytes a6 /* bytes in this iteration */ +#define room a7 /* len - ctr */ +#define nnibbles t0 /* 2 * nbytes */ +#define accepted t1 /* nibbles accepted in this iteration */ + +#define bytes v1 +#define nibbles v2 /* register group of 2 */ +#define hi v4 /* register group of 2 */ +#define accept_mask v6 +#define compressed v14 /* register group of 2 */ +#define wide v16 /* register group of 8 */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(rej_uniform_eta4_riscv64_asm) +MLD_ASM_FN_SYMBOL(rej_uniform_eta4_riscv64_asm) + + li ctr, 0 + vsetvli chunk, zero, e16, m2, ta, ma + +mld_rej_uniform_eta4_riscv64_loop: + bgeu ctr, len, mld_rej_uniform_eta4_riscv64_done + beqz remaining, mld_rej_uniform_eta4_riscv64_done + + /* nbytes = min(chunk, remaining) */ + mv nbytes, chunk + bgeu remaining, chunk, mld_rej_uniform_eta4_riscv64_load + mv nbytes, remaining +mld_rej_uniform_eta4_riscv64_load: + vsetvli zero, nbytes, e16, m2, ta, ma + vle8.v bytes, (buf) + add buf, buf, nbytes + sub remaining, remaining, nbytes + + /* 16-bit lane i = (byte_i >> 4) << 8 | (byte_i & 15) */ + vzext.vf2 nibbles, bytes + vsrl.vi hi, nibbles, 4 + vsll.vi hi, hi, 8 + vand.vi nibbles, nibbles, 15 + vor.vv nibbles, nibbles, hi + + slli nnibbles, nbytes, 1 + vsetvli zero, nnibbles, e8, m2, ta, ma + /* r = 4 - t */ + vmsleu.vi accept_mask, nibbles, 8 + vrsub.vi nibbles, nibbles, 4 + vcpop.m accepted, accept_mask + vcompress.vm compressed, nibbles, accept_mask + + /* accepted = min(accepted, len - ctr) */ + sub room, len, ctr + bgeu room, accepted, mld_rej_uniform_eta4_riscv64_store + mv accepted, room +mld_rej_uniform_eta4_riscv64_store: + vsetvli zero, accepted, e32, m8, ta, ma + vsext.vf4 wide, compressed + vse32.v wide, (out) + add ctr, ctr, accepted + slli accepted, accepted, 2 // in bytes + add out, out, accepted + j mld_rej_uniform_eta4_riscv64_loop + +mld_rej_uniform_eta4_riscv64_done: + mv a0, ctr // return value + ret +.option pop + +MLD_ASM_FN_SIZE(rej_uniform_eta4_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef out +#undef len +#undef buf +#undef remaining +#undef ctr +#undef chunk +#undef nbytes +#undef room +#undef nnibbles +#undef accepted +#undef bytes +#undef nibbles +#undef hi +#undef accept_mask +#undef compressed +#undef wide + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_KEYPAIR_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLDSA_ETA == 4) && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/mldsa_rej_uniform_riscv64_asm.S b/dev/riscv64/src/mldsa_rej_uniform_riscv64_asm.S new file mode 100644 index 0000000000..faa300e7a1 --- /dev/null +++ b/dev/riscv64/src/mldsa_rej_uniform_riscv64_asm.S @@ -0,0 +1,171 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q + Signature: uint64_t mld_rej_uniform_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer (must be a multiple of 3) + test_with: 840 # MLD_POLY_UNIFORM_NBLOCKS * SHAKE128_RATE = 5 * 168 +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) +/* simpasm: header-end */ + +/* + * Vector-length agnostic: works for any VLEN. + * + * Each iteration unpacks VL 24-bit values from 3*VL bytes of input (see + * rv64v_unpack.inc), where VL = VLMAX for e32,m2 in the main loop, keeps + * the low 23 bits, and accepts those below q. Once fewer than 3*VL bytes + * remain, a single final iteration processes the rest with VL equal to + * the number of remaining values. That VL is below VLMAX, so vsetvli + * returns it unchanged. + */ + +#define out a0 /* output pointer (advancing) */ +#define len a1 +#define buf a2 /* input pointer (advancing) */ +#define remaining a3 /* remaining input bytes */ +#define ctr a4 /* number of sampled coefficients */ +#define nbytes a5 /* bytes per iteration (3*VL) */ +#define vl4 a6 /* 4*VL */ +#define room a7 /* len - ctr */ +#define vl t0 /* values per iteration */ +#define accepted t1 /* values accepted in this iteration */ +#define modulus t2 +#define mask t3 +#define tmp t4 +#define two t5 + +#define accept_mask v0 +#define gidx v8 /* register group of 4 */ +#define gtmp v12 /* register group of 4 */ +#define src v16 /* register group of 2 */ +#define coeffs v20 /* register group of 2 */ +#define compressed v24 /* register group of 2 */ + +#include "rv64v_unpack.inc" + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(rej_uniform_riscv64_asm) +MLD_ASM_FN_SYMBOL(rej_uniform_riscv64_asm) + + li ctr, 0 + li modulus, MLDSA_Q + li two, 2 + + /* Set up for VL = VLMAX. The indices are prefix-consistent, so + * also valid for smaller VL. */ + vsetvli vl, zero, e32, m2, ta, ma + unpack_setup 24 + /* Keep 23 bits */ + li mask, 0x7FFFFF + + /* nbytes = 3*VL */ + slli nbytes, vl, 1 + add nbytes, nbytes, vl + +mld_rej_uniform_riscv64_loop: + bgeu ctr, len, mld_rej_uniform_riscv64_done + bltu remaining, nbytes, mld_rej_uniform_riscv64_tail + +mld_rej_uniform_riscv64_body: + unpack 24 + add buf, buf, nbytes + sub remaining, remaining, nbytes + + vmsltu.vx accept_mask, coeffs, modulus + vcpop.m accepted, accept_mask + vcompress.vm compressed, coeffs, accept_mask + + /* accepted = min(accepted, len - ctr) */ + sub room, len, ctr + bgeu room, accepted, mld_rej_uniform_riscv64_store + mv accepted, room +mld_rej_uniform_riscv64_store: + vsetvli zero, accepted, e32, m2, ta, ma + vse32.v compressed, (out) + add ctr, ctr, accepted + slli accepted, accepted, 2 // in bytes + add out, out, accepted + j mld_rej_uniform_riscv64_loop + +mld_rej_uniform_riscv64_tail: + /* Fewer than 3*VL bytes left. Process all remaining bytes in one + * final iteration with VL = remaining/3. Afterwards remaining = 0, + * so the next pass through the loop exits via the beqz below. */ + beqz remaining, mld_rej_uniform_riscv64_done + mv nbytes, remaining + /* x/3 = (x * 0xAAAB) >> 17, exact for multiples x of 3 below 2^17 */ + li tmp, 0xAAAB + mul vl, remaining, tmp + srli vl, vl, 17 + slli vl4, vl, 2 + j mld_rej_uniform_riscv64_body + +mld_rej_uniform_riscv64_done: + mv a0, ctr // return value + ret +.option pop + +MLD_ASM_FN_SIZE(rej_uniform_riscv64_asm) + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef out +#undef len +#undef buf +#undef remaining +#undef ctr +#undef nbytes +#undef vl4 +#undef room +#undef vl +#undef accepted +#undef modulus +#undef mask +#undef tmp +#undef two +#undef accept_mask +#undef gidx +#undef gtmp +#undef src +#undef coeffs +#undef compressed + +/* simpasm: footer-start */ +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/dev/riscv64/src/rv64v_decompose.inc b/dev/riscv64/src/rv64v_decompose.inc new file mode 100644 index 0000000000..b0da59173c --- /dev/null +++ b/dev/riscv64/src/rv64v_decompose.inc @@ -0,0 +1,62 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * Decompose, shared by poly_decompose_{32,88} and poly_use_hint_{32,88}. + * Mirrors mld_decompose() in the C reference, which documents the + * approximations used. For 0 <= a < q, computes a1, a0 with + * a mod q = a1 * 2 * GAMMA2 + a0: + * + * a1 = (a + 127) >> 7 + * GAMMA2 = (q-1)/32: a1 = ((a1 * 1025 + 2^21) >> 22) & 15 + * GAMMA2 = (q-1)/88: a1 = (a1 * 11275 + 2^23) >> 24; a1 = 0 if a1 > 43 + * a0 = a - a1 * 2 * GAMMA2 + * a0 = a0 - q if a0 > (q-1)/2 + * + * Uses the scalar registers c127, dec_mul, dec_round, two_gamma2, + * half_q, modulus and (for GAMMA2 = (q-1)/88) c43, which the including + * file must define. + */ + +/* Load the scalar constants for \gamma2_div = 32 or 88. */ +.macro decompose_setup gamma2_div + li modulus, MLDSA_Q + li c127, 127 + li half_q, ((MLDSA_Q - 1) / 2) + li two_gamma2, (2 * ((MLDSA_Q - 1) / \gamma2_div)) +.if \gamma2_div == 32 + li dec_mul, 1025 + li dec_round, (1 << 21) +.else + li dec_mul, 11275 + li dec_round, (1 << 23) + li c43, 43 +.endif +.endm + +/* \a1, \a0 = decompose(\a0), at the current LMUL. \t is a temporary. */ +.macro decompose gamma2_div, a1, a0, t + vadd.vx \a1, \a0, c127 + vsra.vi \a1, \a1, 7 + vmul.vx \a1, \a1, dec_mul + vadd.vx \a1, \a1, dec_round +.if \gamma2_div == 32 + vsra.vi \a1, \a1, 22 + vand.vi \a1, \a1, 15 +.else + vsra.vi \a1, \a1, 24 + /* a1 ^= ((43 - a1) >> 31) & a1, that is, a1 = 0 if a1 > 43 */ + vrsub.vx \t, \a1, c43 + vsra.vi \t, \t, 31 + vand.vv \t, \t, \a1 + vxor.vv \a1, \a1, \t +.endif + vnmsac.vx \a0, two_gamma2, \a1 + /* a0 -= ((half_q - a0) >> 31) & q */ + vrsub.vx \t, \a0, half_q + vsra.vi \t, \t, 31 + vand.vx \t, \t, modulus + vsub.vv \a0, \a0, \t +.endm diff --git a/dev/riscv64/src/rv64v_montgomery.inc b/dev/riscv64/src/rv64v_montgomery.inc new file mode 100644 index 0000000000..ea3de18c59 --- /dev/null +++ b/dev/riscv64/src/rv64v_montgomery.inc @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * Signed Montgomery reduction at SEW=32, shared by the pointwise + * multiplications. + * + * For a 64-bit value p = phi * 2^32 + plo with |p| < 2^31 * q, and + * QINV = q^-1 mod 2^32: + * + * m = low(plo * QINV), interpreted as signed + * r = phi - high(m * q) = (p - m * q) / 2^32 + * + * The low words of p and m * q agree, so p - m * q is divisible by 2^32 and + * the high-word difference computes the quotient without a borrow. Hence + * r == p * 2^-32 mod q and |r| < |p| / 2^32 + q / 2 < q. This is the + * computation of mld_montgomery_reduce() in the C reference. + * + * Uses the scalar registers modulus and qinv, which the including file + * must define. + */ + +/* Load the scalar constants. */ +.macro montgomery_setup + li modulus, MLDSA_Q + li qinv, 58728449 // q^-1 mod 2^32 +.endm + +/* \r = Montgomery reduction of \lo, \hi (low and high halves of p), at the + * current LMUL. Clobbers \lo. \r may alias \hi. */ +.macro montgomery_reduce r, lo, hi + vmul.vx \lo, \lo, qinv + vmulh.vx \lo, \lo, modulus + vsub.vv \r, \hi, \lo +.endm diff --git a/dev/riscv64/src/rv64v_ntt.inc b/dev/riscv64/src/rv64v_ntt.inc new file mode 100644 index 0000000000..17774c77e4 --- /dev/null +++ b/dev/riscv64/src/rv64v_ntt.inc @@ -0,0 +1,404 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * Shared definitions for the RISC-V forward and inverse NTT, for VL = 4 + * and VL = 8 lanes per register (VLEN = 128 and VLEN = 256). + * + * # Structure + * + * The polynomial does not fit into the 16 registers v16-v31, so the NTT + * makes two passes over memory, each in batches of 16 registers. Write + * the coefficient index as j = 16 * b + r with b, r = 0..15. + * + * - Pass A (layers 1-4) combines coefficients at distance 128, 64, 32 + * and 16, that is, it operates on b. Register v(16+b) holds coefficients + * 16 * b + o to 16 * b + o + VL - 1, for the batch offset o = 0, VL, ... + * All butterflies within a group of registers share their twiddle, so the + * layers operate on register groups of 8, 4, 2 and 1 (LMUL=8, 4, 2, 1) + * with scalar twiddles. + * - Pass B (layers 5-8) combines coefficients at distance 8, 4, 2 and 1, + * that is, it operates on r. Register v(16+r) holds coefficients + * 16 * b + r for VL consecutive blocks b (a strided load), so the layers + * combine registers at distance 8, 4, 2 and 1. The twiddles depend on b, + * hence differ between lanes. They are replicated across the registers of + * a group, so that layers 5-8 operate on register groups of 4, 4, 2 and 1. + * + * # Barrett multiplication + * + * Twiddles and scaling constants are known in advance, so multiplications + * by them use Barrett multiplication @[NeonNTT]: for a constant c with + * twisted constant c' = round(c * 2^31 / q), + * + * x * c mod q ~ x * c - q * round(x * c' / 2^31) + * + * computed with vmul (low half of x * c), vsmul (rounding high half of + * 2 * x * c', with vxrm = rnu) and vnmsac. Since |c' - c * 2^31 / q| <= 1/2, + * the result is congruent to x * c mod q and bounded by + * q * (|x| / 2^32 + 1/2) in absolute value, hence by q for |x| < 2^31. + * The low 32 bits computed by vmul and vnmsac are exact, as the result fits + * into 32 bits. vsmul cannot saturate, as |c'| <= 2^30. + * + * # Tables + * + * The twiddle tables are generated by scripts/autogen. Each starts with + * the layer 1-4 twiddles in order of use, each followed by its twisted + * constant (16 pairs); see load_scalar. Then follow the layer 5-8 twiddles + * in groups of 16 twiddles, one per block b, followed by their 16 twisted + * constants; see load_lane_twiddles. + */ + +#define data a0 +#define zetas a1 +#define vl_m1 a2 +#define vl_m2 a3 +#define vl_m4 a4 +#define vl_m8 a5 +#define ptr a6 /* data pointer of the current batch */ +#define lane_ptr a7 /* layer 5-8 twiddles of the current batch */ +#define modulus t0 +#define cst t1 /* scalar twiddle or constant */ +#define cst_twisted t2 /* its twisted constant */ +#define tmp_addr t3 +#define stride t4 /* 64: bytes between coefficients j and j + 16 */ +#define batch t5 /* remaining batches */ + +/* Temporaries for pass B (register groups of up to 4) */ +#define tmp_t v0 +#define tmp_q v4 +#define twiddle v8 +#define twiddle_twisted v12 + +.macro vsetvl_m1 + vsetvli zero, vl_m1, e32, m1, ta, ma +.endm + +.macro vsetvl_m2 + vsetvli zero, vl_m2, e32, m2, ta, ma +.endm + +.macro vsetvl_m4 + vsetvli zero, vl_m4, e32, m4, ta, ma +.endm + +.macro vsetvl_m8 + vsetvli zero, vl_m8, e32, m8, ta, ma +.endm + +/* Set up the rounding mode and constants. \vl is the number of 32-bit + * lanes per register. */ +.macro ntt_setup vl + csrwi vxrm, 0 // round-to-nearest-up + li modulus, MLDSA_Q + li stride, 64 + li vl_m1, \vl + li vl_m2, (2 * \vl) + li vl_m4, (4 * \vl) + li vl_m8, (8 * \vl) +.endm + +/* Load the layer 1-4 twiddle with index \idx and its twisted constant. */ +.macro load_scalar idx + lw cst, (8 * \idx)(zetas) + lw cst_twisted, (8 * \idx + 4)(zetas) +.endm + +/* Load the layer 5-8 twiddles of group \grp for the blocks of the current + * batch, and their twisted constants, replicated to register groups of + * \lmul = 1, 2 or 4. Leaves LMUL=1. */ +.macro load_lane_twiddles grp, lmul + vsetvl_m1 + addi tmp_addr, lane_ptr, (128 * \grp) + vle32.v v8, (tmp_addr) + addi tmp_addr, tmp_addr, 64 + vle32.v v12, (tmp_addr) +.if \lmul >= 2 + vmv1r.v v9, v8 + vmv1r.v v13, v12 +.endif +.if \lmul >= 4 + vmv2r.v v10, v8 + vmv2r.v v14, v12 +.endif +.endm + +/* Barrett multiplication of \x by cst, in place, at the current LMUL. + * \t is a temporary of the same register group size. */ +.macro barrett_mul_scalar x, t + vsmul.vx \t, \x, cst_twisted + vmul.vx \x, \x, cst + vnmsac.vx \x, modulus, \t +.endm + +/* Cooley-Tukey butterfly with the scalar twiddle cst, at the current LMUL: + * t = hi * cst mod q; hi = lo - t; lo = lo + t + * \t is a temporary of the same register group size. \hi holds the + * quotient estimate in between, which saves a temporary. */ +.macro ct_bfly_scalar lo, hi, t + vmul.vx \t, \hi, cst + vsmul.vx \hi, \hi, cst_twisted + vnmsac.vx \t, modulus, \hi + vsub.vv \hi, \lo, \t + vadd.vv \lo, \lo, \t +.endm + +/* Cooley-Tukey butterfly with per-lane twiddles, at the current LMUL. */ +.macro ct_bfly_lanes lo, hi + vmul.vv tmp_t, \hi, twiddle + vsmul.vv tmp_q, \hi, twiddle_twisted + vnmsac.vx tmp_t, modulus, tmp_q + vsub.vv \hi, \lo, tmp_t + vadd.vv \lo, \lo, tmp_t +.endm + +/* Gentleman-Sande butterfly with the scalar twiddle cst, at the current + * LMUL: + * t = lo - hi; lo = lo + hi; hi = t * cst mod q + * \t is a temporary of the same register group size. */ +.macro gs_bfly_scalar lo, hi, t + vsub.vv \t, \lo, \hi + vadd.vv \lo, \lo, \hi + vmul.vx \hi, \t, cst + vsmul.vx \t, \t, cst_twisted + vnmsac.vx \hi, modulus, \t +.endm + +/* Gentleman-Sande butterfly with per-lane twiddles, at the current LMUL. */ +.macro gs_bfly_lanes lo, hi + vsub.vv tmp_t, \lo, \hi + vadd.vv \lo, \lo, \hi + vmul.vv \hi, tmp_t, twiddle + vsmul.vv tmp_t, tmp_t, twiddle_twisted + vnmsac.vx \hi, modulus, tmp_t +.endm + +/* Pass A: load or store v16-v31 from/to coefficients 16 * b + (0..VL-1) + * relative to ptr, b = 0..15. Requires LMUL=1. */ +.macro access_pass_a op + mv tmp_addr, ptr + .irp m, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + \op v\m, (tmp_addr) + addi tmp_addr, tmp_addr, 64 + .endr +.endm + +/* Pass B: load or store v16-v31 from/to coefficients 16 * (0..VL-1) + r + * relative to ptr, r = 0..15. Requires LMUL=1. */ +.macro access_pass_b op + mv tmp_addr, ptr + .irp m, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + \op v\m, (tmp_addr), stride + addi tmp_addr, tmp_addr, 4 + .endr +.endm + +/* Forward NTT layers 1-4 on v16-v31. + * + * Bounds: the input is bounded by q in absolute value. Each layer adds at + * most q (the bound of the Barrett product), so after layer L, all + * coefficients are bounded by (L+1)*q. */ +.macro ntt_layers1234 + /* Layer 1: < 2q */ + vsetvl_m8 + load_scalar 0 + ct_bfly_scalar v16, v24, v0 + + /* Layer 2: < 3q */ + vsetvl_m4 + load_scalar 1 + ct_bfly_scalar v16, v20, v0 + load_scalar 2 + ct_bfly_scalar v24, v28, v0 + + /* Layer 3: < 4q */ + vsetvl_m2 + load_scalar 3 + ct_bfly_scalar v16, v18, v0 + load_scalar 4 + ct_bfly_scalar v20, v22, v0 + load_scalar 5 + ct_bfly_scalar v24, v26, v0 + load_scalar 6 + ct_bfly_scalar v28, v30, v0 + + /* Layer 4: < 5q */ + vsetvl_m1 + load_scalar 7 + ct_bfly_scalar v16, v17, v0 + load_scalar 8 + ct_bfly_scalar v18, v19, v0 + load_scalar 9 + ct_bfly_scalar v20, v21, v0 + load_scalar 10 + ct_bfly_scalar v22, v23, v0 + load_scalar 11 + ct_bfly_scalar v24, v25, v0 + load_scalar 12 + ct_bfly_scalar v26, v27, v0 + load_scalar 13 + ct_bfly_scalar v28, v29, v0 + load_scalar 14 + ct_bfly_scalar v30, v31, v0 +.endm + +/* Forward NTT layers 5-8 on v16-v31, on register groups of 4, 4, 2 and 1. + * + * Bounds: continuing from layers 1-4, after layer L, all coefficients are + * bounded by (L+1)*q, so the output is bounded by 9q. */ +.macro ntt_layers5678 + /* Layer 5: < 6q */ + load_lane_twiddles 0, 4 + vsetvl_m4 + ct_bfly_lanes v16, v24 + ct_bfly_lanes v20, v28 + + /* Layer 6: < 7q */ + load_lane_twiddles 1, 4 + vsetvl_m4 + ct_bfly_lanes v16, v20 + load_lane_twiddles 2, 4 + vsetvl_m4 + ct_bfly_lanes v24, v28 + + /* Layer 7: < 8q */ + load_lane_twiddles 3, 2 + vsetvl_m2 + ct_bfly_lanes v16, v18 + load_lane_twiddles 4, 2 + vsetvl_m2 + ct_bfly_lanes v20, v22 + load_lane_twiddles 5, 2 + vsetvl_m2 + ct_bfly_lanes v24, v26 + load_lane_twiddles 6, 2 + vsetvl_m2 + ct_bfly_lanes v28, v30 + + /* Layer 8: < 9q */ + load_lane_twiddles 7, 1 + ct_bfly_lanes v16, v17 + load_lane_twiddles 8, 1 + ct_bfly_lanes v18, v19 + load_lane_twiddles 9, 1 + ct_bfly_lanes v20, v21 + load_lane_twiddles 10, 1 + ct_bfly_lanes v22, v23 + load_lane_twiddles 11, 1 + ct_bfly_lanes v24, v25 + load_lane_twiddles 12, 1 + ct_bfly_lanes v26, v27 + load_lane_twiddles 13, 1 + ct_bfly_lanes v28, v29 + load_lane_twiddles 14, 1 + ct_bfly_lanes v30, v31 +.endm + +/* Inverse NTT layers 8-5 on v16-v31, on register groups of 1, 2, 4 and 4. + * Leaves LMUL=4. + * + * Bounds: the input is bounded by q in absolute value. A Gentleman-Sande + * butterfly at most doubles the bound of the sum and reduces the difference + * to below q, so after layer L, all coefficients are bounded by + * 2^(9-L) * q. */ +.macro intt_layers8765 + /* Layer 8: < 2q */ + load_lane_twiddles 0, 1 + gs_bfly_lanes v16, v17 + load_lane_twiddles 1, 1 + gs_bfly_lanes v18, v19 + load_lane_twiddles 2, 1 + gs_bfly_lanes v20, v21 + load_lane_twiddles 3, 1 + gs_bfly_lanes v22, v23 + load_lane_twiddles 4, 1 + gs_bfly_lanes v24, v25 + load_lane_twiddles 5, 1 + gs_bfly_lanes v26, v27 + load_lane_twiddles 6, 1 + gs_bfly_lanes v28, v29 + load_lane_twiddles 7, 1 + gs_bfly_lanes v30, v31 + + /* Layer 7: < 4q */ + load_lane_twiddles 8, 2 + vsetvl_m2 + gs_bfly_lanes v16, v18 + load_lane_twiddles 9, 2 + vsetvl_m2 + gs_bfly_lanes v20, v22 + load_lane_twiddles 10, 2 + vsetvl_m2 + gs_bfly_lanes v24, v26 + load_lane_twiddles 11, 2 + vsetvl_m2 + gs_bfly_lanes v28, v30 + + /* Layer 6: < 8q */ + load_lane_twiddles 12, 4 + vsetvl_m4 + gs_bfly_lanes v16, v20 + load_lane_twiddles 13, 4 + vsetvl_m4 + gs_bfly_lanes v24, v28 + + /* Layer 5: < 16q */ + load_lane_twiddles 14, 4 + vsetvl_m4 + gs_bfly_lanes v16, v24 + gs_bfly_lanes v20, v28 +.endm + +/* Inverse NTT layers 4-1 on v16-v31, followed by the scaling by + * 2^32 / 256 mod q. + * + * Bounds: continuing from layers 8-5, after layer L, all coefficients are + * bounded by 2^(9-L) * q, hence by 256q < 2^31 after layer 1. Layer 1 + * includes the scaling in its twiddle, and the sums are scaled + * separately, so the output is bounded by q. */ +.macro intt_layers4321 + /* Layer 4: < 32q */ + vsetvl_m1 + load_scalar 0 + gs_bfly_scalar v16, v17, v0 + load_scalar 1 + gs_bfly_scalar v18, v19, v0 + load_scalar 2 + gs_bfly_scalar v20, v21, v0 + load_scalar 3 + gs_bfly_scalar v22, v23, v0 + load_scalar 4 + gs_bfly_scalar v24, v25, v0 + load_scalar 5 + gs_bfly_scalar v26, v27, v0 + load_scalar 6 + gs_bfly_scalar v28, v29, v0 + load_scalar 7 + gs_bfly_scalar v30, v31, v0 + + /* Layer 3: < 64q */ + vsetvl_m2 + load_scalar 8 + gs_bfly_scalar v16, v18, v0 + load_scalar 9 + gs_bfly_scalar v20, v22, v0 + load_scalar 10 + gs_bfly_scalar v24, v26, v0 + load_scalar 11 + gs_bfly_scalar v28, v30, v0 + + /* Layer 2: < 128q */ + vsetvl_m4 + load_scalar 12 + gs_bfly_scalar v16, v20, v0 + load_scalar 13 + gs_bfly_scalar v24, v28, v0 + + /* Layer 1 and scaling: < q */ + vsetvl_m8 + load_scalar 14 + gs_bfly_scalar v16, v24, v0 + load_scalar 15 + barrett_mul_scalar v16, v0 +.endm diff --git a/dev/riscv64/src/rv64v_pointwise_acc.inc b/dev/riscv64/src/rv64v_pointwise_acc.inc new file mode 100644 index 0000000000..399d3d871d --- /dev/null +++ b/dev/riscv64/src/rv64v_pointwise_acc.inc @@ -0,0 +1,80 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * Shared body of polyvecl_pointwise_acc_montgomery_l{4,5,7}: computes + * + * w[i] = (sum_l u[l][i] * v[l][i]) * 2^-32 mod q + * + * accumulating the products in 64 bits and reducing once, as the C + * reference does; see rv64v_montgomery.inc. + * + * Vector-length agnostic: works for any VLEN. The 32-bit inputs use + * register groups of 4 (LMUL=4), the 64-bit accumulator groups of 8. VLMAX + * for e32,m4 is a power of 2, so it either divides 256 or exceeds it. + * Hence a single vsetvli with AVL = 256 yields a VL that is used unchanged + * for every iteration. + * + * Bounds: 0 <= u < q and |v| < MLD_NTT_BOUND, so for L <= 7 the sum is + * bounded by 7 * q * MLD_NTT_BOUND < 2^31 * q in absolute value, and the + * result by q. + */ + +#define w_ptr a0 +#define u_ptr a1 +#define v_ptr a2 +#define count a3 +#define u_cur a4 +#define v_cur a5 +#define poly_bytes a6 +#define vl t0 +#define modulus t1 +#define qinv t2 +#define stride t3 +#define shift32 t4 + +#define u_vec v0 /* register group of 4 */ +#define v_vec v4 /* register group of 4 */ +#define acc v8 /* register group of 8 (64-bit lanes) */ +#define acc_lo v16 /* register group of 4 */ +#define acc_hi v20 /* register group of 4 */ + +#include "rv64v_montgomery.inc" + +/* \len: number of polynomials L. \loop: loop label. */ +.macro pointwise_acc_montgomery len, loop + montgomery_setup + li count, MLDSA_N + li shift32, 32 + li poly_bytes, (4 * MLDSA_N) + + vsetvli vl, count, e32, m4, ta, ma + slli stride, vl, 2 + +\loop: + vle32.v u_vec, (u_ptr) + vle32.v v_vec, (v_ptr) + vwmul.vv acc, u_vec, v_vec + mv u_cur, u_ptr + mv v_cur, v_ptr + .rept \len - 1 + add u_cur, u_cur, poly_bytes + add v_cur, v_cur, poly_bytes + vle32.v u_vec, (u_cur) + vle32.v v_vec, (v_cur) + vwmacc.vv acc, u_vec, v_vec + .endr + + vnsrl.wi acc_lo, acc, 0 + vnsra.wx acc_hi, acc, shift32 + montgomery_reduce acc_lo, acc_lo, acc_hi + vse32.v acc_lo, (w_ptr) + + add w_ptr, w_ptr, stride + add u_ptr, u_ptr, stride + add v_ptr, v_ptr, stride + sub count, count, vl + bnez count, \loop +.endm diff --git a/dev/riscv64/src/rv64v_unpack.inc b/dev/riscv64/src/rv64v_unpack.inc new file mode 100644 index 0000000000..cbda1b8eb4 --- /dev/null +++ b/dev/riscv64/src/rv64v_unpack.inc @@ -0,0 +1,61 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * Unpacking of bit-packed little-endian values of 18, 20 or 24 bits into + * 32-bit lanes, shared by polyz_unpack_{17,19} and rej_uniform. + * + * Lane i holds bits [b*i, b*i + b) of the input, where b is the bit width. + * These lie within the three bytes starting at off_i = (b*i) >> 3, at + * shift s_i = (b*i) & 7, since s_i + b <= 24 for the supported widths. + * The bytes are gathered into the lanes with vrgatherei16: byte k of lane i + * is taken from off_i + min(k, 2). The fourth byte only duplicates the + * third and is masked away, so the gather never reads beyond the + * b*VL/8 bytes loaded. Each lane is then shifted right by s_i and masked + * to b bits. + * + * The 32-bit lanes use register groups of 2 (LMUL=2), the bytes groups of + * 2 at SEW=8, and the 16-bit gather indices groups of 4. + * + * Uses the scalar registers buf, vl, vl4 (= 4 * vl), nbytes, mask and + * tmp, and the vector registers gidx, gtmp, shifts, src and coeffs, which + * the including file must define. The indices and shifts depend only on + * the lane, so the setup for VL = VLMAX also serves any smaller VL. + */ + +/* Set up the gather indices and shifts for \bits-bit values and VL = vl. */ +.macro unpack_setup bits + slli vl4, vl, 2 + li tmp, \bits + vsetvli zero, vl4, e16, m4, ta, ma + vid.v gidx + vsrl.vi gtmp, gidx, 2 + vmul.vx gtmp, gtmp, tmp + vsrl.vi gtmp, gtmp, 3 + vand.vi gidx, gidx, 3 + vminu.vx gidx, gidx, two + vadd.vv gidx, gidx, gtmp +.if \bits != 24 + vsetvli zero, vl, e32, m2, ta, ma + vid.v shifts + vmul.vx shifts, shifts, tmp + vand.vi shifts, shifts, 7 +.endif + li mask, ((1 << \bits) - 1) +.endm + +/* Unpack vl \bits-bit values from the nbytes = \bits*vl/8 bytes at buf into + * coeffs. Leaves VL = vl at e32,m2. */ +.macro unpack bits + vsetvli zero, nbytes, e8, m2, ta, ma + vle8.v src, (buf) + vsetvli zero, vl4, e8, m2, ta, ma + vrgatherei16.vv coeffs, src, gidx + vsetvli zero, vl, e32, m2, ta, ma +.if \bits != 24 + vsrl.vv coeffs, coeffs, shifts +.endif + vand.vx coeffs, coeffs, mask +.endm diff --git a/dev/riscv64/src/rv64v_zetas.c b/dev/riscv64/src/rv64v_zetas.c new file mode 100644 index 0000000000..4803582c94 --- /dev/null +++ b/dev/riscv64/src/rv64v_zetas.c @@ -0,0 +1,245 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_riscv64.h" + +/* + * Table of twiddles used in the RISC-V forward NTT + * See autogen for details. + */ +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t + mld_rv64v_ntt_zetas[512] = { + -3572223, -915382907, 3765607, 964937599, 3761513, + 963888510, -3201494, -820383522, -2883726, -738955404, + -3145678, -806080660, -3201430, -820367122, -601683, + -154181397, 3542485, 907762539, 2682288, 687336873, + 2129892, 545785280, 3764867, 964747974, -1005239, + -257592709, 557458, 142848732, -1221177, -312926867, + 0, 0, -3370349, -4063053, 2663378, + -1674615, -3524442, -434125, 676590, -1335936, + -3227876, 1714295, 2453983, 1460718, -642628, + -3585098, 2815639, 2283733, -863652652, -1041158200, + 682491182, -429120452, -903139016, -111244624, 173376332, + -342333886, -827143915, 439288460, 628833668, 374309300, + -164673562, -918682129, 721508096, 585207070, 3602218, + 2740543, -3110818, 3704823, 394148, 1095468, + 2071829, 3241972, 3415069, -817536, 3756790, + -1716988, -2897314, 556856, 2917338, 3345963, + 923069133, 702264730, -797147778, 949361686, 101000509, + 280713909, 530906624, 830756018, 875112161, -209493775, + 962678241, -439978542, -742437332, 142694469, 747568486, + 857403734, 3182878, -3586446, 2101410, 1159875, + 928749, -3506380, -4018989, 2156050, 1759347, + -3574466, -1935799, -3950053, 3192354, 3870317, + 1853806, 1858416, 815613168, -919027554, 538486762, + 297218217, 237992130, -898510625, -1029866791, 552488273, + 450833045, -915957677, -496048908, -1012201926, 818041395, + 991769559, 475038184, 476219497, 3073009, 4183372, + 2508980, 2811291, 1528066, -565603, -4166425, + 1736313, -2579253, 3482206, -1317678, 3901472, + 1356448, -3467665, 348812, -3818627, 787459213, + 1071989969, 642926661, 720393920, 391567239, -144935890, + -1067647297, 444930577, -660934133, 892316032, -337655269, + 999753034, 347590090, -888589898, 89383150, -978523985, + 1277625, -3222807, 2028118, -2983781, 482649, + 169688, -3488383, 235407, 1787943, -4182915, + 2461387, -1226661, -2775755, 2312838, -327848, + -1922253, 327391679, -825844983, 519705671, -764594519, + 123678909, 43482586, -893898890, 60323094, 458160776, + -1071872863, 630730945, -314332144, -711287812, 592665232, + -84011120, -492577742, -2635473, -3121440, 1937570, + -1109516, 1148858, 2462444, 1987814, -3250154, + -2391089, -1300016, 3035980, 2925816, 2683270, + -653275, 1011223, -2236726, -675340520, -799869667, + 496502727, -284313712, 294395108, 631001801, 509377762, + -832852657, -612717067, -333129378, 777970524, 749740976, + 687588511, -167401858, 259126110, -573161516, 3852015, + -274060, -3815725, 4158088, -2962264, -3334383, + -3197248, 3258457, -2254727, -2362063, 621164, + 3374250, -2778788, -459163, -2354215, 1744507, + 987079667, -70227934, -977780347, 1065510939, -759080783, + -854436357, -819295484, 834980303, -577774276, -605279149, + 159173408, 864652284, -712065019, -117660617, -603268097, + 447030292, 1753, 1182243, -27812, -1685153, + -4146264, 3747250, 2998219, -458740, -1623354, + -1476985, -3033742, 3980599, 3994671, 3105558, + -2185084, -303005, 449207, 302950022, -7126831, + -431820817, -1062481036, 960233614, 768294260, -117552223, + -415984810, -378477722, -777397036, 1020029345, 1023635298, + 795799901, -559928242, -77645096, -1935420, 87208, + 822541, -3410568, -1772588, 2296099, 141835, + -1921994, 2105286, 1994046, -338420, 2569011, + -11879, -1103344, 3123762, 3531229, -495951789, + 22347069, 210776307, -873958779, -454226054, 588375860, + 36345249, -492511373, 539479988, 510974714, -86720197, + 658309618, -3043996, -282732136, 800464680, 904878186, + -2659525, 636927, 1009365, 2678278, 2192938, + 1239911, -89301, 4040196, -2374402, 2491325, + 2647994, -1615530, -1370517, 508145, 2358373, + -3974485, -681503850, 163212680, 258649997, 686309310, + 561940831, 317727459, -22883400, 1035301089, -608441020, + 638402564, 678549029, -413979908, -351195274, 130212265, + 604333585, -1018462631, -1455890, -3965306, -2454145, + -3768948, -1727088, -3838479, 2513018, -3472069, + -2033807, -1393159, 3009748, 1723229, 3020393, + -553718, -2193087, -3773731, -373072124, -1016110510, + -628875181, -965793731, -442566669, -983611064, 643961400, + -889718424, -521163479, -356997292, 771248568, 441577800, + 773976352, -141890356, -561979013, -967019376, 2660408, + -3956745, -1979497, -3551006, 2387513, 3195676, + -1354892, 2039144, 586241, 507927, -2612853, + 1665318, 3363542, 860144, -3014420, 1900052, + 681730119, -1013916752, -507246529, -909946047, 611800717, + 818892658, -347191365, 522531086, 150224382, 130156402, + -669544140, 426738094, 861908357, 220412084, -772445769, + 486888731, -1780227, -2296397, 1596822, 635956, + -3611750, 2642980, 613238, -1879878, -1179613, + -1187885, 4148469, 2028038, 214880, 3430436, + -1716814, -781875, -456183549, -588452222, 409185979, + 162963861, -925511710, 677264190, 157142369, -481719139, + -302276083, -304395785, 1063046068, 519685171, 55063046, + 879049958, -439933955, -200355636, -59148, -3284915, + -3956944, -250446, -268456, 1254190, -1310261, + -818761, 527981, -724804, 749577, 1163598, + 545376, 140244, 2926054, 1054478, -15156688, + -841760171, -1013967746, -64176841, -68791907, 321386456, + -335754661, -209807681, 135295244, -185731180, 192079267, + 298172236, 139752717, 35937555, 749801963, 270210213, + 2772600, -3716946, -3759465, -2455377, -3180456, + -12417, -2218467, -2178965, -2743411, -1834526, + -4022750, -3369273, -770441, -1514152, -392707, + -731434, 710479343, -952468207, -963363710, -629190881, + -814992530, -3181859, -568482643, -558360247, -702999655, + -470097680, -1030830548, -863376927, -197425671, -388001774, + -100631253, -187430119, +}; + +/* + * Table of twiddles used in the RISC-V inverse NTT + * See autogen for details. + */ +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t + mld_rv64v_intt_zetas[512] = { + 1221177, 312926867, -557458, -142848732, 1005239, + 257592709, -3764867, -964747974, -2129892, -545785280, + -2682288, -687336873, -3542485, -907762539, 601683, + 154181397, 3201430, 820367122, 3145678, 806080660, + 2883726, 738955404, 3201494, 820383522, -3761513, + -963888510, -3765607, -964937599, -294725, -75523344, + 16382, 4197891, 731434, 392707, 1514152, + 770441, 3369273, 4022750, 1834526, 2743411, + 2178965, 2218467, 12417, 3180456, 2455377, + 3759465, 3716946, -2772600, 187430119, 100631253, + 388001774, 197425671, 863376927, 1030830548, 470097680, + 702999655, 558360247, 568482643, 3181859, 814992530, + 629190881, 963363710, 952468207, -710479343, -1054478, + -2926054, -140244, -545376, -1163598, -749577, + 724804, -527981, 818761, 1310261, -1254190, + 268456, 250446, 3956944, 3284915, 59148, + -270210213, -749801963, -35937555, -139752717, -298172236, + -192079267, 185731180, -135295244, 209807681, 335754661, + -321386456, 68791907, 64176841, 1013967746, 841760171, + 15156688, 781875, 1716814, -3430436, -214880, + -2028038, -4148469, 1187885, 1179613, 1879878, + -613238, -2642980, 3611750, -635956, -1596822, + 2296397, 1780227, 200355636, 439933955, -879049958, + -55063046, -519685171, -1063046068, 304395785, 302276083, + 481719139, -157142369, -677264190, 925511710, -162963861, + -409185979, 588452222, 456183549, -1900052, 3014420, + -860144, -3363542, -1665318, 2612853, -507927, + -586241, -2039144, 1354892, -3195676, -2387513, + 3551006, 1979497, 3956745, -2660408, -486888731, + 772445769, -220412084, -861908357, -426738094, 669544140, + -130156402, -150224382, -522531086, 347191365, -818892658, + -611800717, 909946047, 507246529, 1013916752, -681730119, + 3773731, 2193087, 553718, -3020393, -1723229, + -3009748, 1393159, 2033807, 3472069, -2513018, + 3838479, 1727088, 3768948, 2454145, 3965306, + 1455890, 967019376, 561979013, 141890356, -773976352, + -441577800, -771248568, 356997292, 521163479, 889718424, + -643961400, 983611064, 442566669, 965793731, 628875181, + 1016110510, 373072124, 3974485, -2358373, -508145, + 1370517, 1615530, -2647994, -2491325, 2374402, + -4040196, 89301, -1239911, -2192938, -2678278, + -1009365, -636927, 2659525, 1018462631, -604333585, + -130212265, 351195274, 413979908, -678549029, -638402564, + 608441020, -1035301089, 22883400, -317727459, -561940831, + -686309310, -258649997, -163212680, 681503850, -3531229, + -3123762, 1103344, 11879, -2569011, 338420, + -1994046, -2105286, 1921994, -141835, -2296099, + 1772588, 3410568, -822541, -87208, 1935420, + -904878186, -800464680, 282732136, 3043996, -658309618, + 86720197, -510974714, -539479988, 492511373, -36345249, + -588375860, 454226054, 873958779, -210776307, -22347069, + 495951789, 303005, 2185084, -3105558, -3994671, + -3980599, 3033742, 1476985, 1623354, 458740, + -2998219, -3747250, 4146264, 1685153, 27812, + -1182243, -1753, 77645096, 559928242, -795799901, + -1023635298, -1020029345, 777397036, 378477722, 415984810, + 117552223, -768294260, -960233614, 1062481036, 431820817, + 7126831, -302950022, -449207, -1744507, 2354215, + 459163, 2778788, -3374250, -621164, 2362063, + 2254727, -3258457, 3197248, 3334383, 2962264, + -4158088, 3815725, 274060, -3852015, -447030292, + 603268097, 117660617, 712065019, -864652284, -159173408, + 605279149, 577774276, -834980303, 819295484, 854436357, + 759080783, -1065510939, 977780347, 70227934, -987079667, + 2236726, -1011223, 653275, -2683270, -2925816, + -3035980, 1300016, 2391089, 3250154, -1987814, + -2462444, -1148858, 1109516, -1937570, 3121440, + 2635473, 573161516, -259126110, 167401858, -687588511, + -749740976, -777970524, 333129378, 612717067, 832852657, + -509377762, -631001801, -294395108, 284313712, -496502727, + 799869667, 675340520, 1922253, 327848, -2312838, + 2775755, 1226661, -2461387, 4182915, -1787943, + -235407, 3488383, -169688, -482649, 2983781, + -2028118, 3222807, -1277625, 492577742, 84011120, + -592665232, 711287812, 314332144, -630730945, 1071872863, + -458160776, -60323094, 893898890, -43482586, -123678909, + 764594519, -519705671, 825844983, -327391679, 3818627, + -348812, 3467665, -1356448, -3901472, 1317678, + -3482206, 2579253, -1736313, 4166425, 565603, + -1528066, -2811291, -2508980, -4183372, -3073009, + 978523985, -89383150, 888589898, -347590090, -999753034, + 337655269, -892316032, 660934133, -444930577, 1067647297, + 144935890, -391567239, -720393920, -642926661, -1071989969, + -787459213, -1858416, -1853806, -3870317, -3192354, + 3950053, 1935799, 3574466, -1759347, -2156050, + 4018989, 3506380, -928749, -1159875, -2101410, + 3586446, -3182878, -476219497, -475038184, -991769559, + -818041395, 1012201926, 496048908, 915957677, -450833045, + -552488273, 1029866791, 898510625, -237992130, -297218217, + -538486762, 919027554, -815613168, -3345963, -2917338, + -556856, 2897314, 1716988, -3756790, 817536, + -3415069, -3241972, -2071829, -1095468, -394148, + -3704823, 3110818, -2740543, -3602218, -857403734, + -747568486, -142694469, 742437332, 439978542, -962678241, + 209493775, -875112161, -830756018, -530906624, -280713909, + -101000509, -949361686, 797147778, -702264730, -923069133, + -2283733, -2815639, 3585098, 642628, -1460718, + -2453983, -1714295, 3227876, 1335936, -676590, + 434125, 3524442, 1674615, -2663378, 4063053, + 3370349, -585207070, -721508096, 918682129, 164673562, + -374309300, -628833668, -439288460, 827143915, 342333886, + -173376332, 111244624, 903139016, 429120452, -682491182, + 1041158200, 863652652, +}; + +#else /* MLD_ARITH_BACKEND_RISCV64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +MLD_EMPTY_CU(rv64v_zetas) + +#endif /* !(MLD_ARITH_BACKEND_RISCV64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/flake.nix b/flake.nix index b943750562..c8be6d1c84 100644 --- a/flake.nix +++ b/flake.nix @@ -166,7 +166,7 @@ # autogen shell with cross compiler for the "other" architecture devShells.cross-autogen = util.mkShell { - packages = builtins.attrValues { inherit (config.packages) linters toolchain_riscv32; inherit (pkgs) gcc-arm-embedded; } + packages = builtins.attrValues { inherit (config.packages) linters toolchain_riscv32 toolchain_riscv64; inherit (pkgs) gcc-arm-embedded; } ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isx86_64 [ config.packages.toolchain_aarch64 ] ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isAarch64 [ config.packages.toolchain_x86_64 ]; }; diff --git a/mldsa/mldsa_native.c b/mldsa/mldsa_native.c index 78e80c0684..1d2f399f07 100644 --- a/mldsa/mldsa_native.c +++ b/mldsa/mldsa_native.c @@ -85,6 +85,9 @@ #include "src/native/x86_64/src/consts.c" #include "src/native/x86_64/src/rej_uniform_table.c" #endif +#if defined(MLD_SYS_RISCV64) +#include "src/native/riscv64/src/rv64v_zetas.c" +#endif #if defined(MLD_SYS_RISCV32) #include "src/native/rv32im/src/rv32im_zetas.c" #endif @@ -101,6 +104,9 @@ #include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c" #include "src/fips202/native/armv81m/src/keccakf1600_round_constants.c" #endif +#if defined(MLD_SYS_RISCV64) +#include "src/fips202/native/riscv64/src/keccakf1600_round_constants.c" +#endif #endif /* MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ /* Macro #undef's @@ -657,6 +663,21 @@ #undef mld_keccak_f1600_x4_state_xor_bytes_asm #undef mld_keccakf1600_round_constants #endif /* MLD_SYS_ARMV81M_MVE */ +#if defined(MLD_SYS_RISCV64) +/* + * Undefine macros from native code (FIPS202, RISC-V 64) + */ +/* mldsa/src/fips202/native/riscv64/auto.h */ +#undef MLD_FIPS202_NATIVE_RISCV64_AUTO_H +/* mldsa/src/fips202/native/riscv64/src/fips202_native_riscv64.h */ +#undef MLD_FIPS202_NATIVE_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H +#undef mld_keccak_f1600_x1_zbb_riscv64_asm +#undef mld_keccakf1600_round_constants +/* mldsa/src/fips202/native/riscv64/x1_zbb.h */ +#undef MLD_FIPS202_NATIVE_RISCV64_X1_ZBB_H +#undef MLD_FIPS202_RISCV64_NEED_X1_ZBB +#undef MLD_USE_NATIVE_FIPS202_X1 +#endif /* MLD_SYS_RISCV64 */ #endif /* MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) /* mldsa/src/native/api.h */ @@ -782,6 +803,54 @@ #undef MLD_NATIVE_X86_64_SRC_CONSTS_H #undef mld_qdata #endif /* MLD_SYS_X86_64 */ +#if defined(MLD_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mldsa/src/native/riscv64/meta.h */ +#undef MLD_ARITH_BACKEND_RISCV64 +#undef MLD_NATIVE_RISCV64_META_H +#undef MLD_USE_NATIVE_INTT +#undef MLD_USE_NATIVE_NTT +#undef MLD_USE_NATIVE_POINTWISE_MONTGOMERY +#undef MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4 +#undef MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5 +#undef MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7 +#undef MLD_USE_NATIVE_POLYZ_UNPACK_17 +#undef MLD_USE_NATIVE_POLYZ_UNPACK_19 +#undef MLD_USE_NATIVE_POLY_CADDQ +#undef MLD_USE_NATIVE_POLY_CHKNORM +#undef MLD_USE_NATIVE_POLY_DECOMPOSE_32 +#undef MLD_USE_NATIVE_POLY_DECOMPOSE_88 +#undef MLD_USE_NATIVE_POLY_USE_HINT_32 +#undef MLD_USE_NATIVE_POLY_USE_HINT_88 +#undef MLD_USE_NATIVE_REJ_UNIFORM +#undef MLD_USE_NATIVE_REJ_UNIFORM_ETA2 +#undef MLD_USE_NATIVE_REJ_UNIFORM_ETA4 +/* mldsa/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLD_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mld_intt_vlen128_riscv64_asm +#undef mld_intt_vlen256_riscv64_asm +#undef mld_ntt_vlen128_riscv64_asm +#undef mld_ntt_vlen256_riscv64_asm +#undef mld_poly_caddq_riscv64_asm +#undef mld_poly_chknorm_riscv64_asm +#undef mld_poly_decompose_32_riscv64_asm +#undef mld_poly_decompose_88_riscv64_asm +#undef mld_poly_pointwise_montgomery_riscv64_asm +#undef mld_poly_use_hint_32_riscv64_asm +#undef mld_poly_use_hint_88_riscv64_asm +#undef mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm +#undef mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm +#undef mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm +#undef mld_polyz_unpack_17_riscv64_asm +#undef mld_polyz_unpack_19_riscv64_asm +#undef mld_rej_uniform_eta2_riscv64_asm +#undef mld_rej_uniform_eta4_riscv64_asm +#undef mld_rej_uniform_riscv64_asm +#undef mld_rv64v_intt_zetas +#undef mld_rv64v_ntt_zetas +#endif /* MLD_SYS_RISCV64 */ #if defined(MLD_SYS_RISCV32) /* * Undefine macros from native code (Arith, RV32IM) diff --git a/mldsa/mldsa_native_asm.S b/mldsa/mldsa_native_asm.S index b3ccb8b474..57d04fc821 100644 --- a/mldsa/mldsa_native_asm.S +++ b/mldsa/mldsa_native_asm.S @@ -98,6 +98,27 @@ #include "src/native/x86_64/src/mldsa_rej_uniform_eta2_avx2_asm.S" #include "src/native/x86_64/src/mldsa_rej_uniform_eta4_avx2_asm.S" #endif /* MLD_SYS_X86_64 */ +#if defined(MLD_SYS_RISCV64) +#include "src/native/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_caddq_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S" +#include "src/native/riscv64/src/mldsa_rej_uniform_riscv64_asm.S" +#endif /* MLD_SYS_RISCV64 */ #if defined(MLD_SYS_RISCV32) #include "src/native/rv32im/src/mldsa_intt_rv32im_asm.S" #include "src/native/rv32im/src/mldsa_intt_rv32im_slowmul_asm.S" @@ -123,6 +144,9 @@ #include "src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S" #include "src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S" #endif +#if defined(MLD_SYS_RISCV64) +#include "src/fips202/native/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S" +#endif #endif /* MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ /* Mark the stack non-executable even if no assembly is included above */ @@ -695,6 +719,21 @@ #undef mld_keccak_f1600_x4_state_xor_bytes_asm #undef mld_keccakf1600_round_constants #endif /* MLD_SYS_ARMV81M_MVE */ +#if defined(MLD_SYS_RISCV64) +/* + * Undefine macros from native code (FIPS202, RISC-V 64) + */ +/* mldsa/src/fips202/native/riscv64/auto.h */ +#undef MLD_FIPS202_NATIVE_RISCV64_AUTO_H +/* mldsa/src/fips202/native/riscv64/src/fips202_native_riscv64.h */ +#undef MLD_FIPS202_NATIVE_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H +#undef mld_keccak_f1600_x1_zbb_riscv64_asm +#undef mld_keccakf1600_round_constants +/* mldsa/src/fips202/native/riscv64/x1_zbb.h */ +#undef MLD_FIPS202_NATIVE_RISCV64_X1_ZBB_H +#undef MLD_FIPS202_RISCV64_NEED_X1_ZBB +#undef MLD_USE_NATIVE_FIPS202_X1 +#endif /* MLD_SYS_RISCV64 */ #endif /* MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ #if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) /* mldsa/src/native/api.h */ @@ -820,6 +859,54 @@ #undef MLD_NATIVE_X86_64_SRC_CONSTS_H #undef mld_qdata #endif /* MLD_SYS_X86_64 */ +#if defined(MLD_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mldsa/src/native/riscv64/meta.h */ +#undef MLD_ARITH_BACKEND_RISCV64 +#undef MLD_NATIVE_RISCV64_META_H +#undef MLD_USE_NATIVE_INTT +#undef MLD_USE_NATIVE_NTT +#undef MLD_USE_NATIVE_POINTWISE_MONTGOMERY +#undef MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4 +#undef MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5 +#undef MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7 +#undef MLD_USE_NATIVE_POLYZ_UNPACK_17 +#undef MLD_USE_NATIVE_POLYZ_UNPACK_19 +#undef MLD_USE_NATIVE_POLY_CADDQ +#undef MLD_USE_NATIVE_POLY_CHKNORM +#undef MLD_USE_NATIVE_POLY_DECOMPOSE_32 +#undef MLD_USE_NATIVE_POLY_DECOMPOSE_88 +#undef MLD_USE_NATIVE_POLY_USE_HINT_32 +#undef MLD_USE_NATIVE_POLY_USE_HINT_88 +#undef MLD_USE_NATIVE_REJ_UNIFORM +#undef MLD_USE_NATIVE_REJ_UNIFORM_ETA2 +#undef MLD_USE_NATIVE_REJ_UNIFORM_ETA4 +/* mldsa/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLD_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mld_intt_vlen128_riscv64_asm +#undef mld_intt_vlen256_riscv64_asm +#undef mld_ntt_vlen128_riscv64_asm +#undef mld_ntt_vlen256_riscv64_asm +#undef mld_poly_caddq_riscv64_asm +#undef mld_poly_chknorm_riscv64_asm +#undef mld_poly_decompose_32_riscv64_asm +#undef mld_poly_decompose_88_riscv64_asm +#undef mld_poly_pointwise_montgomery_riscv64_asm +#undef mld_poly_use_hint_32_riscv64_asm +#undef mld_poly_use_hint_88_riscv64_asm +#undef mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm +#undef mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm +#undef mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm +#undef mld_polyz_unpack_17_riscv64_asm +#undef mld_polyz_unpack_19_riscv64_asm +#undef mld_rej_uniform_eta2_riscv64_asm +#undef mld_rej_uniform_eta4_riscv64_asm +#undef mld_rej_uniform_riscv64_asm +#undef mld_rv64v_intt_zetas +#undef mld_rv64v_ntt_zetas +#endif /* MLD_SYS_RISCV64 */ #if defined(MLD_SYS_RISCV32) /* * Undefine macros from native code (Arith, RV32IM) diff --git a/mldsa/src/fips202/native/auto.h b/mldsa/src/fips202/native/auto.h index 70de3c4cb8..a1cc0a6f55 100644 --- a/mldsa/src/fips202/native/auto.h +++ b/mldsa/src/fips202/native/auto.h @@ -16,6 +16,10 @@ #include "aarch64/auto.h" #endif +#if defined(MLD_SYS_RISCV64) +#include "riscv64/auto.h" +#endif + /* The x86_64 backend requires toolchain support for the SysV ABI */ #if defined(MLD_SYS_X86_64_AVX2) && defined(MLD_SYSV_ABI_SUPPORTED) && \ (!defined(MLD_CONFIG_NO_KEYPAIR_API) || \ diff --git a/mldsa/src/fips202/native/riscv64/auto.h b/mldsa/src/fips202/native/riscv64/auto.h new file mode 100644 index 0000000000..fe784469d4 --- /dev/null +++ b/mldsa/src/fips202/native/riscv64/auto.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_FIPS202_NATIVE_RISCV64_AUTO_H +#define MLD_FIPS202_NATIVE_RISCV64_AUTO_H +/* Default FIPS202 assembly profile for RISC-V 64 systems */ + +#if defined(__riscv_zbb) +#include "x1_zbb.h" +#endif + +#endif /* !MLD_FIPS202_NATIVE_RISCV64_AUTO_H */ diff --git a/mldsa/src/fips202/native/riscv64/src/fips202_native_riscv64.h b/mldsa/src/fips202/native/riscv64/src/fips202_native_riscv64.h new file mode 100644 index 0000000000..52665c3c5c --- /dev/null +++ b/mldsa/src/fips202/native/riscv64/src/fips202_native_riscv64.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_FIPS202_NATIVE_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H +#define MLD_FIPS202_NATIVE_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +#define mld_keccakf1600_round_constants \ + MLD_NAMESPACE(keccakf1600_round_constants) +MLD_INTERNAL_DATA_DECLARATION const uint64_t + mld_keccakf1600_round_constants[24]; + +#define mld_keccak_f1600_x1_zbb_riscv64_asm \ + MLD_NAMESPACE(keccak_f1600_x1_zbb_riscv64_asm) +void mld_keccak_f1600_x1_zbb_riscv64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/keccak_f1600_x1_zbb_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mld_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#endif /* !MLD_FIPS202_NATIVE_RISCV64_SRC_FIPS202_NATIVE_RISCV64_H */ diff --git a/mldsa/src/fips202/native/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S b/mldsa/src/fips202/native/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S new file mode 100644 index 0000000000..d4d28938b0 --- /dev/null +++ b/mldsa/src/fips202/native/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S @@ -0,0 +1,422 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: keccak_f1600_x1_zbb_riscv64_asm + Description: RV64 scalar implementation of the Keccak-f[1600] permutation for a single state, using Zbb + Signature: void mld_keccak_f1600_x1_zbb_riscv64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [ZBB] + a0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + a1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 144 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLD_SYS_RISCV64) && defined(__riscv_zbb) && \ + defined(MLD_FIPS202_RISCV64_NEED_X1_ZBB) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/fips202/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(keccak_f1600_x1_zbb_riscv64_asm) +MLD_ASM_FN_SYMBOL(keccak_f1600_x1_zbb_riscv64_asm) + + addi sp, sp, -0x90 + sd ra, 0x0(sp) + sd s0, 0x8(sp) + sd s1, 0x10(sp) + sd s2, 0x18(sp) + sd s3, 0x20(sp) + sd s4, 0x28(sp) + sd s5, 0x30(sp) + sd s6, 0x38(sp) + sd s7, 0x40(sp) + sd s8, 0x48(sp) + sd s9, 0x50(sp) + sd s10, 0x58(sp) + sd s11, 0x60(sp) + sd a0, 0x68(sp) + sd a1, 0x70(sp) + addi t0, a1, 0xc0 + sd t0, 0x78(sp) + ld a1, 0x8(a0) + ld a2, 0x10(a0) + ld a3, 0x18(a0) + ld a4, 0x20(a0) + ld a5, 0x28(a0) + ld a6, 0x30(a0) + ld a7, 0x38(a0) + ld s2, 0x40(a0) + ld s3, 0x48(a0) + ld s4, 0x50(a0) + ld s5, 0x58(a0) + ld s6, 0x60(a0) + ld s7, 0x68(a0) + ld s8, 0x70(a0) + ld s9, 0x78(a0) + ld s10, 0x80(a0) + ld s11, 0x88(a0) + ld s0, 0x90(a0) + ld s1, 0x98(a0) + ld t3, 0xa0(a0) + ld t4, 0xa8(a0) + ld t5, 0xb0(a0) + ld t6, 0xb8(a0) + ld ra, 0xc0(a0) + ld a0, 0x0(a0) + +mld_keccak_f1600_x1_zbb_riscv64_loop: + xor t0, a0, a5 + xor t0, t0, s4 + xor t0, t0, s9 + xor t0, t0, t3 + sd t0, 0x80(sp) + xor t0, a2, a7 + xor t0, t0, s6 + xor t0, t0, s11 + xor t0, t0, t5 + sd t0, 0x88(sp) + xor t0, a1, a6 + xor t0, t0, s5 + xor t0, t0, s10 + xor t0, t0, t4 + xor t1, a4, s3 + xor t1, t1, s8 + xor t1, t1, s1 + xor t1, t1, ra + rori t2, t0, 0x3f + xor t2, t2, t1 + rori t1, t1, 0x3f + xor a0, a0, t2 + xor a5, a5, t2 + xor s4, s4, t2 + xor s9, s9, t2 + xor t3, t3, t2 + ld t2, 0x88(sp) + xor t1, t1, t2 + xor t2, a3, s2 + xor t2, t2, s7 + xor t2, t2, s0 + xor t2, t2, t6 + xor a3, a3, t1 + xor s2, s2, t1 + xor s7, s7, t1 + xor s0, s0, t1 + xor t6, t6, t1 + rori t1, t2, 0x3f + xor t0, t0, t1 + ld t1, 0x80(sp) + xor a2, a2, t0 + xor a7, a7, t0 + xor s6, s6, t0 + xor s11, s11, t0 + xor t5, t5, t0 + rori t0, t1, 0x3f + xor t2, t2, t0 + ld t0, 0x88(sp) + xor a4, a4, t2 + xor s3, s3, t2 + xor s8, s8, t2 + xor s1, s1, t2 + xor ra, ra, t2 + rori t0, t0, 0x3f + xor t0, t0, t1 + xor t2, a1, t0 + xor a6, a6, t0 + xor s5, s5, t0 + xor s10, s10, t0 + xor t4, t4, t0 + rori a1, a6, 0x14 + rori a6, s3, 0x2c + rori s3, t5, 0x3 + rori t5, s8, 0x19 + rori s8, t3, 0x2e + rori t3, a2, 0x2 + rori a2, s6, 0x15 + rori s6, s7, 0x27 + rori s7, s1, 0x38 + rori s1, t6, 0x8 + rori t6, s9, 0x17 + rori s9, a4, 0x25 + rori a4, ra, 0x32 + rori ra, t4, 0x3e + rori t4, s2, 0x9 + rori s2, s10, 0x13 + rori s10, a5, 0x1c + rori a5, a3, 0x24 + rori a3, s0, 0x2b + rori s0, s11, 0x31 + rori s11, s5, 0x36 + rori s5, a7, 0x3a + rori a7, s4, 0x3d + rori s4, t2, 0x3f + andn t0, a0, a4 + andn t1, a1, a0 + andn t2, a2, a1 + xor a0, a0, t2 + andn t2, a3, a2 + xor a1, a1, t2 + andn t2, a4, a3 + xor a2, a2, t2 + xor a3, a3, t0 + xor a4, a4, t1 + andn t0, a5, s3 + andn t1, a6, a5 + andn t2, a7, a6 + xor a5, a5, t2 + andn t2, s2, a7 + xor a6, a6, t2 + andn t2, s3, s2 + xor a7, a7, t2 + xor s2, s2, t0 + xor s3, s3, t1 + andn t0, s4, s8 + andn t1, s5, s4 + andn t2, s6, s5 + xor s4, s4, t2 + andn t2, s7, s6 + xor s5, s5, t2 + andn t2, s8, s7 + xor s6, s6, t2 + xor s7, s7, t0 + xor s8, s8, t1 + andn t0, s9, s1 + andn t1, s10, s9 + andn t2, s11, s10 + xor s9, s9, t2 + andn t2, s0, s11 + xor s10, s10, t2 + andn t2, s1, s0 + xor s11, s11, t2 + xor s0, s0, t0 + xor s1, s1, t1 + andn t0, t3, ra + andn t1, t4, t3 + andn t2, t5, t4 + xor t3, t3, t2 + andn t2, t6, t5 + xor t4, t4, t2 + andn t2, ra, t6 + xor t5, t5, t2 + xor t6, t6, t0 + xor ra, ra, t1 + ld t0, 0x70(sp) + ld t1, 0x0(t0) + xor a0, a0, t1 + xor t0, a0, a5 + xor t0, t0, s4 + xor t0, t0, s9 + xor t0, t0, t3 + sd t0, 0x80(sp) + xor t0, a2, a7 + xor t0, t0, s6 + xor t0, t0, s11 + xor t0, t0, t5 + sd t0, 0x88(sp) + xor t0, a1, a6 + xor t0, t0, s5 + xor t0, t0, s10 + xor t0, t0, t4 + xor t1, a4, s3 + xor t1, t1, s8 + xor t1, t1, s1 + xor t1, t1, ra + rori t2, t0, 0x3f + xor t2, t2, t1 + rori t1, t1, 0x3f + xor a0, a0, t2 + xor a5, a5, t2 + xor s4, s4, t2 + xor s9, s9, t2 + xor t3, t3, t2 + ld t2, 0x88(sp) + xor t1, t1, t2 + xor t2, a3, s2 + xor t2, t2, s7 + xor t2, t2, s0 + xor t2, t2, t6 + xor a3, a3, t1 + xor s2, s2, t1 + xor s7, s7, t1 + xor s0, s0, t1 + xor t6, t6, t1 + rori t1, t2, 0x3f + xor t0, t0, t1 + ld t1, 0x80(sp) + xor a2, a2, t0 + xor a7, a7, t0 + xor s6, s6, t0 + xor s11, s11, t0 + xor t5, t5, t0 + rori t0, t1, 0x3f + xor t2, t2, t0 + ld t0, 0x88(sp) + xor a4, a4, t2 + xor s3, s3, t2 + xor s8, s8, t2 + xor s1, s1, t2 + xor ra, ra, t2 + rori t0, t0, 0x3f + xor t0, t0, t1 + xor t2, a1, t0 + xor a6, a6, t0 + xor s5, s5, t0 + xor s10, s10, t0 + xor t4, t4, t0 + rori a1, a6, 0x14 + rori a6, s3, 0x2c + rori s3, t5, 0x3 + rori t5, s8, 0x19 + rori s8, t3, 0x2e + rori t3, a2, 0x2 + rori a2, s6, 0x15 + rori s6, s7, 0x27 + rori s7, s1, 0x38 + rori s1, t6, 0x8 + rori t6, s9, 0x17 + rori s9, a4, 0x25 + rori a4, ra, 0x32 + rori ra, t4, 0x3e + rori t4, s2, 0x9 + rori s2, s10, 0x13 + rori s10, a5, 0x1c + rori a5, a3, 0x24 + rori a3, s0, 0x2b + rori s0, s11, 0x31 + rori s11, s5, 0x36 + rori s5, a7, 0x3a + rori a7, s4, 0x3d + rori s4, t2, 0x3f + andn t0, a0, a4 + andn t1, a1, a0 + andn t2, a2, a1 + xor a0, a0, t2 + andn t2, a3, a2 + xor a1, a1, t2 + andn t2, a4, a3 + xor a2, a2, t2 + xor a3, a3, t0 + xor a4, a4, t1 + andn t0, a5, s3 + andn t1, a6, a5 + andn t2, a7, a6 + xor a5, a5, t2 + andn t2, s2, a7 + xor a6, a6, t2 + andn t2, s3, s2 + xor a7, a7, t2 + xor s2, s2, t0 + xor s3, s3, t1 + andn t0, s4, s8 + andn t1, s5, s4 + andn t2, s6, s5 + xor s4, s4, t2 + andn t2, s7, s6 + xor s5, s5, t2 + andn t2, s8, s7 + xor s6, s6, t2 + xor s7, s7, t0 + xor s8, s8, t1 + andn t0, s9, s1 + andn t1, s10, s9 + andn t2, s11, s10 + xor s9, s9, t2 + andn t2, s0, s11 + xor s10, s10, t2 + andn t2, s1, s0 + xor s11, s11, t2 + xor s0, s0, t0 + xor s1, s1, t1 + andn t0, t3, ra + andn t1, t4, t3 + andn t2, t5, t4 + xor t3, t3, t2 + andn t2, t6, t5 + xor t4, t4, t2 + andn t2, ra, t6 + xor t5, t5, t2 + xor t6, t6, t0 + xor ra, ra, t1 + ld t0, 0x70(sp) + ld t1, 0x8(t0) + xor a0, a0, t1 + addi t0, t0, 0x10 + sd t0, 0x70(sp) + ld t1, 0x78(sp) + bne t0, t1, mld_keccak_f1600_x1_zbb_riscv64_loop + ld t0, 0x68(sp) + sd a1, 0x8(t0) + sd a2, 0x10(t0) + sd a3, 0x18(t0) + sd a4, 0x20(t0) + sd a5, 0x28(t0) + sd a6, 0x30(t0) + sd a7, 0x38(t0) + sd s2, 0x40(t0) + sd s3, 0x48(t0) + sd s4, 0x50(t0) + sd s5, 0x58(t0) + sd s6, 0x60(t0) + sd s7, 0x68(t0) + sd s8, 0x70(t0) + sd s9, 0x78(t0) + sd s10, 0x80(t0) + sd s11, 0x88(t0) + sd s0, 0x90(t0) + sd s1, 0x98(t0) + sd t3, 0xa0(t0) + sd t4, 0xa8(t0) + sd t5, 0xb0(t0) + sd t6, 0xb8(t0) + sd ra, 0xc0(t0) + sd a0, 0x0(t0) + ld ra, 0x0(sp) + ld s0, 0x8(sp) + ld s1, 0x10(sp) + ld s2, 0x18(sp) + ld s3, 0x20(sp) + ld s4, 0x28(sp) + ld s5, 0x30(sp) + ld s6, 0x38(sp) + ld s7, 0x40(sp) + ld s8, 0x48(sp) + ld s9, 0x50(sp) + ld s10, 0x58(sp) + ld s11, 0x60(sp) + addi sp, sp, 0x90 + ret + +.option pop +MLD_ASM_FN_SIZE(keccak_f1600_x1_zbb_riscv64_asm) + +#endif /* MLD_SYS_RISCV64 && __riscv_zbb && MLD_FIPS202_RISCV64_NEED_X1_ZBB && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/fips202/native/riscv64/src/keccakf1600_round_constants.c b/mldsa/src/fips202/native/riscv64/src/keccakf1600_round_constants.c new file mode 100644 index 0000000000..065203eea5 --- /dev/null +++ b/mldsa/src/fips202/native/riscv64/src/keccakf1600_round_constants.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" + +#if defined(MLD_FIPS202_RISCV64_NEED_X1_ZBB) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_riscv64.h" + +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint64_t + mld_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +#else /* MLD_FIPS202_RISCV64_NEED_X1_ZBB && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLD_EMPTY_CU(fips202_riscv64_round_constants) + +#endif /* !(MLD_FIPS202_RISCV64_NEED_X1_ZBB && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/mldsa/src/fips202/native/riscv64/x1_zbb.h b/mldsa/src/fips202/native/riscv64/x1_zbb.h new file mode 100644 index 0000000000..48d2050d0d --- /dev/null +++ b/mldsa/src/fips202/native/riscv64/x1_zbb.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_FIPS202_NATIVE_RISCV64_X1_ZBB_H +#define MLD_FIPS202_NATIVE_RISCV64_X1_ZBB_H + +/* Part of backend API */ +#define MLD_USE_NATIVE_FIPS202_X1 +/* Guard for assembly file */ +#define MLD_FIPS202_RISCV64_NEED_X1_ZBB + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_riscv64.h" +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_keccak_f1600_x1_native(uint64_t *state) +{ + mld_keccak_f1600_x1_zbb_riscv64_asm(state, mld_keccakf1600_round_constants); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_FIPS202_NATIVE_RISCV64_X1_ZBB_H */ diff --git a/mldsa/src/native/meta.h b/mldsa/src/native/meta.h index 00d200c108..d6711ba235 100644 --- a/mldsa/src/native/meta.h +++ b/mldsa/src/native/meta.h @@ -21,6 +21,10 @@ #include "x86_64/meta.h" #endif +#if defined(MLD_SYS_RISCV64_RVV) +#include "riscv64/meta.h" +#endif + #if defined(MLD_SYS_RISCV32_M) #include "rv32im/fastmul.h" #endif diff --git a/mldsa/src/native/riscv64/meta.h b/mldsa/src/native/riscv64/meta.h new file mode 100644 index 0000000000..db2d20dca7 --- /dev/null +++ b/mldsa/src/native/riscv64/meta.h @@ -0,0 +1,270 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_NATIVE_RISCV64_META_H +#define MLD_NATIVE_RISCV64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLD_ARITH_BACKEND_RISCV64 + +/* Set of primitives that this backend replaces */ +#define MLD_USE_NATIVE_NTT +#define MLD_USE_NATIVE_INTT +#define MLD_USE_NATIVE_REJ_UNIFORM +#define MLD_USE_NATIVE_REJ_UNIFORM_ETA2 +#define MLD_USE_NATIVE_REJ_UNIFORM_ETA4 +#define MLD_USE_NATIVE_POLY_DECOMPOSE_32 +#define MLD_USE_NATIVE_POLY_DECOMPOSE_88 +#define MLD_USE_NATIVE_POLY_CADDQ +#define MLD_USE_NATIVE_POLY_USE_HINT_32 +#define MLD_USE_NATIVE_POLY_USE_HINT_88 +#define MLD_USE_NATIVE_POLY_CHKNORM +#define MLD_USE_NATIVE_POLYZ_UNPACK_17 +#define MLD_USE_NATIVE_POLYZ_UNPACK_19 +#define MLD_USE_NATIVE_POINTWISE_MONTGOMERY +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4 +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5 +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7 + +#if !defined(__ASSEMBLER__) +#if !defined(CBMC) +#include +#endif + +#include "../../common.h" +#include "../api.h" +#include "src/arith_native_riscv64.h" + +/* Returns the vector register length VLEN in bits. + * CBMC proofs use the contract instead of the body. */ +static MLD_INLINE int mld_rv64v_vlen(void) +__contract__( + ensures(return_value >= 0) +) +{ +#if !defined(CBMC) + return (int)(32 * __riscv_vsetvlmax_e32m1()); +#else + return 0; +#endif +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N]) +{ + /* VLEN = 128 and VLEN = 256 only for now */ + const int vlen = mld_rv64v_vlen(); + if (vlen == 256) + { + mld_ntt_vlen256_riscv64_asm(data, mld_rv64v_ntt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + if (vlen == 128) + { + mld_ntt_vlen128_riscv64_asm(data, mld_rv64v_ntt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + return MLD_NATIVE_FUNC_FALLBACK; +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N]) +{ + /* VLEN = 128 and VLEN = 256 only for now */ + const int vlen = mld_rv64v_vlen(); + if (vlen == 256) + { + mld_intt_vlen256_riscv64_asm(data, mld_rv64v_intt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + if (vlen == 128) + { + mld_intt_vlen128_riscv64_asm(data, mld_rv64v_intt_zetas); + return MLD_NATIVE_FUNC_SUCCESS; + } + return MLD_NATIVE_FUNC_FALLBACK; +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + /* Safety: The return value is at most len <= MLDSA_N, hence the cast + * is safe. */ + return (int)mld_rej_uniform_riscv64_asm(r, len, buf, buflen); +} + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_eta2_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + uint64_t outlen; + /* Constant time: Inputs and outputs to this function are secret. + * It is safe to leak which coefficients are accepted/rejected. + * The assembly implementation must not leak any other information about the + * accepted coefficients. Constant-time testing cannot cover this, and we + * hence have to manually verify the assembly. + * We declassify prior the input data and mark the outputs as secret. + */ + MLD_CT_TESTING_DECLASSIFY(buf, buflen); + outlen = mld_rej_uniform_eta2_riscv64_asm(r, len, buf, buflen); + MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen); + /* Safety: outlen is at most len <= MLDSA_N, hence the cast is safe. */ + return (int)outlen; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + uint64_t outlen; + /* Constant time: see mld_rej_uniform_eta2_native(). */ + MLD_CT_TESTING_DECLASSIFY(buf, buflen); + outlen = mld_rej_uniform_eta4_riscv64_asm(r, len, buf, buflen); + MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen); + /* Safety: outlen is at most len <= MLDSA_N, hence the cast is safe. */ + return (int)outlen; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4 */ +#endif /* !MLD_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0) +{ + mld_poly_decompose_32_riscv64_asm(a1, a0); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0) +{ + mld_poly_decompose_88_riscv64_asm(a1, a0); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_SIGN_API */ + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N]) +{ + mld_poly_caddq_riscv64_asm(a); + return MLD_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h) +{ + mld_poly_use_hint_32_riscv64_asm(a, h); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h) +{ + mld_poly_use_hint_88_riscv64_asm(a, h); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_VERIFY_API */ + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B) +{ + return mld_poly_chknorm_riscv64_asm(a, B); +} + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *buf) +{ + mld_polyz_unpack_17_riscv64_asm(r, buf); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *buf) +{ + mld_polyz_unpack_19_riscv64_asm(r, buf); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_pointwise_montgomery_native( + int32_t a[MLDSA_N], const int32_t b[MLDSA_N]) +{ + mld_poly_pointwise_montgomery_riscv64_asm(a, b); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native( + int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], + const int32_t v[4][MLDSA_N]) +{ + mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(w, u, v); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native( + int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], + const int32_t v[5][MLDSA_N]) +{ + mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(w, u, v); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native( + int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], + const int32_t v[7][MLDSA_N]) +{ + mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(w, u, v); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_NATIVE_RISCV64_META_H */ diff --git a/mldsa/src/native/riscv64/src/arith_native_riscv64.h b/mldsa/src/native/riscv64/src/arith_native_riscv64.h new file mode 100644 index 0000000000..82474ef049 --- /dev/null +++ b/mldsa/src/native/riscv64/src/arith_native_riscv64.h @@ -0,0 +1,343 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#define MLD_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H + +#include "../../../cbmc.h" +#include "../../../common.h" + +#define mld_rv64v_ntt_zetas MLD_NAMESPACE(rv64v_ntt_zetas) +#define mld_rv64v_intt_zetas MLD_NAMESPACE(rv64v_intt_zetas) + +MLD_INTERNAL_DATA_DECLARATION const int32_t mld_rv64v_ntt_zetas[512]; +MLD_INTERNAL_DATA_DECLARATION const int32_t mld_rv64v_intt_zetas[512]; + +#define mld_ntt_vlen256_riscv64_asm MLD_NAMESPACE(ntt_vlen256_riscv64_asm) +void mld_ntt_vlen256_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_ntt_vlen256_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_ntt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + ensures(array_abs_bound(p, 0, MLDSA_N, 94279698)) +); + +#define mld_ntt_vlen128_riscv64_asm MLD_NAMESPACE(ntt_vlen128_riscv64_asm) +void mld_ntt_vlen128_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_ntt_vlen128_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_ntt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + ensures(array_abs_bound(p, 0, MLDSA_N, 94279698)) +); + +#define mld_intt_vlen256_riscv64_asm MLD_NAMESPACE(intt_vlen256_riscv64_asm) +void mld_intt_vlen256_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_intt_vlen256_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_intt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) +); + +#define mld_intt_vlen128_riscv64_asm MLD_NAMESPACE(intt_vlen128_riscv64_asm) +void mld_intt_vlen128_riscv64_asm(int32_t p[MLDSA_N], const int32_t zetas[512]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_intt_vlen128_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) + requires(zetas == mld_rv64v_intt_zetas) + assigns(memory_slice(p, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(p, 0, MLDSA_N, MLDSA_Q)) +); + +#define mld_rej_uniform_riscv64_asm MLD_NAMESPACE(rej_uniform_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +uint64_t mld_rej_uniform_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_rej_uniform_riscv64_asm.ml */ +__contract__( + requires(len <= MLDSA_N && buflen <= 840 && buflen % 3 == 0) + requires(memory_no_alias(buf, buflen)) + requires(memory_no_alias(r, sizeof(int32_t) * len)) + assigns(memory_slice(r, sizeof(int32_t) * len)) + ensures(return_value <= len) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLDSA_Q)) +); + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2 +#define mld_rej_uniform_eta2_riscv64_asm \ + MLD_NAMESPACE(rej_uniform_eta2_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +uint64_t mld_rej_uniform_eta2_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_rej_uniform_eta2_riscv64_asm.ml */ +__contract__( + requires(len <= MLDSA_N && buflen <= 272) + requires(memory_no_alias(buf, buflen)) + requires(memory_no_alias(r, sizeof(int32_t) * len)) + assigns(memory_slice(r, sizeof(int32_t) * len)) + ensures(return_value <= len) + ensures(array_abs_bound(r, 0, (unsigned) return_value, 3)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4 +#define mld_rej_uniform_eta4_riscv64_asm \ + MLD_NAMESPACE(rej_uniform_eta4_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +uint64_t mld_rej_uniform_eta4_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_rej_uniform_eta4_riscv64_asm.ml */ +__contract__( + requires(len <= MLDSA_N && buflen <= 272) + requires(memory_no_alias(buf, buflen)) + requires(memory_no_alias(r, sizeof(int32_t) * len)) + assigns(memory_slice(r, sizeof(int32_t) * len)) + ensures(return_value <= len) + ensures(array_abs_bound(r, 0, (unsigned) return_value, 5)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4 */ +#endif /* !MLD_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +#define mld_poly_decompose_32_riscv64_asm \ + MLD_NAMESPACE(poly_decompose_32_riscv64_asm) +void mld_poly_decompose_32_riscv64_asm(int32_t a1[MLDSA_N], int32_t a0[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_decompose_32_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a1, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a0, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a0, 0, MLDSA_N, 0, MLDSA_Q)) + assigns(memory_slice(a1, sizeof(int32_t) * MLDSA_N)) + assigns(memory_slice(a0, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 16 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 32)) */ + ensures(array_bound(a1, 0, MLDSA_N, 0, 16)) + /* check-magic: 261889 == (MLDSA_Q - 1) / 32 + 1 */ + ensures(array_abs_bound(a0, 0, MLDSA_N, 261889)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +#define mld_poly_decompose_88_riscv64_asm \ + MLD_NAMESPACE(poly_decompose_88_riscv64_asm) +void mld_poly_decompose_88_riscv64_asm(int32_t a1[MLDSA_N], int32_t a0[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_decompose_88_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a1, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a0, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a0, 0, MLDSA_N, 0, MLDSA_Q)) + assigns(memory_slice(a1, sizeof(int32_t) * MLDSA_N)) + assigns(memory_slice(a0, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 44 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 88)) */ + ensures(array_bound(a1, 0, MLDSA_N, 0, 44)) + /* check-magic: 95233 == (MLDSA_Q - 1) / 88 + 1 */ + ensures(array_abs_bound(a0, 0, MLDSA_N, 95233)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_SIGN_API */ + +#define mld_poly_caddq_riscv64_asm MLD_NAMESPACE(poly_caddq_riscv64_asm) +void mld_poly_caddq_riscv64_asm(int32_t a[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_caddq_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(array_abs_bound(a, 0, MLDSA_N, MLDSA_Q)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) +); + +#if !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +#define mld_poly_use_hint_32_riscv64_asm \ + MLD_NAMESPACE(poly_use_hint_32_riscv64_asm) +void mld_poly_use_hint_32_riscv64_asm(int32_t a[MLDSA_N], + const int32_t h[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_use_hint_32_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(h, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) + requires(array_bound(h, 0, MLDSA_N, 0, 2)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(a, 0, MLDSA_N, 0, 16)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +#define mld_poly_use_hint_88_riscv64_asm \ + MLD_NAMESPACE(poly_use_hint_88_riscv64_asm) +void mld_poly_use_hint_88_riscv64_asm(int32_t a[MLDSA_N], + const int32_t h[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_use_hint_88_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(h, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) + requires(array_bound(h, 0, MLDSA_N, 0, 2)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(a, 0, MLDSA_N, 0, 44)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_VERIFY_API */ + +#define mld_poly_chknorm_riscv64_asm MLD_NAMESPACE(poly_chknorm_riscv64_asm) +MLD_MUST_CHECK_RETURN_VALUE +int mld_poly_chknorm_riscv64_asm(const int32_t a[MLDSA_N], int64_t B) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_poly_chknorm_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(forall(k0, 0, MLDSA_N, a[k0] > INT32_MIN)) + requires(0 <= B && B <= INT32_MAX) + ensures(return_value == 0 || return_value == 1) + ensures((return_value == 0) == array_abs_bound(a, 0, MLDSA_N, B)) +); + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +#define mld_polyz_unpack_17_riscv64_asm \ + MLD_NAMESPACE(polyz_unpack_17_riscv64_asm) +void mld_polyz_unpack_17_riscv64_asm(int32_t r[MLDSA_N], const uint8_t buf[576]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_polyz_unpack_17_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, 576)) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(r, 0, MLDSA_N, -((1 << 17) - 1), (1 << 17) + 1)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +#define mld_polyz_unpack_19_riscv64_asm \ + MLD_NAMESPACE(polyz_unpack_19_riscv64_asm) +void mld_polyz_unpack_19_riscv64_asm(int32_t r[MLDSA_N], const uint8_t buf[640]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/riscv64/proofs/mldsa_polyz_unpack_19_riscv64_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, 640)) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(r, 0, MLDSA_N, -((1 << 19) - 1), (1 << 19) + 1)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +#define mld_poly_pointwise_montgomery_riscv64_asm \ + MLD_NAMESPACE(poly_pointwise_montgomery_riscv64_asm) +void mld_poly_pointwise_montgomery_riscv64_asm(int32_t a[MLDSA_N], + const int32_t b[MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_poly_pointwise_montgomery_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(b, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(array_abs_bound(a, 0, MLDSA_N, 94279698)) + requires(array_abs_bound(b, 0, MLDSA_N, 94279698)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(a, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4 +#define mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm \ + MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) +void mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm( + int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], + const int32_t v[4][MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(w, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(u, sizeof(int32_t) * 4 * MLDSA_N)) + requires(memory_no_alias(v, sizeof(int32_t) * 4 * MLDSA_N)) + requires(forall(l0, 0, 4, array_bound(u[l0], 0, MLDSA_N, 0, MLDSA_Q))) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(forall(l1, 0, 4, array_abs_bound(v[l1], 0, MLDSA_N, 94279698))) + assigns(memory_slice(w, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(w, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5 +#define mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm \ + MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) +void mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm( + int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], + const int32_t v[5][MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(w, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(u, sizeof(int32_t) * 5 * MLDSA_N)) + requires(memory_no_alias(v, sizeof(int32_t) * 5 * MLDSA_N)) + requires(forall(l0, 0, 5, array_bound(u[l0], 0, MLDSA_N, 0, MLDSA_Q))) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(forall(l1, 0, 5, array_abs_bound(v[l1], 0, MLDSA_N, 94279698))) + assigns(memory_slice(w, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(w, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7 +#define mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm \ + MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) +void mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm( + int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], + const int32_t v[7][MLDSA_N]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/riscv64/proofs/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.ml. + */ +__contract__( + requires(memory_no_alias(w, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(u, sizeof(int32_t) * 7 * MLDSA_N)) + requires(memory_no_alias(v, sizeof(int32_t) * 7 * MLDSA_N)) + requires(forall(l0, 0, 7, array_bound(u[l0], 0, MLDSA_N, 0, MLDSA_Q))) + /* check-magic: 94279698 == 9 * ((5 * MLDSA_Q + 3) / 4) */ + requires(forall(l1, 0, 7, array_abs_bound(v[l1], 0, MLDSA_N, 94279698))) + assigns(memory_slice(w, sizeof(int32_t) * MLDSA_N)) + ensures(array_abs_bound(w, 0, MLDSA_N, MLDSA_Q)) +); +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7 */ + +#endif /* !MLD_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H */ diff --git a/mldsa/src/native/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S new file mode 100644 index 0000000000..4c0ad01aa0 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S @@ -0,0 +1,511 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: intt_vlen128_riscv64_asm + Description: Inverse negacyclic NTT of a polynomial in place, input in bitreversed order, scaled by 2^32 (VLEN=128 only) + Signature: void mld_intt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_intt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(intt_vlen128_riscv64_asm) +MLD_ASM_FN_SYMBOL(intt_vlen128_riscv64_asm) + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x4 + li a3, 0x8 + li a4, 0x10 + li a5, 0x20 + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x4 + +mld_intt_vlen128_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vv v17, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v17, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vv v19, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v19, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vv v21, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v21, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vv v23, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v23, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vv v25, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v25, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vv v27, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v27, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vv v29, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v29, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vv v31, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v31, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vv v18, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v18, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vv v22, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v22, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vv v26, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v26, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vv v30, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v30, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vv v20, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v20, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vv v24, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v24, t0, v0 + vsub.vv v0, v20, v28 + vadd.vv v20, v20, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x100 + addi a7, a7, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen128_riscv64_pass_b + mv a6, a0 + li t5, 0x4 + +mld_intt_vlen128_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vx v17, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v17, t0, v0 + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vx v19, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v19, t0, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vx v21, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v21, t0, v0 + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vx v23, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v23, t0, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vx v25, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v25, t0, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vx v27, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v27, t0, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vx v29, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v29, t0, v0 + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vx v31, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v31, t0, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vx v18, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v18, t0, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vx v22, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v22, t0, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vx v26, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v26, t0, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vx v30, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v30, t0, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vx v20, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v20, t0, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vx v28, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v28, t0, v0 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vx v24, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v24, t0, v0 + lw t1, 0x78(a1) + lw t2, 0x7c(a1) + vsmul.vx v0, v16, t2 + vmul.vx v16, v16, t1 + vnmsac.vx v16, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen128_riscv64_pass_a + ret + +.option pop +MLD_ASM_FN_SIZE(intt_vlen128_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S new file mode 100644 index 0000000000..7639e569d9 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S @@ -0,0 +1,511 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: intt_vlen256_riscv64_asm + Description: Inverse negacyclic NTT of a polynomial in place, input in bitreversed order, scaled by 2^32 (VLEN=256 only) + Signature: void mld_intt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_intt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(intt_vlen256_riscv64_asm) +MLD_ASM_FN_SYMBOL(intt_vlen256_riscv64_asm) + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x8 + li a3, 0x10 + li a4, 0x20 + li a5, 0x40 + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x2 + +mld_intt_vlen256_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vv v17, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v17, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vv v19, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v19, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vv v21, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v21, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vv v23, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v23, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vv v25, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v25, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vv v27, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v27, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vv v29, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v29, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vv v31, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v31, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vv v18, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v18, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vv v22, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v22, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vv v26, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v26, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vv v30, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v30, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vv v20, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v20, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vv v24, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v24, t0, v0 + vsub.vv v0, v20, v28 + vadd.vv v20, v20, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x200 + addi a7, a7, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen256_riscv64_pass_b + mv a6, a0 + li t5, 0x2 + +mld_intt_vlen256_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vx v17, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v17, t0, v0 + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vx v19, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v19, t0, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vx v21, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v21, t0, v0 + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vx v23, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v23, t0, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vx v25, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v25, t0, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vx v27, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v27, t0, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vx v29, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v29, t0, v0 + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vx v31, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v31, t0, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vx v18, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v18, t0, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vx v22, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v22, t0, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vx v26, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v26, t0, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vx v30, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v30, t0, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vx v20, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v20, t0, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vx v28, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v28, t0, v0 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vx v24, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v24, t0, v0 + lw t1, 0x78(a1) + lw t2, 0x7c(a1) + vsmul.vx v0, v16, t2 + vmul.vx v16, v16, t1 + vnmsac.vx v16, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen256_riscv64_pass_a + ret + +.option pop +MLD_ASM_FN_SIZE(intt_vlen256_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S new file mode 100644 index 0000000000..66132244e8 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S @@ -0,0 +1,506 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: ntt_vlen128_riscv64_asm + Description: Forward negacyclic NTT of a polynomial in place, output in bitreversed order (VLEN=128 only) + Signature: void mld_ntt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_ntt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(ntt_vlen128_riscv64_asm) +MLD_ASM_FN_SYMBOL(ntt_vlen128_riscv64_asm) + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x4 + li a3, 0x8 + li a4, 0x10 + li a5, 0x20 + mv a6, a0 + li t5, 0x4 + +mld_ntt_vlen128_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vmul.vx v0, v24, t1 + vsmul.vx v24, v24, t2 + vnmsac.vx v0, t0, v24 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vmul.vx v0, v20, t1 + vsmul.vx v20, v20, t2 + vnmsac.vx v0, t0, v20 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vmul.vx v0, v28, t1 + vsmul.vx v28, v28, t2 + vnmsac.vx v0, t0, v28 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vmul.vx v0, v18, t1 + vsmul.vx v18, v18, t2 + vnmsac.vx v0, t0, v18 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vmul.vx v0, v22, t1 + vsmul.vx v22, v22, t2 + vnmsac.vx v0, t0, v22 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vmul.vx v0, v26, t1 + vsmul.vx v26, v26, t2 + vnmsac.vx v0, t0, v26 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vmul.vx v0, v30, t1 + vsmul.vx v30, v30, t2 + vnmsac.vx v0, t0, v30 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vmul.vx v0, v17, t1 + vsmul.vx v17, v17, t2 + vnmsac.vx v0, t0, v17 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vmul.vx v0, v19, t1 + vsmul.vx v19, v19, t2 + vnmsac.vx v0, t0, v19 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vmul.vx v0, v21, t1 + vsmul.vx v21, v21, t2 + vnmsac.vx v0, t0, v21 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vmul.vx v0, v23, t1 + vsmul.vx v23, v23, t2 + vnmsac.vx v0, t0, v23 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vmul.vx v0, v25, t1 + vsmul.vx v25, v25, t2 + vnmsac.vx v0, t0, v25 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vmul.vx v0, v27, t1 + vsmul.vx v27, v27, t2 + vnmsac.vx v0, t0, v27 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vmul.vx v0, v29, t1 + vsmul.vx v29, v29, t2 + vnmsac.vx v0, t0, v29 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vmul.vx v0, v31, t1 + vsmul.vx v31, v31, t2 + vnmsac.vx v0, t0, v31 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen128_riscv64_pass_a + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x4 + +mld_ntt_vlen128_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v24, v8 + vsmul.vv v4, v24, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v20, v8 + vsmul.vv v4, v20, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v18, v8 + vsmul.vv v4, v18, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v22, v8 + vsmul.vv v4, v22, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v26, v8 + vsmul.vv v4, v26, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v30, v8 + vsmul.vv v4, v30, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v17, v8 + vsmul.vv v4, v17, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v19, v8 + vsmul.vv v4, v19, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v21, v8 + vsmul.vv v4, v21, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v23, v8 + vsmul.vv v4, v23, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v25, v8 + vsmul.vv v4, v25, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v27, v8 + vsmul.vv v4, v27, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v29, v8 + vsmul.vv v4, v29, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v31, v8 + vsmul.vv v4, v31, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x100 + addi a7, a7, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen128_riscv64_pass_b + ret + +.option pop +MLD_ASM_FN_SIZE(ntt_vlen128_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S new file mode 100644 index 0000000000..f00462c42d --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S @@ -0,0 +1,506 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: ntt_vlen256_riscv64_asm + Description: Forward negacyclic NTT of a polynomial in place, output in bitreversed order (VLEN=256 only) + Signature: void mld_ntt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_ntt_zetas) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(ntt_vlen256_riscv64_asm) +MLD_ASM_FN_SYMBOL(ntt_vlen256_riscv64_asm) + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x8 + li a3, 0x10 + li a4, 0x20 + li a5, 0x40 + mv a6, a0 + li t5, 0x2 + +mld_ntt_vlen256_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vmul.vx v0, v24, t1 + vsmul.vx v24, v24, t2 + vnmsac.vx v0, t0, v24 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vmul.vx v0, v20, t1 + vsmul.vx v20, v20, t2 + vnmsac.vx v0, t0, v20 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vmul.vx v0, v28, t1 + vsmul.vx v28, v28, t2 + vnmsac.vx v0, t0, v28 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vmul.vx v0, v18, t1 + vsmul.vx v18, v18, t2 + vnmsac.vx v0, t0, v18 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vmul.vx v0, v22, t1 + vsmul.vx v22, v22, t2 + vnmsac.vx v0, t0, v22 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vmul.vx v0, v26, t1 + vsmul.vx v26, v26, t2 + vnmsac.vx v0, t0, v26 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vmul.vx v0, v30, t1 + vsmul.vx v30, v30, t2 + vnmsac.vx v0, t0, v30 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vmul.vx v0, v17, t1 + vsmul.vx v17, v17, t2 + vnmsac.vx v0, t0, v17 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vmul.vx v0, v19, t1 + vsmul.vx v19, v19, t2 + vnmsac.vx v0, t0, v19 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vmul.vx v0, v21, t1 + vsmul.vx v21, v21, t2 + vnmsac.vx v0, t0, v21 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vmul.vx v0, v23, t1 + vsmul.vx v23, v23, t2 + vnmsac.vx v0, t0, v23 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vmul.vx v0, v25, t1 + vsmul.vx v25, v25, t2 + vnmsac.vx v0, t0, v25 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vmul.vx v0, v27, t1 + vsmul.vx v27, v27, t2 + vnmsac.vx v0, t0, v27 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vmul.vx v0, v29, t1 + vsmul.vx v29, v29, t2 + vnmsac.vx v0, t0, v29 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vmul.vx v0, v31, t1 + vsmul.vx v31, v31, t2 + vnmsac.vx v0, t0, v31 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen256_riscv64_pass_a + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x2 + +mld_ntt_vlen256_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v24, v8 + vsmul.vv v4, v24, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v20, v8 + vsmul.vv v4, v20, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v18, v8 + vsmul.vv v4, v18, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v22, v8 + vsmul.vv v4, v22, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v26, v8 + vsmul.vv v4, v26, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v30, v8 + vsmul.vv v4, v30, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v17, v8 + vsmul.vv v4, v17, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v19, v8 + vsmul.vv v4, v19, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v21, v8 + vsmul.vv v4, v21, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v23, v8 + vsmul.vv v4, v23, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v25, v8 + vsmul.vv v4, v25, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v27, v8 + vsmul.vv v4, v27, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v29, v8 + vsmul.vv v4, v29, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v31, v8 + vsmul.vv v4, v31, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x200 + addi a7, a7, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen256_riscv64_pass_b + ret + +.option pop +MLD_ASM_FN_SIZE(ntt_vlen256_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_caddq_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_caddq_riscv64_asm.S new file mode 100644 index 0000000000..94df851137 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_caddq_riscv64_asm.S @@ -0,0 +1,64 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_caddq_riscv64_asm + Description: Add q to all negative coefficients of a polynomial + Signature: void mld_poly_caddq_riscv64_asm(int32_t a[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_caddq_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_caddq_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_caddq_riscv64_asm) + + lui t1, 0x7fe + addiw t1, t1, 0x1 + li a1, 0x100 + vsetvli t0, a1, e32, m4, ta, ma + slli t2, t0, 0x2 + +mld_poly_caddq_riscv64_loop: + vle32.v v0, (a0) + vsra.vi v4, v0, 0x1f + vand.vx v4, v4, t1 + vadd.vv v0, v0, v4 + vse32.v v0, (a0) + add a0, a0, t2 + sub a1, a1, t0 + bnez a1, mld_poly_caddq_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(poly_caddq_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S new file mode 100644 index 0000000000..cac6b9c3a9 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S @@ -0,0 +1,72 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_chknorm_riscv64_asm + Description: Infinity-norm bound check on polynomial coefficients + Signature: int mld_poly_chknorm_riscv64_asm(const int32_t a[256], int64_t B) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t a[256] + description: Input polynomial + a1: + type: scalar + c_parameter: int64_t B + description: Norm bound + test_with: 131072 # representative non-negative bound (1 << 17) +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_chknorm_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_chknorm_riscv64_asm) + + li a2, 0x100 + vsetvli t0, a2, e32, m4, ta, ma + slli t1, t0, 0x2 + vmv.v.i v8, 0x0 + +mld_poly_chknorm_riscv64_loop: + vle32.v v0, (a0) + vrsub.vi v4, v0, 0x0 + vmax.vv v0, v0, v4 + vmaxu.vv v8, v8, v0 + add a0, a0, t1 + sub a2, a2, t0 + bnez a2, mld_poly_chknorm_riscv64_loop + vmv.s.x v12, zero + vredmaxu.vs v12, v8, v12 + vmv.x.s t2, v12 + slt a0, t2, a1 + xori a0, a0, 0x1 + ret + +.option pop +MLD_ASM_FN_SIZE(poly_chknorm_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S new file mode 100644 index 0000000000..c1cfa7e5dc --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S @@ -0,0 +1,90 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_decompose_32_riscv64_asm + Description: Decompose polynomial coefficients into high and low bits for GAMMA2 = (q-1)/32 + Signature: void mld_poly_decompose_32_riscv64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output polynomial with high bits + a1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial, output polynomial with low bits +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_SIGN_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_decompose_32_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_decompose_32_riscv64_asm) + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x80 + addiw t5, t5, -0x200 + li t6, 0x401 + lui a3, 0x200 + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_decompose_32_riscv64_loop: + vle32.v v8, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x16 + vand.vi v0, v0, 0xf + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vse32.v v0, (a0) + vse32.v v8, (a1) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_decompose_32_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(poly_decompose_32_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_SIGN_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S new file mode 100644 index 0000000000..a22cb67bbc --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S @@ -0,0 +1,95 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_decompose_88_riscv64_asm + Description: Decompose polynomial coefficients into high and low bits for GAMMA2 = (q-1)/88 + Signature: void mld_poly_decompose_88_riscv64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output polynomial with high bits + a1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial, output polynomial with low bits +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_SIGN_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_decompose_88_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_decompose_88_riscv64_asm) + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x2f + addiw t5, t5, -0x800 + lui t6, 0x3 + addiw t6, t6, -0x3f5 + lui a3, 0x800 + li a4, 0x2b + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_decompose_88_riscv64_loop: + vle32.v v8, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x18 + vrsub.vx v16, v0, a4 + vsra.vi v16, v16, 0x1f + vand.vv v16, v16, v0 + vxor.vv v0, v0, v16 + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vse32.v v0, (a0) + vse32.v v8, (a1) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_decompose_88_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(poly_decompose_88_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_SIGN_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 44) && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S new file mode 100644 index 0000000000..be596c4e64 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S @@ -0,0 +1,80 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_pointwise_montgomery_riscv64_asm + Description: Pointwise Montgomery multiplication of two polynomials + Signature: void mld_poly_pointwise_montgomery_riscv64_asm(int32_t a[256], const int32_t b[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t b[256] + description: Input polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_pointwise_montgomery_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_pointwise_montgomery_riscv64_asm) + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t3, t0, 0x2 + +mld_poly_pointwise_montgomery_riscv64_loop: + vle32.v v0, (a0) + vle32.v v8, (a1) + vmul.vv v16, v0, v8 + vmulh.vv v24, v0, v8 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v0, v24, v16 + vse32.v v0, (a0) + add a0, a0, t3 + add a1, a1, t3 + sub a2, a2, t0 + bnez a2, mld_poly_pointwise_montgomery_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(poly_pointwise_montgomery_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (!MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S new file mode 100644 index 0000000000..9f2eb06857 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S @@ -0,0 +1,96 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_use_hint_32_riscv64_asm + Description: Use hint polynomial to correct the high bits of a polynomial for GAMMA2 = (q-1)/32 + Signature: void mld_poly_use_hint_32_riscv64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_VERIFY_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_use_hint_32_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_use_hint_32_riscv64_asm) + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x80 + addiw t5, t5, -0x200 + li t6, 0x401 + lui a3, 0x200 + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_use_hint_32_riscv64_loop: + vle32.v v8, (a0) + vle32.v v24, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x16 + vand.vi v0, v0, 0xf + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vadd.vi v8, v8, -0x1 + vsra.vi v8, v8, 0x1f + vxor.vv v16, v24, v8 + vsub.vv v16, v16, v8 + vadd.vv v0, v0, v16 + vand.vi v0, v0, 0xf + vse32.v v0, (a0) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_use_hint_32_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(poly_use_hint_32_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_VERIFY_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S new file mode 100644 index 0000000000..12869dfa82 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S @@ -0,0 +1,108 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_use_hint_88_riscv64_asm + Description: Use hint polynomial to correct the high bits of a polynomial for GAMMA2 = (q-1)/88 + Signature: void mld_poly_use_hint_88_riscv64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_VERIFY_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(poly_use_hint_88_riscv64_asm) +MLD_ASM_FN_SYMBOL(poly_use_hint_88_riscv64_asm) + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x2f + addiw t5, t5, -0x800 + lui t6, 0x3 + addiw t6, t6, -0x3f5 + lui a3, 0x800 + li a4, 0x2b + li a5, 0x2c + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_use_hint_88_riscv64_loop: + vle32.v v8, (a0) + vle32.v v24, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x18 + vrsub.vx v16, v0, a4 + vsra.vi v16, v16, 0x1f + vand.vv v16, v16, v0 + vxor.vv v0, v0, v16 + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vadd.vi v8, v8, -0x1 + vsra.vi v8, v8, 0x1f + vxor.vv v16, v24, v8 + vsub.vv v16, v16, v8 + vadd.vv v0, v0, v16 + vsra.vi v16, v0, 0x1f + vand.vx v16, v16, a5 + vadd.vv v0, v0, v16 + vrsub.vx v16, v0, a4 + vsra.vi v16, v16, 0x1f + vand.vv v16, v16, v0 + vxor.vv v0, v0, v16 + vse32.v v0, (a0) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_use_hint_88_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(poly_use_hint_88_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_VERIFY_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLD_CONFIG_PARAMETER_SET == 44) && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S new file mode 100644 index 0000000000..0584532c38 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S @@ -0,0 +1,105 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l4_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 4 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(int32_t w[256], const int32_t u[4][256], const int32_t v[4][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t u[4][256] + description: First input vector + a2: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t v[4][256] + description: Second input vector +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a3, 0x100 + li t4, 0x20 + li a6, 0x400 + vsetvli t0, a3, e32, m4, ta, ma + slli t3, t0, 0x2 + +mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_loop: + vle32.v v0, (a1) + vle32.v v4, (a2) + vwmul.vv v8, v0, v4 + mv a4, a1 + mv a5, a2 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + vnsrl.wi v16, v8, 0x0 + vnsra.wx v20, v8, t4 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v16, v20, v16 + vse32.v v16, (a0) + add a0, a0, t3 + add a1, a1, t3 + add a2, a2, t3 + sub a3, a3, t0 + bnez a3, mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(polyvecl_pointwise_acc_montgomery_l4_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S new file mode 100644 index 0000000000..7fa8ef6322 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S @@ -0,0 +1,110 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l5_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 5 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(int32_t w[256], const int32_t u[5][256], const int32_t v[5][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t u[5][256] + description: First input vector + a2: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t v[5][256] + description: Second input vector +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a3, 0x100 + li t4, 0x20 + li a6, 0x400 + vsetvli t0, a3, e32, m4, ta, ma + slli t3, t0, 0x2 + +mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_loop: + vle32.v v0, (a1) + vle32.v v4, (a2) + vwmul.vv v8, v0, v4 + mv a4, a1 + mv a5, a2 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + vnsrl.wi v16, v8, 0x0 + vnsra.wx v20, v8, t4 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v16, v20, v16 + vse32.v v16, (a0) + add a0, a0, t3 + add a1, a1, t3 + add a2, a2, t3 + sub a3, a3, t0 + bnez a3, mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(polyvecl_pointwise_acc_montgomery_l5_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S new file mode 100644 index 0000000000..ce71a9ffe7 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S @@ -0,0 +1,120 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l7_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 7 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(int32_t w[256], const int32_t u[7][256], const int32_t v[7][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t u[7][256] + description: First input vector + a2: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t v[7][256] + description: Second input vector +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a3, 0x100 + li t4, 0x20 + li a6, 0x400 + vsetvli t0, a3, e32, m4, ta, ma + slli t3, t0, 0x2 + +mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_loop: + vle32.v v0, (a1) + vle32.v v4, (a2) + vwmul.vv v8, v0, v4 + mv a4, a1 + mv a5, a2 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + vnsrl.wi v16, v8, 0x0 + vnsra.wx v20, v8, t4 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v16, v20, v16 + vse32.v v16, (a0) + add a0, a0, t3 + add a1, a1, t3 + add a2, a2, t3 + sub a3, a3, t0 + bnez a3, mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(polyvecl_pointwise_acc_montgomery_l7_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S new file mode 100644 index 0000000000..5a454730de --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S @@ -0,0 +1,98 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyz_unpack_17_riscv64_asm + Description: Unpack polynomial z with 18-bit packed coefficients for GAMMA1 = 2^17 + Signature: void mld_polyz_unpack_17_riscv64_asm(int32_t r[256], const uint8_t buf[576]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 576 + permissions: read-only + c_parameter: const uint8_t buf[576] + description: Packed input bytes +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyz_unpack_17_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyz_unpack_17_riscv64_asm) + + li a2, 0x100 + li a5, 0x2 + lui a6, 0x20 + vsetvli t0, a2, e32, m2, ta, ma + slli t3, t0, 0x2 + slli a3, t0, 0x3 + add a3, a3, t0 + srli a3, a3, 0x2 + slli a4, t0, 0x2 + li t2, 0x12 + vsetvli zero, a4, e16, m4, ta, ma + vid.v v8 + vsrl.vi v12, v8, 0x2 + vmul.vx v12, v12, t2 + vsrl.vi v12, v12, 0x3 + vand.vi v8, v8, 0x3 + vminu.vx v8, v8, a5 + vadd.vv v8, v8, v12 + vsetvli zero, t0, e32, m2, ta, ma + vid.v v4 + vmul.vx v4, v4, t2 + vand.vi v4, v4, 0x7 + lui t1, 0x40 + addiw t1, t1, -0x1 + +mld_polyz_unpack_17_riscv64_loop: + vsetvli zero, a3, e8, m2, ta, ma + vle8.v v16, (a1) + vsetvli zero, a4, e8, m2, ta, ma + vrgatherei16.vv v20, v16, v8 + vsetvli zero, t0, e32, m2, ta, ma + vsrl.vv v20, v20, v4 + vand.vx v20, v20, t1 + vrsub.vx v20, v20, a6 + vse32.v v20, (a0) + add a1, a1, a3 + add a0, a0, t3 + sub a2, a2, t0 + bnez a2, mld_polyz_unpack_17_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(polyz_unpack_17_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (!MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API) && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44) \ + && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S new file mode 100644 index 0000000000..e34b0daca7 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S @@ -0,0 +1,99 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyz_unpack_19_riscv64_asm + Description: Unpack polynomial z with 20-bit packed coefficients for GAMMA1 = 2^19 + Signature: void mld_polyz_unpack_19_riscv64_asm(int32_t r[256], const uint8_t buf[640]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 640 + permissions: read-only + c_parameter: const uint8_t buf[640] + description: Packed input bytes +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + (!defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(polyz_unpack_19_riscv64_asm) +MLD_ASM_FN_SYMBOL(polyz_unpack_19_riscv64_asm) + + li a2, 0x100 + li a5, 0x2 + lui a6, 0x80 + vsetvli t0, a2, e32, m2, ta, ma + slli t3, t0, 0x2 + slli a3, t0, 0x2 + add a3, a3, t0 + srli a3, a3, 0x1 + slli a4, t0, 0x2 + li t2, 0x14 + vsetvli zero, a4, e16, m4, ta, ma + vid.v v8 + vsrl.vi v12, v8, 0x2 + vmul.vx v12, v12, t2 + vsrl.vi v12, v12, 0x3 + vand.vi v8, v8, 0x3 + vminu.vx v8, v8, a5 + vadd.vv v8, v8, v12 + vsetvli zero, t0, e32, m2, ta, ma + vid.v v4 + vmul.vx v4, v4, t2 + vand.vi v4, v4, 0x7 + lui t1, 0x100 + addiw t1, t1, -0x1 + +mld_polyz_unpack_19_riscv64_loop: + vsetvli zero, a3, e8, m2, ta, ma + vle8.v v16, (a1) + vsetvli zero, a4, e8, m2, ta, ma + vrgatherei16.vv v20, v16, v8 + vsetvli zero, t0, e32, m2, ta, ma + vsrl.vv v20, v20, v4 + vand.vx v20, v20, t1 + vrsub.vx v20, v20, a6 + vse32.v v20, (a0) + add a1, a1, a3 + add a0, a0, t3 + sub a2, a2, t0 + bnez a2, mld_polyz_unpack_19_riscv64_loop + ret + +.option pop +MLD_ASM_FN_SIZE(polyz_unpack_19_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + (!MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API) && \ + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87) && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S new file mode 100644 index 0000000000..a6f99b80c0 --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S @@ -0,0 +1,114 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_eta2_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers in [-2, +2] + Signature: uint64_t mld_rej_uniform_eta2_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer + test_with: 272 # 2 * SHAKE256_RATE +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_KEYPAIR_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(rej_uniform_eta2_riscv64_asm) +MLD_ASM_FN_SYMBOL(rej_uniform_eta2_riscv64_asm) + + li a4, 0x0 + li t2, 0x34 + li t3, 0x5 + vsetvli a5, zero, e16, m2, ta, ma + +mld_rej_uniform_eta2_riscv64_loop: + bgeu a4, a1, mld_rej_uniform_eta2_riscv64_done + beqz a3, mld_rej_uniform_eta2_riscv64_done + mv a6, a5 + bgeu a3, a5, mld_rej_uniform_eta2_riscv64_load + mv a6, a3 + +mld_rej_uniform_eta2_riscv64_load: + vsetvli zero, a6, e16, m2, ta, ma + vle8.v v1, (a2) + add a2, a2, a6 + sub a3, a3, a6 + vzext.vf2 v2, v1 + vsrl.vi v4, v2, 0x4 + vsll.vi v4, v4, 0x8 + vand.vi v2, v2, 0xf + vor.vv v2, v2, v4 + slli t0, a6, 0x1 + vsetvli zero, t0, e8, m2, ta, ma + vmsleu.vi v6, v2, 0xe + vmulhu.vx v12, v2, t2 + vmul.vx v12, v12, t3 + vsub.vv v2, v2, v12 + vrsub.vi v2, v2, 0x2 + vcpop.m t1, v6 + vcompress.vm v14, v2, v6 + sub a7, a1, a4 + bgeu a7, t1, mld_rej_uniform_eta2_riscv64_store + mv t1, a7 + +mld_rej_uniform_eta2_riscv64_store: + vsetvli zero, t1, e32, m8, ta, ma + vsext.vf4 v16, v14 + vse32.v v16, (a0) + add a4, a4, t1 + slli t1, t1, 0x2 + add a0, a0, t1 + j mld_rej_uniform_eta2_riscv64_loop + +mld_rej_uniform_eta2_riscv64_done: + mv a0, a4 + ret + +.option pop +MLD_ASM_FN_SIZE(rej_uniform_eta2_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_KEYPAIR_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLDSA_ETA == 2) && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S new file mode 100644 index 0000000000..df0bb1d11a --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S @@ -0,0 +1,109 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_eta4_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers in [-4, +4] + Signature: uint64_t mld_rej_uniform_eta4_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer + test_with: 272 # 2 * SHAKE256_RATE +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_NO_KEYPAIR_API) && \ + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(rej_uniform_eta4_riscv64_asm) +MLD_ASM_FN_SYMBOL(rej_uniform_eta4_riscv64_asm) + + li a4, 0x0 + vsetvli a5, zero, e16, m2, ta, ma + +mld_rej_uniform_eta4_riscv64_loop: + bgeu a4, a1, mld_rej_uniform_eta4_riscv64_done + beqz a3, mld_rej_uniform_eta4_riscv64_done + mv a6, a5 + bgeu a3, a5, mld_rej_uniform_eta4_riscv64_load + mv a6, a3 + +mld_rej_uniform_eta4_riscv64_load: + vsetvli zero, a6, e16, m2, ta, ma + vle8.v v1, (a2) + add a2, a2, a6 + sub a3, a3, a6 + vzext.vf2 v2, v1 + vsrl.vi v4, v2, 0x4 + vsll.vi v4, v4, 0x8 + vand.vi v2, v2, 0xf + vor.vv v2, v2, v4 + slli t0, a6, 0x1 + vsetvli zero, t0, e8, m2, ta, ma + vmsleu.vi v6, v2, 0x8 + vrsub.vi v2, v2, 0x4 + vcpop.m t1, v6 + vcompress.vm v14, v2, v6 + sub a7, a1, a4 + bgeu a7, t1, mld_rej_uniform_eta4_riscv64_store + mv t1, a7 + +mld_rej_uniform_eta4_riscv64_store: + vsetvli zero, t1, e32, m8, ta, ma + vsext.vf4 v16, v14 + vse32.v v16, (a0) + add a4, a4, t1 + slli t1, t1, 0x2 + add a0, a0, t1 + j mld_rej_uniform_eta4_riscv64_loop + +mld_rej_uniform_eta4_riscv64_done: + mv a0, a4 + ret + +.option pop +MLD_ASM_FN_SIZE(rej_uniform_eta4_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_NO_KEYPAIR_API && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLDSA_ETA == 4) && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/mldsa_rej_uniform_riscv64_asm.S b/mldsa/src/native/riscv64/src/mldsa_rej_uniform_riscv64_asm.S new file mode 100644 index 0000000000..ca7437516d --- /dev/null +++ b/mldsa/src/native/riscv64/src/mldsa_rej_uniform_riscv64_asm.S @@ -0,0 +1,127 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q + Signature: uint64_t mld_rej_uniform_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer (must be a multiple of 3) + test_with: 840 # MLD_POLY_UNIFORM_NBLOCKS * SHAKE128_RATE = 5 * 168 +*/ + +#include "../../../common.h" +#if defined(MLD_SYS_RISCV64_RVV) && defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_rej_uniform_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +.global MLD_ASM_NAMESPACE(rej_uniform_riscv64_asm) +MLD_ASM_FN_SYMBOL(rej_uniform_riscv64_asm) + + li a4, 0x0 + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t5, 0x2 + vsetvli t0, zero, e32, m2, ta, ma + slli a6, t0, 0x2 + li t4, 0x18 + vsetvli zero, a6, e16, m4, ta, ma + vid.v v8 + vsrl.vi v12, v8, 0x2 + vmul.vx v12, v12, t4 + vsrl.vi v12, v12, 0x3 + vand.vi v8, v8, 0x3 + vminu.vx v8, v8, t5 + vadd.vv v8, v8, v12 + lui t3, 0x1000 + addiw t3, t3, -0x1 + lui t3, 0x800 + addiw t3, t3, -0x1 + slli a5, t0, 0x1 + add a5, a5, t0 + +mld_rej_uniform_riscv64_loop: + bgeu a4, a1, mld_rej_uniform_riscv64_done + bltu a3, a5, mld_rej_uniform_riscv64_tail + +mld_rej_uniform_riscv64_body: + vsetvli zero, a5, e8, m2, ta, ma + vle8.v v16, (a2) + vsetvli zero, a6, e8, m2, ta, ma + vrgatherei16.vv v20, v16, v8 + vsetvli zero, t0, e32, m2, ta, ma + vand.vx v20, v20, t3 + add a2, a2, a5 + sub a3, a3, a5 + vmsltu.vx v0, v20, t2 + vcpop.m t1, v0 + vcompress.vm v24, v20, v0 + sub a7, a1, a4 + bgeu a7, t1, mld_rej_uniform_riscv64_store + mv t1, a7 + +mld_rej_uniform_riscv64_store: + vsetvli zero, t1, e32, m2, ta, ma + vse32.v v24, (a0) + add a4, a4, t1 + slli t1, t1, 0x2 + add a0, a0, t1 + j mld_rej_uniform_riscv64_loop + +mld_rej_uniform_riscv64_tail: + beqz a3, mld_rej_uniform_riscv64_done + mv a5, a3 + lui t4, 0xb + addiw t4, t4, -0x555 + mul t0, a3, t4 + srli t0, t0, 0x11 + slli a6, t0, 0x2 + j mld_rej_uniform_riscv64_body + +mld_rej_uniform_riscv64_done: + mv a0, a4 + ret + +.option pop +MLD_ASM_FN_SIZE(rej_uniform_riscv64_asm) + +#endif /* MLD_SYS_RISCV64_RVV && MLD_ARITH_BACKEND_RISCV64 && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/mldsa/src/native/riscv64/src/rv64v_zetas.c b/mldsa/src/native/riscv64/src/rv64v_zetas.c new file mode 100644 index 0000000000..4803582c94 --- /dev/null +++ b/mldsa/src/native/riscv64/src/rv64v_zetas.c @@ -0,0 +1,245 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLD_ARITH_BACKEND_RISCV64) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_riscv64.h" + +/* + * Table of twiddles used in the RISC-V forward NTT + * See autogen for details. + */ +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t + mld_rv64v_ntt_zetas[512] = { + -3572223, -915382907, 3765607, 964937599, 3761513, + 963888510, -3201494, -820383522, -2883726, -738955404, + -3145678, -806080660, -3201430, -820367122, -601683, + -154181397, 3542485, 907762539, 2682288, 687336873, + 2129892, 545785280, 3764867, 964747974, -1005239, + -257592709, 557458, 142848732, -1221177, -312926867, + 0, 0, -3370349, -4063053, 2663378, + -1674615, -3524442, -434125, 676590, -1335936, + -3227876, 1714295, 2453983, 1460718, -642628, + -3585098, 2815639, 2283733, -863652652, -1041158200, + 682491182, -429120452, -903139016, -111244624, 173376332, + -342333886, -827143915, 439288460, 628833668, 374309300, + -164673562, -918682129, 721508096, 585207070, 3602218, + 2740543, -3110818, 3704823, 394148, 1095468, + 2071829, 3241972, 3415069, -817536, 3756790, + -1716988, -2897314, 556856, 2917338, 3345963, + 923069133, 702264730, -797147778, 949361686, 101000509, + 280713909, 530906624, 830756018, 875112161, -209493775, + 962678241, -439978542, -742437332, 142694469, 747568486, + 857403734, 3182878, -3586446, 2101410, 1159875, + 928749, -3506380, -4018989, 2156050, 1759347, + -3574466, -1935799, -3950053, 3192354, 3870317, + 1853806, 1858416, 815613168, -919027554, 538486762, + 297218217, 237992130, -898510625, -1029866791, 552488273, + 450833045, -915957677, -496048908, -1012201926, 818041395, + 991769559, 475038184, 476219497, 3073009, 4183372, + 2508980, 2811291, 1528066, -565603, -4166425, + 1736313, -2579253, 3482206, -1317678, 3901472, + 1356448, -3467665, 348812, -3818627, 787459213, + 1071989969, 642926661, 720393920, 391567239, -144935890, + -1067647297, 444930577, -660934133, 892316032, -337655269, + 999753034, 347590090, -888589898, 89383150, -978523985, + 1277625, -3222807, 2028118, -2983781, 482649, + 169688, -3488383, 235407, 1787943, -4182915, + 2461387, -1226661, -2775755, 2312838, -327848, + -1922253, 327391679, -825844983, 519705671, -764594519, + 123678909, 43482586, -893898890, 60323094, 458160776, + -1071872863, 630730945, -314332144, -711287812, 592665232, + -84011120, -492577742, -2635473, -3121440, 1937570, + -1109516, 1148858, 2462444, 1987814, -3250154, + -2391089, -1300016, 3035980, 2925816, 2683270, + -653275, 1011223, -2236726, -675340520, -799869667, + 496502727, -284313712, 294395108, 631001801, 509377762, + -832852657, -612717067, -333129378, 777970524, 749740976, + 687588511, -167401858, 259126110, -573161516, 3852015, + -274060, -3815725, 4158088, -2962264, -3334383, + -3197248, 3258457, -2254727, -2362063, 621164, + 3374250, -2778788, -459163, -2354215, 1744507, + 987079667, -70227934, -977780347, 1065510939, -759080783, + -854436357, -819295484, 834980303, -577774276, -605279149, + 159173408, 864652284, -712065019, -117660617, -603268097, + 447030292, 1753, 1182243, -27812, -1685153, + -4146264, 3747250, 2998219, -458740, -1623354, + -1476985, -3033742, 3980599, 3994671, 3105558, + -2185084, -303005, 449207, 302950022, -7126831, + -431820817, -1062481036, 960233614, 768294260, -117552223, + -415984810, -378477722, -777397036, 1020029345, 1023635298, + 795799901, -559928242, -77645096, -1935420, 87208, + 822541, -3410568, -1772588, 2296099, 141835, + -1921994, 2105286, 1994046, -338420, 2569011, + -11879, -1103344, 3123762, 3531229, -495951789, + 22347069, 210776307, -873958779, -454226054, 588375860, + 36345249, -492511373, 539479988, 510974714, -86720197, + 658309618, -3043996, -282732136, 800464680, 904878186, + -2659525, 636927, 1009365, 2678278, 2192938, + 1239911, -89301, 4040196, -2374402, 2491325, + 2647994, -1615530, -1370517, 508145, 2358373, + -3974485, -681503850, 163212680, 258649997, 686309310, + 561940831, 317727459, -22883400, 1035301089, -608441020, + 638402564, 678549029, -413979908, -351195274, 130212265, + 604333585, -1018462631, -1455890, -3965306, -2454145, + -3768948, -1727088, -3838479, 2513018, -3472069, + -2033807, -1393159, 3009748, 1723229, 3020393, + -553718, -2193087, -3773731, -373072124, -1016110510, + -628875181, -965793731, -442566669, -983611064, 643961400, + -889718424, -521163479, -356997292, 771248568, 441577800, + 773976352, -141890356, -561979013, -967019376, 2660408, + -3956745, -1979497, -3551006, 2387513, 3195676, + -1354892, 2039144, 586241, 507927, -2612853, + 1665318, 3363542, 860144, -3014420, 1900052, + 681730119, -1013916752, -507246529, -909946047, 611800717, + 818892658, -347191365, 522531086, 150224382, 130156402, + -669544140, 426738094, 861908357, 220412084, -772445769, + 486888731, -1780227, -2296397, 1596822, 635956, + -3611750, 2642980, 613238, -1879878, -1179613, + -1187885, 4148469, 2028038, 214880, 3430436, + -1716814, -781875, -456183549, -588452222, 409185979, + 162963861, -925511710, 677264190, 157142369, -481719139, + -302276083, -304395785, 1063046068, 519685171, 55063046, + 879049958, -439933955, -200355636, -59148, -3284915, + -3956944, -250446, -268456, 1254190, -1310261, + -818761, 527981, -724804, 749577, 1163598, + 545376, 140244, 2926054, 1054478, -15156688, + -841760171, -1013967746, -64176841, -68791907, 321386456, + -335754661, -209807681, 135295244, -185731180, 192079267, + 298172236, 139752717, 35937555, 749801963, 270210213, + 2772600, -3716946, -3759465, -2455377, -3180456, + -12417, -2218467, -2178965, -2743411, -1834526, + -4022750, -3369273, -770441, -1514152, -392707, + -731434, 710479343, -952468207, -963363710, -629190881, + -814992530, -3181859, -568482643, -558360247, -702999655, + -470097680, -1030830548, -863376927, -197425671, -388001774, + -100631253, -187430119, +}; + +/* + * Table of twiddles used in the RISC-V inverse NTT + * See autogen for details. + */ +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t + mld_rv64v_intt_zetas[512] = { + 1221177, 312926867, -557458, -142848732, 1005239, + 257592709, -3764867, -964747974, -2129892, -545785280, + -2682288, -687336873, -3542485, -907762539, 601683, + 154181397, 3201430, 820367122, 3145678, 806080660, + 2883726, 738955404, 3201494, 820383522, -3761513, + -963888510, -3765607, -964937599, -294725, -75523344, + 16382, 4197891, 731434, 392707, 1514152, + 770441, 3369273, 4022750, 1834526, 2743411, + 2178965, 2218467, 12417, 3180456, 2455377, + 3759465, 3716946, -2772600, 187430119, 100631253, + 388001774, 197425671, 863376927, 1030830548, 470097680, + 702999655, 558360247, 568482643, 3181859, 814992530, + 629190881, 963363710, 952468207, -710479343, -1054478, + -2926054, -140244, -545376, -1163598, -749577, + 724804, -527981, 818761, 1310261, -1254190, + 268456, 250446, 3956944, 3284915, 59148, + -270210213, -749801963, -35937555, -139752717, -298172236, + -192079267, 185731180, -135295244, 209807681, 335754661, + -321386456, 68791907, 64176841, 1013967746, 841760171, + 15156688, 781875, 1716814, -3430436, -214880, + -2028038, -4148469, 1187885, 1179613, 1879878, + -613238, -2642980, 3611750, -635956, -1596822, + 2296397, 1780227, 200355636, 439933955, -879049958, + -55063046, -519685171, -1063046068, 304395785, 302276083, + 481719139, -157142369, -677264190, 925511710, -162963861, + -409185979, 588452222, 456183549, -1900052, 3014420, + -860144, -3363542, -1665318, 2612853, -507927, + -586241, -2039144, 1354892, -3195676, -2387513, + 3551006, 1979497, 3956745, -2660408, -486888731, + 772445769, -220412084, -861908357, -426738094, 669544140, + -130156402, -150224382, -522531086, 347191365, -818892658, + -611800717, 909946047, 507246529, 1013916752, -681730119, + 3773731, 2193087, 553718, -3020393, -1723229, + -3009748, 1393159, 2033807, 3472069, -2513018, + 3838479, 1727088, 3768948, 2454145, 3965306, + 1455890, 967019376, 561979013, 141890356, -773976352, + -441577800, -771248568, 356997292, 521163479, 889718424, + -643961400, 983611064, 442566669, 965793731, 628875181, + 1016110510, 373072124, 3974485, -2358373, -508145, + 1370517, 1615530, -2647994, -2491325, 2374402, + -4040196, 89301, -1239911, -2192938, -2678278, + -1009365, -636927, 2659525, 1018462631, -604333585, + -130212265, 351195274, 413979908, -678549029, -638402564, + 608441020, -1035301089, 22883400, -317727459, -561940831, + -686309310, -258649997, -163212680, 681503850, -3531229, + -3123762, 1103344, 11879, -2569011, 338420, + -1994046, -2105286, 1921994, -141835, -2296099, + 1772588, 3410568, -822541, -87208, 1935420, + -904878186, -800464680, 282732136, 3043996, -658309618, + 86720197, -510974714, -539479988, 492511373, -36345249, + -588375860, 454226054, 873958779, -210776307, -22347069, + 495951789, 303005, 2185084, -3105558, -3994671, + -3980599, 3033742, 1476985, 1623354, 458740, + -2998219, -3747250, 4146264, 1685153, 27812, + -1182243, -1753, 77645096, 559928242, -795799901, + -1023635298, -1020029345, 777397036, 378477722, 415984810, + 117552223, -768294260, -960233614, 1062481036, 431820817, + 7126831, -302950022, -449207, -1744507, 2354215, + 459163, 2778788, -3374250, -621164, 2362063, + 2254727, -3258457, 3197248, 3334383, 2962264, + -4158088, 3815725, 274060, -3852015, -447030292, + 603268097, 117660617, 712065019, -864652284, -159173408, + 605279149, 577774276, -834980303, 819295484, 854436357, + 759080783, -1065510939, 977780347, 70227934, -987079667, + 2236726, -1011223, 653275, -2683270, -2925816, + -3035980, 1300016, 2391089, 3250154, -1987814, + -2462444, -1148858, 1109516, -1937570, 3121440, + 2635473, 573161516, -259126110, 167401858, -687588511, + -749740976, -777970524, 333129378, 612717067, 832852657, + -509377762, -631001801, -294395108, 284313712, -496502727, + 799869667, 675340520, 1922253, 327848, -2312838, + 2775755, 1226661, -2461387, 4182915, -1787943, + -235407, 3488383, -169688, -482649, 2983781, + -2028118, 3222807, -1277625, 492577742, 84011120, + -592665232, 711287812, 314332144, -630730945, 1071872863, + -458160776, -60323094, 893898890, -43482586, -123678909, + 764594519, -519705671, 825844983, -327391679, 3818627, + -348812, 3467665, -1356448, -3901472, 1317678, + -3482206, 2579253, -1736313, 4166425, 565603, + -1528066, -2811291, -2508980, -4183372, -3073009, + 978523985, -89383150, 888589898, -347590090, -999753034, + 337655269, -892316032, 660934133, -444930577, 1067647297, + 144935890, -391567239, -720393920, -642926661, -1071989969, + -787459213, -1858416, -1853806, -3870317, -3192354, + 3950053, 1935799, 3574466, -1759347, -2156050, + 4018989, 3506380, -928749, -1159875, -2101410, + 3586446, -3182878, -476219497, -475038184, -991769559, + -818041395, 1012201926, 496048908, 915957677, -450833045, + -552488273, 1029866791, 898510625, -237992130, -297218217, + -538486762, 919027554, -815613168, -3345963, -2917338, + -556856, 2897314, 1716988, -3756790, 817536, + -3415069, -3241972, -2071829, -1095468, -394148, + -3704823, 3110818, -2740543, -3602218, -857403734, + -747568486, -142694469, 742437332, 439978542, -962678241, + 209493775, -875112161, -830756018, -530906624, -280713909, + -101000509, -949361686, 797147778, -702264730, -923069133, + -2283733, -2815639, 3585098, 642628, -1460718, + -2453983, -1714295, 3227876, 1335936, -676590, + 434125, 3524442, 1674615, -2663378, 4063053, + 3370349, -585207070, -721508096, 918682129, 164673562, + -374309300, -628833668, -439288460, 827143915, 342333886, + -173376332, 111244624, 903139016, 429120452, -682491182, + 1041158200, 863652652, +}; + +#else /* MLD_ARITH_BACKEND_RISCV64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */ + +MLD_EMPTY_CU(rv64v_zetas) + +#endif /* !(MLD_ARITH_BACKEND_RISCV64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/proofs/cbmc/intt_native_riscv64/Makefile b/proofs/cbmc/intt_native_riscv64/Makefile new file mode 100644 index 0000000000..f851edd382 --- /dev/null +++ b/proofs/cbmc/intt_native_riscv64/Makefile @@ -0,0 +1,61 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = intt_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = intt_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c $(SRCDIR)/mldsa/src/native/riscv64/src/rv64v_zetas.c + +CHECK_FUNCTION_CONTRACTS=mld_intt_native +USE_FUNCTION_CONTRACTS=mld_intt_vlen256_riscv64_asm +USE_FUNCTION_CONTRACTS+=mld_intt_vlen128_riscv64_asm +USE_FUNCTION_CONTRACTS+=mld_rv64v_vlen +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = intt_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/intt_native_riscv64/intt_native_riscv64_harness.c b/proofs/cbmc/intt_native_riscv64/intt_native_riscv64_harness.c new file mode 100644 index 0000000000..4664df9e69 --- /dev/null +++ b/proofs/cbmc/intt_native_riscv64/intt_native_riscv64_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + + +int mld_intt_native(int32_t data[MLDSA_N]); + +void harness(void) +{ + int32_t *r; + int t; + t = mld_intt_native(r); +} diff --git a/proofs/cbmc/keccak_f1600_x1_native_riscv64/Makefile b/proofs/cbmc/keccak_f1600_x1_native_riscv64/Makefile new file mode 100644 index 0000000000..653492c6a9 --- /dev/null +++ b/proofs/cbmc/keccak_f1600_x1_native_riscv64/Makefile @@ -0,0 +1,57 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = keccak_f1600_x1_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = keccak_f1600_x1_native_riscv64 + +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 -DMLD_CONFIG_FIPS202_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/fips202/native/riscv64/x1_zbb.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/fips202/native/riscv64/src/keccakf1600_round_constants.c + +CHECK_FUNCTION_CONTRACTS=mld_keccak_f1600_x1_native +USE_FUNCTION_CONTRACTS=mld_keccak_f1600_x1_zbb_riscv64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:bitwuzla \ + LP64:z3_no_bv_extract +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = keccak_f1600_x1_native + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/keccak_f1600_x1_native_riscv64/keccak_f1600_x1_native_riscv64_harness.c b/proofs/cbmc/keccak_f1600_x1_native_riscv64/keccak_f1600_x1_native_riscv64_harness.c new file mode 100644 index 0000000000..71a26c00c3 --- /dev/null +++ b/proofs/cbmc/keccak_f1600_x1_native_riscv64/keccak_f1600_x1_native_riscv64_harness.c @@ -0,0 +1,13 @@ +// Copyright (c) The mldsa-native project authors +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 + +#include + +int mld_keccak_f1600_x1_native(uint64_t *state); + +void harness(void) +{ + uint64_t *s; + int t = mld_keccak_f1600_x1_native(s); +} diff --git a/proofs/cbmc/ntt_native_riscv64/Makefile b/proofs/cbmc/ntt_native_riscv64/Makefile new file mode 100644 index 0000000000..e9da16e94e --- /dev/null +++ b/proofs/cbmc/ntt_native_riscv64/Makefile @@ -0,0 +1,61 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = ntt_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = ntt_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c $(SRCDIR)/mldsa/src/native/riscv64/src/rv64v_zetas.c + +CHECK_FUNCTION_CONTRACTS=mld_ntt_native +USE_FUNCTION_CONTRACTS=mld_ntt_vlen256_riscv64_asm +USE_FUNCTION_CONTRACTS+=mld_ntt_vlen128_riscv64_asm +USE_FUNCTION_CONTRACTS+=mld_rv64v_vlen +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = ntt_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/ntt_native_riscv64/ntt_native_riscv64_harness.c b/proofs/cbmc/ntt_native_riscv64/ntt_native_riscv64_harness.c new file mode 100644 index 0000000000..a985c3492d --- /dev/null +++ b/proofs/cbmc/ntt_native_riscv64/ntt_native_riscv64_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + + +int mld_ntt_native(int32_t data[MLDSA_N]); + +void harness(void) +{ + int32_t *r; + int t; + t = mld_ntt_native(r); +} diff --git a/proofs/cbmc/pointwise_acc_native_riscv64/Makefile b/proofs/cbmc/pointwise_acc_native_riscv64/Makefile new file mode 100644 index 0000000000..85376b5b13 --- /dev/null +++ b/proofs/cbmc/pointwise_acc_native_riscv64/Makefile @@ -0,0 +1,54 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = pointwise_acc_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = pointwise_acc_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/polyvec.c + +ifeq ($(MLD_CONFIG_PARAMETER_SET),44) + CHECK_FUNCTION_CONTRACTS=mld_polyvecl_pointwise_acc_montgomery_l4_native + USE_FUNCTION_CONTRACTS=mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),65) + CHECK_FUNCTION_CONTRACTS=mld_polyvecl_pointwise_acc_montgomery_l5_native + USE_FUNCTION_CONTRACTS=mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),87) + CHECK_FUNCTION_CONTRACTS=mld_polyvecl_pointwise_acc_montgomery_l7_native + USE_FUNCTION_CONTRACTS=mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3_smt_only \ + LP64:cadical +CBMC_DEFAULT_CONFIG = LP64:z3_smt_only + +FUNCTION_NAME = pointwise_acc_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +include ../Makefile.common diff --git a/proofs/cbmc/pointwise_acc_native_riscv64/pointwise_acc_native_riscv64_harness.c b/proofs/cbmc/pointwise_acc_native_riscv64/pointwise_acc_native_riscv64_harness.c new file mode 100644 index 0000000000..791d7f97b2 --- /dev/null +++ b/proofs/cbmc/pointwise_acc_native_riscv64/pointwise_acc_native_riscv64_harness.c @@ -0,0 +1,37 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +#if MLDSA_L == 4 +int mld_polyvecl_pointwise_acc_montgomery_l4_native( + int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], + const int32_t v[4][MLDSA_N]); +#elif MLDSA_L == 5 +int mld_polyvecl_pointwise_acc_montgomery_l5_native( + int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], + const int32_t v[5][MLDSA_N]); +#elif MLDSA_L == 7 +int mld_polyvecl_pointwise_acc_montgomery_l7_native( + int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], + const int32_t v[7][MLDSA_N]); +#endif + +void harness(void) +{ + int32_t *w; + int t; + +#if MLDSA_L == 4 + int32_t (*u)[MLDSA_N], (*v)[MLDSA_N]; + t = mld_polyvecl_pointwise_acc_montgomery_l4_native(w, u, v); +#elif MLDSA_L == 5 + int32_t (*u)[MLDSA_N], (*v)[MLDSA_N]; + t = mld_polyvecl_pointwise_acc_montgomery_l5_native(w, u, v); +#elif MLDSA_L == 7 + int32_t (*u)[MLDSA_N], (*v)[MLDSA_N]; + t = mld_polyvecl_pointwise_acc_montgomery_l7_native(w, u, v); +#endif +} diff --git a/proofs/cbmc/pointwise_native_riscv64/Makefile b/proofs/cbmc/pointwise_native_riscv64/Makefile new file mode 100644 index 0000000000..c89dfdbcac --- /dev/null +++ b/proofs/cbmc/pointwise_native_riscv64/Makefile @@ -0,0 +1,46 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = pointwise_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = pointwise_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c + +CHECK_FUNCTION_CONTRACTS=mld_poly_pointwise_montgomery_native +USE_FUNCTION_CONTRACTS=mld_poly_pointwise_montgomery_riscv64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = pointwise_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +include ../Makefile.common diff --git a/proofs/cbmc/pointwise_native_riscv64/pointwise_native_riscv64_harness.c b/proofs/cbmc/pointwise_native_riscv64/pointwise_native_riscv64_harness.c new file mode 100644 index 0000000000..2202f63580 --- /dev/null +++ b/proofs/cbmc/pointwise_native_riscv64/pointwise_native_riscv64_harness.c @@ -0,0 +1,17 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + + +int mld_poly_pointwise_montgomery_native(int32_t a[MLDSA_N], + const int32_t b[MLDSA_N]); + +void harness(void) +{ + int32_t *a, *b; + int t; + t = mld_poly_pointwise_montgomery_native(a, b); +} diff --git a/proofs/cbmc/poly_caddq_native_riscv64/Makefile b/proofs/cbmc/poly_caddq_native_riscv64/Makefile new file mode 100644 index 0000000000..f7fdf7ed79 --- /dev/null +++ b/proofs/cbmc/poly_caddq_native_riscv64/Makefile @@ -0,0 +1,59 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_caddq_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_caddq_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c + +CHECK_FUNCTION_CONTRACTS=mld_poly_caddq_native +USE_FUNCTION_CONTRACTS = mld_poly_caddq_riscv64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:cvc5_mbqi_enum +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_caddq_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_caddq_native_riscv64/poly_caddq_native_riscv64_harness.c b/proofs/cbmc/poly_caddq_native_riscv64/poly_caddq_native_riscv64_harness.c new file mode 100644 index 0000000000..b2ef4d855d --- /dev/null +++ b/proofs/cbmc/poly_caddq_native_riscv64/poly_caddq_native_riscv64_harness.c @@ -0,0 +1,15 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +int mld_poly_caddq_native(int32_t a[MLDSA_N]); + +void harness(void) +{ + int32_t *a; + int t; + t = mld_poly_caddq_native(a); +} diff --git a/proofs/cbmc/poly_chknorm_native_riscv64/Makefile b/proofs/cbmc/poly_chknorm_native_riscv64/Makefile new file mode 100644 index 0000000000..2ad9870674 --- /dev/null +++ b/proofs/cbmc/poly_chknorm_native_riscv64/Makefile @@ -0,0 +1,59 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_chknorm_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_chknorm_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c + +CHECK_FUNCTION_CONTRACTS=mld_poly_chknorm_native +USE_FUNCTION_CONTRACTS = mld_poly_chknorm_riscv64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_chknorm_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_chknorm_native_riscv64/poly_chknorm_native_riscv64_harness.c b/proofs/cbmc/poly_chknorm_native_riscv64/poly_chknorm_native_riscv64_harness.c new file mode 100644 index 0000000000..e7f99663dd --- /dev/null +++ b/proofs/cbmc/poly_chknorm_native_riscv64/poly_chknorm_native_riscv64_harness.c @@ -0,0 +1,16 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +int mld_poly_chknorm_native(const int32_t a[MLDSA_N], int32_t B); + +void harness(void) +{ + const int32_t *a; + int32_t B; + int t; + t = mld_poly_chknorm_native(a, B); +} diff --git a/proofs/cbmc/poly_decompose_32_native_riscv64/Makefile b/proofs/cbmc/poly_decompose_32_native_riscv64/Makefile new file mode 100644 index 0000000000..9491841fea --- /dev/null +++ b/proofs/cbmc/poly_decompose_32_native_riscv64/Makefile @@ -0,0 +1,68 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_decompose_32_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_decompose_32_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c + +# poly_decompose_32 is only used with ML-DSA-65 and ML-DSA-87 +ifeq ($(MLD_CONFIG_PARAMETER_SET),65) +CHECK_FUNCTION_CONTRACTS=mld_poly_decompose_32_native +USE_FUNCTION_CONTRACTS = mld_poly_decompose_32_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),87) +CHECK_FUNCTION_CONTRACTS=mld_poly_decompose_32_native +USE_FUNCTION_CONTRACTS = mld_poly_decompose_32_riscv64_asm +else +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS = +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:cadical +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_decompose_32_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly_kl.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly_kl.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_decompose_32_native_riscv64/poly_decompose_32_native_riscv64_harness.c b/proofs/cbmc/poly_decompose_32_native_riscv64/poly_decompose_32_native_riscv64_harness.c new file mode 100644 index 0000000000..bfbe8f5890 --- /dev/null +++ b/proofs/cbmc/poly_decompose_32_native_riscv64/poly_decompose_32_native_riscv64_harness.c @@ -0,0 +1,19 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0); + +void harness(void) +{ + /* mld_poly_decompose_32_native is only defined for ML-DSA-65 and ML-DSA-87 */ +#if MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87 + int32_t *a1; + int32_t *a0; + int t; + t = mld_poly_decompose_32_native(a1, a0); +#endif /* MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87 */ +} diff --git a/proofs/cbmc/poly_decompose_88_native_riscv64/Makefile b/proofs/cbmc/poly_decompose_88_native_riscv64/Makefile new file mode 100644 index 0000000000..9758d18fbe --- /dev/null +++ b/proofs/cbmc/poly_decompose_88_native_riscv64/Makefile @@ -0,0 +1,65 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_decompose_88_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_decompose_88_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c + +# poly_decompose_88 is only used with ML-DSA-44 +ifeq ($(MLD_CONFIG_PARAMETER_SET),44) +CHECK_FUNCTION_CONTRACTS=mld_poly_decompose_88_native +USE_FUNCTION_CONTRACTS = mld_poly_decompose_88_riscv64_asm +else +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS = +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_decompose_88_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly_kl.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly_kl.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/poly_decompose_88_native_riscv64/poly_decompose_88_native_riscv64_harness.c b/proofs/cbmc/poly_decompose_88_native_riscv64/poly_decompose_88_native_riscv64_harness.c new file mode 100644 index 0000000000..36951d079d --- /dev/null +++ b/proofs/cbmc/poly_decompose_88_native_riscv64/poly_decompose_88_native_riscv64_harness.c @@ -0,0 +1,19 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0); + +void harness(void) +{ + /* mld_poly_decompose_88_native is only defined for ML-DSA-44 */ +#if MLD_CONFIG_PARAMETER_SET == 44 + int32_t *a1; + int32_t *a0; + int t; + t = mld_poly_decompose_88_native(a1, a0); +#endif /* MLD_CONFIG_PARAMETER_SET == 44 */ +} diff --git a/proofs/cbmc/poly_use_hint_native_riscv64/Makefile b/proofs/cbmc/poly_use_hint_native_riscv64/Makefile new file mode 100644 index 0000000000..238b83a93b --- /dev/null +++ b/proofs/cbmc/poly_use_hint_native_riscv64/Makefile @@ -0,0 +1,54 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = poly_use_hint_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = poly_use_hint_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c + +ifeq ($(MLD_CONFIG_PARAMETER_SET),44) + CHECK_FUNCTION_CONTRACTS=mld_poly_use_hint_88_native + USE_FUNCTION_CONTRACTS=mld_poly_use_hint_88_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),65) + CHECK_FUNCTION_CONTRACTS=mld_poly_use_hint_32_native + USE_FUNCTION_CONTRACTS=mld_poly_use_hint_32_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),87) + CHECK_FUNCTION_CONTRACTS=mld_poly_use_hint_32_native + USE_FUNCTION_CONTRACTS=mld_poly_use_hint_32_riscv64_asm +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = poly_use_hint_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +include ../Makefile.common diff --git a/proofs/cbmc/poly_use_hint_native_riscv64/poly_use_hint_native_riscv64_harness.c b/proofs/cbmc/poly_use_hint_native_riscv64/poly_use_hint_native_riscv64_harness.c new file mode 100644 index 0000000000..bae3059b60 --- /dev/null +++ b/proofs/cbmc/poly_use_hint_native_riscv64/poly_use_hint_native_riscv64_harness.c @@ -0,0 +1,24 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +#if MLDSA_GAMMA2 == ((MLDSA_Q - 1) / 88) +int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h); +#else +int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h); +#endif + +void harness(void) +{ + int32_t *a, *h; + int t; + +#if MLDSA_GAMMA2 == ((MLDSA_Q - 1) / 88) + t = mld_poly_use_hint_88_native(a, h); +#else + t = mld_poly_use_hint_32_native(a, h); +#endif +} diff --git a/proofs/cbmc/polyz_unpack_17_native_riscv64/Makefile b/proofs/cbmc/polyz_unpack_17_native_riscv64/Makefile new file mode 100644 index 0000000000..3a602eb7e9 --- /dev/null +++ b/proofs/cbmc/polyz_unpack_17_native_riscv64/Makefile @@ -0,0 +1,65 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = polyz_unpack_17_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = polyz_unpack_17_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c + +# polyz_unpack_17 is only used with ML-DSA-44 +ifeq ($(MLD_CONFIG_PARAMETER_SET),44) +CHECK_FUNCTION_CONTRACTS=mld_polyz_unpack_17_native +USE_FUNCTION_CONTRACTS = mld_polyz_unpack_17_riscv64_asm +else +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS = +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = polyz_unpack_17_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly_kl.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly_kl.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/polyz_unpack_17_native_riscv64/polyz_unpack_17_native_riscv64_harness.c b/proofs/cbmc/polyz_unpack_17_native_riscv64/polyz_unpack_17_native_riscv64_harness.c new file mode 100644 index 0000000000..ff2983c442 --- /dev/null +++ b/proofs/cbmc/polyz_unpack_17_native_riscv64/polyz_unpack_17_native_riscv64_harness.c @@ -0,0 +1,19 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *a); + +void harness(void) +{ + /* mld_polyz_unpack_17_native is only defined for ML-DSA-44 */ +#if MLD_CONFIG_PARAMETER_SET == 44 + int32_t *r; + const uint8_t *a; + int t; + t = mld_polyz_unpack_17_native(r, a); +#endif /* MLD_CONFIG_PARAMETER_SET == 44 */ +} diff --git a/proofs/cbmc/polyz_unpack_19_native_riscv64/Makefile b/proofs/cbmc/polyz_unpack_19_native_riscv64/Makefile new file mode 100644 index 0000000000..a294a3fa5c --- /dev/null +++ b/proofs/cbmc/polyz_unpack_19_native_riscv64/Makefile @@ -0,0 +1,68 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = polyz_unpack_19_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = polyz_unpack_19_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c + +# polyz_unpack_19 is only used with ML-DSA-65 and ML-DSA-87 +ifeq ($(MLD_CONFIG_PARAMETER_SET),65) +CHECK_FUNCTION_CONTRACTS=mld_polyz_unpack_19_native +USE_FUNCTION_CONTRACTS = mld_polyz_unpack_19_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),87) +CHECK_FUNCTION_CONTRACTS=mld_polyz_unpack_19_native +USE_FUNCTION_CONTRACTS = mld_polyz_unpack_19_riscv64_asm +else +CHECK_FUNCTION_CONTRACTS= +USE_FUNCTION_CONTRACTS = +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:z3 \ + LP64:bitwuzla +CBMC_DEFAULT_CONFIG = LP64:z3 + +FUNCTION_NAME = polyz_unpack_19_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/poly_kl.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly_kl.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/polyz_unpack_19_native_riscv64/polyz_unpack_19_native_riscv64_harness.c b/proofs/cbmc/polyz_unpack_19_native_riscv64/polyz_unpack_19_native_riscv64_harness.c new file mode 100644 index 0000000000..f4cfecced4 --- /dev/null +++ b/proofs/cbmc/polyz_unpack_19_native_riscv64/polyz_unpack_19_native_riscv64_harness.c @@ -0,0 +1,19 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *a); + +void harness(void) +{ + /* mld_polyz_unpack_19_native is only defined for ML-DSA-65 and ML-DSA-87 */ +#if MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87 + int32_t *r; + const uint8_t *a; + int t; + t = mld_polyz_unpack_19_native(r, a); +#endif /* MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87 */ +} diff --git a/proofs/cbmc/rej_uniform_eta_native_riscv64/Makefile b/proofs/cbmc/rej_uniform_eta_native_riscv64/Makefile new file mode 100644 index 0000000000..e5cd311bbb --- /dev/null +++ b/proofs/cbmc/rej_uniform_eta_native_riscv64/Makefile @@ -0,0 +1,55 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = rej_uniform_eta_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = rej_uniform_eta_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly_kl.c + +# ML-DSA-44 and ML-DSA-87 use eta=2; ML-DSA-65 uses eta=4. +ifeq ($(MLD_CONFIG_PARAMETER_SET),44) + CHECK_FUNCTION_CONTRACTS=mld_rej_uniform_eta2_native + USE_FUNCTION_CONTRACTS=mld_rej_uniform_eta2_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),65) + CHECK_FUNCTION_CONTRACTS=mld_rej_uniform_eta4_native + USE_FUNCTION_CONTRACTS=mld_rej_uniform_eta4_riscv64_asm +else ifeq ($(MLD_CONFIG_PARAMETER_SET),87) + CHECK_FUNCTION_CONTRACTS=mld_rej_uniform_eta2_native + USE_FUNCTION_CONTRACTS=mld_rej_uniform_eta2_riscv64_asm +endif +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:bitwuzla \ + LP64:z3_smt_only +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = rej_uniform_eta_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +include ../Makefile.common diff --git a/proofs/cbmc/rej_uniform_eta_native_riscv64/rej_uniform_eta_native_riscv64_harness.c b/proofs/cbmc/rej_uniform_eta_native_riscv64/rej_uniform_eta_native_riscv64_harness.c new file mode 100644 index 0000000000..577d4b3fd5 --- /dev/null +++ b/proofs/cbmc/rej_uniform_eta_native_riscv64/rej_uniform_eta_native_riscv64_harness.c @@ -0,0 +1,29 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + +#if MLDSA_ETA == 2 +int mld_rej_uniform_eta2_native(int32_t *r, unsigned len, const uint8_t *buf, + unsigned buflen); +#elif MLDSA_ETA == 4 +int mld_rej_uniform_eta4_native(int32_t *r, unsigned len, const uint8_t *buf, + unsigned buflen); +#endif + +void harness(void) +{ + int32_t *r; + unsigned len; + const uint8_t *buf; + unsigned buflen; + int t; + +#if MLDSA_ETA == 2 + t = mld_rej_uniform_eta2_native(r, len, buf, buflen); +#elif MLDSA_ETA == 4 + t = mld_rej_uniform_eta4_native(r, len, buf, buflen); +#endif +} diff --git a/proofs/cbmc/rej_uniform_native_riscv64/Makefile b/proofs/cbmc/rej_uniform_native_riscv64/Makefile new file mode 100644 index 0000000000..bdd31cbb6a --- /dev/null +++ b/proofs/cbmc/rej_uniform_native_riscv64/Makefile @@ -0,0 +1,59 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +include ../Makefile_params.common + +HARNESS_ENTRY = harness +HARNESS_FILE = rej_uniform_native_riscv64_harness + +# This should be a unique identifier for this proof, and will appear on the +# Litani dashboard. It can be human-readable and contain spaces if you wish. +PROOF_UID = rej_uniform_native_riscv64 + +# We need to set MLD_CHECK_APIS as otherwise mldsa/src/native/api.h won't be +# included, which contains the CBMC specifications. +DEFINES += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH -DMLD_CONFIG_ARITH_BACKEND_FILE="\"$(SRCDIR)/mldsa/src/native/riscv64/meta.h\"" -DMLD_CHECK_APIS +INCLUDES += + +REMOVE_FUNCTION_BODY += +UNWINDSET += + +PROOF_SOURCES += $(PROOFDIR)/$(HARNESS_FILE).c +PROJECT_SOURCES += $(SRCDIR)/mldsa/src/poly.c + +CHECK_FUNCTION_CONTRACTS=mld_rej_uniform_native +USE_FUNCTION_CONTRACTS=mld_rej_uniform_riscv64_asm +APPLY_LOOP_CONTRACTS=on +USE_DYNAMIC_FRAMES=1 + +# Supported CBMC configurations +CBMC_SUPPORTED_CONFIGS = \ + LP64:bitwuzla \ + LP64:z3_smt_only +CBMC_DEFAULT_CONFIG = LP64:bitwuzla + +FUNCTION_NAME = rej_uniform_native_riscv64 + +# If this proof is found to consume huge amounts of RAM, you can set the +# EXPENSIVE variable. With new enough versions of the proof tools, this will +# restrict the number of EXPENSIVE CBMC jobs running at once. See the +# documentation in Makefile.common under the "Job Pools" heading for details. +# EXPENSIVE = true + +# This function is large enough to need... +CBMC_OBJECT_BITS = 8 + +# If you require access to a file-local ("static") function or object to conduct +# your proof, set the following (and do not include the original source file +# ("mldsa/src/poly.c") in PROJECT_SOURCES). +# REWRITTEN_SOURCES = $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i +# include ../Makefile.common +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_SOURCE = $(SRCDIR)/mldsa/src/poly.c +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_FUNCTIONS = foo bar +# $(PROOFDIR)/<__SOURCE_FILE_BASENAME__>.i_OBJECTS = baz +# Care is required with variables on the left-hand side: REWRITTEN_SOURCES must +# be set before including Makefile.common, but any use of variables on the +# left-hand side requires those variables to be defined. Hence, _SOURCE, +# _FUNCTIONS, _OBJECTS is set after including Makefile.common. + +include ../Makefile.common diff --git a/proofs/cbmc/rej_uniform_native_riscv64/rej_uniform_native_riscv64_harness.c b/proofs/cbmc/rej_uniform_native_riscv64/rej_uniform_native_riscv64_harness.c new file mode 100644 index 0000000000..19737ff6b6 --- /dev/null +++ b/proofs/cbmc/rej_uniform_native_riscv64/rej_uniform_native_riscv64_harness.c @@ -0,0 +1,20 @@ +// Copyright (c) The mldsa-native project authors +// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +#include +#include "cbmc.h" +#include "params.h" + + +int mld_rej_uniform_native(int32_t *r, unsigned len, const uint8_t *buf, + unsigned buflen); + +void harness(void) +{ + int32_t *r; + unsigned len; + const uint8_t *buf; + unsigned buflen; + + mld_rej_uniform_native(r, len, buf, buflen); +} diff --git a/proofs/hol_light/riscv64/README.md b/proofs/hol_light/riscv64/README.md new file mode 100644 index 0000000000..6cf2dfaa4e --- /dev/null +++ b/proofs/hol_light/riscv64/README.md @@ -0,0 +1,69 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# HOL Light proofs for the RISC-V 64 backend (in preparation) + +This directory will hold the HOL Light proofs for the RISC-V 64 assembly in +[dev/riscv64/src](../../../dev/riscv64/src). No proofs exist yet. + +## Status + +- [mldsa/](mldsa) contains the assembly that the proofs will verify. It is + generated by `scripts/autogen` from `dev/riscv64/src` using `scripts/simpasm`, + in the same way as for [aarch64](../aarch64), [x86_64](../x86_64) and + [riscv32](../riscv32). +- The assembly is built with `.option norvc` and `.option norelax`, so the byte + code contains no compressed instructions and the linker cannot rewrite it. + The object code verified by HOL Light is therefore exactly the code that is + linked into the library. + +## Prerequisite: an RV64 + RVV model in s2n-bignum + +The [riscv32](../riscv32) proofs use the RV32IM model of +[s2n-bignum](https://github.com/awslabs/s2n-bignum). +The proofs here additionally need: + +1. **RV64IM:** 64-bit registers and addresses, and the RV64-only instructions + (`addiw` and similar). The base and M instructions used by the kernels are + few. +2. **The vector extension (RVV 1.0):** the vector register file, the `vl`, + `vtype` and `vxrm` state, `vsetvli`, and the vector instructions used by the + kernels, including fixed-point (`vsmul`), widening (`vwmul`, `vwmacc`), + narrowing (`vnsrl`, `vnsra`), strided (`vlse32`, `vsse32`) and permutation + (`vrgatherei16`, `vcompress`) instructions. The model must reflect two kinds + of freedom the specification gives to implementations: + - For `VLMAX < AVL < 2*VLMAX`, `vsetvli` may return any `vl` in + `[ceil(AVL/2), VLMAX]`. + - With the tail-agnostic and mask-agnostic policies (`ta`, `ma`), inactive + elements may be left unchanged or overwritten with all ones. + + Both should be modelled nondeterministically, so that a proof covers every + conforming implementation. + +`VLEN` should be a parameter of the model. Kernels written for a fixed `VLEN` +(NTT and inverse NTT, `VLEN=128` and `VLEN=256`) can then be proved for that +value, and the vector-length agnostic kernels for all `VLEN`. + +## Planned specifications + +The specifications follow the AArch64 ones in [aarch64/proofs](../aarch64/proofs) +and the contracts in +[arith_native_riscv64.h](../../../dev/riscv64/src/arith_native_riscv64.h). All +functions except the rejection samplers also need constant-time proofs; for +`rej_uniform_eta{2,4}`, only the positions of the accepted coefficients may +leak. + +| Function | VLEN | Specification | +|----------|------|---------------| +| `ntt_vlen128_riscv64_asm`, `ntt_vlen256_riscv64_asm` | 128, 256 | For inputs bounded by q in absolute value, the output is congruent mod q to the NTT of the input, in bitreversed order, and bounded by 9q. | +| `intt_vlen128_riscv64_asm`, `intt_vlen256_riscv64_asm` | 128, 256 | For inputs bounded by q in absolute value, the output is congruent mod q to the inverse NTT of the input, times 2^32, and bounded by q. | +| `poly_pointwise_montgomery_riscv64_asm` | any | For inputs bounded by `MLD_NTT_BOUND`, each output coefficient is congruent mod q to the product of the input coefficients, times 2^-32, and bounded by q. | +| `polyvecl_pointwise_acc_montgomery_l{4,5,7}_riscv64_asm` | any | For `u` in [0, q) and `v` bounded by `MLD_NTT_BOUND`, the output is congruent mod q to the scalar product of `u` and `v`, times 2^-32, and bounded by q. | +| `poly_caddq_riscv64_asm` | any | Each output coefficient is the input coefficient plus q if negative. | +| `poly_chknorm_riscv64_asm` | any | Returns 0 if all coefficients are below `B` in absolute value, and 1 otherwise. | +| `poly_decompose_{32,88}_riscv64_asm` | any | As `mld_decompose()` in the C reference, coefficient by coefficient. | +| `poly_use_hint_{32,88}_riscv64_asm` | any | As `mld_use_hint()` in the C reference, coefficient by coefficient. | +| `polyz_unpack_{17,19}_riscv64_asm` | any | As `mld_polyz_unpack()` in the C reference. | +| `keccak_f1600_x1_zbb_riscv64_asm` | n/a (scalar, Zbb) | The state is replaced by the Keccak-f[1600] permutation of the input state. | +| `rej_uniform_riscv64_asm` | any | The output is the first `len` 23-bit values from `buf` that are below q; the function returns their number. No constant-time proof is needed (public data). | +| `rej_uniform_eta{2,4}_riscv64_asm` | any | As `mld_rej_eta()` in the C reference. | + diff --git a/proofs/hol_light/riscv64/mldsa/keccak_f1600_x1_zbb_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/keccak_f1600_x1_zbb_riscv64_asm.S new file mode 100644 index 0000000000..421e3d2b3f --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/keccak_f1600_x1_zbb_riscv64_asm.S @@ -0,0 +1,418 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: keccak_f1600_x1_zbb_riscv64_asm + Description: RV64 scalar implementation of the Keccak-f[1600] permutation for a single state, using Zbb + Signature: void mld_keccak_f1600_x1_zbb_riscv64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [ZBB] + a0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + a1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 144 + description: register preservation and temporary storage +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/fips202/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_keccak_f1600_x1_zbb_riscv64_asm +_mld_keccak_f1600_x1_zbb_riscv64_asm: +#else +.global mld_keccak_f1600_x1_zbb_riscv64_asm +mld_keccak_f1600_x1_zbb_riscv64_asm: +#endif + + addi sp, sp, -0x90 + sd ra, 0x0(sp) + sd s0, 0x8(sp) + sd s1, 0x10(sp) + sd s2, 0x18(sp) + sd s3, 0x20(sp) + sd s4, 0x28(sp) + sd s5, 0x30(sp) + sd s6, 0x38(sp) + sd s7, 0x40(sp) + sd s8, 0x48(sp) + sd s9, 0x50(sp) + sd s10, 0x58(sp) + sd s11, 0x60(sp) + sd a0, 0x68(sp) + sd a1, 0x70(sp) + addi t0, a1, 0xc0 + sd t0, 0x78(sp) + ld a1, 0x8(a0) + ld a2, 0x10(a0) + ld a3, 0x18(a0) + ld a4, 0x20(a0) + ld a5, 0x28(a0) + ld a6, 0x30(a0) + ld a7, 0x38(a0) + ld s2, 0x40(a0) + ld s3, 0x48(a0) + ld s4, 0x50(a0) + ld s5, 0x58(a0) + ld s6, 0x60(a0) + ld s7, 0x68(a0) + ld s8, 0x70(a0) + ld s9, 0x78(a0) + ld s10, 0x80(a0) + ld s11, 0x88(a0) + ld s0, 0x90(a0) + ld s1, 0x98(a0) + ld t3, 0xa0(a0) + ld t4, 0xa8(a0) + ld t5, 0xb0(a0) + ld t6, 0xb8(a0) + ld ra, 0xc0(a0) + ld a0, 0x0(a0) + +mld_keccak_f1600_x1_zbb_riscv64_loop: + xor t0, a0, a5 + xor t0, t0, s4 + xor t0, t0, s9 + xor t0, t0, t3 + sd t0, 0x80(sp) + xor t0, a2, a7 + xor t0, t0, s6 + xor t0, t0, s11 + xor t0, t0, t5 + sd t0, 0x88(sp) + xor t0, a1, a6 + xor t0, t0, s5 + xor t0, t0, s10 + xor t0, t0, t4 + xor t1, a4, s3 + xor t1, t1, s8 + xor t1, t1, s1 + xor t1, t1, ra + rori t2, t0, 0x3f + xor t2, t2, t1 + rori t1, t1, 0x3f + xor a0, a0, t2 + xor a5, a5, t2 + xor s4, s4, t2 + xor s9, s9, t2 + xor t3, t3, t2 + ld t2, 0x88(sp) + xor t1, t1, t2 + xor t2, a3, s2 + xor t2, t2, s7 + xor t2, t2, s0 + xor t2, t2, t6 + xor a3, a3, t1 + xor s2, s2, t1 + xor s7, s7, t1 + xor s0, s0, t1 + xor t6, t6, t1 + rori t1, t2, 0x3f + xor t0, t0, t1 + ld t1, 0x80(sp) + xor a2, a2, t0 + xor a7, a7, t0 + xor s6, s6, t0 + xor s11, s11, t0 + xor t5, t5, t0 + rori t0, t1, 0x3f + xor t2, t2, t0 + ld t0, 0x88(sp) + xor a4, a4, t2 + xor s3, s3, t2 + xor s8, s8, t2 + xor s1, s1, t2 + xor ra, ra, t2 + rori t0, t0, 0x3f + xor t0, t0, t1 + xor t2, a1, t0 + xor a6, a6, t0 + xor s5, s5, t0 + xor s10, s10, t0 + xor t4, t4, t0 + rori a1, a6, 0x14 + rori a6, s3, 0x2c + rori s3, t5, 0x3 + rori t5, s8, 0x19 + rori s8, t3, 0x2e + rori t3, a2, 0x2 + rori a2, s6, 0x15 + rori s6, s7, 0x27 + rori s7, s1, 0x38 + rori s1, t6, 0x8 + rori t6, s9, 0x17 + rori s9, a4, 0x25 + rori a4, ra, 0x32 + rori ra, t4, 0x3e + rori t4, s2, 0x9 + rori s2, s10, 0x13 + rori s10, a5, 0x1c + rori a5, a3, 0x24 + rori a3, s0, 0x2b + rori s0, s11, 0x31 + rori s11, s5, 0x36 + rori s5, a7, 0x3a + rori a7, s4, 0x3d + rori s4, t2, 0x3f + andn t0, a0, a4 + andn t1, a1, a0 + andn t2, a2, a1 + xor a0, a0, t2 + andn t2, a3, a2 + xor a1, a1, t2 + andn t2, a4, a3 + xor a2, a2, t2 + xor a3, a3, t0 + xor a4, a4, t1 + andn t0, a5, s3 + andn t1, a6, a5 + andn t2, a7, a6 + xor a5, a5, t2 + andn t2, s2, a7 + xor a6, a6, t2 + andn t2, s3, s2 + xor a7, a7, t2 + xor s2, s2, t0 + xor s3, s3, t1 + andn t0, s4, s8 + andn t1, s5, s4 + andn t2, s6, s5 + xor s4, s4, t2 + andn t2, s7, s6 + xor s5, s5, t2 + andn t2, s8, s7 + xor s6, s6, t2 + xor s7, s7, t0 + xor s8, s8, t1 + andn t0, s9, s1 + andn t1, s10, s9 + andn t2, s11, s10 + xor s9, s9, t2 + andn t2, s0, s11 + xor s10, s10, t2 + andn t2, s1, s0 + xor s11, s11, t2 + xor s0, s0, t0 + xor s1, s1, t1 + andn t0, t3, ra + andn t1, t4, t3 + andn t2, t5, t4 + xor t3, t3, t2 + andn t2, t6, t5 + xor t4, t4, t2 + andn t2, ra, t6 + xor t5, t5, t2 + xor t6, t6, t0 + xor ra, ra, t1 + ld t0, 0x70(sp) + ld t1, 0x0(t0) + xor a0, a0, t1 + xor t0, a0, a5 + xor t0, t0, s4 + xor t0, t0, s9 + xor t0, t0, t3 + sd t0, 0x80(sp) + xor t0, a2, a7 + xor t0, t0, s6 + xor t0, t0, s11 + xor t0, t0, t5 + sd t0, 0x88(sp) + xor t0, a1, a6 + xor t0, t0, s5 + xor t0, t0, s10 + xor t0, t0, t4 + xor t1, a4, s3 + xor t1, t1, s8 + xor t1, t1, s1 + xor t1, t1, ra + rori t2, t0, 0x3f + xor t2, t2, t1 + rori t1, t1, 0x3f + xor a0, a0, t2 + xor a5, a5, t2 + xor s4, s4, t2 + xor s9, s9, t2 + xor t3, t3, t2 + ld t2, 0x88(sp) + xor t1, t1, t2 + xor t2, a3, s2 + xor t2, t2, s7 + xor t2, t2, s0 + xor t2, t2, t6 + xor a3, a3, t1 + xor s2, s2, t1 + xor s7, s7, t1 + xor s0, s0, t1 + xor t6, t6, t1 + rori t1, t2, 0x3f + xor t0, t0, t1 + ld t1, 0x80(sp) + xor a2, a2, t0 + xor a7, a7, t0 + xor s6, s6, t0 + xor s11, s11, t0 + xor t5, t5, t0 + rori t0, t1, 0x3f + xor t2, t2, t0 + ld t0, 0x88(sp) + xor a4, a4, t2 + xor s3, s3, t2 + xor s8, s8, t2 + xor s1, s1, t2 + xor ra, ra, t2 + rori t0, t0, 0x3f + xor t0, t0, t1 + xor t2, a1, t0 + xor a6, a6, t0 + xor s5, s5, t0 + xor s10, s10, t0 + xor t4, t4, t0 + rori a1, a6, 0x14 + rori a6, s3, 0x2c + rori s3, t5, 0x3 + rori t5, s8, 0x19 + rori s8, t3, 0x2e + rori t3, a2, 0x2 + rori a2, s6, 0x15 + rori s6, s7, 0x27 + rori s7, s1, 0x38 + rori s1, t6, 0x8 + rori t6, s9, 0x17 + rori s9, a4, 0x25 + rori a4, ra, 0x32 + rori ra, t4, 0x3e + rori t4, s2, 0x9 + rori s2, s10, 0x13 + rori s10, a5, 0x1c + rori a5, a3, 0x24 + rori a3, s0, 0x2b + rori s0, s11, 0x31 + rori s11, s5, 0x36 + rori s5, a7, 0x3a + rori a7, s4, 0x3d + rori s4, t2, 0x3f + andn t0, a0, a4 + andn t1, a1, a0 + andn t2, a2, a1 + xor a0, a0, t2 + andn t2, a3, a2 + xor a1, a1, t2 + andn t2, a4, a3 + xor a2, a2, t2 + xor a3, a3, t0 + xor a4, a4, t1 + andn t0, a5, s3 + andn t1, a6, a5 + andn t2, a7, a6 + xor a5, a5, t2 + andn t2, s2, a7 + xor a6, a6, t2 + andn t2, s3, s2 + xor a7, a7, t2 + xor s2, s2, t0 + xor s3, s3, t1 + andn t0, s4, s8 + andn t1, s5, s4 + andn t2, s6, s5 + xor s4, s4, t2 + andn t2, s7, s6 + xor s5, s5, t2 + andn t2, s8, s7 + xor s6, s6, t2 + xor s7, s7, t0 + xor s8, s8, t1 + andn t0, s9, s1 + andn t1, s10, s9 + andn t2, s11, s10 + xor s9, s9, t2 + andn t2, s0, s11 + xor s10, s10, t2 + andn t2, s1, s0 + xor s11, s11, t2 + xor s0, s0, t0 + xor s1, s1, t1 + andn t0, t3, ra + andn t1, t4, t3 + andn t2, t5, t4 + xor t3, t3, t2 + andn t2, t6, t5 + xor t4, t4, t2 + andn t2, ra, t6 + xor t5, t5, t2 + xor t6, t6, t0 + xor ra, ra, t1 + ld t0, 0x70(sp) + ld t1, 0x8(t0) + xor a0, a0, t1 + addi t0, t0, 0x10 + sd t0, 0x70(sp) + ld t1, 0x78(sp) + bne t0, t1, mld_keccak_f1600_x1_zbb_riscv64_loop + ld t0, 0x68(sp) + sd a1, 0x8(t0) + sd a2, 0x10(t0) + sd a3, 0x18(t0) + sd a4, 0x20(t0) + sd a5, 0x28(t0) + sd a6, 0x30(t0) + sd a7, 0x38(t0) + sd s2, 0x40(t0) + sd s3, 0x48(t0) + sd s4, 0x50(t0) + sd s5, 0x58(t0) + sd s6, 0x60(t0) + sd s7, 0x68(t0) + sd s8, 0x70(t0) + sd s9, 0x78(t0) + sd s10, 0x80(t0) + sd s11, 0x88(t0) + sd s0, 0x90(t0) + sd s1, 0x98(t0) + sd t3, 0xa0(t0) + sd t4, 0xa8(t0) + sd t5, 0xb0(t0) + sd t6, 0xb8(t0) + sd ra, 0xc0(t0) + sd a0, 0x0(t0) + ld ra, 0x0(sp) + ld s0, 0x8(sp) + ld s1, 0x10(sp) + ld s2, 0x18(sp) + ld s3, 0x20(sp) + ld s4, 0x28(sp) + ld s5, 0x30(sp) + ld s6, 0x38(sp) + ld s7, 0x40(sp) + ld s8, 0x48(sp) + ld s9, 0x50(sp) + ld s10, 0x58(sp) + ld s11, 0x60(sp) + addi sp, sp, 0x90 + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen128_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen128_riscv64_asm.S new file mode 100644 index 0000000000..25cdc6cd78 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen128_riscv64_asm.S @@ -0,0 +1,508 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: intt_vlen128_riscv64_asm + Description: Inverse negacyclic NTT of a polynomial in place, input in bitreversed order, scaled by 2^32 (VLEN=128 only) + Signature: void mld_intt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_intt_zetas) +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_intt_vlen128_riscv64_asm +_mld_intt_vlen128_riscv64_asm: +#else +.global mld_intt_vlen128_riscv64_asm +mld_intt_vlen128_riscv64_asm: +#endif + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x4 + li a3, 0x8 + li a4, 0x10 + li a5, 0x20 + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x4 + +mld_intt_vlen128_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vv v17, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v17, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vv v19, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v19, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vv v21, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v21, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vv v23, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v23, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vv v25, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v25, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vv v27, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v27, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vv v29, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v29, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vv v31, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v31, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vv v18, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v18, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vv v22, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v22, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vv v26, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v26, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vv v30, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v30, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vv v20, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v20, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vv v24, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v24, t0, v0 + vsub.vv v0, v20, v28 + vadd.vv v20, v20, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x100 + addi a7, a7, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen128_riscv64_pass_b + mv a6, a0 + li t5, 0x4 + +mld_intt_vlen128_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vx v17, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v17, t0, v0 + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vx v19, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v19, t0, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vx v21, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v21, t0, v0 + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vx v23, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v23, t0, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vx v25, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v25, t0, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vx v27, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v27, t0, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vx v29, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v29, t0, v0 + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vx v31, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v31, t0, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vx v18, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v18, t0, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vx v22, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v22, t0, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vx v26, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v26, t0, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vx v30, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v30, t0, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vx v20, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v20, t0, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vx v28, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v28, t0, v0 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vx v24, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v24, t0, v0 + lw t1, 0x78(a1) + lw t2, 0x7c(a1) + vsmul.vx v0, v16, t2 + vmul.vx v16, v16, t1 + vnmsac.vx v16, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen128_riscv64_pass_a + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen256_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen256_riscv64_asm.S new file mode 100644 index 0000000000..ff14c323b3 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_intt_vlen256_riscv64_asm.S @@ -0,0 +1,508 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: intt_vlen256_riscv64_asm + Description: Inverse negacyclic NTT of a polynomial in place, input in bitreversed order, scaled by 2^32 (VLEN=256 only) + Signature: void mld_intt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_intt_zetas) +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_intt_vlen256_riscv64_asm +_mld_intt_vlen256_riscv64_asm: +#else +.global mld_intt_vlen256_riscv64_asm +mld_intt_vlen256_riscv64_asm: +#endif + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x8 + li a3, 0x10 + li a4, 0x20 + li a5, 0x40 + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x2 + +mld_intt_vlen256_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vv v17, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v17, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vv v19, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v19, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vv v21, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v21, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vv v23, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v23, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vv v25, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v25, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vv v27, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v27, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vv v29, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v29, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vv v31, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v31, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vv v18, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v18, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vv v22, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v22, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vv v26, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v26, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vv v30, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v30, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vv v20, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v20, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vv v24, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v24, t0, v0 + vsub.vv v0, v20, v28 + vadd.vv v20, v20, v28 + vmul.vv v28, v0, v8 + vsmul.vv v0, v0, v12 + vnmsac.vx v28, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x200 + addi a7, a7, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen256_riscv64_pass_b + mv a6, a0 + li t5, 0x2 + +mld_intt_vlen256_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vsub.vv v0, v16, v17 + vadd.vv v16, v16, v17 + vmul.vx v17, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v17, t0, v0 + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vsub.vv v0, v18, v19 + vadd.vv v18, v18, v19 + vmul.vx v19, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v19, t0, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vsub.vv v0, v20, v21 + vadd.vv v20, v20, v21 + vmul.vx v21, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v21, t0, v0 + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vsub.vv v0, v22, v23 + vadd.vv v22, v22, v23 + vmul.vx v23, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v23, t0, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vsub.vv v0, v24, v25 + vadd.vv v24, v24, v25 + vmul.vx v25, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v25, t0, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vsub.vv v0, v26, v27 + vadd.vv v26, v26, v27 + vmul.vx v27, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v27, t0, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vsub.vv v0, v28, v29 + vadd.vv v28, v28, v29 + vmul.vx v29, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v29, t0, v0 + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vsub.vv v0, v30, v31 + vadd.vv v30, v30, v31 + vmul.vx v31, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v31, t0, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vsub.vv v0, v16, v18 + vadd.vv v16, v16, v18 + vmul.vx v18, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v18, t0, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vsub.vv v0, v20, v22 + vadd.vv v20, v20, v22 + vmul.vx v22, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v22, t0, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vsub.vv v0, v24, v26 + vadd.vv v24, v24, v26 + vmul.vx v26, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v26, t0, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vsub.vv v0, v28, v30 + vadd.vv v28, v28, v30 + vmul.vx v30, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v30, t0, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vsub.vv v0, v16, v20 + vadd.vv v16, v16, v20 + vmul.vx v20, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v20, t0, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vsub.vv v0, v24, v28 + vadd.vv v24, v24, v28 + vmul.vx v28, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v28, t0, v0 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vsub.vv v0, v16, v24 + vadd.vv v16, v16, v24 + vmul.vx v24, v0, t1 + vsmul.vx v0, v0, t2 + vnmsac.vx v24, t0, v0 + lw t1, 0x78(a1) + lw t2, 0x7c(a1) + vsmul.vx v0, v16, t2 + vmul.vx v16, v16, t1 + vnmsac.vx v16, t0, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_intt_vlen256_riscv64_pass_a + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen128_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen128_riscv64_asm.S new file mode 100644 index 0000000000..7deeba1864 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen128_riscv64_asm.S @@ -0,0 +1,503 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: ntt_vlen128_riscv64_asm + Description: Forward negacyclic NTT of a polynomial in place, output in bitreversed order (VLEN=128 only) + Signature: void mld_ntt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_ntt_zetas) +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_ntt_vlen128_riscv64_asm +_mld_ntt_vlen128_riscv64_asm: +#else +.global mld_ntt_vlen128_riscv64_asm +mld_ntt_vlen128_riscv64_asm: +#endif + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x4 + li a3, 0x8 + li a4, 0x10 + li a5, 0x20 + mv a6, a0 + li t5, 0x4 + +mld_ntt_vlen128_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vmul.vx v0, v24, t1 + vsmul.vx v24, v24, t2 + vnmsac.vx v0, t0, v24 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vmul.vx v0, v20, t1 + vsmul.vx v20, v20, t2 + vnmsac.vx v0, t0, v20 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vmul.vx v0, v28, t1 + vsmul.vx v28, v28, t2 + vnmsac.vx v0, t0, v28 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vmul.vx v0, v18, t1 + vsmul.vx v18, v18, t2 + vnmsac.vx v0, t0, v18 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vmul.vx v0, v22, t1 + vsmul.vx v22, v22, t2 + vnmsac.vx v0, t0, v22 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vmul.vx v0, v26, t1 + vsmul.vx v26, v26, t2 + vnmsac.vx v0, t0, v26 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vmul.vx v0, v30, t1 + vsmul.vx v30, v30, t2 + vnmsac.vx v0, t0, v30 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vmul.vx v0, v17, t1 + vsmul.vx v17, v17, t2 + vnmsac.vx v0, t0, v17 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vmul.vx v0, v19, t1 + vsmul.vx v19, v19, t2 + vnmsac.vx v0, t0, v19 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vmul.vx v0, v21, t1 + vsmul.vx v21, v21, t2 + vnmsac.vx v0, t0, v21 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vmul.vx v0, v23, t1 + vsmul.vx v23, v23, t2 + vnmsac.vx v0, t0, v23 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vmul.vx v0, v25, t1 + vsmul.vx v25, v25, t2 + vnmsac.vx v0, t0, v25 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vmul.vx v0, v27, t1 + vsmul.vx v27, v27, t2 + vnmsac.vx v0, t0, v27 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vmul.vx v0, v29, t1 + vsmul.vx v29, v29, t2 + vnmsac.vx v0, t0, v29 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vmul.vx v0, v31, t1 + vsmul.vx v31, v31, t2 + vnmsac.vx v0, t0, v31 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen128_riscv64_pass_a + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x4 + +mld_ntt_vlen128_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v24, v8 + vsmul.vv v4, v24, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v20, v8 + vsmul.vv v4, v20, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v18, v8 + vsmul.vv v4, v18, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v22, v8 + vsmul.vv v4, v22, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v26, v8 + vsmul.vv v4, v26, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v30, v8 + vsmul.vv v4, v30, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v17, v8 + vsmul.vv v4, v17, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v19, v8 + vsmul.vv v4, v19, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v21, v8 + vsmul.vv v4, v21, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v23, v8 + vsmul.vv v4, v23, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v25, v8 + vsmul.vv v4, v25, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v27, v8 + vsmul.vv v4, v27, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v29, v8 + vsmul.vv v4, v29, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v31, v8 + vsmul.vv v4, v31, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x100 + addi a7, a7, 0x10 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen128_riscv64_pass_b + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen256_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen256_riscv64_asm.S new file mode 100644 index 0000000000..8ac53c11c8 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_ntt_vlen256_riscv64_asm.S @@ -0,0 +1,503 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: ntt_vlen256_riscv64_asm + Description: Forward negacyclic NTT of a polynomial in place, output in bitreversed order (VLEN=256 only) + Signature: void mld_ntt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t p[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int32_t zetas[512] + description: Twiddle table (mld_rv64v_ntt_zetas) +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_ntt_vlen256_riscv64_asm +_mld_ntt_vlen256_riscv64_asm: +#else +.global mld_ntt_vlen256_riscv64_asm +mld_ntt_vlen256_riscv64_asm: +#endif + + csrwi vxrm, 0x0 + lui t0, 0x7fe + addiw t0, t0, 0x1 + li t4, 0x40 + li a2, 0x8 + li a3, 0x10 + li a4, 0x20 + li a5, 0x40 + mv a6, a0 + li t5, 0x2 + +mld_ntt_vlen256_riscv64_pass_a: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vle32.v v16, (t3) + addi t3, t3, 0x40 + vle32.v v17, (t3) + addi t3, t3, 0x40 + vle32.v v18, (t3) + addi t3, t3, 0x40 + vle32.v v19, (t3) + addi t3, t3, 0x40 + vle32.v v20, (t3) + addi t3, t3, 0x40 + vle32.v v21, (t3) + addi t3, t3, 0x40 + vle32.v v22, (t3) + addi t3, t3, 0x40 + vle32.v v23, (t3) + addi t3, t3, 0x40 + vle32.v v24, (t3) + addi t3, t3, 0x40 + vle32.v v25, (t3) + addi t3, t3, 0x40 + vle32.v v26, (t3) + addi t3, t3, 0x40 + vle32.v v27, (t3) + addi t3, t3, 0x40 + vle32.v v28, (t3) + addi t3, t3, 0x40 + vle32.v v29, (t3) + addi t3, t3, 0x40 + vle32.v v30, (t3) + addi t3, t3, 0x40 + vle32.v v31, (t3) + addi t3, t3, 0x40 + vsetvli zero, a5, e32, m8, ta, ma + lw t1, 0x0(a1) + lw t2, 0x4(a1) + vmul.vx v0, v24, t1 + vsmul.vx v24, v24, t2 + vnmsac.vx v0, t0, v24 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a4, e32, m4, ta, ma + lw t1, 0x8(a1) + lw t2, 0xc(a1) + vmul.vx v0, v20, t1 + vsmul.vx v20, v20, t2 + vnmsac.vx v0, t0, v20 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x10(a1) + lw t2, 0x14(a1) + vmul.vx v0, v28, t1 + vsmul.vx v28, v28, t2 + vnmsac.vx v0, t0, v28 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a3, e32, m2, ta, ma + lw t1, 0x18(a1) + lw t2, 0x1c(a1) + vmul.vx v0, v18, t1 + vsmul.vx v18, v18, t2 + vnmsac.vx v0, t0, v18 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x20(a1) + lw t2, 0x24(a1) + vmul.vx v0, v22, t1 + vsmul.vx v22, v22, t2 + vnmsac.vx v0, t0, v22 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x28(a1) + lw t2, 0x2c(a1) + vmul.vx v0, v26, t1 + vsmul.vx v26, v26, t2 + vnmsac.vx v0, t0, v26 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x30(a1) + lw t2, 0x34(a1) + vmul.vx v0, v30, t1 + vsmul.vx v30, v30, t2 + vnmsac.vx v0, t0, v30 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + lw t1, 0x38(a1) + lw t2, 0x3c(a1) + vmul.vx v0, v17, t1 + vsmul.vx v17, v17, t2 + vnmsac.vx v0, t0, v17 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + lw t1, 0x40(a1) + lw t2, 0x44(a1) + vmul.vx v0, v19, t1 + vsmul.vx v19, v19, t2 + vnmsac.vx v0, t0, v19 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + lw t1, 0x48(a1) + lw t2, 0x4c(a1) + vmul.vx v0, v21, t1 + vsmul.vx v21, v21, t2 + vnmsac.vx v0, t0, v21 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + lw t1, 0x50(a1) + lw t2, 0x54(a1) + vmul.vx v0, v23, t1 + vsmul.vx v23, v23, t2 + vnmsac.vx v0, t0, v23 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + lw t1, 0x58(a1) + lw t2, 0x5c(a1) + vmul.vx v0, v25, t1 + vsmul.vx v25, v25, t2 + vnmsac.vx v0, t0, v25 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + lw t1, 0x60(a1) + lw t2, 0x64(a1) + vmul.vx v0, v27, t1 + vsmul.vx v27, v27, t2 + vnmsac.vx v0, t0, v27 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + lw t1, 0x68(a1) + lw t2, 0x6c(a1) + vmul.vx v0, v29, t1 + vsmul.vx v29, v29, t2 + vnmsac.vx v0, t0, v29 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + lw t1, 0x70(a1) + lw t2, 0x74(a1) + vmul.vx v0, v31, t1 + vsmul.vx v31, v31, t2 + vnmsac.vx v0, t0, v31 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vse32.v v16, (t3) + addi t3, t3, 0x40 + vse32.v v17, (t3) + addi t3, t3, 0x40 + vse32.v v18, (t3) + addi t3, t3, 0x40 + vse32.v v19, (t3) + addi t3, t3, 0x40 + vse32.v v20, (t3) + addi t3, t3, 0x40 + vse32.v v21, (t3) + addi t3, t3, 0x40 + vse32.v v22, (t3) + addi t3, t3, 0x40 + vse32.v v23, (t3) + addi t3, t3, 0x40 + vse32.v v24, (t3) + addi t3, t3, 0x40 + vse32.v v25, (t3) + addi t3, t3, 0x40 + vse32.v v26, (t3) + addi t3, t3, 0x40 + vse32.v v27, (t3) + addi t3, t3, 0x40 + vse32.v v28, (t3) + addi t3, t3, 0x40 + vse32.v v29, (t3) + addi t3, t3, 0x40 + vse32.v v30, (t3) + addi t3, t3, 0x40 + vse32.v v31, (t3) + addi t3, t3, 0x40 + addi a6, a6, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen256_riscv64_pass_a + mv a6, a0 + addi a7, a1, 0x80 + li t5, 0x2 + +mld_ntt_vlen256_riscv64_pass_b: + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vlse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vlse32.v v31, (t3), t4 + addi t3, t3, 0x4 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a7 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v24, v8 + vsmul.vv v4, v24, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v24, v16, v0 + vadd.vv v16, v16, v0 + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x80 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v20, v8 + vsmul.vv v4, v20, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v20, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x100 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vmv2r.v v10, v8 + vmv2r.v v14, v12 + vsetvli zero, a4, e32, m4, ta, ma + vmul.vv v0, v28, v8 + vsmul.vv v4, v28, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v28, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x180 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v18, v8 + vsmul.vv v4, v18, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v18, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x200 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v22, v8 + vsmul.vv v4, v22, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v22, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x280 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v26, v8 + vsmul.vv v4, v26, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v26, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x300 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmv1r.v v9, v8 + vmv1r.v v13, v12 + vsetvli zero, a3, e32, m2, ta, ma + vmul.vv v0, v30, v8 + vsmul.vv v4, v30, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v30, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x380 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v17, v8 + vsmul.vv v4, v17, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v17, v16, v0 + vadd.vv v16, v16, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x400 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v19, v8 + vsmul.vv v4, v19, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v19, v18, v0 + vadd.vv v18, v18, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x480 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v21, v8 + vsmul.vv v4, v21, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v21, v20, v0 + vadd.vv v20, v20, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x500 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v23, v8 + vsmul.vv v4, v23, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v23, v22, v0 + vadd.vv v22, v22, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x580 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v25, v8 + vsmul.vv v4, v25, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v25, v24, v0 + vadd.vv v24, v24, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x600 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v27, v8 + vsmul.vv v4, v27, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v27, v26, v0 + vadd.vv v26, v26, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x680 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v29, v8 + vsmul.vv v4, v29, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v29, v28, v0 + vadd.vv v28, v28, v0 + vsetvli zero, a2, e32, m1, ta, ma + addi t3, a7, 0x700 + vle32.v v8, (t3) + addi t3, t3, 0x40 + vle32.v v12, (t3) + vmul.vv v0, v31, v8 + vsmul.vv v4, v31, v12 + vnmsac.vx v0, t0, v4 + vsub.vv v31, v30, v0 + vadd.vv v30, v30, v0 + vsetvli zero, a2, e32, m1, ta, ma + mv t3, a6 + vsse32.v v16, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v17, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v18, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v19, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v20, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v21, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v22, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v23, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v24, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v25, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v26, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v27, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v28, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v29, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v30, (t3), t4 + addi t3, t3, 0x4 + vsse32.v v31, (t3), t4 + addi t3, t3, 0x4 + addi a6, a6, 0x200 + addi a7, a7, 0x20 + addi t5, t5, -0x1 + bnez t5, mld_ntt_vlen256_riscv64_pass_b + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_caddq_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_caddq_riscv64_asm.S new file mode 100644 index 0000000000..20a2a596af --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_caddq_riscv64_asm.S @@ -0,0 +1,61 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_caddq_riscv64_asm + Description: Add q to all negative coefficients of a polynomial + Signature: void mld_poly_caddq_riscv64_asm(int32_t a[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_caddq_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_caddq_riscv64_asm +_mld_poly_caddq_riscv64_asm: +#else +.global mld_poly_caddq_riscv64_asm +mld_poly_caddq_riscv64_asm: +#endif + + lui t1, 0x7fe + addiw t1, t1, 0x1 + li a1, 0x100 + vsetvli t0, a1, e32, m4, ta, ma + slli t2, t0, 0x2 + +mld_poly_caddq_riscv64_loop: + vle32.v v0, (a0) + vsra.vi v4, v0, 0x1f + vand.vx v4, v4, t1 + vadd.vv v0, v0, v4 + vse32.v v0, (a0) + add a0, a0, t2 + sub a1, a1, t0 + bnez a1, mld_poly_caddq_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_chknorm_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_chknorm_riscv64_asm.S new file mode 100644 index 0000000000..510e92105e --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_chknorm_riscv64_asm.S @@ -0,0 +1,69 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_chknorm_riscv64_asm + Description: Infinity-norm bound check on polynomial coefficients + Signature: int mld_poly_chknorm_riscv64_asm(const int32_t a[256], int64_t B) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t a[256] + description: Input polynomial + a1: + type: scalar + c_parameter: int64_t B + description: Norm bound + test_with: 131072 # representative non-negative bound (1 << 17) +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_chknorm_riscv64_asm +_mld_poly_chknorm_riscv64_asm: +#else +.global mld_poly_chknorm_riscv64_asm +mld_poly_chknorm_riscv64_asm: +#endif + + li a2, 0x100 + vsetvli t0, a2, e32, m4, ta, ma + slli t1, t0, 0x2 + vmv.v.i v8, 0x0 + +mld_poly_chknorm_riscv64_loop: + vle32.v v0, (a0) + vrsub.vi v4, v0, 0x0 + vmax.vv v0, v0, v4 + vmaxu.vv v8, v8, v0 + add a0, a0, t1 + sub a2, a2, t0 + bnez a2, mld_poly_chknorm_riscv64_loop + vmv.s.x v12, zero + vredmaxu.vs v12, v8, v12 + vmv.x.s t2, v12 + slt a0, t2, a1 + xori a0, a0, 0x1 + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_32_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_32_riscv64_asm.S new file mode 100644 index 0000000000..7a752d5abf --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_32_riscv64_asm.S @@ -0,0 +1,83 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_decompose_32_riscv64_asm + Description: Decompose polynomial coefficients into high and low bits for GAMMA2 = (q-1)/32 + Signature: void mld_poly_decompose_32_riscv64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output polynomial with high bits + a1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial, output polynomial with low bits +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_decompose_32_riscv64_asm +_mld_poly_decompose_32_riscv64_asm: +#else +.global mld_poly_decompose_32_riscv64_asm +mld_poly_decompose_32_riscv64_asm: +#endif + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x80 + addiw t5, t5, -0x200 + li t6, 0x401 + lui a3, 0x200 + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_decompose_32_riscv64_loop: + vle32.v v8, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x16 + vand.vi v0, v0, 0xf + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vse32.v v0, (a0) + vse32.v v8, (a1) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_decompose_32_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_88_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_88_riscv64_asm.S new file mode 100644 index 0000000000..eacd1c20b6 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_decompose_88_riscv64_asm.S @@ -0,0 +1,88 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_decompose_88_riscv64_asm + Description: Decompose polynomial coefficients into high and low bits for GAMMA2 = (q-1)/88 + Signature: void mld_poly_decompose_88_riscv64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output polynomial with high bits + a1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial, output polynomial with low bits +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_decompose_88_riscv64_asm +_mld_poly_decompose_88_riscv64_asm: +#else +.global mld_poly_decompose_88_riscv64_asm +mld_poly_decompose_88_riscv64_asm: +#endif + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x2f + addiw t5, t5, -0x800 + lui t6, 0x3 + addiw t6, t6, -0x3f5 + lui a3, 0x800 + li a4, 0x2b + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_decompose_88_riscv64_loop: + vle32.v v8, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x18 + vrsub.vx v16, v0, a4 + vsra.vi v16, v16, 0x1f + vand.vv v16, v16, v0 + vxor.vv v0, v0, v16 + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vse32.v v0, (a0) + vse32.v v8, (a1) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_decompose_88_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_pointwise_montgomery_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_pointwise_montgomery_riscv64_asm.S new file mode 100644 index 0000000000..683dcc913c --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_pointwise_montgomery_riscv64_asm.S @@ -0,0 +1,73 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_pointwise_montgomery_riscv64_asm + Description: Pointwise Montgomery multiplication of two polynomials + Signature: void mld_poly_pointwise_montgomery_riscv64_asm(int32_t a[256], const int32_t b[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t b[256] + description: Input polynomial +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_pointwise_montgomery_riscv64_asm +_mld_poly_pointwise_montgomery_riscv64_asm: +#else +.global mld_poly_pointwise_montgomery_riscv64_asm +mld_poly_pointwise_montgomery_riscv64_asm: +#endif + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t3, t0, 0x2 + +mld_poly_pointwise_montgomery_riscv64_loop: + vle32.v v0, (a0) + vle32.v v8, (a1) + vmul.vv v16, v0, v8 + vmulh.vv v24, v0, v8 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v0, v24, v16 + vse32.v v0, (a0) + add a0, a0, t3 + add a1, a1, t3 + sub a2, a2, t0 + bnez a2, mld_poly_pointwise_montgomery_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_32_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_32_riscv64_asm.S new file mode 100644 index 0000000000..d5ed837ffc --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_32_riscv64_asm.S @@ -0,0 +1,89 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_use_hint_32_riscv64_asm + Description: Use hint polynomial to correct the high bits of a polynomial for GAMMA2 = (q-1)/32 + Signature: void mld_poly_use_hint_32_riscv64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_use_hint_32_riscv64_asm +_mld_poly_use_hint_32_riscv64_asm: +#else +.global mld_poly_use_hint_32_riscv64_asm +mld_poly_use_hint_32_riscv64_asm: +#endif + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x80 + addiw t5, t5, -0x200 + li t6, 0x401 + lui a3, 0x200 + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_use_hint_32_riscv64_loop: + vle32.v v8, (a0) + vle32.v v24, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x16 + vand.vi v0, v0, 0xf + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vadd.vi v8, v8, -0x1 + vsra.vi v8, v8, 0x1f + vxor.vv v16, v24, v8 + vsub.vv v16, v16, v8 + vadd.vv v0, v0, v16 + vand.vi v0, v0, 0xf + vse32.v v0, (a0) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_use_hint_32_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_88_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_88_riscv64_asm.S new file mode 100644 index 0000000000..ff2a37a2b6 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_poly_use_hint_88_riscv64_asm.S @@ -0,0 +1,101 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_use_hint_88_riscv64_asm + Description: Use hint polynomial to correct the high bits of a polynomial for GAMMA2 = (q-1)/88 + Signature: void mld_poly_use_hint_88_riscv64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial + a1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_poly_use_hint_88_riscv64_asm +_mld_poly_use_hint_88_riscv64_asm: +#else +.global mld_poly_use_hint_88_riscv64_asm +mld_poly_use_hint_88_riscv64_asm: +#endif + + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t3, 0x7f + lui t4, 0x3ff + lui t5, 0x2f + addiw t5, t5, -0x800 + lui t6, 0x3 + addiw t6, t6, -0x3f5 + lui a3, 0x800 + li a4, 0x2b + li a5, 0x2c + li a2, 0x100 + vsetvli t0, a2, e32, m8, ta, ma + slli t1, t0, 0x2 + +mld_poly_use_hint_88_riscv64_loop: + vle32.v v8, (a0) + vle32.v v24, (a1) + vadd.vx v0, v8, t3 + vsra.vi v0, v0, 0x7 + vmul.vx v0, v0, t6 + vadd.vx v0, v0, a3 + vsra.vi v0, v0, 0x18 + vrsub.vx v16, v0, a4 + vsra.vi v16, v16, 0x1f + vand.vv v16, v16, v0 + vxor.vv v0, v0, v16 + vnmsac.vx v8, t5, v0 + vrsub.vx v16, v8, t4 + vsra.vi v16, v16, 0x1f + vand.vx v16, v16, t2 + vsub.vv v8, v8, v16 + vadd.vi v8, v8, -0x1 + vsra.vi v8, v8, 0x1f + vxor.vv v16, v24, v8 + vsub.vv v16, v16, v8 + vadd.vv v0, v0, v16 + vsra.vi v16, v0, 0x1f + vand.vx v16, v16, a5 + vadd.vv v0, v0, v16 + vrsub.vx v16, v0, a4 + vsra.vi v16, v16, 0x1f + vand.vv v16, v16, v0 + vxor.vv v0, v0, v16 + vse32.v v0, (a0) + add a0, a0, t1 + add a1, a1, t1 + sub a2, a2, t0 + bnez a2, mld_poly_use_hint_88_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S new file mode 100644 index 0000000000..ece471fd3a --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S @@ -0,0 +1,100 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l4_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 4 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(int32_t w[256], const int32_t u[4][256], const int32_t v[4][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t u[4][256] + description: First input vector + a2: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t v[4][256] + description: Second input vector +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm +_mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm: +#else +.global mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm +mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm: +#endif + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a3, 0x100 + li t4, 0x20 + li a6, 0x400 + vsetvli t0, a3, e32, m4, ta, ma + slli t3, t0, 0x2 + +mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_loop: + vle32.v v0, (a1) + vle32.v v4, (a2) + vwmul.vv v8, v0, v4 + mv a4, a1 + mv a5, a2 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + vnsrl.wi v16, v8, 0x0 + vnsra.wx v20, v8, t4 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v16, v20, v16 + vse32.v v16, (a0) + add a0, a0, t3 + add a1, a1, t3 + add a2, a2, t3 + sub a3, a3, t0 + bnez a3, mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S new file mode 100644 index 0000000000..130c3fbdd8 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S @@ -0,0 +1,105 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l5_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 5 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(int32_t w[256], const int32_t u[5][256], const int32_t v[5][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t u[5][256] + description: First input vector + a2: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t v[5][256] + description: Second input vector +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm +_mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm: +#else +.global mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm +mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm: +#endif + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a3, 0x100 + li t4, 0x20 + li a6, 0x400 + vsetvli t0, a3, e32, m4, ta, ma + slli t3, t0, 0x2 + +mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_loop: + vle32.v v0, (a1) + vle32.v v4, (a2) + vwmul.vv v8, v0, v4 + mv a4, a1 + mv a5, a2 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + vnsrl.wi v16, v8, 0x0 + vnsra.wx v20, v8, t4 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v16, v20, v16 + vse32.v v16, (a0) + add a0, a0, t3 + add a1, a1, t3 + add a2, a2, t3 + sub a3, a3, t0 + bnez a3, mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S new file mode 100644 index 0000000000..2a8665696e --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S @@ -0,0 +1,115 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l7_riscv64_asm + Description: Pointwise multiplication and accumulation of two vectors of 7 polynomials with Montgomery reduction + Signature: void mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(int32_t w[256], const int32_t u[7][256], const int32_t v[7][256]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t w[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t u[7][256] + description: First input vector + a2: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t v[7][256] + description: Second input vector +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm +_mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm: +#else +.global mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm +mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm: +#endif + + lui t1, 0x7fe + addiw t1, t1, 0x1 + lui t2, 0x3802 + addiw t2, t2, 0x1 + li a3, 0x100 + li t4, 0x20 + li a6, 0x400 + vsetvli t0, a3, e32, m4, ta, ma + slli t3, t0, 0x2 + +mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_loop: + vle32.v v0, (a1) + vle32.v v4, (a2) + vwmul.vv v8, v0, v4 + mv a4, a1 + mv a5, a2 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + add a4, a4, a6 + add a5, a5, a6 + vle32.v v0, (a4) + vle32.v v4, (a5) + vwmacc.vv v8, v0, v4 + vnsrl.wi v16, v8, 0x0 + vnsra.wx v20, v8, t4 + vmul.vx v16, v16, t2 + vmulh.vx v16, v16, t1 + vsub.vv v16, v20, v16 + vse32.v v16, (a0) + add a0, a0, t3 + add a1, a1, t3 + add a2, a2, t3 + sub a3, a3, t0 + bnez a3, mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_17_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_17_riscv64_asm.S new file mode 100644 index 0000000000..e519b0c7dd --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_17_riscv64_asm.S @@ -0,0 +1,91 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyz_unpack_17_riscv64_asm + Description: Unpack polynomial z with 18-bit packed coefficients for GAMMA1 = 2^17 + Signature: void mld_polyz_unpack_17_riscv64_asm(int32_t r[256], const uint8_t buf[576]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 576 + permissions: read-only + c_parameter: const uint8_t buf[576] + description: Packed input bytes +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_polyz_unpack_17_riscv64_asm +_mld_polyz_unpack_17_riscv64_asm: +#else +.global mld_polyz_unpack_17_riscv64_asm +mld_polyz_unpack_17_riscv64_asm: +#endif + + li a2, 0x100 + li a5, 0x2 + lui a6, 0x20 + vsetvli t0, a2, e32, m2, ta, ma + slli t3, t0, 0x2 + slli a3, t0, 0x3 + add a3, a3, t0 + srli a3, a3, 0x2 + slli a4, t0, 0x2 + li t2, 0x12 + vsetvli zero, a4, e16, m4, ta, ma + vid.v v8 + vsrl.vi v12, v8, 0x2 + vmul.vx v12, v12, t2 + vsrl.vi v12, v12, 0x3 + vand.vi v8, v8, 0x3 + vminu.vx v8, v8, a5 + vadd.vv v8, v8, v12 + vsetvli zero, t0, e32, m2, ta, ma + vid.v v4 + vmul.vx v4, v4, t2 + vand.vi v4, v4, 0x7 + lui t1, 0x40 + addiw t1, t1, -0x1 + +mld_polyz_unpack_17_riscv64_loop: + vsetvli zero, a3, e8, m2, ta, ma + vle8.v v16, (a1) + vsetvli zero, a4, e8, m2, ta, ma + vrgatherei16.vv v20, v16, v8 + vsetvli zero, t0, e32, m2, ta, ma + vsrl.vv v20, v20, v4 + vand.vx v20, v20, t1 + vrsub.vx v20, v20, a6 + vse32.v v20, (a0) + add a1, a1, a3 + add a0, a0, t3 + sub a2, a2, t0 + bnez a2, mld_polyz_unpack_17_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_19_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_19_riscv64_asm.S new file mode 100644 index 0000000000..e59b25fda8 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_polyz_unpack_19_riscv64_asm.S @@ -0,0 +1,91 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: polyz_unpack_19_riscv64_asm + Description: Unpack polynomial z with 20-bit packed coefficients for GAMMA1 = 2^19 + Signature: void mld_polyz_unpack_19_riscv64_asm(int32_t r[256], const uint8_t buf[640]) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial + a1: + type: buffer + size_bytes: 640 + permissions: read-only + c_parameter: const uint8_t buf[640] + description: Packed input bytes +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_polyz_unpack_19_riscv64_asm +_mld_polyz_unpack_19_riscv64_asm: +#else +.global mld_polyz_unpack_19_riscv64_asm +mld_polyz_unpack_19_riscv64_asm: +#endif + + li a2, 0x100 + li a5, 0x2 + lui a6, 0x80 + vsetvli t0, a2, e32, m2, ta, ma + slli t3, t0, 0x2 + slli a3, t0, 0x2 + add a3, a3, t0 + srli a3, a3, 0x1 + slli a4, t0, 0x2 + li t2, 0x14 + vsetvli zero, a4, e16, m4, ta, ma + vid.v v8 + vsrl.vi v12, v8, 0x2 + vmul.vx v12, v12, t2 + vsrl.vi v12, v12, 0x3 + vand.vi v8, v8, 0x3 + vminu.vx v8, v8, a5 + vadd.vv v8, v8, v12 + vsetvli zero, t0, e32, m2, ta, ma + vid.v v4 + vmul.vx v4, v4, t2 + vand.vi v4, v4, 0x7 + lui t1, 0x100 + addiw t1, t1, -0x1 + +mld_polyz_unpack_19_riscv64_loop: + vsetvli zero, a3, e8, m2, ta, ma + vle8.v v16, (a1) + vsetvli zero, a4, e8, m2, ta, ma + vrgatherei16.vv v20, v16, v8 + vsetvli zero, t0, e32, m2, ta, ma + vsrl.vv v20, v20, v4 + vand.vx v20, v20, t1 + vrsub.vx v20, v20, a6 + vse32.v v20, (a0) + add a1, a1, a3 + add a0, a0, t3 + sub a2, a2, t0 + bnez a2, mld_polyz_unpack_19_riscv64_loop + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta2_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta2_riscv64_asm.S new file mode 100644 index 0000000000..3f61de9933 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta2_riscv64_asm.S @@ -0,0 +1,108 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_eta2_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers in [-2, +2] + Signature: uint64_t mld_rej_uniform_eta2_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer + test_with: 272 # 2 * SHAKE256_RATE +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_rej_uniform_eta2_riscv64_asm +_mld_rej_uniform_eta2_riscv64_asm: +#else +.global mld_rej_uniform_eta2_riscv64_asm +mld_rej_uniform_eta2_riscv64_asm: +#endif + + li a4, 0x0 + li t2, 0x34 + li t3, 0x5 + vsetvli a5, zero, e16, m2, ta, ma + +mld_rej_uniform_eta2_riscv64_loop: + bgeu a4, a1, mld_rej_uniform_eta2_riscv64_done + beqz a3, mld_rej_uniform_eta2_riscv64_done + mv a6, a5 + bgeu a3, a5, mld_rej_uniform_eta2_riscv64_load + mv a6, a3 + +mld_rej_uniform_eta2_riscv64_load: + vsetvli zero, a6, e16, m2, ta, ma + vle8.v v1, (a2) + add a2, a2, a6 + sub a3, a3, a6 + vzext.vf2 v2, v1 + vsrl.vi v4, v2, 0x4 + vsll.vi v4, v4, 0x8 + vand.vi v2, v2, 0xf + vor.vv v2, v2, v4 + slli t0, a6, 0x1 + vsetvli zero, t0, e8, m2, ta, ma + vmsleu.vi v6, v2, 0xe + vmulhu.vx v12, v2, t2 + vmul.vx v12, v12, t3 + vsub.vv v2, v2, v12 + vrsub.vi v2, v2, 0x2 + vcpop.m t1, v6 + vcompress.vm v14, v2, v6 + sub a7, a1, a4 + bgeu a7, t1, mld_rej_uniform_eta2_riscv64_store + mv t1, a7 + +mld_rej_uniform_eta2_riscv64_store: + vsetvli zero, t1, e32, m8, ta, ma + vsext.vf4 v16, v14 + vse32.v v16, (a0) + add a4, a4, t1 + slli t1, t1, 0x2 + add a0, a0, t1 + j mld_rej_uniform_eta2_riscv64_loop + +mld_rej_uniform_eta2_riscv64_done: + mv a0, a4 + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta4_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta4_riscv64_asm.S new file mode 100644 index 0000000000..5e575391b2 --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_eta4_riscv64_asm.S @@ -0,0 +1,103 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_eta4_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers in [-4, +4] + Signature: uint64_t mld_rej_uniform_eta4_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer + test_with: 272 # 2 * SHAKE256_RATE +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_rej_uniform_eta4_riscv64_asm +_mld_rej_uniform_eta4_riscv64_asm: +#else +.global mld_rej_uniform_eta4_riscv64_asm +mld_rej_uniform_eta4_riscv64_asm: +#endif + + li a4, 0x0 + vsetvli a5, zero, e16, m2, ta, ma + +mld_rej_uniform_eta4_riscv64_loop: + bgeu a4, a1, mld_rej_uniform_eta4_riscv64_done + beqz a3, mld_rej_uniform_eta4_riscv64_done + mv a6, a5 + bgeu a3, a5, mld_rej_uniform_eta4_riscv64_load + mv a6, a3 + +mld_rej_uniform_eta4_riscv64_load: + vsetvli zero, a6, e16, m2, ta, ma + vle8.v v1, (a2) + add a2, a2, a6 + sub a3, a3, a6 + vzext.vf2 v2, v1 + vsrl.vi v4, v2, 0x4 + vsll.vi v4, v4, 0x8 + vand.vi v2, v2, 0xf + vor.vv v2, v2, v4 + slli t0, a6, 0x1 + vsetvli zero, t0, e8, m2, ta, ma + vmsleu.vi v6, v2, 0x8 + vrsub.vi v2, v2, 0x4 + vcpop.m t1, v6 + vcompress.vm v14, v2, v6 + sub a7, a1, a4 + bgeu a7, t1, mld_rej_uniform_eta4_riscv64_store + mv t1, a7 + +mld_rej_uniform_eta4_riscv64_store: + vsetvli zero, t1, e32, m8, ta, ma + vsext.vf4 v16, v14 + vse32.v v16, (a0) + add a4, a4, t1 + slli t1, t1, 0x2 + add a0, a0, t1 + j mld_rej_uniform_eta4_riscv64_loop + +mld_rej_uniform_eta4_riscv64_done: + mv a0, a4 + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_riscv64_asm.S b/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_riscv64_asm.S new file mode 100644 index 0000000000..a74d883b8d --- /dev/null +++ b/proofs/hol_light/riscv64/mldsa/mldsa_rej_uniform_riscv64_asm.S @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_riscv64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q + Signature: uint64_t mld_rej_uniform_riscv64_asm(int32_t *r, uint64_t len, const uint8_t *buf, uint64_t buflen) + ABI: + Architecture: riscv64 + CallingConvention: LP64D + Features: [RVV] + a0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (4 * len bytes) + a1: + type: scalar + c_parameter: uint64_t len + description: Maximum number of coefficients to sample + test_with: 256 # MLDSA_N + a2: + type: buffer + size_bytes: a3 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + a3: + type: scalar + c_parameter: uint64_t buflen + description: Length of input buffer (must be a multiple of 3) + test_with: 840 # MLD_POLY_UNIFORM_NBLOCKS * SHAKE128_RATE = 5 * 168 +*/ + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/riscv64/src/mldsa_rej_uniform_riscv64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.option push +.option norvc +.option norelax +.balign 4 +#ifdef __APPLE__ +.global _mld_rej_uniform_riscv64_asm +_mld_rej_uniform_riscv64_asm: +#else +.global mld_rej_uniform_riscv64_asm +mld_rej_uniform_riscv64_asm: +#endif + + li a4, 0x0 + lui t2, 0x7fe + addiw t2, t2, 0x1 + li t5, 0x2 + vsetvli t0, zero, e32, m2, ta, ma + slli a6, t0, 0x2 + li t4, 0x18 + vsetvli zero, a6, e16, m4, ta, ma + vid.v v8 + vsrl.vi v12, v8, 0x2 + vmul.vx v12, v12, t4 + vsrl.vi v12, v12, 0x3 + vand.vi v8, v8, 0x3 + vminu.vx v8, v8, t5 + vadd.vv v8, v8, v12 + lui t3, 0x1000 + addiw t3, t3, -0x1 + lui t3, 0x800 + addiw t3, t3, -0x1 + slli a5, t0, 0x1 + add a5, a5, t0 + +mld_rej_uniform_riscv64_loop: + bgeu a4, a1, mld_rej_uniform_riscv64_done + bltu a3, a5, mld_rej_uniform_riscv64_tail + +mld_rej_uniform_riscv64_body: + vsetvli zero, a5, e8, m2, ta, ma + vle8.v v16, (a2) + vsetvli zero, a6, e8, m2, ta, ma + vrgatherei16.vv v20, v16, v8 + vsetvli zero, t0, e32, m2, ta, ma + vand.vx v20, v20, t3 + add a2, a2, a5 + sub a3, a3, a5 + vmsltu.vx v0, v20, t2 + vcpop.m t1, v0 + vcompress.vm v24, v20, v0 + sub a7, a1, a4 + bgeu a7, t1, mld_rej_uniform_riscv64_store + mv t1, a7 + +mld_rej_uniform_riscv64_store: + vsetvli zero, t1, e32, m2, ta, ma + vse32.v v24, (a0) + add a4, a4, t1 + slli t1, t1, 0x2 + add a0, a0, t1 + j mld_rej_uniform_riscv64_loop + +mld_rej_uniform_riscv64_tail: + beqz a3, mld_rej_uniform_riscv64_done + mv a5, a3 + lui t4, 0xb + addiw t4, t4, -0x555 + mul t0, a3, t4 + srli t0, t0, 0x11 + slli a6, t0, 0x2 + j mld_rej_uniform_riscv64_body + +mld_rej_uniform_riscv64_done: + mv a0, a4 + ret + +.option pop +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/scripts/autogen b/scripts/autogen index 854be1b13e..4065e7b158 100755 --- a/scripts/autogen +++ b/scripts/autogen @@ -1072,6 +1072,117 @@ def gen_rv32im_zeta_file(): update_file("dev/riscv32/src/rv32im_zetas.c", "\n".join(gen())) +# RISC-V 64 (inverse) NTT twiddles +# +# The NTT and inverse NTT use the butterflies of the C reference: in layer +# L = 1..8 (distance d = 256 >> L), the butterflies on coefficients j and +# j + d belong to group s = j / (2 * d). The forward NTT multiplies by +# zeta(2^(L-1) + s), the inverse NTT by -zeta(2^L - 1 - s), where +# zeta(k) = root^bitrev8(k) mod q is the plain (non-Montgomery) twiddle. +# +# All twiddles are used with Barrett multiplication: multiplying x by a +# known constant z computes +# +# x * z - q * round(x * z' / 2^31) with z' = round(z * 2^31 / q) +# +# using vmul, vsmul (with rounding mode rnu) and vnmsac. Tables store each +# twiddle z as a signed canonical representative, together with z'. +# +# Layers 1-4 use scalar twiddles: the tables start with 16 pairs (z, z') +# in order of use. Layers 5-8 operate on coefficients j = 16 * b + r, with +# the block index b = 0..15 in the vector lanes and r = 0..15 in the +# registers. Their twiddles depend on the block and on the high bits of r, +# and are stored as groups of 16 twiddles z (one per block) followed by the +# 16 twisted constants z', in order of use. A kernel with VL lanes per +# register loads VL consecutive entries of each group. + + +def riscv64_zeta(k): + return signed_reduce(pow(root_of_unity, bitreverse(k, 8), modulus)) + + +def riscv64_barrett_twist(z): + return round(z * 2**31 / modulus) + + +def riscv64_ntt_twiddle(layer, s, inv): + if inv: + return signed_reduce(-riscv64_zeta(2**layer - 1 - s)) + return riscv64_zeta(2 ** (layer - 1) + s) + + +# INTT scaling: 2^32 / 256 mod q (Montgomery factor times 1/256) +riscv64_intt_scale = signed_reduce(2**24) + + +def riscv64_ntt_layout(inv): + out = [] + # Layers 1-4: (layer, s) in order of use + if inv: + scalars = [(4, s) for s in range(8)] + scalars += [(3, s) for s in range(4)] + scalars += [(2, s) for s in range(2)] + zs = [riscv64_ntt_twiddle(layer, s, inv) for (layer, s) in scalars] + # Layer 1 includes the scaling, followed by the scaling itself + zs += [signed_reduce(riscv64_ntt_twiddle(1, 0, inv) * riscv64_intt_scale)] + zs += [riscv64_intt_scale] + else: + scalars = [(1, 0)] + scalars += [(2, s) for s in range(2)] + scalars += [(3, s) for s in range(4)] + scalars += [(4, s) for s in range(8)] + zs = [riscv64_ntt_twiddle(layer, s, inv) for (layer, s) in scalars] + zs += [0] + for z in zs: + out += [z, riscv64_barrett_twist(z)] + # Layers 5-8: groups g of 16 blocks; s = 2^(L-5) * b + g + layers = [8, 7, 6, 5] if inv else [5, 6, 7, 8] + for layer in layers: + for g in range(2 ** (layer - 5)): + zs = [ + riscv64_ntt_twiddle(layer, 2 ** (layer - 5) * b + g, inv) + for b in range(16) + ] + out += zs + [riscv64_barrett_twist(z) for z in zs] + return out + + +def gen_riscv64_zeta_file(): + def gen(): + yield from gen_header() + yield '#include "../../../common.h"' + yield "" + yield "#if defined(MLD_ARITH_BACKEND_RISCV64) && \\" + yield " !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)" + yield "" + yield '#include "arith_native_riscv64.h"' + yield "" + yield "/*" + yield " * Table of twiddles used in the RISC-V forward NTT" + yield " * See autogen for details." + yield " */" + yield from emit_c_array( + "const int32_t", "mld_rv64v_ntt_zetas", riscv64_ntt_layout(inv=False) + ) + yield "" + yield "/*" + yield " * Table of twiddles used in the RISC-V inverse NTT" + yield " * See autogen for details." + yield " */" + yield from emit_c_array( + "const int32_t", "mld_rv64v_intt_zetas", riscv64_ntt_layout(inv=True) + ) + yield "" + yield "#else" + yield "" + yield "MLD_EMPTY_CU(rv64v_zetas)" + yield "" + yield "#endif" + yield "" + + update_file("dev/riscv64/src/rv64v_zetas.c", "\n".join(gen())) + + def gen_rv32im_hol_light_zeta_file(): def gen(): yield from gen_hol_light_header() @@ -1626,6 +1737,36 @@ def gen_aarch64_keccak_constants_c_file(): ) +def gen_riscv64_keccak_constants_c_file(): + def gen_c(): + yield from gen_header() + yield '#include "../../../../common.h"' + yield "" + yield "#if defined(MLD_FIPS202_RISCV64_NEED_X1_ZBB) && \\" + yield " !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)" + yield "" + yield '#include "fips202_native_riscv64.h"' + yield "" + yield from print_c_uint64_array( + "mld_keccakf1600_round_constants", + gen_keccak_round_constants(), + entries_per_line=3, + ) + yield "" + yield "#else" + yield "" + yield "MLD_EMPTY_CU(fips202_riscv64_round_constants)" + yield "" + yield "#endif" + yield "" + + update_file( + "dev/fips202/riscv64/src/keccakf1600_round_constants.c", + "\n".join(gen_c()), + force_format=True, + ) + + def gen_avx2_keccak_constants_c_file(): def gen_c(): yield from gen_header() @@ -2095,12 +2236,17 @@ def native_fips202_armv81m(c): return native_fips202(c) and armv81m(c) +def native_fips202_riscv64(c): + return native_fips202(c) and riscv64(c) + + def native_fips202_core(c): return ( native_fips202(c) and not native_fips202_x86_64(c) and not native_fips202_aarch64(c) and not native_fips202_armv81m(c) + and not native_fips202_riscv64(c) ) @@ -2227,6 +2373,11 @@ def gen_macro_undefs(extra_notes=None): filt=native_fips202_armv81m, desc="native code (FIPS202, Armv8.1-M)" ) yield "#endif" + yield "#if defined(MLD_SYS_RISCV64)" + yield from gen_monolithic_undef_all_core( + filt=native_fips202_riscv64, desc="native code (FIPS202, RISC-V 64)" + ) + yield "#endif" yield "#endif" yield "#if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH)" yield from gen_monolithic_undef_all_core(filt=native_arith_core, desc="") @@ -2240,6 +2391,11 @@ def gen_macro_undefs(extra_notes=None): filt=native_arith_x86_64, desc="native code (Arith, X86_64)" ) yield "#endif" + yield "#if defined(MLD_SYS_RISCV64)" + yield from gen_monolithic_undef_all_core( + filt=native_arith_riscv64, desc="native code (Arith, RISC-V 64)" + ) + yield "#endif" yield "#if defined(MLD_SYS_RISCV32)" yield from gen_monolithic_undef_all_core( filt=native_arith_rv32im, desc="native code (Arith, RV32IM)" @@ -2324,6 +2480,10 @@ def gen_monolithic_source_file(): for c in filter(native_arith_x86_64, c_sources): yield f'#include "{c}"' yield "#endif" + yield "#if defined(MLD_SYS_RISCV64)" + for c in filter(native_arith_riscv64, c_sources): + yield f'#include "{c}"' + yield "#endif" yield "#if defined(MLD_SYS_RISCV32)" for c in filter(native_arith_rv32im, c_sources): yield f'#include "{c}"' @@ -2343,6 +2503,10 @@ def gen_monolithic_source_file(): for c in filter(native_fips202_armv81m, c_sources): yield f'#include "{c}"' yield "#endif" + yield "#if defined(MLD_SYS_RISCV64)" + for c in filter(native_fips202_riscv64, c_sources): + yield f'#include "{c}"' + yield "#endif" yield "#endif" yield "" yield from gen_macro_undefs() @@ -2411,6 +2575,10 @@ def gen_monolithic_asm_file(): for c in filter(native_arith_x86_64, asm_sources): yield f'#include "{c}"' yield "#endif" + yield "#if defined(MLD_SYS_RISCV64)" + for c in filter(native_arith_riscv64, asm_sources): + yield f'#include "{c}"' + yield "#endif" yield "#if defined(MLD_SYS_RISCV32)" for c in filter(native_arith_rv32im, asm_sources): yield f'#include "{c}"' @@ -2430,6 +2598,10 @@ def gen_monolithic_asm_file(): for c in filter(native_fips202_armv81m, asm_sources): yield f'#include "{c}"' yield "#endif" + yield "#if defined(MLD_SYS_RISCV64)" + for c in filter(native_fips202_riscv64, asm_sources): + yield f'#include "{c}"' + yield "#endif" yield "#endif" yield "" yield "/* Mark the stack non-executable even if no assembly is included above */" @@ -2793,7 +2965,7 @@ def normalize_asm_macro_syntax(): # Architectures autogen knows how to (cross-)compile assembly for. Used to # expand a bare `--force-cross` (no value) into "force all architectures". -FORCE_CROSS_ALL_ARCHES = {"aarch64", "x86_64", "armv81m", "riscv32"} +FORCE_CROSS_ALL_ARCHES = {"aarch64", "x86_64", "armv81m", "riscv32", "riscv64"} def resolve_force_cross(value): @@ -2847,6 +3019,8 @@ def update_via_simpasm( source_arch = "armv81m" elif "riscv32" in infile_full or "rv32im" in infile_full: source_arch = "riscv32" + elif "riscv64" in infile_full: + source_arch = "riscv64" else: raise Exception(f"Could not detect architecture of source file {infile_full}.") # Check native architecture @@ -2891,6 +3065,8 @@ def update_via_simpasm( arch = "armv81m" elif "riscv32" in infile_full or "rv32im" in infile_full: arch = "riscv32" + elif "riscv64" in infile_full: + arch = "riscv64" else: arch = "x86_64" @@ -2903,7 +3079,9 @@ def update_via_simpasm( "-o", tmp.name, ] - cmd += ["--cfify"] + # TODO: Support CFI for riscv64 + if arch != "riscv64": + cmd += ["--cfify"] if cross_prefix is not None: # Stick with llvm-objdump for disassembly cmd += ["--cc", cross_prefix + "gcc"] @@ -2972,6 +3150,8 @@ def upstream_src_dir_for_dev_dir(dev_dir): return "mldsa/src/native/x86_64/src" if dev_dir.startswith("dev/riscv32/"): return "mldsa/src/native/rv32im/src" + if dev_dir.startswith("dev/riscv64/"): + return "mldsa/src/native/riscv64/src" raise ValueError( f"Unrecognised dev/ source directory {dev_dir!r}; " "extend upstream_src_dir_for_dev_dir() to map it to mldsa/src/..." @@ -3271,7 +3451,43 @@ def hol_light_asm_joblist(): ), ] - return joblist_aarch64 + joblist_x86_64 + joblist_riscv32 + riscv64_flags = ( + "-march=rv64gcv -mabi=lp64d -Idev/riscv64/src -Imldsa/src/native/riscv64/src" + ) + joblist_riscv64 = [ + (f"mldsa_{name}_riscv64_asm.S", "dev/riscv64/src", riscv64_flags, "riscv64") + for name in ( + "intt_vlen128", + "intt_vlen256", + "ntt_vlen128", + "ntt_vlen256", + "poly_caddq", + "poly_chknorm", + "poly_decompose_32", + "poly_decompose_88", + "poly_pointwise_montgomery", + "poly_use_hint_32", + "poly_use_hint_88", + "polyvecl_pointwise_acc_montgomery_l4", + "polyvecl_pointwise_acc_montgomery_l5", + "polyvecl_pointwise_acc_montgomery_l7", + "polyz_unpack_17", + "polyz_unpack_19", + "rej_uniform", + "rej_uniform_eta2", + "rej_uniform_eta4", + ) + ] + [ + ( + "keccak_f1600_x1_zbb_riscv64_asm.S", + "dev/fips202/riscv64/src", + "-march=rv64gc_zbb -mabi=lp64d -Idev/fips202/riscv64/src " + "-Imldsa/src/fips202/native/riscv64/src", + "riscv64", + ) + ] + + return joblist_aarch64 + joblist_x86_64 + joblist_riscv32 + joblist_riscv64 def gen_hol_light_asm(): @@ -3286,7 +3502,7 @@ def check_hol_light_asm_autogenerated(): } stray = sorted( os.path.join(d, fn) - for arch in ("aarch64", "x86_64", "riscv32") + for arch in ("aarch64", "x86_64", "riscv32", "riscv64") for d in [os.path.join("proofs/hol_light", arch, "mldsa")] if os.path.isdir(d) for fn in os.listdir(d) @@ -3329,6 +3545,10 @@ def synchronize_file(f, in_dir, out_dir, delete=False, no_simplify=False, **kwar if delete is True: return basename + # Skip .inc files when simplification is enabled (they get inlined by simpasm) + if no_simplify is False and f.endswith(".inc"): + return None + if no_simplify is False and f.endswith(".S"): update_via_simpasm(f, out_dir, **kwargs) else: @@ -3411,6 +3631,14 @@ def synchronize_backends( ), ) + update_via_copy( + "dev/riscv64/meta.h", + "mldsa/src/native/riscv64/meta.h", + transform=lambda c: adjust_header_guard_for_filename( + c, "mldsa/src/native/riscv64/meta.h" + ), + ) + for header in ["fastmul.h", "slowmul.h"]: update_via_copy( f"dev/riscv32/{header}", @@ -3509,6 +3737,30 @@ def synchronize_backends( no_simplify=no_simplify, cflags="-Idev/riscv32/src -Imldsa/src/native/rv32im/src -march=rv32im -mabi=ilp32", ) + synchronize_backend( + "dev/fips202/riscv64/src", + "mldsa/src/fips202/native/riscv64/src", + delete=delete, + force_cross=force_cross, + no_simplify=no_simplify, + cflags="-Idev/fips202/riscv64/src -Imldsa/src/fips202/native/riscv64/src -march=rv64gc_zbb -mabi=lp64d", + ) + synchronize_backend( + "dev/fips202/riscv64", + "mldsa/src/fips202/native/riscv64", + delete=delete, + force_cross=force_cross, + no_simplify=no_simplify, + cflags="-Idev/fips202/riscv64 -Imldsa/src/fips202/native/riscv64 -march=rv64gc_zbb -mabi=lp64d", + ) + synchronize_backend( + "dev/riscv64/src", + "mldsa/src/native/riscv64/src", + delete=delete, + force_cross=force_cross, + no_simplify=no_simplify, + cflags="-Idev/riscv64/src -Imldsa/src/native/riscv64/src -march=rv64gcv -mabi=lp64d", + ) def adjust_header_guard_for_filename(content, header_file): @@ -4602,11 +4854,29 @@ ABI_CAPS = { "test/abicheck/riscv32/selftest_riscv32.S", ], }, + # The RISC-V backend has no runtime capability check; it is enabled + # purely at compile time. + "ZBB": { + "asm_subdir": "riscv64", + "guard": "__riscv_zbb", + "syscap": None, + "description": "RISC-V Zbb extension", + "cflags": ["-march=rv64gc_zbb"], + "aux_files": [], + }, + "RVV": { + "asm_subdir": "riscv64", + "guard": "MLD_SYS_RISCV64_RVV", + "syscap": None, + "description": "RISC-V Vector extension 1.0", + "cflags": ["-march=rv64gcv"], + "aux_files": [], + }, } -# RISC-V integer-register ABI names and architectural indices. -RISCV32_GPR_NUMBERS = { +# RISC-V integer-register ABI names and architectural indices (RV32 and RV64). +RISCV_GPR_NUMBERS = { "zero": 0, "ra": 1, "sp": 2, @@ -4701,7 +4971,19 @@ ARCH_CALLINGS = { "init_func": "init_riscv32_register_state", "stub_func": "call_stub_riscv32", "check_func": "check_riscv32_ilp32_compliance", - "reg_to_field": lambda reg: f"gpr[{RISCV32_GPR_NUMBERS[reg]}]", + "reg_to_field": lambda reg: f"gpr[{RISCV_GPR_NUMBERS[reg]}]", + }, + ("riscv64", "LP64D"): { + "arch_guard": "MLD_SYS_RISCV64", + "extra_guards": [], + "asm_subdir": "riscv64", + "state_type": "struct riscv64_register_state", + "gpr_int_type": "uint64_t", + "fn_qualifier": "", + "init_func": "init_riscv64_register_state", + "stub_func": "asm_call_stub_riscv64", + "check_func": "check_riscv64_lp64d_compliance", + "reg_to_field": lambda reg: f"gpr[{RISCV_GPR_NUMBERS[reg]}]", }, } @@ -5288,6 +5570,7 @@ def _main(): gen_aarch64_zeta_file() gen_rv32im_zeta_file() gen_rv32im_hol_light_zeta_file() + gen_riscv64_zeta_file() gen_aarch64_hol_light_zeta_file() gen_aarch64_rej_uniform_table() gen_hol_light_rej_uniform_table() @@ -5300,6 +5583,7 @@ def _main(): gen_avx2_hol_light_rej_uniform_table() gen_hol_light_keccak_constants_file() gen_aarch64_keccak_constants_c_file() + gen_riscv64_keccak_constants_c_file() gen_avx2_keccak_constants_c_file() gen_avx2_keccak_hol_light_constants_file() diff --git a/scripts/check-contracts b/scripts/check-contracts index 6eb8be3bd5..0d0be7be22 100755 --- a/scripts/check-contracts +++ b/scripts/check-contracts @@ -113,7 +113,9 @@ def is_exception(funcname): # External functions return True - if funcname in ["zeroize"]: + # The VLEN query in the RISC-V backend uses an intrinsic that CBMC + # cannot model; callers use its contract. + if funcname in ["zeroize", "rv64v_vlen"]: # Implemented using inline ASM or external functions return True diff --git a/scripts/simpasm b/scripts/simpasm index 62cca9adf0..ba88501cee 100755 --- a/scripts/simpasm +++ b/scripts/simpasm @@ -295,7 +295,16 @@ def simplify(logger, args, asm_input, asm_output=None): if args.arch == "armv81m": autogen_header += [".thumb", ".syntax unified", ""] - autogen_header += [".text", ".balign 4"] + autogen_header += [".text"] + + # RISC-V: Pin the byte code by disabling compressed instructions and + # linker relaxation, matching the directives in the dev/ sources. + # This must precede .balign, which otherwise emits relaxable padding. + if args.arch == "riscv64": + autogen_header += [".option push", ".option norvc", ".option norelax"] + simplified = simplified + [".option pop"] + + autogen_header += [".balign 4"] # Add syntax specifier for Intel syntax if args.syntax and args.syntax.lower() == "intel": @@ -374,8 +383,8 @@ def simplify(logger, args, asm_input, asm_output=None): logger.debug("Checking that byte-code is unchanged ...") - # When CFI is enabled or for Armv8.1-M, compare only the __text section content - if args.cfify or args.arch == "armv81m": + # When CFI is enabled or for Armv8.1-M/riscv64, compare only the __text section content + if args.cfify or args.arch in ("armv81m", "riscv64"): logger.debug("Comparing __text section content for CFI comparison...") # Extract __text section from both files diff --git a/test/abicheck/abicheck.c b/test/abicheck/abicheck.c index 99da51c27a..f33dec5337 100644 --- a/test/abicheck/abicheck.c +++ b/test/abicheck/abicheck.c @@ -14,6 +14,8 @@ #include "armv81m/checks_armv81m_all.h" #elif defined(MLD_SYS_RISCV32) #include "riscv32/checks_riscv32_all.h" +#elif defined(MLD_SYS_RISCV64) +#include "riscv64/checks_riscv64_all.h" #else /* No abicheck support on this architecture - empty registry, driver runs zero * kernel checks (selftest still runs). */ diff --git a/test/abicheck/abicheck.mk b/test/abicheck/abicheck.mk index 0d7a7eaab3..d1f56e9a09 100644 --- a/test/abicheck/abicheck.mk +++ b/test/abicheck/abicheck.mk @@ -11,4 +11,5 @@ include test/abicheck/aarch64/abicheck_aarch64.mk include test/abicheck/armv81m/abicheck_armv81m.mk include test/abicheck/riscv32/abicheck_riscv32.mk +include test/abicheck/riscv64/abicheck_riscv64.mk include test/abicheck/x86_64/abicheck_x86_64.mk diff --git a/test/abicheck/riscv64/abicheck_riscv64.c b/test/abicheck/riscv64/abicheck_riscv64.c new file mode 100644 index 0000000000..976306f9be --- /dev/null +++ b/test/abicheck/riscv64/abicheck_riscv64.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [RVPSABI] + * RISC-V ABIs Specification + * RISC-V International + * https://github.com/riscv-non-isa/riscv-elf-psabi-doc + */ + +#include + +#include "../../notrandombytes/notrandombytes.h" +#include "abicheck_riscv64.h" + +#if defined(MLD_SYS_RISCV64) + +/* Callee-saved set per @[RVPSABI, Section "Integer Register Convention"] and + * @[RVPSABI, Section "Floating-point Register Convention"]. All vector + * registers and vector CSRs are caller-saved. */ +int check_riscv64_lp64d_compliance(struct riscv64_register_state *before, + struct riscv64_register_state *after, + int quiet) +{ + int violations = 0; + int i; + + /* gp (x3) and tp (x4) are reserved for the platform. The call stub does + * not seed them, but kernels must leave them unchanged. */ + if (before->gpr[3] != after->gpr[3]) + { + MLD_ABI_VIOLATION(quiet, "%s modified\n", "gp"); + violations++; + } + if (before->gpr[4] != after->gpr[4]) + { + MLD_ABI_VIOLATION(quiet, "%s modified\n", "tp"); + violations++; + } + + /* s0-s1 (x8-x9) and s2-s11 (x18-x27). */ + for (i = 0; i < 32; i++) + { + if ((i == 8 || i == 9 || (i >= 18 && i <= 27)) && + before->gpr[i] != after->gpr[i]) + { + MLD_ABI_VIOLATION(quiet, "x%d modified\n", i); + violations++; + } + } + +#if defined(__riscv_float_abi_double) + /* fs0-fs1 (f8-f9) and fs2-fs11 (f18-f27). */ + for (i = 0; i < 32; i++) + { + if ((i == 8 || i == 9 || (i >= 18 && i <= 27)) && + before->fpr[i] != after->fpr[i]) + { + MLD_ABI_VIOLATION(quiet, "f%d modified\n", i); + violations++; + } + } +#endif /* __riscv_float_abi_double */ + + return violations; +} + +void init_riscv64_register_state(struct riscv64_register_state *state) +{ + randombytes((uint8_t *)state, sizeof(*state)); +} + +#endif /* MLD_SYS_RISCV64 */ diff --git a/test/abicheck/riscv64/abicheck_riscv64.h b/test/abicheck/riscv64/abicheck_riscv64.h new file mode 100644 index 0000000000..13bfe28007 --- /dev/null +++ b/test/abicheck/riscv64/abicheck_riscv64.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_TEST_ABICHECK_RISCV64_ABICHECK_RISCV64_H +#define MLD_TEST_ABICHECK_RISCV64_ABICHECK_RISCV64_H + +#include "../abicheck_common.h" + +#if defined(MLD_SYS_RISCV64) + +struct riscv64_register_state +{ + uint64_t gpr[32]; /* x0-x31 */ + uint64_t fpr[32]; /* f0-f31 (64-bit, only used with the LP64D ABI) */ +}; + +/* quiet: MLD_ABICHECK_VERBOSE or MLD_ABICHECK_QUIET (see abicheck_common.h). */ +int check_riscv64_lp64d_compliance(struct riscv64_register_state *before, + struct riscv64_register_state *after, + int quiet); +void init_riscv64_register_state(struct riscv64_register_state *state); + +extern void asm_call_stub_riscv64(struct riscv64_register_state *input, + struct riscv64_register_state *output, + void (*function_ptr)(void)); + +#endif /* MLD_SYS_RISCV64 */ + +#endif /* !MLD_TEST_ABICHECK_RISCV64_ABICHECK_RISCV64_H */ diff --git a/test/abicheck/riscv64/abicheck_riscv64.mk b/test/abicheck/riscv64/abicheck_riscv64.mk new file mode 100644 index 0000000000..d1e2e0f32f --- /dev/null +++ b/test/abicheck/riscv64/abicheck_riscv64.mk @@ -0,0 +1,48 @@ +# Copyright (c) The mldsa-native project authors +# SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + +# WARNING: This file is auto-generated from scripts/autogen +# in the mldsa-native repository. +# Do not modify it directly. +# +# Edit the YAML 'ABI.Features:' list in dev//src/.S +# and re-run scripts/autogen instead. +# +# For each capability declared by a kernel's ABI.Features list, this +# file appends the capability's CFLAGS to that kernel's .S object +# under mldsa/src/. + +# Default each cap's file list to empty so the unconditional appends +# below are safe even when a cap has no kernels on this arch. +ABICHECK_REQ_RVV_FILES := +ABICHECK_REQ_ZBB_FILES := + +# RVV: RISC-V Vector extension 1.0 +ABICHECK_REQ_RVV_FILES := \ + mldsa/src/native/riscv64/src/mldsa_intt_vlen128_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_intt_vlen256_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_ntt_vlen128_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_ntt_vlen256_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_caddq_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_chknorm_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_decompose_32_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_decompose_88_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_pointwise_montgomery_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_use_hint_32_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_poly_use_hint_88_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_polyz_unpack_17_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_polyz_unpack_19_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta2_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_rej_uniform_eta4_riscv64_asm.S \ + mldsa/src/native/riscv64/src/mldsa_rej_uniform_riscv64_asm.S +ABICHECK_REQ_RVV_OBJS := $(call MAKE_OBJS,$(ABICHECK_DIR),$(ABICHECK_REQ_RVV_FILES)) +$(ABICHECK_REQ_RVV_OBJS): CFLAGS += -march=rv64gcv + +# ZBB: RISC-V Zbb extension +ABICHECK_REQ_ZBB_FILES := \ + mldsa/src/fips202/native/riscv64/src/keccak_f1600_x1_zbb_riscv64_asm.S +ABICHECK_REQ_ZBB_OBJS := $(call MAKE_OBJS,$(ABICHECK_DIR),$(ABICHECK_REQ_ZBB_FILES)) +$(ABICHECK_REQ_ZBB_OBJS): CFLAGS += -march=rv64gc_zbb diff --git a/test/abicheck/riscv64/callstub_riscv64.S b/test/abicheck/riscv64/callstub_riscv64.S new file mode 100644 index 0000000000..200fbe64f4 --- /dev/null +++ b/test/abicheck/riscv64/callstub_riscv64.S @@ -0,0 +1,151 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../mldsa/src/sys.h" + +#if defined(MLD_SYS_RISCV64) + +#if defined(__ELF__) +.section .note.GNU-stack,"",@progbits +#endif + +/* + * Function: asm_call_stub_riscv64 + * Description: RISC-V 64 ABI compliance testing stub that captures register + * state before and after function calls to verify LP64D + * compliance + * + * C Signature: void asm_call_stub_riscv64(struct riscv64_register_state *input, + * struct riscv64_register_state *output, + * void (*function_ptr)(void)) + * + * Register state layout: gpr[32] at offset 0, fpr[32] at offset 256. + * + * All GPRs except ra, sp, gp and tp are seeded from the input state. gp and + * tp are reserved for the platform: their live values are written back to + * the input state as the baseline instead. FPRs are seeded only for the + * LP64D ABI. + * + * Stack Usage: 224 bytes total (16-byte aligned) + * - 104 bytes for ra and the callee-saved GPRs s0-s11 + * - 96 bytes for the callee-saved FPRs fs0-fs11 + * - 16 bytes for the output state pointer and the function pointer + * - 8 bytes padding + */ + +#define STACK_SIZE 224 +#define STACK_BASE_GPRS 0 +#define STACK_BASE_FPRS 104 +#define STACK_OUTPUT 200 +#define STACK_FN 208 + +#define FPR_OFFSET 256 + +/* Save ra and s0-s11, with CFI offsets relative to CFA = sp + STACK_SIZE. */ +.macro save_gprs + sd ra, (STACK_BASE_GPRS + 0)(sp) + .cfi_offset ra, -(STACK_SIZE - STACK_BASE_GPRS) + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + sd s\i, (STACK_BASE_GPRS + 8 + 8*\i)(sp) + .cfi_offset s\i, -(STACK_SIZE - STACK_BASE_GPRS - 8 - 8*\i) + .endr +.endm + +.macro restore_gprs + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + ld s\i, (STACK_BASE_GPRS + 8 + 8*\i)(sp) + .cfi_restore s\i + .endr + ld ra, (STACK_BASE_GPRS + 0)(sp) + .cfi_restore ra +.endm + +#if defined(__riscv_float_abi_double) +.macro save_fprs + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + fsd fs\i, (STACK_BASE_FPRS + 8*\i)(sp) + .endr +.endm + +.macro restore_fprs + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + fld fs\i, (STACK_BASE_FPRS + 8*\i)(sp) + .endr +.endm + +/* Seed f0-f31 from the input state pointed to by \base. */ +.macro load_fprs base + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + fld f\i, (FPR_OFFSET + 8*\i)(\base) + .endr +.endm + +/* Capture f0-f31 in the output state pointed to by \base. */ +.macro store_fprs base + .irp i, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + fsd f\i, (FPR_OFFSET + 8*\i)(\base) + .endr +.endm +#else /* __riscv_float_abi_double */ +.macro save_fprs +.endm +.macro restore_fprs +.endm +.macro load_fprs base +.endm +.macro store_fprs base +.endm +#endif /* !__riscv_float_abi_double */ + +.text +.balign 4 +.global asm_call_stub_riscv64 +asm_call_stub_riscv64: + .cfi_startproc + addi sp, sp, -STACK_SIZE + .cfi_def_cfa_offset STACK_SIZE + + save_gprs + save_fprs + sd a1, STACK_OUTPUT(sp) + sd a2, STACK_FN(sp) + + /* gp and tp are not seeded: record their live values as baseline. */ + sd gp, (8*3)(a0) + sd tp, (8*4)(a0) + + load_fprs a0 + + /* Seed x5-x31 from the input state, loading a0 (x10) last since it + * holds the input pointer. ra holds the function pointer. */ + ld ra, STACK_FN(sp) + .irp i, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, \ + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld x\i, (8*\i)(a0) + .endr + ld a0, (8*10)(a0) + + jalr ra, 0(ra) + + /* Capture gp, tp and x5-x31 in the output state, using ra as the + * base register. */ + ld ra, STACK_OUTPUT(sp) + .irp i, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, \ + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + sd x\i, (8*\i)(ra) + .endr + + store_fprs ra + + restore_fprs + restore_gprs + addi sp, sp, STACK_SIZE + .cfi_def_cfa_offset 0 + ret + .cfi_endproc + +#endif /* MLD_SYS_RISCV64 */ diff --git a/test/abicheck/riscv64/checks/check_intt_vlen128_riscv64_asm.c b/test/abicheck/riscv64/checks/check_intt_vlen128_riscv64_asm.c new file mode 100644 index 0000000000..a3432cbfd8 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_intt_vlen128_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_intt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]); + +int check_intt_vlen128_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[2048]; /* Twiddle table (mld_rv64v_intt_zetas) */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 2048); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_intt_vlen128_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for intt_vlen128_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_intt_vlen256_riscv64_asm.c b/test/abicheck/riscv64/checks/check_intt_vlen256_riscv64_asm.c new file mode 100644 index 0000000000..fa1e1ab4ff --- /dev/null +++ b/test/abicheck/riscv64/checks/check_intt_vlen256_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_intt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]); + +int check_intt_vlen256_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[2048]; /* Twiddle table (mld_rv64v_intt_zetas) */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 2048); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_intt_vlen256_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for intt_vlen256_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_keccak_f1600_x1_zbb_riscv64_asm.c b/test/abicheck/riscv64/checks/check_keccak_f1600_x1_zbb_riscv64_asm.c new file mode 100644 index 0000000000..10476b3522 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_keccak_f1600_x1_zbb_riscv64_asm.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(__riscv_zbb) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_keccak_f1600_x1_zbb_riscv64_asm(uint64_t state[25], + const uint64_t rc[24]); + +int check_keccak_f1600_x1_zbb_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[200]; /* Keccak state (25 x uint64_t) */ + MLD_ALIGN uint8_t buf_a1[192]; /* Round constants (24 x uint64_t) */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 200); + randombytes(buf_a1, 192); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_keccak_f1600_x1_zbb_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for keccak_f1600_x1_zbb_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && __riscv_zbb */ diff --git a/test/abicheck/riscv64/checks/check_ntt_vlen128_riscv64_asm.c b/test/abicheck/riscv64/checks/check_ntt_vlen128_riscv64_asm.c new file mode 100644 index 0000000000..0d3d9c7ba8 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_ntt_vlen128_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_ntt_vlen128_riscv64_asm(int32_t p[256], const int32_t zetas[512]); + +int check_ntt_vlen128_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[2048]; /* Twiddle table (mld_rv64v_ntt_zetas) */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 2048); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_ntt_vlen128_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for ntt_vlen128_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_ntt_vlen256_riscv64_asm.c b/test/abicheck/riscv64/checks/check_ntt_vlen256_riscv64_asm.c new file mode 100644 index 0000000000..46b21acd06 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_ntt_vlen256_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_ntt_vlen256_riscv64_asm(int32_t p[256], const int32_t zetas[512]); + +int check_ntt_vlen256_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[2048]; /* Twiddle table (mld_rv64v_ntt_zetas) */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 2048); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_ntt_vlen256_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for ntt_vlen256_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_caddq_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_caddq_riscv64_asm.c new file mode 100644 index 0000000000..1bbe218878 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_caddq_riscv64_asm.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_poly_caddq_riscv64_asm(int32_t a[256]); + +int check_poly_caddq_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_poly_caddq_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_caddq_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_chknorm_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_chknorm_riscv64_asm.c new file mode 100644 index 0000000000..f2249dd3b9 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_chknorm_riscv64_asm.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +int mld_poly_chknorm_riscv64_asm(const int32_t a[256], int64_t B); + +int check_poly_chknorm_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input polynomial */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = 131072; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_poly_chknorm_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_chknorm_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_decompose_32_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_decompose_32_riscv64_asm.c new file mode 100644 index 0000000000..ff8c1c71fa --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_decompose_32_riscv64_asm.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_poly_decompose_32_riscv64_asm(int32_t a1[256], int32_t a0[256]); + +int check_poly_decompose_32_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial with high bits */ + MLD_ALIGN uint8_t + buf_a1[1024]; /* Input polynomial, output polynomial with low bits */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_poly_decompose_32_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_decompose_32_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_decompose_88_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_decompose_88_riscv64_asm.c new file mode 100644 index 0000000000..9c8a5ece13 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_decompose_88_riscv64_asm.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_poly_decompose_88_riscv64_asm(int32_t a1[256], int32_t a0[256]); + +int check_poly_decompose_88_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial with high bits */ + MLD_ALIGN uint8_t + buf_a1[1024]; /* Input polynomial, output polynomial with low bits */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_poly_decompose_88_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_decompose_88_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_pointwise_montgomery_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_pointwise_montgomery_riscv64_asm.c new file mode 100644 index 0000000000..845655a198 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_pointwise_montgomery_riscv64_asm.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_poly_pointwise_montgomery_riscv64_asm(int32_t a[256], + const int32_t b[256]); + +int check_poly_pointwise_montgomery_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[1024]; /* Input polynomial */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64( + &input_state, &output_state, + (void (*)(void))mld_poly_pointwise_montgomery_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_pointwise_montgomery_riscv64_asm " + "(iteration %d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_use_hint_32_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_use_hint_32_riscv64_asm.c new file mode 100644 index 0000000000..f5934264f6 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_use_hint_32_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_poly_use_hint_32_riscv64_asm(int32_t a[256], const int32_t h[256]); + +int check_poly_use_hint_32_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[1024]; /* Hint polynomial */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_poly_use_hint_32_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_use_hint_32_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_poly_use_hint_88_riscv64_asm.c b/test/abicheck/riscv64/checks/check_poly_use_hint_88_riscv64_asm.c new file mode 100644 index 0000000000..5860754951 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_poly_use_hint_88_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_poly_use_hint_88_riscv64_asm(int32_t a[256], const int32_t h[256]); + +int check_poly_use_hint_88_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Input/output polynomial */ + MLD_ALIGN uint8_t buf_a1[1024]; /* Hint polynomial */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 1024); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_poly_use_hint_88_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for poly_use_hint_88_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.c b/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.c new file mode 100644 index 0000000000..d96b6732b6 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm( + int32_t w[256], const int32_t u[4][256], const int32_t v[4][256]); + +int check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial */ + MLD_ALIGN uint8_t buf_a1[4096]; /* First input vector */ + MLD_ALIGN uint8_t buf_a2[4096]; /* Second input vector */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 4096); + randombytes(buf_a2, 4096); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + input_state.gpr[12] = (uint64_t)buf_a2; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64( + &input_state, &output_state, + (void (*)(void))mld_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for " + "polyvecl_pointwise_acc_montgomery_l4_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.c b/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.c new file mode 100644 index 0000000000..615c6dd83a --- /dev/null +++ b/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm( + int32_t w[256], const int32_t u[5][256], const int32_t v[5][256]); + +int check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial */ + MLD_ALIGN uint8_t buf_a1[5120]; /* First input vector */ + MLD_ALIGN uint8_t buf_a2[5120]; /* Second input vector */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 5120); + randombytes(buf_a2, 5120); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + input_state.gpr[12] = (uint64_t)buf_a2; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64( + &input_state, &output_state, + (void (*)(void))mld_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for " + "polyvecl_pointwise_acc_montgomery_l5_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.c b/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.c new file mode 100644 index 0000000000..48e20808e4 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm( + int32_t w[256], const int32_t u[7][256], const int32_t v[7][256]); + +int check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial */ + MLD_ALIGN uint8_t buf_a1[7168]; /* First input vector */ + MLD_ALIGN uint8_t buf_a2[7168]; /* Second input vector */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 7168); + randombytes(buf_a2, 7168); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + input_state.gpr[12] = (uint64_t)buf_a2; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64( + &input_state, &output_state, + (void (*)(void))mld_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for " + "polyvecl_pointwise_acc_montgomery_l7_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_polyz_unpack_17_riscv64_asm.c b/test/abicheck/riscv64/checks/check_polyz_unpack_17_riscv64_asm.c new file mode 100644 index 0000000000..fb94fc2e60 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_polyz_unpack_17_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_polyz_unpack_17_riscv64_asm(int32_t r[256], const uint8_t buf[576]); + +int check_polyz_unpack_17_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial */ + MLD_ALIGN uint8_t buf_a1[576]; /* Packed input bytes */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 576); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_polyz_unpack_17_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for polyz_unpack_17_riscv64_asm (iteration %d): " + "%d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_polyz_unpack_19_riscv64_asm.c b/test/abicheck/riscv64/checks/check_polyz_unpack_19_riscv64_asm.c new file mode 100644 index 0000000000..103bdb9437 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_polyz_unpack_19_riscv64_asm.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +void mld_polyz_unpack_19_riscv64_asm(int32_t r[256], const uint8_t buf[640]); + +int check_polyz_unpack_19_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output polynomial */ + MLD_ALIGN uint8_t buf_a1[640]; /* Packed input bytes */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a1, 640); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = (uint64_t)buf_a1; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_polyz_unpack_19_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for polyz_unpack_19_riscv64_asm (iteration %d): " + "%d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_rej_uniform_eta2_riscv64_asm.c b/test/abicheck/riscv64/checks/check_rej_uniform_eta2_riscv64_asm.c new file mode 100644 index 0000000000..59f585ab77 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_rej_uniform_eta2_riscv64_asm.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +uint64_t mld_rej_uniform_eta2_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen); + +int check_rej_uniform_eta2_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output buffer (4 * len bytes) */ + MLD_ALIGN uint8_t buf_a2[272]; /* Input buffer */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a2, 272); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = 256; + input_state.gpr[12] = (uint64_t)buf_a2; + input_state.gpr[13] = 272; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_rej_uniform_eta2_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for rej_uniform_eta2_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_rej_uniform_eta4_riscv64_asm.c b/test/abicheck/riscv64/checks/check_rej_uniform_eta4_riscv64_asm.c new file mode 100644 index 0000000000..0962dfd0c4 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_rej_uniform_eta4_riscv64_asm.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +uint64_t mld_rej_uniform_eta4_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen); + +int check_rej_uniform_eta4_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output buffer (4 * len bytes) */ + MLD_ALIGN uint8_t buf_a2[272]; /* Input buffer */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a2, 272); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = 256; + input_state.gpr[12] = (uint64_t)buf_a2; + input_state.gpr[13] = 272; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_rej_uniform_eta4_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for rej_uniform_eta4_riscv64_asm (iteration " + "%d): %d violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks/check_rej_uniform_riscv64_asm.c b/test/abicheck/riscv64/checks/check_rej_uniform_riscv64_asm.c new file mode 100644 index 0000000000..eb53545f46 --- /dev/null +++ b/test/abicheck/riscv64/checks/check_rej_uniform_riscv64_asm.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include + +#include "../abicheck_riscv64.h" +#include "../checks_riscv64_all.h" + +#if defined(MLD_SYS_RISCV64) && defined(MLD_SYS_RISCV64_RVV) + +#include "../../../notrandombytes/notrandombytes.h" + +typedef struct riscv64_register_state reg_state; + +uint64_t mld_rej_uniform_riscv64_asm(int32_t *r, uint64_t len, + const uint8_t *buf, uint64_t buflen); + +int check_rej_uniform_riscv64_asm(void) +{ + int test_iter; + reg_state input_state, output_state; + int violations; + MLD_ALIGN uint8_t buf_a0[1024]; /* Output buffer (4 * len bytes) */ + MLD_ALIGN uint8_t buf_a2[840]; /* Input buffer */ + + for (test_iter = 0; test_iter < MLD_ABICHECK_NUM_TESTS; test_iter++) + { + /* Initialize random register state */ + init_riscv64_register_state(&input_state); + + randombytes(buf_a0, 1024); + randombytes(buf_a2, 840); + + /* Set up register state for function arguments */ + input_state.gpr[10] = (uint64_t)buf_a0; + input_state.gpr[11] = 256; + input_state.gpr[12] = (uint64_t)buf_a2; + input_state.gpr[13] = 840; + + /* Call function through ABI test stub */ + asm_call_stub_riscv64(&input_state, &output_state, + (void (*)(void))mld_rej_uniform_riscv64_asm); + + /* Check ABI compliance */ + violations = check_riscv64_lp64d_compliance(&input_state, &output_state, + MLD_ABICHECK_VERBOSE); + if (violations > 0) + { + fprintf(stderr, + "ABI test FAILED for rej_uniform_riscv64_asm (iteration %d): %d " + "violations\n", + test_iter + 1, violations); + return MLD_ABICHECK_FAILED; + } + } + + return MLD_ABICHECK_PASSED; +} + +#endif /* MLD_SYS_RISCV64 && MLD_SYS_RISCV64_RVV */ diff --git a/test/abicheck/riscv64/checks_riscv64_all.h b/test/abicheck/riscv64/checks_riscv64_all.h new file mode 100644 index 0000000000..816761e766 --- /dev/null +++ b/test/abicheck/riscv64/checks_riscv64_all.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + + +#ifndef MLD_TEST_ABICHECK_CHECKS_RISCV64_ALL_H +#define MLD_TEST_ABICHECK_CHECKS_RISCV64_ALL_H + +#include +#include "../abicheck_common.h" + +#if defined(MLD_SYS_RISCV64) + +#if defined(__riscv_zbb) +int check_keccak_f1600_x1_zbb_riscv64_asm(void); +#endif +#if defined(MLD_SYS_RISCV64_RVV) +int check_intt_vlen128_riscv64_asm(void); +int check_intt_vlen256_riscv64_asm(void); +int check_ntt_vlen128_riscv64_asm(void); +int check_ntt_vlen256_riscv64_asm(void); +int check_poly_caddq_riscv64_asm(void); +int check_poly_chknorm_riscv64_asm(void); +int check_poly_decompose_32_riscv64_asm(void); +int check_poly_decompose_88_riscv64_asm(void); +int check_poly_pointwise_montgomery_riscv64_asm(void); +int check_poly_use_hint_32_riscv64_asm(void); +int check_poly_use_hint_88_riscv64_asm(void); +int check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm(void); +int check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm(void); +int check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm(void); +int check_polyz_unpack_17_riscv64_asm(void); +int check_polyz_unpack_19_riscv64_asm(void); +int check_rej_uniform_eta2_riscv64_asm(void); +int check_rej_uniform_eta4_riscv64_asm(void); +int check_rej_uniform_riscv64_asm(void); +#endif /* MLD_SYS_RISCV64_RVV */ + +static const abicheck_entry_t all_checks[] = { +#if defined(__riscv_zbb) + {"keccak_f1600_x1_zbb_riscv64_asm", check_keccak_f1600_x1_zbb_riscv64_asm}, +#endif +#if defined(MLD_SYS_RISCV64_RVV) + {"intt_vlen128_riscv64_asm", check_intt_vlen128_riscv64_asm}, + {"intt_vlen256_riscv64_asm", check_intt_vlen256_riscv64_asm}, + {"ntt_vlen128_riscv64_asm", check_ntt_vlen128_riscv64_asm}, + {"ntt_vlen256_riscv64_asm", check_ntt_vlen256_riscv64_asm}, + {"poly_caddq_riscv64_asm", check_poly_caddq_riscv64_asm}, + {"poly_chknorm_riscv64_asm", check_poly_chknorm_riscv64_asm}, + {"poly_decompose_32_riscv64_asm", check_poly_decompose_32_riscv64_asm}, + {"poly_decompose_88_riscv64_asm", check_poly_decompose_88_riscv64_asm}, + {"poly_pointwise_montgomery_riscv64_asm", + check_poly_pointwise_montgomery_riscv64_asm}, + {"poly_use_hint_32_riscv64_asm", check_poly_use_hint_32_riscv64_asm}, + {"poly_use_hint_88_riscv64_asm", check_poly_use_hint_88_riscv64_asm}, + {"polyvecl_pointwise_acc_montgomery_l4_riscv64_asm", + check_polyvecl_pointwise_acc_montgomery_l4_riscv64_asm}, + {"polyvecl_pointwise_acc_montgomery_l5_riscv64_asm", + check_polyvecl_pointwise_acc_montgomery_l5_riscv64_asm}, + {"polyvecl_pointwise_acc_montgomery_l7_riscv64_asm", + check_polyvecl_pointwise_acc_montgomery_l7_riscv64_asm}, + {"polyz_unpack_17_riscv64_asm", check_polyz_unpack_17_riscv64_asm}, + {"polyz_unpack_19_riscv64_asm", check_polyz_unpack_19_riscv64_asm}, + {"rej_uniform_eta2_riscv64_asm", check_rej_uniform_eta2_riscv64_asm}, + {"rej_uniform_eta4_riscv64_asm", check_rej_uniform_eta4_riscv64_asm}, + {"rej_uniform_riscv64_asm", check_rej_uniform_riscv64_asm}, +#endif /* MLD_SYS_RISCV64_RVV */ + {NULL, NULL} /* Sentinel */ +}; + +#endif /* MLD_SYS_RISCV64 */ + +#endif /* !MLD_TEST_ABICHECK_CHECKS_RISCV64_ALL_H */ diff --git a/test/abicheck/riscv64/selftest_riscv64.S b/test/abicheck/riscv64/selftest_riscv64.S new file mode 100644 index 0000000000..a02c9a8816 --- /dev/null +++ b/test/abicheck/riscv64/selftest_riscv64.S @@ -0,0 +1,74 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * RISC-V 64 LP64D selftest stubs. + * + * Each function clobbers one callee-saved register and returns: GPRs via + * `not`, FPRs via `fneg.d`. The result always differs from the input. + * + * There are no corrupters for gp and tp: the C runtime relies on them (tp + * points to thread-local storage), and the call stub does not restore them. + */ + +#include "../../../mldsa/src/sys.h" + +#if defined(__ELF__) +.section .note.GNU-stack,"",@progbits +#endif + +#if defined(MLD_SYS_RISCV64) + +.text + +.global selftest_riscv64_noop +selftest_riscv64_noop: + ret + +/* GPR corrupters s0-s11 */ +.macro corrupt_gpr reg +.global selftest_riscv64_corrupt_\reg +selftest_riscv64_corrupt_\reg: + not \reg, \reg + ret +.endm + +corrupt_gpr s0 +corrupt_gpr s1 +corrupt_gpr s2 +corrupt_gpr s3 +corrupt_gpr s4 +corrupt_gpr s5 +corrupt_gpr s6 +corrupt_gpr s7 +corrupt_gpr s8 +corrupt_gpr s9 +corrupt_gpr s10 +corrupt_gpr s11 + +#if defined(__riscv_float_abi_double) +/* FPR corrupters fs0-fs11 */ +.macro corrupt_fpr reg +.global selftest_riscv64_corrupt_\reg +selftest_riscv64_corrupt_\reg: + fneg.d \reg, \reg + ret +.endm + +corrupt_fpr fs0 +corrupt_fpr fs1 +corrupt_fpr fs2 +corrupt_fpr fs3 +corrupt_fpr fs4 +corrupt_fpr fs5 +corrupt_fpr fs6 +corrupt_fpr fs7 +corrupt_fpr fs8 +corrupt_fpr fs9 +corrupt_fpr fs10 +corrupt_fpr fs11 +#endif /* __riscv_float_abi_double */ + +#endif /* MLD_SYS_RISCV64 */ diff --git a/test/abicheck/selftest.c b/test/abicheck/selftest.c index 5f072cc439..4646c71ee8 100644 --- a/test/abicheck/selftest.c +++ b/test/abicheck/selftest.c @@ -37,6 +37,7 @@ #include "aarch64/abicheck_aarch64.h" #include "armv81m/abicheck_armv81m.h" #include "riscv32/abicheck_riscv32.h" +#include "riscv64/abicheck_riscv64.h" #include "x86_64/abicheck_x86_64.h" /* Shared registry shape: per-arch tables of (name, fn-ptr, expected count). @@ -246,8 +247,71 @@ static const selftest_entry_t riscv32_entries[] = { {NULL, NULL, 0}, }; +#elif defined(MLD_SYS_RISCV64) + +extern void selftest_riscv64_noop(void); +/* GPRs s0..s11 (12) */ +extern void selftest_riscv64_corrupt_s0(void); +extern void selftest_riscv64_corrupt_s1(void); +extern void selftest_riscv64_corrupt_s2(void); +extern void selftest_riscv64_corrupt_s3(void); +extern void selftest_riscv64_corrupt_s4(void); +extern void selftest_riscv64_corrupt_s5(void); +extern void selftest_riscv64_corrupt_s6(void); +extern void selftest_riscv64_corrupt_s7(void); +extern void selftest_riscv64_corrupt_s8(void); +extern void selftest_riscv64_corrupt_s9(void); +extern void selftest_riscv64_corrupt_s10(void); +extern void selftest_riscv64_corrupt_s11(void); +#if defined(__riscv_float_abi_double) +/* FPRs fs0..fs11 (12) */ +extern void selftest_riscv64_corrupt_fs0(void); +extern void selftest_riscv64_corrupt_fs1(void); +extern void selftest_riscv64_corrupt_fs2(void); +extern void selftest_riscv64_corrupt_fs3(void); +extern void selftest_riscv64_corrupt_fs4(void); +extern void selftest_riscv64_corrupt_fs5(void); +extern void selftest_riscv64_corrupt_fs6(void); +extern void selftest_riscv64_corrupt_fs7(void); +extern void selftest_riscv64_corrupt_fs8(void); +extern void selftest_riscv64_corrupt_fs9(void); +extern void selftest_riscv64_corrupt_fs10(void); +extern void selftest_riscv64_corrupt_fs11(void); +#endif /* __riscv_float_abi_double */ + +static const selftest_entry_t riscv64_entries[] = { + {"noop", selftest_riscv64_noop, 0}, + {"corrupt_s0", selftest_riscv64_corrupt_s0, 1}, + {"corrupt_s1", selftest_riscv64_corrupt_s1, 1}, + {"corrupt_s2", selftest_riscv64_corrupt_s2, 1}, + {"corrupt_s3", selftest_riscv64_corrupt_s3, 1}, + {"corrupt_s4", selftest_riscv64_corrupt_s4, 1}, + {"corrupt_s5", selftest_riscv64_corrupt_s5, 1}, + {"corrupt_s6", selftest_riscv64_corrupt_s6, 1}, + {"corrupt_s7", selftest_riscv64_corrupt_s7, 1}, + {"corrupt_s8", selftest_riscv64_corrupt_s8, 1}, + {"corrupt_s9", selftest_riscv64_corrupt_s9, 1}, + {"corrupt_s10", selftest_riscv64_corrupt_s10, 1}, + {"corrupt_s11", selftest_riscv64_corrupt_s11, 1}, +#if defined(__riscv_float_abi_double) + {"corrupt_fs0", selftest_riscv64_corrupt_fs0, 1}, + {"corrupt_fs1", selftest_riscv64_corrupt_fs1, 1}, + {"corrupt_fs2", selftest_riscv64_corrupt_fs2, 1}, + {"corrupt_fs3", selftest_riscv64_corrupt_fs3, 1}, + {"corrupt_fs4", selftest_riscv64_corrupt_fs4, 1}, + {"corrupt_fs5", selftest_riscv64_corrupt_fs5, 1}, + {"corrupt_fs6", selftest_riscv64_corrupt_fs6, 1}, + {"corrupt_fs7", selftest_riscv64_corrupt_fs7, 1}, + {"corrupt_fs8", selftest_riscv64_corrupt_fs8, 1}, + {"corrupt_fs9", selftest_riscv64_corrupt_fs9, 1}, + {"corrupt_fs10", selftest_riscv64_corrupt_fs10, 1}, + {"corrupt_fs11", selftest_riscv64_corrupt_fs11, 1}, +#endif /* __riscv_float_abi_double */ + {NULL, NULL, 0}, +}; + #endif /* !MLD_SYS_AARCH64 && !(MLD_SYS_X86_64 && MLD_SYSV_ABI_SUPPORTED) && \ - !MLD_SYS_ARMV81M_MVE && MLD_SYS_RISCV32 */ + !MLD_SYS_ARMV81M_MVE && !MLD_SYS_RISCV32 && MLD_SYS_RISCV64 */ int abicheck_selftest(void) { @@ -286,11 +350,16 @@ int abicheck_selftest(void) init_riscv32_register_state, call_stub_riscv32, check_riscv32_ilp32_compliance, riscv32_entries, (void (*)(void))); +#elif defined(MLD_SYS_RISCV64) + SELFTEST_RUN_ARCH("riscv64", struct riscv64_register_state, + init_riscv64_register_state, asm_call_stub_riscv64, + check_riscv64_lp64d_compliance, riscv64_entries, + (void (*)(void))); #else /* !MLD_SYS_AARCH64 && !(MLD_SYS_X86_64 && MLD_SYSV_ABI_SUPPORTED) && \ - !MLD_SYS_ARMV81M_MVE && MLD_SYS_RISCV32 */ + !MLD_SYS_ARMV81M_MVE && !MLD_SYS_RISCV32 && MLD_SYS_RISCV64 */ /* No abicheck support on this architecture. */ #endif /* !MLD_SYS_AARCH64 && !(MLD_SYS_X86_64 && MLD_SYSV_ABI_SUPPORTED) && \ - !MLD_SYS_ARMV81M_MVE && !MLD_SYS_RISCV32 */ + !MLD_SYS_ARMV81M_MVE && !MLD_SYS_RISCV32 && !MLD_SYS_RISCV64 */ return failures; } diff --git a/test/mk/auto.mk b/test/mk/auto.mk index c82b8b5d47..6beea4567b 100644 --- a/test/mk/auto.mk +++ b/test/mk/auto.mk @@ -74,6 +74,9 @@ ifeq ($(ARCH),riscv64) # Test RVV support using C with inline assembly MK_COMPILER_SUPPORTS_RVV ?= $(shell echo 'int main() { __asm__("vadd.vv v0, v1, v2"); return 0; }' | $(CC) -march=rv64gcv -x c - -c -o /dev/null 2>/dev/null && echo 1 || echo 0) +# Test Zbb support using C with inline assembly +MK_COMPILER_SUPPORTS_ZBB ?= $(shell echo 'int main() { __asm__("rori a0, a0, 1"); return 0; }' | $(CC) -march=rv64gc_zbb -x c - -c -o /dev/null 2>/dev/null && echo 1 || echo 0) + endif # riscv64 compiler detection # Define HOST_PLATFORM if not already defined @@ -142,12 +145,15 @@ ifeq ($(HOST_PLATFORM),Linux-riscv64) # Format: rv64imafdcv_sscofpmf_... -- extract single-letter extensions # (before first '_') and check for 'v' MK_HOST_SUPPORTS_RVV := $(shell sed -n '/^isa/{s/.*rv64//;s/_.*//;p;q}' /proc/cpuinfo 2>/dev/null | grep -q v && echo 1 || echo 0) +MK_HOST_SUPPORTS_ZBB := $(shell sed -n '/^isa/{p;q}' /proc/cpuinfo 2>/dev/null | grep -q '_zbb' && echo 1 || echo 0) else ifneq ($(CROSS_PREFIX),) # Cross-compilation: assume all features are supported MK_HOST_SUPPORTS_RVV := 1 +MK_HOST_SUPPORTS_ZBB := 1 else # Other platforms: assume no support MK_HOST_SUPPORTS_RVV := 0 +MK_HOST_SUPPORTS_ZBB := 0 endif # HOST_PLATFORM riscv64 endif # riscv64 @@ -188,9 +194,19 @@ endif # aarch64_be ifeq ($(ARCH),riscv64) CFLAGS += -DMLD_FORCE_RISCV64 -# Add RVV flags only if both compiler and host support it +# Add RVV and Zbb flags only if both compiler and host support them +MK_RISCV64_MARCH := rv64gc ifeq ($(MK_COMPILER_SUPPORTS_RVV)$(MK_HOST_SUPPORTS_RVV),11) -CFLAGS += -march=rv64gcv +MK_RISCV64_MARCH := $(MK_RISCV64_MARCH)v +endif +ifeq ($(MK_COMPILER_SUPPORTS_ZBB)$(MK_HOST_SUPPORTS_ZBB),11) +MK_RISCV64_MARCH := $(MK_RISCV64_MARCH)_zbb +endif +# Respect an -march given by the user +ifeq ($(findstring -march=,$(CFLAGS)),) +ifneq ($(MK_RISCV64_MARCH),rv64gc) +CFLAGS += -march=$(MK_RISCV64_MARCH) +endif endif endif # riscv64 diff --git a/test/mk/components.mk b/test/mk/components.mk index c3a51aadb9..bbb1130b8f 100644 --- a/test/mk/components.mk +++ b/test/mk/components.mk @@ -8,14 +8,16 @@ ifeq ($(OPT),1) $(wildcard mldsa/src/fips202/native/aarch64/src/*.c) \ $(wildcard mldsa/src/fips202/native/x86_64/src/*.c) \ $(wildcard mldsa/src/fips202/native/x86_64/src/*.S) \ - $(wildcard mldsa/src/fips202/native/armv81m/src/*.[csS]) + $(wildcard mldsa/src/fips202/native/armv81m/src/*.[csS]) \ + $(wildcard mldsa/src/fips202/native/riscv64/src/*.[csS]) endif SOURCES += $(wildcard mldsa/src/*.c) ifeq ($(OPT),1) SOURCES += $(wildcard mldsa/src/native/aarch64/src/*.[csS]) \ $(wildcard mldsa/src/native/x86_64/src/*.[csS]) \ - $(wildcard mldsa/src/native/rv32im/src/*.[csS]) + $(wildcard mldsa/src/native/rv32im/src/*.[csS]) \ + $(wildcard mldsa/src/native/riscv64/src/*.[csS]) CFLAGS += -DMLD_CONFIG_USE_NATIVE_BACKEND_ARITH \ -DMLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 endif @@ -202,6 +204,9 @@ else ifeq ($(ABICHECK_ARCH),armv81m) ABICHECK_ASM_SOURCES := $(wildcard mldsa/src/fips202/native/armv81m/src/*.S) else ifeq ($(ABICHECK_ARCH),riscv32) ABICHECK_ASM_SOURCES := $(wildcard mldsa/src/native/rv32im/src/*.S) +else ifeq ($(ABICHECK_ARCH),riscv64) +ABICHECK_ASM_SOURCES := $(wildcard mldsa/src/native/riscv64/src/*.S) \ + $(wildcard mldsa/src/fips202/native/riscv64/src/*.S) else ABICHECK_ASM_SOURCES := endif @@ -255,6 +260,7 @@ ABICHECK_ASM_CFLAGS := \ -DMLD_ARITH_BACKEND_AARCH64 \ -DMLD_ARITH_BACKEND_X86_64_DEFAULT \ -DMLD_ARITH_BACKEND_RV32IM \ + -DMLD_ARITH_BACKEND_RISCV64 \ -DMLD_RV32IM_NEED_FASTMUL \ -DMLD_RV32IM_NEED_SLOWMUL \ -DMLD_FIPS202_AARCH64_NEED_X1_SCALAR \ @@ -263,6 +269,7 @@ ABICHECK_ASM_CFLAGS := \ -DMLD_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID \ -DMLD_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID \ -DMLD_FIPS202_ARMV81M_NEED_X4 \ + -DMLD_FIPS202_RISCV64_NEED_X1_ZBB \ -DMLD_FIPS202_X86_64_NEED_X4_AVX2 ABICHECK_ASM_OBJS = $(call MAKE_OBJS,$(ABICHECK_DIR),$(ABICHECK_ASM_SOURCES))