Lines Matching +full:inline +full:- +full:crypto +full:- +full:engine
1 // SPDX-License-Identifier: GPL-2.0
8 #include <crypto/engine.h>
9 #include <crypto/gcm.h>
10 #include <crypto/internal/aead.h>
11 #include <crypto/internal/skcipher.h>
12 #include <crypto/scatterwalk.h>
13 #include "jh7110-cryp.h"
52 static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp) in starfive_aes_wait_busy()
56 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, in starfive_aes_wait_busy()
60 static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp) in starfive_aes_wait_keydone()
64 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, in starfive_aes_wait_keydone()
68 static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp) in starfive_aes_wait_gcmdone()
72 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, in starfive_aes_wait_gcmdone()
76 static inline int is_gcm(struct starfive_cryp_dev *cryp) in is_gcm()
78 return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM; in is_gcm()
81 static inline bool is_encrypt(struct starfive_cryp_dev *cryp) in is_encrypt()
83 return cryp->flags & FLG_ENCRYPT; in is_encrypt()
88 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_aead_hw_start()
93 value = readl(ctx->cryp->base + STARFIVE_AES_CSR); in starfive_aes_aead_hw_start()
95 writel(value, cryp->base + STARFIVE_AES_CSR); in starfive_aes_aead_hw_start()
99 value = readl(ctx->cryp->base + STARFIVE_AES_CSR); in starfive_aes_aead_hw_start()
101 writel(value, cryp->base + STARFIVE_AES_CSR); in starfive_aes_aead_hw_start()
106 static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx) in starfive_aes_set_alen()
108 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_set_alen()
110 writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0); in starfive_aes_set_alen()
111 writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1); in starfive_aes_set_alen()
114 static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx) in starfive_aes_set_mlen()
116 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_set_mlen()
118 writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0); in starfive_aes_set_mlen()
119 writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1); in starfive_aes_set_mlen()
122 static inline int starfive_aes_ccm_check_iv(const u8 *iv) in starfive_aes_ccm_check_iv()
126 return -EINVAL; in starfive_aes_ccm_check_iv()
133 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_write_iv()
135 writel(iv[0], cryp->base + STARFIVE_AES_IV0); in starfive_aes_write_iv()
136 writel(iv[1], cryp->base + STARFIVE_AES_IV1); in starfive_aes_write_iv()
137 writel(iv[2], cryp->base + STARFIVE_AES_IV2); in starfive_aes_write_iv()
141 return -ETIMEDOUT; in starfive_aes_write_iv()
146 writel(iv[3], cryp->base + STARFIVE_AES_IV3); in starfive_aes_write_iv()
151 static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv) in starfive_aes_get_iv()
153 iv[0] = readl(cryp->base + STARFIVE_AES_IV0); in starfive_aes_get_iv()
154 iv[1] = readl(cryp->base + STARFIVE_AES_IV1); in starfive_aes_get_iv()
155 iv[2] = readl(cryp->base + STARFIVE_AES_IV2); in starfive_aes_get_iv()
156 iv[3] = readl(cryp->base + STARFIVE_AES_IV3); in starfive_aes_get_iv()
159 static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce) in starfive_aes_write_nonce()
161 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_write_nonce()
163 writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0); in starfive_aes_write_nonce()
164 writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1); in starfive_aes_write_nonce()
165 writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2); in starfive_aes_write_nonce()
166 writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3); in starfive_aes_write_nonce()
171 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_write_key()
172 u32 *key = (u32 *)ctx->key; in starfive_aes_write_key()
174 if (ctx->keylen >= AES_KEYSIZE_128) { in starfive_aes_write_key()
175 writel(key[0], cryp->base + STARFIVE_AES_KEY0); in starfive_aes_write_key()
176 writel(key[1], cryp->base + STARFIVE_AES_KEY1); in starfive_aes_write_key()
177 writel(key[2], cryp->base + STARFIVE_AES_KEY2); in starfive_aes_write_key()
178 writel(key[3], cryp->base + STARFIVE_AES_KEY3); in starfive_aes_write_key()
181 if (ctx->keylen >= AES_KEYSIZE_192) { in starfive_aes_write_key()
182 writel(key[4], cryp->base + STARFIVE_AES_KEY4); in starfive_aes_write_key()
183 writel(key[5], cryp->base + STARFIVE_AES_KEY5); in starfive_aes_write_key()
186 if (ctx->keylen >= AES_KEYSIZE_256) { in starfive_aes_write_key()
187 writel(key[6], cryp->base + STARFIVE_AES_KEY6); in starfive_aes_write_key()
188 writel(key[7], cryp->base + STARFIVE_AES_KEY7); in starfive_aes_write_key()
192 return -ETIMEDOUT; in starfive_aes_write_key()
199 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_ccm_init()
203 memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE); in starfive_aes_ccm_init()
204 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); in starfive_aes_ccm_init()
209 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); in starfive_aes_ccm_init()
211 if (cryp->assoclen) in starfive_aes_ccm_init()
214 textlen = cryp->total_in; in starfive_aes_ccm_init()
216 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; in starfive_aes_ccm_init()
217 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; in starfive_aes_ccm_init()
226 struct starfive_cryp_request_ctx *rctx = ctx->rctx; in starfive_aes_hw_init()
227 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_hw_init()
231 rctx->csr.aes.v = 0; in starfive_aes_hw_init()
232 rctx->csr.aes.aesrst = 1; in starfive_aes_hw_init()
233 writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); in starfive_aes_hw_init()
236 hw_mode = cryp->flags & FLG_MODE_MASK; in starfive_aes_hw_init()
238 rctx->csr.aes.v = 0; in starfive_aes_hw_init()
240 switch (ctx->keylen) { in starfive_aes_hw_init()
242 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128; in starfive_aes_hw_init()
245 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192; in starfive_aes_hw_init()
248 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256; in starfive_aes_hw_init()
252 rctx->csr.aes.mode = hw_mode; in starfive_aes_hw_init()
253 rctx->csr.aes.cmode = !is_encrypt(cryp); in starfive_aes_hw_init()
254 rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; in starfive_aes_hw_init()
256 if (cryp->side_chan) { in starfive_aes_hw_init()
257 rctx->csr.aes.delay_aes = 1; in starfive_aes_hw_init()
258 rctx->csr.aes.vaes_start = 1; in starfive_aes_hw_init()
261 writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); in starfive_aes_hw_init()
263 cryp->err = starfive_aes_write_key(ctx); in starfive_aes_hw_init()
264 if (cryp->err) in starfive_aes_hw_init()
265 return cryp->err; in starfive_aes_hw_init()
271 writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); in starfive_aes_hw_init()
273 starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv); in starfive_aes_hw_init()
283 starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv); in starfive_aes_hw_init()
289 return cryp->err; in starfive_aes_hw_init()
294 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_read_authtag()
295 struct starfive_cryp_request_ctx *rctx = ctx->rctx; in starfive_aes_read_authtag()
299 return dev_err_probe(cryp->dev, -ETIMEDOUT, in starfive_aes_read_authtag()
302 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) { in starfive_aes_read_authtag()
303 cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0); in starfive_aes_read_authtag()
304 cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1); in starfive_aes_read_authtag()
305 cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2); in starfive_aes_read_authtag()
306 cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3); in starfive_aes_read_authtag()
309 cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); in starfive_aes_read_authtag()
313 scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg, in starfive_aes_read_authtag()
314 cryp->total_in, cryp->authsize, 1); in starfive_aes_read_authtag()
316 if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize)) in starfive_aes_read_authtag()
317 return -EBADMSG; in starfive_aes_read_authtag()
325 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_finish_req()
326 int err = cryp->err; in starfive_aes_finish_req()
328 if (!err && cryp->authsize) in starfive_aes_finish_req()
331 if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC || in starfive_aes_finish_req()
332 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR)) in starfive_aes_finish_req()
333 starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv); in starfive_aes_finish_req()
335 if (cryp->authsize) in starfive_aes_finish_req()
336 crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err); in starfive_aes_finish_req()
338 crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq, in starfive_aes_finish_req()
344 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_gcm_write_adata()
345 struct starfive_cryp_request_ctx *rctx = ctx->rctx; in starfive_aes_gcm_write_adata()
349 total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int); in starfive_aes_gcm_write_adata()
350 buffer = (u32 *)rctx->adata; in starfive_aes_gcm_write_adata()
353 writel(*buffer, cryp->base + STARFIVE_AES_NONCE0); in starfive_aes_gcm_write_adata()
355 writel(*buffer, cryp->base + STARFIVE_AES_NONCE1); in starfive_aes_gcm_write_adata()
357 writel(*buffer, cryp->base + STARFIVE_AES_NONCE2); in starfive_aes_gcm_write_adata()
359 writel(*buffer, cryp->base + STARFIVE_AES_NONCE3); in starfive_aes_gcm_write_adata()
364 return dev_err_probe(cryp->dev, -ETIMEDOUT, in starfive_aes_gcm_write_adata()
372 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_ccm_write_adata()
373 struct starfive_cryp_request_ctx *rctx = ctx->rctx; in starfive_aes_ccm_write_adata()
378 total_len = cryp->assoclen; in starfive_aes_ccm_write_adata()
380 ci = rctx->adata; in starfive_aes_ccm_write_adata()
381 writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); in starfive_aes_ccm_write_adata()
383 writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); in starfive_aes_ccm_write_adata()
385 total_len -= 2; in starfive_aes_ccm_write_adata()
389 writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); in starfive_aes_ccm_write_adata()
391 total_len -= 12; in starfive_aes_ccm_write_adata()
395 writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); in starfive_aes_ccm_write_adata()
397 total_len -= AES_BLOCK_SIZE; in starfive_aes_ccm_write_adata()
401 return dev_err_probe(cryp->dev, -ETIMEDOUT, in starfive_aes_ccm_write_adata()
411 complete(&cryp->dma_done); in starfive_aes_dma_done()
416 cryp->cfg_in.direction = DMA_MEM_TO_DEV; in starfive_aes_dma_init()
417 cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; in starfive_aes_dma_init()
418 cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; in starfive_aes_dma_init()
419 cryp->cfg_in.src_maxburst = cryp->dma_maxburst; in starfive_aes_dma_init()
420 cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; in starfive_aes_dma_init()
421 cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; in starfive_aes_dma_init()
423 dmaengine_slave_config(cryp->tx, &cryp->cfg_in); in starfive_aes_dma_init()
425 cryp->cfg_out.direction = DMA_DEV_TO_MEM; in starfive_aes_dma_init()
426 cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; in starfive_aes_dma_init()
427 cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; in starfive_aes_dma_init()
428 cryp->cfg_out.src_maxburst = 4; in starfive_aes_dma_init()
429 cryp->cfg_out.dst_maxburst = 4; in starfive_aes_dma_init()
430 cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; in starfive_aes_dma_init()
432 dmaengine_slave_config(cryp->rx, &cryp->cfg_out); in starfive_aes_dma_init()
434 init_completion(&cryp->dma_done); in starfive_aes_dma_init()
449 writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); in starfive_aes_dma_xfer()
454 writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); in starfive_aes_dma_xfer()
455 writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET); in starfive_aes_dma_xfer()
460 out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM, in starfive_aes_dma_xfer()
463 ret = -EINVAL; in starfive_aes_dma_xfer()
467 out_desc->callback = starfive_aes_dma_done; in starfive_aes_dma_xfer()
468 out_desc->callback_param = cryp; in starfive_aes_dma_xfer()
470 reinit_completion(&cryp->dma_done); in starfive_aes_dma_xfer()
472 dma_async_issue_pending(cryp->rx); in starfive_aes_dma_xfer()
474 in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV, in starfive_aes_dma_xfer()
477 ret = -EINVAL; in starfive_aes_dma_xfer()
482 dma_async_issue_pending(cryp->tx); in starfive_aes_dma_xfer()
484 if (!wait_for_completion_timeout(&cryp->dma_done, in starfive_aes_dma_xfer()
486 ret = -ETIMEDOUT; in starfive_aes_dma_xfer()
494 writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); in starfive_aes_dma_xfer()
505 unsigned int remain = cryp->total_in; in starfive_aes_map_sg()
512 src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); in starfive_aes_map_sg()
514 return dev_err_probe(cryp->dev, -ENOMEM, in starfive_aes_map_sg()
521 dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); in starfive_aes_map_sg()
525 remain -= len; in starfive_aes_map_sg()
529 src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); in starfive_aes_map_sg()
531 return dev_err_probe(cryp->dev, -ENOMEM, in starfive_aes_map_sg()
534 dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); in starfive_aes_map_sg()
536 return dev_err_probe(cryp->dev, -ENOMEM, in starfive_aes_map_sg()
543 dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); in starfive_aes_map_sg()
544 dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); in starfive_aes_map_sg()
548 remain -= len; in starfive_aes_map_sg()
552 if (sg_dma_len(stsg) - len) { in starfive_aes_map_sg()
555 } else if (sg_dma_len(dtsg) - len) { in starfive_aes_map_sg()
568 static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) in starfive_aes_do_one_req() argument
575 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_do_one_req()
578 cryp->req.sreq = req; in starfive_aes_do_one_req()
579 cryp->total_in = req->cryptlen; in starfive_aes_do_one_req()
580 cryp->total_out = req->cryptlen; in starfive_aes_do_one_req()
581 cryp->assoclen = 0; in starfive_aes_do_one_req()
582 cryp->authsize = 0; in starfive_aes_do_one_req()
584 rctx->in_sg = req->src; in starfive_aes_do_one_req()
585 rctx->out_sg = req->dst; in starfive_aes_do_one_req()
587 ctx->rctx = rctx; in starfive_aes_do_one_req()
593 if (!cryp->total_in) in starfive_aes_do_one_req()
598 ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); in starfive_aes_do_one_req()
613 ctx->cryp = starfive_cryp_find_dev(ctx); in starfive_aes_init_tfm()
614 if (!ctx->cryp) in starfive_aes_init_tfm()
615 return -ENODEV; in starfive_aes_init_tfm()
617 ctx->skcipher_fbk = crypto_alloc_skcipher(alg_name, 0, in starfive_aes_init_tfm()
619 if (IS_ERR(ctx->skcipher_fbk)) in starfive_aes_init_tfm()
620 return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk), in starfive_aes_init_tfm()
625 crypto_skcipher_reqsize(ctx->skcipher_fbk)); in starfive_aes_init_tfm()
634 crypto_free_skcipher(ctx->skcipher_fbk); in starfive_aes_exit_tfm()
637 static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq) in starfive_aes_aead_do_one_req() argument
643 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_aead_do_one_req()
648 cryp->req.areq = req; in starfive_aes_aead_do_one_req()
649 cryp->assoclen = req->assoclen; in starfive_aes_aead_do_one_req()
650 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); in starfive_aes_aead_do_one_req()
652 rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen); in starfive_aes_aead_do_one_req()
653 if (req->src == req->dst) in starfive_aes_aead_do_one_req()
654 rctx->out_sg = rctx->in_sg; in starfive_aes_aead_do_one_req()
656 rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen); in starfive_aes_aead_do_one_req()
659 cryp->total_in = req->cryptlen; in starfive_aes_aead_do_one_req()
660 cryp->total_out = req->cryptlen; in starfive_aes_aead_do_one_req()
662 cryp->total_in = req->cryptlen - cryp->authsize; in starfive_aes_aead_do_one_req()
663 cryp->total_out = cryp->total_in; in starfive_aes_aead_do_one_req()
664 scatterwalk_map_and_copy(cryp->tag_in, req->src, in starfive_aes_aead_do_one_req()
665 cryp->total_in + cryp->assoclen, in starfive_aes_aead_do_one_req()
666 cryp->authsize, 0); in starfive_aes_aead_do_one_req()
669 if (cryp->assoclen) { in starfive_aes_aead_do_one_req()
670 rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); in starfive_aes_aead_do_one_req()
671 if (!rctx->adata) in starfive_aes_aead_do_one_req()
672 return dev_err_probe(cryp->dev, -ENOMEM, in starfive_aes_aead_do_one_req()
675 if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), in starfive_aes_aead_do_one_req()
676 rctx->adata, cryp->assoclen) != cryp->assoclen) in starfive_aes_aead_do_one_req()
677 return -EINVAL; in starfive_aes_aead_do_one_req()
680 if (cryp->total_in) in starfive_aes_aead_do_one_req()
681 sg_zero_buffer(rctx->in_sg, sg_nents(rctx->in_sg), in starfive_aes_aead_do_one_req()
682 sg_dma_len(rctx->in_sg) - cryp->total_in, in starfive_aes_aead_do_one_req()
683 cryp->total_in); in starfive_aes_aead_do_one_req()
685 ctx->rctx = rctx; in starfive_aes_aead_do_one_req()
691 if (!cryp->assoclen) in starfive_aes_aead_do_one_req()
694 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) in starfive_aes_aead_do_one_req()
699 kfree(rctx->adata); in starfive_aes_aead_do_one_req()
705 if (!cryp->total_in) in starfive_aes_aead_do_one_req()
710 ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); in starfive_aes_aead_do_one_req()
724 ctx->cryp = starfive_cryp_find_dev(ctx); in starfive_aes_aead_init_tfm()
725 if (!ctx->cryp) in starfive_aes_aead_init_tfm()
726 return -ENODEV; in starfive_aes_aead_init_tfm()
728 ctx->aead_fbk = crypto_alloc_aead(alg_name, 0, in starfive_aes_aead_init_tfm()
730 if (IS_ERR(ctx->aead_fbk)) in starfive_aes_aead_init_tfm()
731 return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk), in starfive_aes_aead_init_tfm()
736 crypto_aead_reqsize(ctx->aead_fbk)); in starfive_aes_aead_init_tfm()
745 crypto_free_aead(ctx->aead_fbk); in starfive_aes_aead_exit_tfm()
756 if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || in starfive_aes_check_unaligned()
757 (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && in starfive_aes_check_unaligned()
763 if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || in starfive_aes_check_unaligned()
764 (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && in starfive_aes_check_unaligned()
777 skcipher_request_set_tfm(subreq, ctx->skcipher_fbk); in starfive_aes_do_fallback()
778 skcipher_request_set_callback(subreq, req->base.flags, in starfive_aes_do_fallback()
779 req->base.complete, in starfive_aes_do_fallback()
780 req->base.data); in starfive_aes_do_fallback()
781 skcipher_request_set_crypt(subreq, req->src, req->dst, in starfive_aes_do_fallback()
782 req->cryptlen, req->iv); in starfive_aes_do_fallback()
792 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_crypt()
793 unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1; in starfive_aes_crypt()
795 cryp->flags = flags; in starfive_aes_crypt()
797 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB || in starfive_aes_crypt()
798 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC) in starfive_aes_crypt()
799 if (req->cryptlen & blocksize_align) in starfive_aes_crypt()
800 return -EINVAL; in starfive_aes_crypt()
802 if (starfive_aes_check_unaligned(cryp, req->src, req->dst)) in starfive_aes_crypt()
805 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); in starfive_aes_crypt()
814 aead_request_set_tfm(subreq, ctx->aead_fbk); in starfive_aes_aead_do_fallback()
815 aead_request_set_callback(subreq, req->base.flags, in starfive_aes_aead_do_fallback()
816 req->base.complete, in starfive_aes_aead_do_fallback()
817 req->base.data); in starfive_aes_aead_do_fallback()
818 aead_request_set_crypt(subreq, req->src, req->dst, in starfive_aes_aead_do_fallback()
819 req->cryptlen, req->iv); in starfive_aes_aead_do_fallback()
820 aead_request_set_ad(subreq, req->assoclen); in starfive_aes_aead_do_fallback()
829 struct starfive_cryp_dev *cryp = ctx->cryp; in starfive_aes_aead_crypt()
832 cryp->flags = flags; in starfive_aes_aead_crypt()
834 /* aes-ccm does not support tag verification for non-aligned text, in starfive_aes_aead_crypt()
837 if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) && in starfive_aes_aead_crypt()
841 src = scatterwalk_ffwd(_src, req->src, req->assoclen); in starfive_aes_aead_crypt()
843 if (req->src == req->dst) in starfive_aes_aead_crypt()
846 dst = scatterwalk_ffwd(_dst, req->dst, req->assoclen); in starfive_aes_aead_crypt()
851 return crypto_transfer_aead_request_to_engine(cryp->engine, req); in starfive_aes_aead_crypt()
860 return -EINVAL; in starfive_aes_setkey()
865 return -EINVAL; in starfive_aes_setkey()
867 memcpy(ctx->key, key, keylen); in starfive_aes_setkey()
868 ctx->keylen = keylen; in starfive_aes_setkey()
870 return crypto_skcipher_setkey(ctx->skcipher_fbk, key, keylen); in starfive_aes_setkey()
879 return -EINVAL; in starfive_aes_aead_setkey()
884 return -EINVAL; in starfive_aes_aead_setkey()
886 memcpy(ctx->key, key, keylen); in starfive_aes_aead_setkey()
887 ctx->keylen = keylen; in starfive_aes_aead_setkey()
889 return crypto_aead_setkey(ctx->aead_fbk, key, keylen); in starfive_aes_aead_setkey()
902 return crypto_aead_setauthsize(ctx->aead_fbk, authsize); in starfive_aes_gcm_setauthsize()
920 return -EINVAL; in starfive_aes_ccm_setauthsize()
923 return crypto_aead_setauthsize(ctx->aead_fbk, authsize); in starfive_aes_ccm_setauthsize()
970 ret = starfive_aes_ccm_check_iv(req->iv); in starfive_aes_ccm_encrypt()
981 ret = starfive_aes_ccm_check_iv(req->iv); in starfive_aes_ccm_decrypt()
990 return starfive_aes_init_tfm(tfm, "ecb(aes-generic)"); in starfive_aes_ecb_init_tfm()
995 return starfive_aes_init_tfm(tfm, "cbc(aes-generic)"); in starfive_aes_cbc_init_tfm()
1000 return starfive_aes_init_tfm(tfm, "ctr(aes-generic)"); in starfive_aes_ctr_init_tfm()
1005 return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-generic),cbcmac(aes-generic))"); in starfive_aes_ccm_init_tfm()
1010 return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-generic),ghash-generic)"); in starfive_aes_gcm_init_tfm()
1024 .cra_driver_name = "starfive-ecb-aes",
1047 .cra_driver_name = "starfive-cbc-aes",
1070 .cra_driver_name = "starfive-ctr-aes",
1097 .cra_driver_name = "starfive-gcm-aes",
1120 .cra_driver_name = "starfive-ccm-aes",