From 0a2e427964ad53c33831c66fd7566d2713754373 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Krotowski?= Date: Fri, 25 Apr 2014 14:15:51 +0200 Subject: [PATCH] - padmcrypt updated from origin for 3.14 --- bio-kmalloc-align.patch | 37 ++++++ dm-crypt-dont-allocate-partial-pages.patch | 99 ++++++++------- dm-crypt-fix-allocation-deadlock.patch | 41 +++--- dm-crypt-offload-writes-to-thread.patch | 36 +++--- dm-crypt-per_bio_data.patch | 138 +++++++++++++++++++++ dm-crypt-remove-io-pool.patch | 93 ++++++++++++++ dm-crypt-remove-percpu.patch | 56 +++++---- dm-crypt-sort-requests.patch | 28 ++--- dm-crypt-unbound-workqueue.patch | 12 +- kernel.spec | 16 ++- 10 files changed, 426 insertions(+), 130 deletions(-) create mode 100644 bio-kmalloc-align.patch create mode 100644 dm-crypt-per_bio_data.patch create mode 100644 dm-crypt-remove-io-pool.patch diff --git a/bio-kmalloc-align.patch b/bio-kmalloc-align.patch new file mode 100644 index 00000000..372b3cef --- /dev/null +++ b/bio-kmalloc-align.patch @@ -0,0 +1,37 @@ +bio: use kmalloc alignment for bio slab + +Various subsystems can ask the bio subsystem to create a bio slab cache +with some free space before the bio. This free space can be used for any +purpose. Device mapper uses this feature to place some target-specific and +device-mapper specific data before the bio, so that the target-specific +data doesn't have to be allocated separatedly. + +This mechanism is used in place of kmalloc, so we need that the allocated +slab have the same memory alignment as memory allocated with kmalloc. + +This patch changes the function bio_find_or_create_slab so that it uses +ARCH_KMALLOC_MINALIGN alignment when creating the slab cache. This patch +is needed so that dm-crypt can use per-bio data for encryption - the +crypto subsystem assumes that these data have the same alignment as +kmallocated memory. + +Signed-off-by: Mikulas Patocka + +--- + fs/bio.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +Index: linux-3.14-rc3/fs/bio.c +=================================================================== +--- linux-3.14-rc3.orig/fs/bio.c 2014-02-23 23:53:50.000000000 +0100 ++++ linux-3.14-rc3/fs/bio.c 2014-02-23 23:55:00.000000000 +0100 +@@ -112,7 +112,8 @@ static struct kmem_cache *bio_find_or_cr + bslab = &bio_slabs[entry]; + + snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry); +- slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL); ++ slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN, ++ SLAB_HWCACHE_ALIGN, NULL); + if (!slab) + goto out_unlock; + diff --git a/dm-crypt-dont-allocate-partial-pages.patch b/dm-crypt-dont-allocate-partial-pages.patch index b14bc222..b387f5d6 100644 --- a/dm-crypt-dont-allocate-partial-pages.patch +++ b/dm-crypt-dont-allocate-partial-pages.patch @@ -11,22 +11,22 @@ Note: the next patch is needed to fix a theoretical deadlock Signed-off-by: Mikulas Patocka --- - drivers/md/dm-crypt.c | 134 +++++++++----------------------------------------- - 1 file changed, 26 insertions(+), 108 deletions(-) + drivers/md/dm-crypt.c | 139 ++++++++++---------------------------------------- + 1 file changed, 30 insertions(+), 109 deletions(-) -Index: linux-3.10.4-fast/drivers/md/dm-crypt.c +Index: linux-3.14/drivers/md/dm-crypt.c =================================================================== ---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:18.000000000 +0200 -+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:21.000000000 +0200 -@@ -59,7 +59,6 @@ struct dm_crypt_io { +--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 20:48:50.000000000 +0200 ++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 20:57:36.000000000 +0200 +@@ -58,7 +58,6 @@ struct dm_crypt_io { atomic_t io_pending; int error; sector_t sector; - struct dm_crypt_io *base_io; - }; + } CRYPTO_MINALIGN_ATTR; struct dm_crypt_request { -@@ -162,7 +161,6 @@ struct crypt_config { +@@ -172,7 +171,6 @@ struct crypt_config { }; #define MIN_IOS 16 @@ -34,7 +34,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c static struct kmem_cache *_crypt_io_pool; -@@ -777,14 +775,13 @@ static int crypt_convert(struct crypt_co +@@ -951,14 +949,13 @@ static int crypt_convert(struct crypt_co return 0; } @@ -52,7 +52,14 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c { struct crypt_config *cc = io->cc; struct bio *clone; -@@ -798,37 +795,23 @@ static struct bio *crypt_alloc_buffer(st +@@ -966,41 +963,27 @@ static struct bio *crypt_alloc_buffer(st + gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; + unsigned i, len; + struct page *page; ++ struct bio_vec *bvec; + + clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); + if (!clone) return NULL; clone_init(io, clone); @@ -74,35 +81,35 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c len = (size > PAGE_SIZE) ? PAGE_SIZE : size; - if (!bio_add_page(clone, page, len, 0)) { -+ DMERR("bio_add_page failed for page %d: the underlying device has stricter limits than dm-crypt target", i); - mempool_free(page, cc->page_pool); +- if (!bio_add_page(clone, page, len, 0)) { +- mempool_free(page, cc->page_pool); - break; -+ crypt_free_buffer_pages(cc, clone); -+ bio_put(clone); -+ return NULL; - } +- } ++ bvec = &clone->bi_io_vec[clone->bi_vcnt++]; ++ bvec->bv_page = page; ++ bvec->bv_len = len; ++ bvec->bv_offset = 0; - size -= len; - } +- size -= len; +- } ++ clone->bi_iter.bi_size += len; -- if (!clone->bi_size) { +- if (!clone->bi_iter.bi_size) { - bio_put(clone); - return NULL; -- } -- - return clone; - } ++ size -= len; + } -@@ -854,7 +837,6 @@ static struct dm_crypt_io *crypt_io_allo + return clone; +@@ -1025,7 +1008,6 @@ static void crypt_io_init(struct dm_cryp io->base_bio = bio; io->sector = sector; io->error = 0; - io->base_io = NULL; io->ctx.req = NULL; atomic_set(&io->io_pending, 0); - -@@ -869,13 +851,11 @@ static void crypt_inc_pending(struct dm_ + } +@@ -1038,13 +1020,11 @@ static void crypt_inc_pending(struct dm_ /* * One of the bios was finished. Check for completion of * the whole request and correctly clean up the buffer. @@ -116,9 +123,9 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c int error = io->error; if (!atomic_dec_and_test(&io->io_pending)) -@@ -885,13 +865,7 @@ static void crypt_dec_pending(struct dm_ - mempool_free(io->ctx.req, cc->req_pool); - mempool_free(io, cc->io_pool); +@@ -1055,13 +1035,7 @@ static void crypt_dec_pending(struct dm_ + if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size)) + mempool_free(io, cc->io_pool); - if (likely(!base_io)) - bio_endio(base_bio, error); @@ -131,18 +138,18 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c } /* -@@ -1027,10 +1001,7 @@ static void kcryptd_crypt_write_convert( +@@ -1197,10 +1171,7 @@ static void kcryptd_crypt_write_convert( { struct crypt_config *cc = io->cc; struct bio *clone; - struct dm_crypt_io *new_io; int crypt_finished; - unsigned out_of_pages = 0; -- unsigned remaining = io->base_bio->bi_size; +- unsigned remaining = io->base_bio->bi_iter.bi_size; sector_t sector = io->sector; int r; -@@ -1040,81 +1011,28 @@ static void kcryptd_crypt_write_convert( +@@ -1210,80 +1181,30 @@ static void kcryptd_crypt_write_convert( crypt_inc_pending(io); crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); @@ -158,9 +165,9 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c - } - - io->ctx.bio_out = clone; -- io->ctx.idx_out = 0; +- io->ctx.iter_out = clone->bi_iter; - -- remaining -= clone->bi_size; +- remaining -= clone->bi_iter.bi_size; - sector += bio_sectors(clone); - - crypt_inc_pending(io); @@ -181,7 +188,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c - */ - if (unlikely(r < 0)) - break; -+ clone = crypt_alloc_buffer(io, io->base_bio->bi_size); ++ clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); + if (unlikely(!clone)) { + io->error = -EIO; + goto dec; @@ -190,7 +197,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c - io->sector = sector; - } + io->ctx.bio_out = clone; -+ io->ctx.idx_out = 0; ++ io->ctx.iter_out = clone->bi_iter; - /* - * Out of memory -> run queues @@ -205,13 +212,12 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c - * between fragments, so switch to a new dm_crypt_io structure. - */ - if (unlikely(!crypt_finished && remaining)) { -- new_io = crypt_io_alloc(io->cc, io->base_bio, -- sector); +- new_io = mempool_alloc(cc->io_pool, GFP_NOIO); +- crypt_io_init(new_io, io->cc, io->base_bio, sector); - crypt_inc_pending(new_io); - crypt_convert_init(cc, &new_io->ctx, NULL, - io->base_bio, sector); -- new_io->ctx.idx_in = io->ctx.idx_in; -- new_io->ctx.offset_in = io->ctx.offset_in; +- new_io->ctx.iter_in = io->ctx.iter_in; - - /* - * Fragments after the first use the base_io @@ -232,18 +238,19 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c - io = new_io; - } -- } + /* Encryption was already finished, submit io now */ -+ if (crypt_finished) ++ if (crypt_finished) { + kcryptd_crypt_write_io_submit(io, 0); ++ io->sector = sector; + } +dec: crypt_dec_pending(io); } -@@ -1553,7 +1471,7 @@ static int crypt_ctr(struct dm_target *t - goto bad; - } +@@ -1738,7 +1659,7 @@ static int crypt_ctr(struct dm_target *t + sizeof(struct dm_crypt_io) + cc->dmreq_start + + sizeof(struct dm_crypt_request) + cc->iv_size; - cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); + cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0); diff --git a/dm-crypt-fix-allocation-deadlock.patch b/dm-crypt-fix-allocation-deadlock.patch index 9971e09b..80994da6 100644 --- a/dm-crypt-fix-allocation-deadlock.patch +++ b/dm-crypt-fix-allocation-deadlock.patch @@ -17,14 +17,14 @@ to a blocking allocation with a mutex. Signed-off-by: Mikulas Patocka --- - drivers/md/dm-crypt.c | 36 +++++++++++++++++++++++++++++++++--- - 1 file changed, 33 insertions(+), 3 deletions(-) + drivers/md/dm-crypt.c | 41 ++++++++++++++++++++++++++++++++++++----- + 1 file changed, 36 insertions(+), 5 deletions(-) -Index: linux-3.10.4-fast/drivers/md/dm-crypt.c +Index: linux-3.14/drivers/md/dm-crypt.c =================================================================== ---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:21.000000000 +0200 -+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:24.000000000 +0200 -@@ -118,6 +118,7 @@ struct crypt_config { +--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 20:59:46.000000000 +0200 ++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:04:40.000000000 +0200 +@@ -124,6 +124,7 @@ struct crypt_config { mempool_t *req_pool; mempool_t *page_pool; struct bio_set *bs; @@ -32,7 +32,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c struct workqueue_struct *io_queue; struct workqueue_struct *crypt_queue; -@@ -780,24 +781,46 @@ static void crypt_free_buffer_pages(stru +@@ -954,27 +955,51 @@ static void crypt_free_buffer_pages(stru /* * Generate a new unfragmented bio with the given size * This should never violate the device limitations @@ -55,9 +55,11 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c struct bio *clone; unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; +- unsigned i, len; + gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; - unsigned i, len; ++ unsigned i, len, remaining_size; struct page *page; + struct bio_vec *bvec; +retry: + if (unlikely(gfp_mask & __GFP_WAIT)) @@ -70,6 +72,8 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c clone_init(io, clone); ++ remaining_size = size; ++ for (i = 0; i < nr_iovecs; i++) { page = mempool_alloc(cc->page_pool, gfp_mask); + if (!page) { @@ -79,18 +83,17 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c + goto retry; + } - len = (size > PAGE_SIZE) ? PAGE_SIZE : size; +- len = (size > PAGE_SIZE) ? PAGE_SIZE : size; ++ len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; + + bvec = &clone->bi_io_vec[clone->bi_vcnt++]; + bvec->bv_page = page; +@@ -983,9 +1008,13 @@ static struct bio *crypt_alloc_buffer(st -@@ -806,12 +829,17 @@ static struct bio *crypt_alloc_buffer(st - mempool_free(page, cc->page_pool); - crypt_free_buffer_pages(cc, clone); - bio_put(clone); -- return NULL; -+ clone = NULL; -+ goto return_clone; - } + clone->bi_iter.bi_size += len; - size -= len; +- size -= len; ++ remaining_size -= len; } +return_clone: @@ -100,7 +103,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c return clone; } -@@ -1483,6 +1511,8 @@ static int crypt_ctr(struct dm_target *t +@@ -1671,6 +1700,8 @@ static int crypt_ctr(struct dm_target *t goto bad; } diff --git a/dm-crypt-offload-writes-to-thread.patch b/dm-crypt-offload-writes-to-thread.patch index 2fe475c1..8fdec56c 100644 --- a/dm-crypt-offload-writes-to-thread.patch +++ b/dm-crypt-offload-writes-to-thread.patch @@ -21,11 +21,11 @@ Signed-off-by: Mikulas Patocka drivers/md/dm-crypt.c | 120 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 97 insertions(+), 23 deletions(-) -Index: linux-3.10.4-fast/drivers/md/dm-crypt.c +Index: linux-3.14/drivers/md/dm-crypt.c =================================================================== ---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:24.000000000 +0200 -+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:27.000000000 +0200 -@@ -17,6 +17,7 @@ +--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:05:40.000000000 +0200 ++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:06:22.000000000 +0200 +@@ -18,6 +18,7 @@ #include #include #include @@ -33,16 +33,16 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c #include #include #include -@@ -59,6 +60,8 @@ struct dm_crypt_io { +@@ -58,6 +59,8 @@ struct dm_crypt_io { atomic_t io_pending; int error; sector_t sector; + + struct list_head list; - }; + } CRYPTO_MINALIGN_ATTR; struct dm_crypt_request { -@@ -123,6 +126,10 @@ struct crypt_config { +@@ -128,6 +131,10 @@ struct crypt_config { struct workqueue_struct *io_queue; struct workqueue_struct *crypt_queue; @@ -53,7 +53,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c char *cipher; char *cipher_string; -@@ -975,37 +982,89 @@ static int kcryptd_io_read(struct dm_cry +@@ -1141,37 +1148,89 @@ static int kcryptd_io_read(struct dm_cry return 0; } @@ -159,9 +159,9 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c if (unlikely(io->error < 0)) { crypt_free_buffer_pages(cc, clone); -@@ -1019,10 +1078,10 @@ static void kcryptd_crypt_write_io_submi +@@ -1185,10 +1244,10 @@ static void kcryptd_crypt_write_io_submi - clone->bi_sector = cc->start + io->sector; + clone->bi_iter.bi_sector = cc->start + io->sector; - if (async) - kcryptd_queue_io(io); @@ -174,16 +174,16 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c } static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) -@@ -1058,7 +1117,7 @@ static void kcryptd_crypt_write_convert( +@@ -1224,7 +1283,7 @@ static void kcryptd_crypt_write_convert( /* Encryption was already finished, submit io now */ - if (crypt_finished) + if (crypt_finished) { - kcryptd_crypt_write_io_submit(io, 0); + kcryptd_crypt_write_io_submit(io); + io->sector = sector; + } - dec: - crypt_dec_pending(io); -@@ -1116,7 +1175,7 @@ static void kcryptd_async_done(struct cr +@@ -1284,7 +1343,7 @@ static void kcryptd_async_done(struct cr if (bio_data_dir(io->base_bio) == READ) kcryptd_crypt_read_done(io); else @@ -192,7 +192,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c } static void kcryptd_crypt(struct work_struct *work) -@@ -1260,6 +1319,9 @@ static void crypt_dtr(struct dm_target * +@@ -1431,6 +1490,9 @@ static void crypt_dtr(struct dm_target * if (!cc) return; @@ -202,7 +202,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c if (cc->io_queue) destroy_workqueue(cc->io_queue); if (cc->crypt_queue) -@@ -1576,6 +1638,18 @@ static int crypt_ctr(struct dm_target *t +@@ -1745,6 +1807,18 @@ static int crypt_ctr(struct dm_target *t goto bad; } @@ -221,7 +221,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c ti->num_flush_bios = 1; ti->discard_zeroes_data_unsupported = true; -@@ -1607,7 +1681,7 @@ static int crypt_map(struct dm_target *t +@@ -1779,7 +1853,7 @@ static int crypt_map(struct dm_target *t if (bio_data_dir(io->base_bio) == READ) { if (kcryptd_io_read(io, GFP_NOWAIT)) diff --git a/dm-crypt-per_bio_data.patch b/dm-crypt-per_bio_data.patch new file mode 100644 index 00000000..bfb02887 --- /dev/null +++ b/dm-crypt-per_bio_data.patch @@ -0,0 +1,138 @@ +dm-crypt: use per-bio data + +This patch changes dm-crypt so that it uses auxiliary data allocated with +the bio. + +Dm-crypt requires two allocations per request - struct dm_crypt_io and +struct ablkcipher_request (with other data appended to it). It used +mempool for the allocation. + +Some requests may require more dm_crypt_ios and ablkcipher_requests, +however most requests need just one of each of these two structures to +complete. + +This patch changes it so that the first dm_crypt_io and ablkcipher_request +and allocated with the bio (using target per_bio_data_size option). If the +request needs additional values, they are allocated from the mempool. + +Signed-off-by: Mikulas Patocka + +--- + drivers/md/dm-crypt.c | 40 ++++++++++++++++++++++++++-------------- + 1 file changed, 26 insertions(+), 14 deletions(-) + +Index: linux-3.14-rc4/drivers/md/dm-crypt.c +=================================================================== +--- linux-3.14-rc4.orig/drivers/md/dm-crypt.c 2014-02-27 17:48:31.000000000 +0100 ++++ linux-3.14-rc4/drivers/md/dm-crypt.c 2014-02-27 17:48:31.000000000 +0100 +@@ -59,7 +59,7 @@ struct dm_crypt_io { + int error; + sector_t sector; + struct dm_crypt_io *base_io; +-}; ++} CRYPTO_MINALIGN_ATTR; + + struct dm_crypt_request { + struct convert_context *ctx; +@@ -162,6 +162,8 @@ struct crypt_config { + */ + unsigned int dmreq_start; + ++ unsigned int per_bio_data_size; ++ + unsigned long flags; + unsigned int key_size; + unsigned int key_parts; /* independent parts in key buffer */ +@@ -895,6 +897,14 @@ static void crypt_alloc_req(struct crypt + kcryptd_async_done, dmreq_of_req(cc, ctx->req)); + } + ++static void crypt_free_req(struct crypt_config *cc, ++ struct ablkcipher_request *req, struct bio *base_bio) ++{ ++ struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); ++ if ((struct ablkcipher_request *)(io + 1) != req) ++ mempool_free(req, cc->req_pool); ++} ++ + /* + * Encrypt / decrypt data from one bio to another one (can be the same one) + */ +@@ -1008,12 +1018,9 @@ static void crypt_free_buffer_pages(stru + } + } + +-static struct dm_crypt_io *crypt_io_alloc(struct crypt_config *cc, +- struct bio *bio, sector_t sector) ++static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc, ++ struct bio *bio, sector_t sector) + { +- struct dm_crypt_io *io; +- +- io = mempool_alloc(cc->io_pool, GFP_NOIO); + io->cc = cc; + io->base_bio = bio; + io->sector = sector; +@@ -1021,8 +1028,6 @@ static struct dm_crypt_io *crypt_io_allo + io->base_io = NULL; + io->ctx.req = NULL; + atomic_set(&io->io_pending, 0); +- +- return io; + } + + static void crypt_inc_pending(struct dm_crypt_io *io) +@@ -1046,8 +1051,9 @@ static void crypt_dec_pending(struct dm_ + return; + + if (io->ctx.req) +- mempool_free(io->ctx.req, cc->req_pool); +- mempool_free(io, cc->io_pool); ++ crypt_free_req(cc, io->ctx.req, base_bio); ++ if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size)) ++ mempool_free(io, cc->io_pool); + + if (likely(!base_io)) + bio_endio(base_bio, error); +@@ -1255,8 +1261,8 @@ static void kcryptd_crypt_write_convert( + * between fragments, so switch to a new dm_crypt_io structure. + */ + if (unlikely(!crypt_finished && remaining)) { +- new_io = crypt_io_alloc(io->cc, io->base_bio, +- sector); ++ new_io = mempool_alloc(cc->io_pool, GFP_NOIO); ++ crypt_io_init(new_io, io->cc, io->base_bio, sector); + crypt_inc_pending(new_io); + crypt_convert_init(cc, &new_io->ctx, NULL, + io->base_bio, sector); +@@ -1325,7 +1331,7 @@ static void kcryptd_async_done(struct cr + if (error < 0) + io->error = -EIO; + +- mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); ++ crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio); + + if (!atomic_dec_and_test(&ctx->cc_pending)) + return; +@@ -1728,6 +1734,10 @@ static int crypt_ctr(struct dm_target *t + goto bad; + } + ++ cc->per_bio_data_size = ti->per_bio_data_size = ++ sizeof(struct dm_crypt_io) + cc->dmreq_start + ++ sizeof(struct dm_crypt_request) + cc->iv_size; ++ + cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); + if (!cc->page_pool) { + ti->error = "Cannot allocate page mempool"; +@@ -1824,7 +1834,9 @@ static int crypt_map(struct dm_target *t + return DM_MAPIO_REMAPPED; + } + +- io = crypt_io_alloc(cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector)); ++ io = dm_per_bio_data(bio, cc->per_bio_data_size); ++ crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector)); ++ io->ctx.req = (struct ablkcipher_request *)(io + 1); + + if (bio_data_dir(io->base_bio) == READ) { + if (kcryptd_io_read(io, GFP_NOWAIT)) diff --git a/dm-crypt-remove-io-pool.patch b/dm-crypt-remove-io-pool.patch new file mode 100644 index 00000000..5217724b --- /dev/null +++ b/dm-crypt-remove-io-pool.patch @@ -0,0 +1,93 @@ +dm-crypt: remove io_pool + +Remove io_pool and _crypt_io_pool because they are unused. + +Signed-off-by: Mikulas Patocka + +--- + drivers/md/dm-crypt.c | 21 +-------------------- + 1 file changed, 1 insertion(+), 20 deletions(-) + +Index: linux-3.14/drivers/md/dm-crypt.c +=================================================================== +--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:04:40.000000000 +0200 ++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:05:40.000000000 +0200 +@@ -120,7 +120,6 @@ struct crypt_config { + * pool for per bio private data, crypto requests and + * encryption requeusts/buffer pages + */ +- mempool_t *io_pool; + mempool_t *req_pool; + mempool_t *page_pool; + struct bio_set *bs; +@@ -173,8 +172,6 @@ struct crypt_config { + + #define MIN_IOS 16 + +-static struct kmem_cache *_crypt_io_pool; +- + static void clone_init(struct dm_crypt_io *, struct bio *); + static void kcryptd_queue_crypt(struct dm_crypt_io *io); + static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq); +@@ -1061,8 +1058,6 @@ static void crypt_dec_pending(struct dm_ + + if (io->ctx.req) + crypt_free_req(cc, io->ctx.req, base_bio); +- if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size)) +- mempool_free(io, cc->io_pool); + + bio_endio(base_bio, error); + } +@@ -1450,8 +1445,6 @@ static void crypt_dtr(struct dm_target * + mempool_destroy(cc->page_pool); + if (cc->req_pool) + mempool_destroy(cc->req_pool); +- if (cc->io_pool) +- mempool_destroy(cc->io_pool); + + if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) + cc->iv_gen_ops->dtr(cc); +@@ -1664,19 +1657,13 @@ static int crypt_ctr(struct dm_target *t + if (ret < 0) + goto bad; + +- ret = -ENOMEM; +- cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); +- if (!cc->io_pool) { +- ti->error = "Cannot allocate crypt io mempool"; +- goto bad; +- } +- + cc->dmreq_start = sizeof(struct ablkcipher_request); + cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc)); + cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); + cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) & + ~(crypto_tfm_ctx_alignment() - 1); + ++ ret = -ENOMEM; + cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + + sizeof(struct dm_crypt_request) + cc->iv_size); + if (!cc->req_pool) { +@@ -1938,14 +1925,9 @@ static int __init dm_crypt_init(void) + { + int r; + +- _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); +- if (!_crypt_io_pool) +- return -ENOMEM; +- + r = dm_register_target(&crypt_target); + if (r < 0) { + DMERR("register failed %d", r); +- kmem_cache_destroy(_crypt_io_pool); + } + + return r; +@@ -1954,7 +1936,6 @@ static int __init dm_crypt_init(void) + static void __exit dm_crypt_exit(void) + { + dm_unregister_target(&crypt_target); +- kmem_cache_destroy(_crypt_io_pool); + } + + module_init(dm_crypt_init); diff --git a/dm-crypt-remove-percpu.patch b/dm-crypt-remove-percpu.patch index d08a5069..1589b523 100644 --- a/dm-crypt-remove-percpu.patch +++ b/dm-crypt-remove-percpu.patch @@ -1,19 +1,31 @@ dm-crypt: remove per-cpu structure -Remove per-cpu structure and make it per-convert_context instead. -This allows moving requests between different cpus. +Dm-crypt used per-cpu structures to hold pointers to ablkcipher_request. +The code assumed that the work item keeps executing on a single CPU, so it +used no synchronization when accessing this structure. + +When we disable a CPU by writing zero to +/sys/devices/system/cpu/cpu*/online, the work item could be moved to +another CPU. This causes crashes in dm-crypt because the code starts using +a wrong ablkcipher_request. + +This patch fixes this bug by removing the percpu definition. The structure +ablkcipher_request is accessed via a pointer from convert_context. +Consequently, if the work item is rescheduled to a different CPU, the +thread still uses the same ablkcipher_request. Signed-off-by: Mikulas Patocka +Cc: stable@vger.kernel.org --- drivers/md/dm-crypt.c | 61 +++++++++----------------------------------------- 1 file changed, 12 insertions(+), 49 deletions(-) -Index: linux-3.10.4-fast/drivers/md/dm-crypt.c +Index: linux-3.14-rc1/drivers/md/dm-crypt.c =================================================================== ---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 16:59:48.000000000 +0200 -+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:10.000000000 +0200 -@@ -18,7 +18,6 @@ +--- linux-3.14-rc1.orig/drivers/md/dm-crypt.c 2014-02-03 19:18:23.000000000 +0100 ++++ linux-3.14-rc1/drivers/md/dm-crypt.c 2014-02-03 19:21:35.000000000 +0100 +@@ -19,7 +19,6 @@ #include #include #include @@ -21,15 +33,15 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c #include #include #include -@@ -44,6 +43,7 @@ struct convert_context { - unsigned int idx_out; +@@ -43,6 +42,7 @@ struct convert_context { + struct bvec_iter iter_out; sector_t cc_sector; atomic_t cc_pending; + struct ablkcipher_request *req; }; /* -@@ -105,15 +105,7 @@ struct iv_lmk_private { +@@ -111,15 +111,7 @@ struct iv_tcw_private { enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; /* @@ -46,7 +58,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c */ struct crypt_config { struct dm_dev *dev; -@@ -143,12 +135,6 @@ struct crypt_config { +@@ -150,12 +142,6 @@ struct crypt_config { sector_t iv_offset; unsigned int iv_size; @@ -59,7 +71,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c /* ESSIV: struct crypto_cipher *essiv_tfm */ void *iv_private; struct crypto_ablkcipher **tfms; -@@ -184,11 +170,6 @@ static void clone_init(struct dm_crypt_i +@@ -192,11 +178,6 @@ static void clone_init(struct dm_crypt_i static void kcryptd_queue_crypt(struct dm_crypt_io *io); static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq); @@ -71,7 +83,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c /* * Use this to access cipher attributes that are the same for each CPU. */ -@@ -738,16 +719,15 @@ static void kcryptd_async_done(struct cr +@@ -903,16 +884,15 @@ static void kcryptd_async_done(struct cr static void crypt_alloc_req(struct crypt_config *cc, struct convert_context *ctx) { @@ -93,7 +105,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c } /* -@@ -756,7 +736,6 @@ static void crypt_alloc_req(struct crypt +@@ -921,7 +901,6 @@ static void crypt_alloc_req(struct crypt static int crypt_convert(struct crypt_config *cc, struct convert_context *ctx) { @@ -101,7 +113,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c int r; atomic_set(&ctx->cc_pending, 1); -@@ -768,7 +747,7 @@ static int crypt_convert(struct crypt_co +@@ -932,7 +911,7 @@ static int crypt_convert(struct crypt_co atomic_inc(&ctx->cc_pending); @@ -110,8 +122,8 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c switch (r) { /* async */ -@@ -777,7 +756,7 @@ static int crypt_convert(struct crypt_co - INIT_COMPLETION(ctx->restart); +@@ -941,7 +920,7 @@ static int crypt_convert(struct crypt_co + reinit_completion(&ctx->restart); /* fall through*/ case -EINPROGRESS: - this_cc->req = NULL; @@ -119,7 +131,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c ctx->cc_sector++; continue; -@@ -876,6 +855,7 @@ static struct dm_crypt_io *crypt_io_allo +@@ -1040,6 +1019,7 @@ static struct dm_crypt_io *crypt_io_allo io->sector = sector; io->error = 0; io->base_io = NULL; @@ -127,7 +139,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c atomic_set(&io->io_pending, 0); return io; -@@ -901,6 +881,8 @@ static void crypt_dec_pending(struct dm_ +@@ -1065,6 +1045,8 @@ static void crypt_dec_pending(struct dm_ if (!atomic_dec_and_test(&io->io_pending)) return; @@ -136,7 +148,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c mempool_free(io, cc->io_pool); if (likely(!base_io)) -@@ -1326,8 +1308,6 @@ static int crypt_wipe_key(struct crypt_c +@@ -1492,8 +1474,6 @@ static int crypt_wipe_key(struct crypt_c static void crypt_dtr(struct dm_target *ti) { struct crypt_config *cc = ti->private; @@ -145,7 +157,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c ti->private = NULL; -@@ -1339,13 +1319,6 @@ static void crypt_dtr(struct dm_target * +@@ -1505,13 +1485,6 @@ static void crypt_dtr(struct dm_target * if (cc->crypt_queue) destroy_workqueue(cc->crypt_queue); @@ -159,7 +171,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c crypt_free_tfms(cc); if (cc->bs) -@@ -1364,9 +1337,6 @@ static void crypt_dtr(struct dm_target * +@@ -1530,9 +1503,6 @@ static void crypt_dtr(struct dm_target * if (cc->dev) dm_put_device(ti, cc->dev); @@ -169,7 +181,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c kzfree(cc->cipher); kzfree(cc->cipher_string); -@@ -1421,13 +1391,6 @@ static int crypt_ctr_cipher(struct dm_ta +@@ -1588,13 +1558,6 @@ static int crypt_ctr_cipher(struct dm_ta if (tmp) DMWARN("Ignoring unexpected additional cipher options"); diff --git a/dm-crypt-sort-requests.patch b/dm-crypt-sort-requests.patch index 8ee2da74..5dc98863 100644 --- a/dm-crypt-sort-requests.patch +++ b/dm-crypt-sort-requests.patch @@ -13,11 +13,11 @@ Signed-off-by: Mikulas Patocka drivers/md/dm-crypt.c | 50 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 15 deletions(-) -Index: linux-3.10.4-fast/drivers/md/dm-crypt.c +Index: linux-3.14/drivers/md/dm-crypt.c =================================================================== ---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:27.000000000 +0200 -+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:30.000000000 +0200 -@@ -21,6 +21,7 @@ +--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:06:22.000000000 +0200 ++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:06:55.000000000 +0200 +@@ -22,6 +22,7 @@ #include #include #include @@ -25,16 +25,16 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c #include #include #include -@@ -61,7 +62,7 @@ struct dm_crypt_io { +@@ -60,7 +61,7 @@ struct dm_crypt_io { int error; sector_t sector; - struct list_head list; + struct rb_node rb_node; - }; + } CRYPTO_MINALIGN_ATTR; struct dm_crypt_request { -@@ -128,7 +129,7 @@ struct crypt_config { +@@ -133,7 +134,7 @@ struct crypt_config { struct task_struct *write_thread; wait_queue_head_t write_thread_wait; @@ -43,7 +43,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c char *cipher; char *cipher_string; -@@ -1011,7 +1012,7 @@ static int dmcrypt_write(void *data) +@@ -1177,7 +1178,7 @@ static int dmcrypt_write(void *data) { struct crypt_config *cc = data; while (1) { @@ -52,7 +52,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c struct blk_plug plug; DECLARE_WAITQUEUE(wait, current); -@@ -1019,7 +1020,7 @@ static int dmcrypt_write(void *data) +@@ -1185,7 +1186,7 @@ static int dmcrypt_write(void *data) spin_lock_irq(&cc->write_thread_wait.lock); continue_locked: @@ -61,7 +61,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c goto pop_from_list; __set_current_state(TASK_INTERRUPTIBLE); -@@ -1041,20 +1042,23 @@ continue_locked: +@@ -1207,20 +1208,23 @@ continue_locked: goto continue_locked; pop_from_list: @@ -94,7 +94,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c blk_finish_plug(&plug); } return 0; -@@ -1065,6 +1069,8 @@ static void kcryptd_crypt_write_io_submi +@@ -1231,6 +1235,8 @@ static void kcryptd_crypt_write_io_submi struct bio *clone = io->ctx.bio_out; struct crypt_config *cc = io->cc; unsigned long flags; @@ -103,8 +103,8 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c if (unlikely(io->error < 0)) { crypt_free_buffer_pages(cc, clone); -@@ -1079,7 +1085,21 @@ static void kcryptd_crypt_write_io_submi - clone->bi_sector = cc->start + io->sector; +@@ -1245,7 +1251,21 @@ static void kcryptd_crypt_write_io_submi + clone->bi_iter.bi_sector = cc->start + io->sector; spin_lock_irqsave(&cc->write_thread_wait.lock, flags); - list_add_tail(&io->list, &cc->write_thread_list); @@ -126,7 +126,7 @@ Index: linux-3.10.4-fast/drivers/md/dm-crypt.c wake_up_locked(&cc->write_thread_wait); spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); } -@@ -1639,7 +1659,7 @@ static int crypt_ctr(struct dm_target *t +@@ -1808,7 +1828,7 @@ static int crypt_ctr(struct dm_target *t } init_waitqueue_head(&cc->write_thread_wait); diff --git a/dm-crypt-unbound-workqueue.patch b/dm-crypt-unbound-workqueue.patch index 4ebd48f4..0a0a0adb 100644 --- a/dm-crypt-unbound-workqueue.patch +++ b/dm-crypt-unbound-workqueue.patch @@ -6,14 +6,14 @@ available CPUs. Signed-off-by: Mikulas Patocka --- - drivers/md/dm-crypt.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) + drivers/md/dm-crypt.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) -Index: linux-3.10.4-fast/drivers/md/dm-crypt.c +Index: linux-3.14-rc8/drivers/md/dm-crypt.c =================================================================== ---- linux-3.12/drivers/md/dm-crypt.c.orig 2013-11-12 11:02:03.770083857 +0100 -+++ linux-3.12/drivers/md/dm-crypt.c 2013-11-12 12:03:46.890126367 +0100 -@@ -1615,7 +1615,7 @@ +--- linux-3.14-rc8.orig/drivers/md/dm-crypt.c 2014-03-25 22:57:00.000000000 +0100 ++++ linux-3.14-rc8/drivers/md/dm-crypt.c 2014-03-28 17:09:14.000000000 +0100 +@@ -1800,7 +1800,7 @@ static int crypt_ctr(struct dm_target *t } cc->crypt_queue = alloc_workqueue("kcryptd", diff --git a/kernel.spec b/kernel.spec index feae87b6..8280d0bd 100644 --- a/kernel.spec +++ b/kernel.spec @@ -227,11 +227,14 @@ Patch400: kernel-virtio-gl-accel.patch # http://people.redhat.com/mpatocka/patches/kernel/dm-crypt-paralelizace/current/series.html Patch500: dm-crypt-remove-percpu.patch -Patch501: dm-crypt-unbound-workqueue.patch -Patch502: dm-crypt-dont-allocate-partial-pages.patch -Patch503: dm-crypt-fix-allocation-deadlock.patch -Patch504: dm-crypt-offload-writes-to-thread.patch -Patch505: dm-crypt-sort-requests.patch +Patch501: bio-kmalloc-align.patch +Patch502: dm-crypt-per_bio_data.patch +Patch503: dm-crypt-unbound-workqueue.patch +Patch504: dm-crypt-dont-allocate-partial-pages.patch +Patch505: dm-crypt-fix-allocation-deadlock.patch +Patch506: dm-crypt-remove-io-pool.patch +Patch507: dm-crypt-offload-writes-to-thread.patch +Patch508: dm-crypt-sort-requests.patch Patch2000: kernel-small_fixes.patch Patch2001: kernel-pwc-uncompress.patch @@ -740,6 +743,9 @@ cd linux-%{basever} %patch503 -p1 %patch504 -p1 %patch505 -p1 +%patch506 -p1 +%patch507 -p1 +%patch508 -p1 %endif %endif # vanilla -- 2.44.0