--- /dev/null
+bio: use kmalloc alignment for bio slab
+
+Various subsystems can ask the bio subsystem to create a bio slab cache
+with some free space before the bio. This free space can be used for any
+purpose. Device mapper uses this feature to place some target-specific and
+device-mapper specific data before the bio, so that the target-specific
+data doesn't have to be allocated separatedly.
+
+This mechanism is used in place of kmalloc, so we need that the allocated
+slab have the same memory alignment as memory allocated with kmalloc.
+
+This patch changes the function bio_find_or_create_slab so that it uses
+ARCH_KMALLOC_MINALIGN alignment when creating the slab cache. This patch
+is needed so that dm-crypt can use per-bio data for encryption - the
+crypto subsystem assumes that these data have the same alignment as
+kmallocated memory.
+
+Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
+
+---
+ fs/bio.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+Index: linux-3.14-rc3/fs/bio.c
+===================================================================
+--- linux-3.14-rc3.orig/fs/bio.c 2014-02-23 23:53:50.000000000 +0100
++++ linux-3.14-rc3/fs/bio.c 2014-02-23 23:55:00.000000000 +0100
+@@ -112,7 +112,8 @@ static struct kmem_cache *bio_find_or_cr
+ bslab = &bio_slabs[entry];
+
+ snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
+- slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL);
++ slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
++ SLAB_HWCACHE_ALIGN, NULL);
+ if (!slab)
+ goto out_unlock;
+
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
---
- drivers/md/dm-crypt.c | 134 +++++++++-----------------------------------------
- 1 file changed, 26 insertions(+), 108 deletions(-)
+ drivers/md/dm-crypt.c | 139 ++++++++++----------------------------------------
+ 1 file changed, 30 insertions(+), 109 deletions(-)
-Index: linux-3.10.4-fast/drivers/md/dm-crypt.c
+Index: linux-3.14/drivers/md/dm-crypt.c
===================================================================
---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:18.000000000 +0200
-+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:21.000000000 +0200
-@@ -59,7 +59,6 @@ struct dm_crypt_io {
+--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 20:48:50.000000000 +0200
++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 20:57:36.000000000 +0200
+@@ -58,7 +58,6 @@ struct dm_crypt_io {
atomic_t io_pending;
int error;
sector_t sector;
- struct dm_crypt_io *base_io;
- };
+ } CRYPTO_MINALIGN_ATTR;
struct dm_crypt_request {
-@@ -162,7 +161,6 @@ struct crypt_config {
+@@ -172,7 +171,6 @@ struct crypt_config {
};
#define MIN_IOS 16
static struct kmem_cache *_crypt_io_pool;
-@@ -777,14 +775,13 @@ static int crypt_convert(struct crypt_co
+@@ -951,14 +949,13 @@ static int crypt_convert(struct crypt_co
return 0;
}
{
struct crypt_config *cc = io->cc;
struct bio *clone;
-@@ -798,37 +795,23 @@ static struct bio *crypt_alloc_buffer(st
+@@ -966,41 +963,27 @@ static struct bio *crypt_alloc_buffer(st
+ gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
+ unsigned i, len;
+ struct page *page;
++ struct bio_vec *bvec;
+
+ clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
+ if (!clone)
return NULL;
clone_init(io, clone);
len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
- if (!bio_add_page(clone, page, len, 0)) {
-+ DMERR("bio_add_page failed for page %d: the underlying device has stricter limits than dm-crypt target", i);
- mempool_free(page, cc->page_pool);
+- if (!bio_add_page(clone, page, len, 0)) {
+- mempool_free(page, cc->page_pool);
- break;
-+ crypt_free_buffer_pages(cc, clone);
-+ bio_put(clone);
-+ return NULL;
- }
+- }
++ bvec = &clone->bi_io_vec[clone->bi_vcnt++];
++ bvec->bv_page = page;
++ bvec->bv_len = len;
++ bvec->bv_offset = 0;
- size -= len;
- }
+- size -= len;
+- }
++ clone->bi_iter.bi_size += len;
-- if (!clone->bi_size) {
+- if (!clone->bi_iter.bi_size) {
- bio_put(clone);
- return NULL;
-- }
--
- return clone;
- }
++ size -= len;
+ }
-@@ -854,7 +837,6 @@ static struct dm_crypt_io *crypt_io_allo
+ return clone;
+@@ -1025,7 +1008,6 @@ static void crypt_io_init(struct dm_cryp
io->base_bio = bio;
io->sector = sector;
io->error = 0;
- io->base_io = NULL;
io->ctx.req = NULL;
atomic_set(&io->io_pending, 0);
-
-@@ -869,13 +851,11 @@ static void crypt_inc_pending(struct dm_
+ }
+@@ -1038,13 +1020,11 @@ static void crypt_inc_pending(struct dm_
/*
* One of the bios was finished. Check for completion of
* the whole request and correctly clean up the buffer.
int error = io->error;
if (!atomic_dec_and_test(&io->io_pending))
-@@ -885,13 +865,7 @@ static void crypt_dec_pending(struct dm_
- mempool_free(io->ctx.req, cc->req_pool);
- mempool_free(io, cc->io_pool);
+@@ -1055,13 +1035,7 @@ static void crypt_dec_pending(struct dm_
+ if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size))
+ mempool_free(io, cc->io_pool);
- if (likely(!base_io))
- bio_endio(base_bio, error);
}
/*
-@@ -1027,10 +1001,7 @@ static void kcryptd_crypt_write_convert(
+@@ -1197,10 +1171,7 @@ static void kcryptd_crypt_write_convert(
{
struct crypt_config *cc = io->cc;
struct bio *clone;
- struct dm_crypt_io *new_io;
int crypt_finished;
- unsigned out_of_pages = 0;
-- unsigned remaining = io->base_bio->bi_size;
+- unsigned remaining = io->base_bio->bi_iter.bi_size;
sector_t sector = io->sector;
int r;
-@@ -1040,81 +1011,28 @@ static void kcryptd_crypt_write_convert(
+@@ -1210,80 +1181,30 @@ static void kcryptd_crypt_write_convert(
crypt_inc_pending(io);
crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
- }
-
- io->ctx.bio_out = clone;
-- io->ctx.idx_out = 0;
+- io->ctx.iter_out = clone->bi_iter;
-
-- remaining -= clone->bi_size;
+- remaining -= clone->bi_iter.bi_size;
- sector += bio_sectors(clone);
-
- crypt_inc_pending(io);
- */
- if (unlikely(r < 0))
- break;
-+ clone = crypt_alloc_buffer(io, io->base_bio->bi_size);
++ clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
+ if (unlikely(!clone)) {
+ io->error = -EIO;
+ goto dec;
- io->sector = sector;
- }
+ io->ctx.bio_out = clone;
-+ io->ctx.idx_out = 0;
++ io->ctx.iter_out = clone->bi_iter;
- /*
- * Out of memory -> run queues
- * between fragments, so switch to a new dm_crypt_io structure.
- */
- if (unlikely(!crypt_finished && remaining)) {
-- new_io = crypt_io_alloc(io->cc, io->base_bio,
-- sector);
+- new_io = mempool_alloc(cc->io_pool, GFP_NOIO);
+- crypt_io_init(new_io, io->cc, io->base_bio, sector);
- crypt_inc_pending(new_io);
- crypt_convert_init(cc, &new_io->ctx, NULL,
- io->base_bio, sector);
-- new_io->ctx.idx_in = io->ctx.idx_in;
-- new_io->ctx.offset_in = io->ctx.offset_in;
+- new_io->ctx.iter_in = io->ctx.iter_in;
-
- /*
- * Fragments after the first use the base_io
- io = new_io;
- }
-- }
+ /* Encryption was already finished, submit io now */
-+ if (crypt_finished)
++ if (crypt_finished) {
+ kcryptd_crypt_write_io_submit(io, 0);
++ io->sector = sector;
+ }
+dec:
crypt_dec_pending(io);
}
-@@ -1553,7 +1471,7 @@ static int crypt_ctr(struct dm_target *t
- goto bad;
- }
+@@ -1738,7 +1659,7 @@ static int crypt_ctr(struct dm_target *t
+ sizeof(struct dm_crypt_io) + cc->dmreq_start +
+ sizeof(struct dm_crypt_request) + cc->iv_size;
- cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
+ cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0);
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
---
- drivers/md/dm-crypt.c | 36 +++++++++++++++++++++++++++++++++---
- 1 file changed, 33 insertions(+), 3 deletions(-)
+ drivers/md/dm-crypt.c | 41 ++++++++++++++++++++++++++++++++++++-----
+ 1 file changed, 36 insertions(+), 5 deletions(-)
-Index: linux-3.10.4-fast/drivers/md/dm-crypt.c
+Index: linux-3.14/drivers/md/dm-crypt.c
===================================================================
---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:21.000000000 +0200
-+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:24.000000000 +0200
-@@ -118,6 +118,7 @@ struct crypt_config {
+--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 20:59:46.000000000 +0200
++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:04:40.000000000 +0200
+@@ -124,6 +124,7 @@ struct crypt_config {
mempool_t *req_pool;
mempool_t *page_pool;
struct bio_set *bs;
struct workqueue_struct *io_queue;
struct workqueue_struct *crypt_queue;
-@@ -780,24 +781,46 @@ static void crypt_free_buffer_pages(stru
+@@ -954,27 +955,51 @@ static void crypt_free_buffer_pages(stru
/*
* Generate a new unfragmented bio with the given size
* This should never violate the device limitations
struct bio *clone;
unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
- gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
+- unsigned i, len;
+ gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
- unsigned i, len;
++ unsigned i, len, remaining_size;
struct page *page;
+ struct bio_vec *bvec;
+retry:
+ if (unlikely(gfp_mask & __GFP_WAIT))
clone_init(io, clone);
++ remaining_size = size;
++
for (i = 0; i < nr_iovecs; i++) {
page = mempool_alloc(cc->page_pool, gfp_mask);
+ if (!page) {
+ goto retry;
+ }
- len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
+- len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
++ len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size;
+
+ bvec = &clone->bi_io_vec[clone->bi_vcnt++];
+ bvec->bv_page = page;
+@@ -983,9 +1008,13 @@ static struct bio *crypt_alloc_buffer(st
-@@ -806,12 +829,17 @@ static struct bio *crypt_alloc_buffer(st
- mempool_free(page, cc->page_pool);
- crypt_free_buffer_pages(cc, clone);
- bio_put(clone);
-- return NULL;
-+ clone = NULL;
-+ goto return_clone;
- }
+ clone->bi_iter.bi_size += len;
- size -= len;
+- size -= len;
++ remaining_size -= len;
}
+return_clone:
return clone;
}
-@@ -1483,6 +1511,8 @@ static int crypt_ctr(struct dm_target *t
+@@ -1671,6 +1700,8 @@ static int crypt_ctr(struct dm_target *t
goto bad;
}
drivers/md/dm-crypt.c | 120 ++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 97 insertions(+), 23 deletions(-)
-Index: linux-3.10.4-fast/drivers/md/dm-crypt.c
+Index: linux-3.14/drivers/md/dm-crypt.c
===================================================================
---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:24.000000000 +0200
-+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:27.000000000 +0200
-@@ -17,6 +17,7 @@
+--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:05:40.000000000 +0200
++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:06:22.000000000 +0200
+@@ -18,6 +18,7 @@
#include <linux/slab.h>
#include <linux/crypto.h>
#include <linux/workqueue.h>
#include <linux/backing-dev.h>
#include <linux/atomic.h>
#include <linux/scatterlist.h>
-@@ -59,6 +60,8 @@ struct dm_crypt_io {
+@@ -58,6 +59,8 @@ struct dm_crypt_io {
atomic_t io_pending;
int error;
sector_t sector;
+
+ struct list_head list;
- };
+ } CRYPTO_MINALIGN_ATTR;
struct dm_crypt_request {
-@@ -123,6 +126,10 @@ struct crypt_config {
+@@ -128,6 +131,10 @@ struct crypt_config {
struct workqueue_struct *io_queue;
struct workqueue_struct *crypt_queue;
char *cipher;
char *cipher_string;
-@@ -975,37 +982,89 @@ static int kcryptd_io_read(struct dm_cry
+@@ -1141,37 +1148,89 @@ static int kcryptd_io_read(struct dm_cry
return 0;
}
if (unlikely(io->error < 0)) {
crypt_free_buffer_pages(cc, clone);
-@@ -1019,10 +1078,10 @@ static void kcryptd_crypt_write_io_submi
+@@ -1185,10 +1244,10 @@ static void kcryptd_crypt_write_io_submi
- clone->bi_sector = cc->start + io->sector;
+ clone->bi_iter.bi_sector = cc->start + io->sector;
- if (async)
- kcryptd_queue_io(io);
}
static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
-@@ -1058,7 +1117,7 @@ static void kcryptd_crypt_write_convert(
+@@ -1224,7 +1283,7 @@ static void kcryptd_crypt_write_convert(
/* Encryption was already finished, submit io now */
- if (crypt_finished)
+ if (crypt_finished) {
- kcryptd_crypt_write_io_submit(io, 0);
+ kcryptd_crypt_write_io_submit(io);
+ io->sector = sector;
+ }
- dec:
- crypt_dec_pending(io);
-@@ -1116,7 +1175,7 @@ static void kcryptd_async_done(struct cr
+@@ -1284,7 +1343,7 @@ static void kcryptd_async_done(struct cr
if (bio_data_dir(io->base_bio) == READ)
kcryptd_crypt_read_done(io);
else
}
static void kcryptd_crypt(struct work_struct *work)
-@@ -1260,6 +1319,9 @@ static void crypt_dtr(struct dm_target *
+@@ -1431,6 +1490,9 @@ static void crypt_dtr(struct dm_target *
if (!cc)
return;
if (cc->io_queue)
destroy_workqueue(cc->io_queue);
if (cc->crypt_queue)
-@@ -1576,6 +1638,18 @@ static int crypt_ctr(struct dm_target *t
+@@ -1745,6 +1807,18 @@ static int crypt_ctr(struct dm_target *t
goto bad;
}
ti->num_flush_bios = 1;
ti->discard_zeroes_data_unsupported = true;
-@@ -1607,7 +1681,7 @@ static int crypt_map(struct dm_target *t
+@@ -1779,7 +1853,7 @@ static int crypt_map(struct dm_target *t
if (bio_data_dir(io->base_bio) == READ) {
if (kcryptd_io_read(io, GFP_NOWAIT))
--- /dev/null
+dm-crypt: use per-bio data
+
+This patch changes dm-crypt so that it uses auxiliary data allocated with
+the bio.
+
+Dm-crypt requires two allocations per request - struct dm_crypt_io and
+struct ablkcipher_request (with other data appended to it). It used
+mempool for the allocation.
+
+Some requests may require more dm_crypt_ios and ablkcipher_requests,
+however most requests need just one of each of these two structures to
+complete.
+
+This patch changes it so that the first dm_crypt_io and ablkcipher_request
+and allocated with the bio (using target per_bio_data_size option). If the
+request needs additional values, they are allocated from the mempool.
+
+Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
+
+---
+ drivers/md/dm-crypt.c | 40 ++++++++++++++++++++++++++--------------
+ 1 file changed, 26 insertions(+), 14 deletions(-)
+
+Index: linux-3.14-rc4/drivers/md/dm-crypt.c
+===================================================================
+--- linux-3.14-rc4.orig/drivers/md/dm-crypt.c 2014-02-27 17:48:31.000000000 +0100
++++ linux-3.14-rc4/drivers/md/dm-crypt.c 2014-02-27 17:48:31.000000000 +0100
+@@ -59,7 +59,7 @@ struct dm_crypt_io {
+ int error;
+ sector_t sector;
+ struct dm_crypt_io *base_io;
+-};
++} CRYPTO_MINALIGN_ATTR;
+
+ struct dm_crypt_request {
+ struct convert_context *ctx;
+@@ -162,6 +162,8 @@ struct crypt_config {
+ */
+ unsigned int dmreq_start;
+
++ unsigned int per_bio_data_size;
++
+ unsigned long flags;
+ unsigned int key_size;
+ unsigned int key_parts; /* independent parts in key buffer */
+@@ -895,6 +897,14 @@ static void crypt_alloc_req(struct crypt
+ kcryptd_async_done, dmreq_of_req(cc, ctx->req));
+ }
+
++static void crypt_free_req(struct crypt_config *cc,
++ struct ablkcipher_request *req, struct bio *base_bio)
++{
++ struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
++ if ((struct ablkcipher_request *)(io + 1) != req)
++ mempool_free(req, cc->req_pool);
++}
++
+ /*
+ * Encrypt / decrypt data from one bio to another one (can be the same one)
+ */
+@@ -1008,12 +1018,9 @@ static void crypt_free_buffer_pages(stru
+ }
+ }
+
+-static struct dm_crypt_io *crypt_io_alloc(struct crypt_config *cc,
+- struct bio *bio, sector_t sector)
++static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
++ struct bio *bio, sector_t sector)
+ {
+- struct dm_crypt_io *io;
+-
+- io = mempool_alloc(cc->io_pool, GFP_NOIO);
+ io->cc = cc;
+ io->base_bio = bio;
+ io->sector = sector;
+@@ -1021,8 +1028,6 @@ static struct dm_crypt_io *crypt_io_allo
+ io->base_io = NULL;
+ io->ctx.req = NULL;
+ atomic_set(&io->io_pending, 0);
+-
+- return io;
+ }
+
+ static void crypt_inc_pending(struct dm_crypt_io *io)
+@@ -1046,8 +1051,9 @@ static void crypt_dec_pending(struct dm_
+ return;
+
+ if (io->ctx.req)
+- mempool_free(io->ctx.req, cc->req_pool);
+- mempool_free(io, cc->io_pool);
++ crypt_free_req(cc, io->ctx.req, base_bio);
++ if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size))
++ mempool_free(io, cc->io_pool);
+
+ if (likely(!base_io))
+ bio_endio(base_bio, error);
+@@ -1255,8 +1261,8 @@ static void kcryptd_crypt_write_convert(
+ * between fragments, so switch to a new dm_crypt_io structure.
+ */
+ if (unlikely(!crypt_finished && remaining)) {
+- new_io = crypt_io_alloc(io->cc, io->base_bio,
+- sector);
++ new_io = mempool_alloc(cc->io_pool, GFP_NOIO);
++ crypt_io_init(new_io, io->cc, io->base_bio, sector);
+ crypt_inc_pending(new_io);
+ crypt_convert_init(cc, &new_io->ctx, NULL,
+ io->base_bio, sector);
+@@ -1325,7 +1331,7 @@ static void kcryptd_async_done(struct cr
+ if (error < 0)
+ io->error = -EIO;
+
+- mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool);
++ crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
+
+ if (!atomic_dec_and_test(&ctx->cc_pending))
+ return;
+@@ -1728,6 +1734,10 @@ static int crypt_ctr(struct dm_target *t
+ goto bad;
+ }
+
++ cc->per_bio_data_size = ti->per_bio_data_size =
++ sizeof(struct dm_crypt_io) + cc->dmreq_start +
++ sizeof(struct dm_crypt_request) + cc->iv_size;
++
+ cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
+ if (!cc->page_pool) {
+ ti->error = "Cannot allocate page mempool";
+@@ -1824,7 +1834,9 @@ static int crypt_map(struct dm_target *t
+ return DM_MAPIO_REMAPPED;
+ }
+
+- io = crypt_io_alloc(cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
++ io = dm_per_bio_data(bio, cc->per_bio_data_size);
++ crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
++ io->ctx.req = (struct ablkcipher_request *)(io + 1);
+
+ if (bio_data_dir(io->base_bio) == READ) {
+ if (kcryptd_io_read(io, GFP_NOWAIT))
--- /dev/null
+dm-crypt: remove io_pool
+
+Remove io_pool and _crypt_io_pool because they are unused.
+
+Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
+
+---
+ drivers/md/dm-crypt.c | 21 +--------------------
+ 1 file changed, 1 insertion(+), 20 deletions(-)
+
+Index: linux-3.14/drivers/md/dm-crypt.c
+===================================================================
+--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:04:40.000000000 +0200
++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:05:40.000000000 +0200
+@@ -120,7 +120,6 @@ struct crypt_config {
+ * pool for per bio private data, crypto requests and
+ * encryption requeusts/buffer pages
+ */
+- mempool_t *io_pool;
+ mempool_t *req_pool;
+ mempool_t *page_pool;
+ struct bio_set *bs;
+@@ -173,8 +172,6 @@ struct crypt_config {
+
+ #define MIN_IOS 16
+
+-static struct kmem_cache *_crypt_io_pool;
+-
+ static void clone_init(struct dm_crypt_io *, struct bio *);
+ static void kcryptd_queue_crypt(struct dm_crypt_io *io);
+ static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
+@@ -1061,8 +1058,6 @@ static void crypt_dec_pending(struct dm_
+
+ if (io->ctx.req)
+ crypt_free_req(cc, io->ctx.req, base_bio);
+- if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size))
+- mempool_free(io, cc->io_pool);
+
+ bio_endio(base_bio, error);
+ }
+@@ -1450,8 +1445,6 @@ static void crypt_dtr(struct dm_target *
+ mempool_destroy(cc->page_pool);
+ if (cc->req_pool)
+ mempool_destroy(cc->req_pool);
+- if (cc->io_pool)
+- mempool_destroy(cc->io_pool);
+
+ if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
+ cc->iv_gen_ops->dtr(cc);
+@@ -1664,19 +1657,13 @@ static int crypt_ctr(struct dm_target *t
+ if (ret < 0)
+ goto bad;
+
+- ret = -ENOMEM;
+- cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
+- if (!cc->io_pool) {
+- ti->error = "Cannot allocate crypt io mempool";
+- goto bad;
+- }
+-
+ cc->dmreq_start = sizeof(struct ablkcipher_request);
+ cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc));
+ cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment());
+ cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) &
+ ~(crypto_tfm_ctx_alignment() - 1);
+
++ ret = -ENOMEM;
+ cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start +
+ sizeof(struct dm_crypt_request) + cc->iv_size);
+ if (!cc->req_pool) {
+@@ -1938,14 +1925,9 @@ static int __init dm_crypt_init(void)
+ {
+ int r;
+
+- _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0);
+- if (!_crypt_io_pool)
+- return -ENOMEM;
+-
+ r = dm_register_target(&crypt_target);
+ if (r < 0) {
+ DMERR("register failed %d", r);
+- kmem_cache_destroy(_crypt_io_pool);
+ }
+
+ return r;
+@@ -1954,7 +1936,6 @@ static int __init dm_crypt_init(void)
+ static void __exit dm_crypt_exit(void)
+ {
+ dm_unregister_target(&crypt_target);
+- kmem_cache_destroy(_crypt_io_pool);
+ }
+
+ module_init(dm_crypt_init);
dm-crypt: remove per-cpu structure
-Remove per-cpu structure and make it per-convert_context instead.
-This allows moving requests between different cpus.
+Dm-crypt used per-cpu structures to hold pointers to ablkcipher_request.
+The code assumed that the work item keeps executing on a single CPU, so it
+used no synchronization when accessing this structure.
+
+When we disable a CPU by writing zero to
+/sys/devices/system/cpu/cpu*/online, the work item could be moved to
+another CPU. This causes crashes in dm-crypt because the code starts using
+a wrong ablkcipher_request.
+
+This patch fixes this bug by removing the percpu definition. The structure
+ablkcipher_request is accessed via a pointer from convert_context.
+Consequently, if the work item is rescheduled to a different CPU, the
+thread still uses the same ablkcipher_request.
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
+Cc: stable@vger.kernel.org
---
drivers/md/dm-crypt.c | 61 +++++++++-----------------------------------------
1 file changed, 12 insertions(+), 49 deletions(-)
-Index: linux-3.10.4-fast/drivers/md/dm-crypt.c
+Index: linux-3.14-rc1/drivers/md/dm-crypt.c
===================================================================
---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 16:59:48.000000000 +0200
-+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:10.000000000 +0200
-@@ -18,7 +18,6 @@
+--- linux-3.14-rc1.orig/drivers/md/dm-crypt.c 2014-02-03 19:18:23.000000000 +0100
++++ linux-3.14-rc1/drivers/md/dm-crypt.c 2014-02-03 19:21:35.000000000 +0100
+@@ -19,7 +19,6 @@
#include <linux/crypto.h>
#include <linux/workqueue.h>
#include <linux/backing-dev.h>
#include <linux/atomic.h>
#include <linux/scatterlist.h>
#include <asm/page.h>
-@@ -44,6 +43,7 @@ struct convert_context {
- unsigned int idx_out;
+@@ -43,6 +42,7 @@ struct convert_context {
+ struct bvec_iter iter_out;
sector_t cc_sector;
atomic_t cc_pending;
+ struct ablkcipher_request *req;
};
/*
-@@ -105,15 +105,7 @@ struct iv_lmk_private {
+@@ -111,15 +111,7 @@ struct iv_tcw_private {
enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
/*
*/
struct crypt_config {
struct dm_dev *dev;
-@@ -143,12 +135,6 @@ struct crypt_config {
+@@ -150,12 +142,6 @@ struct crypt_config {
sector_t iv_offset;
unsigned int iv_size;
/* ESSIV: struct crypto_cipher *essiv_tfm */
void *iv_private;
struct crypto_ablkcipher **tfms;
-@@ -184,11 +170,6 @@ static void clone_init(struct dm_crypt_i
+@@ -192,11 +178,6 @@ static void clone_init(struct dm_crypt_i
static void kcryptd_queue_crypt(struct dm_crypt_io *io);
static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
/*
* Use this to access cipher attributes that are the same for each CPU.
*/
-@@ -738,16 +719,15 @@ static void kcryptd_async_done(struct cr
+@@ -903,16 +884,15 @@ static void kcryptd_async_done(struct cr
static void crypt_alloc_req(struct crypt_config *cc,
struct convert_context *ctx)
{
}
/*
-@@ -756,7 +736,6 @@ static void crypt_alloc_req(struct crypt
+@@ -921,7 +901,6 @@ static void crypt_alloc_req(struct crypt
static int crypt_convert(struct crypt_config *cc,
struct convert_context *ctx)
{
int r;
atomic_set(&ctx->cc_pending, 1);
-@@ -768,7 +747,7 @@ static int crypt_convert(struct crypt_co
+@@ -932,7 +911,7 @@ static int crypt_convert(struct crypt_co
atomic_inc(&ctx->cc_pending);
switch (r) {
/* async */
-@@ -777,7 +756,7 @@ static int crypt_convert(struct crypt_co
- INIT_COMPLETION(ctx->restart);
+@@ -941,7 +920,7 @@ static int crypt_convert(struct crypt_co
+ reinit_completion(&ctx->restart);
/* fall through*/
case -EINPROGRESS:
- this_cc->req = NULL;
ctx->cc_sector++;
continue;
-@@ -876,6 +855,7 @@ static struct dm_crypt_io *crypt_io_allo
+@@ -1040,6 +1019,7 @@ static struct dm_crypt_io *crypt_io_allo
io->sector = sector;
io->error = 0;
io->base_io = NULL;
atomic_set(&io->io_pending, 0);
return io;
-@@ -901,6 +881,8 @@ static void crypt_dec_pending(struct dm_
+@@ -1065,6 +1045,8 @@ static void crypt_dec_pending(struct dm_
if (!atomic_dec_and_test(&io->io_pending))
return;
mempool_free(io, cc->io_pool);
if (likely(!base_io))
-@@ -1326,8 +1308,6 @@ static int crypt_wipe_key(struct crypt_c
+@@ -1492,8 +1474,6 @@ static int crypt_wipe_key(struct crypt_c
static void crypt_dtr(struct dm_target *ti)
{
struct crypt_config *cc = ti->private;
ti->private = NULL;
-@@ -1339,13 +1319,6 @@ static void crypt_dtr(struct dm_target *
+@@ -1505,13 +1485,6 @@ static void crypt_dtr(struct dm_target *
if (cc->crypt_queue)
destroy_workqueue(cc->crypt_queue);
crypt_free_tfms(cc);
if (cc->bs)
-@@ -1364,9 +1337,6 @@ static void crypt_dtr(struct dm_target *
+@@ -1530,9 +1503,6 @@ static void crypt_dtr(struct dm_target *
if (cc->dev)
dm_put_device(ti, cc->dev);
kzfree(cc->cipher);
kzfree(cc->cipher_string);
-@@ -1421,13 +1391,6 @@ static int crypt_ctr_cipher(struct dm_ta
+@@ -1588,13 +1558,6 @@ static int crypt_ctr_cipher(struct dm_ta
if (tmp)
DMWARN("Ignoring unexpected additional cipher options");
drivers/md/dm-crypt.c | 50 +++++++++++++++++++++++++++++++++++---------------
1 file changed, 35 insertions(+), 15 deletions(-)
-Index: linux-3.10.4-fast/drivers/md/dm-crypt.c
+Index: linux-3.14/drivers/md/dm-crypt.c
===================================================================
---- linux-3.10.4-fast.orig/drivers/md/dm-crypt.c 2013-07-31 17:03:27.000000000 +0200
-+++ linux-3.10.4-fast/drivers/md/dm-crypt.c 2013-07-31 17:03:30.000000000 +0200
-@@ -21,6 +21,7 @@
+--- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:06:22.000000000 +0200
++++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:06:55.000000000 +0200
+@@ -22,6 +22,7 @@
#include <linux/backing-dev.h>
#include <linux/atomic.h>
#include <linux/scatterlist.h>
#include <asm/page.h>
#include <asm/unaligned.h>
#include <crypto/hash.h>
-@@ -61,7 +62,7 @@ struct dm_crypt_io {
+@@ -60,7 +61,7 @@ struct dm_crypt_io {
int error;
sector_t sector;
- struct list_head list;
+ struct rb_node rb_node;
- };
+ } CRYPTO_MINALIGN_ATTR;
struct dm_crypt_request {
-@@ -128,7 +129,7 @@ struct crypt_config {
+@@ -133,7 +134,7 @@ struct crypt_config {
struct task_struct *write_thread;
wait_queue_head_t write_thread_wait;
char *cipher;
char *cipher_string;
-@@ -1011,7 +1012,7 @@ static int dmcrypt_write(void *data)
+@@ -1177,7 +1178,7 @@ static int dmcrypt_write(void *data)
{
struct crypt_config *cc = data;
while (1) {
struct blk_plug plug;
DECLARE_WAITQUEUE(wait, current);
-@@ -1019,7 +1020,7 @@ static int dmcrypt_write(void *data)
+@@ -1185,7 +1186,7 @@ static int dmcrypt_write(void *data)
spin_lock_irq(&cc->write_thread_wait.lock);
continue_locked:
goto pop_from_list;
__set_current_state(TASK_INTERRUPTIBLE);
-@@ -1041,20 +1042,23 @@ continue_locked:
+@@ -1207,20 +1208,23 @@ continue_locked:
goto continue_locked;
pop_from_list:
blk_finish_plug(&plug);
}
return 0;
-@@ -1065,6 +1069,8 @@ static void kcryptd_crypt_write_io_submi
+@@ -1231,6 +1235,8 @@ static void kcryptd_crypt_write_io_submi
struct bio *clone = io->ctx.bio_out;
struct crypt_config *cc = io->cc;
unsigned long flags;
if (unlikely(io->error < 0)) {
crypt_free_buffer_pages(cc, clone);
-@@ -1079,7 +1085,21 @@ static void kcryptd_crypt_write_io_submi
- clone->bi_sector = cc->start + io->sector;
+@@ -1245,7 +1251,21 @@ static void kcryptd_crypt_write_io_submi
+ clone->bi_iter.bi_sector = cc->start + io->sector;
spin_lock_irqsave(&cc->write_thread_wait.lock, flags);
- list_add_tail(&io->list, &cc->write_thread_list);
wake_up_locked(&cc->write_thread_wait);
spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags);
}
-@@ -1639,7 +1659,7 @@ static int crypt_ctr(struct dm_target *t
+@@ -1808,7 +1828,7 @@ static int crypt_ctr(struct dm_target *t
}
init_waitqueue_head(&cc->write_thread_wait);
Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
---
- drivers/md/dm-crypt.c | 5 +++--
- 1 file changed, 3 insertions(+), 2 deletions(-)
+ drivers/md/dm-crypt.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
-Index: linux-3.10.4-fast/drivers/md/dm-crypt.c
+Index: linux-3.14-rc8/drivers/md/dm-crypt.c
===================================================================
---- linux-3.12/drivers/md/dm-crypt.c.orig 2013-11-12 11:02:03.770083857 +0100
-+++ linux-3.12/drivers/md/dm-crypt.c 2013-11-12 12:03:46.890126367 +0100
-@@ -1615,7 +1615,7 @@
+--- linux-3.14-rc8.orig/drivers/md/dm-crypt.c 2014-03-25 22:57:00.000000000 +0100
++++ linux-3.14-rc8/drivers/md/dm-crypt.c 2014-03-28 17:09:14.000000000 +0100
+@@ -1800,7 +1800,7 @@ static int crypt_ctr(struct dm_target *t
}
cc->crypt_queue = alloc_workqueue("kcryptd",
# http://people.redhat.com/mpatocka/patches/kernel/dm-crypt-paralelizace/current/series.html
Patch500: dm-crypt-remove-percpu.patch
-Patch501: dm-crypt-unbound-workqueue.patch
-Patch502: dm-crypt-dont-allocate-partial-pages.patch
-Patch503: dm-crypt-fix-allocation-deadlock.patch
-Patch504: dm-crypt-offload-writes-to-thread.patch
-Patch505: dm-crypt-sort-requests.patch
+Patch501: bio-kmalloc-align.patch
+Patch502: dm-crypt-per_bio_data.patch
+Patch503: dm-crypt-unbound-workqueue.patch
+Patch504: dm-crypt-dont-allocate-partial-pages.patch
+Patch505: dm-crypt-fix-allocation-deadlock.patch
+Patch506: dm-crypt-remove-io-pool.patch
+Patch507: dm-crypt-offload-writes-to-thread.patch
+Patch508: dm-crypt-sort-requests.patch
Patch2000: kernel-small_fixes.patch
Patch2001: kernel-pwc-uncompress.patch
%patch503 -p1
%patch504 -p1
%patch505 -p1
+%patch506 -p1
+%patch507 -p1
+%patch508 -p1
%endif
%endif # vanilla
projects / packages / kernel.git / commitdiff