dm-crypt-dont-allocate-partial-pages.patch

   1 dm-crypt: don't allocate pages for a partial request.
   2
   3 This patch changes crypt_alloc_buffer so that it always allocates pages for
   4 a full request.
   5
   6 This change enables further simplification and removing of one refcounts
   7 in the next patches.
   8
   9 Note: the next patch is needed to fix a theoretical deadlock
  10
  11 Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
  12
  13 ---
  14  drivers/md/dm-crypt.c |  139 ++++++++++----------------------------------------
  15  1 file changed, 30 insertions(+), 109 deletions(-)
  16
  17 Index: linux-3.14/drivers/md/dm-crypt.c
  18 ===================================================================
  19 --- linux-3.14.orig/drivers/md/dm-crypt.c       2014-04-04 20:48:50.000000000 +0200
  20 +++ linux-3.14/drivers/md/dm-crypt.c    2014-04-04 20:57:36.000000000 +0200
  21 @@ -58,7 +58,6 @@ struct dm_crypt_io {
  22         atomic_t io_pending;
  23         int error;
  24         sector_t sector;
  25 -       struct dm_crypt_io *base_io;
  26  } CRYPTO_MINALIGN_ATTR;
  27
  28  struct dm_crypt_request {
  29 @@ -172,7 +171,6 @@ struct crypt_config {
  30  };
  31
  32  #define MIN_IOS        16
  33 -#define MIN_POOL_PAGES 32
  34
  35  static struct kmem_cache *_crypt_io_pool;
  36
  37 @@ -951,14 +949,13 @@ static int crypt_convert(struct crypt_co
  38         return 0;
  39  }
  40
  41 +static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
  42 +
  43  /*
  44   * Generate a new unfragmented bio with the given size
  45   * This should never violate the device limitations
  46 - * May return a smaller bio when running out of pages, indicated by
  47 - * *out_of_pages set to 1.
  48   */
  49 -static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size,
  50 -                                     unsigned *out_of_pages)
  51 +static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
  52  {
  53         struct crypt_config *cc = io->cc;
  54         struct bio *clone;
  55 @@ -966,41 +963,27 @@ static struct bio *crypt_alloc_buffer(st
  56         gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
  57         unsigned i, len;
  58         struct page *page;
  59 +       struct bio_vec *bvec;
  60
  61         clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
  62         if (!clone)
  63                 return NULL;
  64
  65         clone_init(io, clone);
  66 -       *out_of_pages = 0;
  67
  68         for (i = 0; i < nr_iovecs; i++) {
  69                 page = mempool_alloc(cc->page_pool, gfp_mask);
  70 -               if (!page) {
  71 -                       *out_of_pages = 1;
  72 -                       break;
  73 -               }
  74 -
  75 -               /*
  76 -                * If additional pages cannot be allocated without waiting,
  77 -                * return a partially-allocated bio.  The caller will then try
  78 -                * to allocate more bios while submitting this partial bio.
  79 -                */
  80 -               gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
  81
  82                 len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
  83
  84 -               if (!bio_add_page(clone, page, len, 0)) {
  85 -                       mempool_free(page, cc->page_pool);
  86 -                       break;
  87 -               }
  88 +               bvec = &clone->bi_io_vec[clone->bi_vcnt++];
  89 +               bvec->bv_page = page;
  90 +               bvec->bv_len = len;
  91 +               bvec->bv_offset = 0;
  92
  93 -               size -= len;
  94 -       }
  95 +               clone->bi_iter.bi_size += len;
  96
  97 -       if (!clone->bi_iter.bi_size) {
  98 -               bio_put(clone);
  99 -               return NULL;
 100 +               size -= len;
 101         }
 102
 103         return clone;
 104 @@ -1025,7 +1008,6 @@ static void crypt_io_init(struct dm_cryp
 105         io->base_bio = bio;
 106         io->sector = sector;
 107         io->error = 0;
 108 -       io->base_io = NULL;
 109         io->ctx.req = NULL;
 110         atomic_set(&io->io_pending, 0);
 111  }
 112 @@ -1038,13 +1020,11 @@ static void crypt_inc_pending(struct dm_
 113  /*
 114   * One of the bios was finished. Check for completion of
 115   * the whole request and correctly clean up the buffer.
 116 - * If base_io is set, wait for the last fragment to complete.
 117   */
 118  static void crypt_dec_pending(struct dm_crypt_io *io)
 119  {
 120         struct crypt_config *cc = io->cc;
 121         struct bio *base_bio = io->base_bio;
 122 -       struct dm_crypt_io *base_io = io->base_io;
 123         int error = io->error;
 124
 125         if (!atomic_dec_and_test(&io->io_pending))
 126 @@ -1055,13 +1035,7 @@ static void crypt_dec_pending(struct dm_
 127         if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size))
 128                 mempool_free(io, cc->io_pool);
 129
 130 -       if (likely(!base_io))
 131 -               bio_endio(base_bio, error);
 132 -       else {
 133 -               if (error && !base_io->error)
 134 -                       base_io->error = error;
 135 -               crypt_dec_pending(base_io);
 136 -       }
 137 +       bio_endio(base_bio, error);
 138  }
 139
 140  /*
 141 @@ -1197,10 +1171,7 @@ static void kcryptd_crypt_write_convert(
 142  {
 143         struct crypt_config *cc = io->cc;
 144         struct bio *clone;
 145 -       struct dm_crypt_io *new_io;
 146         int crypt_finished;
 147 -       unsigned out_of_pages = 0;
 148 -       unsigned remaining = io->base_bio->bi_iter.bi_size;
 149         sector_t sector = io->sector;
 150         int r;
 151
 152 @@ -1210,80 +1181,30 @@ static void kcryptd_crypt_write_convert(
 153         crypt_inc_pending(io);
 154         crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
 155
 156 -       /*
 157 -        * The allocated buffers can be smaller than the whole bio,
 158 -        * so repeat the whole process until all the data can be handled.
 159 -        */
 160 -       while (remaining) {
 161 -               clone = crypt_alloc_buffer(io, remaining, &out_of_pages);
 162 -               if (unlikely(!clone)) {
 163 -                       io->error = -ENOMEM;
 164 -                       break;
 165 -               }
 166 -
 167 -               io->ctx.bio_out = clone;
 168 -               io->ctx.iter_out = clone->bi_iter;
 169 -
 170 -               remaining -= clone->bi_iter.bi_size;
 171 -               sector += bio_sectors(clone);
 172 -
 173 -               crypt_inc_pending(io);
 174 -
 175 -               r = crypt_convert(cc, &io->ctx);
 176 -               if (r < 0)
 177 -                       io->error = -EIO;
 178 -
 179 -               crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
 180 -
 181 -               /* Encryption was already finished, submit io now */
 182 -               if (crypt_finished) {
 183 -                       kcryptd_crypt_write_io_submit(io, 0);
 184 -
 185 -                       /*
 186 -                        * If there was an error, do not try next fragments.
 187 -                        * For async, error is processed in async handler.
 188 -                        */
 189 -                       if (unlikely(r < 0))
 190 -                               break;
 191 +       clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
 192 +       if (unlikely(!clone)) {
 193 +               io->error = -EIO;
 194 +               goto dec;
 195 +       }
 196
 197 -                       io->sector = sector;
 198 -               }
 199 +       io->ctx.bio_out = clone;
 200 +       io->ctx.iter_out = clone->bi_iter;
 201
 202 -               /*
 203 -                * Out of memory -> run queues
 204 -                * But don't wait if split was due to the io size restriction
 205 -                */
 206 -               if (unlikely(out_of_pages))
 207 -                       congestion_wait(BLK_RW_ASYNC, HZ/100);
 208 +       sector += bio_sectors(clone);
 209
 210 -               /*
 211 -                * With async crypto it is unsafe to share the crypto context
 212 -                * between fragments, so switch to a new dm_crypt_io structure.
 213 -                */
 214 -               if (unlikely(!crypt_finished && remaining)) {
 215 -                       new_io = mempool_alloc(cc->io_pool, GFP_NOIO);
 216 -                       crypt_io_init(new_io, io->cc, io->base_bio, sector);
 217 -                       crypt_inc_pending(new_io);
 218 -                       crypt_convert_init(cc, &new_io->ctx, NULL,
 219 -                                          io->base_bio, sector);
 220 -                       new_io->ctx.iter_in = io->ctx.iter_in;
 221 -
 222 -                       /*
 223 -                        * Fragments after the first use the base_io
 224 -                        * pending count.
 225 -                        */
 226 -                       if (!io->base_io)
 227 -                               new_io->base_io = io;
 228 -                       else {
 229 -                               new_io->base_io = io->base_io;
 230 -                               crypt_inc_pending(io->base_io);
 231 -                               crypt_dec_pending(io);
 232 -                       }
 233 +       crypt_inc_pending(io);
 234 +       r = crypt_convert(cc, &io->ctx);
 235 +       if (r)
 236 +               io->error = -EIO;
 237 +       crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
 238
 239 -                       io = new_io;
 240 -               }
 241 +       /* Encryption was already finished, submit io now */
 242 +       if (crypt_finished) {
 243 +               kcryptd_crypt_write_io_submit(io, 0);
 244 +               io->sector = sector;
 245         }
 246
 247 +dec:
 248         crypt_dec_pending(io);
 249  }
 250
 251 @@ -1738,7 +1659,7 @@ static int crypt_ctr(struct dm_target *t
 252                                 sizeof(struct dm_crypt_io) + cc->dmreq_start +
 253                                 sizeof(struct dm_crypt_request) + cc->iv_size;
 254
 255 -       cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
 256 +       cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0);
 257         if (!cc->page_pool) {
 258                 ti->error = "Cannot allocate page mempool";
 259                 goto bad;