dm-crypt-dont-allocate-partial-pages.patch

   1 dm-crypt: don't allocate pages for a partial request.
   2
   3 This patch changes crypt_alloc_buffer so that it always allocates pages for
   4 a full request.
   5
   6 This change enables further simplification and removing of one refcounts
   7 in the next patches.
   8
   9 Note: the next patch is needed to fix a theoretical deadlock
  10
  11 Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
  12
  13 ---
  14  drivers/md/dm-crypt.c |  133 +++++++++-----------------------------------------
  15  1 file changed, 25 insertions(+), 108 deletions(-)
  16
  17 Index: linux-3.9.2-fast/drivers/md/dm-crypt.c
  18 ===================================================================
  19 --- linux-3.9.2-fast.orig/drivers/md/dm-crypt.c 2013-05-15 21:47:30.000000000 +0200
  20 +++ linux-3.9.2-fast/drivers/md/dm-crypt.c      2013-05-15 22:49:13.000000000 +0200
  21 @@ -59,7 +59,6 @@ struct dm_crypt_io {
  22         atomic_t io_pending;
  23         int error;
  24         sector_t sector;
  25 -       struct dm_crypt_io *base_io;
  26  };
  27
  28  struct dm_crypt_request {
  29 @@ -162,7 +161,6 @@ struct crypt_config {
  30  };
  31
  32  #define MIN_IOS        16
  33 -#define MIN_POOL_PAGES 32
  34
  35  static struct kmem_cache *_crypt_io_pool;
  36
  37 @@ -777,14 +775,13 @@ static int crypt_convert(struct crypt_co
  38         return 0;
  39  }
  40
  41 +static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
  42 +
  43  /*
  44   * Generate a new unfragmented bio with the given size
  45   * This should never violate the device limitations
  46 - * May return a smaller bio when running out of pages, indicated by
  47 - * *out_of_pages set to 1.
  48   */
  49 -static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size,
  50 -                                     unsigned *out_of_pages)
  51 +static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
  52  {
  53         struct crypt_config *cc = io->cc;
  54         struct bio *clone;
  55 @@ -798,37 +795,22 @@ static struct bio *crypt_alloc_buffer(st
  56                 return NULL;
  57
  58         clone_init(io, clone);
  59 -       *out_of_pages = 0;
  60
  61         for (i = 0; i < nr_iovecs; i++) {
  62                 page = mempool_alloc(cc->page_pool, gfp_mask);
  63 -               if (!page) {
  64 -                       *out_of_pages = 1;
  65 -                       break;
  66 -               }
  67 -
  68 -               /*
  69 -                * If additional pages cannot be allocated without waiting,
  70 -                * return a partially-allocated bio.  The caller will then try
  71 -                * to allocate more bios while submitting this partial bio.
  72 -                */
  73 -               gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
  74
  75                 len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
  76
  77                 if (!bio_add_page(clone, page, len, 0)) {
  78                         mempool_free(page, cc->page_pool);
  79 -                       break;
  80 +                       crypt_free_buffer_pages(cc, clone);
  81 +                       bio_put(clone);
  82 +                       return NULL;
  83                 }
  84
  85                 size -= len;
  86         }
  87
  88 -       if (!clone->bi_size) {
  89 -               bio_put(clone);
  90 -               return NULL;
  91 -       }
  92 -
  93         return clone;
  94  }
  95
  96 @@ -855,7 +837,6 @@ static struct dm_crypt_io *crypt_io_allo
  97         io->base_bio = bio;
  98         io->sector = sector;
  99         io->error = 0;
 100 -       io->base_io = NULL;
 101         io->ctx.req = NULL;
 102         atomic_set(&io->io_pending, 0);
 103
 104 @@ -870,13 +851,11 @@ static void crypt_inc_pending(struct dm_
 105  /*
 106   * One of the bios was finished. Check for completion of
 107   * the whole request and correctly clean up the buffer.
 108 - * If base_io is set, wait for the last fragment to complete.
 109   */
 110  static void crypt_dec_pending(struct dm_crypt_io *io)
 111  {
 112         struct crypt_config *cc = io->cc;
 113         struct bio *base_bio = io->base_bio;
 114 -       struct dm_crypt_io *base_io = io->base_io;
 115         int error = io->error;
 116
 117         if (!atomic_dec_and_test(&io->io_pending))
 118 @@ -886,13 +865,7 @@ static void crypt_dec_pending(struct dm_
 119                 mempool_free(io->ctx.req, cc->req_pool);
 120         mempool_free(io, cc->io_pool);
 121
 122 -       if (likely(!base_io))
 123 -               bio_endio(base_bio, error);
 124 -       else {
 125 -               if (error && !base_io->error)
 126 -                       base_io->error = error;
 127 -               crypt_dec_pending(base_io);
 128 -       }
 129 +       bio_endio(base_bio, error);
 130  }
 131
 132  /*
 133 @@ -1030,10 +1003,7 @@ static void kcryptd_crypt_write_convert(
 134  {
 135         struct crypt_config *cc = io->cc;
 136         struct bio *clone;
 137 -       struct dm_crypt_io *new_io;
 138         int crypt_finished;
 139 -       unsigned out_of_pages = 0;
 140 -       unsigned remaining = io->base_bio->bi_size;
 141         sector_t sector = io->sector;
 142         int r;
 143
 144 @@ -1043,81 +1013,28 @@ static void kcryptd_crypt_write_convert(
 145         crypt_inc_pending(io);
 146         crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
 147
 148 -       /*
 149 -        * The allocated buffers can be smaller than the whole bio,
 150 -        * so repeat the whole process until all the data can be handled.
 151 -        */
 152 -       while (remaining) {
 153 -               clone = crypt_alloc_buffer(io, remaining, &out_of_pages);
 154 -               if (unlikely(!clone)) {
 155 -                       io->error = -ENOMEM;
 156 -                       break;
 157 -               }
 158 -
 159 -               io->ctx.bio_out = clone;
 160 -               io->ctx.idx_out = 0;
 161 -
 162 -               remaining -= clone->bi_size;
 163 -               sector += bio_sectors(clone);
 164 -
 165 -               crypt_inc_pending(io);
 166 -
 167 -               r = crypt_convert(cc, &io->ctx);
 168 -               if (r < 0)
 169 -                       io->error = -EIO;
 170 -
 171 -               crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
 172 -
 173 -               /* Encryption was already finished, submit io now */
 174 -               if (crypt_finished) {
 175 -                       kcryptd_crypt_write_io_submit(io, 0);
 176 -
 177 -                       /*
 178 -                        * If there was an error, do not try next fragments.
 179 -                        * For async, error is processed in async handler.
 180 -                        */
 181 -                       if (unlikely(r < 0))
 182 -                               break;
 183 +       clone = crypt_alloc_buffer(io, io->base_bio->bi_size);
 184 +       if (unlikely(!clone)) {
 185 +               io->error = -ENOMEM;
 186 +               goto dec;
 187 +       }
 188
 189 -                       io->sector = sector;
 190 -               }
 191 +       io->ctx.bio_out = clone;
 192 +       io->ctx.idx_out = 0;
 193
 194 -               /*
 195 -                * Out of memory -> run queues
 196 -                * But don't wait if split was due to the io size restriction
 197 -                */
 198 -               if (unlikely(out_of_pages))
 199 -                       congestion_wait(BLK_RW_ASYNC, HZ/100);
 200 +       sector += bio_sectors(clone);
 201
 202 -               /*
 203 -                * With async crypto it is unsafe to share the crypto context
 204 -                * between fragments, so switch to a new dm_crypt_io structure.
 205 -                */
 206 -               if (unlikely(!crypt_finished && remaining)) {
 207 -                       new_io = crypt_io_alloc(io->cc, io->base_bio,
 208 -                                               sector);
 209 -                       crypt_inc_pending(new_io);
 210 -                       crypt_convert_init(cc, &new_io->ctx, NULL,
 211 -                                          io->base_bio, sector);
 212 -                       new_io->ctx.idx_in = io->ctx.idx_in;
 213 -                       new_io->ctx.offset_in = io->ctx.offset_in;
 214 -
 215 -                       /*
 216 -                        * Fragments after the first use the base_io
 217 -                        * pending count.
 218 -                        */
 219 -                       if (!io->base_io)
 220 -                               new_io->base_io = io;
 221 -                       else {
 222 -                               new_io->base_io = io->base_io;
 223 -                               crypt_inc_pending(io->base_io);
 224 -                               crypt_dec_pending(io);
 225 -                       }
 226 +       crypt_inc_pending(io);
 227 +       r = crypt_convert(cc, &io->ctx);
 228 +       if (r)
 229 +               io->error = -EIO;
 230 +       crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
 231
 232 -                       io = new_io;
 233 -               }
 234 -       }
 235 +       /* Encryption was already finished, submit io now */
 236 +       if (crypt_finished)
 237 +               kcryptd_crypt_write_io_submit(io, 0);
 238
 239 +dec:
 240         crypt_dec_pending(io);
 241  }
 242
 243 @@ -1556,7 +1473,7 @@ static int crypt_ctr(struct dm_target *t
 244                 goto bad;
 245         }
 246
 247 -       cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
 248 +       cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0);
 249         if (!cc->page_pool) {
 250                 ti->error = "Cannot allocate page mempool";
 251                 goto bad;