loop-jari-2.4.21.0.patch

   1 diff -Nur linux-2.4.18/drivers/block/loop.c linux-int-2.4.18/drivers/block/loop.c
   2 --- linux-2.4.18/drivers/block/loop.c   Fri Dec 21 18:41:53 2001
   3 +++ linux-int-2.4.18/drivers/block/loop.c       Thu May 30 14:50:16 2002
   4 @@ -49,6 +49,18 @@
   5   *   problem above. Encryption modules that used to rely on the old scheme
   6   *   should just call ->i_mapping->bmap() to calculate the physical block
   7   *   number.
   8 + *
   9 + * AES transfer added. IV is now passed as (512 byte) sector number.
  10 + * Jari Ruusu <jari.ruusu@pp.inet.fi>, May 18 2001
  11 + *
  12 + * External encryption module locking bug fixed.
  13 + * Ingo Rohloff <rohloff@in.tum.de>, June 21 2001
  14 + *
  15 + * Make device backed loop work with swap (pre-allocated buffers + queue rewrite).
  16 + * Jari Ruusu <jari.ruusu@pp.inet.fi>, September 2 2001
  17 + *
  18 + * File backed code now uses file->f_op->read/write. Based on Andrew Morton's idea.
  19 + * Jari Ruusu <jari.ruusu@pp.inet.fi>, May 23 2002
  20   */
  21
  22  #include <linux/config.h>
  23 @@ -79,6 +91,7 @@
  24  static struct loop_device *loop_dev;
  25  static int *loop_sizes;
  26  static int *loop_blksizes;
  27 +static int *loop_hardsizes;
  28  static devfs_handle_t devfs_handle;      /*  For the directory */
  29
  30  /*
  31 @@ -93,7 +106,8 @@
  32                 else
  33                         memcpy(raw_buf, loop_buf, size);
  34         }
  35 -
  36 +       if (current->need_resched)
  37 +               schedule();
  38         return 0;
  39  }
  40
  41 @@ -115,12 +129,13 @@
  42         keysize = lo->lo_encrypt_key_size;
  43         for (i = 0; i < size; i++)
  44                 *out++ = *in++ ^ key[(i & 511) % keysize];
  45 +       if (current->need_resched)
  46 +               schedule();
  47         return 0;
  48  }
  49
  50  static int none_status(struct loop_device *lo, struct loop_info *info)
  51  {
  52 -       lo->lo_flags |= LO_FLAGS_BH_REMAP;
  53         return 0;
  54  }
  55
  56 @@ -146,320 +161,332 @@
  57  /* xfer_funcs[0] is special - its release function is never called */
  58  struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
  59         &none_funcs,
  60 -       &xor_funcs
  61 +       &xor_funcs,
  62  };
  63
  64 -#define MAX_DISK_SIZE 1024*1024*1024
  65 -
  66 -static int compute_loop_size(struct loop_device *lo, struct dentry * lo_dentry, kdev_t lodev)
  67 -{
  68 -       if (S_ISREG(lo_dentry->d_inode->i_mode))
  69 -               return (lo_dentry->d_inode->i_size - lo->lo_offset) >> BLOCK_SIZE_BITS;
  70 -       if (blk_size[MAJOR(lodev)])
  71 -               return blk_size[MAJOR(lodev)][MINOR(lodev)] -
  72 -                                (lo->lo_offset >> BLOCK_SIZE_BITS);
  73 -       return MAX_DISK_SIZE;
  74 +/*
  75 + *  First number of 'lo_prealloc' is the default number of RAM pages
  76 + *  to pre-allocate for each device backed loop. Every (configured)
  77 + *  device backed loop pre-allocates this amount of RAM pages unless
  78 + *  later 'lo_prealloc' numbers provide an override. 'lo_prealloc'
  79 + *  overrides are defined in pairs: loop_index,number_of_pages
  80 + */
  81 +static int lo_prealloc[9] = { 125, 999, 0, 999, 0, 999, 0, 999, 0 };
  82 +#define LO_PREALLOC_MIN 4    /* minimum user defined pre-allocated RAM pages */
  83 +#define LO_PREALLOC_MAX 512  /* maximum user defined pre-allocated RAM pages */
  84 +
  85 +#ifdef MODULE
  86 +MODULE_PARM(lo_prealloc, "1-9i");
  87 +MODULE_PARM_DESC(lo_prealloc, "Number of pre-allocated pages [,index,pages]...");
  88 +#else
  89 +static int __init lo_prealloc_setup(char *str)
  90 +{
  91 +       int x, y, z;
  92 +
  93 +       for (x = 0; x < (sizeof(lo_prealloc) / sizeof(int)); x++) {
  94 +               z = get_option(&str, &y);
  95 +               if (z > 0)
  96 +                       lo_prealloc[x] = y;
  97 +               if (z < 2)
  98 +                       break;
  99 +       }
 100 +       return 1;
 101  }
 102 +__setup("lo_prealloc=", lo_prealloc_setup);
 103 +#endif
 104
 105 -static void figure_loop_size(struct loop_device *lo)
 106 +typedef struct {
 107 +       struct buffer_head      **q0;
 108 +       struct buffer_head      **q1;
 109 +       struct buffer_head      **q2;
 110 +       int                     x0;
 111 +       int                     x1;
 112 +       int                     x2;
 113 +} que_look_up_table;
 114 +
 115 +static void loop_prealloc_cleanup(struct loop_device *lo)
 116  {
 117 -       loop_sizes[lo->lo_number] = compute_loop_size(lo,
 118 -                                       lo->lo_backing_file->f_dentry,
 119 -                                       lo->lo_device);
 120 +       struct buffer_head *bh;
 121 +
 122 +       while ((bh = lo->lo_bh_free)) {
 123 +               __free_page(bh->b_page);
 124 +               lo->lo_bh_free = bh->b_reqnext;
 125 +               bh->b_reqnext = NULL;
 126 +               kmem_cache_free(bh_cachep, bh);
 127 +       }
 128  }
 129
 130 -static int lo_send(struct loop_device *lo, struct buffer_head *bh, int bsize,
 131 -                  loff_t pos)
 132 +static int loop_prealloc_init(struct loop_device *lo, int y)
 133  {
 134 -       struct file *file = lo->lo_backing_file; /* kudos to NFsckingS */
 135 -       struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
 136 -       struct address_space_operations *aops = mapping->a_ops;
 137 -       struct page *page;
 138 -       char *kaddr, *data;
 139 -       unsigned long index;
 140 -       unsigned size, offset;
 141 -       int len;
 142 -
 143 -       down(&mapping->host->i_sem);
 144 -       index = pos >> PAGE_CACHE_SHIFT;
 145 -       offset = pos & (PAGE_CACHE_SIZE - 1);
 146 -       len = bh->b_size;
 147 -       data = bh->b_data;
 148 -       while (len > 0) {
 149 -               int IV = index * (PAGE_CACHE_SIZE/bsize) + offset/bsize;
 150 -               int transfer_result;
 151 +       struct buffer_head *bh;
 152 +       int x;
 153
 154 -               size = PAGE_CACHE_SIZE - offset;
 155 -               if (size > len)
 156 -                       size = len;
 157 +       if(!y) {
 158 +               y = lo_prealloc[0];
 159 +               for (x = 1; x < (sizeof(lo_prealloc) / sizeof(int)); x += 2) {
 160 +                       if (lo_prealloc[x + 1] && (lo->lo_number == lo_prealloc[x])) {
 161 +                               y = lo_prealloc[x + 1];
 162 +                               break;
 163 +                       }
 164 +               }
 165 +       }
 166 +       lo->lo_bh_flsh = (y * 3) / 4;
 167
 168 -               page = grab_cache_page(mapping, index);
 169 -               if (!page)
 170 -                       goto fail;
 171 -               kaddr = kmap(page);
 172 -               if (aops->prepare_write(file, page, offset, offset+size))
 173 -                       goto unlock;
 174 -               flush_dcache_page(page);
 175 -               transfer_result = lo_do_transfer(lo, WRITE, kaddr + offset, data, size, IV);
 176 -               if (transfer_result) {
 177 -                       /*
 178 -                        * The transfer failed, but we still write the data to
 179 -                        * keep prepare/commit calls balanced.
 180 -                        */
 181 -                       printk(KERN_ERR "loop: transfer error block %ld\n", index);
 182 -                       memset(kaddr + offset, 0, size);
 183 +       for (x = 0; x < y; x++) {
 184 +               bh = kmem_cache_alloc(bh_cachep, SLAB_KERNEL);
 185 +               if (!bh) {
 186 +                       loop_prealloc_cleanup(lo);
 187 +                       return 1;
 188                 }
 189 -               if (aops->commit_write(file, page, offset, offset+size))
 190 -                       goto unlock;
 191 -               if (transfer_result)
 192 -                       goto unlock;
 193 -               kunmap(page);
 194 -               data += size;
 195 -               len -= size;
 196 -               offset = 0;
 197 -               index++;
 198 -               pos += size;
 199 -               UnlockPage(page);
 200 -               page_cache_release(page);
 201 +               bh->b_page = alloc_page(GFP_KERNEL);
 202 +               if (!bh->b_page) {
 203 +                       bh->b_reqnext = NULL;
 204 +                       kmem_cache_free(bh_cachep, bh);
 205 +                       loop_prealloc_cleanup(lo);
 206 +                       return 1;
 207 +               }
 208 +               bh->b_reqnext = lo->lo_bh_free;
 209 +               lo->lo_bh_free = bh;
 210         }
 211 -       up(&mapping->host->i_sem);
 212         return 0;
 213 -
 214 -unlock:
 215 -       kunmap(page);
 216 -       UnlockPage(page);
 217 -       page_cache_release(page);
 218 -fail:
 219 -       up(&mapping->host->i_sem);
 220 -       return -1;
 221 -}
 222 -
 223 -struct lo_read_data {
 224 -       struct loop_device *lo;
 225 -       char *data;
 226 -       int bsize;
 227 -};
 228 -
 229 -static int lo_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
 230 -{
 231 -       char *kaddr;
 232 -       unsigned long count = desc->count;
 233 -       struct lo_read_data *p = (struct lo_read_data*)desc->buf;
 234 -       struct loop_device *lo = p->lo;
 235 -       int IV = page->index * (PAGE_CACHE_SIZE/p->bsize) + offset/p->bsize;
 236 -
 237 -       if (size > count)
 238 -               size = count;
 239 -
 240 -       kaddr = kmap(page);
 241 -       if (lo_do_transfer(lo, READ, kaddr + offset, p->data, size, IV)) {
 242 -               size = 0;
 243 -               printk(KERN_ERR "loop: transfer error block %ld\n",page->index);
 244 -               desc->error = -EINVAL;
 245 -       }
 246 -       kunmap(page);
 247 -
 248 -       desc->count = count - size;
 249 -       desc->written += size;
 250 -       p->data += size;
 251 -       return size;
 252 -}
 253 -
 254 -static int lo_receive(struct loop_device *lo, struct buffer_head *bh, int bsize,
 255 -                     loff_t pos)
 256 -{
 257 -       struct lo_read_data cookie;
 258 -       read_descriptor_t desc;
 259 -       struct file *file;
 260 -
 261 -       cookie.lo = lo;
 262 -       cookie.data = bh->b_data;
 263 -       cookie.bsize = bsize;
 264 -       desc.written = 0;
 265 -       desc.count = bh->b_size;
 266 -       desc.buf = (char*)&cookie;
 267 -       desc.error = 0;
 268 -       spin_lock_irq(&lo->lo_lock);
 269 -       file = lo->lo_backing_file;
 270 -       spin_unlock_irq(&lo->lo_lock);
 271 -       do_generic_file_read(file, &pos, &desc, lo_read_actor);
 272 -       return desc.error;
 273  }
 274
 275 -static inline int loop_get_bs(struct loop_device *lo)
 276 +static void loop_add_queue_last(struct loop_device *lo, struct buffer_head *bh, struct buffer_head **q)
 277  {
 278 -       int bs = 0;
 279 +       unsigned long flags;
 280
 281 -       if (blksize_size[MAJOR(lo->lo_device)])
 282 -               bs = blksize_size[MAJOR(lo->lo_device)][MINOR(lo->lo_device)];
 283 -       if (!bs)
 284 -               bs = BLOCK_SIZE;
 285 +       spin_lock_irqsave(&lo->lo_lock, flags);
 286 +       if (*q) {
 287 +               bh->b_reqnext = (*q)->b_reqnext;
 288 +               (*q)->b_reqnext = bh;
 289 +       } else {
 290 +               bh->b_reqnext = bh;
 291 +       }
 292 +       *q = bh;
 293 +       spin_unlock_irqrestore(&lo->lo_lock, flags);
 294
 295 -       return bs;
 296 +       if (waitqueue_active(&lo->lo_bh_wait))
 297 +               wake_up_interruptible(&lo->lo_bh_wait);
 298  }
 299
 300 -static inline unsigned long loop_get_iv(struct loop_device *lo,
 301 -                                       unsigned long sector)
 302 +static void loop_add_queue_first(struct loop_device *lo, struct buffer_head *bh, struct buffer_head **q)
 303  {
 304 -       int bs = loop_get_bs(lo);
 305 -       unsigned long offset, IV;
 306 -
 307 -       IV = sector / (bs >> 9) + lo->lo_offset / bs;
 308 -       offset = ((sector % (bs >> 9)) << 9) + lo->lo_offset % bs;
 309 -       if (offset >= bs)
 310 -               IV++;
 311 -
 312 -       return IV;
 313 +       spin_lock_irq(&lo->lo_lock);
 314 +       if (*q) {
 315 +               bh->b_reqnext = (*q)->b_reqnext;
 316 +               (*q)->b_reqnext = bh;
 317 +       } else {
 318 +               bh->b_reqnext = bh;
 319 +               *q = bh;
 320 +       }
 321 +       spin_unlock_irq(&lo->lo_lock);
 322  }
 323
 324 -static int do_bh_filebacked(struct loop_device *lo, struct buffer_head *bh, int rw)
 325 +static struct buffer_head *loop_get_bh(struct loop_device *lo, int *list_nr,
 326 +                                       que_look_up_table *qt)
 327  {
 328 -       loff_t pos;
 329 -       int ret;
 330 -
 331 -       pos = ((loff_t) bh->b_rsector << 9) + lo->lo_offset;
 332 +       struct buffer_head *bh = NULL, *last;
 333
 334 -       if (rw == WRITE)
 335 -               ret = lo_send(lo, bh, loop_get_bs(lo), pos);
 336 -       else
 337 -               ret = lo_receive(lo, bh, loop_get_bs(lo), pos);
 338 -
 339 -       return ret;
 340 -}
 341 -
 342 -static void loop_end_io_transfer(struct buffer_head *bh, int uptodate);
 343 -static void loop_put_buffer(struct buffer_head *bh)
 344 -{
 345 -       /*
 346 -        * check b_end_io, may just be a remapped bh and not an allocated one
 347 -        */
 348 -       if (bh && bh->b_end_io == loop_end_io_transfer) {
 349 -               __free_page(bh->b_page);
 350 -               kmem_cache_free(bh_cachep, bh);
 351 +       spin_lock_irq(&lo->lo_lock);
 352 +       if ((last = *qt->q0)) {
 353 +               bh = last->b_reqnext;
 354 +               if (bh == last)
 355 +                       *qt->q0 = NULL;
 356 +               else
 357 +                       last->b_reqnext = bh->b_reqnext;
 358 +               bh->b_reqnext = NULL;
 359 +               *list_nr = qt->x0;
 360 +       } else if ((last = *qt->q1)) {
 361 +               bh = last->b_reqnext;
 362 +               if (bh == last)
 363 +                       *qt->q1 = NULL;
 364 +               else
 365 +                       last->b_reqnext = bh->b_reqnext;
 366 +               bh->b_reqnext = NULL;
 367 +               *list_nr = qt->x1;
 368 +       } else if ((last = *qt->q2)) {
 369 +               bh = last->b_reqnext;
 370 +               if (bh == last)
 371 +                       *qt->q2 = NULL;
 372 +               else
 373 +                       last->b_reqnext = bh->b_reqnext;
 374 +               bh->b_reqnext = NULL;
 375 +               *list_nr = qt->x2;
 376         }
 377 +       spin_unlock_irq(&lo->lo_lock);
 378 +       return bh;
 379  }
 380
 381 -/*
 382 - * Add buffer_head to back of pending list
 383 - */
 384 -static void loop_add_bh(struct loop_device *lo, struct buffer_head *bh)
 385 +static void loop_put_buffer(struct loop_device *lo, struct buffer_head *b)
 386  {
 387         unsigned long flags;
 388 +       int wk;
 389
 390         spin_lock_irqsave(&lo->lo_lock, flags);
 391 -       if (lo->lo_bhtail) {
 392 -               lo->lo_bhtail->b_reqnext = bh;
 393 -               lo->lo_bhtail = bh;
 394 -       } else
 395 -               lo->lo_bh = lo->lo_bhtail = bh;
 396 +       b->b_reqnext = lo->lo_bh_free;
 397 +       lo->lo_bh_free = b;
 398 +       wk = lo->lo_bh_need;
 399         spin_unlock_irqrestore(&lo->lo_lock, flags);
 400
 401 -       up(&lo->lo_bh_mutex);
 402 +       if (wk && waitqueue_active(&lo->lo_bh_wait))
 403 +               wake_up_interruptible(&lo->lo_bh_wait);
 404  }
 405
 406 -/*
 407 - * Grab first pending buffer
 408 - */
 409 -static struct buffer_head *loop_get_bh(struct loop_device *lo)
 410 +static void loop_end_io_transfer_wr(struct buffer_head *bh, int uptodate)
 411  {
 412 -       struct buffer_head *bh;
 413 -
 414 -       spin_lock_irq(&lo->lo_lock);
 415 -       if ((bh = lo->lo_bh)) {
 416 -               if (bh == lo->lo_bhtail)
 417 -                       lo->lo_bhtail = NULL;
 418 -               lo->lo_bh = bh->b_reqnext;
 419 -               bh->b_reqnext = NULL;
 420 -       }
 421 -       spin_unlock_irq(&lo->lo_lock);
 422 +       struct loop_device *lo = &loop_dev[MINOR(bh->b_dev)];
 423 +       struct buffer_head *rbh = bh->b_private;
 424
 425 -       return bh;
 426 +       rbh->b_reqnext = NULL;
 427 +       rbh->b_end_io(rbh, uptodate);
 428 +       loop_put_buffer(lo, bh);
 429 +       if (atomic_dec_and_test(&lo->lo_pending))
 430 +               wake_up_interruptible(&lo->lo_bh_wait);
 431  }
 432
 433 -/*
 434 - * when buffer i/o has completed. if BH_Dirty is set, this was a WRITE
 435 - * and lo->transfer stuff has already been done. if not, it was a READ
 436 - * so queue it for the loop thread and let it do the transfer out of
 437 - * b_end_io context (we don't want to do decrypt of a page with irqs
 438 - * disabled)
 439 - */
 440 -static void loop_end_io_transfer(struct buffer_head *bh, int uptodate)
 441 +static void loop_end_io_transfer_rd(struct buffer_head *bh, int uptodate)
 442  {
 443         struct loop_device *lo = &loop_dev[MINOR(bh->b_dev)];
 444
 445 -       if (!uptodate || test_bit(BH_Dirty, &bh->b_state)) {
 446 -               struct buffer_head *rbh = bh->b_private;
 447 -
 448 -               rbh->b_end_io(rbh, uptodate);
 449 -               if (atomic_dec_and_test(&lo->lo_pending))
 450 -                       up(&lo->lo_bh_mutex);
 451 -               loop_put_buffer(bh);
 452 -       } else
 453 -               loop_add_bh(lo, bh);
 454 +       if (!uptodate)
 455 +               loop_end_io_transfer_wr(bh, uptodate);
 456 +       else
 457 +               loop_add_queue_last(lo, bh, &lo->lo_bhQue0);
 458  }
 459
 460  static struct buffer_head *loop_get_buffer(struct loop_device *lo,
 461 -                                          struct buffer_head *rbh)
 462 +               struct buffer_head *rbh, int from_thread, int rw)
 463  {
 464         struct buffer_head *bh;
 465 +       struct page *p;
 466 +       unsigned long flags;
 467
 468 -       /*
 469 -        * for xfer_funcs that can operate on the same bh, do that
 470 -        */
 471 -       if (lo->lo_flags & LO_FLAGS_BH_REMAP) {
 472 -               bh = rbh;
 473 -               goto out_bh;
 474 +       spin_lock_irqsave(&lo->lo_lock, flags);
 475 +       bh = lo->lo_bh_free;
 476 +       if (bh) {
 477 +               lo->lo_bh_free = bh->b_reqnext;
 478 +               if (from_thread)
 479 +                       lo->lo_bh_need = 0;
 480 +       } else {
 481 +               if (from_thread)
 482 +                       lo->lo_bh_need = 1;
 483         }
 484 +       spin_unlock_irqrestore(&lo->lo_lock, flags);
 485 +       if (!bh)
 486 +               return (struct buffer_head *)0;
 487
 488 -       do {
 489 -               bh = kmem_cache_alloc(bh_cachep, SLAB_NOIO);
 490 -               if (bh)
 491 -                       break;
 492 -
 493 -               run_task_queue(&tq_disk);
 494 -               set_current_state(TASK_INTERRUPTIBLE);
 495 -               schedule_timeout(HZ);
 496 -       } while (1);
 497 -       memset(bh, 0, sizeof(*bh));
 498 +       p = bh->b_page;
 499 +       memset(bh, 0, sizeof(struct buffer_head));
 500 +       bh->b_page = p;
 501
 502 +       bh->b_private = rbh;
 503         bh->b_size = rbh->b_size;
 504         bh->b_dev = rbh->b_rdev;
 505 +       bh->b_rdev = lo->lo_device;
 506         bh->b_state = (1 << BH_Req) | (1 << BH_Mapped) | (1 << BH_Lock);
 507 +       bh->b_data = page_address(bh->b_page);
 508 +       bh->b_end_io = (rw == WRITE) ? loop_end_io_transfer_wr : loop_end_io_transfer_rd;
 509 +       bh->b_rsector = rbh->b_rsector + (lo->lo_offset >> 9);
 510 +       init_waitqueue_head(&bh->b_wait);
 511
 512 -       /*
 513 -        * easy way out, although it does waste some memory for < PAGE_SIZE
 514 -        * blocks... if highmem bounce buffering can get away with it,
 515 -        * so can we :-)
 516 -        */
 517 -       do {
 518 -               bh->b_page = alloc_page(GFP_NOIO);
 519 -               if (bh->b_page)
 520 -                       break;
 521 +       return bh;
 522 +}
 523
 524 -               run_task_queue(&tq_disk);
 525 -               set_current_state(TASK_INTERRUPTIBLE);
 526 -               schedule_timeout(HZ);
 527 -       } while (1);
 528 +#define MAX_DISK_SIZE 1024*1024*1024
 529
 530 -       bh->b_data = page_address(bh->b_page);
 531 -       bh->b_end_io = loop_end_io_transfer;
 532 -       bh->b_private = rbh;
 533 -       init_waitqueue_head(&bh->b_wait);
 534 +static unsigned long compute_loop_size(struct loop_device *lo, struct dentry * lo_dentry, kdev_t lodev)
 535 +{
 536 +       if (S_ISREG(lo_dentry->d_inode->i_mode))
 537 +               return (lo_dentry->d_inode->i_size - lo->lo_offset) >> BLOCK_SIZE_BITS;
 538 +       if (blk_size[MAJOR(lodev)])
 539 +               return blk_size[MAJOR(lodev)][MINOR(lodev)] -
 540 +                                (lo->lo_offset >> BLOCK_SIZE_BITS);
 541 +       return MAX_DISK_SIZE;
 542 +}
 543
 544 -out_bh:
 545 -       bh->b_rsector = rbh->b_rsector + (lo->lo_offset >> 9);
 546 -       spin_lock_irq(&lo->lo_lock);
 547 -       bh->b_rdev = lo->lo_device;
 548 -       spin_unlock_irq(&lo->lo_lock);
 549 +static void figure_loop_size(struct loop_device *lo)
 550 +{
 551 +       loop_sizes[lo->lo_number] = compute_loop_size(lo,
 552 +                                       lo->lo_backing_file->f_dentry,
 553 +                                       lo->lo_device);
 554 +}
 555
 556 -       return bh;
 557 +static int loop_file_io(struct file *file, char *buf, int size, loff_t *ppos, int w)
 558 +{
 559 +       mm_segment_t fs;
 560 +       int x, y, z;
 561 +
 562 +       y = 0;
 563 +       do {
 564 +               z = size - y;
 565 +               fs = get_fs();
 566 +               set_fs(get_ds());
 567 +               if (w)
 568 +                       x = file->f_op->write(file, buf + y, z, ppos);
 569 +               else
 570 +                       x = file->f_op->read(file, buf + y, z, ppos);
 571 +               set_fs(fs);
 572 +               if (x < 0) {
 573 +                       if ((x == -EAGAIN) || (x == -ENOMEM) || (x == -ERESTART) || (x == -EINTR)) {
 574 +                               run_task_queue(&tq_disk);
 575 +                               set_current_state(TASK_INTERRUPTIBLE);
 576 +                               schedule_timeout(HZ / 2);
 577 +                               continue;
 578 +                       }
 579 +                       return 1;
 580 +               }
 581 +               y += x;
 582 +       } while (y < size);
 583 +       return 0;
 584 +}
 585 +
 586 +static int do_bh_filebacked(struct loop_device *lo, struct buffer_head *bh, int rw)
 587 +{
 588 +       loff_t pos;
 589 +       struct file *file = lo->lo_backing_file;
 590 +       char *data, *buf;
 591 +       unsigned int size, len;
 592 +       int IV;
 593 +
 594 +       pos = ((loff_t) bh->b_rsector << 9) + lo->lo_offset;
 595 +       buf = page_address(lo->lo_bh_free->b_page);
 596 +       len = bh->b_size;
 597 +       data = bh->b_data;
 598 +       IV = bh->b_rsector + (lo->lo_offset >> 9);
 599 +       while (len > 0) {
 600 +               size = PAGE_CACHE_SIZE;
 601 +               if (size > len)
 602 +                       size = len;
 603 +               if (rw == WRITE) {
 604 +                       if (lo_do_transfer(lo, WRITE, buf, data, size, IV)) {
 605 +                               printk(KERN_ERR "loop: write transfer error, sector %d\n", IV);
 606 +                               return 1;
 607 +                       }
 608 +                       if (loop_file_io(file, buf, size, &pos, 1)) {
 609 +                               printk(KERN_ERR "loop: write i/o error, sector %d\n", IV);
 610 +                               return 1;
 611 +                       }
 612 +               } else {
 613 +                       if (loop_file_io(file, buf, size, &pos, 0)) {
 614 +                               printk(KERN_ERR "loop: read i/o error, sector %d\n", IV);
 615 +                               return 1;
 616 +                       }
 617 +                       if (lo_do_transfer(lo, READ, buf, data, size, IV)) {
 618 +                               printk(KERN_ERR "loop: read transfer error, sector %d\n", IV);
 619 +                               return 1;
 620 +                       }
 621 +               }
 622 +               data += size;
 623 +               len -= size;
 624 +               IV += size >> 9;
 625 +       }
 626 +       return 0;
 627  }
 628
 629  static int loop_make_request(request_queue_t *q, int rw, struct buffer_head *rbh)
 630  {
 631 -       struct buffer_head *bh = NULL;
 632 +       struct buffer_head *bh;
 633         struct loop_device *lo;
 634 -       unsigned long IV;
 635
 636         if (!buffer_locked(rbh))
 637                 BUG();
 638 @@ -488,35 +519,43 @@
 639          * file backed, queue for loop_thread to handle
 640          */
 641         if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
 642 -               /*
 643 -                * rbh locked at this point, noone else should clear
 644 -                * the dirty flag
 645 -                */
 646 -               if (rw == WRITE)
 647 -                       set_bit(BH_Dirty, &rbh->b_state);
 648 -               loop_add_bh(lo, rbh);
 649 +               loop_add_queue_last(lo, rbh, (rw == WRITE) ? &lo->lo_bhQue1 : &lo->lo_bhQue0);
 650 +               return 0;
 651 +       }
 652 +
 653 +       /*
 654 +        * device backed, just remap rdev & rsector for NONE transfer
 655 +        */
 656 +       if (lo->lo_encrypt_type == LO_CRYPT_NONE) {
 657 +               rbh->b_rsector += lo->lo_offset >> 9;
 658 +               rbh->b_rdev = lo->lo_device;
 659 +               generic_make_request(rw, rbh);
 660 +               if (atomic_dec_and_test(&lo->lo_pending))
 661 +                       wake_up_interruptible(&lo->lo_bh_wait);
 662                 return 0;
 663         }
 664
 665         /*
 666 -        * piggy old buffer on original, and submit for I/O
 667 +        * device backed, start reads and writes now if buffer available
 668          */
 669 -       bh = loop_get_buffer(lo, rbh);
 670 -       IV = loop_get_iv(lo, rbh->b_rsector);
 671 +       bh = loop_get_buffer(lo, rbh, 0, rw);
 672 +       if (!bh) {
 673 +               /* just queue request and let thread handle alloc later */
 674 +               loop_add_queue_last(lo, rbh, (rw == WRITE) ? &lo->lo_bhQue1 : &lo->lo_bhQue2);
 675 +               return 0;
 676 +       }
 677         if (rw == WRITE) {
 678 -               set_bit(BH_Dirty, &bh->b_state);
 679 -               if (lo_do_transfer(lo, WRITE, bh->b_data, rbh->b_data,
 680 -                                  bh->b_size, IV))
 681 +               if (lo_do_transfer(lo, WRITE, bh->b_data, rbh->b_data, bh->b_size, bh->b_rsector)) {
 682 +                       loop_put_buffer(lo, bh);
 683                         goto err;
 684 +               }
 685         }
 686 -
 687         generic_make_request(rw, bh);
 688         return 0;
 689
 690  err:
 691         if (atomic_dec_and_test(&lo->lo_pending))
 692 -               up(&lo->lo_bh_mutex);
 693 -       loop_put_buffer(bh);
 694 +               wake_up_interruptible(&lo->lo_bh_wait);
 695  out:
 696         buffer_IO_error(rbh);
 697         return 0;
 698 @@ -525,30 +564,6 @@
 699         goto out;
 700  }
 701
 702 -static inline void loop_handle_bh(struct loop_device *lo,struct buffer_head *bh)
 703 -{
 704 -       int ret;
 705 -
 706 -       /*
 707 -        * For block backed loop, we know this is a READ
 708 -        */
 709 -       if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
 710 -               int rw = !!test_and_clear_bit(BH_Dirty, &bh->b_state);
 711 -
 712 -               ret = do_bh_filebacked(lo, bh, rw);
 713 -               bh->b_end_io(bh, !ret);
 714 -       } else {
 715 -               struct buffer_head *rbh = bh->b_private;
 716 -               unsigned long IV = loop_get_iv(lo, rbh->b_rsector);
 717 -
 718 -               ret = lo_do_transfer(lo, READ, bh->b_data, rbh->b_data,
 719 -                                    bh->b_size, IV);
 720 -
 721 -               rbh->b_end_io(rbh, !ret);
 722 -               loop_put_buffer(bh);
 723 -       }
 724 -}
 725 -
 726  /*
 727   * worker thread that handles reads/writes to file backed loop devices,
 728   * to avoid blocking in our make_request_fn. it also does loop decrypting
 729 @@ -558,8 +573,17 @@
 730  static int loop_thread(void *data)
 731  {
 732         struct loop_device *lo = data;
 733 -       struct buffer_head *bh;
 734 +       struct buffer_head *bh, *xbh;
 735 +       int x, rw, qi = 0, flushcnt = 0;
 736 +       wait_queue_t waitq;
 737 +       que_look_up_table qt[4] = {
 738 +               { &lo->lo_bhQue0, &lo->lo_bhQue1, &lo->lo_bhQue2, 0, 1, 2 },
 739 +               { &lo->lo_bhQue2, &lo->lo_bhQue0, &lo->lo_bhQue1, 2, 0, 1 },
 740 +               { &lo->lo_bhQue0, &lo->lo_bhQue2, &lo->lo_bhQue1, 0, 2, 1 },
 741 +               { &lo->lo_bhQue1, &lo->lo_bhQue0, &lo->lo_bhQue2, 1, 0, 2 }
 742 +       };
 743
 744 +       init_waitqueue_entry(&waitq, current);
 745         daemonize();
 746         exit_files(current);
 747         reparent_to_init();
 748 @@ -586,23 +611,104 @@
 749         up(&lo->lo_sem);
 750
 751         for (;;) {
 752 -               down_interruptible(&lo->lo_bh_mutex);
 753 +               add_wait_queue(&lo->lo_bh_wait, &waitq);
 754 +               for (;;) {
 755 +                       set_current_state(TASK_INTERRUPTIBLE);
 756 +                       if (!atomic_read(&lo->lo_pending))
 757 +                               break;
 758 +
 759 +                       x = 0;
 760 +                       spin_lock_irq(&lo->lo_lock);
 761 +                       if (lo->lo_bhQue0) {
 762 +                               x = 1;
 763 +                       } else if (lo->lo_bhQue1 || lo->lo_bhQue2) {
 764 +                               /* file backed works too because lo->lo_bh_need == 0 */
 765 +                               if (lo->lo_bh_free || !lo->lo_bh_need)
 766 +                                       x = 1;
 767 +                       }
 768 +                       spin_unlock_irq(&lo->lo_lock);
 769 +                       if (x)
 770 +                               break;
 771 +
 772 +                       schedule();
 773 +               }
 774 +               current->state = TASK_RUNNING;
 775 +               remove_wait_queue(&lo->lo_bh_wait, &waitq);
 776 +
 777                 /*
 778 -                * could be upped because of tear-down, not because of
 779 +                * could be woken because of tear-down, not because of
 780                  * pending work
 781                  */
 782                 if (!atomic_read(&lo->lo_pending))
 783                         break;
 784
 785 -               bh = loop_get_bh(lo);
 786 -               if (!bh) {
 787 -                       printk("loop: missing bh\n");
 788 +               /*
 789 +                * read queues using alternating order to prevent starvation
 790 +                */
 791 +               bh = loop_get_bh(lo, &x, &qt[++qi & 3]);
 792 +               if (!bh)
 793 +                       continue;
 794 +
 795 +               /*
 796 +                *  x  list tag       usage(buffer-allocated)
 797 +                * --- -------------  -----------------------
 798 +                *  0  lo->lo_bhQue0  dev-read(y) / file-read
 799 +                *  1  lo->lo_bhQue1  dev-write(n) / file-write
 800 +                *  2  lo->lo_bhQue2  dev-read(n)
 801 +                */
 802 +               rw = (x == 1) ? WRITE : READ;
 803 +               if ((x >= 1) && !(lo->lo_flags & LO_FLAGS_DO_BMAP)) {
 804 +                       /* loop_make_request didn't allocate a buffer, do that now */
 805 +                       xbh = loop_get_buffer(lo, bh, 1, rw);
 806 +                       if (!xbh) {
 807 +                               run_task_queue(&tq_disk);
 808 +                               flushcnt = 0;
 809 +                               loop_add_queue_first(lo, bh, (rw == WRITE) ? &lo->lo_bhQue1 : &lo->lo_bhQue2);
 810 +                               /* lo->lo_bh_need should be 1 now, go back to sleep */
 811 +                               continue;
 812 +                       }
 813 +                       if (rw == WRITE) {
 814 +                               if (lo_do_transfer(lo, WRITE, xbh->b_data, bh->b_data, xbh->b_size, xbh->b_rsector)) {
 815 +                                       loop_put_buffer(lo, xbh);
 816 +                                       buffer_IO_error(bh);
 817 +                                       atomic_dec(&lo->lo_pending);
 818 +                                       continue;
 819 +                               }
 820 +                       }
 821 +                       generic_make_request(rw, xbh);
 822 +
 823 +                       /* start I/O if there are no more requests lacking buffers */
 824 +                       x = 0;
 825 +                       spin_lock_irq(&lo->lo_lock);
 826 +                       if (!lo->lo_bhQue1 && !lo->lo_bhQue2)
 827 +                               x = 1;
 828 +                       spin_unlock_irq(&lo->lo_lock);
 829 +                       if (x || (++flushcnt >= lo->lo_bh_flsh)) {
 830 +                               run_task_queue(&tq_disk);
 831 +                               flushcnt = 0;
 832 +                       }
 833 +
 834 +                       /* request not completely processed yet */
 835                         continue;
 836                 }
 837 -               loop_handle_bh(lo, bh);
 838 +               if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
 839 +                       /* request is for file backed device */
 840 +                       x = do_bh_filebacked(lo, bh, rw);
 841 +                       bh->b_reqnext = NULL;
 842 +                       bh->b_end_io(bh, !x);
 843 +               } else {
 844 +                       /* device backed read has completed, do decrypt now */
 845 +                       xbh = bh->b_private;
 846 +                       /* must not use bh->b_rsector as IV, as it may be modified by LVM at this point */
 847 +                       /* instead, recompute IV from original request */
 848 +                       x = lo_do_transfer(lo, READ, bh->b_data, xbh->b_data, bh->b_size, xbh->b_rsector + (lo->lo_offset >> 9));
 849 +                       xbh->b_reqnext = NULL;
 850 +                       xbh->b_end_io(xbh, !x);
 851 +                       loop_put_buffer(lo, bh);
 852 +               }
 853
 854                 /*
 855 -                * upped both for pending work and tear-down, lo_pending
 856 +                * woken both for pending work and tear-down, lo_pending
 857                  * will hit zero then
 858                  */
 859                 if (atomic_dec_and_test(&lo->lo_pending))
 860 @@ -619,7 +725,7 @@
 861         struct file     *file;
 862         struct inode    *inode;
 863         kdev_t          lo_device;
 864 -       int             lo_flags = 0;
 865 +       int             lo_flags = 0, hardsz = 512;
 866         int             error;
 867         int             bs;
 868
 869 @@ -640,33 +746,44 @@
 870         if (!(file->f_mode & FMODE_WRITE))
 871                 lo_flags |= LO_FLAGS_READ_ONLY;
 872
 873 +       lo->lo_bh_free = lo->lo_bhQue2 = lo->lo_bhQue1 = lo->lo_bhQue0 = NULL;
 874 +       lo->lo_bh_need = lo->lo_bh_flsh = 0;
 875 +       init_waitqueue_head(&lo->lo_bh_wait);
 876         if (S_ISBLK(inode->i_mode)) {
 877                 lo_device = inode->i_rdev;
 878                 if (lo_device == dev) {
 879                         error = -EBUSY;
 880                         goto out_putf;
 881 +               }
 882 +               if (loop_prealloc_init(lo, 0)) {
 883 +                       error = -ENOMEM;
 884 +                       goto out_putf;
 885                 }
 886 +               hardsz = get_hardsect_size(lo_device);
 887         } else if (S_ISREG(inode->i_mode)) {
 888 -               struct address_space_operations *aops = inode->i_mapping->a_ops;
 889                 /*
 890                  * If we can't read - sorry. If we only can't write - well,
 891                  * it's going to be read-only.
 892                  */
 893 -               if (!aops->readpage)
 894 +               if (!file->f_op || !file->f_op->read)
 895                         goto out_putf;
 896
 897 -               if (!aops->prepare_write || !aops->commit_write)
 898 +               if (!file->f_op->write)
 899                         lo_flags |= LO_FLAGS_READ_ONLY;
 900
 901                 lo_device = inode->i_dev;
 902                 lo_flags |= LO_FLAGS_DO_BMAP;
 903 +               if (loop_prealloc_init(lo, 1)) {
 904 +                       error = -ENOMEM;
 905 +                       goto out_putf;
 906 +               }
 907                 error = 0;
 908         } else
 909                 goto out_putf;
 910
 911         get_file(file);
 912
 913 -       if (IS_RDONLY (inode) || is_read_only(lo_device)
 914 +       if ((S_ISREG(inode->i_mode) && IS_RDONLY(inode)) || is_read_only(lo_device)
 915             || !(lo_file->f_mode & FMODE_WRITE))
 916                 lo_flags |= LO_FLAGS_READ_ONLY;
 917
 918 @@ -687,9 +804,19 @@
 919         if (!bs)
 920                 bs = BLOCK_SIZE;
 921
 922 +       if (S_ISREG(inode->i_mode)) {
 923 +               int x = loop_sizes[lo->lo_number];
 924 +
 925 +               if ((bs == 8192) && (x & 7))
 926 +                       bs = 4096;
 927 +               if ((bs == 4096) && (x & 3))
 928 +                       bs = 2048;
 929 +               if ((bs == 2048) && (x & 1))
 930 +                       bs = 1024;
 931 +       }
 932 +       loop_hardsizes[MINOR(dev)] = hardsz;
 933         set_blocksize(dev, bs);
 934
 935 -       lo->lo_bh = lo->lo_bhtail = NULL;
 936         kernel_thread(loop_thread, lo, CLONE_FS | CLONE_FILES | CLONE_SIGHAND);
 937         down(&lo->lo_sem);
 938
 939 @@ -748,11 +875,12 @@
 940         spin_lock_irq(&lo->lo_lock);
 941         lo->lo_state = Lo_rundown;
 942         if (atomic_dec_and_test(&lo->lo_pending))
 943 -               up(&lo->lo_bh_mutex);
 944 +               wake_up_interruptible(&lo->lo_bh_wait);
 945         spin_unlock_irq(&lo->lo_lock);
 946
 947         down(&lo->lo_sem);
 948
 949 +       loop_prealloc_cleanup(lo);
 950         lo->lo_backing_file = NULL;
 951
 952         loop_release_xfer(lo);
 953 @@ -902,7 +1030,7 @@
 954  static int lo_open(struct inode *inode, struct file *file)
 955  {
 956         struct loop_device *lo;
 957 -       int     dev, type;
 958 +       int     dev;
 959
 960         if (!inode)
 961                 return -EINVAL;
 962 @@ -917,10 +1045,6 @@
 963         lo = &loop_dev[dev];
 964         MOD_INC_USE_COUNT;
 965         down(&lo->lo_ctl_mutex);
 966 -
 967 -       type = lo->lo_encrypt_type;
 968 -       if (type && xfer_funcs[type] && xfer_funcs[type]->lock)
 969 -               xfer_funcs[type]->lock(lo);
 970         lo->lo_refcnt++;
 971         up(&lo->lo_ctl_mutex);
 972         return 0;
 973 @@ -929,7 +1053,7 @@
 974  static int lo_release(struct inode *inode, struct file *file)
 975  {
 976         struct loop_device *lo;
 977 -       int     dev, type;
 978 +       int     dev;
 979
 980         if (!inode)
 981                 return 0;
 982 @@ -944,11 +1068,7 @@
 983
 984         lo = &loop_dev[dev];
 985         down(&lo->lo_ctl_mutex);
 986 -       type = lo->lo_encrypt_type;
 987         --lo->lo_refcnt;
 988 -       if (xfer_funcs[type] && xfer_funcs[type]->unlock)
 989 -               xfer_funcs[type]->unlock(lo);
 990 -
 991         up(&lo->lo_ctl_mutex);
 992         MOD_DEC_USE_COUNT;
 993         return 0;
 994 @@ -1031,6 +1151,10 @@
 995         if (!loop_blksizes)
 996                 goto out_blksizes;
 997
 998 +       loop_hardsizes = kmalloc(max_loop * sizeof(int), GFP_KERNEL);
 999 +       if (!loop_hardsizes)
1000 +               goto out_hardsizes;
1001 +
1002         blk_queue_make_request(BLK_DEFAULT_QUEUE(MAJOR_NR), loop_make_request);
1003
1004         for (i = 0; i < max_loop; i++) {
1005 @@ -1038,27 +1162,38 @@
1006                 memset(lo, 0, sizeof(struct loop_device));
1007                 init_MUTEX(&lo->lo_ctl_mutex);
1008                 init_MUTEX_LOCKED(&lo->lo_sem);
1009 -               init_MUTEX_LOCKED(&lo->lo_bh_mutex);
1010                 lo->lo_number = i;
1011                 spin_lock_init(&lo->lo_lock);
1012         }
1013
1014         memset(loop_sizes, 0, max_loop * sizeof(int));
1015         memset(loop_blksizes, 0, max_loop * sizeof(int));
1016 +       memset(loop_hardsizes, 0, max_loop * sizeof(int));
1017         blk_size[MAJOR_NR] = loop_sizes;
1018         blksize_size[MAJOR_NR] = loop_blksizes;
1019 +       hardsect_size[MAJOR_NR] = loop_hardsizes;
1020         for (i = 0; i < max_loop; i++)
1021                 register_disk(NULL, MKDEV(MAJOR_NR, i), 1, &lo_fops, 0);
1022
1023         devfs_handle = devfs_mk_dir(NULL, "loop", NULL);
1024         devfs_register_series(devfs_handle, "%u", max_loop, DEVFS_FL_DEFAULT,
1025                               MAJOR_NR, 0,
1026                               S_IFBLK | S_IRUSR | S_IWUSR | S_IRGRP,
1027                               &lo_fops, NULL);
1028
1029 +       for (i = 0; i < (sizeof(lo_prealloc) / sizeof(int)); i += 2) {
1030 +               if (!lo_prealloc[i])
1031 +                       continue;
1032 +               if (lo_prealloc[i] < LO_PREALLOC_MIN)
1033 +                       lo_prealloc[i] = LO_PREALLOC_MIN;
1034 +               if (lo_prealloc[i] > LO_PREALLOC_MAX)
1035 +                       lo_prealloc[i] = LO_PREALLOC_MAX;
1036 +       }
1037         printk(KERN_INFO "loop: loaded (max %d devices)\n", max_loop);
1038         return 0;
1039
1040 +out_hardsizes:
1041 +       kfree(loop_blksizes);
1042  out_blksizes:
1043         kfree(loop_sizes);
1044  out_sizes:
1045 @@ -1067,9 +1202,13 @@
1046         devfs_unregister(devfs_handle);
1047         if (devfs_unregister_blkdev(MAJOR_NR, "loop"))
1048                 printk(KERN_WARNING "loop: cannot unregister blkdev\n");
1049 +       blk_size[MAJOR_NR] = 0;
1050 +       blksize_size[MAJOR_NR] = 0;
1051 +       hardsect_size[MAJOR_NR] = 0;
1052         kfree(loop_dev);
1053         kfree(loop_sizes);
1054         kfree(loop_blksizes);
1055 +       kfree(loop_hardsizes);
1056  }
1057
1058  module_init(loop_init);
1059 diff -Nur linux-2.4.18/include/linux/loop.h linux-int-2.4.18/include/linux/loop.h
1060 --- linux-2.4.18/include/linux/loop.h   Mon Sep 17 22:16:30 2001
1061 +++ linux-int-2.4.18/include/linux/loop.h       Thu May 30 14:48:01 2002
1062 @@ -17,6 +17,11 @@
1063
1064  #ifdef __KERNEL__
1065
1066 +/* definitions for IV metric -- cryptoapi specific */
1067 +#define LOOP_IV_SECTOR_BITS 9
1068 +#define LOOP_IV_SECTOR_SIZE (1 << LOOP_IV_SECTOR_BITS)
1069 +typedef int loop_iv_t;
1070 +
1071  /* Possible states of device */
1072  enum {
1073         Lo_unbound,
1074 @@ -49,13 +54,17 @@
1075         int             old_gfp_mask;
1076
1077         spinlock_t              lo_lock;
1078 -       struct buffer_head      *lo_bh;
1079 -       struct buffer_head      *lo_bhtail;
1080 +       struct buffer_head      *lo_bhQue0;
1081 +       struct buffer_head      *lo_bhQue1;
1082         int                     lo_state;
1083         struct semaphore        lo_sem;
1084         struct semaphore        lo_ctl_mutex;
1085 -       struct semaphore        lo_bh_mutex;
1086         atomic_t                lo_pending;
1087 +       struct buffer_head      *lo_bhQue2;
1088 +       struct buffer_head      *lo_bh_free;
1089 +       int                     lo_bh_flsh;
1090 +       int                     lo_bh_need;
1091 +       wait_queue_head_t       lo_bh_wait;
1092  };
1093
1094  typedef        int (* transfer_proc_t)(struct loop_device *, int cmd,
1095 @@ -77,7 +86,6 @@
1096   */
1097  #define LO_FLAGS_DO_BMAP       1
1098  #define LO_FLAGS_READ_ONLY     2
1099 -#define LO_FLAGS_BH_REMAP      4
1100
1101  /*
1102   * Note that this structure gets the wrong offsets when directly used
1103 @@ -122,6 +130,8 @@
1104  #define LO_CRYPT_IDEA     6
1105  #define LO_CRYPT_DUMMY    9
1106  #define LO_CRYPT_SKIPJACK 10
1107 +#define LO_CRYPT_AES      16
1108 +#define LO_CRYPT_CRYPTOAPI 18
1109  #define MAX_LO_CRYPT   20
1110
1111  #ifdef __KERNEL__