1 dm-crypt: offload writes to thread
3 Submitting write bios directly in the encryption thread caused serious
4 performance degradation. On multiprocessor machine encryption requests
5 finish in a different order than they were submitted in. Consequently, write
6 requests would be submitted in a different order and it could cause severe
7 performance degradation.
9 This patch moves submitting write requests to a separate thread so that
10 the requests can be sorted before submitting.
12 Sorting is implemented in the next patch.
14 Note: it is required that a previous patch "dm-crypt: don't allocate pages
15 for a partial request." is applied before applying this patch. Without
16 that, this patch could introduce a crash.
18 Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
21 drivers/md/dm-crypt.c | 120 ++++++++++++++++++++++++++++++++++++++++----------
22 1 file changed, 97 insertions(+), 23 deletions(-)
24 Index: linux-3.14/drivers/md/dm-crypt.c
25 ===================================================================
26 --- linux-3.14.orig/drivers/md/dm-crypt.c 2014-04-04 21:05:40.000000000 +0200
27 +++ linux-3.14/drivers/md/dm-crypt.c 2014-04-04 21:06:22.000000000 +0200
29 #include <linux/slab.h>
30 #include <linux/crypto.h>
31 #include <linux/workqueue.h>
32 +#include <linux/kthread.h>
33 #include <linux/backing-dev.h>
34 #include <linux/atomic.h>
35 #include <linux/scatterlist.h>
36 @@ -58,6 +59,8 @@ struct dm_crypt_io {
41 + struct list_head list;
42 } CRYPTO_MINALIGN_ATTR;
44 struct dm_crypt_request {
45 @@ -128,6 +131,10 @@ struct crypt_config {
46 struct workqueue_struct *io_queue;
47 struct workqueue_struct *crypt_queue;
49 + struct task_struct *write_thread;
50 + wait_queue_head_t write_thread_wait;
51 + struct list_head write_thread_list;
56 @@ -1141,37 +1148,89 @@ static int kcryptd_io_read(struct dm_cry
60 +static void kcryptd_io_read_work(struct work_struct *work)
62 + struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
64 + crypt_inc_pending(io);
65 + if (kcryptd_io_read(io, GFP_NOIO))
66 + io->error = -ENOMEM;
67 + crypt_dec_pending(io);
70 +static void kcryptd_queue_read(struct dm_crypt_io *io)
72 + struct crypt_config *cc = io->cc;
74 + INIT_WORK(&io->work, kcryptd_io_read_work);
75 + queue_work(cc->io_queue, &io->work);
78 static void kcryptd_io_write(struct dm_crypt_io *io)
80 struct bio *clone = io->ctx.bio_out;
82 generic_make_request(clone);
85 -static void kcryptd_io(struct work_struct *work)
86 +static int dmcrypt_write(void *data)
88 - struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
89 + struct crypt_config *cc = data;
91 + struct list_head local_list;
92 + struct blk_plug plug;
94 - if (bio_data_dir(io->base_bio) == READ) {
95 - crypt_inc_pending(io);
96 - if (kcryptd_io_read(io, GFP_NOIO))
97 - io->error = -ENOMEM;
98 - crypt_dec_pending(io);
100 - kcryptd_io_write(io);
102 + DECLARE_WAITQUEUE(wait, current);
104 -static void kcryptd_queue_io(struct dm_crypt_io *io)
106 - struct crypt_config *cc = io->cc;
107 + spin_lock_irq(&cc->write_thread_wait.lock);
110 - INIT_WORK(&io->work, kcryptd_io);
111 - queue_work(cc->io_queue, &io->work);
112 + if (!list_empty(&cc->write_thread_list))
113 + goto pop_from_list;
115 + __set_current_state(TASK_INTERRUPTIBLE);
116 + __add_wait_queue(&cc->write_thread_wait, &wait);
118 + spin_unlock_irq(&cc->write_thread_wait.lock);
120 + if (unlikely(kthread_should_stop())) {
121 + set_task_state(current, TASK_RUNNING);
122 + remove_wait_queue(&cc->write_thread_wait, &wait);
128 + set_task_state(current, TASK_RUNNING);
129 + spin_lock_irq(&cc->write_thread_wait.lock);
130 + __remove_wait_queue(&cc->write_thread_wait, &wait);
131 + goto continue_locked;
134 + local_list = cc->write_thread_list;
135 + local_list.next->prev = &local_list;
136 + local_list.prev->next = &local_list;
137 + INIT_LIST_HEAD(&cc->write_thread_list);
139 + spin_unlock_irq(&cc->write_thread_wait.lock);
141 + blk_start_plug(&plug);
143 + struct dm_crypt_io *io = container_of(local_list.next,
144 + struct dm_crypt_io, list);
145 + list_del(&io->list);
146 + kcryptd_io_write(io);
147 + } while (!list_empty(&local_list));
148 + blk_finish_plug(&plug);
153 -static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
154 +static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io)
156 struct bio *clone = io->ctx.bio_out;
157 struct crypt_config *cc = io->cc;
158 + unsigned long flags;
160 if (unlikely(io->error < 0)) {
161 crypt_free_buffer_pages(cc, clone);
162 @@ -1185,10 +1244,10 @@ static void kcryptd_crypt_write_io_submi
164 clone->bi_iter.bi_sector = cc->start + io->sector;
167 - kcryptd_queue_io(io);
169 - generic_make_request(clone);
170 + spin_lock_irqsave(&cc->write_thread_wait.lock, flags);
171 + list_add_tail(&io->list, &cc->write_thread_list);
172 + wake_up_locked(&cc->write_thread_wait);
173 + spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags);
176 static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
177 @@ -1224,7 +1283,7 @@ static void kcryptd_crypt_write_convert(
179 /* Encryption was already finished, submit io now */
180 if (crypt_finished) {
181 - kcryptd_crypt_write_io_submit(io, 0);
182 + kcryptd_crypt_write_io_submit(io);
186 @@ -1284,7 +1343,7 @@ static void kcryptd_async_done(struct cr
187 if (bio_data_dir(io->base_bio) == READ)
188 kcryptd_crypt_read_done(io);
190 - kcryptd_crypt_write_io_submit(io, 1);
191 + kcryptd_crypt_write_io_submit(io);
194 static void kcryptd_crypt(struct work_struct *work)
195 @@ -1431,6 +1490,9 @@ static void crypt_dtr(struct dm_target *
199 + if (cc->write_thread)
200 + kthread_stop(cc->write_thread);
203 destroy_workqueue(cc->io_queue);
205 @@ -1745,6 +1807,18 @@ static int crypt_ctr(struct dm_target *t
209 + init_waitqueue_head(&cc->write_thread_wait);
210 + INIT_LIST_HEAD(&cc->write_thread_list);
212 + cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write");
213 + if (IS_ERR(cc->write_thread)) {
214 + ret = PTR_ERR(cc->write_thread);
215 + cc->write_thread = NULL;
216 + ti->error = "Couldn't spawn write thread";
219 + wake_up_process(cc->write_thread);
221 ti->num_flush_bios = 1;
222 ti->discard_zeroes_data_unsupported = true;
224 @@ -1779,7 +1853,7 @@ static int crypt_map(struct dm_target *t
226 if (bio_data_dir(io->base_bio) == READ) {
227 if (kcryptd_io_read(io, GFP_NOWAIT))
228 - kcryptd_queue_io(io);
229 + kcryptd_queue_read(io);
231 kcryptd_queue_crypt(io);