1 --- rsync-3.1.3/checksum.c~ 2018-01-15 05:55:07.000000000 +0200
2 +++ rsync-3.1.3/checksum.c 2018-12-04 00:27:15.382240696 +0200
4 extern int proper_seed_order;
5 extern char *checksum_choice;
7 +#ifdef WITH_DROP_CACHE
8 +#define close(fd) fadv_close(fd)
12 #define CSUM_MD4_ARCHAIC 1
13 #define CSUM_MD4_BUSTED 2
14 --- rsync-3.1.2.orig/cleanup.c 2015-08-08 22:47:03.000000000 +0300
15 +++ rsync-3.1.2/cleanup.c 2016-10-24 15:38:28.002415712 +0300
22 +#ifdef WITH_DROP_CACHE
25 +#ifdef SHUTDOWN_ALL_SOCKETS
26 max_fd = sysconf(_SC_OPEN_MAX) - 1;
27 for (fd = max_fd; fd >= 0; fd--) {
28 if ((ret = do_fstat(fd, &st)) == 0) {
29 diff -ru rsync-3.1.2.orig/config.h.in rsync-3.1.2/config.h.in
30 --- rsync-3.1.2.orig/config.h.in 2015-12-21 22:20:53.000000000 +0200
31 +++ rsync-3.1.2/config.h.in 2016-10-24 15:38:28.006415712 +0300
33 /* Define to 1 if you have the <memory.h> header file. */
36 +/* Define to 1 if you have the `mincore' function. */
39 /* Define to 1 if you have the `mkfifo' function. */
43 /* Define to 1 if the system has the type `mode_t'. */
46 +/* Define to 1 if you have the `mmap' function. */
49 /* Define to 1 if you have the `mtrace' function. */
53 /* true if you have posix ACLs */
54 #undef HAVE_POSIX_ACLS
56 +/* Define to 1 if you have the `posix_fadvise64' function. */
57 +#undef HAVE_POSIX_FADVISE64
59 /* Define to 1 if you have the `posix_fallocate' function. */
60 #undef HAVE_POSIX_FALLOCATE
62 diff -ru rsync-3.1.2.orig/configure.ac rsync-3.1.2/configure.ac
63 --- rsync-3.1.2.orig/configure.ac 2015-12-21 22:00:49.000000000 +0200
64 +++ rsync-3.1.2/configure.ac 2016-10-24 15:38:28.006415712 +0300
66 setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
67 seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
68 extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
69 + mmap mincore posix_fadvise64 \
70 initgroups utimensat posix_fallocate attropen setvbuf usleep)
72 dnl cygwin iconv.h defines iconv_open as libiconv_open
73 diff -ru rsync-3.1.2.orig/configure.sh rsync-3.1.2/configure.sh
74 --- rsync-3.1.2.orig/configure.sh 2015-12-21 22:20:53.000000000 +0200
75 +++ rsync-3.1.2/configure.sh 2016-10-24 15:38:28.006415712 +0300
77 setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
78 seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
79 extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
80 + mmap mincore posix_fadvise64 \
81 initgroups utimensat posix_fallocate attropen setvbuf usleep
83 as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
84 diff -ru rsync-3.1.2.orig/fileio.c rsync-3.1.2/fileio.c
85 --- rsync-3.1.2.orig/fileio.c 2015-08-08 22:47:03.000000000 +0300
86 +++ rsync-3.1.2/fileio.c 2016-10-24 15:38:28.006415712 +0300
91 - ret = write(f, "", 1);
92 + ret = fadv_write(f, "", 1);
93 } while (ret < 0 && errno == EINTR);
95 ret = ret <= 0 ? -1 : 0;
97 do_lseek(f, sparse_seek, SEEK_CUR);
100 - while ((ret = write(f, buf + l1, len - (l1+l2))) <= 0) {
101 + while ((ret = fadv_write(f, buf + l1, len - (l1+l2))) <= 0) {
102 if (ret < 0 && errno == EINTR)
106 char *bp = wf_writeBuf;
108 while (wf_writeBufCnt > 0) {
109 - if ((ret = write(f, bp, wf_writeBufCnt)) < 0) {
110 + if ((ret = fadv_write(f, bp, wf_writeBufCnt)) < 0) {
115 map->p_len = window_size;
117 while (read_size > 0) {
118 - int32 nread = read(map->fd, map->p + read_offset, read_size);
119 + int32 nread = fadv_read(map->fd, map->p + read_offset, read_size);
122 map->status = nread ? errno : ENODATA;
123 diff -ru rsync-3.1.2.orig/generator.c rsync-3.1.2/generator.c
124 --- rsync-3.1.2.orig/generator.c 2015-12-05 21:10:24.000000000 +0200
125 +++ rsync-3.1.2/generator.c 2016-10-24 15:38:28.006415712 +0300
127 static int need_retouch_dir_perms;
128 static const char *solo_file = NULL;
130 +#ifdef WITH_DROP_CACHE
131 +#define close(fd) fadv_close(fd)
135 TYPE_DIR, TYPE_SPECIAL, TYPE_DEVICE, TYPE_SYMLINK
137 diff -ru rsync-3.1.2.orig/options.c rsync-3.1.2/options.c
138 --- rsync-3.1.2.orig/options.c 2015-12-19 00:46:28.000000000 +0200
139 +++ rsync-3.1.2/options.c 2016-10-24 15:38:28.006415712 +0300
141 int preserve_gid = 0;
142 int preserve_times = 0;
144 +#ifdef WITH_DROP_CACHE
151 rprintf(F," --backup-dir=DIR make backups into hierarchy based in DIR\n");
152 rprintf(F," --suffix=SUFFIX set backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX);
153 rprintf(F," -u, --update skip files that are newer on the receiver\n");
154 +#ifdef WITH_DROP_CACHE
155 + rprintf(F," --drop-cache do not cache rsync files (POSIX_FADV_DONTNEED)\n");
157 rprintf(F," --inplace update destination files in-place (SEE MAN PAGE)\n");
158 rprintf(F," --append append data onto shorter files\n");
159 rprintf(F," --append-verify like --append, but with old data in file checksum\n");
161 {"no-one-file-system",0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 },
162 {"no-x", 0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 },
163 {"update", 'u', POPT_ARG_NONE, &update_only, 0, 0, 0 },
164 +#ifdef WITH_DROP_CACHE
165 + {"drop-cache", 0, POPT_ARG_NONE, &drop_cache, 0, 0, 0 },
167 {"existing", 0, POPT_ARG_NONE, &ignore_non_existing, 0, 0, 0 },
168 {"ignore-non-existing",0,POPT_ARG_NONE, &ignore_non_existing, 0, 0, 0 },
169 {"ignore-existing", 0, POPT_ARG_NONE, &ignore_existing, 0, 0, 0 },
170 @@ -1065,6 +1074,9 @@
171 rprintf(F," --log-file=FILE override the \"log file\" setting\n");
172 rprintf(F," --log-file-format=FMT override the \"log format\" setting\n");
173 rprintf(F," --sockopts=OPTIONS specify custom TCP options\n");
174 +#ifdef WITH_DROP_CACHE
175 + rprintf(F," --drop-cache do not cache rsync files (POSIX_FADV_DONTNEED)\n");
177 rprintf(F," -v, --verbose increase verbosity\n");
178 rprintf(F," -4, --ipv4 prefer IPv4\n");
179 rprintf(F," -6, --ipv6 prefer IPv6\n");
180 @@ -1089,6 +1101,9 @@
181 {"log-file", 0, POPT_ARG_STRING, &logfile_name, 0, 0, 0 },
182 {"log-file-format", 0, POPT_ARG_STRING, &logfile_format, 0, 0, 0 },
183 {"port", 0, POPT_ARG_INT, &rsync_port, 0, 0, 0 },
184 +#ifdef WITH_DROP_CACHE
185 + {"drop-cache", 0, POPT_ARG_NONE, &drop_cache, 0, 0, 0 },
187 {"sockopts", 0, POPT_ARG_STRING, &sockopts, 0, 0, 0 },
188 {"protocol", 0, POPT_ARG_INT, &protocol_version, 0, 0, 0 },
189 {"server", 0, POPT_ARG_NONE, &am_server, 0, 0, 0 },
190 @@ -2376,6 +2391,11 @@
192 args[ac++] = "--sender";
194 +#ifdef WITH_DROP_CACHE
196 + args[ac++] = "--drop-cache";
202 diff -ru rsync-3.1.2.orig/proto.h rsync-3.1.2/proto.h
203 --- rsync-3.1.2.orig/proto.h 2015-12-21 22:22:53.000000000 +0200
204 +++ rsync-3.1.2/proto.h 2016-10-24 15:38:28.010415712 +0300
206 uid_t recv_user_name(int f, uid_t uid);
207 gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr);
208 void recv_id_list(int f, struct file_list *flist);
209 +ssize_t fadv_write(int fd, const void *buf, size_t count);
210 +ssize_t fadv_read(int fd, void *buf, size_t count);
211 +void fadv_close_all(void);
212 +int fadv_close(int fd);
213 void parse_name_map(char *map, BOOL usernames);
214 const char *getallgroups(uid_t uid, item_list *gid_list);
215 void set_nonblocking(int fd);
216 diff -ru rsync-3.1.2.orig/receiver.c rsync-3.1.2/receiver.c
217 --- rsync-3.1.2.orig/receiver.c 2015-09-07 20:07:17.000000000 +0300
218 +++ rsync-3.1.2/receiver.c 2016-10-24 15:38:28.010415712 +0300
220 extern struct file_list *cur_flist, *first_flist, *dir_flist;
221 extern filter_rule_list daemon_filter_list;
223 +#ifdef WITH_DROP_CACHE
224 +#define close(fd) fadv_close(fd)
227 static struct bitbag *delayed_bits = NULL;
228 static int phase = 0, redoing = 0;
229 static flist_ndx_list batch_redo_list;
230 --- rsync-3.1.3/rsync.1~ 2018-12-04 00:26:25.000000000 +0200
231 +++ rsync-3.1.3/rsync.1 2018-12-04 00:27:43.557037661 +0200
233 \-\-super receiver attempts super\-user activities
234 \-\-fake\-super store/recover privileged attrs using xattrs
235 \-S, \-\-sparse turn sequences of nulls into sparse blocks
236 + \-\-drop\-cache drop cache continuosly using fadvise
237 \-\-preallocate allocate dest files before writing
238 \-n, \-\-dry\-run perform a trial run with no changes made
239 \-W, \-\-whole\-file copy files whole (w/o delta\-xfer algorithm)
240 @@ -1426,6 +1427,13 @@
241 up less space on the destination. Conflicts with \fB\-\-inplace\fP because it\(cq\&s
242 not possible to overwrite data in a sparse fashion.
244 +.IP "\fB\-\-drop\-cache\fP"
245 +Stop rsync from filling up the file system cache with the files it copies\&. Without this
246 +option other processes, that had been crunching along happily on your system, will suddenly
247 +become slow as they find their data being outsed from the cache. The \fB\-\-drop\-cache\fP function
248 +uses posix_fadvise64 and mincore todo its work\&. It will only get compiled if configure can find posix_fadvise64 and mincore\&.
249 +Rsync will tries only to drop data from cache that has not been cached before.
251 .IP "\fB\-\-preallocate\fP"
252 This tells the receiver to allocate each destination
253 file to its eventual size before writing data to the file. Rsync will only use
254 diff -ru rsync-3.1.2.orig/rsync.h rsync-3.1.2/rsync.h
255 --- rsync-3.1.2.orig/rsync.h 2015-08-08 22:47:03.000000000 +0300
256 +++ rsync-3.1.2/rsync.h 2016-10-24 15:38:28.010415712 +0300
257 @@ -1291,3 +1291,13 @@
258 #ifdef MAINTAINER_MODE
259 const char *get_panic_action(void);
262 +#if defined HAVE_POSIX_FADVISE64 && defined HAVE_MINCORE && defined HAVE_MMAP
263 +#define WITH_DROP_CACHE 1
264 +#include <sys/mman.h>
265 +int fadv_close(int fd);
266 +void fadv_close_all(void);
269 +ssize_t fadv_write(int fd, const void *buf, size_t count);
270 +ssize_t fadv_read(int fd, void *buf, size_t count);
271 diff -ru rsync-3.1.2.orig/rsync.yo rsync-3.1.2/rsync.yo
272 --- rsync-3.1.2.orig/rsync.yo 2015-12-21 22:00:49.000000000 +0200
273 +++ rsync-3.1.2/rsync.yo 2016-10-24 15:38:28.010415712 +0300
274 @@ -1244,6 +1244,17 @@
275 up less space on the destination. Conflicts with bf(--inplace) because it's
276 not possible to overwrite data in a sparse fashion.
278 +dit(bf(--drop-cache)) Stop rsync from disturbing the file system cache with
279 +the data from the files it copies. Without this option other processes, that
280 +had been crunching along happily using cached data, will suddenly become
281 +slow as they find their favorite data blocks data being evicted from the
282 +cache by the files read and written by rsync. Since rsync has to wait until
283 +the data is written to disk, before it can drop the cache, this option will
284 +slow rsync down considerably, especially with small files and short copy
285 +jobs. The bf(--drop-cache) function uses posix_fadvise64 and mincore todo
286 +its work. It will only get compiled if configure can find posix_fadvise64
289 dit(bf(--preallocate)) This tells the receiver to allocate each destination
290 file to its eventual size before writing data to the file. Rsync will only use
291 the real filesystem-level preallocation support provided by Linux's
292 diff -ru rsync-3.1.2.orig/sender.c rsync-3.1.2/sender.c
293 --- rsync-3.1.2.orig/sender.c 2015-09-07 20:07:17.000000000 +0300
294 +++ rsync-3.1.2/sender.c 2016-10-24 15:38:28.010415712 +0300
296 extern int file_old_total;
297 extern struct stats stats;
298 extern struct file_list *cur_flist, *first_flist, *dir_flist;
299 +#ifdef WITH_DROP_CACHE
300 +#define close(fd) fadv_close(fd)
303 BOOL extra_flist_sending_enabled;
305 diff -ru rsync-3.1.2.orig/t_unsafe.c rsync-3.1.2/t_unsafe.c
306 --- rsync-3.1.2.orig/t_unsafe.c 2015-08-08 22:47:03.000000000 +0300
307 +++ rsync-3.1.2/t_unsafe.c 2016-10-24 15:38:28.010415712 +0300
316 diff -ru rsync-3.1.2.orig/util.c rsync-3.1.2/util.c
317 --- rsync-3.1.2.orig/util.c 2015-12-21 20:54:02.000000000 +0200
318 +++ rsync-3.1.2/util.c 2016-10-24 15:38:28.014415712 +0300
320 extern unsigned int module_dirlen;
321 extern char *partial_dir;
322 extern filter_rule_list daemon_filter_list;
323 +#ifdef WITH_DROP_CACHE
324 +#include <sys/mman.h>
325 +extern int drop_cache;
328 int sanitize_paths = 0;
331 unsigned int curr_dir_len;
332 int curr_dir_depth; /* This is only set for a sanitizing daemon. */
334 +#ifdef WITH_DROP_CACHE
335 +#define FADV_BUFFER_SIZE 1024*1024*16
337 +static struct stat fadv_fd_stat[1024];
338 +static off_t fadv_fd_pos[1024];
339 +static unsigned char *fadv_core_ptr[1024];
340 +static int fadv_max_fd = 0;
341 +static int fadv_close_ring_tail = 0;
342 +static int fadv_close_ring_head = 0;
343 +static int fadv_close_ring_size = 0;
344 +static int fadv_close_ring[1024];
345 +static int fadv_close_buffer_size = 0;
346 +static size_t fadv_pagesize;
348 +static void fadv_fd_init_func(void)
350 + static int fadv_fd_init = 0;
351 + if (fadv_fd_init == 0){
354 + fadv_pagesize = getpagesize();
355 + if (fadv_max_fd == 0){
356 + fadv_max_fd = sysconf(_SC_OPEN_MAX) - 20;
357 + if (fadv_max_fd < 0)
359 + if (fadv_max_fd > 1000)
360 + fadv_max_fd = 1000;
362 + for (i=0;i<fadv_max_fd;i++){
363 + fadv_fd_pos[i] = 0;
364 + fadv_fd_stat[i].st_dev = 0;
365 + fadv_fd_stat[i].st_ino = 0;
366 + fadv_fd_stat[i].st_size = 0;
367 + fadv_core_ptr[i] = NULL;
372 +static void fadv_get_core(int fd)
378 + if ( fadv_fd_stat[fd].st_dev == stat.st_dev && fadv_fd_stat[fd].st_ino == stat.st_ino ) {
381 + fadv_fd_stat[fd].st_dev = stat.st_dev;
382 + fadv_fd_stat[fd].st_ino = stat.st_ino;
383 + fadv_fd_stat[fd].st_size = stat.st_size;
385 + if (fadv_core_ptr[fd]!=NULL){
386 + free (fadv_core_ptr[fd]);
389 + pa = mmap((void *)0, stat.st_size, PROT_READ, MAP_SHARED, fd, 0);
390 + if (MAP_FAILED == pa) {
393 + fadv_core_ptr[fd] = calloc(1, (stat.st_size+fadv_pagesize)/fadv_pagesize);
394 + if ( fadv_core_ptr[fd] == NULL ){
397 + if ( mincore(pa, stat.st_size, (fadv_core_ptr[fd])) != 0){
399 + free(fadv_core_ptr[fd]);
400 + fadv_core_ptr[fd]=(unsigned char*)0;
401 + } else if (DEBUG_GTE(IO, 4)) {
403 + rprintf(FINFO, "fadv_get_core(fd=%d): ", fd);
404 + for (pi = 0; pi <= stat.st_size/fadv_pagesize; pi++) {
405 + if ((fadv_core_ptr[fd])[pi]&1) {
406 + rprintf(FINFO,"%lu ", (unsigned long)pi);
409 + rprintf(FINFO,"\n");
411 + munmap(pa, stat.st_size);
416 +static void fadv_drop(int fd, int sync)
418 + /* trail 1 MB behind in dropping. we do this to make
419 + sure that the same block or stripe does not have
420 + to be written twice */
421 + off_t pos = lseek(fd,0,SEEK_CUR) - 1024*1024;
422 + if (fd > fadv_max_fd){
425 + if ( fadv_fd_pos[fd] < pos - FADV_BUFFER_SIZE ) {
427 + /* if the file is not flushed to disk before calling fadvise,
428 + then the Cache will not be freed and the advise gets ignored
429 + this does give a severe hit on performance. If only there
430 + was a way to mark cache so that it gets release once the data
431 + is written to disk. */
434 + if (fadv_core_ptr[fd] != NULL) {
436 + if (pos < fadv_fd_stat[fd].st_size){
437 + for (pi = fadv_fd_pos[fd]/fadv_pagesize; pi <= pos/fadv_pagesize; pi++) {
438 + if (! (fadv_core_ptr[fd][pi]&1)) {
439 + posix_fadvise64(fd, pi*fadv_pagesize, fadv_pagesize, POSIX_FADV_DONTNEED);
443 + posix_fadvise64(fd, fadv_fd_stat[fd].st_size, pos-fadv_fd_stat[fd].st_size, POSIX_FADV_DONTNEED);
446 + posix_fadvise64(fd, 0, pos, POSIX_FADV_DONTNEED);
448 + fadv_fd_pos[fd] = pos;
454 +ssize_t fadv_write(int fd, const void *buf, size_t count)
456 + int ret = write(fd, buf, count);
457 +#ifdef WITH_DROP_CACHE
465 +ssize_t fadv_read(int fd, void *buf, size_t count)
468 +#ifdef WITH_DROP_CACHE
470 + fadv_fd_init_func();
474 + ret = read(fd, buf, count);
475 +#ifdef WITH_DROP_CACHE
483 +#ifdef WITH_DROP_CACHE
484 +void fadv_close_all(void)
486 + /* printf ("%i\n",fadv_close_ring_size); */
487 + while (fadv_close_ring_size > 0){
488 + fdatasync(fadv_close_ring[fadv_close_ring_tail]);
489 + if (fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]]){
491 + for (pi = 0; pi <= fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_size/fadv_pagesize; pi++) {
492 + if (!(fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]][pi]&1)) {
493 + posix_fadvise64(fadv_close_ring[fadv_close_ring_tail], pi*fadv_pagesize, fadv_pagesize, POSIX_FADV_DONTNEED);
496 + /* if the file has grown, drop the rest */
497 + //posix_fadvise64(fadv_close_ring[fadv_close_ring_tail], fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_size,0, POSIX_FADV_DONTNEED);
499 + free(fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]]);
500 + fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]] = NULL;
501 + fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_size = 0;
502 + fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_ino = 0;
503 + fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_dev = 0;
505 + posix_fadvise64(fadv_close_ring[fadv_close_ring_tail], 0, 0,POSIX_FADV_DONTNEED);
507 + fadv_close_ring_size--;
508 + close(fadv_close_ring[fadv_close_ring_tail]);
509 + fadv_close_ring_tail = (fadv_close_ring_tail + 1) % fadv_max_fd;
510 + fadv_close_buffer_size = 0;
514 +int fadv_close(int fd)
517 + /* if the file is not flushed to disk before calling fadvise,
518 + then the Cache will not be freed and the advise gets ignored
519 + this does give a severe hit on performance. So instead of doing
520 + it right away, we save us a copy of the filehandle and do it
521 + some time before we are out of filehandles. This speeds
522 + up operation for small files massively. It is directly
523 + related to the number of spare file handles you have. */
524 + int newfd = dup(fd);
525 + off_t pos = lseek(fd,0,SEEK_CUR);
526 + fadv_fd_init_func();
527 + fadv_core_ptr[newfd] = fadv_core_ptr[fd];
528 + fadv_fd_stat[newfd].st_size = fadv_fd_stat[fd].st_size ;
529 + fadv_core_ptr[fd] = NULL;
530 + fadv_close_buffer_size += pos - fadv_fd_pos[fd];
531 + fadv_close_ring[fadv_close_ring_head] = newfd;
532 + fadv_close_ring_head = (fadv_close_ring_head + 1) % fadv_max_fd;
533 + fadv_close_ring_size ++;
534 + if (fadv_close_ring_size == fadv_max_fd || fadv_close_buffer_size > 1024*1024 ){
535 + /* it seems fastest to drop things 'in groups' */
543 +#define close(fd) fadv_close(fd)
546 /* Set a fd into nonblocking mode. */
547 void set_nonblocking(int fd)
553 - int written = write(desc, ptr, len);
554 + int written = fadv_write(desc, ptr, len);
562 - n_chars = read(desc, ptr, len);
563 + n_chars = fadv_read(desc, ptr, len);
564 } while (n_chars < 0 && errno == EINTR);