1 From: Matthew Grosso < mgrosso at looksmart dot net>
2 To: cronolog-users@icarus.demon.co.uk
3 Subject: CRONOLOG: jumbo patch, includes launching external program to handle logs
4 Date: Fri, 10 Oct 2003 13:02:14 -0400
6 I've put together a jumbo patch with some features that are on the todo
7 list and most of the outstanding patches on the site. specifically:
9 the todo list items completed (which motivated the work for my companies use) are:
11 - ability to launch a helper program on every log file after it is closed
12 if multiple cronologs writing to the same file, only one helper is
15 - signal handling to ensure that every log file has a chance to be handled
16 by that helper and that cronolog doesnt ever hang apache restarts
18 - alarm() used so log files rotate on time period boundary even if there is no
21 I also implemented (with tweaks) many of the outstanding patches, some of
22 which are also on the todo list:
24 - uid/gid patch from Isaac Wilcox <iwilcox at eatstatic dot net>
25 minus the configure changes which broke the build for me.
27 - the large file patch from matt lanier <mlanier at danger dot .com>
28 with tweaks from me to work with my own changes above.
30 - incorporated a slightly different implementation of
31 the SIGUSR1 patch from Prakash Kailasa <PKailasa at seisint dot com>
33 - the s/stat/lstat/ fix from Victor Martinez <victor at kablinkteam dot com>
35 the changes have only been tested on my Linux 2.4 box, and should be considered
36 beta until more users have tried it. That said, we'll be starting to qa this at
37 looksmart soon, but again, only on linux.
39 I'm including a sample script that can be used as a helper. It will first
40 compress, then scp a log to multiple destinations, then remove it.
42 Still todo: I need to put my new options into the man page, not just the usage,
43 and see if the configure.in needs modification.
45 the patch is available below, and at this url:
46 http://www.falconweb.com/~mattg/code/cronolog-1.6.2-jumbo-20031010.diff
48 diff -Nur cronolog-1.6.2/src/cronolog.c cronolog-1.6.2-jumbo-20031008/src/cronolog.c
49 --- cronolog-1.6.2/src/cronolog.c 2001-05-03 12:42:48.000000000 -0400
50 +++ cronolog-1.6.2-jumbo-20031008/src/cronolog.c 2003-10-08 00:22:10.000000000 -0400
52 * written to "file" (e.g. /dev/console) or to stderr if "file" is "-".
56 +#define _GNU_SOURCE 1
57 +#define OPEN_EXCLUSIVE O_WRONLY|O_CREAT|O_EXCL|O_APPEND|O_LARGEFILE
58 +#define OPEN_SHARED O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE
60 +#define OPEN_EXCLUSIVE O_WRONLY|O_CREAT|O_EXCL|O_APPEND
61 +#define OPEN_SHARED O_WRONLY|O_CREAT|O_APPEND
64 +#include <sys/types.h>
65 +#include <sys/wait.h>
67 #include "cronoutils.h"
71 int new_log_file(const char *, const char *, mode_t, const char *,
72 PERIODICITY, int, int, char *, size_t, time_t, time_t *);
76 +void fork_to_handle_file();
78 +int openwrapper( const char *filename );
81 +void setsig_handler( int signum, void (*action)(int, siginfo_t *, void *));
82 +void set_signal_handlers();
85 /* Definition of version and usage messages */
88 #define VERSION_MSG "cronolog version 0.1\n"
92 +#define SETUGID_USAGE " -u USER, --set-uid=USER change to USER before doing anything (name or UID)\n" \
93 + " -g GROUP, --set-gid=GROUP change to GROUP before doing anything (name or GID)\n"
95 +#define SETUGID_USAGE ""
98 #define USAGE_MSG "usage: %s [OPTIONS] logfile-spec\n" \
101 " -o, --once-only create single output log from template (not rotated)\n" \
102 " -x FILE, --debug=FILE write debug messages to FILE\n" \
103 " ( or to standard error if FILE is \"-\")\n" \
104 + " -r, --helper=SCRIPT post rotation helper script to fork exec on old files\n" \
105 + " ( will be called like \"SCRIPT <oldlog>\" )\n" \
106 + " ( not tested on windows )\n" \
107 + " -G, --helper-arg=ARG argument passed to rotation helper script\n" \
109 " -a, --american American date formats\n" \
110 " -e, --european European date formats (default)\n" \
111 " -s, --start-time=TIME starting time\n" \
114 /* Definition of the short and long program options */
116 -char *short_options = "ad:eop:s:z:H:P:S:l:hVx:";
117 +char *short_options = "ad:eop:s:z:H:P:S:l:hVx:r:G:u:g:";
120 struct option long_options[] =
121 @@ -137,12 +170,23 @@
122 { "link", required_argument, NULL, 'l' },
123 { "period", required_argument, NULL, 'p' },
124 { "delay", required_argument, NULL, 'd' },
125 + { "helper", required_argument, NULL, 'r' },
126 + { "helper-arg", required_argument, NULL, 'G' },
127 + { "set-uid", required_argument, NULL, 'u' },
128 + { "set-gid", required_argument, NULL, 'g' },
129 { "once-only", no_argument, NULL, 'o' },
130 { "help", no_argument, NULL, 'h' },
131 { "version", no_argument, NULL, 'V' }
135 +static char handler[MAX_PATH];
136 +static char handler_arg[MAX_PATH];
137 +static char filename[MAX_PATH];
138 +static int use_handler =0;
139 +static int use_handler_arg =0;
140 +static int i_am_handler =0;
145 @@ -155,11 +199,16 @@
146 int use_american_date_formats = 0;
147 char read_buf[BUFSIZE];
149 - char filename[MAX_PATH];
150 char *start_time = NULL;
152 char *linkname = NULL;
153 char *prevlinkname = NULL;
157 + int change_uid = 0;
158 + int change_gid = 0;
164 time_t time_offset = 0;
165 time_t next_period = 0;
168 + memset( handler, '\0', MAX_PATH );
169 + memset( handler_arg, '\0', MAX_PATH );
170 + memset( filename, '\0', MAX_PATH );
173 while ((ch = getopt_long(argc, argv, short_options, long_options, NULL)) != EOF)
180 + new_uid = parse_uid(optarg, argv[0]);
184 + new_gid = parse_gid(optarg, argv[0]);
189 periodicity = ONCE_ONLY;
192 debug_file = fopen(optarg, "a+");
197 + strncat(handler, optarg, MAX_PATH );
201 + strncat(handler_arg, optarg, MAX_PATH );
206 fprintf(stderr, VERSION_MSG);
213 + if (change_gid && setgid(new_gid) == -1) {
214 + fprintf(stderr, "setgid: unable to change to gid: %d\n", new_gid);
217 + if (change_uid && setuid(new_uid) == -1) {
218 + fprintf(stderr, "setuid: unable to change to uid: %d\n", new_uid);
223 DEBUG((VERSION_MSG "\n"));
227 DEBUG(("Rotation period is per %d %s\n", period_multiple, periods[periodicity]));
231 + set_signal_handlers();
234 /* Loop, waiting for data on standard input */
237 @@ -316,15 +402,17 @@
238 n_bytes_read = read(0, read_buf, sizeof read_buf);
239 if (n_bytes_read == 0)
248 + * fall through, it may have been alarm, in which case it will be time to rotate.
251 else if (n_bytes_read < 0)
257 time_now = time(NULL) + time_offset;
262 + fork_to_handle_file();
265 /* If there is no log file open then open a new one.
267 log_fd = new_log_file(template, linkname, linktype, prevlinkname,
268 periodicity, period_multiple, period_delay,
269 filename, sizeof (filename), time_now, &next_period);
270 + alarm( next_period - time_now );
273 DEBUG(("%s (%d): wrote message; next period starts at %s (%d) in %d secs\n",
274 @@ -354,10 +444,10 @@
276 /* Write out the log data to the current log file.
278 - if (write(log_fd, read_buf, n_bytes_read) != n_bytes_read)
279 + if (n_bytes_read && write(log_fd, read_buf, n_bytes_read) != n_bytes_read)
292 start_of_period = start_of_this_period(time_now, periodicity, period_multiple);
293 tm = localtime(&start_of_period);
294 strftime(pfilename, BUFSIZE, template, tm);
295 @@ -394,13 +485,13 @@
296 timestamp(*pnext_period), *pnext_period,
297 *pnext_period - time_now));
299 - log_fd = open(pfilename, O_WRONLY|O_CREAT|O_APPEND, FILE_MODE);
300 + log_fd = openwrapper(pfilename);
302 #ifndef DONT_CREATE_SUBDIRS
303 if ((log_fd < 0) && (errno == ENOENT))
305 create_subdirs(pfilename);
306 - log_fd = open(pfilename, O_WRONLY|O_CREAT|O_APPEND, FILE_MODE);
307 + log_fd = openwrapper(pfilename);
311 @@ -416,3 +507,179 @@
317 + * fork, then exec an external handler to deal with rotated file.
320 +fork_to_handle_file()
323 + static int childpid=0;
325 + if( ! use_handler || !i_am_handler || handler[0] =='\0' || filename[0] == '\0' )
332 + perror("couldnt fork");
339 + * collect zombies. run twice, in case one or more children took longer than
340 + * the rotation period for a while, this will eventually clean them up.
341 + * Of course, if handler children take longer than rotation period to handle
342 + * things, you will eventually have a big problem.
345 + (void) waitpid( 0, NULL, WNOHANG | WUNTRACED );
346 + (void) waitpid( 0, NULL, WNOHANG | WUNTRACED );
349 + return; /* parent */
352 + /* dont muck with stdin or out of parent, but allow stderr to be commingled */
359 + * exec an external handler to deal with rotated file.
365 + if( ! use_handler || !i_am_handler || handler[0] =='\0' || filename[0] == '\0' )
369 + if ( use_handler_arg == 0 )
371 + exec_argv = malloc( sizeof( char *)*3);
372 + exec_argv[0] = strdup( handler );
373 + exec_argv[1] = strdup( filename );
374 + exec_argv[2] = NULL;
377 + exec_argv = malloc( sizeof( char *)*4);
378 + exec_argv[0] = strdup( handler );
379 + exec_argv[1] = strdup( handler_arg );
380 + exec_argv[2] = strdup( filename );
381 + exec_argv[3] = NULL ;
383 + execvp( exec_argv[0], exec_argv );
384 + perror("cant execvp");
392 + * wrapper to be called as signal handler.
395 +handle_file_on_sig( int sig, siginfo_t *si, void *v )
403 + * wrapper to be called for alarm signal
406 +alarm_signal_handler( int sig, siginfo_t *si, void *v )
410 + * do nothing; the key thing is that the alarm will cause the read()
411 + * to fail with errno=EINTR. this empty handler is required, because the
412 + * default handler will exit(1)
418 +set_signal_handlers()
421 + * all signals which usually kill a process that can be caught are
422 + * set to handle_file when received. This will make apache shutdowns more
423 + * graceful even if use_handler is false.
425 + setsig_handler( SIGHUP, handle_file_on_sig );
426 + setsig_handler( SIGINT, handle_file_on_sig );
427 + setsig_handler( SIGQUIT, handle_file_on_sig );
428 + setsig_handler( SIGILL, handle_file_on_sig );
429 + setsig_handler( SIGABRT, handle_file_on_sig );
430 + setsig_handler( SIGBUS, handle_file_on_sig );
431 + setsig_handler( SIGFPE, handle_file_on_sig );
432 + setsig_handler( SIGPIPE, handle_file_on_sig );
433 + setsig_handler( SIGTERM, handle_file_on_sig );
434 + setsig_handler( SIGUSR1, handle_file_on_sig );
436 + /* sigalrm is used to break out of read() when it is time to rotate the log. */
437 + setsig_handler( SIGALRM, alarm_signal_handler );
441 +setsig_handler( int signum, void (*action)(int, siginfo_t *, void *))
443 + struct sigaction siga ;
444 + memset( &siga, '\0', sizeof( struct sigaction ));
445 + siga.sa_sigaction= action ;
446 + siga.sa_flags = SA_SIGINFO ;
447 + if( -1== sigaction( signum, &siga, NULL ))
449 + perror( "cant set sigaction" );
459 +cleanup( int exit_status )
466 + * only the first cronolog process to open a particular file is responsible
467 + * for starting the cleanup process later. This wrapper sets i_am_handler
468 + * according to that logic.
471 +openwrapper( const char *ofilename )
474 + if( use_handler !=1 )
476 + return open(ofilename, OPEN_SHARED, S_IRWXU );
478 + ret = open(ofilename, OPEN_EXCLUSIVE, S_IRWXU );
481 + ret = open(ofilename, OPEN_SHARED, S_IRWXU );
491 diff -Nur cronolog-1.6.2/src/cronoutils.c cronolog-1.6.2-jumbo-20031008/src/cronoutils.c
492 --- cronolog-1.6.2/src/cronoutils.c 2001-05-03 12:43:21.000000000 -0400
493 +++ cronolog-1.6.2-jumbo-20031008/src/cronoutils.c 2003-10-08 00:22:10.000000000 -0400
494 @@ -195,11 +195,11 @@
496 struct stat stat_buf;
498 - if (stat(prevlinkname, &stat_buf) == 0)
499 + if (lstat(prevlinkname, &stat_buf) == 0)
501 unlink(prevlinkname);
503 - if (stat(linkname, &stat_buf) == 0)
504 + if (lstat(linkname, &stat_buf) == 0)
507 rename(linkname, prevlinkname);
515 +/* Turn a string specifying either a username or UID into an actual
516 + * uid_t for use in setuid(). A string is assumed to be a UID if
517 + * it contains only decimal digits. */
519 +parse_uid(char *user, char *argv0)
521 + char *probe = user;
522 + struct passwd *ent;
524 + while (*probe && isdigit(*probe)) {
530 + if (!(ent = getpwnam(user))) {
531 + fprintf(stderr, "%s: Bad username %s\n", argv0, user);
534 + return (ent->pw_uid);
538 +/* Turn a string specifying either a group name or GID into an actual
539 + * gid_t for use in setgid(). A string is assumed to be a GID if
540 + * it contains only decimal digits. */
542 +parse_gid(char *group, char *argv0)
544 + char *probe = group;
547 + while (*probe && isdigit(*probe)) {
551 + return atoi(group);
553 + if (!(ent = getgrnam(group))) {
554 + fprintf(stderr, "%s: Bad group name %s\n", argv0, group);
557 + return (ent->gr_gid);
560 diff -Nur cronolog-1.6.2/src/cronoutils.h cronolog-1.6.2-jumbo-20031008/src/cronoutils.h
561 --- cronolog-1.6.2/src/cronoutils.h 2001-05-03 12:40:12.000000000 -0400
562 +++ cronolog-1.6.2-jumbo-20031008/src/cronoutils.h 2003-10-08 00:22:10.000000000 -0400
573 void print_debug_msg(char *msg, ...);
574 time_t parse_time(char *time_str, int);
575 char *timestamp(time_t thetime);
577 +uid_t parse_uid(char *user, char *argv0);
578 +gid_t parse_gid(char *group, char *argv0);
580 /* Global variables */
582 diff -Nur cronolog-1.6.2/src/zip_send_rm.sh cronolog-1.6.2-jumbo-20031008/src/zip_send_rm.sh
583 --- cronolog-1.6.2/src/zip_send_rm.sh 1969-12-31 19:00:00.000000000 -0500
584 +++ cronolog-1.6.2-jumbo-20031008/src/zip_send_rm.sh 2003-10-10 12:37:22.000000000 -0400
587 +## ----------------------------------------------------------------------
588 +## ----------------------------------------------------------------------
590 +## File: zip_send_rm
592 +## Created: Fri Oct 3 18:18:38 EDT 2003 on dhcp-172-18-102-101.looksmart.com
593 +## Project: apache, cronolog
594 +## Purpose: make damn sure the log file arrives at its destinations
596 +## Copyright (c) 2003 LookSmart. All Rights Reserved.
599 +## ----------------------------------------------------------------------
600 +## ----------------------------------------------------------------------
603 +function do_or_die()
606 + if [ $? -ne 0 ] ; then
607 + logger -s -p local0.crit "$0 puking on [ $@ ]"
612 +if [ $# == 2 ] ; then
620 +do_or_die gzip $FILE
622 +if [ -z $SCPDESTLIST ] ; then
628 +# spawn subshells, so that each scp can suceed or fail on its own.
629 +# while this is less efficient from a bandwidth and cpu perspective, it
630 +# is safer because it ensures that a hanging dest cant prevent the other
631 +# dests from working.
633 +SCPDESTINATIONS=$( echo $SCPDESTLIST | sed -e 's/,/ /g' )
634 +for DEST in $SCPDESTINATIONS ; do
636 + HDEST=$( echo $DEST | cut -d ':' -f 1 )
637 + BACKUP_LINK=${HDEST}-$FILE
638 + do_or_die ln $FILE $BACKUP_LINK
639 + do_or_die scp $FILE $DEST
640 + do_or_die rm $BACKUP_LINK
644 +# we dont delete the file unless it arrive at all destinations
645 +# because if one destination failed, then ${DEST}-$FILE will still exist
646 +# even after we rm $FILE
656 #######################################################################
657 # Matt Grosso cell (201)780-9592 mgrosso@looksmart.net #
658 # work ny (212)324-1904 fax ny (212)324-1910 work sf (415)348-7756 #
659 # pgp public key is at http://www.falconweb.com/~mattg/publickey.pgp #
660 #######################################################################