From: Matthew Grosso < mgrosso at looksmart dot net> To: cronolog-users@icarus.demon.co.uk Subject: CRONOLOG: jumbo patch, includes launching external program to handle logs Date: Fri, 10 Oct 2003 13:02:14 -0400 I've put together a jumbo patch with some features that are on the todo list and most of the outstanding patches on the site. specifically: the todo list items completed (which motivated the work for my companies use) are: - ability to launch a helper program on every log file after it is closed if multiple cronologs writing to the same file, only one helper is launched. - signal handling to ensure that every log file has a chance to be handled by that helper and that cronolog doesnt ever hang apache restarts - alarm() used so log files rotate on time period boundary even if there is no traffic I also implemented (with tweaks) many of the outstanding patches, some of which are also on the todo list: - uid/gid patch from Isaac Wilcox minus the configure changes which broke the build for me. - the large file patch from matt lanier with tweaks from me to work with my own changes above. - incorporated a slightly different implementation of the SIGUSR1 patch from Prakash Kailasa - the s/stat/lstat/ fix from Victor Martinez the changes have only been tested on my Linux 2.4 box, and should be considered beta until more users have tried it. That said, we'll be starting to qa this at looksmart soon, but again, only on linux. I'm including a sample script that can be used as a helper. It will first compress, then scp a log to multiple destinations, then remove it. Still todo: I need to put my new options into the man page, not just the usage, and see if the configure.in needs modification. the patch is available below, and at this url: http://www.falconweb.com/~mattg/code/cronolog-1.6.2-jumbo-20031010.diff diff -Nur cronolog-1.6.2/src/cronolog.c cronolog-1.6.2-jumbo-20031008/src/cronolog.c --- cronolog-1.6.2/src/cronolog.c 2001-05-03 12:42:48.000000000 -0400 +++ cronolog-1.6.2-jumbo-20031008/src/cronolog.c 2003-10-08 00:22:10.000000000 -0400 @@ -82,6 +82,18 @@ * written to "file" (e.g. /dev/console) or to stderr if "file" is "-". */ +#ifndef _WIN32 +#define _GNU_SOURCE 1 +#define OPEN_EXCLUSIVE O_WRONLY|O_CREAT|O_EXCL|O_APPEND|O_LARGEFILE +#define OPEN_SHARED O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE +#else +#define OPEN_EXCLUSIVE O_WRONLY|O_CREAT|O_EXCL|O_APPEND +#define OPEN_SHARED O_WRONLY|O_CREAT|O_APPEND +#endif + +#include +#include +#include #include "cronoutils.h" #include "getopt.h" @@ -91,6 +103,16 @@ int new_log_file(const char *, const char *, mode_t, const char *, PERIODICITY, int, int, char *, size_t, time_t, time_t *); +void cleanup(int ); +void handle_file(); +void fork_to_handle_file(); + +int openwrapper( const char *filename ); + +#ifndef _WIN32 +void setsig_handler( int signum, void (*action)(int, siginfo_t *, void *)); +void set_signal_handlers(); +#endif /* Definition of version and usage messages */ @@ -100,6 +122,12 @@ #define VERSION_MSG "cronolog version 0.1\n" #endif +#ifndef _WIN32 +#define SETUGID_USAGE " -u USER, --set-uid=USER change to USER before doing anything (name or UID)\n" \ + " -g GROUP, --set-gid=GROUP change to GROUP before doing anything (name or GID)\n" +#else +#define SETUGID_USAGE "" +#endif #define USAGE_MSG "usage: %s [OPTIONS] logfile-spec\n" \ "\n" \ @@ -113,6 +141,11 @@ " -o, --once-only create single output log from template (not rotated)\n" \ " -x FILE, --debug=FILE write debug messages to FILE\n" \ " ( or to standard error if FILE is \"-\")\n" \ + " -r, --helper=SCRIPT post rotation helper script to fork exec on old files\n" \ + " ( will be called like \"SCRIPT \" )\n" \ + " ( not tested on windows )\n" \ + " -G, --helper-arg=ARG argument passed to rotation helper script\n" \ + SETUGID_USAGE \ " -a, --american American date formats\n" \ " -e, --european European date formats (default)\n" \ " -s, --start-time=TIME starting time\n" \ @@ -122,7 +155,7 @@ /* Definition of the short and long program options */ -char *short_options = "ad:eop:s:z:H:P:S:l:hVx:"; +char *short_options = "ad:eop:s:z:H:P:S:l:hVx:r:G:u:g:"; #ifndef _WIN32 struct option long_options[] = @@ -137,12 +170,23 @@ { "link", required_argument, NULL, 'l' }, { "period", required_argument, NULL, 'p' }, { "delay", required_argument, NULL, 'd' }, + { "helper", required_argument, NULL, 'r' }, + { "helper-arg", required_argument, NULL, 'G' }, + { "set-uid", required_argument, NULL, 'u' }, + { "set-gid", required_argument, NULL, 'g' }, { "once-only", no_argument, NULL, 'o' }, { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, 'V' } }; #endif +static char handler[MAX_PATH]; +static char handler_arg[MAX_PATH]; +static char filename[MAX_PATH]; +static int use_handler =0; +static int use_handler_arg =0; +static int i_am_handler =0; + /* Main function. */ int @@ -155,11 +199,16 @@ int use_american_date_formats = 0; char read_buf[BUFSIZE]; char tzbuf[BUFSIZE]; - char filename[MAX_PATH]; char *start_time = NULL; char *template; char *linkname = NULL; char *prevlinkname = NULL; +#ifndef _WIN32 + uid_t new_uid = 0; + gid_t new_gid = 0; + int change_uid = 0; + int change_gid = 0; +#endif mode_t linktype = 0; int n_bytes_read; int ch; @@ -167,6 +216,10 @@ time_t time_offset = 0; time_t next_period = 0; int log_fd = -1; + + memset( handler, '\0', MAX_PATH ); + memset( handler_arg, '\0', MAX_PATH ); + memset( filename, '\0', MAX_PATH ); #ifndef _WIN32 while ((ch = getopt_long(argc, argv, short_options, long_options, NULL)) != EOF) @@ -234,6 +287,16 @@ } break; +#ifndef _WIN32 + case 'u': + new_uid = parse_uid(optarg, argv[0]); + change_uid = 1; + break; + case 'g': + new_gid = parse_gid(optarg, argv[0]); + change_gid = 1; + break; +#endif case 'o': periodicity = ONCE_ONLY; break; @@ -248,7 +311,15 @@ debug_file = fopen(optarg, "a+"); } break; - + case 'r': + strncat(handler, optarg, MAX_PATH ); + use_handler=1; + break; + case 'G': + strncat(handler_arg, optarg, MAX_PATH ); + use_handler_arg=1; + break; + case 'V': fprintf(stderr, VERSION_MSG); exit(0); @@ -266,6 +337,17 @@ exit(1); } +#ifndef _WIN32 + if (change_gid && setgid(new_gid) == -1) { + fprintf(stderr, "setgid: unable to change to gid: %d\n", new_gid); + exit(1); + } + if (change_uid && setuid(new_uid) == -1) { + fprintf(stderr, "setuid: unable to change to uid: %d\n", new_uid); + exit(1); + } +#endif + DEBUG((VERSION_MSG "\n")); if (start_time) @@ -306,6 +388,10 @@ DEBUG(("Rotation period is per %d %s\n", period_multiple, periods[periodicity])); +#ifndef _WIN32 + set_signal_handlers(); +#endif + /* Loop, waiting for data on standard input */ for (;;) @@ -316,15 +402,17 @@ n_bytes_read = read(0, read_buf, sizeof read_buf); if (n_bytes_read == 0) { - exit(3); + cleanup(3); } if (errno == EINTR) { - continue; + /* + * fall through, it may have been alarm, in which case it will be time to rotate. + * */ } else if (n_bytes_read < 0) { - exit(4); + cleanup(4); } time_now = time(NULL) + time_offset; @@ -336,6 +424,7 @@ { close(log_fd); log_fd = -1; + fork_to_handle_file(); } /* If there is no log file open then open a new one. @@ -345,6 +434,7 @@ log_fd = new_log_file(template, linkname, linktype, prevlinkname, periodicity, period_multiple, period_delay, filename, sizeof (filename), time_now, &next_period); + alarm( next_period - time_now ); } DEBUG(("%s (%d): wrote message; next period starts at %s (%d) in %d secs\n", @@ -354,10 +444,10 @@ /* Write out the log data to the current log file. */ - if (write(log_fd, read_buf, n_bytes_read) != n_bytes_read) + if (n_bytes_read && write(log_fd, read_buf, n_bytes_read) != n_bytes_read) { perror(filename); - exit(5); + cleanup(5); } } @@ -383,6 +473,7 @@ struct tm *tm; int log_fd; + start_of_period = start_of_this_period(time_now, periodicity, period_multiple); tm = localtime(&start_of_period); strftime(pfilename, BUFSIZE, template, tm); @@ -394,13 +485,13 @@ timestamp(*pnext_period), *pnext_period, *pnext_period - time_now)); - log_fd = open(pfilename, O_WRONLY|O_CREAT|O_APPEND, FILE_MODE); + log_fd = openwrapper(pfilename); #ifndef DONT_CREATE_SUBDIRS if ((log_fd < 0) && (errno == ENOENT)) { create_subdirs(pfilename); - log_fd = open(pfilename, O_WRONLY|O_CREAT|O_APPEND, FILE_MODE); + log_fd = openwrapper(pfilename); } #endif @@ -416,3 +507,179 @@ } return log_fd; } + +/* + * fork, then exec an external handler to deal with rotated file. + */ +void +fork_to_handle_file() +{ + int fk ; + static int childpid=0; + + if( ! use_handler || !i_am_handler || handler[0] =='\0' || filename[0] == '\0' ) + { + return; + } + fk=fork(); + if( fk < 0 ) + { + perror("couldnt fork"); + exit(2); + }else if( fk > 0 ) + { + if( childpid ) + { + /* + * collect zombies. run twice, in case one or more children took longer than + * the rotation period for a while, this will eventually clean them up. + * Of course, if handler children take longer than rotation period to handle + * things, you will eventually have a big problem. + * + * */ + (void) waitpid( 0, NULL, WNOHANG | WUNTRACED ); + (void) waitpid( 0, NULL, WNOHANG | WUNTRACED ); + } + childpid=fk; + return; /* parent */ + } + /* child */ + /* dont muck with stdin or out of parent, but allow stderr to be commingled */ + close(0); + close(1); + handle_file(); +} + +/* + * exec an external handler to deal with rotated file. + */ +void +handle_file() +{ + char **exec_argv ; + if( ! use_handler || !i_am_handler || handler[0] =='\0' || filename[0] == '\0' ) + { + return; + } + if ( use_handler_arg == 0 ) + { + exec_argv = malloc( sizeof( char *)*3); + exec_argv[0] = strdup( handler ); + exec_argv[1] = strdup( filename ); + exec_argv[2] = NULL; + }else + { + exec_argv = malloc( sizeof( char *)*4); + exec_argv[0] = strdup( handler ); + exec_argv[1] = strdup( handler_arg ); + exec_argv[2] = strdup( filename ); + exec_argv[3] = NULL ; + } + execvp( exec_argv[0], exec_argv ); + perror("cant execvp"); + exit(2); +} + + + +#ifndef _WIN32 +/* + * wrapper to be called as signal handler. + */ +void +handle_file_on_sig( int sig, siginfo_t *si, void *v ) +{ + handle_file(); + /* not reached */ + exit( 3 ); +}; + +/* + * wrapper to be called for alarm signal + */ +void +alarm_signal_handler( int sig, siginfo_t *si, void *v ) +{ + ; + /* + * do nothing; the key thing is that the alarm will cause the read() + * to fail with errno=EINTR. this empty handler is required, because the + * default handler will exit(1) + * + */ +}; + +void +set_signal_handlers() +{ + /* + * all signals which usually kill a process that can be caught are + * set to handle_file when received. This will make apache shutdowns more + * graceful even if use_handler is false. + */ + setsig_handler( SIGHUP, handle_file_on_sig ); + setsig_handler( SIGINT, handle_file_on_sig ); + setsig_handler( SIGQUIT, handle_file_on_sig ); + setsig_handler( SIGILL, handle_file_on_sig ); + setsig_handler( SIGABRT, handle_file_on_sig ); + setsig_handler( SIGBUS, handle_file_on_sig ); + setsig_handler( SIGFPE, handle_file_on_sig ); + setsig_handler( SIGPIPE, handle_file_on_sig ); + setsig_handler( SIGTERM, handle_file_on_sig ); + setsig_handler( SIGUSR1, handle_file_on_sig ); + + /* sigalrm is used to break out of read() when it is time to rotate the log. */ + setsig_handler( SIGALRM, alarm_signal_handler ); +} + +void +setsig_handler( int signum, void (*action)(int, siginfo_t *, void *)) +{ + struct sigaction siga ; + memset( &siga, '\0', sizeof( struct sigaction )); + siga.sa_sigaction= action ; + siga.sa_flags = SA_SIGINFO ; + if( -1== sigaction( signum, &siga, NULL )) + { + perror( "cant set sigaction" ); + } +} +#endif + + +/* + * cleanup + */ +void +cleanup( int exit_status ) +{ + handle_file(); + exit(exit_status); +} + +/* + * only the first cronolog process to open a particular file is responsible + * for starting the cleanup process later. This wrapper sets i_am_handler + * according to that logic. + * */ +int +openwrapper( const char *ofilename ) +{ + int ret; + if( use_handler !=1 ) + { + return open(ofilename, OPEN_SHARED, S_IRWXU ); + } + ret = open(ofilename, OPEN_EXCLUSIVE, S_IRWXU ); + if( ret < 0 ) + { + ret = open(ofilename, OPEN_SHARED, S_IRWXU ); + i_am_handler= 0; + } + else + { + i_am_handler=1; + } + return ret; +} + diff -Nur cronolog-1.6.2/src/cronoutils.c cronolog-1.6.2-jumbo-20031008/src/cronoutils.c --- cronolog-1.6.2/src/cronoutils.c 2001-05-03 12:43:21.000000000 -0400 +++ cronolog-1.6.2-jumbo-20031008/src/cronoutils.c 2003-10-08 00:22:10.000000000 -0400 @@ -195,11 +195,11 @@ { struct stat stat_buf; - if (stat(prevlinkname, &stat_buf) == 0) + if (lstat(prevlinkname, &stat_buf) == 0) { unlink(prevlinkname); } - if (stat(linkname, &stat_buf) == 0) + if (lstat(linkname, &stat_buf) == 0) { if (prevlinkname) { rename(linkname, prevlinkname); @@ -710,4 +710,50 @@ return retval; } - + +#ifndef _WIN32 +/* Turn a string specifying either a username or UID into an actual + * uid_t for use in setuid(). A string is assumed to be a UID if + * it contains only decimal digits. */ +uid_t +parse_uid(char *user, char *argv0) +{ + char *probe = user; + struct passwd *ent; + + while (*probe && isdigit(*probe)) { + probe++; + } + if (!(*probe)) { + return atoi(user); + } + if (!(ent = getpwnam(user))) { + fprintf(stderr, "%s: Bad username %s\n", argv0, user); + exit(1); + } + return (ent->pw_uid); +} + + +/* Turn a string specifying either a group name or GID into an actual + * gid_t for use in setgid(). A string is assumed to be a GID if + * it contains only decimal digits. */ +gid_t +parse_gid(char *group, char *argv0) +{ + char *probe = group; + struct group *ent; + + while (*probe && isdigit(*probe)) { + probe++; + } + if (!(*probe)) { + return atoi(group); + } + if (!(ent = getgrnam(group))) { + fprintf(stderr, "%s: Bad group name %s\n", argv0, group); + exit(1); + } + return (ent->gr_gid); +} +#endif /* _WIN32 */ diff -Nur cronolog-1.6.2/src/cronoutils.h cronolog-1.6.2-jumbo-20031008/src/cronoutils.h --- cronolog-1.6.2/src/cronoutils.h 2001-05-03 12:40:12.000000000 -0400 +++ cronolog-1.6.2-jumbo-20031008/src/cronoutils.h 2003-10-08 00:22:10.000000000 -0400 @@ -84,6 +84,8 @@ #include #ifndef _WIN32 #include +#include +#include #else #include #include @@ -172,7 +174,8 @@ void print_debug_msg(char *msg, ...); time_t parse_time(char *time_str, int); char *timestamp(time_t thetime); - +uid_t parse_uid(char *user, char *argv0); +gid_t parse_gid(char *group, char *argv0); /* Global variables */ diff -Nur cronolog-1.6.2/src/zip_send_rm.sh cronolog-1.6.2-jumbo-20031008/src/zip_send_rm.sh --- cronolog-1.6.2/src/zip_send_rm.sh 1969-12-31 19:00:00.000000000 -0500 +++ cronolog-1.6.2-jumbo-20031008/src/zip_send_rm.sh 2003-10-10 12:37:22.000000000 -0400 @@ -0,0 +1,64 @@ +#!/bin/bash +## ---------------------------------------------------------------------- +## ---------------------------------------------------------------------- +## +## File: zip_send_rm +## Author: mgrosso +## Created: Fri Oct 3 18:18:38 EDT 2003 on dhcp-172-18-102-101.looksmart.com +## Project: apache, cronolog +## Purpose: make damn sure the log file arrives at its destinations +## +## Copyright (c) 2003 LookSmart. All Rights Reserved. +## +## $Id$ +## ---------------------------------------------------------------------- +## ---------------------------------------------------------------------- + + +function do_or_die() +{ + $@ + if [ $? -ne 0 ] ; then + logger -s -p local0.crit "$0 puking on [ $@ ]" + exit 1 + fi +} + +if [ $# == 2 ] ; then + SCPDESTLIST=$1 + shift +fi +FILE=$1 + + + +do_or_die gzip $FILE + +if [ -z $SCPDESTLIST ] ; then + exit 0 +fi + +FILE=${FILE}.gz + +# spawn subshells, so that each scp can suceed or fail on its own. +# while this is less efficient from a bandwidth and cpu perspective, it +# is safer because it ensures that a hanging dest cant prevent the other +# dests from working. + +SCPDESTINATIONS=$( echo $SCPDESTLIST | sed -e 's/,/ /g' ) +for DEST in $SCPDESTINATIONS ; do + ( + HDEST=$( echo $DEST | cut -d ':' -f 1 ) + BACKUP_LINK=${HDEST}-$FILE + do_or_die ln $FILE $BACKUP_LINK + do_or_die scp $FILE $DEST + do_or_die rm $BACKUP_LINK + ) & +done +wait +# we dont delete the file unless it arrive at all destinations +# because if one destination failed, then ${DEST}-$FILE will still exist +# even after we rm $FILE +do_or_die rm $FILE +exit 0 + -- ####################################################################### # Matt Grosso cell (201)780-9592 mgrosso@looksmart.net # # work ny (212)324-1904 fax ny (212)324-1910 work sf (415)348-7756 # # pgp public key is at http://www.falconweb.com/~mattg/publickey.pgp # #######################################################################