2 https://lists.balabit.hu/pipermail/syslog-ng/2009-August/013325.html
5 [syslog-ng] PATCH: Log file size limit (was: Logfile Rotation)
7 Balazs Scheidler wrote:
9 >> Personally I prefer to arrange log rotation based on the file size
10 >> rather than some fixed time interval, so I have added that feature
11 >> to syslog-ng. If anyone is interested I can explain in more details
12 >> and post the patch.
14 > Please do. That's what open source is about.
17 The following patch introduces two new configuration options:
18 'file_size_limit' to be used inside global 'options' and 'size_limit' to
19 be used inside 'file' destination. Each option specifies log file size
20 limit in bytes. If the global option is set to a value greater than zero
21 it applies to all 'file' destinations. A particular file destination can
22 remove the limit by setting it to zero. For example:
24 # set the global file size limit
25 options { file_size_limit(123456); };
27 # set a different size limit for a particular file destination
28 destination log1 { file("/var/log/log1.log" size_limit(456789)); };
30 # remove size limit for a particular file destination
31 # (only useful if there is a global size limit set)
32 destination log2 { file("/var/log/log2.log" size_limit(0)); };
34 The file size is checked after writing each log message and if the file
35 has grown up to or above the size limit the file is renamed and a new
36 empty file is created to continue logging to. The name format the
37 "overgrown" log file is renamed to is "<p>-<s>.<m>-<r>", where <p> - the
38 full path and name of the original log file, <s> - current time in UNIX
39 format (seconds since Jan 1, 1970), <m> - fractional part of the current
40 time (microseconds, 6 digits), <r> - a random number (10 digits).
42 The intended use is to have incrond or another similar mechanism to
43 detect when there is a new "renamed" log file and to process it in
44 whatever way necessary (gzip it, parse it, send it my email, etc.).
46 *********** BEGIN PATCH ***********
47 diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.c syslog-ng-3.0.4/src/affile.c
48 --- syslog-ng-3.0.4-orig/src/affile.c 2009-07-31 11:41:20.000000000 +0200
49 +++ syslog-ng-3.0.4/src/affile.c 2009-08-17 11:02:41.000000000 +0200
52 return log_queue_get_length(((LogWriter *) self->writer)->queue) == 0;
56 +affile_dw_open_file(AFFileDestWriter *self, int *fd)
58 + return affile_open_file(self->filename->str, (self->owner->flags & AFFILE_PIPE)?
59 + (O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE):
60 + (O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE),
61 + self->owner->file_uid, self->owner->file_gid, self->owner->file_perm,
62 + self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm,
63 + !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), fd);
67 +affile_dw_transport_callback(gint fd, void *context)
69 + static const size_t extra_length = 32;
70 + AFFileDestWriter *self = (AFFileDestWriter *) context;
71 + off_t size = lseek(fd, 0, SEEK_CUR);
75 + if ((size > 0) && (size >= self->owner->size_limit))
77 + /* TODO: use g_file_read_link() */
78 + g_get_current_time(&time);
79 + name = g_string_sized_new(self->filename->len + extra_length);
80 + g_string_printf(name, "%s-%lu.%06lu-%010lu", self->filename->str, (gulong)time.tv_sec, (gulong)time.tv_usec, (gulong)g_random_int());
81 + if (rename(self->filename->str, name->str) == 0)
83 + if (affile_dw_open_file(self, &reopen_fd))
85 + if (dup2(reopen_fd, fd) < 0)
87 + msg_error("Error swithing to new log file",
88 + evt_tag_str("filename", self->filename->str),
89 + evt_tag_errno(EVT_TAG_OSERROR, errno),
96 + msg_error("Error opening file for writing",
97 + evt_tag_str("filename", self->filename->str),
98 + evt_tag_errno(EVT_TAG_OSERROR, errno),
104 + msg_error("Error renaming overgrown file",
105 + evt_tag_str("filename", self->filename->str),
106 + evt_tag_errno(EVT_TAG_OSERROR, errno),
109 + g_string_free(name, TRUE);
114 affile_dw_init(LogPipe *s)
116 AFFileDestWriter *self = (AFFileDestWriter *) s;
120 GlobalConfig *cfg = log_pipe_get_config(s);
123 self->time_reopen = cfg->time_reopen;
124 @@ -454,20 +512,12 @@
125 evt_tag_int("overwrite_if_older", self->owner->overwrite_if_older),
127 unlink(self->filename->str);
130 - if (self->owner->flags & AFFILE_PIPE)
131 - flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE;
133 - flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE;
135 self->last_open_stamp = time(NULL);
136 - if (affile_open_file(self->filename->str, flags,
137 - self->owner->file_uid, self->owner->file_gid, self->owner->file_perm,
138 - self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm,
139 - !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), &fd))
140 + if (affile_dw_open_file(self, &fd))
146 @@ -482,11 +532,11 @@
151 write_flags = ((self->owner->flags & AFFILE_FSYNC) ? LTF_FSYNC : 0) | LTF_APPEND;
152 - log_writer_reopen(self->writer, log_proto_plain_new_client(log_transport_plain_new(fd, write_flags)));
153 + log_writer_reopen(self->writer, log_proto_plain_new_client(((self->owner->size_limit > 0) && !(self->owner->flags & AFFILE_PIPE))? log_transport_plain_new_with_callback(fd, write_flags, affile_dw_transport_callback, self): log_transport_plain_new(fd, write_flags)));
157 msg_error("Error opening file for writing",
158 evt_tag_str("filename", self->filename->str),
159 @@ -678,10 +728,18 @@
160 AFFileDestDriver *self = (AFFileDestDriver *) s;
162 self->local_time_zone = g_strdup(local_time_zone);
166 +affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit)
168 + AFFileDestDriver *self = (AFFileDestDriver *) s;
170 + self->size_limit = file_size_limit;
173 static inline gchar *
174 affile_dd_format_persist_name(AFFileDestDriver *self)
176 static gchar persist_name[1024];
178 @@ -766,10 +824,12 @@
179 self->dir_gid = cfg->dir_gid;
180 if (self->dir_perm == (mode_t) -1)
181 self->dir_perm = cfg->dir_perm;
182 if (self->time_reap == -1)
183 self->time_reap = cfg->time_reap;
184 + if (self->size_limit == -1)
185 + self->size_limit = cfg->file_size_limit;
187 self->use_time_recvd = cfg->use_time_recvd;
189 if (self->local_time_zone_info)
190 time_zone_info_free(self->local_time_zone_info);
192 if (strchr(filename, '$') == NULL)
194 self->flags |= AFFILE_NO_EXPAND;
196 self->time_reap = -1;
197 + self->size_limit = -1;
200 diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.h syslog-ng-3.0.4/src/affile.h
201 --- syslog-ng-3.0.4-orig/src/affile.h 2009-04-30 12:22:53.000000000 +0200
202 +++ syslog-ng-3.0.4/src/affile.h 2009-08-17 10:10:39.000000000 +0200
205 gint overwrite_if_older;
206 gboolean use_time_recvd;
212 LogDriver *affile_dd_new(gchar *filename, guint32 flags);
214 void affile_dd_set_compress(LogDriver *s, gboolean compress);
216 void affile_dd_set_dir_perm(LogDriver *s, mode_t dir_perm);
217 void affile_dd_set_create_dirs(LogDriver *s, gboolean create_dirs);
218 void affile_dd_set_fsync(LogDriver *s, gboolean enable);
219 void affile_dd_set_overwrite_if_older(LogDriver *s, gint overwrite_if_older);
220 void affile_dd_set_local_time_zone(LogDriver *s, const gchar *local_time_zone);
221 +void affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit);
224 diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-grammar.y syslog-ng-3.0.4/src/cfg-grammar.y
225 --- syslog-ng-3.0.4-orig/src/cfg-grammar.y 2009-08-05 12:34:18.000000000 +0200
226 +++ syslog-ng-3.0.4/src/cfg-grammar.y 2009-08-17 10:10:39.000000000 +0200
227 @@ -154,10 +154,11 @@
228 %token KW_DIR_OWNER KW_DIR_GROUP KW_DIR_PERM
229 %token KW_TEMPLATE KW_TEMPLATE_ESCAPE
230 %token KW_FOLLOW_FREQ
231 %token KW_OVERWRITE_IF_OLDER
232 %token KW_DEFAULT_FACILITY KW_DEFAULT_LEVEL
233 +%token KW_FILE_SIZE_LIMIT KW_SIZE_LIMIT
235 /* socket related options */
236 %token KW_KEEP_ALIVE KW_MAX_CONNECTIONS
237 %token KW_LOCALIP KW_IP KW_LOCALPORT KW_PORT KW_DESTPORT
238 %token KW_IP_TTL KW_SO_BROADCAST KW_IP_TOS KW_SO_SNDBUF KW_SO_RCVBUF KW_SO_KEEPALIVE KW_SPOOF_SOURCE
239 @@ -799,10 +800,11 @@
240 | KW_DIR_PERM '(' LL_NUMBER ')' { affile_dd_set_dir_perm(last_driver, $3); }
241 | KW_CREATE_DIRS '(' yesno ')' { affile_dd_set_create_dirs(last_driver, $3); }
242 | KW_OVERWRITE_IF_OLDER '(' LL_NUMBER ')' { affile_dd_set_overwrite_if_older(last_driver, $3); }
243 | KW_FSYNC '(' yesno ')' { affile_dd_set_fsync(last_driver, $3); }
244 | KW_LOCAL_TIME_ZONE '(' string ')' { affile_dd_set_local_time_zone(last_driver, $3); free($3); }
245 + | KW_SIZE_LIMIT '(' LL_NUMBER ')' { affile_dd_set_file_size_limit(last_driver, $3); }
249 : KW_PIPE '(' dest_afpipe_params ')' { $$ = $3; }
251 @@ -1141,10 +1143,11 @@
252 | KW_FILE_TEMPLATE '(' string ')' { configuration->file_template_name = g_strdup($3); free($3); }
253 | KW_PROTO_TEMPLATE '(' string ')' { configuration->proto_template_name = g_strdup($3); free($3); }
254 | KW_RECV_TIME_ZONE '(' string ')' { configuration->recv_time_zone = g_strdup($3); free($3); }
255 | KW_SEND_TIME_ZONE '(' string ')' { configuration->send_time_zone = g_strdup($3); free($3); }
256 | KW_LOCAL_TIME_ZONE '(' string ')' { configuration->local_time_zone = g_strdup($3); free($3); }
257 + | KW_FILE_SIZE_LIMIT '(' LL_NUMBER ')' { configuration->file_size_limit = $3; }
260 /* BEGIN MARK: tls */
262 : tls_option tls_options
263 diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-lex.l syslog-ng-3.0.4/src/cfg-lex.l
264 --- syslog-ng-3.0.4-orig/src/cfg-lex.l 2009-05-06 11:20:22.000000000 +0200
265 +++ syslog-ng-3.0.4/src/cfg-lex.l 2009-08-17 10:10:39.000000000 +0200
266 @@ -126,10 +126,12 @@
267 { "recv_time_zone", KW_RECV_TIME_ZONE },
268 { "send_time_zone", KW_SEND_TIME_ZONE },
269 { "local_time_zone", KW_LOCAL_TIME_ZONE },
270 { "use_time_recvd", KW_USE_TIME_RECVD, KWS_OBSOLETE, "Use R_ or S_ prefixed macros in templates" },
271 { "use_fqdn", KW_USE_FQDN },
272 + { "size_limit", KW_SIZE_LIMIT },
273 + { "file_size_limit", KW_FILE_SIZE_LIMIT },
274 { "use_dns", KW_USE_DNS },
275 { "gc_threshold", KW_GC_BUSY_THRESHOLD },
276 { "gc_busy_threshold", KW_GC_BUSY_THRESHOLD },
277 { "gc_idle_threshold", KW_GC_IDLE_THRESHOLD },
278 { "time_reopen", KW_TIME_REOPEN },
279 diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.c syslog-ng-3.0.4/src/cfg.c
280 --- syslog-ng-3.0.4-orig/src/cfg.c 2009-04-30 12:00:54.000000000 +0200
281 +++ syslog-ng-3.0.4/src/cfg.c 2009-08-17 10:10:39.000000000 +0200
282 @@ -298,10 +298,11 @@
284 self->follow_freq = -1;
287 self->file_perm = 0600;
288 + self->file_size_limit = 0;
291 self->dir_perm = 0700;
293 self->use_dns_cache = 1;
294 diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.h syslog-ng-3.0.4/src/cfg.h
295 --- syslog-ng-3.0.4-orig/src/cfg.h 2009-06-03 13:08:27.000000000 +0200
296 +++ syslog-ng-3.0.4/src/cfg.h 2009-08-17 10:10:39.000000000 +0200
299 gboolean create_dirs;
303 + off_t file_size_limit;
309 diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.c syslog-ng-3.0.4/src/logtransport.c
310 --- syslog-ng-3.0.4-orig/src/logtransport.c 2009-04-30 10:53:22.000000000 +0200
311 +++ syslog-ng-3.0.4/src/logtransport.c 2009-08-17 11:01:33.000000000 +0200
313 typedef struct _LogTransportPlain LogTransportPlain;
315 struct _LogTransportPlain
318 + LogTransportCallback callback;
323 log_transport_plain_read_method(LogTransport *s, gpointer buf, gsize buflen, GSockAddr **sa)
325 @@ -138,24 +140,32 @@
327 if (self->super.flags & LTF_FSYNC)
328 fsync(self->super.fd);
330 while (rc == -1 && errno == EINTR);
331 + if ((self->callback != NULL) && (rc > 0))
332 + self->callback(self->super.fd, self->context);
338 -log_transport_plain_new(gint fd, guint flags)
339 +log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context)
341 LogTransportPlain *self = g_new0(LogTransportPlain, 1);
344 self->super.cond = 0;
345 self->super.flags = flags;
346 self->super.read = log_transport_plain_read_method;
347 self->super.write = log_transport_plain_write_method;
348 self->super.free_fn = log_transport_free_method;
349 + self->callback = callback;
350 + self->context = context;
356 +log_transport_plain_new(gint fd, guint flags)
358 + return log_transport_plain_new_with_callback(fd, flags, NULL, NULL);
360 diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.h syslog-ng-3.0.4/src/logtransport.h
361 --- syslog-ng-3.0.4-orig/src/logtransport.h 2009-04-30 10:46:55.000000000 +0200
362 +++ syslog-ng-3.0.4/src/logtransport.h 2009-08-17 11:00:58.000000000 +0200
364 log_transport_read(LogTransport *self, gpointer buf, gsize count, GSockAddr **sa)
366 return self->read(self, buf, count, sa);
369 +typedef void (*LogTransportCallback)(gint fd, void *context);
370 +LogTransport *log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context);
371 LogTransport *log_transport_plain_new(gint fd, guint flags);
372 void log_transport_free(LogTransport *s);
373 void log_transport_free_method(LogTransport *s);