001 package net.sf.logdistiller;
002
003 /*
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017 import java.io.*;
018 import java.util.*;
019
020 import org.apache.commons.io.IOUtils;
021 import org.apache.commons.lang.ArrayUtils;
022
023 /**
024 * The class executing a log distillation, using a log distiller, and processing every log events. A log distillation
025 * can be run either with LogDistiller's {@link net.sf.logdistiller.gui GUI} or with {@link net.sf.logdistiller.ant Ant
026 * tasks}.
027 *
028 * @see LogDistiller
029 */
030 public class LogDistillation
031 implements Serializable
032 {
033 private static final long serialVersionUID = -9098523382541148543L;
034
035 public final static String LINE_SEPARATOR = System.getProperty( "line.separator" );
036
037 private final LogDistiller ld;
038
039 private final Group[] groups;
040
041 private final Category[] categories;
042
043 private final String content;
044
045 private final File destinationDirectory;
046
047 private final String version = LogDistiller.getVersion();
048
049 private transient Map/*<String, Group>*/ mapGroups;
050
051 private transient Map/*<String, Category>*/ mapCategories;
052
053 private long beginTime;
054
055 private long endTime;
056
057 private int eventCount = -1;
058
059 private long eventBytes = 0;
060
061 private int skippedEventCount = 0;
062
063 private long skippedEventBytes = 0;
064
065 private transient LogType.Description logtypeDescription;
066
067 public LogDistillation( LogDistiller ld )
068 {
069 this.ld = ld;
070
071 // init categories
072 LogDistiller.Category[] categoryDefinitions = ld.getCategories();
073 int length = categoryDefinitions.length;
074 categories = new Category[length];
075 mapCategories = new HashMap();
076 for ( int i = 0; i < length; i++ )
077 {
078 categories[i] = new Category( categoryDefinitions[i] );
079 mapCategories.put( categoryDefinitions[i].getId(), categories[i] );
080 }
081
082 // init groups
083 LogDistiller.Group[] groupDefinitions = ld.getGroups();
084 length = groupDefinitions.length;
085 groups = new Group[length];
086 for ( int i = 0; i < length; i++ )
087 {
088 LogDistiller.Group groupDefinition = groupDefinitions[i];
089 LogDistiller.Category categoryDefinition = groupDefinition.getCategory();
090 Category category =
091 ( categoryDefinition == null ) ? null : (Category) mapCategories.get( categoryDefinition.getId() );
092 groups[i] = new Group( groupDefinition, category );
093 }
094
095 // misc.
096 content = ld.getOutput().getContent();
097 destinationDirectory = new File( ld.getOutput().getDirectory() );
098 updateRef();
099 }
100
101 /**
102 * Create a logtype description with parameters defined in LogDistiller configuration.
103 * @return a new logtype description
104 */
105 private LogType.Description newLogTypeDescription()
106 {
107 LogDistiller.LogType lt = ld.getLogType();
108
109 LogType logtype = LogTypes.getLogType( lt.getId() );
110 LogType.Description logtypeDescription = logtype.newDescription( ld.getLogType().getParams() );
111 logtypeDescription.setExtensions( ld.getLogType().getExtensions() );
112 ld.checkLogtypeAttributes( logtypeDescription );
113
114 return logtypeDescription;
115 }
116
117 /**
118 * update internal maps, skip value and references from categories and groups
119 */
120 private void updateRef()
121 {
122 String skip = ld.getOutput().getSkip();
123
124 // process categories
125 mapCategories = new HashMap();
126 Category skipCategory = null;
127 int len = categories.length;
128 for ( int i = 0; i < len; i++ )
129 {
130 Category category = categories[i];
131 mapCategories.put( category.getDefinition().getId(), category );
132 category.updateRef( this );
133
134 if ( category.getDefinition().getId().equals( skip ) )
135 {
136 skipCategory = category;
137 }
138 }
139
140 // process groups
141 mapGroups = new HashMap();
142 len = groups.length;
143 for ( int i = 0; i < len; i++ )
144 {
145 Group group = groups[i];
146 mapGroups.put( group.getDefinition().getId(), group );
147 group.updateRef( this );
148
149 if ( group.getDefinition().getId().equals( skip ) )
150 {
151 group.setSkip( true );
152 }
153 }
154
155 if ( skipCategory != null )
156 {
157 skipCategory.setSkip( true );
158 }
159
160 logtypeDescription = newLogTypeDescription();
161 }
162
163 private void readObject( ObjectInputStream in )
164 throws IOException, ClassNotFoundException
165 {
166 in.defaultReadObject();
167 if ( ld.getOutput() != null )
168 {
169 ld.updateRef(); // workaround: a problem with Group deserialization happens
170 updateRef();
171 }
172 }
173
174 /**
175 * begin the distillation
176 *
177 * @throws IOException
178 */
179 public void begin()
180 throws IOException
181 {
182 destinationDirectory.mkdirs();
183 int count = groups.length;
184 for ( int i = 0; i < count; i++ )
185 {
186 groups[i].begin( destinationDirectory );
187 }
188 eventCount = 0;
189 beginTime = System.currentTimeMillis();
190 }
191
192 /**
193 * Process a log event
194 *
195 * @param le LogEvent the log event
196 * @throws IOException if an error occurs when (eventually) saving the log event
197 */
198 public void processLogEvent( LogEvent le )
199 throws IOException
200 {
201 if ( eventCount < 0 )
202 {
203 throw new IllegalStateException( "LogDistiller not initialized: need to call begin() before processing" );
204 }
205 eventCount++;
206 int eventSize = le.getRawLog().length() + LINE_SEPARATOR.length();
207 eventBytes += eventSize;
208 int count = groups.length;
209 for ( int i = 0; i < count; i++ )
210 {
211 Group group = groups[i];
212 if ( group.accept( le ) )
213 {
214 if ( group.isSkip() )
215 {
216 // logevent matched a skipped group => update counters
217 eventCount--;
218 eventBytes -= eventSize;
219 skippedEventCount++;
220 skippedEventBytes += eventSize;
221 break;
222 }
223 if ( !group.getDefinition().continueProcessing() )
224 {
225 break;
226 }
227 }
228 }
229 }
230
231 /**
232 * end the distillation
233 *
234 * @throws IOException if an error occurs when closing the files where log events were (eventually) saved
235 */
236 public void end()
237 throws IOException
238 {
239 endTime = System.currentTimeMillis();
240 int count = groups.length;
241 for ( int i = 0; i < count; i++ )
242 {
243 Group group = groups[i];
244 group.end();
245 }
246 saveTo( newDestinationFile( "result.ser" ) );
247 }
248
249 /**
250 * Saves the distillation to a file, in Java serialized format.
251 *
252 * @param file File destination file.
253 */
254 public void saveTo( File file )
255 throws IOException
256 {
257 OutputStream out = null;
258 try
259 {
260 out = new FileOutputStream( file );
261 saveTo( out );
262 }
263 finally
264 {
265 IOUtils.closeQuietly( out );
266 }
267 }
268
269 public void saveTo( OutputStream out )
270 throws IOException
271 {
272 ObjectOutputStream obj = new ObjectOutputStream( out );
273 obj.writeObject( this );
274 }
275
276 public List listCategoriesToReport()
277 {
278 List reportCategories = new ArrayList();
279 int count = categories.length;
280 for ( int i = 0; i < count; i++ )
281 {
282 Category category = categories[i];
283 if ( ( !category.isSkip() ) && ( category.sumEventCount() > 0 ) )
284 {
285 reportCategories.add( category );
286 }
287 }
288 return reportCategories;
289 }
290
291 public List listGroupsToReport()
292 {
293 List reportGroups = new ArrayList();
294 int count = groups.length;
295 for ( int i = 0; i < count; i++ )
296 {
297 Group group = groups[i];
298 if ( ( !group.isSkip() ) && ( group.getEventCount() > 0 ) && ( group.getCategory() == null ) )
299 {
300 reportGroups.add( group );
301 }
302 }
303 return reportGroups;
304 }
305
306 public LogDistiller getDefinition()
307 {
308 return ld;
309 }
310
311 public Group[] getGroups()
312 {
313 return (Group[]) ArrayUtils.clone( groups );
314 }
315
316 public Group getGroup( String id )
317 {
318 return (Group) mapGroups.get( id );
319 }
320
321 public Category[] getCategories()
322 {
323 return (Category[]) ArrayUtils.clone( categories );
324 }
325
326 public Category getCategory( String id )
327 {
328 return (Category) mapCategories.get( id );
329 }
330
331 public String getContent()
332 {
333 return content;
334 }
335
336 public int getEventCount()
337 {
338 return eventCount;
339 }
340
341 public long getEventBytes()
342 {
343 return eventBytes;
344 }
345
346 public long getBeginTime()
347 {
348 return beginTime;
349 }
350
351 public long getEndTime()
352 {
353 return endTime;
354 }
355
356 public int getSkippedEventCount()
357 {
358 return skippedEventCount;
359 }
360
361 public long getSkippedEventBytes()
362 {
363 return skippedEventBytes;
364 }
365
366 public File newDestinationFile( String filename )
367 {
368 return new File( destinationDirectory, filename );
369 }
370
371 public LogType.Description getLogTypeDescription()
372 {
373 return logtypeDescription;
374 }
375
376 /**
377 * What is the version of <b>LogDistiller</b> that was used to create this instance?
378 *
379 * @return String the version
380 */
381 public String getVersion()
382 {
383 return ( version == null ) ? "unknown" : version;
384 }
385
386 public class Group
387 implements Serializable
388 {
389 private static final long serialVersionUID = 5788707827350407084L;
390
391 private final LogDistiller.Group definition;
392
393 private final Category category;
394
395 private transient LogDistillation logdistillation;
396
397 private boolean skip = false;
398
399 /**
400 * maximum number of events saved, -1 = no limit
401 *
402 * @since 0.7
403 */
404 private final int maxSaveCount;
405
406 /**
407 * maximum size sum of events saved in bytes, -1 = no limit
408 *
409 * @since 0.7
410 */
411 private final long maxSaveBytes;
412
413 private final Plugin[] plugins;
414
415 private File logFile;
416
417 private transient PrintStream logStream;
418
419 private int eventCount = 0;
420
421 private int savedEventCount = 0;
422
423 private long bytes = 0;
424
425 public Group( LogDistiller.Group definition, Category category )
426 {
427 this.definition = definition;
428 this.category = category;
429 maxSaveCount = Integer.parseInt( definition.getParam( "maxSave.count", "-1" ) );
430 maxSaveBytes = Long.parseLong( definition.getParam( "maxSave.size", "-1" ) ) * 1024;
431 LogDistiller.Plugin[] pluginDefs = definition.getPlugins();
432 plugins = new Plugin[pluginDefs.length];
433 for ( int i = 0; i < pluginDefs.length; i++ )
434 {
435 plugins[i] = Plugins.newInstance( pluginDefs[i] );
436 }
437 }
438
439 private void updateRef( LogDistillation logdistillation )
440 {
441 this.logdistillation = logdistillation;
442 if ( category != null )
443 {
444 category.addGroup( this );
445 }
446 int len = plugins.length;
447 for ( int i = 0; i < len; i++ )
448 {
449 plugins[i].updateRef( this );
450 }
451 }
452
453 public LogDistillation getLogdistillation()
454 {
455 return logdistillation;
456 }
457
458 public Category getCategory()
459 {
460 return category;
461 }
462
463 public void setSkip( boolean skip )
464 {
465 this.skip = skip;
466 }
467
468 public boolean isSkip()
469 {
470 return skip;
471 }
472
473 /**
474 * Check if given LogEvent meet one of the conditions defined.
475 *
476 * @param le LogEvent the LogEvent to check
477 * @throws IOException
478 * @return boolean <code>true</code> if the LogEvent meets a condition
479 */
480 public boolean accept( LogEvent le )
481 throws IOException
482 {
483 Condition condition = definition.accept( le );
484 if ( condition != null )
485 {
486 addLogEvent( le, condition );
487 return true;
488 }
489 return false;
490 }
491
492 /**
493 * Add a LogEvent to the current Group.
494 *
495 * @param le LogEvent
496 * @param condition the condition that matched the LogEvent
497 * @throws IOException
498 */
499 protected void addLogEvent( LogEvent le, Condition condition )
500 throws IOException
501 {
502 eventCount++;
503 bytes += le.getRawLog().length() + LINE_SEPARATOR.length();
504 if ( definition.getSave() && ( ( maxSaveCount < 0 ) || ( eventCount <= maxSaveCount ) )
505 && ( ( maxSaveBytes < 0 ) || ( bytes <= maxSaveBytes ) ) )
506 {
507 if ( logStream == null )
508 {
509 logStream = new PrintStream( new FileOutputStream( logFile ) );
510 }
511 logStream.println( le.getRawLog() );
512 savedEventCount++;
513 }
514 int len = plugins.length;
515 for ( int i = 0; i < len; i++ )
516 {
517 plugins[i].addLogEvent( le, condition );
518 }
519 }
520
521 void begin( File destinationDirectory )
522 throws IOException
523 {
524 logFile = newDestinationFile( definition.getId() + ".log" );
525 int len = plugins.length;
526 for ( int i = 0; i < len; i++ )
527 {
528 plugins[i].begin( destinationDirectory );
529 }
530 }
531
532 void end()
533 throws IOException
534 {
535 if ( logStream != null )
536 {
537 logStream.close();
538 }
539 int len = plugins.length;
540 for ( int i = 0; i < len; i++ )
541 {
542 plugins[i].end();
543 }
544 }
545
546 public LogDistiller.Group getDefinition()
547 {
548 return definition;
549 }
550
551 public int getEventCount()
552 {
553 return eventCount;
554 }
555
556 public int getSavedEventCount()
557 {
558 return savedEventCount;
559 }
560
561 public long getBytes()
562 {
563 return bytes;
564 }
565
566 public File getLogFile()
567 {
568 return logFile;
569 }
570
571 public int getMaxSaveCount()
572 {
573 return maxSaveCount;
574 }
575
576 public long getMaxSaveBytes()
577 {
578 return maxSaveBytes;
579 }
580
581 public Plugin[] getPlugins()
582 {
583 return (Plugin[]) ArrayUtils.clone( plugins );
584 }
585 }
586
587 public static class Category
588 implements Serializable
589 {
590 private static final long serialVersionUID = 7098883103191641753L;
591
592 private final LogDistiller.Category definition;
593
594 private transient LogDistillation logdistillation;
595
596 private transient List groups;
597
598 private boolean skip = false;
599
600 public Category( LogDistiller.Category definition )
601 {
602 this.definition = definition;
603 }
604
605 private void updateRef( LogDistillation logdistillation )
606 {
607 this.logdistillation = logdistillation;
608 groups = new ArrayList();
609 }
610
611 public LogDistillation getLogdistillation()
612 {
613 return logdistillation;
614 }
615
616 public LogDistiller.Category getDefinition()
617 {
618 return definition;
619 }
620
621 private void addGroup( Group group )
622 {
623 groups.add( group );
624 }
625
626 public List getGroups()
627 {
628 return groups;
629 }
630
631 public void setSkip( boolean skip )
632 {
633 this.skip = skip;
634 Iterator iter = groups.iterator();
635 while ( iter.hasNext() )
636 {
637 Group group = (Group) iter.next();
638 group.setSkip( skip );
639 }
640 }
641
642 public boolean isSkip()
643 {
644 return skip;
645 }
646
647 public int sumEventCount()
648 {
649 int sum = 0;
650 Iterator iter = groups.iterator();
651 while ( iter.hasNext() )
652 {
653 Group group = (Group) iter.next();
654 sum += group.getEventCount();
655 }
656 return sum;
657 }
658
659 public long sumBytes()
660 {
661 long sum = 0;
662 Iterator iter = groups.iterator();
663 while ( iter.hasNext() )
664 {
665 Group group = (Group) iter.next();
666 sum += group.getBytes();
667 }
668 return sum;
669 }
670
671 public List listGroupsToReport()
672 {
673 List reportGroups = new ArrayList();
674 Iterator iter = groups.iterator();
675 while ( iter.hasNext() )
676 {
677 Group group = (Group) iter.next();
678 if ( group.getEventCount() > 0 )
679 {
680 reportGroups.add( group );
681 }
682 }
683 return reportGroups;
684 }
685 }
686
687 public static abstract class Plugin
688 implements Serializable
689 {
690 private static final long serialVersionUID = -722318023109374109L;
691
692 protected final LogDistiller.Plugin definition;
693
694 protected transient Group group;
695
696 public Plugin( LogDistiller.Plugin definition )
697 {
698 this.definition = definition;
699 }
700
701 private void updateRef( Group group )
702 {
703 this.group = group;
704 }
705
706 public Group getGroup()
707 {
708 return group;
709 }
710
711 public LogDistiller.Plugin getDefinition()
712 {
713 return definition;
714 }
715
716 public String getId()
717 {
718 return getDefinition().getType();
719 }
720
721 /**
722 * Begin the log distillation for this plugin instance.
723 *
724 * @param destinationDirectory the directory where reports will be stored.
725 * @throws IOException
726 */
727 abstract public void begin( File destinationDirectory )
728 throws IOException;
729
730 /**
731 * Add a log event to this plugin instance.
732 *
733 * @param le the log event.
734 * @throws IOException
735 */
736 abstract public void addLogEvent( LogEvent le )
737 throws IOException;
738
739 /**
740 * Add a log event to this plugin instance, with information on which condition was matched.
741 *
742 * @param le the log event.
743 * @param condition the condition which was matched
744 * @since 1.1
745 */
746 public void addLogEvent( LogEvent le, Condition condition )
747 throws IOException
748 {
749 addLogEvent( le ); // default implementation does ignore condition
750 }
751
752 /**
753 * End the log distillation for this plugin instance.
754 *
755 * @throws IOException
756 */
757 abstract public void end()
758 throws IOException;
759
760 /**
761 * Append a summary of this plugin instance results to the group report.
762 *
763 * @param report the plugin report formatter
764 */
765 abstract public void appendGroupReport( ReportFormat.PluginReport report );
766
767 /**
768 * Append a summary of this plugin instance results to the global report.
769 *
770 * @param report the plugin report formatter
771 */
772 abstract public void appendGlobalReport( ReportFormat.PluginReport report );
773 }
774 }