package net.sf.logdistiller.plugins;

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.commons.lang.StringUtils;

import net.sf.logdistiller.LogDistillation;
import net.sf.logdistiller.LogDistiller;
import net.sf.logdistiller.LogEvent;
import net.sf.logdistiller.LogType;
import net.sf.logdistiller.Plugin;
import net.sf.logdistiller.PluginConfigException;
import net.sf.logdistiller.ReportFormat;

/**
 * Counts the frequency of each attribute's value.
 * <p>
 * Parameters:
 * </p>
 * <ul>
 * <li><b>value</b> (deprecated since 1.1): definition of the value that will be taken into account
 * <ul>
 * <li><code>attribute</code> (mandatory): the attribute's name
 * <li><code>regexp</code>: use a regexp to extract value taken in account for the first attribute
 * <li><code>regexp.group</code> (default: -1): which group to extract? If negative, all groups will be concatenated
 * <li><code>attribute2</code> (default: none): second attribute to append
 * <li><code>attribute3</code> (default: none): third attribute to append
 * </ul>
 * </li>
 * <li><b>value</b>: definition of the value that will be taken into account
 * <ul>
 * <li><code>attributes</code> (mandatory): the attributes' names, comma separated (no count limit)
 * </li>
 * </ul>
 * <li><b>sorting options</b>
 * <ul>
 * <li><code>sortItem</code> (default: <code>count</code>): sort on which item? <code>count</code>, <code>value</code>,
 * <code>valueLength</code>
 * <li><code>reverseOrder</code> (default: <code>false</code>): by default, most frequent value are displayed first
 * </ul>
 * </li>
 * <li><b>reporting</b>
 * <ul>
 * <li><code>valueDescriptions</code>: description to insert in the report instead of corresponding value (in java
 * properties format)
 * <li><code>maxGlobalReport</code> (default: 5): maximum number of values inserted in global report
 * <li><code>maxGroupReport</code> (default: 25): maximum number of values inserted in group report
 * </ul>
 * </li>
 * </ul>
 *
 * @see SamplingPlugin
 * @since 0.6
 */
public class FreqPlugin
    extends LogDistillation.Plugin
{
    private static final long serialVersionUID = -5162563816094662936L;

    public final static String ID = "freq";

    public final static Plugin TYPE = new Type();

    private final static String[] SORT_ITEM_VALUES = { "count", "value", "valueLength" };

    private final static String[] SORT_ITEM_DESCRIPTIONS = { "count", "attribute's value", "attribute's value length" };

    private final static FreqOrder[] SORT_ITEM_COMPARATORS =
        { new CountOrder(), new ValueOrder(), new ValueLengthOrder() };

    private final Map<String, Freq> freqs = new HashMap<String, Freq>();

    protected SortedSet<Freq> sorted; // sorted Freq

    /** @deprecated since 1.1, use attributes instead */
    private final String attribute;

    /** @since 0.8
     * @deprecated since 1.1, use attributes instead */
    private final String attribute2;

    /** @since 0.8
     * @deprecated since 1.1, use attributes instead */
    private final String attribute3;

    /** @deprecated since 1.1, use attributes instead */
    private transient LogType.AttributeInfo attributeInfo;

    /** @deprecated since 1.1, use attributes instead */
    private transient LogType.AttributeInfo attributeInfo2;

    /** @deprecated since 1.1, use attributes instead */
    private transient LogType.AttributeInfo attributeInfo3;

    private final int sortItem; // index in SORT_ITEM_VALUES

    private final boolean reverseOrder;

    private final Properties valueDescriptions = new Properties();

    private final int maxGlobalReport;

    private final int maxGroupReport;

    /** @deprecated since 1.1, use attributes instead */
    private final String regexpString;

    /** @deprecated since 1.1, use attributes instead */
    private transient Pattern regexp;

    /** @deprecated since 1.1, use attributes instead */
    private final int regexpGroup;

    /** @since 1.1 */
    protected final String[] attributes;

    protected transient LogType.AttributeInfo[] attributesInfo;

    public FreqPlugin( LogDistiller.Plugin definition )
    {
        this( definition, true );
    }

    protected FreqPlugin( LogDistiller.Plugin definition, boolean checkSampling )
    {
        super( definition );
        LogDistiller.Group group = definition.getGroup();
        String context = "(logdistiller/group[id='" + ( ( group == null ) ? "default-id" : group.getId() )
            + "]/plugin[type='" + definition.getType() + "'])";

        attribute = definition.getParam( "attribute" );
        attribute2 = definition.getParam( "attribute2", null );
        attribute3 = definition.getParam( "attribute3", null );
        regexpString = definition.getParam( "regexp" );
        updatePattern();
        regexpGroup = Integer.parseInt( definition.getParam( "regexp.group", "-1" ) );
        String attrs = definition.getParam( "attributes" );
        attributes = ( attrs == null ) ? null : StringUtils.split( attrs, ", " );

        String attr = definition.getParam( "sortItem" );
        int sortItem_ = ( attr == null ) ? 0 : Arrays.binarySearch( SORT_ITEM_VALUES, attr );
        if ( sortItem_ < 0 )
        {
            LogDistiller ld = group.getLogdistiller();
            ld.addWarning( context + " unsupported '" + attr
                + "' sortItem value: choose one of [ 'count', 'value', 'valueLength' ], defaulting to 'count'" );
            sortItem_ = 0;
        }
        sortItem = sortItem_;

        attr = definition.getParam( "reverseOrder", "false" );
        reverseOrder = Boolean.valueOf( attr ).booleanValue();
        attr = definition.getParam( "valueDescriptions" );
        if ( attr != null )
        {
            try
            {
                valueDescriptions.load( new ByteArrayInputStream( attr.getBytes( "ISO-8859-1" ) ) );
            }
            catch ( IOException ioe )
            {
                ioe.printStackTrace();
            }
        }
        maxGlobalReport = Integer.parseInt( definition.getParam( "maxGlobalReport", "5" ) );
        maxGroupReport = Integer.parseInt( definition.getParam( "maxGroupReport", "25" ) );

        if ( checkSampling
            && ( ( definition.getParam( "sampling.maxCount" ) != null )
                || ( definition.getParam( "sampling.maxSize" ) != null )
                || ( definition.getParam( "sampling.filename" ) != null ) ) )
        {
            LogDistiller ld = group.getLogdistiller();
            ld.addWarning( context + " sampling features (sampling.* parameters) disabled from freq plugin: "
                           + "moved to sampling plugin since LogDistiller 0.9" );
        }

        if ( attribute != null )
        {
            LogDistiller ld = group.getLogdistiller();
            ld.addWarning( context + " 'attribute' parameter deprecated in " + getId() + " plugin since LogDistiller 1.1: use 'attributes'" );
            if ( regexp != null )
            {
                ld.addWarning( context + " 'regexp' parameter deprecated in " + getId() + " plugin since LogDistiller 1.1: use extended attributes" );
            }
        }
    }

    private void readObject( ObjectInputStream in )
        throws IOException, ClassNotFoundException
    {
        in.defaultReadObject();
        updatePattern();
    }

    /** @deprecated since 1.1, use attributes instead */
    private void updatePattern()
    {
        try
        {
            regexp = ( regexpString == null ) ? null : Pattern.compile( regexpString );
        }
        catch ( PatternSyntaxException pse )
        {
            throw new PluginConfigException( "regexp parameter contains bad pattern '" + regexpString
                + "': caused an exception " + pse.getMessage() );
        }
    }

    public void begin( File destinationDirectory )
        throws FileNotFoundException
    {
    }

    public void addLogEvent( LogEvent logEvent )
        throws IOException
    {
        String[] values = extractAttributes( logEvent );

        String key = Freq.asKey( values );
        Freq freq = (Freq) freqs.get( key );
        if ( freq == null )
        {
            freq = new Freq( key, values );
            freqs.put( key, freq );
        }

        addLogEventToFreq( logEvent, freq );
    }

    private String[] extractAttributes( LogEvent logEvent )
    {
        if ( attributes == null )
        {
            return new String[]{ oldComputeValue( logEvent ) };
        }

        // initialize attributesInfo if necessary
        if ( attributesInfo == null )
        {
            attributesInfo = new LogType.AttributeInfo[attributes.length];
            LogType.Description description = logEvent.getFactory().getDescription();

            for ( int i = 0; i < attributes.length; i++ )
            {
                attributesInfo[i] = description.getAttributeInfo( attributes[i] );
            }
        }

        String[] values = new String[attributesInfo.length];
        for ( int i = 0; i < attributesInfo.length; i++ )
        {
            values[i] = logEvent.getValue( attributesInfo[i] );
        }
        return values;
    }

    /** @deprecated since 1.1, use attributes instead */
    private String oldComputeValue( LogEvent logEvent )
    {
        // initialize attributeInfo if necessary
        if ( attributeInfo == null )
        {
            LogType.Description description = logEvent.getFactory().getDescription();
            attributeInfo = description.getAttributeInfo( attribute );
            if ( StringUtils.isNotEmpty( attribute2 ) )
            {
                attributeInfo2 = description.getAttributeInfo( attribute2 );
                if ( StringUtils.isNotEmpty( attribute3 ) )
                {
                    attributeInfo3 = description.getAttributeInfo( attribute3 );
                }
            }
        }

        // calculate freq value
        String value = logEvent.getValue( attributeInfo );
        if ( attributeInfo2 != null )
        {
            value += '-' + logEvent.getValue( attributeInfo2 );
            if ( attributeInfo3 != null )
            {
                value += '-' + logEvent.getValue( attributeInfo3 );
            }
        }
        if ( regexp != null )
        {
            Matcher matcher = regexp.matcher( value );
            if ( matcher.find() && ( regexpGroup <= matcher.groupCount() ) )
            {
                if ( regexpGroup >= 0 )
                {
                    value = matcher.group( regexpGroup );
                }
                else if ( matcher.groupCount() > 1 )
                {
                    StringBuffer buff = new StringBuffer( matcher.group( 1 ) );
                    for ( int i = 2; i <= matcher.groupCount(); i++ )
                    {
                        buff.append( '-' ).append( matcher.group( i ) );
                    }
                    value = buff.toString();
                }
                else if ( matcher.groupCount() == 1 )
                {
                    value = matcher.group( 1 );
                }
                else
                {
                    value = matcher.group( 0 );
                }
            }
            else
            {
                value = "";
            }
            if ( value == null )
            {
                value = "";
            }
        }

        return value;
    }

    /**
     * @since 1.1
     */
    protected void addLogEventToFreq( LogEvent logEvent, Freq freq )
        throws IOException
    {
        // increase freq count
        freq.count++;
        int bytes = logEvent.getRawLog().length() + LogDistillation.LINE_SEPARATOR.length();
        freq.bytes += bytes;
    }

    public void end()
        throws IOException
    {
        FreqOrder comparator = SORT_ITEM_COMPARATORS[sortItem];
        comparator = reverseOrder ? new ReverseOrder( comparator ) : comparator;
        sorted = new TreeSet<Freq>( comparator );
        sorted.addAll( freqs.values() );
    }

    public void appendGroupReport( ReportFormat.PluginReport report )
    {
        appendReport( report, maxGroupReport );
    }

    public void appendGlobalReport( ReportFormat.PluginReport report )
    {
        appendReport( report, maxGlobalReport );
    }

    SortedSet<Freq> getSorted()
    {
        return sorted;
    }

    private String describeResult( int maxCount )
    {
        String count =
            ( ( maxCount < 0 ) || ( maxCount >= sorted.size() ) ) ? String.valueOf( sorted.size() )
                            : ( maxCount + "/" + sorted.size() );
        String attrs;
        if ( attributes == null )
        {
            attrs = attribute
                + ( ( attribute2 == null ) ? "" :
                    ( "'-'" + attribute2 + ( ( attribute3 == null ) ? "" : ( "'-'" + attribute3 ) ) ) );
        }
        else
        {
            attrs = StringUtils.join( attributes, "'-'" );
        }
        return "values of " + ( regexpString == null ? "" : "regexp on " ) + "attribute '" + attrs + "'"
            + " sorted by" + ( reverseOrder ? " reverse " : " " ) + SORT_ITEM_DESCRIPTIONS[sortItem] + " (" + count
            + " different values)";
    }

    private void addItems( ReportFormat.PluginReport report, int maxCount )
    {
        Iterator<Freq> iter = sorted.iterator();
        while ( iter.hasNext() && ( ( maxCount < 0 ) || ( maxCount-- > 0 ) ) )
        {
            Freq freq = iter.next();

            // value displayed in the report
            String value = ( freq.values == null ) ? freq.key : StringUtils.join( freq.values, '-' );

            report.addItem( freq.count, valueDescriptions.getProperty( value, value ) );
        }
    }

    private void appendReport( ReportFormat.PluginReport report, int maxCount )
    {
        report.beginPluginReport( this, describeResult( maxCount ) );
        appendLinkToFreqReport( report );
        addItems( report, maxCount );
        report.endPluginReport();
    }

    /**
     * Append links to freq reports, for freq extensions.
     *
     * @param report the report formatter
     */
    protected void appendLinkToFreqReport( ReportFormat.PluginReport report )
    {
        // no additionnal link for freq: it's intended for subclasses
    }

    public static class Freq
        implements Serializable
    {
        private static final long serialVersionUID = 9195786567021952968L;

        public final String key;

        public final String[] values;

        public int count = 0;

        public long bytes = 0;

        public Freq( String key, String[] values )
        {
            this.key = key;
            this.values = values;

            // get values Strings as substring from key, to avoid keeping a pointer to the initial log String
            int index = 0;
            for ( int i = 0; i < values.length; i++ )
            {
                int start = index;
                index += values[i].length();
                values[i] = key.substring( start, index );
                index++;
            }
        }

        public static String asKey( String[] values )
        {
            return StringUtils.join( values, '\n' );
        }
    }

    private abstract static class FreqOrder
        implements Comparator<Freq>, Serializable
    {
        private static final long serialVersionUID = -2574387085671510158L;
    }

    /**
     * Order on freq count, then on key when count are equal.
     */
    private static class CountOrder
        extends FreqOrder
    {
        private static final long serialVersionUID = -933179294932120994L;

        public int compare( Freq freq1, Freq freq2 )
        {
            int diff = freq2.count - freq1.count;
            return ( diff == 0 ) ? freq1.key.compareTo( freq2.key ) : diff;
        }
    }

    /**
     * Order on key (lexical order).
     */
    private static class ValueOrder
        extends FreqOrder
    {
        private static final long serialVersionUID = 1824757961838977443L;

        public int compare( Freq freq1, Freq freq2 )
        {
            return freq1.key.compareTo( freq2.key );
        }
    }

    /**
     * Order on key length, then key lexical order.
     * example: 1234 &gt; 234 &gt; 233 &gt; A5 &gt; 45
     */
    private static class ValueLengthOrder
        extends FreqOrder
    {
        private static final long serialVersionUID = 651766724239811116L;

        public int compare( Freq freq1, Freq freq2 )
        {
            int diff = freq2.key.length() - freq1.key.length();
            return ( diff == 0 ) ? freq2.key.compareTo( freq1.key ) : diff;
        }
    }

    private static class ReverseOrder
        extends FreqOrder
    {
        private static final long serialVersionUID = 539304109704333616L;

        private final FreqOrder comparator;

        public ReverseOrder( FreqOrder comparator )
        {
            this.comparator = comparator;
        }

        public int compare( Freq freq1, Freq freq2 )
        {
            return comparator.compare( freq2, freq1 );
        }
    }

    private static class Type
        extends Plugin
    {
        public Type()
        {
            super( ID );
        }

        public LogDistillation.Plugin newInstance( LogDistiller.Plugin conf )
        {
            return new FreqPlugin( conf );
        }
    }
}