    001/*
    002    Licensed to the Apache Software Foundation (ASF) under one
    003    or more contributor license agreements.  See the NOTICE file
    004    distributed with this work for additional information
    005    regarding copyright ownership.  The ASF licenses this file
    006    to you under the Apache License, Version 2.0 (the
    007    "License"); you may not use this file except in compliance
    008    with the License.  You may obtain a copy of the License at
    009
    010       http://www.apache.org/licenses/LICENSE-2.0
    011
    012    Unless required by applicable law or agreed to in writing,
    013    software distributed under the License is distributed on an
    014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    015    KIND, either express or implied.  See the License for the
    016    specific language governing permissions and limitations
    017    under the License.
    018 */
    019package org.apache.wiki.filters;
    020
    021import com.github.difflib.DiffUtils;
    022import com.github.difflib.patch.AbstractDelta;
    023import com.github.difflib.patch.Patch;
    024import org.apache.commons.lang3.StringUtils;
    025import org.apache.commons.lang3.time.StopWatch;
    026import org.apache.logging.log4j.LogManager;
    027import org.apache.logging.log4j.Logger;
    028import org.apache.oro.text.regex.MalformedPatternException;
    029import org.apache.oro.text.regex.MatchResult;
    030import org.apache.oro.text.regex.Pattern;
    031import org.apache.oro.text.regex.PatternCompiler;
    032import org.apache.oro.text.regex.PatternMatcher;
    033import org.apache.oro.text.regex.Perl5Compiler;
    034import org.apache.oro.text.regex.Perl5Matcher;
    035import org.apache.wiki.InternalWikiException;
    036import org.apache.wiki.api.core.Attachment;
    037import org.apache.wiki.api.core.Context;
    038import org.apache.wiki.api.core.ContextEnum;
    039import org.apache.wiki.api.core.Engine;
    040import org.apache.wiki.api.core.Page;
    041import org.apache.wiki.api.exceptions.ProviderException;
    042import org.apache.wiki.api.exceptions.RedirectException;
    043import org.apache.wiki.api.filters.BasePageFilter;
    044import org.apache.wiki.api.providers.WikiProvider;
    045import org.apache.wiki.attachment.AttachmentManager;
    046import org.apache.wiki.auth.user.UserProfile;
    047import org.apache.wiki.pages.PageManager;
    048import org.apache.wiki.ui.EditorManager;
    049import org.apache.wiki.util.FileUtil;
    050import org.apache.wiki.util.HttpUtil;
    051import org.apache.wiki.util.TextUtil;
    052
    053import jakarta.servlet.http.HttpServletRequest;
    054import jakarta.servlet.http.HttpServletResponse;
    055import jakarta.servlet.jsp.PageContext;
    056import java.io.BufferedReader;
    057import java.io.IOException;
    058import java.io.InputStream;
    059import java.io.InputStreamReader;
    060import java.io.StringReader;
    061import java.io.StringWriter;
    062import java.nio.charset.StandardCharsets;
    063import java.util.ArrayList;
    064import java.util.Arrays;
    065import java.util.Collection;
    066import java.util.Date;
    067import java.util.Iterator;
    068import java.util.List;
    069import java.util.Properties;
    070import java.util.Random;
    071import java.util.StringTokenizer;
    072import java.util.Vector;
    073import java.util.concurrent.ThreadLocalRandom;
    074import net.thauvin.erik.akismet.Akismet;
    075import net.thauvin.erik.akismet.AkismetComment;
    076
    077
    078/**
    079 *  This is Herb, the JSPWiki spamfilter that can also do choke modifications.
    080 *
    081 *  Parameters:
    082 *  <ul>
    083 *    <li>wordlist - Page name where the spamword regexps are found.  Use [{SET spamwords='regexp list separated with spaces'}] on
    084 *     that page.  Default is "SpamFilterWordList".
    085 *    <li>IPlist - Page name where the IP regexps are found.  Use [{SET ips='regexp list separated with spaces'}] on
    086 *     that page.  Default is "SpamFilterIPList".
    087 *    <li>maxpagenamelength - Maximum page name length. Default is 100.
    088 *    <li>blacklist - The name of an attachment containing the list of spam patterns, one per line. Default is
    089 *        "SpamFilterWordList/blacklist.txt"</li>
    090 *    <li>errorpage - The page to which the user is redirected.  Has a special variable $msg which states the reason. Default is "RejectedMessage".
    091 *    <li>pagechangesinminute - How many page changes are allowed/minute.  Default is 5.</li>
    092 *    <li>similarchanges - How many similar page changes are allowed before the host is banned.  Default is 2.  (since 2.4.72)</li>
    093 *    <li>bantime - How long an IP address stays on the temporary ban list (default is 60 for 60 minutes).</li>
    094 *    <li>maxurls - How many URLs can be added to the page before it is considered spam (default is 5)</li>
    095 *    <li>akismet-apikey - The Akismet API key (see akismet.org)</li>
    096 *    <li>ignoreauthenticated - If set to "true", all authenticated users are ignored and never caught in SpamFilter</li>
    097 *    <li>captcha - Sets the captcha technology to use.  Current allowed values are "none". "asirra" was previously supported however that service has been discontinued.</li>
    098 *    <li>strategy - Sets the filtering strategy to use.  If set to "eager", will stop at the first probable
    099 *        match, and won't consider any other tests.  This is the default, as it's considerably lighter. If set to "score", will go through all of the tests
    100 *        and calculates a score for the spam, which is then compared to a filter level value.
    101 *  </ul>
    102 *
    103 *  <p>Please see the default editors/plain.jsp for examples on how the SpamFilter integrates
    104 *  with the editor system.</p>
    105 *
    106 *  <p>Changes by admin users are ignored in any case.</p>
    107 *
    108 *  @since 2.1.112
    109 */
    110public class SpamFilter extends BasePageFilter {
    111
    112    private static final String ATTR_SPAMFILTER_SCORE = "spamfilter.score";
    113    private static final String REASON_REGEXP = "Regexp";
    114    private static final String REASON_IP_BANNED_TEMPORARILY = "IPBannedTemporarily";
    115    private static final String REASON_IP_BANNED_PERMANENTLY = "IPBannedPermanently";
    116    private static final String REASON_BOT_TRAP = "BotTrap";
    117    private static final String REASON_AKISMET = "Akismet";
    118    private static final String REASON_TOO_MANY_URLS = "TooManyUrls";
    119    private static final String REASON_SIMILAR_MODIFICATIONS = "SimilarModifications";
    120    private static final String REASON_TOO_MANY_MODIFICATIONS = "TooManyModifications";
    121    private static final String REASON_PAGENAME_TOO_LONG = "PageNameTooLong";
    122    private static final String REASON_UTF8_TRAP = "UTF8Trap";
    123
    124    private static final String LISTVAR = "spamwords";
    125    private static final String LISTIPVAR = "ips";
    126
    127    private static final Random RANDOM = ThreadLocalRandom.current();
    128
    129    /** The filter property name for specifying the page which contains the list of spamwords. Value is <tt>{@value}</tt>. */
    130    public static final String  PROP_WORDLIST              = "wordlist";
    131
    132    /** The filter property name for specifying the page which contains the list of IPs to ban. Value is <tt>{@value}</tt>. */
    133    public static final String  PROP_IPLIST                = "IPlist";
    134
    135    /** The filter property name for specifying the maximum page name length.  Value is <tt>{@value}</tt>. */
    136    public static final String  PROP_MAX_PAGENAME_LENGTH   = "maxpagenamelength";
    137
    138    /** The filter property name for the page to which you are directed if Herb rejects your edit.  Value is <tt>{@value}</tt>. */
    139    public static final String  PROP_ERRORPAGE             = "errorpage";
    140
    141    /** The filter property name for specifying how many changes is any given IP address
    142     *  allowed to do per minute.  Value is <tt>{@value}</tt>.
    143     */
    144    public static final String  PROP_PAGECHANGES           = "pagechangesinminute";
    145
    146    /** The filter property name for specifying how many similar changes are allowed before a host is banned.  Value is <tt>{@value}</tt>. */
    147    public static final String  PROP_SIMILARCHANGES        = "similarchanges";
    148
    149    /** The filter property name for specifying how long a host is banned.  Value is <tt>{@value}</tt>.*/
    150    public static final String  PROP_BANTIME               = "bantime";
    151
    152    /** The filter property name for the attachment containing the blacklist.  Value is <tt>{@value}</tt>.*/
    153    public static final String  PROP_BLACKLIST             = "blacklist";
    154
    155    /** The filter property name for specifying how many URLs can any given edit contain. Value is <tt>{@value}</tt> */
    156    public static final String  PROP_MAXURLS               = "maxurls";
    157
    158    /** The filter property name for specifying the Akismet API-key.  Value is <tt>{@value}</tt>. */
    159    public static final String  PROP_AKISMET_API_KEY       = "akismet-apikey";
    160
    161    /** The filter property name for specifying whether authenticated users should be ignored. Value is <tt>{@value}</tt>. */
    162    public static final String  PROP_IGNORE_AUTHENTICATED  = "ignoreauthenticated";
    163
    164    /** The filter property name for specifying groups allowed to bypass the spam filter. Value is <tt>{@value}</tt>. */
    165    public static final String PROP_ALLOWED_GROUPS = "jspwiki.filters.spamfilter.allowedgroups";
    166
    167    /** The filter property name for specifying which captcha technology should be used. Value is <tt>{@value}</tt>. */
    168    public static final String  PROP_CAPTCHA               = "captcha";
    169
    170    /** The filter property name for specifying which filter strategy should be used.  Value is <tt>{@value}</tt>. */
    171    public static final String  PROP_FILTERSTRATEGY        = "strategy";
    172
    173    /** The string specifying the "eager" strategy. Value is <tt>{@value}</tt>. */
    174    public static final String  STRATEGY_EAGER             = "eager";
    175
    176    /** The string specifying the "score" strategy. Value is <tt>{@value}</tt>. */
    177    public static final String  STRATEGY_SCORE             = "score";
    178
    179    private static final String URL_REGEXP = "(http://|https://|mailto:)([A-Za-z0-9_/\\.\\+\\?\\#\\-\\@=&;]+)";
    180
    181    private String          m_forbiddenWordsPage = "SpamFilterWordList";
    182    private String          m_forbiddenIPsPage   = "SpamFilterIPList";
    183    private String          m_pageNameMaxLength  = "100";
    184    private String          m_errorPage          = "RejectedMessage";
    185    private String          m_blacklist          = "SpamFilterWordList/blacklist.txt";
    186
    187    private final PatternMatcher  m_matcher = new Perl5Matcher();
    188    private final PatternCompiler m_compiler = new Perl5Compiler();
    189
    190    private Collection<Pattern> m_spamPatterns;
    191    private Collection<Pattern> m_IPPatterns;
    192
    193    private Date m_lastRebuild = new Date( 0L );
    194
    195    private static final Logger C_SPAMLOG = LogManager.getLogger( "SpamLog" );
    196    private static final Logger LOG = LogManager.getLogger( SpamFilter.class );
    197
    198    private final Vector<Host>    m_temporaryBanList = new Vector<>();
    199
    200    private int             m_banTime = 60; // minutes
    201
    202    private final Vector<Host>    m_lastModifications = new Vector<>();
    203
    204    /** How many times a single IP address can change a page per minute? */
    205    private int             m_limitSinglePageChanges = 5;
    206
    207    /** How many times can you add the exact same string to a page? */
    208    private int             m_limitSimilarChanges = 2;
    209
    210    /** How many URLs can be added at maximum. */
    211    private int             m_maxUrls = 10;
    212
    213    private Pattern         m_urlPattern;
    214    private Akismet         m_akismet;
    215
    216    private String          m_akismetAPIKey;
    217
    218    /** The limit at which we consider something to be spam. */
    219    private final int             m_scoreLimit = 1;
    220
    221    /** If set to true, will ignore anyone who is in Authenticated role. */
    222    private boolean         m_ignoreAuthenticated;
    223
    224    /** Groups allowed to bypass the filter */
    225    private String[]         m_allowedGroups;
    226
    227    private boolean         m_stopAtFirstMatch = true;
    228
    229    private static String   c_hashName;
    230    private static long     c_lastUpdate;
    231
    232    /** The HASH_DELAY value is a maximum amount of time that an user can keep
    233     *  a session open, because after the value has expired, we will invent a new
    234     *  hash field name.  By default this is {@value} hours, which should be ample
    235     *  time for someone.
    236     */
    237    private static final long HASH_DELAY = 24;
    238
    239
    240    /**
    241     *  {@inheritDoc}
    242     */
    243    @Override
    244    public void initialize( final Engine engine, final Properties properties ) {
    245        m_forbiddenWordsPage = properties.getProperty( PROP_WORDLIST, m_forbiddenWordsPage );
    246        m_forbiddenIPsPage = properties.getProperty( PROP_IPLIST, m_forbiddenIPsPage);
    247        m_pageNameMaxLength = properties.getProperty( PROP_MAX_PAGENAME_LENGTH, m_pageNameMaxLength);
    248        m_errorPage = properties.getProperty( PROP_ERRORPAGE, m_errorPage );
    249        m_limitSinglePageChanges = TextUtil.getIntegerProperty( properties, PROP_PAGECHANGES, m_limitSinglePageChanges );
    250
    251        m_limitSimilarChanges = TextUtil.getIntegerProperty( properties, PROP_SIMILARCHANGES, m_limitSimilarChanges );
    252
    253        m_maxUrls = TextUtil.getIntegerProperty( properties, PROP_MAXURLS, m_maxUrls );
    254        m_banTime = TextUtil.getIntegerProperty( properties, PROP_BANTIME, m_banTime );
    255        m_blacklist = properties.getProperty( PROP_BLACKLIST, m_blacklist );
    256
    257        m_ignoreAuthenticated = TextUtil.getBooleanProperty( properties, PROP_IGNORE_AUTHENTICATED, m_ignoreAuthenticated );
    258        m_allowedGroups = StringUtils.split( StringUtils.defaultString( properties.getProperty( PROP_ALLOWED_GROUPS, m_blacklist ) ), ',' );
    259
    260        try {
    261            m_urlPattern = m_compiler.compile( URL_REGEXP );
    262        } catch( final MalformedPatternException e ) {
    263            LOG.fatal( "Internal error: Someone put in a faulty pattern.", e );
    264            throw new InternalWikiException( "Faulty pattern." , e);
    265        }
    266
    267        m_akismetAPIKey = TextUtil.getStringProperty( properties, PROP_AKISMET_API_KEY, m_akismetAPIKey );
    268        m_stopAtFirstMatch = TextUtil.getStringProperty( properties, PROP_FILTERSTRATEGY, STRATEGY_EAGER ).equals( STRATEGY_EAGER );
    269
    270        LOG.info( "# Spam filter initialized.  Temporary ban time " + m_banTime +
    271                  " mins, max page changes/minute: " + m_limitSinglePageChanges );
    272    }
    273
    274    private static final int REJECT = 0;
    275    private static final int ACCEPT = 1;
    276    private static final int NOTE   = 2;
    277
    278    private static String log( final Context ctx, final int type, final String source, String message ) {
    279        message = TextUtil.replaceString( message, "\r\n", "\\r\\n" );
    280        message = TextUtil.replaceString( message, "\"", "\\\"" );
    281
    282        final String uid = getUniqueID();
    283        final String page   = ctx.getPage().getName();
    284        final String addr   = ctx.getHttpRequest() != null ? HttpUtil.getRemoteAddress( ctx.getHttpRequest() ) : "-";
    285        final String reason;
    286        switch( type ) {
    287            case REJECT: reason = "REJECTED";
    288                break;
    289            case ACCEPT: reason = "ACCEPTED";
    290                break;
    291            case NOTE: reason = "NOTE";
    292                break;
    293            default: throw new InternalWikiException( "Illegal type " + type );
    294        }
    295        C_SPAMLOG.info( reason + " " + source + " " + uid + " " + addr + " \"" + page + "\" " + message );
    296
    297        return uid;
    298    }
    299
    300    /** {@inheritDoc} */
    301    @Override
    302    public String preSave( final Context context, final String content ) throws RedirectException {
    303        cleanBanList();
    304        refreshBlacklists( context );
    305        final Change change = getChange( context, content );
    306
    307        if( !ignoreThisUser( context ) ) {
    308            checkBanList( context, change );
    309            checkSinglePageChange( context, change );
    310            checkIPList( context );
    311            checkPatternList( context, change );
    312            checkPageName( context);
    313        }
    314
    315        if( !m_stopAtFirstMatch ) {
    316            final Integer score = context.getVariable( ATTR_SPAMFILTER_SCORE );
    317
    318            if( score != null && score >= m_scoreLimit ) {
    319                throw new RedirectException( "Herb says you got too many points", getRedirectPage( context ) );
    320            }
    321        }
    322
    323        log( context, ACCEPT, "-", change.toString() );
    324        return content;
    325    }
    326
    327    private void checkPageName(final Context context ) throws RedirectException {
    328        final Page page = context.getPage();
    329        final String pageName = page.getName();
    330        final int maxlength = Integer.parseInt(m_pageNameMaxLength);
    331        if ( pageName.length() > maxlength) {
    332            //
    333            //  Spam filter has a match.
    334            //
    335
    336            final String uid = log( context, REJECT, REASON_PAGENAME_TOO_LONG + "(" + m_pageNameMaxLength + ")" , pageName);
    337
    338            LOG.info("SPAM:PageNameTooLong (" + uid + "). The length of the page name is too large (" + pageName.length() + " , limit is " + m_pageNameMaxLength + ")");
    339            checkStrategy( context, "Herb says '" + pageName + "' is a bad pageName and I trust Herb! (Incident code " + uid + ")" );
    340
    341        }
    342    }
    343
    344    private void checkStrategy(final Context context, final String message ) throws RedirectException {
    345        if( m_stopAtFirstMatch ) {
    346            throw new RedirectException( message, getRedirectPage( context ) );
    347        }
    348
    349        Integer score = context.getVariable( ATTR_SPAMFILTER_SCORE );
    350        if( score != null ) {
    351            score = score + 1;
    352        } else {
    353            score = 1;
    354        }
    355
    356        context.setVariable( ATTR_SPAMFILTER_SCORE, score );
    357    }
    358
    359    /**
    360     *  Parses a list of patterns and returns a Collection of compiled Pattern objects.
    361     *
    362     * @param source page containing the list of patterns.
    363     * @param list list of patterns.
    364     * @return A Collection of the Patterns that were found from the lists.
    365     */
    366    private Collection< Pattern > parseWordList( final Page source, final String list ) {
    367        final ArrayList< Pattern > compiledpatterns = new ArrayList<>();
    368
    369        if( list != null ) {
    370            final StringTokenizer tok = new StringTokenizer( list, " \t\n" );
    371
    372            while( tok.hasMoreTokens() ) {
    373                final String pattern = tok.nextToken();
    374
    375                try {
    376                    compiledpatterns.add( m_compiler.compile( pattern ) );
    377                } catch( final MalformedPatternException e ) {
    378                    LOG.debug( "Malformed spam filter pattern " + pattern );
    379                    source.setAttribute("error", "Malformed spam filter pattern " + pattern);
    380                }
    381            }
    382        }
    383
    384        return compiledpatterns;
    385    }
    386
    387    /**
    388     *  Takes a MT-Blacklist -formatted blacklist and returns a list of compiled Pattern objects.
    389     *
    390     *  @param list list of patterns.
    391     *  @return The parsed blacklist patterns.
    392     */
    393    private Collection< Pattern > parseBlacklist( final String list ) {
    394        final ArrayList< Pattern > compiledpatterns = new ArrayList<>();
    395
    396        if( list != null ) {
    397            try {
    398                final BufferedReader in = new BufferedReader( new StringReader(list) );
    399                String line;
    400                while( (line = in.readLine() ) != null ) {
    401                    line = line.trim();
    402                    if( line.isEmpty() ) continue; // Empty line
    403                    if( line.startsWith("#") ) continue; // It's a comment
    404
    405                    int ws = line.indexOf( ' ' );
    406                    if( ws == -1 ) ws = line.indexOf( '\t' );
    407                    if( ws != -1 ) line = line.substring( 0, ws );
    408
    409                    try {
    410                        compiledpatterns.add( m_compiler.compile( line ) );
    411                    } catch( final MalformedPatternException e ) {
    412                        LOG.debug( "Malformed spam filter pattern " + line );
    413                    }
    414                }
    415            } catch( final IOException e ) {
    416                LOG.info( "Could not read patterns; returning what I got" , e );
    417            }
    418        }
    419
    420        return compiledpatterns;
    421    }
    422
    423    /**
    424     * Takes a single page change and performs a load of tests on the content change. An admin can modify anything.
    425     *
    426     * @param context page Context
    427     * @param change page change
    428     * @throws RedirectException spam filter rejects the page change.
    429     */
    430    private synchronized void checkSinglePageChange(final Context context, final Change change )
    431            throws RedirectException {
    432        final HttpServletRequest req = context.getHttpRequest();
    433
    434        if( req != null ) {
    435            final String addr = HttpUtil.getRemoteAddress( req );
    436            int hostCounter = 0;
    437            int changeCounter = 0;
    438
    439            LOG.debug( "Change is " + change.m_change );
    440
    441            final long time = System.currentTimeMillis() - 60*1000L; // 1 minute
    442
    443            for( final Iterator< Host > i = m_lastModifications.iterator(); i.hasNext(); ) {
    444                final Host host = i.next();
    445
    446                //  Check if this item is invalid
    447                if( host.getAddedTime() < time ) {
    448                    LOG.debug( "Removed host " + host.getAddress() + " from modification queue (expired)" );
    449                    i.remove();
    450                    continue;
    451                }
    452
    453                // Check if this IP address has been seen before
    454                if( host.getAddress().equals( addr ) ) {
    455                    hostCounter++;
    456                }
    457
    458                //  Check, if this change has been seen before
    459                if( host.getChange() != null && host.getChange().equals( change ) ) {
    460                    changeCounter++;
    461                }
    462            }
    463
    464            //  Now, let's check against the limits.
    465            if( hostCounter >= m_limitSinglePageChanges ) {
    466                final Host host = new Host( addr, null );
    467                m_temporaryBanList.add( host );
    468
    469                final String uid = log( context, REJECT, REASON_TOO_MANY_MODIFICATIONS, change.m_change );
    470                LOG.info( "SPAM:TooManyModifications (" + uid + "). Added host " + addr + " to temporary ban list for doing too many modifications/minute" );
    471                checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code " + uid + ")" );
    472            }
    473
    474            if( changeCounter >= m_limitSimilarChanges ) {
    475                final Host host = new Host( addr, null );
    476                m_temporaryBanList.add( host );
    477
    478                final String uid = log( context, REJECT, REASON_SIMILAR_MODIFICATIONS, change.m_change );
    479                LOG.info( "SPAM:SimilarModifications (" + uid + "). Added host " + addr + " to temporary ban list for doing too many similar modifications" );
    480                checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code "+uid+")");
    481            }
    482
    483            //  Calculate the number of links in the addition.
    484            String tstChange  = change.toString();
    485            int urlCounter = 0;
    486            while( m_matcher.contains( tstChange,m_urlPattern ) ) {
    487                final MatchResult m = m_matcher.getMatch();
    488                tstChange = tstChange.substring( m.endOffset(0) );
    489                urlCounter++;
    490            }
    491
    492            if( urlCounter > m_maxUrls ) {
    493                final Host host = new Host( addr, null );
    494                m_temporaryBanList.add( host );
    495
    496                final String uid = log( context, REJECT, REASON_TOO_MANY_URLS, change.toString() );
    497                LOG.info( "SPAM:TooManyUrls (" + uid + "). Added host " + addr + " to temporary ban list for adding too many URLs" );
    498                checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code " + uid + ")" );
    499            }
    500
    501            //  Check bot trap
    502            checkBotTrap( context, change );
    503
    504            //  Check UTF-8 mangling
    505            checkUTF8( context, change );
    506
    507            //  Do Akismet check.  This is good to be the last, because this is the most expensive operation.
    508            checkAkismet( context, change );
    509
    510            m_lastModifications.add( new Host( addr, change ) );
    511        }
    512    }
    513
    514
    515    /**
    516     *  Checks against the akismet system.
    517     *
    518     * @param context page Context
    519     * @throws RedirectException spam filter rejects the page change.
    520     */
    521    private void checkAkismet( final Context context, final Change change ) throws RedirectException {
    522        if( m_akismetAPIKey != null ) {
    523            if( m_akismet == null ) {
    524                LOG.info( "Initializing Akismet spam protection." );
    525                String fullPageUrl = context.getHttpRequest().getRequestURL().toString();
    526                String fragment = context.getEngine().getBaseURL();
    527                fullPageUrl = fullPageUrl.substring(0, fullPageUrl.indexOf(fragment) + fragment.length());
    528                m_akismet = new Akismet( m_akismetAPIKey, fullPageUrl );
    529
    530                if( !m_akismet.verifyKey() ) {
    531                    LOG.error( "Akismet API key cannot be verified.  Please check your config." );
    532                    m_akismetAPIKey = null;
    533                    m_akismet = null;
    534                }
    535            }
    536
    537            final HttpServletRequest req = context.getHttpRequest();
    538
    539            //  Akismet will mark all empty statements as spam, so we'll just ignore them.
    540            if( change.m_adds == 0 && change.m_removals > 0 ) {
    541                return;
    542            }
    543
    544            if( req != null && m_akismet != null ) {
    545                LOG.debug( "Calling Akismet to check for spam..." );
    546
    547                final StopWatch sw = new StopWatch();
    548                sw.start();
    549
    550                final String ipAddress     = HttpUtil.getRemoteAddress( req );
    551                final String userAgent     = req.getHeader( "User-Agent" );
    552                final String referrer      = req.getHeader( "Referer");
    553                final String permalink     = context.getViewURL( context.getPage().getName() );
    554                final String commentType   = context.getRequestContext().equals( ContextEnum.PAGE_COMMENT.getRequestContext() ) ? "comment" : "edit";
    555                final String commentAuthor = context.getCurrentUser().getName();
    556                final String commentAuthorEmail = null;
    557                final String commentAuthorURL   = null;
    558                AkismetComment comment = new AkismetComment(ipAddress, userAgent);
    559                comment.setAuthor(commentAuthor);
    560                comment.setAuthorEmail(commentAuthorEmail);
    561                comment.setAuthorUrl(commentAuthorURL);
    562                comment.setContent(change.toString());
    563                comment.setPermalink(permalink);
    564                comment.setReferrer(referrer);
    565                comment.setType(commentType);
    566
    567                final boolean isSpam = m_akismet.checkComment(comment);
    568
    569                sw.stop();
    570                LOG.debug( "Akismet request done in: " + sw );
    571
    572                if( isSpam ) {
    573                    // Host host = new Host( ipAddress, null );
    574                    // m_temporaryBanList.add( host );
    575
    576                    final String uid = log( context, REJECT, REASON_AKISMET, change.toString() );
    577                    LOG.info( "SPAM:Akismet (" + uid + "). Akismet thinks this change is spam; added host to temporary ban list." );
    578                    checkStrategy( context, "Akismet tells Herb you're a spammer, Herb trusts Akismet, and I trust Herb! (Incident code " + uid + ")" );
    579                }
    580            }
    581        }
    582    }
    583
    584    /**
    585     * Returns a static string which can be used to detect spambots which just wildly fill in all the fields.
    586     *
    587     * @return A string
    588     */
    589    public static String getBotFieldName() {
    590        return "submit_auth";
    591    }
    592
    593    /**
    594     * This checks whether an invisible field is available in the request, and whether it's contents are suspected spam.
    595     *
    596     * @param context page Context
    597     * @param change page change
    598     * @throws RedirectException spam filter rejects the page change.
    599     */
    600    private void checkBotTrap( final Context context, final Change change ) throws RedirectException {
    601        final HttpServletRequest request = context.getHttpRequest();
    602        if( request != null ) {
    603            final String unspam = request.getParameter( getBotFieldName() );
    604            if( unspam != null && !unspam.isEmpty() ) {
    605                final String uid = log( context, REJECT, REASON_BOT_TRAP, change.toString() );
    606
    607                LOG.info( "SPAM:BotTrap (" + uid + ").  Wildly behaving bot detected." );
    608                checkStrategy( context, "Spamming attempt detected. (Incident code " + uid + ")" );
    609            }
    610        }
    611    }
    612
    613    private void checkUTF8( final Context context, final Change change ) throws RedirectException {
    614        final HttpServletRequest request = context.getHttpRequest();
    615        if( request != null ) {
    616            final String utf8field = request.getParameter( "encodingcheck" );
    617            if( utf8field != null && !utf8field.equals( "\u3041" ) ) {
    618                final String uid = log( context, REJECT, REASON_UTF8_TRAP, change.toString() );
    619
    620                LOG.info( "SPAM:UTF8Trap (" + uid + ").  Wildly posting dumb bot detected." );
    621                checkStrategy( context, "Spamming attempt detected. (Incident code " + uid + ")" );
    622            }
    623        }
    624    }
    625
    626    /** Goes through the ban list and cleans away any host which has expired from it. */
    627    private synchronized void cleanBanList() {
    628        final long now = System.currentTimeMillis();
    629        for( final Iterator< Host > i = m_temporaryBanList.iterator(); i.hasNext(); ) {
    630            final Host host = i.next();
    631
    632            if( host.getReleaseTime() < now ) {
    633                LOG.debug( "Removed host " + host.getAddress() + " from temporary ban list (expired)" );
    634                i.remove();
    635            }
    636        }
    637    }
    638
    639    /**
    640     *  Checks the ban list if the IP address of the changer is already on it.
    641     *
    642     *  @param context page context
    643     *  @throws RedirectException spam filter rejects the page change.
    644     */
    645    private void checkBanList( final Context context, final Change change ) throws RedirectException {
    646        final HttpServletRequest req = context.getHttpRequest();
    647
    648        if( req != null ) {
    649            final String remote = HttpUtil.getRemoteAddress(req);
    650            final long now = System.currentTimeMillis();
    651
    652            for( final Host host : m_temporaryBanList ) {
    653                if( host.getAddress().equals( remote ) ) {
    654                    final long timeleft = ( host.getReleaseTime() - now ) / 1000L;
    655
    656                    log( context, REJECT, REASON_IP_BANNED_TEMPORARILY, change.m_change );
    657                    checkStrategy( context,
    658                            "You have been temporarily banned from modifying this wiki. (" + timeleft + " seconds of ban left)" );
    659                }
    660            }
    661        }
    662    }
    663
    664    /**
    665     *  If the spam filter notices changes in the black list page, it will refresh them automatically.
    666     *
    667     *  @param context associated WikiContext
    668     */
    669    private void refreshBlacklists( final Context context ) {
    670        try {
    671            boolean rebuild = false;
    672
    673            //  Rebuild, if the spam words page, the attachment or the IP ban page has changed since.
    674            final Page sourceSpam = context.getEngine().getManager( PageManager.class ).getPage( m_forbiddenWordsPage );
    675            if( sourceSpam != null ) {
    676                if( m_spamPatterns == null || m_spamPatterns.isEmpty() || sourceSpam.getLastModified().after( m_lastRebuild ) ) {
    677                    rebuild = true;
    678                }
    679            }
    680
    681            final Attachment att = context.getEngine().getManager( AttachmentManager.class ).getAttachmentInfo( context, m_blacklist );
    682            if( att != null ) {
    683                if( m_spamPatterns == null || m_spamPatterns.isEmpty() || att.getLastModified().after( m_lastRebuild ) ) {
    684                    rebuild = true;
    685                }
    686            }
    687
    688            final Page sourceIPs = context.getEngine().getManager( PageManager.class ).getPage( m_forbiddenIPsPage );
    689            if( sourceIPs != null ) {
    690                if( m_IPPatterns == null || m_IPPatterns.isEmpty() || sourceIPs.getLastModified().after( m_lastRebuild ) ) {
    691                    rebuild = true;
    692                }
    693            }
    694
    695            //  Do the actual rebuilding.  For simplicity's sake, we always rebuild the complete filter list regardless of what changed.
    696            if( rebuild ) {
    697                m_lastRebuild = new Date();
    698                m_spamPatterns = parseWordList( sourceSpam, ( sourceSpam != null ) ? sourceSpam.getAttribute( LISTVAR ) : null );
    699
    700                LOG.info( "Spam filter reloaded - recognizing " + m_spamPatterns.size() + " patterns from page " + m_forbiddenWordsPage );
    701
    702                m_IPPatterns = parseWordList( sourceIPs,  ( sourceIPs != null ) ? sourceIPs.getAttribute( LISTIPVAR ) : null );
    703                LOG.info( "IP filter reloaded - recognizing " + m_IPPatterns.size() + " patterns from page " + m_forbiddenIPsPage );
    704
    705                if( att != null ) {
    706                    final InputStream in = context.getEngine().getManager( AttachmentManager.class ).getAttachmentStream(att);
    707                    final StringWriter out = new StringWriter();
    708                    FileUtil.copyContents( new InputStreamReader( in, StandardCharsets.UTF_8 ), out );
    709                    final Collection< Pattern > blackList = parseBlacklist( out.toString() );
    710                    LOG.info( "...recognizing additional " + blackList.size() + " patterns from blacklist " + m_blacklist );
    711                    m_spamPatterns.addAll( blackList );
    712                }
    713            }
    714        } catch( final IOException ex ) {
    715            LOG.info( "Unable to read attachment data, continuing...", ex );
    716        } catch( final ProviderException ex ) {
    717            LOG.info( "Failed to read spam filter attachment, continuing...", ex );
    718        }
    719    }
    720
    721    /**
    722     * Does a check against a known pattern list.
    723     *
    724     * @param context page Context
    725     * @param change page change
    726     * @throws RedirectException spam filter rejects the page change.
    727     */
    728    private void checkPatternList( final Context context, final Change change ) throws RedirectException {
    729        // If we have no spam patterns defined, or we're trying to save the page containing the patterns, just return.
    730        if( m_spamPatterns == null || context.getPage().getName().equals( m_forbiddenWordsPage ) ) {
    731            return;
    732        }
    733
    734        String ch = change.toString();
    735        if( context.getHttpRequest() != null ) {
    736            ch += HttpUtil.getRemoteAddress( context.getHttpRequest() );
    737        }
    738
    739        for( final Pattern p : m_spamPatterns ) {
    740            // LOG.debug("Attempting to match page contents with "+p.getPattern());
    741
    742            if( m_matcher.contains( ch, p ) ) {
    743                //  Spam filter has a match.
    744                final String uid = log( context, REJECT, REASON_REGEXP + "(" + p.getPattern() + ")", ch );
    745
    746                LOG.info( "SPAM:Regexp (" + uid + "). Content matches the spam filter '" + p.getPattern() + "'" );
    747                checkStrategy( context, "Herb says '" + p.getPattern() + "' is a bad spam word and I trust Herb! (Incident code " + uid + ")" );
    748            }
    749        }
    750    }
    751
    752
    753    /**
    754     *  Does a check against a pattern list of IPs.
    755     *
    756     *  @param context page context
    757     *  @throws RedirectException spam filter rejects the page change.
    758     */
    759    private void checkIPList( final Context context ) throws RedirectException {
    760        //  If we have no IP patterns defined, or we're trying to save the page containing the IP patterns, just return.
    761        if( m_IPPatterns == null || context.getPage().getName().equals( m_forbiddenIPsPage ) ) {
    762            return;
    763        }
    764
    765        final String remoteIP = HttpUtil.getRemoteAddress( context.getHttpRequest() );
    766        LOG.info("Attempting to match remoteIP " + remoteIP + " against " + m_IPPatterns.size() + " patterns");
    767
    768        for( final Pattern p : m_IPPatterns ) {
    769             LOG.debug("Attempting to match remoteIP with " + p.getPattern());
    770
    771            if( m_matcher.contains( remoteIP, p ) ) {
    772
    773                //  IP filter has a match.
    774                //
    775                final String uid = log( context, REJECT, REASON_IP_BANNED_PERMANENTLY + "(" + p.getPattern() + ")", remoteIP );
    776
    777                LOG.info( "SPAM:IPBanList (" + uid + "). remoteIP matches the IP filter '" + p.getPattern() + "'" );
    778                checkStrategy( context, "Herb says '" + p.getPattern() + "' is a banned IP and I trust Herb! (Incident code " + uid + ")" );
    779            }
    780        }
    781    }
    782
    783    private void checkPatternList( final Context context, final String change ) throws RedirectException {
    784        final Change c = new Change();
    785        c.m_change = change;
    786        checkPatternList( context, c );
    787    }
    788
    789    /**
    790     *  Creates a simple text string describing the added content.
    791     *
    792     *  @param context page context
    793     *  @param newText added content
    794     *  @return Empty string, if there is no change.
    795     */
    796    private static Change getChange( final Context context, final String newText ) {
    797        final Page page = context.getPage();
    798        final StringBuffer change = new StringBuffer();
    799        final Engine engine = context.getEngine();
    800        // Get current page version
    801
    802        final Change ch = new Change();
    803
    804        try {
    805            final String oldText = engine.getManager( PageManager.class ).getPureText( page.getName(), WikiProvider.LATEST_VERSION );
    806            Patch<String> patch = DiffUtils.diffInline(oldText, newText);
    807
    808
    809            if( patch == null ) {
    810                return ch;
    811            }
    812            int lineNumber = 1;
    813            int currentOriginalLine = 0;
    814            int currentModifiedLine = 0;
    815
    816            for (AbstractDelta<String> delta : patch.getDeltas()) {
    817                int originalPosition = delta.getSource().getPosition();
    818                int modifiedPosition = delta.getTarget().getPosition();
    819
    820                // Output unchanged lines before the delta
    821                while (currentOriginalLine < originalPosition && currentModifiedLine < modifiedPosition) {
    822                    lineNumber++;
    823                    currentOriginalLine++;
    824                    currentModifiedLine++;
    825                }
    826
    827                List<String> originalLines = delta.getSource().getLines();
    828                List<String> revisedLines = delta.getTarget().getLines();
    829
    830                for (String line : originalLines) {
    831                    change.append("- " + lineNumber + ": " + line + "\r\n");
    832                    ch.m_removals++;
    833                    lineNumber++;
    834                    currentOriginalLine++;
    835                }
    836
    837                for (String line : revisedLines) {
    838                    change.append("+ " + lineNumber + ": " + line + "\r\n");
    839                    lineNumber++;
    840                    currentModifiedLine++;
    841                    ch.m_adds++;
    842
    843                }
    844            }
    845
    846        } catch (final Exception e) {
    847            LOG.error("Diff failed", e);
    848        }
    849
    850        //  Don't forget to include the change note, too
    851        final String changeNote = page.getAttribute( Page.CHANGENOTE );
    852        if( changeNote != null ) {
    853            change.append( "\r\n" );
    854            change.append( changeNote );
    855        }
    856
    857        //  And author as well
    858        if( page.getAuthor() != null ) {
    859            change.append( "\r\n" ).append( page.getAuthor() );
    860        }
    861
    862        ch.m_change = change.toString();
    863        return ch;
    864    }
    865
    866    /**
    867     * Returns true, if this user should be ignored.  For example, admin users.
    868     *
    869     * @param context page context
    870     * @return True, if this user should be ignored.
    871     */
    872    private boolean ignoreThisUser( final Context context ) {
    873        if( context.hasAdminPermissions() ) {
    874            return true;
    875        }
    876
    877        final List< String > groups = Arrays.asList( m_allowedGroups );
    878        if( Arrays.stream( context.getWikiSession().getRoles() ).anyMatch( role -> groups.contains( role.getName() ) ) ) {
    879            return true;
    880        }
    881
    882        if( m_ignoreAuthenticated && context.getWikiSession().isAuthenticated() ) {
    883            return true;
    884        }
    885
    886        return context.getVariable("captcha") != null;
    887    }
    888
    889    /**
    890     *  Returns a random string of six uppercase characters.
    891     *
    892     *  @return A random string
    893     */
    894    private static String getUniqueID() {
    895        final StringBuilder sb = new StringBuilder();
    896        for( int i = 0; i < 6; i++ ) {
    897            final char x = ( char )( 'A' + RANDOM.nextInt( 26 ) );
    898            sb.append( x );
    899        }
    900
    901        return sb.toString();
    902    }
    903
    904    /**
    905     *  Returns a page to which we shall redirect, based on the current value of the "captcha" parameter.
    906     *
    907     *  @param ctx WikiContext
    908     *  @return An URL to redirect to
    909     */
    910    private String getRedirectPage( final Context ctx ) {
    911
    912        return ctx.getURL( ContextEnum.PAGE_VIEW.getRequestContext(), m_errorPage );
    913    }
    914
    915    /**
    916     *  Checks whether the UserProfile matches certain checks.
    917     *
    918     *  @param profile The profile to check
    919     *  @param context The WikiContext
    920     *  @return False, if this userprofile is suspect and should not be allowed to be added.
    921     *  @since 2.6.1
    922     */
    923    public boolean isValidUserProfile( final Context context, final UserProfile profile ) {
    924        try {
    925            checkPatternList( context, profile.getEmail() );
    926            checkPatternList( context, profile.getFullname() );
    927            checkPatternList( context, profile.getLoginName() );
    928        } catch( final RedirectException e ) {
    929            LOG.info("Detected attempt to create a spammer user account (see above for rejection reason)");
    930            return false;
    931        }
    932
    933        return true;
    934    }
    935
    936    /**
    937     *  This method is used to calculate an unique code when submitting the page to detect edit conflicts.
    938     *  It currently incorporates the last-modified date of the page, and the IP address of the submitter.
    939     *
    940     *  @param page The WikiPage under edit
    941     *  @param request The HTTP Request
    942     *  @since 2.6
    943     *  @return A hash value for this page and session
    944     */
    945    public static String getSpamHash( final Page page, final HttpServletRequest request ) {
    946        long lastModified = 0;
    947
    948        if( page.getLastModified() != null ) {
    949            lastModified = page.getLastModified().getTime();
    950        }
    951        final long remote = HttpUtil.getRemoteAddress( request ).hashCode();
    952
    953        return Long.toString( lastModified ^ remote );
    954    }
    955
    956    /**
    957     *  Returns the name of the hash field to be used in this request. The value is unique per session, and once
    958     *  the session has expired, you cannot edit anymore.
    959     *
    960     *  @param request The page request
    961     *  @return The name to be used in the hash field
    962     *  @since  2.6
    963     */
    964    public static String getHashFieldName( final HttpServletRequest request ) {
    965        String hash = null;
    966
    967        if( request.getSession() != null ) {
    968            hash = ( String )request.getSession().getAttribute( "_hash" );
    969
    970            if( hash == null ) {
    971                hash = c_hashName;
    972                request.getSession().setAttribute( "_hash", hash );
    973            }
    974        }
    975
    976        if( c_hashName == null || c_lastUpdate < ( System.currentTimeMillis() - HASH_DELAY * 60 * 60 * 1000 ) ) {
    977            c_hashName = getUniqueID().toLowerCase();
    978            c_lastUpdate = System.currentTimeMillis();
    979        }
    980
    981        return hash != null ? hash : c_hashName;
    982    }
    983
    984
    985    /**
    986     *  This method checks if the hash value is still valid, i.e. if it exists at all. This can occur in two cases:
    987     *  either this is a spam bot which is not adaptive, or it is someone who has been editing one page for too long,
    988     *  and their session has expired.
    989     *  <p>
    990     *  This method puts a redirect to the http response field to page "SessionExpired" and logs the incident in
    991     *  the spam log (it may or may not be spam, but it's rather likely that it is).
    992     *
    993     *  @param context The WikiContext
    994     *  @param pageContext The JSP PageContext.
    995     *  @return True, if hash is okay.  False, if hash is not okay, and you need to redirect.
    996     *  @throws IOException If redirection fails
    997     *  @since 2.6
    998     */
    999    public static boolean checkHash( final Context context, final PageContext pageContext ) throws IOException {
    1000        final String hashName = getHashFieldName( (HttpServletRequest)pageContext.getRequest() );
    1001        if( pageContext.getRequest().getParameter(hashName) == null ) {
    1002            if( pageContext.getAttribute( hashName ) == null ) {
    1003                final Change change = getChange( context, EditorManager.getEditedText( pageContext ) );
    1004                log( context, REJECT, "MissingHash", change.m_change );
    1005
    1006                final String redirect = context.getURL( ContextEnum.PAGE_VIEW.getRequestContext(),"SessionExpired" );
    1007                ( ( HttpServletResponse )pageContext.getResponse() ).sendRedirect( redirect );
    1008                return false;
    1009            }
    1010        }
    1011
    1012        return true;
    1013    }
    1014
    1015    /**
    1016     * This helper method adds all the input fields to your editor that the SpamFilter requires
    1017     * to check for spam.  This <i>must</i> be in your editor form if you intend to use the SpamFilter.
    1018     *
    1019     * @param pageContext The PageContext
    1020     * @return A HTML string which contains input fields for the SpamFilter.
    1021     */
    1022    public static String insertInputFields( final PageContext pageContext ) {
    1023        final Context ctx = Context.findContext( pageContext );
    1024        final Engine engine = ctx.getEngine();
    1025        final StringBuilder sb = new StringBuilder();
    1026        if( engine.getContentEncoding().equals( StandardCharsets.UTF_8 ) ) {
    1027            sb.append( "<input name='encodingcheck' type='hidden' value='\u3041' />\n" );
    1028        }
    1029
    1030        return sb.toString();
    1031    }
    1032
    1033    /**
    1034     *  A local class for storing host information.
    1035     */
    1036    private class Host {
    1037
    1038        private final long m_addedTime = System.currentTimeMillis();
    1039        private final long m_releaseTime;
    1040        private final String m_address;
    1041        private final Change m_change;
    1042
    1043        public String getAddress() {
    1044            return m_address;
    1045        }
    1046
    1047        public long getReleaseTime() {
    1048            return m_releaseTime;
    1049        }
    1050
    1051        public long getAddedTime() {
    1052            return m_addedTime;
    1053        }
    1054
    1055        public Change getChange() {
    1056            return m_change;
    1057        }
    1058
    1059        public Host( final String ipaddress, final Change change ) {
    1060            m_address = ipaddress;
    1061            m_change = change;
    1062            m_releaseTime = System.currentTimeMillis() + m_banTime * 60 * 1000L;
    1063        }
    1064
    1065    }
    1066
    1067    private static class Change {
    1068
    1069        public String m_change;
    1070        public int    m_adds;
    1071        public int    m_removals;
    1072
    1073        @Override
    1074        public String toString() {
    1075            return m_change;
    1076        }
    1077
    1078        @Override
    1079        public boolean equals( final Object o ) {
    1080            if( o instanceof Change ) {
    1081                return m_change.equals( ( ( Change )o ).m_change );
    1082            }
    1083            return false;
    1084        }
    1085
    1086        @Override
    1087        public int hashCode() {
    1088            return m_change.hashCode() + 17;
    1089        }
    1090
    1091    }
    1092
    1093}