Custom SA rule FAQ addition.

Drew Marshall drew at THEMARSHALLS.CO.UK
Tue Feb 24 21:20:11 GMT 2004


Skipped content of type multipart/alternative-------------- next part --------------
#!/usr/local/bin/bash
# Version 1.08a Added an option "SINGLE_EMAIL_ONLY" to send a single email only (instead of one per "event") (Thanks, Andrew Ott!).  Fixed SA_RESTART redirect to dev/null. (1.08a: Fixed "line continuation" bug in RDJ update notification)

## This file updates SpamAssassin RuleSet files from the internet.
##
## It is important that you *only* automatically update
## RuleSet files from people that you trust and that you
## *TEST* this.
##
## Note: When running this script interactively, debug mode is enable to allow you to view the results.


# Usage instructions:
# 1) Choose rulesets to update (TRUSTED_RULESETS below)
# 2) Configure Local SpamAssassin settings (SA_DIR, MAIL_ADDRESS, SA_RESTART below)
# 3) Run this script periodically (manually or crontab)
# 3a) To run manually, first make it executable (chmod +x rules_du_jour) then execute (./rules_du_jour)
# 3b) To run via cron, edit your cron (crontab -e) and add a line such as this:
#     28 1 * * *                                      /root/bin/rules_du_jour
#     The crontab line above runs /root/bin/rules_du_jour at 1:28AM every day. (choose a different time, please)
#     Make sure the user who's crontab you are editing has permission to write files to the SA config dir.


# Choose Rulesets from this list:
# BIGEVIL TRIPWIRE POPCORN BACKHAIR WEEDS1 WEEDS2 CHICKENPOX ANTIDRUG EVILNUMBERS

# IMPORTANT: Edit this line to choose which RuleSets to update
[ "${TRUSTED_RULESETS}" ] || \
        TRUSTED_RULESETS="BIGEVIL TRIPWIRE BACKHAIR WEEDS2 CHICKENPOX ANTIDRUG EVILNUMBERS";


#### Local SpamAssassin/system Settings ####
#### Modify these to match your system. ####
[ "${SA_DIR}" ] || SA_DIR="/etc/mail/spamassassin";      # Change this to your SA local config
                                                    # directory, probably /etc/mail/spamassassin.
                                                    # For amavisd chrooted, this may be:
                                                    # /var/amavisd/etc/mail/spamassassin
[ "${MAIL_ADDRESS}" ] || MAIL_ADDRESS="root";       # Where do Email notifications go
[ "${SINGLE_EMAIL_ONLY}" ] || \
    SINGLE_EMAIL_ONLY="true";                       # Set this to "true" to send only one notification
                                                    # email per RDJ run with "interesting"
                                                    # activity. Set to "" to send a separate
                                                    # for each interesting activity.
[ "${SA_LINT}" ] || SA_LINT="spamassassin --lint";  # Command used to lint the rules
[ "${SA_RESTART}" ] || \
    SA_RESTART="/usr/local/etc/rc.d/mailscanner.sh restart";  # Command used to restart spamd
                                                    # May be /etc/rc.d/init.d/spamassassin restart
                                                    # For amavisd, may be /etc/init.d/amavisd restart
[ "${PERL}" ] || PERL="perl";                       # Location of the perl program
[ "${GREP}" ] || GREP="grep";                       # Location of the grep program
                                                    # (solaris users may want to point this to gnu grep)

# DEBUG="true";                                     # Uncomment this to force debug mode on (or use -D)
#### End Local SpamAssassin Settings    ####



TMPDIR="${SA_DIR}/RulesDuJour";                     # Where we store old rulesets.  If you delete
                                                    # this directory, RuleSets may be detected as
                                                    # out of date the next time you run rules_du_jour.
RDJ_URL="http://sandgnat.com/rdj/rules_du_jour";    # URL to update this script

#### CF Files information ####
# These are bash Array Variables ("man bash" for more information)
[ ${CF_URLS} ]  || declare -a CF_URLS;                  # Array that contains URLs of the files.
[ ${CF_FILES} ] || declare -a CF_FILES;                 # Local name of the CF file; eg: bigevil.cf
[ ${CF_NAMES} ] || declare -a CF_NAMES;                 # Happy Name of CF file; eg: "Big Evil"
[ ${PARSE_NEW_VER_SCRIPTS} ] || \
        declare -a PARSE_NEW_VER_SCRIPTS;               # Command to run on the file to retrieve new version info
[ ${CF_MUNGE_SCRIPTS} ] || declare -a CF_MUNGE_SCRIPTS; # This (optionally) modifies the file; eg: lower scores


#########################################
####     Begin Rules File Registry   ####
#########################################

# If you add more RuleSets to your own registry, please contribute the settings to the www.exit0.us wiki
# http://www.exit0.us/index.php/RulesDuJourRuleSets

#### Here are settings for Tripwire. ####

TRIPWIRE=0; # Index of Tripwire data into the arrays is 0
              CF_URLS[0]="http://www.merchantsoverseas.com/wwwroot/gorilla/99_FVGT_Tripwire.cf";
             CF_FILES[0]="tripwire.cf";
             CF_NAMES[0]="TripWire";
PARSE_NEW_VER_SCRIPTS[0]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[0]="nothing necessary for this ruleset.";

#### Here are settings for Big Evil. ####
BIGEVIL=1; # Index of Big Evil is 1
              CF_URLS[1]="http://www.merchantsoverseas.com/wwwroot/gorilla/bigevil.cf";
             CF_FILES[1]="bigevil.cf";
             CF_NAMES[1]="Big Evil";
PARSE_NEW_VER_SCRIPTS[1]="head -1";
#    CF_MUNGE_SCRIPTS[1]="nothing necessary for this ruleset.";

#### Here are settings for Popcorn. ####
# Note that as of 2004/01/21 popcorn is now included in the Backhair ruleset (below)
POPCORN=2; # Index of Popcorn is 2
              CF_URLS[2]="http://www.emtinc.net/includes/popcorn.cf";
             CF_FILES[2]="popcorn.cf";
             CF_NAMES[2]="Jennifer's Popcorn";
PARSE_NEW_VER_SCRIPTS[2]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[2]="nothing for this ruleset.";

#### Here are settings for Backhair. ####
BACKHAIR=3; # Index of Backhair is 3
              CF_URLS[3]="http://www.emtinc.net/includes/backhair.cf";
             CF_FILES[3]="backhair.cf";
             CF_NAMES[3]="Jennifer's Backhair"; # ;-)
PARSE_NEW_VER_SCRIPTS[3]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[3]="nothing for this ruleset.";

#### Here are settings for Weeds 1. Do not install both weeds sets at the same time. ####
WEEDS1=4; # Index of Weeds Set 1 is 4

              CF_URLS[4]="http://www.emtinc.net/includes/weeds.cf";
             CF_FILES[4]="weeds.cf";
             CF_NAMES[4]="Jennifer's Weeds Set (1)";
PARSE_NEW_VER_SCRIPTS[4]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[4]="nothing for this ruleset.";

#### Here are settings for Weeds 2. Do not install both weeds sets at the same time. ####
WEEDS2=5; # Index of Weeds Set 2 is 5
              CF_URLS[5]="http://www.emtinc.net/includes/weeds_2.cf";
             CF_FILES[5]="weeds.cf";
             CF_NAMES[5]="Jennifer's Weeds Set (2)";
PARSE_NEW_VER_SCRIPTS[5]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[5]="nothing for this ruleset.";

#### Here are settings for ChickenPox. ####
CHICKENPOX=6; # Index of ChickenPox is 6
              CF_URLS[6]="http://www.emtinc.net/includes/chickenpox.cf";
             CF_FILES[6]="chickenpox.cf";
             CF_NAMES[6]="Jennifer's ChickenPox";
PARSE_NEW_VER_SCRIPTS[6]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[6]="nothing for this ruleset.";

#### Here are settings for AntiDrug. ####
ANTIDRUG=7; # Index of antidrug is 7
              CF_URLS[7]="http://mywebpages.comcast.net/mkettler/sa/antidrug.cf"
             CF_FILES[7]="antidrug.cf";
             CF_NAMES[7]="Matt Kettler's AntiDrug";
PARSE_NEW_VER_SCRIPTS[7]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[7]="nothing for this ruleset.";

#### Here are settings for evilnumber ####
EVILNUMBERS=8; # Index of evilnumbers data into the arrays is 8
              CF_URLS[8]="http://www.merchantsoverseas.com/wwwroot/gorilla/evilnumbers.cf";
             CF_FILES[8]="evilnumbers.cf";
             CF_NAMES[8]="EvilNumber";
PARSE_NEW_VER_SCRIPTS[8]="${PERL} -ne 'print if /^\s*#.*(vers?|version|rev|revision)[:\.\s]*[0-9]/i;' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[8]="nothing for this ruleset.";

#### Here are settings for sa-blacklist ####
BLACKLIST=9; # Index of sa-blacklist data into the arrays is 9
              CF_URLS[9]="http://www.stearns.org/sa-blacklist/sa-blacklist.current";
             CF_FILES[9]="blacklist.cf";
             CF_NAMES[9]="William Stearn's sa-blacklist";
PARSE_NEW_VER_SCRIPTS[9]="grep -i '^#.*sa-blacklist: 200' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[9]="nothing for this ruleset.";

#### Here are settings for sa-blacklist-uri ####
BLACKLIST_URI=10; # Index of sa-blacklist-uri data into the arrays is 10
              CF_URLS[10]="http://www.stearns.org/sa-blacklist/sa-blacklist.current.uri.cf";
             CF_FILES[10]="blacklist-uri.cf";
             CF_NAMES[10]="William Stearn's URI blacklist";
PARSE_NEW_VER_SCRIPTS[10]="grep -i '^#.*sa-blacklist.uri: 200' | sort | tail -1";
#    CF_MUNGE_SCRIPTS[10]="nothing for this ruleset.";



#########################################
####     End Rules File Registry     ####
#########################################

# Do not update beyond this line unless you know what you are doing.






#########################################
####     Begin rules update code     ####
#########################################

# if invoked with -D, enable DEBUG here.
[ "$1" = "-D" ] && DEBUG="true";
# if running interactively, enable DEBUG here.
[ -t 0 ] && DEBUG="true";

# If we're not running interactively, add a random delay here. This should
# help reduce spikes on the servers hosting the rulesets (Thanks, Bob)
MAXDELAY=3600;
DELAY=0;
[ ! -t 0 ] && [ ${MAXDELAY} -gt 0 ] && let DELAY="${RANDOM} % ${MAXDELAY}";
[ "${DEBUG}" ] && [ ${DELAY} -gt 0 ] && echo "Probably running from cron... sleeping for a random interval (${DELAY} seconds)";
[ ${DELAY} -gt 0 ] && sleep ${DELAY};


# Save old working dir
OLDDIR=`pwd`;

# This variable is used to indicate if we should restart spamd. Currently empty (false).
RESTART_REQUIRED="";
# This variable is used to indicate if we should send an email notification when all is said and done.
# It is toggled on whenever an "interesting" event happens (404, rule updated, etc)
QUEUE_SINGLE_EMAIL="";
# The beginnings of an email and/or debug summary text
MESSAGES="RulesDuJour Run Summary on `hostname`:";

[ ! -e ${TMPDIR} ] && mkdir ${TMPDIR};
cd ${TMPDIR};

[ ! "${DONT_CHECK_FOR_RDJ_UPDATES}" ] && {
    if [ -f ${TMPDIR}/rules_du_jour ] ; then
        wget -N ${RDJ_URL} > ${TMPDIR}/wget.log 2>&1;
        ${GREP} 'saved' ${TMPDIR}/wget.log > /dev/null;             DOWNLOADED=$?;
        [ ${DOWNLOADED} = 0 ] && {
            NEWVER=`grep "^# Version" ${TMPDIR}/rules_du_jour`;
            MSG_RDJ_UPDATED="Rules Du Jour has an update available.  New version is ${NEWVER} and was downloaded to ${TMPDIR}/rules_du_jour";
            [ "${SINGLE_EMAIL_ONLY}" ] && QUEUE_SINGLE_EMAIL="true" || echo "${MSG_RDJ_UPDATED}" | \
                    mail -s "RulesDuJour/`hostname`: new Rules Du Jour version available." ${MAIL_ADDRESS};
            MESSAGES="${MESSAGES}\n${MSG_RDJ_UPDATED}";
        }
    else
        wget -N ${RDJ_URL} > ${TMPDIR}/wget.log 2>&1;
    fi
}

for RULESET_NAME in ${TRUSTED_RULESETS} ; do

    # Set up some array variables
    INDEX=${!RULESET_NAME};
    CF_URL=${CF_URLS[${INDEX}]};
    CF_FILE=${CF_FILES[${INDEX}]};
    CF_NAME=${CF_NAMES[${INDEX}]};
    PARSE_NEW_VER_SCRIPT=${PARSE_NEW_VER_SCRIPTS[${INDEX}]};
    CF_MUNGE_SCRIPT=${CF_MUNGE_SCRIPTS[${INDEX}]};

    # Get the filename the author chose.
    CF_BASENAME=`basename ${CF_URL}`;

    DATE=`date +"%Y%m%d-%H%M"`

    if [ "${DEBUG}" ] ; then
        # Dump the variables to stdout
        echo "";
        echo "------ ${RULESET_NAME} ------";
        echo "RULESET_NAME=${RULESET_NAME}";
        echo "INDEX=${INDEX}";
        echo "CF_URL=${CF_URL}";
        echo "CF_FILE=${CF_FILE}";
        echo "CF_NAME=${CF_NAME}";
        echo "PARSE_NEW_VER_SCRIPT=${PARSE_NEW_VER_SCRIPT}";
        echo "CF_MUNGE_SCRIPT=${CF_MUNGE_SCRIPT}";
    fi

    [ "${DEBUG}" ] && [ -f ${TMPDIR}/${CF_BASENAME} ] && echo "Old ${CF_BASENAME} already existed in ${TMPDIR}...";
    [ "${DEBUG}" ] && [ ! -f ${TMPDIR}/${CF_BASENAME} ] && \
            [ ! -f ${SA_DIR}/${CF_FILE} ] && echo "This is the first time downloading ${CF_BASENAME}...";

    [ "${DEBUG}" ] && [ ! -f ${TMPDIR}/${CF_BASENAME} ] && [ -f ${SA_DIR}/${CF_FILE} ] && \
            echo "Copying from ${SA_DIR}/${CF_FILE} to ${TMPDIR}/${CF_BASENAME}...";
    [ ! -f ${TMPDIR}/${CF_BASENAME} ] && [ -f ${SA_DIR}/${CF_FILE} ] && \
            cp ${SA_DIR}/${CF_FILE} ${TMPDIR}/${CF_BASENAME} && \
            touch -r ${SA_DIR}/${CF_FILE} ${TMPDIR}/${CF_BASENAME};





    [ "${DEBUG}" ] && echo "Retrieving file from ${CF_URL}...";
    # send wget output to a temp file for grepping
    wget -N ${CF_URL} > ${TMPDIR}/wget.log 2>&1;

    ${GREP} 'saved' ${TMPDIR}/wget.log > /dev/null;             DOWNLOADED=$?;
    ${GREP} 'ERROR 4[0-9][0-9]' ${TMPDIR}/wget.log > /dev/null; WAS404=$?;
    ${GREP} -i 'failed: ' ${TMPDIR}/wget.log > /dev/null;       FAILED=$?;

    [ ! ${DOWNLOADED} = 0 ] && DOWNLOADED=; # Unset DOWNLOADED if the file was already current
    [ ! ${WAS404} = 0 ] && WAS404=; # Unset WAS404 if the file didn't return 404.
    [ ! ${FAILED} = 0 ] && FAILED=; # Unset FAILED if wget succeded

    # Append these errors to a variable to be mailed to the admin (later in script)
    [ "${FAILED}" ] && RULES_THAT_404ED="${RULES_THAT_404ED}\n${CF_NAME} had an unknown error:\n`cat ${TMPDIR}/wget.log`";
    [ "${WAS404}" ] && RULES_THAT_404ED="${RULES_THAT_404ED}\n${CF_NAME} not found (404) at ${CF_URL}";

    [ "${DEBUG}" ] && [ ${WAS404} ] && echo "Got 404 from ${CF_NAME} (${CF_URL})...";
    [ "${DEBUG}" ] && [ ! ${WAS404} ] && ([ "${DOWNLOADED}" ] && \
            echo "New version downloaded..." || \
            echo "${CF_BASENAME} was up to date (skipped downloading of ${CF_URL})...");






    # If we downloaded a new version, or if we have the current version,
    # but it is not installed, copy or munge to CF_FILE.2
    if ([ "${DOWNLOADED}" ] || \
            ( [ ! -f ${SA_DIR}/${CF_FILE} ] && \
            [ -f ${TMPDIR}/${CF_BASENAME} ]) ) ; then
        if [ "${CF_MUNGE_SCRIPT}" ] ; then
            [ "${DEBUG}" ] && echo "Munging output using command: ${CF_MUNGE_SCRIPT}";
            sh -c "${CF_MUNGE_SCRIPT}" < ${TMPDIR}/${CF_BASENAME} > ${TMPDIR}/${CF_BASENAME}.2;
        else
            cp ${TMPDIR}/${CF_BASENAME} ${TMPDIR}/${CF_BASENAME}.2;
        fi

        # Set munged file to same timestamp as downloaded file...
        touch -r ${TMPDIR}/${CF_BASENAME} ${TMPDIR}/${CF_BASENAME}.2;
    fi






    # Update SA config dir if this is the first time we've seen the ruleset, or if the ruleset has changed.
    if ( [ -f ${TMPDIR}/${CF_BASENAME}.2 ] && \
            ( [ ! -f ${SA_DIR}/${CF_FILE} ] || \
            ! cmp -s ${TMPDIR}/${CF_BASENAME}.2 ${SA_DIR}/${CF_FILE} ) ); then

        [ "${DEBUG}" ] && [ ! -f ${SA_DIR}/${CF_FILE} ] && echo "Installing new ruleset from ${TMPDIR}/${CF_BASENAME}.2" ;
        [ "${DEBUG}" ] && [ -f ${SA_DIR}/${CF_FILE} ] && echo "Old version ${SA_DIR}/${CF_FILE} differs from new version ${TMPDIR}/${CF_BASENAME}.2" && echo "Backing up old version...";
        [ -f ${SA_DIR}/${CF_FILE} ] && mv -f ${SA_DIR}/${CF_FILE} ${TMPDIR}/${CF_FILE}.${DATE};

        # Save the command that can be used to undo this change, if rules won't --lint
        UNDO_COMMAND="${UNDO_COMMAND} mv -f ${SA_DIR}/${CF_FILE} ${TMPDIR}/${CF_BASENAME}.2;";
        [ -f ${TMPDIR}/${CF_FILE}.${DATE} ] && \
                UNDO_COMMAND="${UNDO_COMMAND} mv -f ${TMPDIR}/${CF_FILE}.${DATE} ${SA_DIR}/${CF_FILE};" || \
                UNDO_COMMAND="${UNDO_COMMAND} rm -f ${SA_DIR}/${CF_FILE};";

        [ "${DEBUG}" ] && [ -f ${TMPDIR}/${CF_BASENAME}.2 ] && echo "Installing new version...";
        [ -f ${TMPDIR}/${CF_BASENAME}.2 ] && mv -f ${TMPDIR}/${CF_BASENAME}.2 ${SA_DIR}/${CF_FILE};

        NEWVER=`sh -c "cat ${SA_DIR}/${CF_FILE} | ${PARSE_NEW_VER_SCRIPT}"`;

        MSG_CHANGED="${CF_NAME} has changed on `hostname`.  The new ${CF_NAME} is ${NEWVER}.";
        MESSAGES="${MESSAGES}\n${MSG_CHANGED}";
        [ "${DEBUG}" ] && echo "${MSG_CHANGED}";
        [ "${SINGLE_EMAIL_ONLY}" ] && QUEUE_SINGLE_EMAIL="true" || \
                echo ${MSG_CHANGED} | mail -s "RulesDuJour/`hostname`: ${CF_NAME} RuleSet has been updated" ${MAIL_ADDRESS}

        RESTART_REQUIRED="true";
    fi
done






# Cleanup, lint, email admin if required, restart SA if required
[ -f ${TMPDIR}/wget.log ] && rm -f ${TMPDIR}/wget.log;
[ "${RULES_THAT_404ED}" ] && {
    MSG_404S="The following rules had 404 errors:${RULES_THAT_404ED}";
    [ "${SINGLE_EMAIL_ONLY}" ] && QUEUE_SINGLE_EMAIL="true" || \
            echo -e "${MSG_404S}" | mail -s "RulesDuJour/`hostname`: 404 errors" ${MAIL_ADDRESS};
    MESSAGES="${MESSAGES}\n\n${MSG_404S}";
}

[ "${RESTART_REQUIRED}" ] && {
    [ "${DEBUG}" ] && echo "Attempting to --lint the rules.";
    ${SA_LINT} > /dev/null 2>&1; LINTFAILED=$?;
    [ "${LINTFAILED}" = "0" ] && LINTFAILED=; # Unset LINTFAILED if lint didn't fail.

    # Lint failed. Run the undo commands, send administrative notification
    if [ ${LINTFAILED} ] ; then
        WARNMSG="***WARNING***: ${SA_LINT} failed.\nRolling configuration files back, not restarting SpamAssassin.\nRollback command is: ${UNDO_COMMAND}";
        MESSAGES="${MESSAGES}\n\n${WARNMSG}";
        sh -c "${UNDO_COMMAND}" && RESTART_REQUIRED= ;
        [ "${SINGLE_EMAIL_ONLY}" ] && QUEUE_SINGLE_EMAIL="true" || \
                echo -e "${WARNMSG}" | mail -s "RulesDuJour/`hostname`: lint failed. Updates rolled back." ${MAIL_ADDRESS};
    else
        [ "${DEBUG}" ] && echo "Restarting SpamAssassin using: sh -c \"${SA_RESTART}\"";
        sh -c "${SA_RESTART}" > /dev/null 2>&1;
    fi
}

[ "${DEBUG}" ] && [ ! "${RESTART_REQUIRED}" ] && echo "No files updated; No restart required.";

[ "${DEBUG}" ] && echo -e "\n\n\n\n\nRules Du Jour Run Summary:${MESSAGES}";

# Send the single consolidated notification email here.
[ "${SINGLE_EMAIL_ONLY}" ] && [ "${QUEUE_SINGLE_EMAIL}" ] && \
        echo -e "${MESSAGES}" | mail -s "RulesDuJour Run Summary on `hostname`" ${MAIL_ADDRESS};

cd ${OLDDIR};


More information about the MailScanner mailing list