Here is a script I did "earlier" (i.e. years and years ago) to remove annoying 404s from the Apache error log.
#!/usr/bin/perl
# ---------------------------------------------------------------------------
# Script kiddies and worms often try URLs behind which one can find
# specific vulnerabilities. This script writes a file to stdout that can then
# be included by httpd.conf so that known probed URLS result in 410s.
#
# See also: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html:
#
# 10.4.11 410 Gone
#
# The requested resource is no longer available at the server and no
# forwarding address is known. This condition is expected to be
# considered permanent. Clients with link editing capabilities SHOULD
# delete references to the Request-URI after user approval. If the
# server does not know, or has no facility to determine, whether or
# not the condition is permanent, the status code 404 (Not Found) SHOULD
# be used instead. This response is cacheable unless indicated otherwise.
#
# The 410 response is primarily intended to assist the task of web
# maintenance by notifying the recipient that the resource is intentionally
# unavailable and that the server owners desire that remote links to that
# resource be removed. Such an event is common for limited-time, promotional
# services and for resources belonging to individuals no longer working at
# the server's site. It is not necessary to mark all permanently unavailable
# resources as "gone" or to keep the mark for any length of time -- that is
# left to the discretion of the server owner.
#
# WHY IS THIS INTERESTING
# -----------------------
#
# This setup removes the requests from the Apache httpd error log (the
# requests no longer generate 'file not found' errors)
#
# This is is a good thing insofar as that list then becomes smaller (good
# for maintenance) and someone trying out more 'refined' vulnerabilities
# becomes visible (good for ringing the warning bell). It also tells
# script kiddies to go look elsewhere. On the other hand, a notable
# increase in probing might go unnoticed.
#
# If the website is configured to redirect requests for unknown URLs to the
# homepage, having this script saves on bandwidth for sure.
#
# Notes
# -----
#
# Probes for errors in HTTP protocol handling (bad headers etc) will still
# show up in the log.
#
# -> analog webanalysis: The files still appear in the analog "failure
# request" log except if you set "STATUSEXCLUDE 410"
#
# -> You want to allow some URLs which are being probed as you really
# might have the corresponding application installed (patched and
# secured beforehand of course). Sometimes, adding further path elements
# might be a solution to discriminate legit requests from probes.
#
# -> Performance impact? I have no idea.
#
# What matches
# ------------
#
# The "gone" URLs are just the start of URLs, so anything with an extension
# will also match. There generally is no need to put the values into goneMatch.
# If you list "/forum3" as "gone", then the following will be marked "gone":
#
# /forum3
# //forum3 (which reduces to /forum3)
# /forum3/x
# /forum3//
#
# but not
#
# /forum3alpha
#
# If the website is configured to redirect requests for unknown URLs to the
# homepage, having this script saves both on bandwidth and noise in the web
# statistics.
# ---------------------------------------------------------------------------
# Direct matching
@gone = makeGoneArray();
# Special matching ANYWHERE WITHIN AN URL.
# Probes may check many versions of "phpMyAdmin" (like "/phpMyAdmin-2.6.0a" etc),
# so, that URL is in the "goneMatch" list:
@goneMatch = qw(xmlrpc.php
nsiislog.dll
jud.cgi
xmlrpc
xmlsrv
w00tw00t
proc/self/environ
etc/passwd
/.ht
/.svn/
/.svn$
^/phpMyAdmin
);
# PRINT IT. The result is supposed to be installed by another script which
# checks whether the script currently in use has different content and
# runs a graceful restart of Apache httpd. It also replaces %COPYMARK%
# with a tag indicating the installation operation.
print "# --------------------------------------------------------------------------n";
print "# Use the perl script 'kiddie_be_gone.pl' to generate the contents below,n";
print "# which are included by httpd.confn";
print "# Even easier, use ~q/httpd/sbin/install_kiddie_be_gone.shn";
print "# %COPYMARK%n";
print "# --------------------------------------------------------------------------n";
{
my $earlier = "";
foreach my $entry (sort @gone) {
# Eliminate duplicates, which is easy as the list is sorted
if ($earlier eq $entry) {
print STDERR "Duplicate 'gone' entry '$entry'n";
}
else {
$earlier = $entry;
if ($entry =~ /^(.*)/$/) {
print STDERR "Terminating slash in '$entry' - removed slashn";
$entry = $1;
}
#
# Some URLS in scan attacks are actually used.
# We just allow them for all.
#
# if ($entry eq '/administrator') {
# # used by Joomla
# }
# elsif ($entry eq '/mysql') {
# # a subdirectory on site
# }
{
#
# Return a "410" - redirect gone
#
# See "http://httpd.apache.org/docs/2.2/mod/mod_alias.html#redirect"
# redirect instruction
# Access to URLs yields error 410 - "Resource gone, no forwarding address"
#
# It would be cool to return a custom error code "444" - probe URL for
# example, but that does not seem to be possible.
#
print "Redirect gone $entryn";
}
}
}
}
#
# Once again, for "RedirectMatch"
#
{
my $earlier = "";
foreach my $entry (sort @goneMatch) {
if ($earlier eq $entry) {
print STDERR "Duplicate 'goneMatch' entry $entryn";
}
else {
$earlier = $entry;
print "RedirectMatch gone $entryn";
}
}
}
#######################################################################################
# Function to set up the URLs to which one responds "gone".
#
# "/.ht" matches URLS which contain "/.ht", e.g. "/.htaccess"
# "/.svn/" matches URLS which contain "/.svn/", i.e. anything requesting stuff under
# .svn
# "/.svn$ matches URLs which terminate in "/.svn", i.e. the request for the dir itself
# How about requests for "encoded" URLs? like "/.s%76n/"? They seem to be caught,
# too.
#
# This list is at the end of the script for easier editing and subsequent appending
# using the usual Unix text processing tools.
#######################################################################################
sub makeGoneArray {
return qw(
/..
/...
/*
/2phpmyadmin
/..%5c..%5cwinnt
/a.asp
/adm
/admin
/admin/config/config.inc.php
/admin/includes/general.js
/admin/phpmyadmin2/index.php
/admin/scripts/setup.php
/adsamples
/appConf.htm
/appserv/main.php
/aws
/awstats
/awstats-cgibin
/awstats.pl
/azenv.php
/AZ.php
/b2b/admin/includes/stylesheet.css
/bb
/BB
/bbs
/BBS1
/BBS2
/BE_config.php
/BFormMail.pl
/bin/msgimport
/blogs
/board
/boards
/bug/login_page.php
/bugtracker/login_page.php
/bulletinboard
/bulletinboards
/butik
/c
/c99.php
/cart
/catalog/admin/includes/stylesheet.css
/cc
/cgi
/cgi-bin
/cgi-bin/..
/cgi-bin2
/cgi-bin/azenv.pl
/cgi-bin/check.bat
/cgi-bin/ip1.cgi
/cgi-bin/judge.cgi
/cgi-bin/prxjdg.cgi
/cgi-local
/chat1
/chat2
/chat3
/chi-bin
/cmd.php
/community
/components/com_roundcube/CHANGELOG
/config
/createemails.inc.php
/cube/bin/msgimport
/cvs
/d
/database
/db
/dbadmin
/dbadmin/config/config.inc.php
/db/js/keyhandler.js
/db/mysql/main.php
/db/pma/main.php
/db/scripts/setup.php
/db/sql/main.php
/default.ida
/discussion
/domains
/drupal
/ecommerce/admin/includes/stylesheet.css
/email/README
/eshop/admin/includes/stylesheet.css
/estadisticas
/exchange
/ezformml.cgi
/fastenv
/fmail.pl
/formmail.cgi
/foros
/forum
/Forum
/forum2
/forum3
/forums
/Forums
/galaxy_
/him.php
/horde
/Horde
/horde2
/horde3
/horde-3.0.5
/horde-3.0.7
/horde-3.0.8
/horde-3.0.9
/iisadmpwd
/ip1.cgi
/ip.cgi
/issue/login_page.php
/issuetracker/login_page.php
/joke
/lib_gor
/M83A
/mail
/mail/bin/msgimport
/mails/README
/mailz/README
/mambo
/mantisbt/login_page.php
/mantis/login_page.php
/members
/_mem_bin
/mod_cbsms_messages.php
/msadc
/MSADC
/msdac
/msgboard
/MSOffice
/MSOffice/cltreq.asp
/mss2/bin/msgimport
/myadmin
/MyAdmin
/myadmin/config/config.inc.php
/myAdmin/config/config.inc.php
/myadmin/scripts/setup.php
/myAdmin/scripts/setup.php
/mysqladmin
/mysql-admin
/mysqladmin/scripts/setup.php
/mysql/config/config.inc.php
/mysqlmanager
/mysql/scripts/setup.php
/nar
/negozio/admin/includes/stylesheet.css
/newboard
/newboards
/NULL.ida
/NULL.IDA
/NULL.idq
/NULL.printer
/ok
/openwebmail
/padmin
/PBServer
/phpadmin
/PHPmyadmin
/phpmanager
/judge112233.php
/phpadmin/js/keyhandler.js
/phpadmin/scripts/setup.php
/phpbb
/phpBB
/phpBB2
/phpdb
/phpgroupware
/phpmanager
/phpmy
/phpmyadmin
/php-myadmin
/php-my-admin
/phpmy-admin
/phpmyAdmin
/PHPmyadmin
/PHPMYADMIN
/phpmyadmin2
/php-my-admin/config/config.inc.php
/phppgadmin
/phppma
/pma
/PMA
/pma2005
/PMA2005
/p/m/a/config/config.inc.php
/pmadmin
/pma/scripts/setup.php
/portal
/program
/projects/login_page.php
/proxyheader.php
/pr.php
/qql
/r
/r57.php
/rc
/README
/round/bin
/roundcube
/roundcube-0.1
/roundcube-0.2
/roundcubemail
/roundcubemail-0.1
/roundcubemail-0.2
/Rpc
/samples
/scgi
/s-cgi
/scgi-bin
/scoreboard
/script
/scripts
/scripts/..
/send_emails.inc.php
/soapCaller.bs
/sql
/SQL
/sqladmin
/sqlmanager
/sqlweb
/sqlweb/config/config.inc.php
/squirrelmail/CHANGELOG
/stats
/stoma.php
/sumthin
/textenv.pl
/thisdoesnotexistahaha.php
/tracker/login_page.php
/trixbox/soapCaller.bs
/trix/soapCaller.bs
/ugboard
/ugboards
/undergraduate
/underground
/usage
/user/soapCaller.bs
/user/templates/footer.tpl
/vhcs2
/vhcs2/domain_default_page/index.html
/vhcs2/soapCaller.bs
/_vti_bin
/_vti_bin/owssvr.dll
/_vti_cnf
/_vti_inf.html
/webadmin
/webdb
/webmai
/webmail
/websql
/webstats
/wordpress
/workflow-activities.php
/wp-admin/index.php
x85x04bsoapCaller.bs
/judge.php
/pr.php
/ip.php
)
}
