Man kann natürlich auch mittel USER_AGENT checken um lästige Dinge loszuwerden, hier mal so als Auszug. Natürlich muss jeder selber entscheiden, was er zulassen möchte und was nicht:
CODE # spambots
RewriteCond %{HTTP_USER_AGENT} compatible\;\ MSIE\ 6.0 [OR]
RewriteCond %{HTTP_USER_AGENT} DTS\ Agent [OR]
RewriteCond %{HTTP_REFERER} iaea\.org [OR]
#RewriteCond %{HTTP_USER_AGENT} ^IE\ \d\.\d\ Compatible.*Browser$ [OR]
RewriteCond %{HTTP_USER_AGENT} Microsoft\ URL\ Control [OR]
RewriteCond %{HTTP_USER_AGENT} NG/2.0 [OR]
RewriteCond %{HTTP_USER_AGENT} RPT-HTTPClient [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*\ Charlotte.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*\ Daumoa.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*\ Exabot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*ACONTBOT.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*adressendeutschland.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*askpeter\_bot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*askpeter\_jeanie.*$ [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^.*Exabot\-Thumbnail.*$ [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^.*HTTrack.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*IDBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*iCcrawler\ \-\ iCjobs.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*jobs\.de\-Robot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*MJ12bot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*MooseBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*NaverBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*OpenX\ Spider.*$ [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^.*PicSpider.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*proximic.*$ [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^.*sexsearcher.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*TopBlogsInfo.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*VoilaBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Alexibot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ArtfaceBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^asterias [OR,NC]
# exploit searcher
RewriteCond %{HTTP_USER_AGENT} ^BackDoorBot [OR,NC]
# exploit searcher
RewriteCond %{HTTP_USER_AGENT} ^Black.Hole [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BlackWidow [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Baiduspider [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^BlowFish [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^boitho\.com\-dc [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BotALot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BotOnParade [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Bot\ mailto:craftbot@yahoo\.com [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BuiltBotTough [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Bullseye [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BunnySlippers [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Cegbfeieh [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^CheeseBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^CherryPicker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ChinaClaw [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Cityreview [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^CMUImageBot/spider\.pl [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Convera [OR,NC]
# Copyright checker
RewriteCond %{HTTP_USER_AGENT} ^CopyRightCheck [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^cosmos [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Crescent [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Custo [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^DISCo [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^DittoSpyder [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Download\ Demon [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^eCatch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EirGrabber [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailCollector.*$ [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailSearch.*$ [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailSiphon.*$ [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailWolf.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EmeraldShield [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^envolk [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EroCrawler [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Eurobot [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Express\ WebPictures [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ExtractorPro [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EyeNetIE [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Faxobot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^FlashGet [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^findlinks.*wortschatz\.uni-leipzig\.de [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Foobot [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^GetRight [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^GetWeb! [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Go!Zilla [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Go-Ahead-Got-It [OR,NC]
# RewriteCond %{HTTP_USER_AGENT} ^Googlebot-Image [OR]
RewriteCond %{HTTP_USER_AGENT} ^GornKer [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^GrabNet [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Grafula [OR,NC]
# Perl Module
RewriteCond %{HTTP_USER_AGENT} ^GT\:\:WWW [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Harvest [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^hloader [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^HMView [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^holmes [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^httplib [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^humanlinks [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^iearthworm [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Image\ Stripper [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Image\ Sucker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Indy\ Library [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^InfoNaviRobot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Infosearch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^InnerpriseBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^InterGET [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Internet\ Ninja [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^IRLbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Irvine [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Isidorus [OR,NC]
# Apache Jakarta std bot
RewriteCond %{HTTP_USER_AGENT} ^Jakarta\ Commons\-HttpClient [OR,NC]
# Java std bot
RewriteCond %{HTTP_USER_AGENT} ^Java/1\.6 [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^JennyBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^JetCar [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^JOC\ Web\ Spider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Kenjin.Spider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Keyword.Density [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Knowledge.com [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^larbin [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^LeechFTP [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^LexiBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^libWeb/clsHTTP [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^LinkextractorPro [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^LinkScan/8.1a.Unix [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^LinkWalker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^lwp-trivial [OR,NC]
# vermutlich Spambot (Forum/Blogs/Co)
RewriteCond %{HTTP_USER_AGENT} ^Mail\.Ru [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Mata.Hari [OR,NC]
# oesterreichische softwarefirma
RewriteCond %{HTTP_USER_AGENT} ^McBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Microsoft [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Microsoft.URL [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^MIDown\ tool [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^MIIxpc [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Mister.PiX [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Mnogosearch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^moget [OR,NC]
# spybot
RewriteCond %{HTTP_USER_AGENT} ^Mozilla.*\ obot [OR]
RewriteCond %{HTTP_USER_AGENT} ^NaverBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Navroad [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NearSite [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Net\ Vampire [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NetAnts [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NetMechanic [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NetSpider [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^NetZIP [OR,NC]
# ein Arcor-Bot
RewriteCond %{HTTP_USER_AGENT} ^NG-Search [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NICErsPRO [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NPBot [OR,NC]
# spybot
RewriteCond %{HTTP_USER_AGENT} ^oBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Ocelli [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Octopus [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Offline\ Explorer [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Offline\ Navigator [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^OmniExplorer [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Openfind [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^PageGrabber [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^panscient\.com [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Papa\ Foto [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^pavuk [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^pcBrowser [OR,NC]
# favicon-browser
RewriteCond %{HTTP_USER_AGENT} ^PEERbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^picscout [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^PingALink [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^PrivacyFinder [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ProPowerBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ProWebWalker [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^psbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^QueryN.Metasearch [OR,NC]
# keine robotstxt
RewriteCond %{HTTP_USER_AGENT} ^RedKernel [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ReGet [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^RepoMonkey [OR,NC]
# spiders
RewriteCond %{HTTP_USER_AGENT} ^RexyoBot [OR,NC]
# spambot
RewriteCond %{HTTP_USER_AGENT} ^RIN\.\ Web\ crawler [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^RMA [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Seekbot [OR,NC]
# favicon-browser
RewriteCond %{HTTP_USER_AGENT} ^Shit [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ShopWiki [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Siphon [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SiteSnagger [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SlySearch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SmartDownload [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Speedy\ Spider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SpankBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^spanner [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Sosospider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SSM [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SuperBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SuperHTTP [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Surfbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SurveyBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^suzuran [OR,NC]
# russische Suchmaschine
RewriteCond %{HTTP_USER_AGENT} ^StackRambler [OR,NC]
# statcrawler@gmail.com
RewriteCond %{HTTP_USER_AGENT} ^stat [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^tAkeOut [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Tasapspider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Teleport [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Telesoft [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^The.Intraformant [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^TheNomad [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^TightTwatBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Titan [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^toCrawl/UrlDispatcher [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Touche [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^True_Robot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^turingos [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^URLy.Warning [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^VCI [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^VoidEYE [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web\ Image\ Collector [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web.Image.Collector [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web\ Sucker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebAuto [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^WebBandit [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^WebCopier [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^WebEMailExtrac.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebEnhancer [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebFetch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebGo\ IS [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebLeacher [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebmasterWorldForumBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebReaper [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebSauger [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Website\ eXtractor [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Website.Quester [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Webster.Pro [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebStripper [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebWhacker [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^WebZip [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web\ Image\ Collector [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web.Image.Collector [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Wget [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Widow [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WikioFeedBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WikioPxyFeedBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WiseWire.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WorQmada [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WWW-Collector-E [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WWWOFFLE [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Xaldon\ WebSpider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Yandex [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Yanga\ WorldSearch\ Bot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Yeti [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Xenu's [OR,NC]
# Car Search Engine
RewriteCond %{HTTP_USER_AGENT} ^Yotta [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^YoudaoBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Zeus
RewriteRule .* - [F,L]