我可以用哪些方法来检测机器人?

您可以在您的页面中放置最终用户不可见或无法点击的链接。许多机器人只是跟随所有链接。一旦有人请求其中一个链接，你几乎肯定有一个爬虫/机器人。

Project Honey Pot保存了一个"坏"机器人的列表。

这是我写的一个类来联系他们的web服务。你将不得不修改一些，因为我有几个专有库在它，但大多数情况下，它应该是好的。有时他们的服务会发回错误，但它确实有助于减少一些不良流量。

using System;
using System.Linq;
using System.Net;
using System.Xml.Linq;
using SeaRisenLib2.Text;
using XmlLib;
/// <summary>
/// Summary description for HoneyPot
/// </summary>
public class HoneyPot
{
    private const string KEY = "blacklistkey"; // blacklist key - need to register at httpbl.org to get it
    private const string HTTPBL = "dnsbl.httpbl.org"; // blacklist lookup host
public HoneyPot()
{
}
public static Score GetScore_ByIP(string ip)
{
    string sendMsg = "", receiveMsg = "";
    int errorCount = 0; // track where in try/catch we fail for debugging
    try
    {
        // for testing: ip = "188.143.232.31";
        //ip = "173.242.116.72";
        if ("127.0.0.1" == ip) return null; // localhost development computer
        IPAddress address;
        if (!IPAddress.TryParse(ip, out address))
            throw new Exception("Invalid IP address to HoneyPot.GetScore_ByIP:" + ip);
        errorCount++; // 1
        string reverseIP = ip.ToArray('.').Reverse().ToStringCSV(".");
        sendMsg = string.Format("{0}.{1}.{2}", KEY, reverseIP, HTTPBL);
        errorCount++; // 2
        //IPHostEntry value = Dns.GetHostByName(sendMsg);
        IPHostEntry value = Dns.GetHostEntry(sendMsg);
        errorCount++; // 3
        address = value.AddressList[0];
        errorCount++; // 4
        receiveMsg = address.ToString();
        errorCount++; // 5
        int[] ipArray = receiveMsg.ToArray('.').Select(s => Convert.ToInt32(s)).ToArray();
        errorCount++; // 6
        if (127 != ipArray[0]) // error
            throw new Exception("HoneyPot error");
        errorCount++; // 7
        Score score = new Score()
        {
            DaysSinceLastSeen = ipArray[1],
            Threat = ipArray[2],
            BotType = ipArray[3]
        };
        errorCount++; // 8
        return score;
    }
    catch (Exception ex)
    {
        Log.Using("VisitorLog/HoneyPotErrors", log =>
        {
            log.SetString("IPrequest", ip);
            log.SetString("SendMsg", sendMsg, XmlFile.ELEMENT);
            log.SetString("RecvMsg", receiveMsg, XmlFile.ELEMENT);
            log.SetString("Exception", ex.Message, XmlFile.ELEMENT);
            log.SetString("ErrorCount", errorCount.ToString());
        });
    }
    return null;
}
// Bitwise values
public enum BotTypeEnum : int
{
    SearchEngine = 0,
    Suspicious = 1,
    Harvester = 2,
    CommentSpammer = 4
}
public class Score
{
    public Score()
    {
        BotType = -1;
        DaysSinceLastSeen = -1;
        Threat = -1;
    }
    public int DaysSinceLastSeen { get; internal set; }
    public int Threat { get; internal set; }
    /// <summary>
    /// Use BotTypeEnum to understand value.
    /// </summary>
    public int BotType { get; internal set; }
    /// <summary>
    /// Convert HoneyPot Score values to String (DaysSinceLastSeen.Threat.BotType)
    /// </summary>
    /// <returns></returns>
    public override string ToString()
    {
        return string.Format("{0}.{1}.{2}",
            DaysSinceLastSeen,
            Threat,
            BotType);
    }
    public static explicit operator XElement(Score score)
    {
        XElement xpot = new XElement("HoneyPot");
        if (null != score)
        {
            if (score.DaysSinceLastSeen >= 0)
                xpot.SetString("Days", score.DaysSinceLastSeen);
            if (score.Threat >= 0)
                xpot.SetString("Threat", score.Threat);
            if (score.BotType >= 0)
                xpot.SetString("Type", score.BotType);
            foreach (BotTypeEnum t in Enum.GetValues(typeof(BotTypeEnum)))
            {
                // Log enum values as string for each bitwise value represented in score.BotType
                int value = (int)t;
                if ((value == score.BotType) || ((value & score.BotType) > 0))
                    xpot.GetCategory(t.ToString());
            }
        }
        return xpot;
    }
    public static explicit operator Score(XElement xpot)
    {
        Score score = null;
        if (null != xpot)
            score = new Score()
            {
                DaysSinceLastSeen = xpot.GetInt("Days"),
                Threat = xpot.GetInt("Threat"),
                BotType = xpot.GetInt("Type")
            };
        return score;
    }
}
/// <summary>
/// Log score value to HoneyPot child Element (if score not null).
/// </summary>
/// <param name="score"></param>
/// <param name="parent"></param>
public static void LogScore(HoneyPot.Score score, XElement parent)
{
    if ((null != score) && (null != parent))
    {
        parent.Add((XElement)score);
    }
}

}

虽然从技术上讲，它不会"检测"bot爬虫，但我有一个有趣的方法来阻止它们。我的方法是创建一个IIS过滤器或Apache插件。你要做的是加密你所有的html, asp, php等…页面。唯一未加密的页面是索引页面。索引页只是安装一个带有加密公钥的cookie，然后重定向到第二个索引页。然后，IIS过滤器或Apache插件将检查每个访问者，以确保他们拥有这个cookie。如果是这样，过滤器将解密请求的页面，然后将页面传递到web服务器进行处理。

这种方法将允许普通访问者查看您的网页，但如果拒绝cookie的bot试图读取您的网页，它们将被加密。

黑名单可能不是一个好方法，最好有一个已知机器人的白名单，允许每秒超过一定数量的点击。如果不在白名单上的人每秒点击次数太多，就开始中断他们的连接几秒钟。这将有助于防止滥用，仍然让未知的机器人扫描你的网站(尽管比你认为重要的慢得多)。

你可以把违规者记录下来，看看谁是反复违反规则的:)