如何使用XPath来挖掘(别人的)编码糟糕的HTML
本文关键字:编码 HTML 别人 XPath 何使用 | 更新日期: 2023-09-27 18:16:38
当我执行这段c#代码时…
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.IO;
using System.Net;
using System.Dynamic;
using HtmlAgilityPack;
namespace entropedizer
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
public String postRequest(string url, string eventTarget)
{
// A "pre-request," sent to gather SessionID and POST data parameters for the main request
HttpWebRequest prequest = (HttpWebRequest)WebRequest.Create("http://www.entropedia.info/Chart.aspx?chart=Chart");
HttpWebResponse presponse = (HttpWebResponse)prequest.GetResponse();
Stream pstream = presponse.GetResponseStream();
StreamReader psr = new StreamReader(pstream);
string phtml = psr.ReadToEnd();
Match viewstate = Regex.Match(phtml, "id='"__VIEWSTATE'".+/>");
Match eventvalidation = Regex.Match(phtml, "id='"__EVENTVALIDATION'".+/>");
ASCIIEncoding encoding = new ASCIIEncoding();
string postData = "__EVENTTARGET=" + eventTarget + "&__VIEWSTATE=" + Uri.EscapeDataString(viewstate.ToString().Substring(24, viewstate.Length - 28)) + "&__EVENTVALIDATION=" + Uri.EscapeDataString(eventvalidation.ToString().Substring(30, eventvalidation.Length - 34));
byte[] data = encoding.GetBytes(postData);
// The main request, intended to retreive the desired HTML
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.entropedia.info/Chart.aspx?chart=Chart");
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.CookieContainer = new CookieContainer();
Cookie sessionId = new Cookie("ASP.NET_SessionId", Regex.Match(presponse.Headers.ToString(), "ASP.NET_SessionId=.+ d").ToString().Substring(18, Regex.Match(presponse.Headers.ToString(), "ASP.NET_SessionId=.+ d").Length - 21), "/", ".entropedia.info");
request.CookieContainer.Add(new Uri("http://www.entropedia.info/Chart.aspx?chart=Chart"), sessionId);
Stream stream = request.GetRequestStream();
stream.Write(data, 0, data.Length);
stream.Close();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
stream = response.GetResponseStream();
StreamReader sr = new StreamReader(stream);
return sr.ReadToEnd();
}
private void Form1_Load(object sender, EventArgs e)
{
System.Net.ServicePointManager.Expect100Continue = false;
HtmlAgilityPack.HtmlDocument hChart = new HtmlAgilityPack.HtmlDocument();
hChart.LoadHtml(postRequest("http://www.entropedia.info/Chart.aspx?chart=Chart", "ctl00%24ContentPlaceHolder1%24DG1%24ctl19%24ctl05"));
HtmlNodeCollection chartStrings = hChart.DocumentNode.SelectNodes("/");
if (chartStrings != null)
{
foreach (HtmlNode i in chartStrings)
{
System.IO.File.WriteAllText("C:/Users/Admin/Desktop/WholeDocument.txt", i.OuterHtml);
}
}
else
{
MessageBox.Show("Error: Null item list.");
}
}
}
}
…下面的HTML被写入一个文本文件。
http://pastebin.com/FALerBWR当我将c#代码中的行更改为HtmlNodeCollection chartStrings = hChart.DocumentNode.SelectNodes("/html/body");
时,主体内的400多行HTML将被写入文本文件。
当我将行更改为HtmlNodeCollection chartStrings = hChart.DocumentNode.SelectNodes("/html/body/form");
时,只有一行代码(带有其属性的表单开始标记)被写入文本文件。它应该写很多行(大部分文档)。我相信HtmlAgilityPack
变得混乱,由于畸形的HTML标签。是否有一种方法可以程序化地解决这个问题?我不想每次运行程序时都手动更正HTML !
这是一个"按设计"的行为。默认情况下,FORM
被视为empty
HTML元素。原因在这里解释(查看我的答案):出于某种原因关闭了自己?
但这也是可配置的,你只需要指示解析器的行为不同,像这样:
HtmlAgilityPack.HtmlDocument hChart = new HtmlAgilityPack.HtmlDocument();
// remove all specific behaviors for the `FORM` element
HtmlAgilityPack.HtmlNode.ElementsFlags.Remove("form");
hChart.LoadHtml(postRequest("http://www.entropedia.info/Chart.aspx?chart=Chart", "ctl00%24ContentPlaceHolder1%24DG1%24ctl19%24ctl05"));
HtmlNodeCollection chartStrings = hChart.DocumentNode.SelectNodes("/");
如果您认为这是由错误的html引起的,您应该首先整理html。这里有一些你可以用的东西…https://github.com/markbeaton/TidyManaged