c#使用控制台爬取笔趣阁小说,以下为效果图
以下为完整代码
using System; using System.IO; using System.Net; using System.Text; using System.Text.RegularExpressions; namespace ConsoleApp3 { class Program { static void Main(string[] args) { string searchbook = "https://www.biqugeu.net/searchbook.php?keyword=<<bookname>>"; string searchurl = null; string searchcontent = null; string baseurl = "https://www.biqugeu.net/"; string nextChapter = null; string html = null; string bookname = null; string bookTitle = null; string ChapterContent; string regex1 = "<h1>(?<bookname>.*?)</h1>"; string regex2 = "<a href=\"/.*?\" target=\"_top\" class=\"pre\">上一章</a> ← <a href=\"/.*?/\" target=\"_top\" title=\"\" class=\"back\">章节列表</a> → <a href=\"(?<nextChapter>.*?)\" target=\"_top\" class=\"next\""; string regex3 = "booktitle = \"(?<booktitle>.*?)\";"; string regex4 = "(?<data>.*?)<br/><br/>"; string regex5 = "<div class=\"image\">\\s*<a href=\"/(?<bookurl>.*?)\""; string regex6 = "<dt>.*?</dt><dd><ahref=\"/(?<bookfirst>.*?)\">.*?</a></dd>"; Console.WriteLine("请输入需要爬取的小说!"); string novelName = Console.ReadLine(); try { searchurl = searchbook.Replace("<<bookname>>", novelName); HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(searchurl); req1.Method = "GET"; req1.Accept = "text/html"; req1.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"; HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse(); using (StreamReader reader = new StreamReader(res1.GetResponseStream())) { html = reader.ReadToEnd(); if (!string.IsNullOrEmpty(html)) { //Console.WriteLine(html); html = html.Replace("\n", "").Replace("\t", "").Replace("\r", ""); searchcontent = Regex.Match(html, regex5).Groups["bookurl"].ToString(); if (searchcontent == "") { Console.WriteLine("没有找到该小说!"); } searchurl = baseurl + searchcontent; } } } catch (WebException we) { Console.WriteLine(we.Message); } try { HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(searchurl); req1.Method = "GET"; req1.Accept = "text/html"; req1.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"; HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse(); using (StreamReader reader = new StreamReader(res1.GetResponseStream())) { html = reader.ReadToEnd(); if (!string.IsNullOrEmpty(html)) { //Console.WriteLine(html); html = html.Replace("\n", "").Replace("\t", "").Replace("\r", "").Replace(" ",""); searchcontent = Regex.Matches(html, regex6)[1].Groups["bookfirst"].ToString(); searchurl = baseurl + searchcontent; } } } catch (Exception) { throw; } do { restart: try { HttpWebRequest req = (HttpWebRequest)WebRequest.Create(searchurl); req.Method = "GET"; req.Accept = "text/html"; req.AllowAutoRedirect = true; req.Headers.Add("Encoding", Encoding.UTF8.ToString()); req.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)"; HttpWebResponse res = (HttpWebResponse)req.GetResponse(); using (StreamReader reader = new StreamReader(res.GetResponseStream())) { html = reader.ReadToEnd(); if (!string.IsNullOrEmpty(html)) { ChapterContent = ""; //获取下一章 nextChapter = Regex.Match(html, regex2).Groups["nextChapter"].ToString(); searchurl = baseurl + nextChapter; //获取章节名 bookname = Regex.Match(html, regex1).Groups["bookname"].ToString(); ChapterContent += "\r\n"; ChapterContent += bookname; ChapterContent += "\r\n"; //获取书名 bookTitle = Regex.Match(html, regex3).Groups["booktitle"].ToString(); //获取内容 MatchCollection match = Regex.Matches(html, regex4); foreach (Match item in match) { string book = Regex.Match(item.Value, regex4).Groups["data"].ToString().Trim(); ChapterContent += book; } Console.WriteLine(bookname + "-------下载完毕!"); AddBookToTXT(ChapterContent, bookTitle); } } } catch (WebException we) { //Console.WriteLine(we.Message); Console.WriteLine("远程主机强迫关闭了一个现有的连接,重新爬取当前章节。。。"); goto restart; } } while (nextChapter.Contains("html"));//当下一章链接没有跳转时结束 } /// <summary> /// 将内容保存到txt文件 /// </summary> /// <param name="logstring">内容</param> /// <param name="pathName">书名</param> public static void AddBookToTXT(string logstring, string pathName) { string path = AppDomain.CurrentDomain.BaseDirectory + pathName + ".txt"; if (!System.IO.File.Exists(path)) { FileStream stream = System.IO.File.Create(path); stream.Close(); stream.Dispose(); } using (StreamWriter writer = new StreamWriter(path, true)) { writer.WriteLine(logstring); } } } }