OprGrab(Guid ruleId) 2 { 3 using (var context = new SupportYunDBContext()) 4 { 5 var ruleInfo = context.CollectionRule.Find(ruleId); 6 if (ruleInfo == null) 7 { Exception(); 9 } listUrl = ruleInfo.ListUrlRule.Split( }); 12 // 获取列表 13 //var client = new HttpClient(); 14 //HttpContent content = new StringContent("flag=" + listUrl[0] + "&uuid=" + listUrl[1],Encoding.UTF8); responseJson = + listUrl[+ listUrl[1], 17 Encoding.UTF8); 18 var responseModel = new JavaScriptSerializer().Deserialize<ResponseModel>(responseJson); (responseModel.success == && responseModel.value != null && 21 (responseModel.value.lastestArticle.Any() || responseModel.value.topArticle.Any())) 22 { 23 var newList = new List<DataModel>(); 24 newList.AddRange(responseModel.value.lastestArticle); 25 newList.AddRange(responseModel.value.topArticle); 26 newList = newList.Distinct(i => i.title, StringComparer.CurrentCultureIgnoreCase).ToList(); 27 foreach (var list in newList) 28 { 29 angleSharpGrabService.GrabDetail(ruleInfo, list.url, list.title, context); 30 } 31 } 32 } 33 }

　　其中第18行，是讲json字符串序列化为对应的object。需要引用System.Web.Extensions

　　注释掉的几行是准备使用HttpClient来实现Post请求，但该接口使用该方法一直报错，无法获取参数。故而自己写了一个PushToWeb方法来实现Post请求：　　

PushToWeb(string weburl, string data, Encoding encode) 2 { 3 var byteArray = encode.GetBytes(data); webRequest = (HttpWebRequest)WebRequest.Create(new Uri(weburl)); ; ; 8 webRequest.ContentLength = byteArray.Length; 9 var newStream = webRequest.GetRequestStream(); 10 newStream.Write(byteArray, 0, byteArray.Length); 11 newStream.Close(); response = (HttpWebResponse)webRequest.GetResponse(); 15 var aspx = new StreamReader(response.GetResponseStream(), encode); 16 return aspx.ReadToEnd(); 17 }

　　抓取详情页的方法GrabDetail，是重构AngleSharpGrabService服务的OprGrab方法提取出来的，具体的AngleSharpGrabService服务源码及思路请参考第二篇文章：

　　记一次企业级爬虫系统升级改造（二）：基于AngleSharp实现的抓取服务

GrabDetail(CollectionRule ruleInfo, string realUrl,string title,SupportYunDBContext context) 2 { 3 if (!IsRepeatedGrab(realUrl, ruleInfo.Id, title)) 4 { 5 string contentDetail = GetHtml(realUrl, ruleInfo.GetCharset()); 6 var detailModel = DetailAnalyse(contentDetail, title, ruleInfo); (!string.IsNullOrEmpty(detailModel.FullContent)) 9 { 10 var ruleModel = context.CollectionRule.Find(ruleInfo.Id); 11 ruleModel.LastGrabTime = DateTime.Now; 12 var newData = new CollectionInitialData() 13 { 14 CollectionRule = ruleModel, 15 CollectionType = ruleModel.CollectionType, 16 Title = detailModel.Title, 17 FullContent = detailModel.FullContent, 18 Url = realUrl, 19 ProcessingProgress = ProcessingProgress.未处理 20 }; 21 if (!IsRepeatedGrab(realUrl, ruleInfo.Id, newData.Title)) 22 { 23 context.CollectionInitialData.Add(newData); 24 context.SaveChanges(); 25 } 26 } 27 } 28 }

 　　该服务我关联到了每日查询6次的group下面，windows服务就会自动每日扫描配置了该api规则的公众号6次，以满足尽快抓取公众号最新文章的需求。

如果你交给某人一个程序，你将折磨他一整天；如果你教给某人如何编写程序， 你将折磨他一辈子！

　　哈哈，我想到了从小听到大的至理名言：授人以鱼，不如授人以渔！

　　过程中参考了很多资料，这里列出博主认为收获最多的一个链接，以供大家参考：

　　 https://www.zhihu.com/question/31285583

原创文章，代码都是从自己项目里贴出来的。转载请注明出处哦，亲~~~