c# 爬虫常用的3个方法，备份一下
代码语言：javascript复制 1 using System;
2 using System.Collections.Generic;
3 using System.IO;
4 using System.Linq;
5 using System.Net;
6 using System.Text;
7
8 namespace MiSuMi
9 {
10 public class HttpHelper
11 {
12 public CookieContainer cookie;
13 public HttpHelper()
14 {
15 cookie = new CookieContainer();
16 }
17
18 public string Get_Request(
19 string strUrl,
20 CookieContainer _cookie = null,
21 string strHost = "",
22 string strRefer = "",
23 string strOrigin = "",
24 Dictionary<string, string> lstHeads = null,
25 string strEncoding = "utf-8",
26 string strContentType = "",
27 string strAccept = "text/html,application/xhtml xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
28 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
29 bool blnAllowAutoRedirect = true,
30 int intTimeout = 1000 * 30)
31 {
32 HttpWebRequest request;
33 HttpWebResponse response;
34 request = (HttpWebRequest)WebRequest.Create(strUrl);
35 request.Accept = strAccept;
36 request.Timeout = intTimeout;
37 request.Method = "GET";
38 request.Credentials = CredentialCache.DefaultCredentials;
39 request.UserAgent = strUserAgent;
40 request.AllowAutoRedirect = blnAllowAutoRedirect;
41 if (!string.IsNullOrEmpty(strContentType))
42 {
43 request.ContentType = strContentType;
44 }
45 if (_cookie != null)
46 {
47 request.CookieContainer = _cookie;
48 }
49 if (!string.IsNullOrEmpty(strHost))
50 {
51 request.Host = strHost;
52 }
53 if (!string.IsNullOrEmpty(strRefer))
54 {
55 request.Referer = strRefer;
56 }
57 if (!string.IsNullOrEmpty(strOrigin))
58 {
59 request.Headers.Add("Origin", strOrigin);
60 }
61 if (lstHeads != null && lstHeads.Count > 0)
62 {
63 foreach (var item in lstHeads)
64 {
65 request.Headers.Add(item.Key, item.Value);
66 }
67 }
68 response = (HttpWebResponse)request.GetResponse();
69 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
70 string strResult = sr.ReadToEnd();
71 sr.Close();
72 request.Abort();
73 response.Close();
74 return strResult;
75
76 }
77
78 public string POST_Request(
79 string strUrl,
80 string postDataStr,
81 CookieContainer _cookie = null,
82 string strHost = "",
83 string strRefer = "",
84 string strOrigin = "",
85 Dictionary<string, string> lstHeads = null,
86 string strEncoding = "utf-8",
87 string strContentType = "",
88 string strAccept = "text/html,application/xhtml xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
89 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
90 bool blnAllowAutoRedirect = true,
91 int intTimeout = 1000 * 30)
92 {
93 HttpWebRequest request;
94 HttpWebResponse response;
95 request = (HttpWebRequest)WebRequest.Create(strUrl);
96 request.Accept = strAccept;
97 request.Timeout = intTimeout;
98 request.Method = "POST";
99 request.Host = strHost;
100 request.UserAgent = strUserAgent;
101 if (_cookie != null)
102 {
103 request.CookieContainer = _cookie;
104 }
105 request.AllowAutoRedirect = blnAllowAutoRedirect;
106 if (!string.IsNullOrEmpty(strContentType))
107 {
108 request.ContentType = strContentType;
109 }
110 if (!string.IsNullOrEmpty(strOrigin))
111 {
112 request.Headers.Add("Origin", strOrigin);
113 }
114 if (!string.IsNullOrEmpty(strRefer))
115 {
116 request.Referer = strRefer;
117 }
118 if (!string.IsNullOrEmpty(strHost))
119 {
120 request.Host = strHost;
121 }
122 if (lstHeads != null && lstHeads.Count > 0)
123 {
124 foreach (var item in lstHeads)
125 {
126 request.Headers.Add(item.Key, item.Value);
127 }
128 }
129 if (!string.IsNullOrEmpty(postDataStr))
130 {
131 request.ContentLength = postDataStr.Length;
132 Stream myRequestStream = request.GetRequestStream();
133 StreamWriter myStreamWriter = new StreamWriter(myRequestStream);
134 myStreamWriter.Write(postDataStr);
135 myStreamWriter.Close();
136 }
137
138 response = (HttpWebResponse)request.GetResponse();
139 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
140 string strResult = sr.ReadToEnd();
141 sr.Close();
142 request.Abort();
143 response.Close();
144 return strResult;
145 }
146
147 private string DownloadFile(
148 string strURLAddress,
149 string strPath,
150 CookieContainer _cookie = null,
151 string strHost = "",
152 string strRefer = "",
153 string strOrigin = "",
154 Dictionary<string, string> lstHeads = null,
155 string strAccept = "",
156 string strUserAgent = "")
157 {
158 try
159 {
160 // 设置参数
161 HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;
162 if (!string.IsNullOrEmpty(strAccept))
163 {
164 request.Accept = strAccept;
165 }
166 if (!string.IsNullOrEmpty(strUserAgent))
167 {
168 request.UserAgent = strUserAgent;
169 }
170 if (_cookie != null)
171 {
172 request.CookieContainer = _cookie;
173 }
174 if (!string.IsNullOrEmpty(strOrigin))
175 {
176 request.Headers.Add("Origin", strOrigin);
177 }
178 if (!string.IsNullOrEmpty(strRefer))
179 {
180 request.Referer = strRefer;
181 }
182 if (!string.IsNullOrEmpty(strHost))
183 {
184 request.Host = strHost;
185 }
186 if (lstHeads != null && lstHeads.Count > 0)
187 {
188 foreach (var item in lstHeads)
189 {
190 request.Headers.Add(item.Key, item.Value);
191 }
192 }
193 //发送请求并获取相应回应数据
194 HttpWebResponse response = request.GetResponse() as HttpWebResponse;
195 string strReceivePath = string.Empty;
196
197 //直到request.GetResponse()程序才开始向目标网页发送Post请求
198 Stream responseStream = response.GetResponseStream();
199 //创建本地文件写入流
200 Stream stream = new FileStream(strPath, FileMode.Create);
201 byte[] bArr = new byte[1024];
202 int size = responseStream.Read(bArr, 0, (int)bArr.Length);
203 while (size > 0)
204 {
205 stream.Write(bArr, 0, size);
206 stream.Flush();
207 size = responseStream.Read(bArr, 0, (int)bArr.Length);
208 }
209 stream.Close();
210 responseStream.Close();
211 return strPath;
212 }
213 catch (Exception ex)
214 {
215 return "";
216 }
217 }
218
219 }
220 }常用的方法，参数就这些了，后面如果有用到其他的再补充了
