private string RequestGet(string TheURL, string TheProxy) { Uri uri = new Uri(TheURL); HttpWebRequest request = (HttpWebRequest) WebRequest.Create(uri); string page; try {
request.KeepAlive = false; request.ProtocolVersion=HttpVersion.Version10;
request.Method = "GET";
request.ContentType = "application/x-www-form-urlencoded";
request.Proxy = System.Net.WebProxy.GetDefaultProxy();
//allow auto redirects from redirect headers request.AllowAutoRedirect=true;
//maximum of 10 auto redirects request.MaximumAutomaticRedirections=10;
//30 second timeout for request request.Timeout=(int) new TimeSpan(0,0,60).TotalMilliseconds;
//give the crawler a name. request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"; //request.UserAgent="Mozilla/3.0 (compatible; My Browser/1.0)";
HttpWebResponse response = (HttpWebResponse) request.GetResponse(); Stream responseStream = response.GetResponseStream(); StreamReader readStream = new StreamReader (responseStream, System.Text.Encoding.Default);
page = readStream.ReadToEnd(); } catch (Exception ee) { page = "Fail message : "+ee.Message; } return page;
}
|