//allow auto redirects from redirect headers request.AllowAutoRedirect=true;
//maximum of 10 auto redirects request.MaximumAutomaticRedirections=10;
//30 second timeout for request request.Timeout=(int) new TimeSpan(0,0,60).TotalMilliseconds;
//give the crawler a name. request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"; //request.UserAgent="Mozilla/3.0 (compatible; My Browser/1.0)";