需求分析：本人最近做一個項目，項目中需要從新聞的索引頁（就是上面有很多鏈接的那種網(wǎng)頁），獲取新聞?wù)捻撛创a，并將新聞?wù)捻撛创a保存到本地數(shù)據(jù)庫中。
    但是由于網(wǎng)絡(luò)穩(wěn)定性的原因，總會出現(xiàn) 404 page not found 類型的error。（但是網(wǎng)頁是確確實實存在的）。而且這種錯誤，往往是在程序運行一段時間后出現(xiàn)的，覺得很不可思議。我在網(wǎng)絡(luò)上查這種問題的解決方案時，發(fā)現(xiàn)沒有一種管用的。本人現(xiàn)在已經(jīng)成功解決該問題，遂將自己的解決方案寫下來和大家分享與探討。
解決方案核心：一旦出現(xiàn)這種錯誤，程序中就遞歸調(diào)用下載函數(shù)本身。代碼說明如下：

 public static string GetDataFromUrl(string url, int nRetryTimes)
        {
            if (nRetryTimes == 0)
                 return string.Empty;

            string result = string.Empty;
            try
            {
                result=GetDataFromUrl(url);
            }
            catch (System.Exception exc)
            {
                if(exc.Message.IndexOf("404")!=-1)
                {
                    result=GetDataFromUrl(url,nRetryTimes-1);
                }
            }
            return result;
        }
其中nRetryTimes 代表出現(xiàn)這種錯誤后，函數(shù)遞歸調(diào)用自己的次數(shù)，也可以理解為遞歸終止的條件。GetDataFromUrl(string url)函數(shù)代碼如下：

 public static string GetDataFromUrl(string url)
       {
           string str = string.Empty;
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
            //設(shè)置http頭
            request.AllowAutoRedirect = true;
            request.AllowWriteStreamBuffering = true;
            request.Referer = "";
            request.Timeout = 10000000;
            request.UserAgent = "";
            ;
            request.KeepAlive = false;//to avoid the error of time out
            HttpWebResponse response = null;
             response = (HttpWebResponse)request.GetResponse();

            //根據(jù)http應(yīng)答的http頭來判斷編碼
            string characterSet = response.CharacterSet;
            Encoding encode;
            if (characterSet != "")
            {
                if (characterSet == "ISO-8859-1")
                {
                    characterSet = "gb2312";
                }
                encode = Encoding.GetEncoding(characterSet);
            }
            else
            {
                encode = Encoding.Default;
             }

            //聲明一個內(nèi)存流來保存http應(yīng)答流
            Stream receiveStream = response.GetResponseStream();
             MemoryStream mStream = new MemoryStream();

            byte[] bf = new byte[255];
            int count = receiveStream.Read(bf, 0, 255);
            while (count > 0)
            {
                mStream.Write(bf, 0, count);
                count = receiveStream.Read(bf, 0, 255);
            }
             receiveStream.Close();

            mStream.Seek(0, SeekOrigin.Begin);

            //從內(nèi)存流里讀取字符串
            StreamReader reader = new StreamReader(mStream, encode);
            char[] buffer = new char[1024];
            count = reader.Read(buffer, 0, 1024);
            while (count > 0)
            {
                str += new String(buffer, 0, count);
                count = reader.Read(buffer, 0, 1024);
             }

            //從解析出的字符串里判斷charset，如果和http應(yīng)答的編碼不一直
            //那么以頁面聲明的為準，再次從內(nèi)存流里重新讀取文本
            Regex reg =
               new Regex(@"<meta[\s\S]+?charset=(.*?)""[\s\S]+?>",
                          RegexOptions.Multiline | RegexOptions.IgnoreCase);
            MatchCollection mc = reg.Matches(str);
            if (mc.Count > 0)
            {
                string tempCharSet = mc[0].Result("$1");
                if (string.Compare(tempCharSet, characterSet, true) != 0)
                {
                    encode = Encoding.GetEncoding(tempCharSet);
                    str = string.Empty;
                    mStream.Seek(0, SeekOrigin.Begin);
                    reader = new StreamReader(mStream, encode);
                    buffer = new char[255];
                    count = reader.Read(buffer, 0, 255);
                    while (count > 0)
                    {
                        str += new String(buffer, 0, count);
                        count = reader.Read(buffer, 0, 255);
                    }
                }
            }
            reader.Close();
            mStream.Close();
            if (response != null)
                 response.Close();

            return str;

       
 
         
       }

值得說明的是：盡管采用了此方法，當你查看數(shù)據(jù)庫的時候，你還是會發(fā)現(xiàn)有些正文源碼沒有下載下來。拿我的數(shù)據(jù)表單來說：我的數(shù)據(jù)庫表單的各個屬性如下 ArticlePageId,--數(shù)據(jù)表的主鍵。ArticlePageTitle--新聞標題，ArticlePageUrl,--新聞?wù)捻揢RL，ArticlePageSource--新聞?wù)捻撛创a，也就是從ArticlePageUrl下載的源碼。如果ArticlePageSource字段為空，則表明，下載失敗。于是，我又加了一個打補丁的模塊。代碼如下：

把補丁的模塊代碼
 public static void PatchingUp(string desTable)
        {
             string conStr = "server=(local);database=ArticleCollection;uid=sa;pwd=123456";
             string des = desTable;
             string select = string.Format("SELECT ArticlePageId,ArticlePageSource,ArticlePageUrl from {0}", des);
             SqlConnection connection = new SqlConnection(conStr);
             SqlDataAdapter adapter = new SqlDataAdapter(select, connection);
             SqlCommandBuilder builder = new SqlCommandBuilder(adapter);
             DataTable table = new DataTable();
            adapter.Fill(table);
             bool isModified = false;
             for (int i = 0; i < table.Rows.Count; i++)
            {
                 if (table.Rows[i]["ArticlePageSource"].ToString() == null || table.Rows[i]["ArticlePageSource"].ToString() == "")
                {
                     Console.WriteLine("在原數(shù)據(jù)庫第{0}行插入失敗", i+1);
                     string url = table.Rows[i]["ArticlePageUrl"].ToString().Trim();
                     table.Rows[i]["ArticlePageSource"] = ClawlerAssist.GetDataFromUrl(url, 5);

                     Console.WriteLine("在原數(shù)據(jù)庫第{0}行更新成功", i+1);
                     if (!isModified)
                    {
                         isModified = true;
                    }

                }
            }
             if (isModified)
            {
                adapter.Update(table);
            }    
 
         }