用C#写一个采集器、蜘蛛

效果图：


代码如下：



using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Web;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;


namespace chinaz
{
    class Program
    {
        static void Main(string[] args)
        {

            string cookie = null;
            using (StreamReader sr = new StreamReader("cookie.txt"))
            {
                cookie = sr.ReadToEnd();
                sr.Close();
            }
            //string tmp = SRWebClient.GetPage("http://bbs.chinaz.com/Members.html?page=1&sort=CreateDate&desc=true&keyword=", Encoding.UTF8, cookie);
            int a = int.Parse(Console.ReadLine());
            int b = int.Parse(Console.ReadLine());
            string url = Console.ReadLine();

            Hashtable hash = new Hashtable();
            Encoding encoding = Encoding.GetEncoding(Console.ReadLine());

            for (int i = a; i <= b; i++)
            {
                string html = SRWebClient.GetPage(string.Format(url, i), encoding, cookie);
                //Console.WriteLine(html);
                if (html != null && html.Length > 1000)
                {
                    Match m = Regex.Match(html, @"\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*", RegexOptions.Compiled | RegexOptions.IgnoreCase);
                    while (m != null && m.Value != null && m.Value.Trim() != string.Empty)
                    {
                        if (!hash.Contains(m.Value))
                        {
                            Console.WriteLine(m.Value);
                            using (StreamWriter sw = new StreamWriter("mail.txt", true))
                            {
                                sw.WriteLine(m.Value);
                                sw.Close();
                            }
                            hash.Add(m.Value, string.Empty);
                        }
                        m = m.NextMatch();
                    }

                }
            }



            Console.Write("完成");
            Console.ReadLine();
        }
    }


    public class SRWebClient
    {
        public CookieCollection cookie;
        public SRWebClient()
        {
            cookie = null;
        }

        #region 从包含多个 Cookie 的字符串读取到 CookieCollection 集合中
        private static void AddCookieWithCookieHead(ref CookieCollection cookieCol, string cookieHead, string defaultDomain)
        {
            if (cookieCol == null) cookieCol = new CookieCollection();
            if (cookieHead == null) return;
            string[] ary = cookieHead.Split(';');
            for (int i = 0; i < ary.Length; i++)
            {
                Cookie ck = GetCookieFromString(ary[i].Trim(), defaultDomain);
                if (ck != null)
                {
                    cookieCol.Add(ck);
                }
            }
        }
        #endregion

        #region 读取某一个 Cookie 字符串到 Cookie 变量中
        private static Cookie GetCookieFromString(string cookieString, string defaultDomain)
        {
            string[] ary = cookieString.Split(',');
            Hashtable hs = new Hashtable();
            for (int i = 0; i < ary.Length; i++)
            {
                string s = ary[i].Trim();
                int index = s.IndexOf("=");
                if (index > 0)
                {
                    hs.Add(s.Substring(0, index), s.Substring(index + 1));
                }
            }
            Cookie ck = new Cookie();
            foreach (object Key in hs.Keys)
            {
                if (Key.ToString() == "path") ck.Path = hs[Key].ToString();

                else if (Key.ToString() == "expires")
                {
                    //ck.Expires=DateTime.Parse(hs[Key].ToString();
                }
                else if (Key.ToString() == "domain") ck.Domain = hs[Key].ToString();
                else
                {
                    ck.Name = Key.ToString();
                    ck.Value = hs[Key].ToString();
                }
            }
            if (ck.Name == "") return null;
            if (ck.Domain == "") ck.Domain = defaultDomain;
            return ck;
        }
        #endregion



        /**/
        /// <TgData>
        ///     <Alias>下载Web源代码</Alias>
        /// </TgData>
        public string DownloadHtml(string URL, bool CreateCookie)
        {
            try
            {
                HttpWebRequest request = HttpWebRequest.Create(URL) as HttpWebRequest;
                if (cookie != null)
                {
                    request.CookieContainer = new CookieContainer();
                    request.CookieContainer.Add(cookie);
                }
                request.AllowAutoRedirect = false;
                //request.MaximumAutomaticRedirections = 3;
                request.Timeout = 20000;

                HttpWebResponse res = (HttpWebResponse)request.GetResponse();
                string r = "";

                System.IO.StreamReader S1 = new System.IO.StreamReader(res.GetResponseStream(), System.Text.Encoding.Default);
                try
                {
                    r = S1.ReadToEnd();
                    if (CreateCookie)
                        cookie = res.Cookies;
                }
                catch (Exception er)
                {
                    //Log l = new Log();
                    //l.writelog("下载Web错误", er.ToString());
                }
                finally
                {
                    res.Close();
                    S1.Close();
                }

                return r;
            }

            catch
            {

            }

            return string.Empty;
        }

        /**/
        /// <TgData>
        ///     <Alias>下载文件</Alias>
        /// </TgData>
        public long DownloadFile(string FileURL, string FileSavePath, bool CreateCookie)
        {
            long Filelength = 0;
            HttpWebRequest req = HttpWebRequest.Create(FileURL) as HttpWebRequest;

            if (cookie != null)
            {
                req.CookieContainer = new CookieContainer();
                req.CookieContainer.Add(cookie);
            }
            req.AllowAutoRedirect = true;

            HttpWebResponse res = req.GetResponse() as HttpWebResponse;
            if (CreateCookie)
                cookie = res.Cookies;
            System.IO.Stream stream = res.GetResponseStream();
            try
            {
                Filelength = res.ContentLength;

                byte[] b = new byte[512];

                int nReadSize = 0;
                nReadSize = stream.Read(b, 0, 512);

                System.IO.FileStream fs = System.IO.File.Create(FileSavePath);
                try
                {
                    while (nReadSize > 0)
                    {
                        fs.Write(b, 0, nReadSize);
                        nReadSize = stream.Read(b, 0, 512);
                    }
                }
                finally
                {
                    fs.Close();
                }
            }
            catch (Exception er)
            {
                //Log l = new Log();
                //l.writelog("下载文件错误", er.ToString());
            }
            finally
            {
                res.Close();
                stream.Close();
            }

            return Filelength;
        }

        /**/
        /// <TgData>
        ///     <Alias>提交数据</Alias>
        /// </TgData>
        public string Request(string RequestPageURL, RequestData Data, bool CreateCookie)
        {
            StreamReader reader = null;
            HttpWebResponse response = null;
            HttpWebRequest request = null;
            try
            {
                string StrUrl = RequestPageURL;
                request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;

                string postdata = Data.GetData();
                request.Referer = RequestPageURL;
                request.AllowAutoRedirect = false;
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
                request.Timeout = 20000;

                if (cookie != null)
                {
                    request.CookieContainer = new CookieContainer();
                    request.CookieContainer.Add(cookie);
                }

                Uri u = new Uri(StrUrl);

                if (postdata.Length > 0) //包含要提交的数据　就使用Post方式
                {
                    request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
                    request.Method = "POST";        //方式就是Post

                    //把提交的数据换成字节数组
                    Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
                    request.ContentLength = B.Length;

                    System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
                    SW.Write(B, 0, B.Length);
                    SW.Close();
                }

                response = request.GetResponse() as HttpWebResponse;
                if (CreateCookie)
                    //cookie = response.Cookies;
                    AddCookieWithCookieHead(ref cookie, response.Headers["Set-Cookie"], request.RequestUri.Host);
                reader = new StreamReader(response.GetResponseStream(), Encoding.Default);

                return reader.ReadToEnd();
            }
            catch (Exception ex)
            {
                string x = ex.StackTrace;
            }
            finally
            {
                if (response != null)
                    response.Close();
            }

            return string.Empty;
        }


        public bool PostDownload(RequestData Data, out string file)
        {
            file = null;
            StreamReader reader = null;
            HttpWebResponse response = null;
            HttpWebRequest request = null;
            try
            {
                string StrUrl = "http://www.imobile.com.cn/wapdiyringdownload.php";
                request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;

                string postdata = Data.GetData();
                request.Referer = StrUrl;
                request.AllowAutoRedirect = false;
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
                request.Timeout = 20000;

                if (cookie != null)
                {
                    request.CookieContainer = new CookieContainer();
                    request.CookieContainer.Add(cookie);
                }

                Uri u = new Uri(StrUrl);

                if (postdata.Length > 0) //包含要提交的数据　就使用Post方式
                {
                    request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
                    request.Method = "POST";        //方式就是Post

                    //把提交的数据换成字节数组
                    Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
                    request.ContentLength = B.Length;

                    System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
                    SW.Write(B, 0, B.Length);
                    SW.Close();
                }

                response = request.GetResponse() as HttpWebResponse;
                string des = response.Headers["Content-Disposition"].Trim();
                file = des.Substring(des.IndexOf("filename=") + 9);
                file = new Random().Next(100).ToString() + "/" + file;

                System.IO.Stream stream = response.GetResponseStream();
                try
                {
                    int Filelength = (int)response.ContentLength;

                    byte[] b = new byte[512];

                    int nReadSize = 0;
                    nReadSize = stream.Read(b, 0, 512);

                    System.IO.FileStream fs = System.IO.File.Create("f:/mobileMusic/" + file);
                    try
                    {
                        while (nReadSize > 0)
                        {
                            fs.Write(b, 0, nReadSize);
                            nReadSize = stream.Read(b, 0, 512);
                        }
                    }
                    finally
                    {
                        fs.Close();
                    }
                }
                catch (Exception er)
                {
                    //Log l = new Log();
                    //l.writelog("下载文件错误", er.ToString());
                }
                finally
                {
                    response.Close();
                    stream.Close();
                }
            }
            catch (Exception ex)
            {
                string x = ex.StackTrace;
            }
            finally
            {
                if (response != null)
                    response.Close();
            }
            return true;
        }
        #region GetPage
        /// <summary>
        /// 获取源代码
        /// </summary>
        /// <param name="url"></param>
        /// <param name="coding"></param>
        /// <param name="TryCount"></param>
        /// <returns></returns>
        public static string GetPage(string url, Encoding encoding, int TryCount)
        {
            for (int i = 0; i < TryCount; i++)
            {
                string result = GetPage(url, encoding, null);
                if (result != null && result != string.Empty)
                    return result;
            }

            return string.Empty;
        }

        /// <summary>
        /// 获取源代码
        /// </summary>
        /// <param name="url"></param>
        /// <param name="coding"></param>
        /// <returns></returns>
        public static string GetPage(string url, Encoding encoding, string cookie)
        {
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(url);
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2;)";
                request.Timeout = 20000;
                request.AllowAutoRedirect = false;
                if (cookie != null)
                    request.Headers["Cookie"] = cookie;

                response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
                {
                    reader = new StreamReader(response.GetResponseStream(), encoding);
                    string html = reader.ReadToEnd();

                    return html;
                }
            }
            catch
            {
            }
            finally
            {

                if (response != null)
                {
                    response.Close();
                    response = null;
                }
                if (reader != null)
                    reader.Close();

                if (request != null)
                    request = null;

            }

            return string.Empty;
        }
        #endregion
    }

    public class RequestData
    {
        Hashtable hash = new Hashtable();

        public RequestData()
        {

        }

        public string GetData()
        {
            string r = "";

            foreach (string key in hash.Keys)
            {
                if (r.Length > 0) r += "&";
                r += key + "=" + hash[key];
            }

            return r;
        }

        public void AddField(string Field, string Value)
        {
            hash[Field] = Value;
        }


    }
}

文章来自: 本站原创
引用通告: 查看所有引用 | 我要引用此文章
Tags: