【原创】Scrapyd 的 .net 客户端

最近项目需要部署Scrapy爬虫，采用最简单的Scrapyd服务进行部署，基于.net core 进行了客户端的封装。

1）Scrapyd API文档：http://scrapyd.readthedocs.io/en/latest/api.html

2）使用HttpClient进行交互

比较麻烦的是部署爬虫所需要的使用 HttpClient 上传文件的代码，使用了  “MultipartFormDataContent”  对象进行上传，添加字符串内容，及StreamContent，直接可以将MVC中的File对象传入，不用自行处理临时文件。

 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));

                content.Add(new StringContent(project), "\"project\"");
                content.Add(new StringContent(version), "\"version\"");
                content.Add(new StreamContent(eggStream), "\"egg\"", "\"" + project + ".egg" + "\"");

                var resp = client.PostAsync("addversion.json", content).Result;

                var strResult = resp.Content.ReadAsStringAsync().Result;

                return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);

完整代码

 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Text;
 using Newtonsoft.Json;
 using Scrapyd.Client.Response;

 namespace Scrapyd.Client
 {
     /// <summary>
     /// A Client used to communicate with the Scrapyd Daemon
     /// </summary>
     public class ScrapydClient
     {
         private readonly string _host;
         private readonly int _port;

         public ScrapydClient(string host, int port)
         {
             _host = host;
             _port = port;
         }

         /// <summary>
         /// 查询状态
         /// </summary>
         /// <returns></returns>
         public DaemonStatusResponse QueryStatus()
         {
             HttpClient client = new HttpClient()
             {
                 BaseAddress = new Uri($"http://{_host}:{_port}")
             };

             try
             {
                 var strResult = client.GetStringAsync("daemonstatus.json").Result;

                 var jresult = JsonConvert.DeserializeObject<DaemonStatusResponse>(strResult);
                 return jresult;
             }
             catch (Exception e)
             {
                 return new DaemonStatusResponse { status = "failed", message = e.Message };
             }
         }

         public AddVersionResponse AddVersion(string project, string version, Stream eggStream)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
             if (eggStream == null) throw new ArgumentNullException(nameof(eggStream));

             var client = new HttpClient()
             {
                 BaseAddress = new Uri($"http://{_host}:{_port}")
             };

             try
             {
                 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));

                 content.Add(new StringContent(project), "\"project\"");
                 content.Add(new StringContent(version), "\"version\"");
                 content.Add(new StreamContent(eggStream), "\"egg\"", "\"" + project + ".egg" + "\"");

                 var resp = client.PostAsync("addversion.json", content).Result;

                 var strResult = resp.Content.ReadAsStringAsync().Result;

                 return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);
             }
             catch (Exception e)
             {
                 return new AddVersionResponse { status = "failed", message = e.Message };
             }
         }

         public ScheduleResponse ScheduleJob(string project, string spider,
             string setting = "",
             string jobid = "",
             string version = "",
             Dictionary<string, string> arguments = null)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
             if (string.IsNullOrEmpty(spider)) throw new ArgumentNullException(nameof(spider));

             var client = new HttpClient()
             {
                 BaseAddress = new Uri($"http://{_host}:{_port}")
             };

             try
             {
                 var data = new Dictionary<string, string>()
                 {
                     {"project", project},
                     {"spider", spider}
                 };

                 if (!string.IsNullOrEmpty(jobid))
                 {
                     data.Add("jobid", jobid);
                 }
                 if (!string.IsNullOrEmpty(setting))
                 {
                     data.Add("setting", setting);
                 }

                 if (!string.IsNullOrEmpty(version))
                 {
                     data.Add("_version", version);
                 }

                 if (arguments != null && arguments.Count > )
                 {
                     foreach (var argument in arguments)
                     {
                         data.Add(argument.Key, argument.Value);
                     }
                 }

                 var strResult =
                     client.PostAsync("schedule.json", new FormUrlEncodedContent(data))
                         .Result.Content.ReadAsStringAsync()
                         .Result;

                 return JsonConvert.DeserializeObject<ScheduleResponse>(strResult);

             }
             catch (Exception e)
             {
                 return new ScheduleResponse { status = "failed", message = e.Message };
             }
         }

         public CancelResponse CancelJob(string project, string jobid)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
             if (string.IsNullOrEmpty(jobid)) throw new ArgumentNullException(nameof(jobid));

             var client = new HttpClient()
             {
                 BaseAddress = new Uri($"http://{_host}:{_port}")
             };

             try
             {
                 var content = new FormUrlEncodedContent(new Dictionary<string, string>()
                 {
                     {"project", project},
                     {"job", jobid}
                 });
                 var strResult = client.PostAsync($"/cancel.json", content).Result.Content.ReadAsStringAsync().Result;

                 return JsonConvert.DeserializeObject<CancelResponse>(strResult);

             }
             catch (Exception e)
             {
                 return new CancelResponse { status = "failed", message = e.Message };
             }
         }

         public ListProjectsResponse ListProjects()
         {
             try
             {
                 var client = new HttpClient
                 {
                     BaseAddress = new Uri($"http://{_host}:{_port}")
                 };
                 var result = client.GetStringAsync("listprojects.json").Result;
                 return JsonConvert.DeserializeObject<ListProjectsResponse>(result);
             }
             catch (Exception e)
             {
                 return new ListProjectsResponse { status = "failed", message = e.Message };
             }
         }

         public ListVersionsResponse ListVersions(string project)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

             try
             {
                 var client = new HttpClient
                 {
                     BaseAddress = new Uri($"http://{_host}:{_port}")
                 };

                 var result = client.GetStringAsync($"listversions.json?project={project}").Result;
                 return JsonConvert.DeserializeObject<ListVersionsResponse>(result);
             }
             catch (Exception e)
             {
                 return new ListVersionsResponse { status = "failed", message = e.Message };
             }
         }

         public ListSpidersResponse ListSpiders(string project, string version = "")
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

             try
             {
                 var client = new HttpClient
                 {
                     BaseAddress = new Uri($"http://{_host}:{_port}")
                 };

                 var result = client.GetStringAsync($"listspiders.json?project={project}&_version={version}").Result;
                 return JsonConvert.DeserializeObject<ListSpidersResponse>(result);
             }
             catch (Exception e)
             {
                 return new ListSpidersResponse { status = "failed", message = e.Message };
             }
         }

         public ListJobsResponse ListJobs(string project)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

             try
             {
                 var client = new HttpClient
                 {
                     BaseAddress = new Uri($"http://{_host}:{_port}")
                 };

                 var result = client.GetStringAsync($"listjobs.json?project={project}").Result;
                 return JsonConvert.DeserializeObject<ListJobsResponse>(result);
             }
             catch (Exception e)
             {
                 return new ListJobsResponse { status = "failed", message = e.Message };
             }
         }

         public ScrapydResponse DeleteVersion(string project, string version)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
             if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version));

             var client = new HttpClient()
             {
                 BaseAddress = new Uri($"http://{_host}:{_port}")
             };

             try
             {
                 var data = new Dictionary<string, string>()
                 {
                     {"project", project},
                     {"version", version}
                 };

                 var strResult =
                     client.PostAsync("delversion.json", new FormUrlEncodedContent(data))
                         .Result.Content.ReadAsStringAsync()
                         .Result;

                 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);

             }
             catch (Exception e)
             {
                 return new ScrapydResponse { status = "failed", message = e.Message };
             }
         }

         public ScrapydResponse DeleteProject(string project)
         {
             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

             var client = new HttpClient()
             {
                 BaseAddress = new Uri($"http://{_host}:{_port}")
             };

             try
             {
                 var data = new Dictionary<string, string>()
                 {
                     {"project", project}
                 };

                 var strResult =
                     client.PostAsync("delproject.json", new FormUrlEncodedContent(data))
                         .Result.Content.ReadAsStringAsync()
                         .Result;

                 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);

             }
             catch (Exception e)
             {
                 return new ScrapydResponse { status = "failed", message = e.Message };
             }
         }

     }

 }

博客地址 : http://www.cnblogs.com/mobwiz/