这几天在部署代理池的时候，用Python写了requests测试IP地址检测连通性的脚本。但是我发现一个问题，requests.getwithproxyrequest有时会失败。一开始以为是代理的问题，后来用curl请求发现代理正常，又用Go写了个测试，发现还是正常的。是请求的问题吗？一时不知道原因，后来用Go写了一个代理测试，于是找到了一个Go语言友好的Http请求包——Gorequest。Go语言中net/http代理请求net/http请求的整体流程并不复杂，使用nrt/http包的get和post方法即可实现。但是在配置代理上，需要单独配置Client客户端：1.//发送请求2.rqt,err:=http.NewRequest("GET",testApi,nil)3.iferr!=nil{4.fmt.Println(err)5.return6.}7.//配置代理8.client:=&http.Client{9.传输：&http.Transport{10.代理：http.ProxyURL(urlProxy),11.},12.}13.response,err:=client.Do(rqt)14.iferr!=nil{15.fmt.Println(err)16.panic(err)17.return18.}直到我找到GorequestGorequest$gogetgithub.com/parnurzeal/gorequest//安装getrequest:1.request:=gorequest.New()2.resp,body,errs:=request.Get(url).End()gorequest代理，很简单1.request:=gorequest.New().Proxy("Proxy")Gorequest获取代理并执行测试。获取代理方法定义一个getRes函数，接收url，发起http请求，返回请求内容1.funcgetRes(urlstring)string{2.deferfunc(){3.err:=recover()4.iferr!=nil{5.fmt.Println(time.Now().Format("2006-01-0215:04:0507"),"【http错误】","返回信息：",err)6.}7.}()8.//获取代理9._,body,errs:=gorequest.New().Get(url).End()10.iferrs!=nil{11.panic(errs)12.}13.returnbody14.}定义一个getIp方法获取代理并处理返回的json（先定义一个结构体处理json）1.typereqinfostruct{2.codeint//返回结果码3.successbool//是否成功参数请求成功4.Msg,RequestIpstring//返回信息和本地请求的IP地址5.Data[]map[string]interface{}//返回的IP类型为带key的字符串，value是一个空接口映射组成的数组6.}1.funcgetiP(getipUrlstring){2.deferfunc(){3.err:=recover()4.iferr!=nil{5.fmt.Println(time).Format("2006-01-0215:04:0507"),"【http错误】","返回信息：",err)6.}7.}()8.body:=getRes(getipUrl)9.fmt.Println(body)10.//处理json11.varinforeqinfo12.err:=json.Unmarshal([]byte(body),&info)13.iferr!=nil{m.t14.Println("jsonerror",err)15.}16.}for循环并使用goroutine多线程检测IP1.for_,v:=rangeinfo.Data{2.IP:=v["ip"]3.port:=v["port"]4.proxyUrl:=fmt.Sprint("http://",IP,":",port)5.fmt.Println(proxyUrl)6.url:=“https://api.myip.la/en?json”7.wg.Add(1)8.goipcheck(url,proxyUrl)9.}10.wg.Wait()1.funcipcheck(urlstring,proxystring){2.request:=gorequest.New().Proxy(proxy)3._,body,errs:=request.Get(url).4.Set("User-Agent",`Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/98.0.4758.82Safari/537.36`)。5.End()6.iferrs!=nil{7.fmt.Println(errs)8.}9.fmt.Println(body)10.deferwg.Done()11.deferfunc(){12.deferfunc(){12:=recover()13.iferr!=nil{14.fmt.Println(time.Now().Format("2006-01-0215:04:0507"),【http错误]","返回信息:",err)15.}16.}()17.}测试结果本次代理测试使用ipidea的代理，覆盖面广，亲测通过率不低。新用户可以白嫖流量。地址：http://www.ipidea.net/完整代码1.packagemain2.3.import(4."encoding/json"5."fmt"6."github.com/parnurzeal/gorequest"7."sync"8."time"9.)10.11.varwgsync.WaitGroup12.13.typereqinfostruct{14.Codeint15.Successbool16.Msg,RequestIp]stringstring[mapstring17.][interface{}18.}19.20.//api21.funcgetRes(urlstring)string{22.deferfunc(){23.err:=recover()24.iferr!=nil{25.tfmPrintln(time.Now().Format("2006-01-0215:04:0507"),"【http错误】","返回信息：",err)26.}27.}()28.//获取代理29._,body,errs:=gorequest.New().Get(url).End()30.iferrs!=nil{31.panic(errs)32.}33.returnbody34.}35.36.funcipcheck(urlstring,proxystring){37.request:=gorequest.New().Proxy(proxy)38._,body,errs:=request.Get(url)。39.Set(“User-Agent”,`Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/98.0.4758.82Safari/537.36`)。40.End()41.iferrs!=nil{42.fmt.Println(errs)43.//panic(errs)44.}45.fmt.Println(body)46.4.Done()7.Done()deferfunc(){48.err:=recover()49.iferr!=nil{50.fmt.Println(time.Now().Format("2006-01-0215:04:0507"),“【http错误]”,“返回信息:”,err)51.}52.}()53.}54.55.funcgetiP(getipUrlstring){56.deferfunc(){57.err:=恢复()58.iferr!=nil{59.fmt.Println(time.Now().Format("2006-01-0215:04:0507"),【http错误]","返回信息：",err)60.}61.}()62.body:=getRes(getipUrl)63.fmt.Println(body)64.//处理json65.varinforeqinfo66.err:=json.Unmarshal([]byte(body),&info)67.iferr!=nil{68.fmt.Println("jsonerror",err)69.}70.for_,v:=rangeinfo.Data{71.IP:=v["ip"]72.端口:=v["端口"]73.proxyUrl:=fmt.Sprint("http://",IP,":",port)74.fmt.Println(proxyUrl)75.url:=“https://api.myip.la/en?json”76.wg.Add(1)77.goipcheck(url,proxyUrl)78.}79.wg.Wait()80.}81.82.funcmain(){83.getipUrl:=“代理链接”84.getiP(getipUrl)85.}
