diff --git a/go.mod b/go.mod index a87ed6a..2d02cf4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,10 @@ go 1.12 require ( 474420502.top/eson/gjson v1.1.3 + github.com/ahmetb/go-httpbin v0.0.0-20190607190512-d5c855a2c9eb + github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 + github.com/gorilla/mux v1.7.3 // indirect + github.com/pkg/errors v0.8.1 // indirect github.com/tidwall/match v1.0.1 // indirect golang.org/x/net v0.0.0-20190902185345-cdfb69ac37fc6fa907650654115ebebb3aae2087 ) diff --git a/go.sum b/go.sum index 8b509c2..459471e 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,15 @@ 474420502.top/eson/gjson v1.1.3 h1:SDeD1/SWm1YknuokcPww8ZmsOOguQqFAYLWnQTMMX98= 474420502.top/eson/gjson v1.1.3/go.mod h1:95mdr7XPHsGvsGZj/FeQ+iT7mggI6LKc3JT/ZnveebI= +github.com/ahmetb/go-httpbin v0.0.0-20190607190512-d5c855a2c9eb h1:MCH9va3v6SaOiA2pLA1YekSGvId2qPvJuJPdMeq/u7Y= +github.com/ahmetb/go-httpbin v0.0.0-20190607190512-d5c855a2c9eb/go.mod h1:iB3NbHoh0P/9AZepPBcH+gM1PhQJGmsres+ZHf72M3k= +github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY= +github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/golang/net v0.0.0-20190902185345-cdfb69ac37fc6fa907650654115ebebb3aae2087 h1:haK1T12C0CO79KUdu+ZzLL9+l9BwM9PRkd2/mQqdg8E= github.com/golang/net v0.0.0-20190902185345-cdfb69ac37fc6fa907650654115ebebb3aae2087/go.mod h1:98y8FxUyMjTdJ5eOj/8vzuiVO14/dkJ98NYhEPG8QGY= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/init_test.go b/init_test.go new file mode 100644 index 0000000..6505df8 --- /dev/null +++ b/init_test.go @@ -0,0 +1,29 @@ +package requests + +import ( + "log" + "net/http" + "os/exec" + "time" + + "github.com/elazarl/goproxy" +) + +var ProxyAddress = "localhost:58080" + +func init() { + log.SetFlags(log.Lshortfile | log.LstdFlags) + go func() { + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + http.ListenAndServe(ProxyAddress, proxy) + }() + + cmd := exec.Command("/bin/bash", "-c", "docker ps | grep httpbin") + _, err := cmd.Output() + if err != nil { + log.Println("recommend docker run -p 80:80 kennethreitz/httpbin ") + } + + time.Sleep(time.Millisecond * 100) +} diff --git a/response.go b/response.go index 7f158da..d8a57b9 100644 --- a/response.go +++ b/response.go @@ -16,7 +16,7 @@ type Response struct { } // FromHTTPResponse 生成Response 从标准http.Response -func FromHTTPResponse(resp *http.Response, IsDecompressNoAccept bool) (*Response, error) { +func FromHTTPResponse(resp *http.Response, isDecompressNoAccept bool) (*Response, error) { var err error // 复制response 返回内容 并且测试是否有解压的需求 srcbuf, err := ioutil.ReadAll(resp.Body) @@ -27,8 +27,7 @@ func FromHTTPResponse(resp *http.Response, IsDecompressNoAccept bool) (*Response content := "" - if IsDecompressNoAccept { - + if isDecompressNoAccept { srcReader := bytes.NewReader(srcbuf) var reader io.ReadCloser if reader, err = gzip.NewReader(srcReader); err == nil { diff --git a/response_test.go b/response_test.go index fd525a0..b573a6c 100644 --- a/response_test.go +++ b/response_test.go @@ -33,7 +33,7 @@ func TestFromHTTPResponse(t *testing.T) { } } -func TestResponseGzipZip(t *testing.T) { +func TestResponseDeflate(t *testing.T) { ses := NewSession() if wf := ses.Get("http://httpbin.org/get"); wf != nil { wf.AddHeader("accept-encoding", "deflate") @@ -41,7 +41,9 @@ func TestResponseGzipZip(t *testing.T) { if err != nil { t.Error(err) } else { - t.Error(resp.Content()) + if gjson.Get(resp.Content(), "headers.Accept-Encoding").String() != "deflate" { + t.Error("Accept-Encoding != deflate ?") + } } } diff --git a/session.go b/session.go index 7614227..1ef4729 100644 --- a/session.go +++ b/session.go @@ -161,7 +161,7 @@ const ( // CDialTimeout 一个Connect过程的Timeout CDialTimeout // 支持time.Duration 和 int(秒为单位) - // CKeepAlives 默认不KeepAlives, 容易被一直KeepAlives 没关闭链接 + // CKeepAlives 默认KeepAlives false, 如果默认为true容易被一直KeepAlives, 没关闭链接 CKeepAlives // CProxy 代理链接 @@ -176,12 +176,12 @@ const ( // CTLS 帐号认证 CTLS // user pwd - // CCookiejar 持久化 CookieJar - CCookiejar // true or false ; default = true + // CIsWithCookiejar 持久化 CookieJar true or false ; default = true + CIsWithCookiejar - // CDecompressNoAccept 解压 当response header 不存在 Accept-Encoding + // CIsDecompressNoAccept 解压 当response header 不存在 Accept-Encoding // 很多特殊情景会不返回Accept-Encoding: Gzip. 如 不按照标准的网站 - CDecompressNoAccept + CIsDecompressNoAccept ) // NewSession 创建Session @@ -222,11 +222,12 @@ func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) { } case CDialTimeout: // TODO: CDialTimeout CRequestTimeout 与 细节 - case CDecompressNoAccept: + case CIsDecompressNoAccept: ses.Is.isDecompressNoAccept = values.(bool) case CKeepAlives: + println(ses.transport.DisableKeepAlives) ses.transport.DisableKeepAlives = !values.(bool) - case CCookiejar: + case CIsWithCookiejar: v := values.(bool) if v { if ses.client.Jar == nil { diff --git a/session_test.go b/session_test.go index c77488a..bf26bd4 100644 --- a/session_test.go +++ b/session_test.go @@ -1,15 +1,15 @@ package requests import ( - "log" "net/http" + "net/url" "regexp" + "strings" "testing" -) + "time" -func init() { - log.SetFlags(log.Lshortfile | log.LstdFlags) -} + "474420502.top/eson/gjson" +) func TestNewSession(t *testing.T) { ses := NewSession() @@ -297,7 +297,7 @@ func TestSession_SetConfig(t *testing.T) { { name: "test proxy", - args: args{typeConfig: CProxy, values: "http://474420502.top:7070"}, + args: args{typeConfig: CProxy, values: "http://" + ProxyAddress}, wantErr: false, }, } @@ -379,3 +379,51 @@ func TestSession_Header(t *testing.T) { } }) } + +func TestSession_ConfigEx(t *testing.T) { + ses := NewSession() + ses.SetConfig(CRequestTimeout, time.Microsecond) + resp, err := ses.Get("http://httpbin.org/get").Execute() + if err == nil { + t.Error(resp) + } else { + if strings.LastIndex(err.Error(), "Client.Timeout exceeded while awaiting headers") < 0 { + t.Error(err) + } + } + ses.SetConfig(CRequestTimeout, float32(0.0000001)) + resp, err = ses.Get("http://httpbin.org/get").Execute() + if err == nil { + t.Error(resp) + } else { + if strings.LastIndex(err.Error(), "Client.Timeout exceeded while awaiting headers") < 0 { + t.Error(err) + } + } + + ses.SetConfig(CKeepAlives, true) + ses.SetConfig(CRequestTimeout, int64(5)) + // jar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + u, err := url.Parse("http://httpbin.org") + if err != nil { + t.Error(err) + } else { + // jar.SetCookies(u, []*http.Cookie{&http.Cookie{Name: "Request", Value: "Cookiejar"}}) + ses.SetConfig(CIsWithCookiejar, true) + ses.SetConfig(CIsWithCookiejar, true) + ses.SetCookies(u, []*http.Cookie{&http.Cookie{Name: "Request", Value: "Cookiejar"}, &http.Cookie{Name: "eson", Value: "bad"}}) + resp, err = ses.Get("http://httpbin.org/get").Execute() + if err != nil { + t.Error(err) + } + + if gjson.Get(resp.Content(), "headers.Cookie").String() != "Request=Cookiejar; eson=bad" { + t.Error(resp.Content()) + } + + if resp.GetSrcResponse().Header["Connection"][0] != "keep-alive" { + t.Error("CKeepAlive is error") + } + } + +} diff --git a/workflow.go b/workflow.go index 2c8b8a0..416be26 100644 --- a/workflow.go +++ b/workflow.go @@ -21,7 +21,6 @@ type Workflow struct { func NewWorkflow(ses *Session, urlstr string) *Workflow { wf := &Workflow{} wf.SwitchSession(ses) - wf.SetRawURL(urlstr) wf.Body = NewBody() diff --git a/workflow_test.go b/workflow_test.go index 4ef4eb9..d9492a3 100644 --- a/workflow_test.go +++ b/workflow_test.go @@ -57,4 +57,5 @@ func TestWorkflow(t *testing.T) { t.Error(resp.readContent) } }) + }