diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..024eefa --- /dev/null +++ b/go.mod @@ -0,0 +1,19 @@ +module downloadlogo + +go 1.20 + +require ( + github.com/474420502/extractor v0.11.1 + github.com/474420502/requests v1.32.1 +) + +require ( + github.com/antchfx/xpath v1.2.4 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/tidwall/gjson v1.12.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/net v0.5.0 // indirect + golang.org/x/text v0.6.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5f51ebc --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/474420502/extractor v0.11.1 h1:w+ow5Jq5MLygmlmXE/L3vCB+RTzobLbahz0HnOXAcfE= +github.com/474420502/extractor v0.11.1/go.mod h1:HqZh4rjOg2gUtS616Chp1dMwP3Upji0Hw6BRo2tAhCY= +github.com/474420502/random v0.4.1 h1:HUUyLXRWMijVb7CJoEC16f0aFQOW25Lkr80Mut6PoKU= +github.com/474420502/requests v1.32.1 h1:ivr9iMXrYqfdKWS0q15HEFlt1Cjeeu75Hjhbmikg9us= +github.com/474420502/requests v1.32.1/go.mod h1:rrDBE6AfPr0Cclo1lL4Jgw1mIW6TKSCvFgXDuEqlw70= +github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= +github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4 h1:lS3P5Nw3oPO05Lk2gFiYUOL3QPaH+fRoI1wFOc4G1UY= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY= +github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= diff --git a/logos/.record b/logos/.record new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/logos/.record @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..470d773 --- /dev/null +++ b/main_test.go @@ -0,0 +1,75 @@ +package downloadlogo + +import ( + "log" + "os" + "strconv" + "strings" + "testing" + + "github.com/474420502/extractor" + "github.com/474420502/requests" +) + +type ItemList struct { + LogoUri []string `exp:"//div[@class='result-group bg-white']//img" method:"AttributeValue,src"` +} + +func TestMain(t *testing.T) { + + ses := requests.NewSession() + tp := ses.Get("https://brandeps.com/logo/Latest?page=500") + + dp, err := os.ReadFile("logos/.record") + if err != nil { + log.Panic(err) + } + startPage, err := strconv.Atoi(string(dp)) + if err != nil { + log.Panic(err) + } + + page := tp.QueryParam("page") + for i := startPage; i < 500; i++ { + log.Println("page", i) + + // strconv.Itoa(i) + err = os.WriteFile("logos/.record", []byte(strconv.Itoa(i)), 0644) + if err != nil { + panic(err) + } + + page.IntSet(int64(i)) + res, err := tp.Execute() + if err != nil { + panic(err) + } + + // log.Println(res.ContentString()) + etor := extractor.ExtractHtml(res.Content()) + itemlist := &ItemList{} + etor.GetObjectByTag(itemlist) + if len(itemlist.LogoUri) <= 0 { + break + } + for _, uri := range itemlist.LogoUri { + + // uri = strings.Replace(uri, ".svg", ".eps", 1) + log.Println(uri) + res, err = ses.Get(uri).Execute() + if err != nil { + panic(err) + } + index := strings.LastIndex(uri, "/") + if index >= 0 { + log.Println(uri[index+1:]) + err = os.WriteFile("logos/"+uri[index+1:], res.Content(), 0644) + if err != nil { + panic(err) + } + } + + } + } + // https://brandeps.com/logo/Latest?page=4 +}