1package main 2 3import ( 4 "fmt" 5 6 "github.com/gocolly/colly/v2" 7 "github.com/gocolly/colly/v2/debug" 8) 9 10func main() { 11 url := "https://httpbin.org/delay/2" 12 13 // Instantiate default collector 14 c := colly.NewCollector( 15 // Turn on asynchronous requests 16 colly.Async(), 17 // Attach a debugger to the collector 18 colly.Debugger(&debug.LogDebugger{}), 19 ) 20 21 // Limit the number of threads started by colly to two 22 // when visiting links which domains' matches "*httpbin.*" glob 23 c.Limit(&colly.LimitRule{ 24 DomainGlob: "*httpbin.*", 25 Parallelism: 2, 26 //Delay: 5 * time.Second, 27 }) 28 29 // Start scraping in five threads on https://httpbin.org/delay/2 30 for i := 0; i < 5; i++ { 31 c.Visit(fmt.Sprintf("%s?n=%d", url, i)) 32 } 33 // Wait until threads are finished 34 c.Wait() 35} 36