1package main
2
3import (
4	"fmt"
5
6	"github.com/gocolly/colly/v2"
7	"github.com/gocolly/colly/v2/debug"
8)
9
10func main() {
11	url := "https://httpbin.org/delay/2"
12
13	// Instantiate default collector
14	c := colly.NewCollector(
15		// Turn on asynchronous requests
16		colly.Async(),
17		// Attach a debugger to the collector
18		colly.Debugger(&debug.LogDebugger{}),
19	)
20
21	// Limit the number of threads started by colly to two
22	// when visiting links which domains' matches "*httpbin.*" glob
23	c.Limit(&colly.LimitRule{
24		DomainGlob:  "*httpbin.*",
25		Parallelism: 2,
26		//Delay:      5 * time.Second,
27	})
28
29	// Start scraping in five threads on https://httpbin.org/delay/2
30	for i := 0; i < 5; i++ {
31		c.Visit(fmt.Sprintf("%s?n=%d", url, i))
32	}
33	// Wait until threads are finished
34	c.Wait()
35}
36