1package urlgetter 2 3import ( 4 "context" 5 "io/ioutil" 6 "net/url" 7 "time" 8 9 "github.com/ooni/probe-cli/v3/internal/engine/model" 10 "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" 11 "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" 12 "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" 13 "github.com/ooni/probe-cli/v3/internal/engine/tunnel" 14) 15 16// The Getter gets the specified target in the context of the 17// given session and with the specified config. 18// 19// Other OONI experiment should use the Getter to factor code when 20// the Getter implements the operations they wanna perform. 21type Getter struct { 22 // Begin is the time when the experiment begun. If you do not 23 // set this field, every target is measured independently. 24 Begin time.Time 25 26 // Config contains settings for this run. If not set, then 27 // we will use the default config. 28 Config Config 29 30 // Session is the session for this run. This field must 31 // be set otherwise the code will panic. 32 Session model.ExperimentSession 33 34 // Target is the thing to measure in this run. This field must 35 // be set otherwise the code won't know what to do. 36 Target string 37 38 // testIOUtilTempDir allows us to mock ioutil.TempDir 39 testIOUtilTempDir func(dir, pattern string) (string, error) 40} 41 42// Get performs the action described by g using the given context 43// and returning the test keys and eventually an error 44func (g Getter) Get(ctx context.Context) (TestKeys, error) { 45 if g.Config.Timeout > 0 { 46 var cancel context.CancelFunc 47 ctx, cancel = context.WithTimeout(ctx, g.Config.Timeout) 48 defer cancel() 49 } 50 if g.Begin.IsZero() { 51 g.Begin = time.Now() 52 } 53 saver := new(trace.Saver) 54 tk, err := g.get(ctx, saver) 55 // Make sure we have an operation in cases where we fail before 56 // hitting our httptransport that does error wrapping. 57 err = errorx.SafeErrWrapperBuilder{ 58 Error: err, 59 Operation: errorx.TopLevelOperation, 60 }.MaybeBuild() 61 tk.FailedOperation = archival.NewFailedOperation(err) 62 tk.Failure = archival.NewFailure(err) 63 events := saver.Read() 64 tk.Queries = append(tk.Queries, archival.NewDNSQueriesList(g.Begin, events)...) 65 tk.NetworkEvents = append( 66 tk.NetworkEvents, archival.NewNetworkEventsList(g.Begin, events)..., 67 ) 68 tk.Requests = append( 69 tk.Requests, archival.NewRequestList(g.Begin, events)..., 70 ) 71 if len(tk.Requests) > 0 { 72 // OONI's convention is that the last request appears first 73 tk.HTTPResponseStatus = tk.Requests[0].Response.Code 74 tk.HTTPResponseBody = tk.Requests[0].Response.Body.Value 75 tk.HTTPResponseLocations = tk.Requests[0].Response.Locations 76 } 77 tk.TCPConnect = append( 78 tk.TCPConnect, archival.NewTCPConnectList(g.Begin, events)..., 79 ) 80 tk.TLSHandshakes = append( 81 tk.TLSHandshakes, archival.NewTLSHandshakesList(g.Begin, events)..., 82 ) 83 return tk, err 84} 85 86// ioutilTempDir calls either g.testIOUtilTempDir or ioutil.TempDir 87func (g Getter) ioutilTempDir(dir, pattern string) (string, error) { 88 if g.testIOUtilTempDir != nil { 89 return g.testIOUtilTempDir(dir, pattern) 90 } 91 return ioutil.TempDir(dir, pattern) 92} 93 94func (g Getter) get(ctx context.Context, saver *trace.Saver) (TestKeys, error) { 95 tk := TestKeys{ 96 Agent: "redirect", 97 Tunnel: g.Config.Tunnel, 98 } 99 if g.Config.DNSCache != "" { 100 tk.DNSCache = []string{g.Config.DNSCache} 101 } 102 if g.Config.NoFollowRedirects { 103 tk.Agent = "agent" 104 } 105 // start tunnel 106 var proxyURL *url.URL 107 if g.Config.Tunnel != "" { 108 // Every new instance of the tunnel goes into a separate 109 // directory within the temporary directory. Calling 110 // Session.Close will delete such a directory. 111 tundir, err := g.ioutilTempDir(g.Session.TempDir(), "urlgetter-tunnel-") 112 if err != nil { 113 return tk, err 114 } 115 tun, err := tunnel.Start(ctx, &tunnel.Config{ 116 Name: g.Config.Tunnel, 117 Session: g.Session, 118 TorArgs: g.Session.TorArgs(), 119 TorBinary: g.Session.TorBinary(), 120 TunnelDir: tundir, 121 }) 122 if err != nil { 123 return tk, err 124 } 125 tk.BootstrapTime = tun.BootstrapTime().Seconds() 126 proxyURL = tun.SOCKS5ProxyURL() 127 tk.SOCKSProxy = proxyURL.String() 128 defer tun.Stop() 129 } 130 // create configuration 131 configurer := Configurer{ 132 Config: g.Config, 133 Logger: g.Session.Logger(), 134 ProxyURL: proxyURL, 135 Saver: saver, 136 } 137 configuration, err := configurer.NewConfiguration() 138 if err != nil { 139 return tk, err 140 } 141 defer configuration.CloseIdleConnections() 142 // run the measurement 143 runner := Runner{ 144 Config: g.Config, 145 HTTPConfig: configuration.HTTPConfig, 146 Target: g.Target, 147 } 148 return tk, runner.Run(ctx) 149} 150