1// Copyright 2015 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// proxy is an HTTP/S proxy configurable via an HTTP API. 16// 17// It can be dynamically configured/queried at runtime by issuing requests to 18// proxy specific paths using JSON. 19// 20// Supported configuration endpoints: 21// 22// POST http://martian.proxy/configure 23// 24// sets the request and response modifier of the proxy; modifiers adhere to the 25// following top-level JSON structure: 26// 27// { 28// "package.Modifier": { 29// "scope": ["request", "response"], 30// "attribute 1": "value", 31// "attribute 2": "value" 32// } 33// } 34// 35// modifiers may be "stacked" to provide support for additional behaviors; for 36// example, to add a "Martian-Test" header with the value "true" for requests 37// with the domain "www.example.com" the JSON message would be: 38// 39// { 40// "url.Filter": { 41// "scope": ["request"], 42// "host": "www.example.com", 43// "modifier": { 44// "header.Modifier": { 45// "name": "Martian-Test", 46// "value": "true" 47// } 48// } 49// } 50// } 51// 52// url.Filter parses the JSON object in the value of the "url.Filter" attribute; 53// the "host" key tells the url.Filter to filter requests if the host explicitly 54// matches "www.example.com" 55// 56// the "modifier" key within the "url.Filter" JSON object contains another 57// modifier message of the type header.Modifier to run iff the filter passes 58// 59// groups may also be used to run multiple modifiers sequentially; for example to 60// log requests and responses after adding the "Martian-Test" header to the 61// request, but only when the host matches www.example.com: 62// 63// { 64// "url.Filter": { 65// "host": "www.example.com", 66// "modifier": { 67// "fifo.Group": { 68// "modifiers": [ 69// { 70// "header.Modifier": { 71// "scope": ["request"], 72// "name": "Martian-Test", 73// "value": "true" 74// } 75// }, 76// { 77// "log.Logger": { } 78// } 79// ] 80// } 81// } 82// } 83// } 84// 85// modifiers are designed to be composed together in ways that allow the user to 86// write a single JSON structure to accomplish a variety of functionality 87// 88// GET http://martian.proxy/verify 89// 90// retrieves the verifications errors as JSON with the following structure: 91// 92// { 93// "errors": [ 94// { 95// "message": "request(url) verification failure" 96// }, 97// { 98// "message": "response(url) verification failure" 99// } 100// ] 101// } 102// 103// verifiers also adhere to the modifier interface and thus can be included in the 104// modifier configuration request; for example, to verify that all requests to 105// "www.example.com" are sent over HTTPS send the following JSON to the 106// configuration endpoint: 107// 108// { 109// "url.Filter": { 110// "scope": ["request"], 111// "host": "www.example.com", 112// "modifier": { 113// "url.Verifier": { 114// "scope": ["request"], 115// "scheme": "https" 116// } 117// } 118// } 119// } 120// 121// sending a request to "http://martian.proxy/verify" will then return errors from the url.Verifier 122// 123// POST http://martian.proxy/verify/reset 124// 125// resets the verifiers to their initial state; note some verifiers may start in 126// a failure state (e.g., pingback.Verifier is failed if no requests have been 127// seen by the proxy) 128// 129// GET http://martian.proxy/authority.cer 130// 131// prompts the user to install the CA certificate used by the proxy if MITM is enabled 132// 133// GET http://martian.proxy/logs 134// 135// retrieves the HAR logs for all requests and responses seen by the proxy if 136// the HAR flag is enabled 137// 138// DELETE http://martian.proxy/logs/reset 139// 140// reset the in-memory HAR log; note that the log will grow unbounded unless it 141// is periodically reset 142// 143// passing the -cors flag will enable CORS support for the endpoints so that they 144// may be called via AJAX 145// 146// Sending a sigint will cause the proxy to stop receiving new connections, 147// finish processing any inflight requests, and close existing connections without 148// reading anymore requests from them. 149// 150// The flags are: 151// -addr=":8080" 152// host:port of the proxy 153// -api-addr=":8181" 154// host:port of the proxy API 155// -tls-addr=":4443" 156// host:port of the proxy over TLS 157// -api="martian.proxy" 158// hostname that can be used to reference the configuration API when 159// configuring through the proxy 160// -cert="" 161// PEM encoded X.509 CA certificate; if set, it will be set as the 162// issuer for dynamically-generated certificates during man-in-the-middle 163// -key="" 164// PEM encoded private key of cert (RSA or ECDSA); if set, the key will be used 165// to sign dynamically-generated certificates during man-in-the-middle 166// -generate-ca-cert=false 167// generates a CA certificate and private key to use for man-in-the-middle; 168// the certificate is only valid while the proxy is running and will be 169// discarded on shutdown 170// -organization="Martian Proxy" 171// organization name set on the dynamically-generated certificates during 172// man-in-the-middle 173// -validity="1h" 174// window of time around the time of request that the dynamically-generated 175// certificate is valid for; the duration is set such that the total valid 176// timeframe is double the value of validity (1h before & 1h after) 177// -cors=false 178// allow the proxy to be configured via CORS requests; such as when 179// configuring the proxy via AJAX 180// -har=false 181// enable logging endpoints for retrieving full request/response logs in 182// HAR format. 183// -traffic-shaping=false 184// enable traffic shaping endpoints for simulating latency and constrained 185// bandwidth conditions (e.g. mobile, exotic network infrastructure, the 186// 90's) 187// -skip-tls-verify=false 188// skip TLS server verification; insecure and intended for testing only 189// -v=0 190// log level for console logs; defaults to error only. 191package main 192 193import ( 194 "crypto/tls" 195 "crypto/x509" 196 "flag" 197 "log" 198 "net" 199 "net/http" 200 "net/url" 201 "os" 202 "os/signal" 203 "path" 204 "strconv" 205 "strings" 206 "time" 207 208 "github.com/google/martian/v3" 209 mapi "github.com/google/martian/v3/api" 210 "github.com/google/martian/v3/cors" 211 "github.com/google/martian/v3/fifo" 212 "github.com/google/martian/v3/har" 213 "github.com/google/martian/v3/httpspec" 214 "github.com/google/martian/v3/marbl" 215 "github.com/google/martian/v3/martianhttp" 216 "github.com/google/martian/v3/martianlog" 217 "github.com/google/martian/v3/mitm" 218 "github.com/google/martian/v3/servemux" 219 "github.com/google/martian/v3/trafficshape" 220 "github.com/google/martian/v3/verify" 221 222 _ "github.com/google/martian/v3/body" 223 _ "github.com/google/martian/v3/cookie" 224 _ "github.com/google/martian/v3/failure" 225 _ "github.com/google/martian/v3/martianurl" 226 _ "github.com/google/martian/v3/method" 227 _ "github.com/google/martian/v3/pingback" 228 _ "github.com/google/martian/v3/port" 229 _ "github.com/google/martian/v3/priority" 230 _ "github.com/google/martian/v3/querystring" 231 _ "github.com/google/martian/v3/skip" 232 _ "github.com/google/martian/v3/stash" 233 _ "github.com/google/martian/v3/static" 234 _ "github.com/google/martian/v3/status" 235) 236 237var ( 238 addr = flag.String("addr", ":8080", "host:port of the proxy") 239 apiAddr = flag.String("api-addr", ":8181", "host:port of the configuration API") 240 tlsAddr = flag.String("tls-addr", ":4443", "host:port of the proxy over TLS") 241 api = flag.String("api", "martian.proxy", "hostname for the API") 242 generateCA = flag.Bool("generate-ca-cert", false, "generate CA certificate and private key for MITM") 243 cert = flag.String("cert", "", "filepath to the CA certificate used to sign MITM certificates") 244 key = flag.String("key", "", "filepath to the private key of the CA used to sign MITM certificates") 245 organization = flag.String("organization", "Martian Proxy", "organization name for MITM certificates") 246 validity = flag.Duration("validity", time.Hour, "window of time that MITM certificates are valid") 247 allowCORS = flag.Bool("cors", false, "allow CORS requests to configure the proxy") 248 harLogging = flag.Bool("har", false, "enable HAR logging API") 249 marblLogging = flag.Bool("marbl", false, "enable MARBL logging API") 250 trafficShaping = flag.Bool("traffic-shaping", false, "enable traffic shaping API") 251 skipTLSVerify = flag.Bool("skip-tls-verify", false, "skip TLS server verification; insecure") 252 dsProxyURL = flag.String("downstream-proxy-url", "", "URL of downstream proxy") 253) 254 255func main() { 256 p := martian.NewProxy() 257 defer p.Close() 258 259 tr := &http.Transport{ 260 Dial: (&net.Dialer{ 261 Timeout: 30 * time.Second, 262 KeepAlive: 30 * time.Second, 263 }).Dial, 264 TLSHandshakeTimeout: 10 * time.Second, 265 ExpectContinueTimeout: time.Second, 266 TLSClientConfig: &tls.Config{ 267 InsecureSkipVerify: *skipTLSVerify, 268 }, 269 } 270 p.SetRoundTripper(tr) 271 272 if *dsProxyURL != "" { 273 u, err := url.Parse(*dsProxyURL) 274 if err != nil { 275 log.Fatal(err) 276 } 277 p.SetDownstreamProxy(u) 278 } 279 280 mux := http.NewServeMux() 281 282 var x509c *x509.Certificate 283 var priv interface{} 284 285 if *generateCA { 286 var err error 287 x509c, priv, err = mitm.NewAuthority("martian.proxy", "Martian Authority", 30*24*time.Hour) 288 if err != nil { 289 log.Fatal(err) 290 } 291 } else if *cert != "" && *key != "" { 292 tlsc, err := tls.LoadX509KeyPair(*cert, *key) 293 if err != nil { 294 log.Fatal(err) 295 } 296 priv = tlsc.PrivateKey 297 298 x509c, err = x509.ParseCertificate(tlsc.Certificate[0]) 299 if err != nil { 300 log.Fatal(err) 301 } 302 } 303 304 if x509c != nil && priv != nil { 305 mc, err := mitm.NewConfig(x509c, priv) 306 if err != nil { 307 log.Fatal(err) 308 } 309 310 mc.SetValidity(*validity) 311 mc.SetOrganization(*organization) 312 mc.SkipTLSVerify(*skipTLSVerify) 313 314 p.SetMITM(mc) 315 316 // Expose certificate authority. 317 ah := martianhttp.NewAuthorityHandler(x509c) 318 configure("/authority.cer", ah, mux) 319 320 // Start TLS listener for transparent MITM. 321 tl, err := net.Listen("tcp", *tlsAddr) 322 if err != nil { 323 log.Fatal(err) 324 } 325 326 go p.Serve(tls.NewListener(tl, mc.TLS())) 327 } 328 329 stack, fg := httpspec.NewStack("martian") 330 331 // wrap stack in a group so that we can forward API requests to the API port 332 // before the httpspec modifiers which include the via modifier which will 333 // trip loop detection 334 topg := fifo.NewGroup() 335 336 // Redirect API traffic to API server. 337 if *apiAddr != "" { 338 apip := strings.Replace(*apiAddr, ":", "", 1) 339 port, err := strconv.Atoi(apip) 340 if err != nil { 341 log.Fatal(err) 342 } 343 344 // Forward traffic that pattern matches in http.DefaultServeMux 345 apif := servemux.NewFilter(mux) 346 apif.SetRequestModifier(mapi.NewForwarder("", port)) 347 topg.AddRequestModifier(apif) 348 } 349 topg.AddRequestModifier(stack) 350 topg.AddResponseModifier(stack) 351 352 p.SetRequestModifier(topg) 353 p.SetResponseModifier(topg) 354 355 m := martianhttp.NewModifier() 356 fg.AddRequestModifier(m) 357 fg.AddResponseModifier(m) 358 359 if *harLogging { 360 hl := har.NewLogger() 361 muxf := servemux.NewFilter(mux) 362 // Only append to HAR logs when the requests are not API requests, 363 // that is, they are not matched in http.DefaultServeMux 364 muxf.RequestWhenFalse(hl) 365 muxf.ResponseWhenFalse(hl) 366 367 stack.AddRequestModifier(muxf) 368 stack.AddResponseModifier(muxf) 369 370 configure("/logs", har.NewExportHandler(hl), mux) 371 configure("/logs/reset", har.NewResetHandler(hl), mux) 372 } 373 374 logger := martianlog.NewLogger() 375 logger.SetDecode(true) 376 377 stack.AddRequestModifier(logger) 378 stack.AddResponseModifier(logger) 379 380 if *marblLogging { 381 lsh := marbl.NewHandler() 382 lsm := marbl.NewModifier(lsh) 383 muxf := servemux.NewFilter(mux) 384 muxf.RequestWhenFalse(lsm) 385 muxf.ResponseWhenFalse(lsm) 386 stack.AddRequestModifier(muxf) 387 stack.AddResponseModifier(muxf) 388 389 // retrieve binary marbl logs 390 mux.Handle("/binlogs", lsh) 391 } 392 393 // Configure modifiers. 394 configure("/configure", m, mux) 395 396 // Verify assertions. 397 vh := verify.NewHandler() 398 vh.SetRequestVerifier(m) 399 vh.SetResponseVerifier(m) 400 configure("/verify", vh, mux) 401 402 // Reset verifications. 403 rh := verify.NewResetHandler() 404 rh.SetRequestVerifier(m) 405 rh.SetResponseVerifier(m) 406 configure("/verify/reset", rh, mux) 407 408 l, err := net.Listen("tcp", *addr) 409 if err != nil { 410 log.Fatal(err) 411 } 412 413 if *trafficShaping { 414 tsl := trafficshape.NewListener(l) 415 tsh := trafficshape.NewHandler(tsl) 416 configure("/shape-traffic", tsh, mux) 417 418 l = tsl 419 } 420 421 lAPI, err := net.Listen("tcp", *apiAddr) 422 if err != nil { 423 log.Fatal(err) 424 } 425 426 log.Printf("martian: starting proxy on %s and api on %s", l.Addr().String(), lAPI.Addr().String()) 427 428 go p.Serve(l) 429 430 go http.Serve(lAPI, mux) 431 432 sigc := make(chan os.Signal, 1) 433 signal.Notify(sigc, os.Interrupt, os.Kill) 434 435 <-sigc 436 437 log.Println("martian: shutting down") 438} 439 440func init() { 441 martian.Init() 442} 443 444// configure installs a configuration handler at path. 445func configure(pattern string, handler http.Handler, mux *http.ServeMux) { 446 if *allowCORS { 447 handler = cors.NewHandler(handler) 448 } 449 450 // register handler for martian.proxy to be forwarded to 451 // local API server 452 mux.Handle(path.Join(*api, pattern), handler) 453 454 // register handler for local API server 455 p := path.Join("localhost"+*apiAddr, pattern) 456 mux.Handle(p, handler) 457} 458