1// Copyright 2016 Keybase, Inc. All rights reserved. Use of
2// this source code is governed by the included BSD license.
3
4package engine
5
6import (
7	"bufio"
8	"encoding/csv"
9	"encoding/gob"
10	"fmt"
11	"io"
12	"os"
13	"strconv"
14	"strings"
15	"time"
16
17	"github.com/keybase/client/go/libkb"
18	keybase1 "github.com/keybase/client/go/protocol/keybase1"
19)
20
21type ScanProofsCacheData struct {
22	// Map from sigid to whether the proof is ok.
23	Proofs map[string]bool
24}
25
26type ScanProofsCache struct {
27	data  ScanProofsCacheData
28	dirty bool
29}
30
31func NewScanProofsCache() *ScanProofsCache {
32	return &ScanProofsCache{
33		data: ScanProofsCacheData{
34			Proofs: make(map[string]bool),
35		},
36		dirty: false,
37	}
38}
39
40func LoadScanProofsCache(filepath string) (*ScanProofsCache, error) {
41	f, err := os.Open(filepath)
42	if err != nil {
43		return nil, err
44	}
45	dec := gob.NewDecoder(f)
46	var c ScanProofsCache
47	err = dec.Decode(&c.data)
48	if err != nil {
49		return nil, err
50	}
51	return &c, nil
52}
53
54func (c *ScanProofsCache) Get(sigID string) bool {
55	return c.data.Proofs[sigID]
56}
57
58func (c *ScanProofsCache) Set(sigID string) {
59	if !c.data.Proofs[sigID] {
60		c.dirty = true
61	}
62	c.data.Proofs[sigID] = true
63}
64
65func (c *ScanProofsCache) Save(filepath string) error {
66	if !c.dirty {
67		// Don't save if nothing has changed
68		return nil
69	}
70	temppath, f, err := libkb.OpenTempFile(filepath, "", 0644)
71	if err != nil {
72		return err
73	}
74	enc := gob.NewEncoder(f)
75	err = enc.Encode(c.data)
76	if err != nil {
77		f.Close()
78		return err
79	}
80	f.Close()
81	err = os.Rename(temppath, filepath)
82	if err != nil {
83		return err
84	}
85	c.dirty = false
86	return nil
87}
88
89type ScanProofsTickers map[keybase1.ProofType]*time.Ticker
90
91type ScanProofsEngine struct {
92	libkb.Contextified
93	infile     string
94	indices    string
95	sigid      string
96	ratelimit  int
97	cachefile  string
98	ignorefile string
99}
100
101var _ Engine2 = (*ScanProofsEngine)(nil)
102
103func NewScanProofsEngine(infile string, indices string, sigid string, ratelimit int, cachefile string, ignorefile string, g *libkb.GlobalContext) *ScanProofsEngine {
104	return &ScanProofsEngine{
105		infile:       infile,
106		indices:      indices,
107		sigid:        sigid,
108		ratelimit:    ratelimit,
109		cachefile:    cachefile,
110		ignorefile:   ignorefile,
111		Contextified: libkb.NewContextified(g),
112	}
113}
114
115func (e *ScanProofsEngine) Name() string {
116	return "ScanProofs"
117}
118
119func (e *ScanProofsEngine) Prereqs() Prereqs {
120	return Prereqs{}
121}
122
123func (e *ScanProofsEngine) RequiredUIs() []libkb.UIKind {
124	return []libkb.UIKind{
125		libkb.LogUIKind,
126	}
127}
128
129func (e *ScanProofsEngine) SubConsumers() []libkb.UIConsumer {
130	return []libkb.UIConsumer{}
131}
132
133func (e *ScanProofsEngine) Run(m libkb.MetaContext) (err error) {
134	defer m.Trace("ScanProofsEngine#Run", &err)()
135
136	var cache *ScanProofsCache
137	saveevery := 10
138	var ignored []string
139
140	if len(e.cachefile) > 0 {
141		lcache, err := LoadScanProofsCache(e.cachefile)
142		if err == nil {
143			m.Info("Using cache: %v (%v entries)", e.cachefile, len(lcache.data.Proofs))
144			cache = lcache
145		} else {
146			m.Warning("Could not load cache: %v", err)
147			cache = NewScanProofsCache()
148		}
149	}
150
151	if len(e.ignorefile) > 0 {
152		ignored, err = LoadScanProofsIgnore(e.ignorefile)
153		if err != nil {
154			return fmt.Errorf("Could not open ignore file: %v", err)
155		}
156		m.Info("Using ignore file: %v (%v entries)", e.ignorefile, len(ignored))
157	}
158
159	if len(e.sigid) > 0 && len(e.indices) > 0 {
160		return fmt.Errorf("Only one of sigid and indices allowed")
161	}
162
163	// One ticker for each proof type.
164	var tickers = make(map[keybase1.ProofType]*time.Ticker)
165	m.Info("Running with ratelimit: %v ms", e.ratelimit)
166	if e.ratelimit < 0 {
167		return fmt.Errorf("Ratelimit value can not be negative: %v", e.ratelimit)
168	}
169	if e.ratelimit > 0 {
170		for _, ptype := range keybase1.ProofTypeMap {
171			switch ptype {
172			case keybase1.ProofType_GENERIC_WEB_SITE, keybase1.ProofType_DNS:
173				// Web sites and DNS do not need a rate limit.
174			default:
175				tickers[ptype] = time.NewTicker(time.Millisecond * time.Duration(e.ratelimit))
176			}
177		}
178	}
179	defer func(tickers *map[keybase1.ProofType]*time.Ticker) {
180		for _, ticker := range *tickers {
181			if ticker != nil {
182				ticker.Stop()
183			}
184		}
185	}(&tickers)
186
187	f, err := os.Open(e.infile)
188	if err != nil {
189		return err
190	}
191	r := csv.NewReader(f)
192
193	var records []map[string]string
194
195	header, err := r.Read()
196	if err != nil {
197		return fmt.Errorf("Could not read header: %v", err)
198	}
199
200	m.Debug("Reading csv... ")
201	for {
202		rec, err := r.Read()
203		if err == io.EOF {
204			break
205		}
206		if err != nil {
207			return err
208		}
209		record := make(map[string]string)
210		for i, val := range rec {
211			record[header[i]] = val
212		}
213		records = append(records, record)
214	}
215	m.Debug("done")
216
217	startindex := 0
218	endindex := len(records)
219	if len(e.indices) > 0 {
220		startindex, endindex, err = e.ParseIndices(e.indices)
221		if err != nil {
222			return err
223		}
224	}
225	if startindex < 0 {
226		return fmt.Errorf("Invalid start index: %v", startindex)
227	}
228	if endindex > len(records) {
229		return fmt.Errorf("Invalid end index: %v (%v records)", endindex, len(records))
230	}
231
232	nrun := 0
233	nok := 0
234
235	for i := startindex; i < endindex; i++ {
236		rec := records[i]
237
238		if len(e.sigid) > 0 && e.sigid != rec["sig_id"] {
239			continue
240		}
241
242		m.Info("i:%v user:%v type:%v sigid:%v", i, rec["username"], rec["proof_type"], rec["sig_id"])
243
244		err := e.ProcessOne(m, i, rec, cache, ignored, tickers)
245		nrun++
246		if err == nil {
247			m.Info("Ok\n")
248			nok++
249			if cache != nil {
250				cache.Set(rec["sig_id"])
251				if i%saveevery == 0 {
252					saveerr := cache.Save(e.cachefile)
253					if saveerr != nil {
254						m.Warning("Could not save cache: %v", saveerr)
255					}
256				}
257			}
258		} else {
259			m.Error("%v FAILED: %v\n", i, err)
260		}
261	}
262
263	m.Info("---")
264	m.Info("proofs checked  : %v", nrun)
265	m.Info("oks             : %v", nok)
266	m.Info("fails           : %v", nrun-nok)
267
268	if cache != nil {
269		saveerr := cache.Save(e.cachefile)
270		if saveerr != nil {
271			m.Warning("Could not save cache: %v", saveerr)
272		}
273	}
274
275	return nil
276}
277
278func (e *ScanProofsEngine) ProcessOne(m libkb.MetaContext, i int, rec map[string]string, cache *ScanProofsCache, ignored []string, tickers ScanProofsTickers) error {
279	serverstate, err := strconv.Atoi(rec["state"])
280	if err != nil {
281		return fmt.Errorf("Could not read server state: %v", err)
282	}
283
284	shouldsucceed := true
285	skip := false
286	var skipreason string
287	badstate := false
288
289	switch keybase1.ProofState(serverstate) {
290	case keybase1.ProofState_NONE:
291		badstate = true
292	case keybase1.ProofState_OK:
293	case keybase1.ProofState_TEMP_FAILURE:
294		shouldsucceed = false
295	case keybase1.ProofState_PERM_FAILURE:
296		shouldsucceed = false
297	case keybase1.ProofState_LOOKING:
298		skip = true
299		skipreason = "server LOOKING"
300	case keybase1.ProofState_SUPERSEDED:
301		skip = true
302		skipreason = "server SUPERSEDED"
303	case keybase1.ProofState_POSTED:
304		badstate = true
305	case keybase1.ProofState_REVOKED:
306		skip = true
307		skipreason = "server REVOKED"
308	case keybase1.ProofState_DELETED:
309		skip = true
310		skipreason = "server DELETED"
311	default:
312		badstate = true
313	}
314
315	if cache != nil && cache.Get(rec["sig_id"]) {
316		skip = true
317		skipreason = "cached success"
318	}
319
320	for _, x := range ignored {
321		if x == rec["sig_id"] {
322			skip = true
323			skipreason = "in ignored list"
324		}
325	}
326
327	if badstate {
328		return fmt.Errorf("Unsupported serverstate: %v", serverstate)
329	}
330
331	if skip {
332		m.Info("skipping: %v", skipreason)
333		return nil
334	}
335
336	deluserstr := "Error loading user: Deleted"
337	perr1, foundhint1, err := e.CheckOne(m, rec, tickers)
338	if err != nil {
339		if err.Error() == deluserstr {
340			m.Info("deleted user")
341			return nil
342		}
343		return err
344	}
345	// Skip the rate limit on the second check.
346	perr2, foundhint2, err := e.CheckOne(m, rec, nil)
347	if err != nil {
348		return err
349	}
350
351	if foundhint1 != foundhint2 {
352		return fmt.Errorf("Local verifiers disagree: foundhint1:%v foundhint:%v (likely timing)",
353			foundhint1, foundhint2)
354	}
355
356	if (perr1 == nil) != (perr2 == nil) {
357		return fmt.Errorf("Local verifiers disagree:\n  %v\n  %v", perr1, perr2)
358	}
359
360	succeeded := foundhint1 && (perr1 == nil)
361
362	if succeeded != shouldsucceed {
363		return fmt.Errorf("Local verifiers disagree with server: server:%v client:%v", serverstate, perr1)
364	}
365
366	return nil
367}
368
369// CheckOne checks one proof using two checkers (default, pvl).
370// NOTE: This doesn't make sense anymore because pvl is the default.
371// Returns nil or an error, whether a hint was found, and any more serious error
372func (e *ScanProofsEngine) CheckOne(m libkb.MetaContext, rec map[string]string, tickers ScanProofsTickers) (libkb.ProofError, bool, error) {
373	uid := keybase1.UID(rec["uid"])
374	sigid := keybase1.SigID(rec["sig_id"])
375
376	foundhint := false
377	hint, err := e.GetSigHint(m, uid, sigid)
378	if err != nil {
379		return nil, foundhint, err
380	}
381	if hint == nil {
382		return nil, foundhint, nil
383	}
384	foundhint = true
385
386	link, err := e.GetRemoteProofChainLink(m, uid, sigid)
387	if err != nil {
388		return nil, foundhint, err
389	}
390
391	pc, err := libkb.MakeProofChecker(m, m.G().GetProofServices(), link)
392	if err != nil {
393		return nil, foundhint, err
394	}
395
396	// Beyond this point, external requests will occur, and rate limiting is used
397	ptype := link.GetProofType()
398	if tickers[ptype] != nil {
399		m.Info("Waiting for ticker: %v (%v)", keybase1.ProofTypeRevMap[ptype], ptype)
400		<-tickers[ptype].C
401	}
402
403	pvlSource := m.G().GetPvlSource()
404	if pvlSource == nil {
405		return nil, foundhint, fmt.Errorf("no pvl source for proof verification")
406	}
407	pvlU, err := pvlSource.GetLatestEntry(m)
408	if err != nil {
409		return nil, foundhint, fmt.Errorf("error getting pvl: %s", err)
410	}
411
412	if _, perr := pc.CheckStatus(m, *hint, libkb.ProofCheckerModeActive, pvlU); perr != nil {
413		return perr, foundhint, nil
414	}
415
416	return nil, foundhint, nil
417}
418
419// GetSigHint gets the SigHint. This can return (nil, nil) if nothing goes wrong but there is no hint.
420func (e *ScanProofsEngine) GetSigHint(m libkb.MetaContext, uid keybase1.UID, sigid keybase1.SigID) (*libkb.SigHint, error) {
421	sighints, err := libkb.LoadAndRefreshSigHints(m, uid)
422	if err != nil {
423		return nil, err
424	}
425
426	sighint := sighints.Lookup(sigid)
427	if sighint == nil {
428		return nil, nil
429	}
430	return sighint, nil
431}
432
433func (e *ScanProofsEngine) GetRemoteProofChainLink(m libkb.MetaContext, uid keybase1.UID, sigid keybase1.SigID) (libkb.RemoteProofChainLink, error) {
434	user, err := libkb.LoadUser(libkb.NewLoadUserArgWithMetaContext(m).WithUID(uid))
435	if err != nil {
436		return nil, fmt.Errorf("Error loading user: %v", err)
437	}
438
439	link := user.LinkFromSigID(sigid)
440	if link == nil {
441		return nil, fmt.Errorf("Could not find link from sigid")
442	}
443
444	tlink, w := libkb.NewTypedChainLink(link)
445	if w != nil {
446		return nil, fmt.Errorf("Could not get typed chain link: %v", w.Warning())
447	}
448
449	switch vlink := tlink.(type) {
450	case libkb.RemoteProofChainLink:
451		return vlink, nil
452	default:
453		return nil, fmt.Errorf("Link is not a RemoteProofChainLink: %v", tlink)
454	}
455}
456
457func (e *ScanProofsEngine) ParseIndices(indices string) (start int, end int, reterr error) {
458	wrap := func(format string, arg ...interface{}) error {
459		f2 := fmt.Sprintf("Invalid indices: %s", format)
460		return fmt.Errorf(f2, arg...)
461	}
462	ss := strings.Split(strings.TrimSpace(indices), ":")
463	if len(ss) != 2 {
464		return start, end, wrap("must be like start:end")
465	}
466	var err error
467	start, err = strconv.Atoi(ss[0])
468	if err != nil {
469		return start, end, wrap("could not convert start: %v", err)
470	}
471	end, err = strconv.Atoi(ss[1])
472	if err != nil {
473		return start, end, wrap("could not convert end: %v", err)
474	}
475	if end <= start {
476		return start, end, wrap("%v <= %v", end, start)
477	}
478	reterr = nil
479	return
480}
481
482// LoadScanProofsIgnore loads an ignore file and returns the list of proofids to ignore.
483func LoadScanProofsIgnore(filepath string) ([]string, error) {
484	f, err := os.Open(filepath)
485	if err != nil {
486		return nil, err
487	}
488	defer f.Close()
489	scanner := bufio.NewScanner(f)
490	scanner.Split(bufio.ScanLines)
491	var ignored []string
492	for scanner.Scan() {
493		x := strings.TrimSpace(scanner.Text())
494		if strings.HasPrefix(x, "//") {
495			continue
496		}
497		ignored = append(ignored, x)
498	}
499	return ignored, nil
500}
501