1// +build linux darwin 2 3/* 4Copyright 2011 The Perkeep Authors 5 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17*/ 18 19// Package fs implements a FUSE filesystem for Perkeep and is 20// used by the pk-mount binary. 21package fs // import "perkeep.org/pkg/fs" 22 23import ( 24 "context" 25 "fmt" 26 "io" 27 "log" 28 "os" 29 "sync" 30 "time" 31 32 "perkeep.org/internal/lru" 33 "perkeep.org/pkg/blob" 34 "perkeep.org/pkg/client" 35 "perkeep.org/pkg/schema" 36 37 "bazil.org/fuse" 38 fusefs "bazil.org/fuse/fs" 39) 40 41var ( 42 serverStart = time.Now() 43 // Logger is used by the package to print all sorts of debugging statements. It 44 // is up to the user of the package to SetOutput the Logger to reduce verbosity. 45 Logger = log.New(os.Stderr, "PerkeepFS: ", log.LstdFlags) 46) 47 48type CamliFileSystem struct { 49 fetcher blob.Fetcher 50 client *client.Client // or nil, if not doing search queries 51 root fusefs.Node 52 53 // IgnoreOwners, if true, collapses all file ownership to the 54 // uid/gid running the fuse filesystem, and sets all the 55 // permissions to 0600/0700. 56 IgnoreOwners bool 57 58 blobToSchema *lru.Cache // ~map[blobstring]*schema.Blob 59 nameToBlob *lru.Cache // ~map[string]blob.Ref 60 nameToAttr *lru.Cache // ~map[string]*fuse.Attr 61} 62 63var _ fusefs.FS = (*CamliFileSystem)(nil) 64 65func newCamliFileSystem(fetcher blob.Fetcher) *CamliFileSystem { 66 return &CamliFileSystem{ 67 fetcher: fetcher, 68 blobToSchema: lru.New(1024), // arbitrary; TODO: tunable/smarter? 69 nameToBlob: lru.New(1024), // arbitrary: TODO: tunable/smarter? 70 nameToAttr: lru.New(1024), // arbitrary: TODO: tunable/smarter? 71 } 72} 73 74// NewDefaultCamliFileSystem returns a filesystem with a generic base, from which 75// users can navigate by blobref, tag, date, etc. 76func NewDefaultCamliFileSystem(client *client.Client, fetcher blob.Fetcher) *CamliFileSystem { 77 if client == nil || fetcher == nil { 78 panic("nil argument") 79 } 80 fs := newCamliFileSystem(fetcher) 81 fs.root = &root{fs: fs} // root.go 82 fs.client = client 83 return fs 84} 85 86// NewRootedCamliFileSystem returns a CamliFileSystem with a node based on a blobref 87// as its base. 88func NewRootedCamliFileSystem(cli *client.Client, fetcher blob.Fetcher, root blob.Ref) (*CamliFileSystem, error) { 89 fs := newCamliFileSystem(fetcher) 90 fs.client = cli 91 92 n, err := fs.newNodeFromBlobRef(root) 93 94 if err != nil { 95 return nil, err 96 } 97 98 fs.root = n 99 100 return fs, nil 101} 102 103// node implements fuse.Node with a read-only Camli "file" or 104// "directory" blob. 105type node struct { 106 fs *CamliFileSystem 107 blobref blob.Ref 108 109 pnodeModTime time.Time // optionally set by recent.go; modtime of permanode 110 111 dmu sync.Mutex // guards dirents. acquire before mu. 112 dirents []fuse.Dirent // nil until populated once 113 114 mu sync.Mutex // guards rest 115 attr fuse.Attr 116 meta *schema.Blob 117 lookMap map[string]blob.Ref 118} 119 120var _ fusefs.Node = (*node)(nil) 121 122func (n *node) Attr(ctx context.Context, a *fuse.Attr) error { 123 if _, err := n.schema(); err != nil { 124 return err 125 } 126 *a = n.attr 127 return nil 128} 129 130func (n *node) addLookupEntry(name string, ref blob.Ref) { 131 n.mu.Lock() 132 defer n.mu.Unlock() 133 if n.lookMap == nil { 134 n.lookMap = make(map[string]blob.Ref) 135 } 136 n.lookMap[name] = ref 137} 138 139var _ fusefs.NodeStringLookuper = (*node)(nil) 140 141func (n *node) Lookup(ctx context.Context, name string) (fusefs.Node, error) { 142 if name == ".quitquitquit" { 143 // TODO: only in dev mode 144 log.Fatalf("Shutting down due to .quitquitquit lookup.") 145 } 146 147 // If we haven't done Readdir yet (dirents isn't set), then force a Readdir 148 // call to populate lookMap. 149 n.dmu.Lock() 150 loaded := n.dirents != nil 151 n.dmu.Unlock() 152 if !loaded { 153 n.ReadDirAll(nil) 154 } 155 156 n.mu.Lock() 157 defer n.mu.Unlock() 158 ref, ok := n.lookMap[name] 159 if !ok { 160 return nil, fuse.ENOENT 161 } 162 return &node{fs: n.fs, blobref: ref}, nil 163} 164 165func (n *node) schema() (*schema.Blob, error) { 166 // TODO: use singleflight library here instead of a lock? 167 n.mu.Lock() 168 defer n.mu.Unlock() 169 if n.meta != nil { 170 return n.meta, nil 171 } 172 blob, err := n.fs.fetchSchemaMeta(context.TODO(), n.blobref) 173 if err == nil { 174 n.meta = blob 175 n.populateAttr() 176 } 177 return blob, err 178} 179 180func isWriteFlags(flags fuse.OpenFlags) bool { 181 // TODO read/writeness are not flags, use O_ACCMODE 182 return flags&fuse.OpenFlags(os.O_WRONLY|os.O_RDWR|os.O_APPEND|os.O_CREATE) != 0 183} 184 185var _ fusefs.NodeOpener = (*node)(nil) 186 187func (n *node) Open(ctx context.Context, req *fuse.OpenRequest, res *fuse.OpenResponse) (fusefs.Handle, error) { 188 Logger.Printf("CAMLI Open on %v: %#v", n.blobref, req) 189 if isWriteFlags(req.Flags) { 190 return nil, fuse.EPERM 191 } 192 ss, err := n.schema() 193 if err != nil { 194 Logger.Printf("open of %v: %v", n.blobref, err) 195 return nil, fuse.EIO 196 } 197 if ss.Type() == "directory" { 198 return n, nil 199 } 200 fr, err := ss.NewFileReader(n.fs.fetcher) 201 if err != nil { 202 // Will only happen if ss.Type != "file" or "bytes" 203 Logger.Printf("NewFileReader(%s) = %v", n.blobref, err) 204 return nil, fuse.EIO 205 } 206 return &nodeReader{n: n, fr: fr}, nil 207} 208 209type nodeReader struct { 210 n *node 211 fr *schema.FileReader 212} 213 214var _ fusefs.HandleReader = (*nodeReader)(nil) 215 216func (nr *nodeReader) Read(ctx context.Context, req *fuse.ReadRequest, res *fuse.ReadResponse) error { 217 Logger.Printf("CAMLI nodeReader READ on %v: %#v", nr.n.blobref, req) 218 if req.Offset >= nr.fr.Size() { 219 return nil 220 } 221 size := req.Size 222 if int64(size)+req.Offset >= nr.fr.Size() { 223 size -= int((int64(size) + req.Offset) - nr.fr.Size()) 224 } 225 buf := make([]byte, size) 226 n, err := nr.fr.ReadAt(buf, req.Offset) 227 if err == io.EOF { 228 err = nil 229 } 230 if err != nil { 231 Logger.Printf("camli read on %v at %d: %v", nr.n.blobref, req.Offset, err) 232 return fuse.EIO 233 } 234 res.Data = buf[:n] 235 return nil 236} 237 238var _ fusefs.HandleReleaser = (*nodeReader)(nil) 239 240func (nr *nodeReader) Release(ctx context.Context, req *fuse.ReleaseRequest) error { 241 Logger.Printf("CAMLI nodeReader RELEASE on %v", nr.n.blobref) 242 nr.fr.Close() 243 return nil 244} 245 246var _ fusefs.HandleReadDirAller = (*node)(nil) 247 248func (n *node) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { 249 Logger.Printf("CAMLI ReadDirAll on %v", n.blobref) 250 n.dmu.Lock() 251 defer n.dmu.Unlock() 252 if n.dirents != nil { 253 return n.dirents, nil 254 } 255 256 ss, err := n.schema() 257 if err != nil { 258 Logger.Printf("camli.ReadDirAll error on %v: %v", n.blobref, err) 259 return nil, fuse.EIO 260 } 261 dr, err := schema.NewDirReader(ctx, n.fs.fetcher, ss.BlobRef()) 262 if err != nil { 263 Logger.Printf("camli.ReadDirAll error on %v: %v", n.blobref, err) 264 return nil, fuse.EIO 265 } 266 schemaEnts, err := dr.Readdir(ctx, -1) 267 if err != nil { 268 Logger.Printf("camli.ReadDirAll error on %v: %v", n.blobref, err) 269 return nil, fuse.EIO 270 } 271 n.dirents = make([]fuse.Dirent, 0) 272 for _, sent := range schemaEnts { 273 if name := sent.FileName(); name != "" { 274 n.addLookupEntry(name, sent.BlobRef()) 275 n.dirents = append(n.dirents, fuse.Dirent{Name: name}) 276 } 277 } 278 return n.dirents, nil 279} 280 281// populateAttr should only be called once n.ss is known to be set and 282// non-nil 283func (n *node) populateAttr() error { 284 meta := n.meta 285 286 n.attr.Mode = meta.FileMode() 287 288 if n.fs.IgnoreOwners { 289 n.attr.Uid = uint32(os.Getuid()) 290 n.attr.Gid = uint32(os.Getgid()) 291 executeBit := n.attr.Mode & 0100 292 n.attr.Mode = (n.attr.Mode ^ n.attr.Mode.Perm()) | 0400 | executeBit 293 } else { 294 n.attr.Uid = uint32(meta.MapUid()) 295 n.attr.Gid = uint32(meta.MapGid()) 296 } 297 298 // TODO: inode? 299 300 if mt := meta.ModTime(); !mt.IsZero() { 301 n.attr.Mtime = mt 302 } else { 303 n.attr.Mtime = n.pnodeModTime 304 } 305 306 switch meta.Type() { 307 case "file": 308 n.attr.Size = uint64(meta.PartsSize()) 309 n.attr.Blocks = 0 // TODO: set? 310 n.attr.Mode |= 0400 311 case "directory": 312 n.attr.Mode |= 0500 313 case "symlink": 314 n.attr.Mode |= 0400 315 default: 316 Logger.Printf("unknown attr ss.Type %q in populateAttr", meta.Type()) 317 } 318 return nil 319} 320 321func (fs *CamliFileSystem) Root() (fusefs.Node, error) { 322 return fs.root, nil 323} 324 325var _ fusefs.FSStatfser = (*CamliFileSystem)(nil) 326 327func (fs *CamliFileSystem) Statfs(ctx context.Context, req *fuse.StatfsRequest, res *fuse.StatfsResponse) error { 328 // Make some stuff up, just to see if it makes "lsof" happy. 329 res.Blocks = 1 << 35 330 res.Bfree = 1 << 34 331 res.Bavail = 1 << 34 332 res.Files = 1 << 29 333 res.Ffree = 1 << 28 334 res.Namelen = 2048 335 res.Bsize = 1024 336 return nil 337} 338 339// Errors returned are: 340// os.ErrNotExist -- blob not found 341// os.ErrInvalid -- not JSON or a camli schema blob 342func (fs *CamliFileSystem) fetchSchemaMeta(ctx context.Context, br blob.Ref) (*schema.Blob, error) { 343 blobStr := br.String() 344 if blob, ok := fs.blobToSchema.Get(blobStr); ok { 345 return blob.(*schema.Blob), nil 346 } 347 348 rc, _, err := fs.fetcher.Fetch(ctx, br) 349 if err != nil { 350 return nil, err 351 } 352 defer rc.Close() 353 blob, err := schema.BlobFromReader(br, rc) 354 if err != nil { 355 Logger.Printf("Error parsing %s as schema blob: %v", br, err) 356 return nil, os.ErrInvalid 357 } 358 if blob.Type() == "" { 359 Logger.Printf("blob %s is JSON but lacks camliType", br) 360 return nil, os.ErrInvalid 361 } 362 fs.blobToSchema.Add(blobStr, blob) 363 return blob, nil 364} 365 366// consolated logic for determining a node to mount based on an arbitrary blobref 367func (fs *CamliFileSystem) newNodeFromBlobRef(root blob.Ref) (fusefs.Node, error) { 368 blob, err := fs.fetchSchemaMeta(context.TODO(), root) 369 if err != nil { 370 return nil, err 371 } 372 373 switch blob.Type() { 374 case "directory": 375 n := &node{fs: fs, blobref: root, meta: blob} 376 n.populateAttr() 377 return n, nil 378 379 case "permanode": 380 // other mutDirs listed in the default fileystem have names and are displayed 381 return &mutDir{fs: fs, permanode: root, name: "-"}, nil 382 } 383 384 return nil, fmt.Errorf("Blobref must be of a directory or permanode got a %v", blob.Type()) 385} 386 387type notImplementDirNode struct{} 388 389var _ fusefs.Node = (*notImplementDirNode)(nil) 390 391func (notImplementDirNode) Attr(ctx context.Context, a *fuse.Attr) error { 392 a.Mode = os.ModeDir | 0000 393 a.Uid = uint32(os.Getuid()) 394 a.Gid = uint32(os.Getgid()) 395 return nil 396} 397 398type staticFileNode string 399 400var _ fusefs.Node = (*notImplementDirNode)(nil) 401 402func (s staticFileNode) Attr(ctx context.Context, a *fuse.Attr) error { 403 a.Mode = 0400 404 a.Uid = uint32(os.Getuid()) 405 a.Gid = uint32(os.Getgid()) 406 a.Size = uint64(len(s)) 407 a.Mtime = serverStart 408 a.Ctime = serverStart 409 a.Crtime = serverStart 410 return nil 411} 412 413var _ fusefs.HandleReader = (*staticFileNode)(nil) 414 415func (s staticFileNode) Read(ctx context.Context, req *fuse.ReadRequest, res *fuse.ReadResponse) error { 416 if req.Offset > int64(len(s)) { 417 return nil 418 } 419 s = s[req.Offset:] 420 size := req.Size 421 if size > len(s) { 422 size = len(s) 423 } 424 res.Data = make([]byte, size) 425 copy(res.Data, s) 426 return nil 427} 428