1/* 2NNCP -- Node to Node copy, utilities for store-and-forward data exchange 3Copyright (C) 2016-2021 Sergey Matveev <stargrave@stargrave.org> 4 5This program is free software: you can redistribute it and/or modify 6it under the terms of the GNU General Public License as published by 7the Free Software Foundation, version 3 of the License. 8 9This program is distributed in the hope that it will be useful, 10but WITHOUT ANY WARRANTY; without even the implied warranty of 11MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12GNU General Public License for more details. 13 14You should have received a copy of the GNU General Public License 15along with this program. If not, see <http://www.gnu.org/licenses/>. 16*/ 17 18// Reassembly chunked file. 19package main 20 21import ( 22 "bufio" 23 "bytes" 24 "encoding/hex" 25 "errors" 26 "flag" 27 "fmt" 28 "hash" 29 "io" 30 "log" 31 "os" 32 "path/filepath" 33 "strconv" 34 "strings" 35 36 xdr "github.com/davecgh/go-xdr/xdr2" 37 "github.com/dustin/go-humanize" 38 "go.cypherpunks.ru/nncp/v8" 39) 40 41func usage() { 42 fmt.Fprintf(os.Stderr, nncp.UsageHeader()) 43 fmt.Fprintf(os.Stderr, "nncp-reass -- reassemble chunked files\n\n") 44 fmt.Fprintf(os.Stderr, "Usage: %s [options] [FILE.nncp.meta]\nOptions:\n", os.Args[0]) 45 flag.PrintDefaults() 46 fmt.Fprint(os.Stderr, ` 47Neither FILE, nor -node nor -all can be set simultaneously, 48but at least one of them must be specified. 49`) 50} 51 52func process(ctx *nncp.Ctx, path string, keep, dryRun, stdout, dumpMeta bool) bool { 53 fd, err := os.Open(path) 54 defer fd.Close() 55 if err != nil { 56 log.Fatalln("Can not open file:", err) 57 } 58 var metaPkt nncp.ChunkedMeta 59 les := nncp.LEs{{K: "Path", V: path}} 60 logMsg := func(les nncp.LEs) string { 61 return fmt.Sprintf("Reassembling chunked file \"%s\"", path) 62 } 63 if _, err = xdr.Unmarshal(fd, &metaPkt); err != nil { 64 ctx.LogE("reass-bad-meta", les, err, func(les nncp.LEs) string { 65 return logMsg(les) + ": bad meta" 66 }) 67 return false 68 } 69 fd.Close() 70 if metaPkt.Magic == nncp.MagicNNCPMv1.B { 71 ctx.LogE("reass", les, nncp.MagicNNCPMv1.TooOld(), logMsg) 72 return false 73 } 74 if metaPkt.Magic != nncp.MagicNNCPMv2.B { 75 ctx.LogE("reass", les, nncp.BadMagic, logMsg) 76 return false 77 } 78 79 metaName := filepath.Base(path) 80 if !strings.HasSuffix(metaName, nncp.ChunkedSuffixMeta) { 81 ctx.LogE("reass", les, errors.New("invalid filename suffix"), logMsg) 82 return false 83 } 84 mainName := strings.TrimSuffix(metaName, nncp.ChunkedSuffixMeta) 85 if dumpMeta { 86 fmt.Printf("Original filename: %s\n", mainName) 87 fmt.Printf( 88 "File size: %s (%d bytes)\n", 89 humanize.IBytes(metaPkt.FileSize), 90 metaPkt.FileSize, 91 ) 92 fmt.Printf( 93 "Chunk size: %s (%d bytes)\n", 94 humanize.IBytes(metaPkt.ChunkSize), 95 metaPkt.ChunkSize, 96 ) 97 fmt.Printf("Number of chunks: %d\n", len(metaPkt.Checksums)) 98 fmt.Println("Checksums:") 99 for chunkNum, checksum := range metaPkt.Checksums { 100 fmt.Printf("\t%d: %s\n", chunkNum, hex.EncodeToString(checksum[:])) 101 } 102 return true 103 } 104 mainDir := filepath.Dir(path) 105 106 chunksPaths := make([]string, 0, len(metaPkt.Checksums)) 107 for i := 0; i < len(metaPkt.Checksums); i++ { 108 chunksPaths = append( 109 chunksPaths, 110 filepath.Join(mainDir, mainName+nncp.ChunkedSuffixPart+strconv.Itoa(i)), 111 ) 112 } 113 114 allChunksExist := true 115 for chunkNum, chunkPath := range chunksPaths { 116 fi, err := os.Stat(chunkPath) 117 lesChunk := append(les, nncp.LE{K: "Chunk", V: chunkNum}) 118 if err != nil && os.IsNotExist(err) { 119 ctx.LogI("reass-chunk-miss", lesChunk, func(les nncp.LEs) string { 120 return fmt.Sprintf("%s: chunk %d missing", logMsg(les), chunkNum) 121 }) 122 allChunksExist = false 123 continue 124 } 125 var badSize bool 126 if chunkNum+1 == len(chunksPaths) { 127 badSize = uint64(fi.Size()) != metaPkt.FileSize%metaPkt.ChunkSize 128 } else { 129 badSize = uint64(fi.Size()) != metaPkt.ChunkSize 130 } 131 if badSize { 132 ctx.LogE( 133 "reass-chunk", 134 lesChunk, 135 errors.New("invalid size"), 136 func(les nncp.LEs) string { 137 return fmt.Sprintf("%s: chunk %d", logMsg(les), chunkNum) 138 }, 139 ) 140 allChunksExist = false 141 } 142 } 143 if !allChunksExist { 144 return false 145 } 146 147 var hsh hash.Hash 148 allChecksumsGood := true 149 for chunkNum, chunkPath := range chunksPaths { 150 fd, err = os.Open(chunkPath) 151 if err != nil { 152 log.Fatalln("Can not open file:", err) 153 } 154 fi, err := fd.Stat() 155 if err != nil { 156 log.Fatalln("Can not stat file:", err) 157 } 158 hsh = nncp.MTHNew(fi.Size(), 0) 159 if _, err = nncp.CopyProgressed( 160 hsh, bufio.NewReader(fd), "check", 161 nncp.LEs{{K: "Pkt", V: chunkPath}, {K: "FullSize", V: fi.Size()}}, 162 ctx.ShowPrgrs, 163 ); err != nil { 164 log.Fatalln(err) 165 } 166 fd.Close() 167 if bytes.Compare(hsh.Sum(nil), metaPkt.Checksums[chunkNum][:]) != 0 { 168 ctx.LogE( 169 "reass-chunk", 170 nncp.LEs{{K: "Path", V: path}, {K: "Chunk", V: chunkNum}}, 171 errors.New("checksum is bad"), 172 func(les nncp.LEs) string { 173 return fmt.Sprintf("%s: chunk %d", logMsg(les), chunkNum) 174 }, 175 ) 176 allChecksumsGood = false 177 } 178 } 179 if !allChecksumsGood { 180 return false 181 } 182 if dryRun { 183 ctx.LogI("reass", nncp.LEs{{K: "path", V: path}}, logMsg) 184 return true 185 } 186 187 var dst io.Writer 188 var tmp *os.File 189 if stdout { 190 dst = os.Stdout 191 les = nncp.LEs{{K: "path", V: path}} 192 } else { 193 tmp, err = nncp.TempFile(mainDir, "reass") 194 if err != nil { 195 log.Fatalln(err) 196 } 197 les = nncp.LEs{{K: "path", V: path}, {K: "Tmp", V: tmp.Name()}} 198 ctx.LogD("reass-tmp-created", les, func(les nncp.LEs) string { 199 return fmt.Sprintf("%s: temporary %s created", logMsg(les), tmp.Name()) 200 }) 201 dst = tmp 202 } 203 dstW := bufio.NewWriter(dst) 204 205 hasErrors := false 206 for chunkNum, chunkPath := range chunksPaths { 207 fd, err = os.Open(chunkPath) 208 if err != nil { 209 log.Fatalln("Can not open file:", err) 210 } 211 fi, err := fd.Stat() 212 if err != nil { 213 log.Fatalln("Can not stat file:", err) 214 } 215 if _, err = nncp.CopyProgressed( 216 dstW, bufio.NewReader(fd), "reass", 217 nncp.LEs{{K: "Pkt", V: chunkPath}, {K: "FullSize", V: fi.Size()}}, 218 ctx.ShowPrgrs, 219 ); err != nil { 220 log.Fatalln(err) 221 } 222 fd.Close() 223 if !keep { 224 if err = os.Remove(chunkPath); err != nil { 225 ctx.LogE( 226 "reass-chunk", 227 append(les, nncp.LE{K: "Chunk", V: chunkNum}), err, 228 func(les nncp.LEs) string { 229 return fmt.Sprintf("%s: chunk %d", logMsg(les), chunkNum) 230 }, 231 ) 232 hasErrors = true 233 } 234 } 235 } 236 if err = dstW.Flush(); err != nil { 237 log.Fatalln("Can not flush:", err) 238 } 239 if tmp != nil { 240 if err = tmp.Sync(); err != nil { 241 log.Fatalln("Can not sync:", err) 242 } 243 if err = tmp.Close(); err != nil { 244 log.Fatalln("Can not close:", err) 245 } 246 } 247 ctx.LogD("reass-written", les, func(les nncp.LEs) string { 248 return logMsg(les) + ": written" 249 }) 250 if !keep { 251 if err = os.Remove(path); err != nil { 252 ctx.LogE("reass-removing", les, err, func(les nncp.LEs) string { 253 return logMsg(les) + ": removing" 254 }) 255 hasErrors = true 256 } 257 } 258 if stdout { 259 ctx.LogI("reass", nncp.LEs{{K: "Path", V: path}}, func(les nncp.LEs) string { 260 return logMsg(les) + ": done" 261 }) 262 return !hasErrors 263 } 264 265 dstPathOrig := filepath.Join(mainDir, mainName) 266 dstPath := dstPathOrig 267 dstPathCtr := 0 268 for { 269 if _, err = os.Stat(dstPath); err != nil { 270 if os.IsNotExist(err) { 271 break 272 } 273 log.Fatalln(err) 274 } 275 dstPath = dstPathOrig + "." + strconv.Itoa(dstPathCtr) 276 dstPathCtr++ 277 } 278 if err = os.Rename(tmp.Name(), dstPath); err != nil { 279 log.Fatalln(err) 280 } 281 if err = nncp.DirSync(mainDir); err != nil { 282 log.Fatalln(err) 283 } 284 ctx.LogI("reass", nncp.LEs{{K: "Path", V: path}}, func(les nncp.LEs) string { 285 return logMsg(les) + ": done" 286 }) 287 return !hasErrors 288} 289 290func findMetas(ctx *nncp.Ctx, dirPath string) []string { 291 dir, err := os.Open(dirPath) 292 defer dir.Close() 293 logMsg := func(les nncp.LEs) string { 294 return "Finding .meta in " + dirPath 295 } 296 if err != nil { 297 ctx.LogE("reass", nncp.LEs{{K: "Path", V: dirPath}}, err, logMsg) 298 return nil 299 } 300 fis, err := dir.Readdir(0) 301 dir.Close() 302 if err != nil { 303 ctx.LogE("reass", nncp.LEs{{K: "Path", V: dirPath}}, err, logMsg) 304 return nil 305 } 306 metaPaths := make([]string, 0) 307 for _, fi := range fis { 308 if strings.HasSuffix(fi.Name(), nncp.ChunkedSuffixMeta) { 309 metaPaths = append(metaPaths, filepath.Join(dirPath, fi.Name())) 310 } 311 } 312 return metaPaths 313} 314 315func main() { 316 var ( 317 cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") 318 allNodes = flag.Bool("all", false, "Process all found chunked files for all nodes") 319 nodeRaw = flag.String("node", "", "Process all found chunked files for that node") 320 keep = flag.Bool("keep", false, "Do not remove chunks while assembling") 321 dryRun = flag.Bool("dryrun", false, "Do not assemble whole file") 322 dumpMeta = flag.Bool("dump", false, "Print decoded human-readable FILE.nncp.meta") 323 stdout = flag.Bool("stdout", false, "Output reassembled FILE to stdout") 324 spoolPath = flag.String("spool", "", "Override path to spool") 325 logPath = flag.String("log", "", "Override path to logfile") 326 quiet = flag.Bool("quiet", false, "Print only errors") 327 showPrgrs = flag.Bool("progress", false, "Force progress showing") 328 omitPrgrs = flag.Bool("noprogress", false, "Omit progress showing") 329 debug = flag.Bool("debug", false, "Print debug messages") 330 version = flag.Bool("version", false, "Print version information") 331 warranty = flag.Bool("warranty", false, "Print warranty information") 332 ) 333 log.SetFlags(log.Lshortfile) 334 flag.Usage = usage 335 flag.Parse() 336 if *warranty { 337 fmt.Println(nncp.Warranty) 338 return 339 } 340 if *version { 341 fmt.Println(nncp.VersionGet()) 342 return 343 } 344 345 ctx, err := nncp.CtxFromCmdline( 346 *cfgPath, 347 *spoolPath, 348 *logPath, 349 *quiet, 350 *showPrgrs, 351 *omitPrgrs, 352 *debug, 353 ) 354 if err != nil { 355 log.Fatalln("Error during initialization:", err) 356 } 357 358 var nodeOnly *nncp.Node 359 if *nodeRaw != "" { 360 nodeOnly, err = ctx.FindNode(*nodeRaw) 361 if err != nil { 362 log.Fatalln("Invalid -node specified:", err) 363 } 364 } 365 366 if !(*allNodes || nodeOnly != nil || flag.NArg() > 0) { 367 usage() 368 os.Exit(1) 369 } 370 if flag.NArg() > 0 && (*allNodes || nodeOnly != nil) { 371 usage() 372 os.Exit(1) 373 } 374 if *allNodes && nodeOnly != nil { 375 usage() 376 os.Exit(1) 377 } 378 379 ctx.Umask() 380 381 if flag.NArg() > 0 { 382 if process(ctx, flag.Arg(0), *keep, *dryRun, *stdout, *dumpMeta) { 383 return 384 } 385 os.Exit(1) 386 } 387 388 hasErrors := false 389 if nodeOnly == nil { 390 seenMetaPaths := make(map[string]struct{}) 391 for _, node := range ctx.Neigh { 392 if node.Incoming == nil { 393 continue 394 } 395 for _, metaPath := range findMetas(ctx, *node.Incoming) { 396 if _, seen := seenMetaPaths[metaPath]; seen { 397 continue 398 } 399 if !process(ctx, metaPath, *keep, *dryRun, false, false) { 400 hasErrors = true 401 } 402 seenMetaPaths[metaPath] = struct{}{} 403 } 404 } 405 } else { 406 if nodeOnly.Incoming == nil { 407 log.Fatalln("Specified -node does not allow incoming") 408 } 409 for _, metaPath := range findMetas(ctx, *nodeOnly.Incoming) { 410 if !process(ctx, metaPath, *keep, *dryRun, false, false) { 411 hasErrors = true 412 } 413 } 414 } 415 if hasErrors { 416 os.Exit(1) 417 } 418} 419