1--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2-- default-rsync.lua 3-- 4-- Syncs with rsync ("classic" Lsyncd) 5-- A (Layer 1) configuration. 6-- 7-- Note: 8-- this is infact just a configuration using Layer 1 configuration 9-- like any other. It only gets compiled into the binary by default. 10-- You can simply use a modified one, by copying everything into a 11-- config file of yours and name it differently. 12-- 13-- License: GPLv2 (see COPYING) or any later version 14-- Authors: Axel Kittenberger <axkibe@gmail.com> 15-- 16--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 17 18 19if not default then error( 'default not loaded' ) end 20 21if default.rsync then error( 'default-rsync already loaded' ) end 22 23 24local rsync = { } 25 26default.rsync = rsync 27 28-- uses default collect 29 30-- 31-- used to ensure there aren't typos in the keys 32-- 33rsync.checkgauge = { 34 35 -- unsets default user action handlers 36 onCreate = false, 37 onModify = false, 38 onDelete = false, 39 onStartup = false, 40 onMove = false, 41 42 delete = true, 43 exclude = true, 44 excludeFrom = true, 45 filter = true, 46 filterFrom = true, 47 target = true, 48 49 rsync = { 50 acls = true, 51 append = true, 52 append_verify = true, 53 archive = true, 54 backup = true, 55 backup_dir = true, 56 binary = true, 57 bwlimit = true, 58 checksum = true, 59 chown = true, 60 chmod = true, 61 compress = true, 62 copy_dirlinks = true, 63 copy_links = true, 64 cvs_exclude = true, 65 dry_run = true, 66 executability = true, 67 existing = true, 68 group = true, 69 groupmap = true, 70 hard_links = true, 71 ignore_times = true, 72 inplace = true, 73 ipv4 = true, 74 ipv6 = true, 75 keep_dirlinks = true, 76 links = true, 77 one_file_system = true, 78 omit_dir_times = true, 79 omit_link_times = true, 80 owner = true, 81 password_file = true, 82 perms = true, 83 protect_args = true, 84 prune_empty_dirs = true, 85 quiet = true, 86 rsh = true, 87 rsync_path = true, 88 sparse = true, 89 suffix = true, 90 temp_dir = true, 91 timeout = true, 92 times = true, 93 update = true, 94 usermap = true, 95 verbose = true, 96 whole_file = true, 97 xattrs = true, 98 _extra = true, 99 }, 100} 101 102 103-- 104-- Returns true for non Init and Blanket events. 105-- 106local eventNotInitBlank = 107 function 108( 109 event 110) 111 return event.etype ~= 'Init' and event.etype ~= 'Blanket' 112end 113 114 115-- 116-- Spawns rsync for a list of events 117-- 118-- Exclusions are already handled by not having 119-- events for them. 120-- 121rsync.action = function 122( 123 inlet 124) 125 local config = inlet.getConfig( ) 126 127 -- gets all events ready for syncing 128 local elist = inlet.getEvents( eventNotInitBlank ) 129 130 -- gets the list of paths for the event list 131 -- deletes create multi match patterns 132 local paths = elist.getPaths( ) 133 134 -- 135 -- Replaces what rsync would consider filter rules by literals 136 -- 137 local function sub 138 ( 139 p -- pattern 140 ) 141 if not p then return end 142 143 return p: 144 gsub( '%?', '\\?' ): 145 gsub( '%*', '\\*' ): 146 gsub( '%[', '\\[' ): 147 gsub( '%]', '\\]' ) 148 end 149 150 -- 151 -- Gets the list of paths for the event list 152 -- 153 -- Deletes create multi match patterns 154 -- 155 local paths = elist.getPaths( 156 function 157 ( 158 etype, -- event type 159 path1, -- path 160 path2 -- path to for move events 161 ) 162 if string.byte( path1, -1 ) == 47 and etype == 'Delete' 163 then 164 return sub( path1 )..'***', sub( path2 ) 165 else 166 return sub( path1 ), sub( path2 ) 167 end 168 end 169 ) 170 171 -- stores all filters by integer index 172 local filterI = { } 173 174 -- stores all filters with path index 175 local filterP = { } 176 177 -- adds one path to the filter 178 local function addToFilter 179 ( 180 path 181 ) 182 if filterP[ path ] then return end 183 184 filterP[ path ] = true 185 186 table.insert( filterI, path ) 187 end 188 189 -- adds a path to the filter. 190 -- 191 -- rsync needs to have entries for all steps in the path, 192 -- so the file for example d1/d2/d3/f1 needs following filters: 193 -- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1' 194 for _, path in ipairs( paths ) 195 do 196 if path and path ~= '' 197 then 198 addToFilter( path ) 199 200 local pp = string.match( path, '^(.*/)[^/]+/?' ) 201 202 while pp 203 do 204 addToFilter( pp ) 205 206 pp = string.match( pp, '^(.*/)[^/]+/?' ) 207 end 208 end 209 end 210 211 log( 212 'Normal', 213 'Calling rsync with filter-list of new/modified files/dirs\n', 214 table.concat( filterI, '\n' ) 215 ) 216 217 local config = inlet.getConfig( ) 218 219 local delete = nil 220 221 if config.delete == true or config.delete == 'running' 222 then 223 delete = { '--delete', '--ignore-errors' } 224 end 225 226 spawn( 227 elist, 228 config.rsync.binary, 229 '<', table.concat( filterI, '\000' ), 230 config.rsync._computed, 231 '-r', 232 delete, 233 '--force', 234 '--from0', 235 '--include-from=-', 236 '--exclude=*', 237 config.source, 238 config.target 239 ) 240end 241 242 243---- 244---- NOTE: This optimized version can be used once 245---- https://bugzilla.samba.org/show_bug.cgi?id=12569 246---- is fixed. 247---- 248---- Spawns rsync for a list of events 249---- 250---- Exclusions are already handled by not having 251---- events for them. 252---- 253--rsync.action = function 254--( 255-- inlet 256--) 257-- local config = inlet.getConfig( ) 258-- 259-- -- gets all events ready for syncing 260-- local elist = inlet.getEvents( eventNotInitBlank ) 261-- 262-- -- gets the list of paths for the event list 263-- -- deletes create multi match patterns 264-- local paths = elist.getPaths( ) 265-- 266-- -- removes trailing slashes from dirs. 267-- for k, v in ipairs( paths ) 268-- do 269-- if string.byte( v, -1 ) == 47 270-- then 271-- paths[ k ] = string.sub( v, 1, -2 ) 272-- end 273-- end 274-- 275-- log( 276-- 'Normal', 277-- 'Calling rsync with filter-list of new/modified files/dirs\n', 278-- table.concat( paths, '\n' ) 279-- ) 280-- 281-- local delete = nil 282-- 283-- if config.delete == true 284-- or config.delete == 'running' 285-- then 286-- delete = { '--delete-missing-args', '--ignore-errors' } 287-- end 288-- 289-- spawn( 290-- elist, 291-- config.rsync.binary, 292-- '<', table.concat( paths, '\000' ), 293-- config.rsync._computed, 294-- delete, 295-- '--force', 296-- '--from0', 297-- '--files-from=-', 298-- config.source, 299-- config.target 300-- ) 301--end 302 303 304-- 305-- Spawns the recursive startup sync. 306-- 307rsync.init = function 308( 309 event 310) 311 local config = event.config 312 313 local inlet = event.inlet 314 315 local excludes = inlet.getExcludes( ) 316 317 local filters = inlet.hasFilters( ) and inlet.getFilters( ) 318 319 local delete = nil 320 321 local target = config.target 322 323 if not target 324 then 325 if not config.host 326 then 327 error('Internal fail, Neither target nor host is configured') 328 end 329 330 target = config.host .. ':' .. config.targetdir 331 end 332 333 if config.delete == true 334 or config.delete == 'startup' 335 then 336 delete = { '--delete', '--ignore-errors' } 337 end 338 339 if not filters and #excludes == 0 340 then 341 -- starts rsync without any filters or excludes 342 log( 343 'Normal', 344 'recursive startup rsync: ', 345 config.source, 346 ' -> ', 347 target 348 ) 349 350 spawn( 351 event, 352 config.rsync.binary, 353 delete, 354 config.rsync._computed, 355 '-r', 356 config.source, 357 target 358 ) 359 360 elseif not filters 361 then 362 -- starts rsync providing an exclusion list 363 -- on stdin 364 local exS = table.concat( excludes, '\n' ) 365 366 log( 367 'Normal', 368 'recursive startup rsync: ', 369 config.source, 370 ' -> ', 371 target, 372 ' excluding\n', 373 exS 374 ) 375 376 spawn( 377 event, 378 config.rsync.binary, 379 '<', exS, 380 '--exclude-from=-', 381 delete, 382 config.rsync._computed, 383 '-r', 384 config.source, 385 target 386 ) 387 else 388 -- starts rsync providing a filter list 389 -- on stdin 390 local fS = table.concat( filters, '\n' ) 391 392 log( 393 'Normal', 394 'recursive startup rsync: ', 395 config.source, 396 ' -> ', 397 target, 398 ' filtering\n', 399 fS 400 ) 401 402 spawn( 403 event, 404 config.rsync.binary, 405 '<', fS, 406 '--filter=. -', 407 delete, 408 config.rsync._computed, 409 '-r', 410 config.source, 411 target 412 ) 413 end 414end 415 416 417-- 418-- Prepares and checks a syncs configuration on startup. 419-- 420rsync.prepare = function 421( 422 config, -- the configuration 423 level, -- additional error level for inherited use ( by rsyncssh ) 424 skipTarget -- used by rsyncssh, do not check for target 425) 426 427 -- First let default.prepare test the checkgauge 428 default.prepare( config, level + 6 ) 429 430 if not skipTarget and not config.target 431 then 432 error( 433 'default.rsync needs "target" configured', 434 level 435 ) 436 end 437 438 -- checks if the _computed argument exists already 439 if config.rsync._computed 440 then 441 error( 442 'please do not use the internal rsync._computed parameter', 443 level 444 ) 445 end 446 447 -- computes the rsync arguments into one list 448 local crsync = config.rsync; 449 450 -- everything implied by archive = true 451 local archiveFlags = { 452 recursive = true, 453 links = true, 454 perms = true, 455 times = true, 456 group = true, 457 owner = true, 458 devices = true, 459 specials = true, 460 hard_links = false, 461 acls = false, 462 xattrs = false, 463 } 464 465 -- if archive is given the implications are filled in 466 if crsync.archive 467 then 468 for k, v in pairs( archiveFlags ) 469 do 470 if crsync[ k ] == nil 471 then 472 crsync[ k ] = v 473 end 474 end 475 end 476 477 crsync._computed = { true } 478 479 local computed = crsync._computed 480 481 local computedN = 2 482 483 local shortFlags = { 484 acls = 'A', 485 backup = 'b', 486 checksum = 'c', 487 compress = 'z', 488 copy_dirlinks = 'k', 489 copy_links = 'L', 490 cvs_exclude = 'C', 491 dry_run = 'n', 492 executability = 'E', 493 group = 'g', 494 hard_links = 'H', 495 ignore_times = 'I', 496 ipv4 = '4', 497 ipv6 = '6', 498 keep_dirlinks = 'K', 499 links = 'l', 500 one_file_system = 'x', 501 omit_dir_times = 'O', 502 omit_link_times = 'J', 503 owner = 'o', 504 perms = 'p', 505 protect_args = 's', 506 prune_empty_dirs = 'm', 507 quiet = 'q', 508 sparse = 'S', 509 times = 't', 510 update = 'u', 511 verbose = 'v', 512 whole_file = 'W', 513 xattrs = 'X', 514 } 515 516 local shorts = { '-' } 517 local shortsN = 2 518 519 if crsync._extra 520 then 521 for k, v in ipairs( crsync._extra ) 522 do 523 computed[ computedN ] = v 524 computedN = computedN + 1 525 end 526 end 527 528 for k, flag in pairs( shortFlags ) 529 do 530 if crsync[ k ] 531 then 532 shorts[ shortsN ] = flag 533 shortsN = shortsN + 1 534 end 535 end 536 537 if crsync.devices and crsync.specials 538 then 539 shorts[ shortsN ] = 'D' 540 shortsN = shortsN + 1 541 else 542 if crsync.devices 543 then 544 computed[ computedN ] = '--devices' 545 computedN = computedN + 1 546 end 547 548 if crsync.specials 549 then 550 computed[ computedN ] = '--specials' 551 computedN = computedN + 1 552 end 553 end 554 555 if crsync.append 556 then 557 computed[ computedN ] = '--append' 558 computedN = computedN + 1 559 end 560 561 if crsync.append_verify 562 then 563 computed[ computedN ] = '--append-verify' 564 computedN = computedN + 1 565 end 566 567 if crsync.backup_dir 568 then 569 computed[ computedN ] = '--backup-dir=' .. crsync.backup_dir 570 computedN = computedN + 1 571 end 572 573 if crsync.bwlimit 574 then 575 computed[ computedN ] = '--bwlimit=' .. crsync.bwlimit 576 computedN = computedN + 1 577 end 578 579 if crsync.chmod 580 then 581 computed[ computedN ] = '--chmod=' .. crsync.chmod 582 computedN = computedN + 1 583 end 584 585 if crsync.chown 586 then 587 computed[ computedN ] = '--chown=' .. crsync.chown 588 computedN = computedN + 1 589 end 590 591 if crsync.groupmap 592 then 593 computed[ computedN ] = '--groupmap=' .. crsync.groupmap 594 computedN = computedN + 1 595 end 596 597 if crsync.existing 598 then 599 computed[ computedN ] = '--existing' 600 computedN = computedN + 1 601 end 602 603 if crsync.inplace 604 then 605 computed[ computedN ] = '--inplace' 606 computedN = computedN + 1 607 end 608 609 if crsync.password_file 610 then 611 computed[ computedN ] = '--password-file=' .. crsync.password_file 612 computedN = computedN + 1 613 end 614 615 if crsync.rsh 616 then 617 computed[ computedN ] = '--rsh=' .. crsync.rsh 618 computedN = computedN + 1 619 end 620 621 if crsync.rsync_path 622 then 623 computed[ computedN ] = '--rsync-path=' .. crsync.rsync_path 624 computedN = computedN + 1 625 end 626 627 if crsync.suffix 628 then 629 computed[ computedN ] = '--suffix=' .. crsync.suffix 630 computedN = computedN + 1 631 end 632 633 if crsync.temp_dir 634 then 635 computed[ computedN ] = '--temp-dir=' .. crsync.temp_dir 636 computedN = computedN + 1 637 end 638 639 if crsync.timeout 640 then 641 computed[ computedN ] = '--timeout=' .. crsync.timeout 642 computedN = computedN + 1 643 end 644 645 if crsync.usermap 646 then 647 computed[ computedN ] = '--usermap=' .. crsync.usermap 648 computedN = computedN + 1 649 end 650 651 if shortsN ~= 2 652 then 653 computed[ 1 ] = table.concat( shorts, '' ) 654 else 655 computed[ 1 ] = { } 656 end 657 658 -- appends a / to target if not present 659 -- and not a ':' for home dir. 660 if not skipTarget 661 and string.sub( config.target, -1 ) ~= '/' 662 and string.sub( config.target, -1 ) ~= ':' 663 then 664 config.target = config.target..'/' 665 end 666end 667 668 669-- 670-- By default do deletes. 671-- 672rsync.delete = true 673 674-- 675-- Rsyncd exitcodes 676-- 677rsync.exitcodes = default.rsyncExitCodes 678 679-- 680-- Calls rsync with this default options 681-- 682rsync.rsync = 683{ 684 -- The rsync binary to be called. 685 binary = '/usr/local/bin/rsync', 686 links = true, 687 times = true, 688 protect_args = true 689} 690 691 692-- 693-- Default delay 694-- 695rsync.delay = 15 696