1#!/usr/local/bin/perl -w 2 3my $grabber_cvs_id = '$Id: tv_grab_zz_sdjson,v 1.1 2017/01/24 00:41:08 rmeden Exp $'; 4 5=head1 NAME 6 7tv_grab_zz_sdjson - Grab TV listings from Schedules Direct SD-JSON service. 8 9=head1 SYNOPSIS 10 11tv_grab_zz_sdjson --help 12 13tv_grab_zz_sdjson --info 14 15tv_grab_zz_sdjson --version 16 17tv_grab_zz_sdjson --capabilities 18 19tv_grab_zz_sdjson --description 20 21 22tv_grab_zz_sdjson [--config-file FILE] 23 [--days N] [--offset N] 24 [--output FILE] [--quiet] [--debug] 25 26tv_grab_zz_sdjson --configure [--config-file FILE] 27 28=head1 DESCRIPTION 29 30This is an XMLTV grabber for the Schedules Direct 31(http://www.schedulesdirect.org) JSON API. 32 33=head1 CONFIGURATION 34 35Run tv_grab_zz_sdjson with the --configure option to create a config file. 36 37MythTV does not use the default XMLTV config file path. If using MythTV you 38should also specify the config file such as: 39 40 tv_grab_zz_sdjson --configure --config-file ~/.mythtv/source_name.xmltv 41 42Doing the XMLTV config from within the MythTV GUI seems very flaky so you 43are probably better off configuring from the command line. 44 45=head1 AUTHOR 46 47Kevin Groeneveld (kgroeneveld at gmail dot com) 48 49=cut 50 51use strict; 52use XMLTV; 53use XMLTV::Options qw(ParseOptions); 54use XMLTV::Configure::Writer; 55use XMLTV::Ask; 56use Cwd; 57use Storable; 58use LWP::UserAgent; 59use JSON; 60use Digest::SHA qw(sha1_hex); 61use DateTime; 62use Scalar::Util qw(looks_like_number); 63use Try::Tiny; 64use Data::Dumper; 65 66my $grabber_name; 67my $grabber_version; 68 69if($grabber_cvs_id =~ m!\$Id: ([^,]+),v (\S+) ([0-9/: -]+) !) { 70 $grabber_name = $1; 71 $grabber_version = "$2 $3"; 72} 73else { 74 $grabber_name = 'tv_grab_zz_sdjson'; 75 $grabber_version = '0.1'; 76} 77 78# The XMLTV::Writer docs only indicate you need to set 'encoding'. However, 79# this value does not get passed to the underlying XML::Writer object. Unless 80# 'ENCODING' is also specified XML::Writer does not actually encode the data! 81my %w_args = ( 82 'encoding' => 'utf-8', 83 'ENCODING' => 'utf-8', 84 'UNSAFE' => 1, 85); 86 87my %tv_attributes = ( 88 'source-info-name' => 'Schedules Direct', 89 'source-info-url' => 'http://www.schedulesdirect.org', 90 'generator-info-name' => "$grabber_name $grabber_version", 91); 92 93my @channel_id_formats = ( 94 [ 'default', 'I%s.json.schedulesdirect.org', 'Default Format' ], 95 [ 'zap2it', 'I%s.labs.zap2it.com', 'tv_grab_na_dd Format' ], 96 [ 'mythtv', '%s', 'MythTV Internal DD Grabber Format' ], 97); 98 99my @previously_shown_formats = ( 100 [ 'date', '%Y%m%d', 'Date Only' ], 101 [ 'datetime', '%Y%m%d%H%M%S %z', 'Date And Time' ], 102); 103 104my $cache_schema = 1; 105 106my $sd_json_baseurl = 'https://json.schedulesdirect.org'; 107my $sd_json_api = '/20141201/'; 108my $sd_json_token; 109my $sd_json_status; 110my $sd_json_request_max = 5000; 111 112my $ua = LWP::UserAgent->new(agent => "$grabber_name $grabber_version"); 113$ua->default_header('accept-encoding' => scalar HTTP::Message::decodable()); 114 115my $debug; 116my $quiet; 117 118# In general we rely on ParseOptions to parse the command line options. However 119# ParseOptions does not pass the options to stage_sub so we check for some 120# options on our own. 121for my $opt (@ARGV) { 122 $debug = 1 if($opt =~ /--debug/i); 123 $quiet = 1 if($opt =~ /--quiet/i); 124} 125 126$quiet = 0 if $debug; 127$ua->show_progress(1) unless $quiet; 128 129my ($opt, $conf) = ParseOptions({ 130 grabber_name => $grabber_name, 131 version => $grabber_cvs_id, 132 description => 'Schedules Direct JSON API', 133 capabilities => [qw/baseline manualconfig preferredmethod/], 134 stage_sub => \&config_stage, 135 listchannels_sub => \&list_channels, 136 preferredmethod => 'allatonce', 137 defaults => { days => -1 }, 138}); 139 140sub get_conf_format { 141 my ($config, $options, $text) = @_; 142 my $result; 143 144 if($conf->{$config}->[0]) { 145 for my $format (@{$options}) { 146 if($format->[0] eq $conf->{$config}->[0]) { 147 $result = $format->[1]; 148 last; 149 } 150 } 151 } 152 153 if(!$result) { 154 print STDERR "Valid $text not specified in config, using default.\n" unless $quiet; 155 $result = $options->[0]->[1]; 156 } 157 158 return $result; 159} 160 161my $channel_id_format = get_conf_format('channel-id-format', \@channel_id_formats, 'channel ID format'); 162my $previously_shown_format = get_conf_format('previously-shown-format', \@previously_shown_formats, 'previously shown format'); 163 164# default days to largish value 165if($opt->{'days'} < 0) { 166 $opt->{'days'} = 100; 167} 168 169sub get_start_stop_time { 170 # calculate start and stop time from offset and days options 171 my $dt_start = DateTime->today(time_zone => 'local'); 172 $dt_start->add(days => $opt->{'offset'}); 173 my $dt_stop = $dt_start->clone(); 174 $dt_stop->add(days => $opt->{'days'}); 175 176 # source data has times in UTC 177 $dt_start->set_time_zone('UTC'); 178 $dt_stop->set_time_zone('UTC'); 179 180 # convert DateTime to seconds from epoch which will allow for a LOT faster 181 # comparisons than comparing DateTime objects 182 return ($dt_start->epoch(), $dt_stop->epoch()); 183} 184my ($time_start, $time_stop) = get_start_stop_time(); 185 186my $cache_file = $conf->{'cache'}->[0]; 187 188sub get_default_cache_file { 189 my $winhome; 190 if(defined $ENV{HOMEDRIVE} && defined $ENV{HOMEPATH}) { 191 $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH}; 192 } 193 my $home = $ENV{HOME} || $winhome || getcwd(); 194 195 return "$home/.xmltv/$grabber_name.cache"; 196} 197 198# days to add to day of month to get days since Jan 1st 199my @days_norm = ( -1, 30, 58, 89, 119, 150, 180, 211, 242, 272, 303, 333 ); 200my @days_leap = ( -1, 30, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 ); 201 202sub is_leap_year { 203 return (!($_[0] % 4) && (($_[0] % 100) || !($_[0] % 400))); 204} 205 206sub parse_airtime { 207 use integer; 208 my ($year, $month, $day, $hour, $min, $sec) = ($_[0] =~ /(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z/); 209 210 # determine number of days since Jan 1st of requested year 211 $month -= 1; 212 $day += is_leap_year($year) ? $days_leap[$month] : $days_norm[$month]; 213 214 # add number of days (minus leap days) for years since 1970 215 $day += ($year - 1970) * 365; 216 217 # add leap days from previous years since year 0 (we already included leap 218 # day for this year), subtract number of leap days between 0 and 1970 (477) 219 $year -= 1; 220 $day += $year / 4 - $year / 100 + $year / 400 - 477; 221 222 return ($day * 86400 + $hour * 3600 + $min * 60 + $sec); 223} 224 225sub format_airtime { 226 my ($sec, $min, $hour, $day, $month, $year) = gmtime($_[0]); 227 return sprintf('%04d%02d%02d%02d%02d%02d +0000', $year + 1900, $month + 1, $day, $hour, $min, $sec); 228} 229 230my $dt_zone_local = DateTime::TimeZone->new(name => 'local'); 231 232# SD-JSON only specifies a date for originalAirDate. Older versions of 233# mythtv need full date and time even though xmltv only requires date. 234# We assume local time as mythtv expects and set the time to noon to 235# minimize the chance of an error causing the day to be off by one. 236sub parse_original_airdate { 237 my ($year, $month, $day) = ($_[0] =~ /(\d+)-(\d+)-(\d+)/); 238 local $Params::Validate::NO_VALIDATION = 1; 239 return DateTime->new( 240 year => $year, 241 month => $month, 242 day => $day, 243 hour => 12, 244 time_zone => $dt_zone_local, 245 ); 246} 247 248sub retry { 249 my ($action) = @_; 250 my $retry = 3; 251 my $result; 252 253 for(;;) { 254 try { 255 $result = $action->(); 256 } 257 catch { 258 if(--$retry) { 259 print STDERR $_, "Retry in 10 seconds...\n" unless $quiet; 260 sleep 10; 261 } 262 else { 263 die $_, "Retry count exceeded."; 264 } 265 }; 266 return $result if $result; 267 } 268} 269 270sub sd_json_request { 271 my ($method, $path, $content) = @_; 272 273 my $url; 274 if($path =~ /^\//) { 275 $url = $sd_json_baseurl . $path; 276 } 277 else { 278 $url = $sd_json_baseurl . $sd_json_api . $path; 279 } 280 281 my @params; 282 push(@params, content_type => 'application/json'); 283 push(@params, token => $sd_json_token) unless $path eq 'token'; 284 push(@params, content => encode_json($content)) if defined $content; 285 286 my $response = $ua->$method($url, @params); 287 if($response->is_success()) { 288 return decode_json($response->decoded_content()); 289 } 290 else { 291 my $msg = $response->decoded_content(); 292 293 if($response->header('content-type') =~ m{application/json}i) { 294 my $error = decode_json($msg); 295 296 # for lineups request don't consider 4102/NO_LINEUPS an error 297 if($path eq 'lineups' && $error->{'code'} == 4102) { 298 return undef; 299 } 300 301 $msg = "Server (ID=$error->{'serverID'} Time=$error->{'datetime'}) returned an error:\n" 302 ."$error->{'message'} ($error->{'code'}/$error->{'response'})"; 303 } 304 305 print STDERR Dumper($response) if $debug; 306 die $msg, "\n"; 307 } 308} 309 310sub sd_json_get_token { 311 my ($username, $password) = @_; 312 313 retry sub { 314 my $response = sd_json_request('post', 'token', { username => $username, password => $password }); 315 if(ref $response ne 'HASH' || !exists $response->{'token'}) { 316 die "Invalid token response.\n"; 317 } 318 return $response->{'token'}; 319 }; 320} 321 322sub sd_json_get_status { 323 retry sub { 324 my $status = sd_json_request('get', 'status'); 325 if(ref $status ne 'HASH' || 326 ref $status->{'systemStatus'} ne 'ARRAY' || ref $status->{'systemStatus'}->[0] ne 'HASH' || 327 ref $status->{'account'} ne 'HASH' || 328 ref $status->{'lineups'} ne 'ARRAY') { 329 die "Invalid status response.\n" 330 } 331 return $status; 332 } 333} 334 335sub sd_json_get_available { 336 my ($type) = @_; 337 my $result = sd_json_request('get', 'available'); 338 339 if($type) { 340 for my $entry (@{$result}) { 341 if($entry->{'type'} eq $type) { 342 return $entry; 343 } 344 } 345 } 346 347 return $result; 348} 349 350sub sd_json_get_lineups { 351 return sd_json_request('get', 'lineups'); 352} 353 354sub sd_json_get_headends { 355 my ($country, $postalcode) = @_; 356 return sd_json_request('get', "headends?country=$country&postalcode=$postalcode"); 357} 358 359sub sd_json_get_transmitters { 360 my ($country) = @_; 361 return sd_json_request('get', "transmitters/$country"); 362} 363 364sub sd_json_add_lineup { 365 my ($lineup) = @_; 366 return sd_json_request('put', "lineups/$lineup"); 367} 368 369sub sd_json_delete_lineup { 370 my ($lineup) = @_; 371 return sd_json_request('delete', "lineups/$lineup"); 372} 373 374sub sd_json_get_lineup { 375 my ($lineup) = @_; 376 retry sub { 377 my $lineup = sd_json_request('get', $lineup); 378 if(ref $lineup ne 'HASH') { 379 die "Invalid lineup response.\n" 380 } 381 return $lineup; 382 } 383} 384 385sub sd_json_get_schedules_md5 { 386 my ($channels) = @_; 387 my @stations; 388 for my $channel (@{$channels}) { 389 push(@stations, { stationID => $channel }); 390 } 391 return sd_json_request('post', 'schedules/md5', \@stations); 392} 393 394sub sd_json_get_schedules { 395 my ($schedules) = @_; 396 return sd_json_request('post', 'schedules', $schedules); 397} 398 399sub sd_json_get_programs { 400 my ($programs) = @_; 401 return sd_json_request('post', 'programs', $programs); 402} 403 404sub sd_json_init { 405 my ($conf) = @_; 406 407 if(!defined $sd_json_status) { 408 $sd_json_token = sd_json_get_token($conf->{'username'}->[0], sha1_hex($conf->{'password'}->[0])); 409 $sd_json_status = sd_json_get_status(); 410 411 my $status = $sd_json_status->{'systemStatus'}->[0]->{'status'}; 412 if($status !~ /online/i) { 413 die "Schedules Direct system status: $status\n"; 414 } 415 } 416} 417 418sub sd_json_get_image_url { 419 my ($url) = @_; 420 421 if($url =~ /^http/) { 422 return $url; 423 } 424 else { 425 return $sd_json_baseurl . $sd_json_api . 'image/' . $url; 426 } 427} 428 429sub get_lineup_description { 430 my ($lineup) = @_; 431 432 my $location = $lineup->{'location'} // 'unknown'; 433 my $transport = $lineup->{'transport'} // 'unknown'; 434 my $name = $lineup->{'name'} // 'unknown'; 435 my $id = $lineup->{'lineup'} // 'unknown'; 436 437 if($lineup->{'isDeleted'}) { 438 return "$id | $name"; 439 } 440 elsif($transport eq 'QAM') { 441 return "$id | $transport"; 442 } 443 else { 444 return "$id | $name | $location | $transport"; 445 } 446} 447 448my %transmitter_countries; 449 450sub ask_search_by_transmitter { 451 my ($country) = @_; 452 453 if(!%transmitter_countries) { 454 my $available = sd_json_get_available('DVB-T'); 455 for ($available->{'description'} =~ /[A-Z]{3}/g) { 456 $transmitter_countries{$_} = undef; 457 } 458 } 459 460 if(exists $transmitter_countries{$country}) { 461 my @options; 462 push(@options, 'transmitter'); 463 push(@options, 'postal' ); 464 465 if(ask_choice('Search by Transmitter or Postal Code:', $options[0], @options) eq $options[0]) { 466 return 1; 467 } 468 } 469 470 return 0; 471} 472 473sub config_stage { 474 my ($stage, $conf) = @_; 475 476 if($stage ne 'start' && $stage ne 'login') { 477 sd_json_init($conf); 478 } 479 480 my $result; 481 my $w = new XMLTV::Configure::Writer(OUTPUT => \$result, %w_args); 482 $w->start(\%tv_attributes); 483 484 if($stage eq 'start') { 485 $w->write_string({ 486 id => 'cache', 487 description => [ [ 'Cache file for lineups, schedules and programs.', 'en' ] ], 488 title => [ [ 'Cache file', 'en' ] ], 489 default => get_default_cache_file(), 490 }); 491 492 $w->start_selectone({ 493 id => 'channel-id-format', 494 description => [ [ 'If you are migrating from a different grabber selecting an alternate channel ID format can make the migration easier.', 'en' ] ], 495 title => [ [ 'Select channel ID format', 'en' ] ], 496 }); 497 for my $format (@channel_id_formats) { 498 $w->write_option({ 499 value => $format->[0], 500 text => [ [ $format->[2].' (eg: '.sprintf($format->[1], 12345).')', 'en' ] ], 501 }); 502 } 503 $w->end_selectone(); 504 505 $w->start_selectone({ 506 id => 'previously-shown-format', 507 description => [ [ 'As the JSON data only includes the previously shown date normally the XML output should only have the date. However some programs such as older versions of MythTV also need a time.', 'en' ] ], 508 title => [ [ 'Select previously shown format', 'en' ] ], 509 }); 510 for my $format (@previously_shown_formats) { 511 $w->write_option({ 512 value => $format->[0], 513 text => [ [ $format->[2], 'en' ] ], 514 }); 515 } 516 $w->end_selectone(); 517 518 $w->end('login'); 519 } 520 elsif($stage eq 'login') { 521 $w->write_string({ 522 id => 'username', 523 description => [ [ 'Schedules Direct username.', 'en' ] ], 524 title => [ [ 'Username', 'en' ] ], 525 }); 526 $w->write_secretstring({ 527 id => 'password', 528 description => [ [ 'Schedules Direct password.', 'en' ] ], 529 title => [ [ 'Password', 'en' ] ], 530 }); 531 532 $w->end('account-lineups'); 533 } 534 elsif($stage eq 'account-lineups') { 535 # This stage doesn't work with configapi and I am not sure if there is 536 # currently any good way to make it work... 537 my $edit; 538 do { 539 my $max = $sd_json_status->{'account'}->{'maxLineups'}; 540 my $lineups = sd_json_get_lineups(); 541 $lineups = $lineups->{'lineups'}; 542 my $count = 0; 543 544 say("This step configures the lineups enabled for your Schedules " 545 ."Direct account. It impacts all other configurations and " 546 ."programs using the JSON API with your account. A maximum of " 547 ."$max lineups can by added to your account. In a later step " 548 ."you will choose which lineups or channels to actually use " 549 ."for this configuration.\n" 550 ."Current lineups enabled for your Schedules Direct account:" 551 ); 552 553 say('#. Lineup ID | Name | Location | Transport'); 554 for my $lineup (@{$lineups}) { 555 $count++; 556 my $desc = get_lineup_description($lineup); 557 say("$count. $desc"); 558 } 559 if(!$count) { 560 say('(none)'); 561 } 562 563 my @options; 564 push(@options, 'continue') if $count; 565 push(@options, 'add' ) if($count < $max); 566 push(@options, 'delete') if $count; 567 $edit = ask_choice('Edit account lineups:', $options[0], @options); 568 569 try 570 { 571 if($edit eq 'add') { 572 my $country = uc(ask('Lineup ID or Country (ISO-3166-1 alpha 3 such as USA or CAN):')); 573 if(length($country) > 3) { 574 sd_json_add_lineup("$country"); 575 } 576 else { 577 my $count = 0; 578 my @lineups; 579 580 if(ask_search_by_transmitter($country)) { 581 my $transmitters = sd_json_get_transmitters($country); 582 583 say('#. Lineup ID | Transmitter'); 584 for my $transmitter (sort(keys %{$transmitters})) { 585 $count++; 586 my $lineup = $transmitters->{$transmitter}; 587 push(@lineups, $lineup); 588 say("$count. $lineup | $transmitter"); 589 } 590 } 591 else { 592 my $postalcode = ask(($country eq 'USA') ? 'Zip Code:' : 'Postal Code:'); 593 my $headends = sd_json_get_headends($country, $postalcode); 594 595 say('#. Lineup ID | Name | Location | Transport'); 596 for my $headend (@{$headends}) { 597 for my $lineup (@{$headend->{'lineups'}}) { 598 $count++; 599 my $id = $lineup->{'lineup'}; 600 push(@lineups, $id); 601 say("$count. $id | $lineup->{'name'} | $headend->{'location'} | $headend->{'transport'}"); 602 } 603 } 604 } 605 606 my $add = ask_choice('Add lineup (0 = none):', 0, (0 .. $count)); 607 if($add) { 608 sd_json_add_lineup($lineups[$add - 1]); 609 } 610 } 611 } 612 elsif($edit eq 'delete') { 613 my $delete = ask_choice('Delete lineup (0 = none):', 0, (0 .. $count)); 614 if($delete) { 615 sd_json_delete_lineup($lineups->[$delete - 1]->{'lineup'}); 616 } 617 } 618 } 619 catch { 620 say($_); 621 }; 622 } 623 while($edit ne 'continue'); 624 625 $w->end('select-mode'); 626 } 627 elsif($stage eq 'select-mode') { 628 $w->start_selectone({ 629 id => 'mode', 630 description => [ [ 'Choose whether you want to include complete lineups or individual channels for this configuration.', 'en' ] ], 631 title => [ [ 'Select mode', 'en' ] ], 632 }); 633 $w->write_option({ 634 value => 'lineup', 635 text => [ [ 'lineups', 'en' ] ], 636 }); 637 $w->write_option({ 638 value => 'channels', 639 text => [ [ 'channels', 'en' ] ], 640 }); 641 $w->end_selectone(); 642 643 $w->end('select-lineups'); 644 } 645 elsif($stage eq 'select-lineups') { 646 my $lineups = sd_json_get_lineups(); 647 $lineups = $lineups->{'lineups'}; 648 649 my $desc; 650 if($conf->{'mode'}->[0] eq 'lineup') { 651 $desc = 'Choose lineups to use for this configuration.'; 652 } 653 else { 654 $desc = 'Choose lineups from which you want to select channels for this configuration.'; 655 } 656 657 $w->start_selectmany({ 658 id => $conf->{'mode'}->[0], 659 description => [ [ $desc, 'en' ] ], 660 title => [ [ 'Select linups', 'en' ] ], 661 }); 662 for my $lineup (@{$lineups}) { 663 my $id = $lineup->{'lineup'}; 664 $w->write_option({ 665 value => $id, 666 text => [ [ $id, 'en' ] ], 667 }); 668 } 669 $w->end_selectmany(); 670 671 $w->end('select-channels'); 672 } 673 else { 674 die "Unknown stage $stage"; 675 } 676 677 return $result; 678} 679 680my $cache; 681my $cache_lineups; 682my $cache_schedules; 683my $cache_programs; 684my %channel_index; 685my %channel_map; 686 687sub cache_load { 688 sub get_hash { 689 my $hash = $cache->{$_[0]}; 690 return (ref $hash eq 'HASH') ? $hash : {}; 691 } 692 693 # make sure the cache file is readable and writable 694 if(open(my $fh, '+>>', $cache_file)) { 695 close($fh); 696 } 697 else { 698 die "Cannot open $cache_file for read/write.\n"; 699 } 700 701 # attempt to retreive cached data 702 try { 703 $cache = retrieve($cache_file); 704 if(ref $cache ne 'HASH') { 705 die "Invalid cache file.\n"; 706 } 707 708 if($cache->{'schema'} == $cache_schema) { 709 $cache_lineups = get_hash('lineups'); 710 $cache_schedules = get_hash('schedules'); 711 $cache_programs = get_hash('programs'); 712 } 713 else { 714 die "Ignoring cache file with old schema.\n"; 715 } 716 } 717 catch { 718 print STDERR unless $quiet; 719 $cache_lineups = {}; 720 $cache_schedules = {}; 721 $cache_programs = {}; 722 }; 723 724 $cache = { schema => $cache_schema, lineups => $cache_lineups, schedules => $cache_schedules, programs => $cache_programs }; 725} 726 727sub cache_update_lineups { 728 print STDERR "Updating lineups...\n" unless $quiet; 729 730 my $now = DateTime->now()->epoch(); 731 my %lineups_enabled; 732 my @lineups_update; 733 734 # check for out of date lineups 735 for my $lineup (@{$sd_json_status->{'lineups'}}) { 736 if(ref $lineup ne 'HASH') { 737 print STDERR "Invalid lineup in account status.\n" unless $quiet; 738 next; 739 } 740 741 my $id = $lineup->{'lineup'}; 742 if(!$id || ref $id) { 743 print STDERR "Invalid lineup in account status.\n" unless $quiet; 744 next; 745 } 746 747 $lineups_enabled{$id} = 1; 748 749 my $metadata = $cache_lineups->{$id}->{'metadata'}; 750 if(ref $metadata ne 'HASH') { 751 print STDERR "lineup $id: new\n" if $debug; 752 push(@lineups_update, $lineup); 753 } 754 elsif($metadata->{'modified'} ne $lineup->{'modified'}) { 755 print STDERR "lineup $id: old\n" if $debug; 756 push(@lineups_update, $lineup); 757 } 758 else { 759 print STDERR "lineup $id: current\n" if $debug; 760 $cache_lineups->{$id}->{'accessed'} = $now; 761 } 762 } 763 764 # check that configured lineups are actually enabled for the account 765 my $lineup_error; 766 for my $lineup (@{$conf->{'lineup'}}, @{$conf->{'channels'}}) { 767 if(!$lineups_enabled{$lineup}) { 768 $lineup_error = 1; 769 print STDERR "Lineup $lineup in the current configuration is not enabled on your account.\n"; 770 } 771 } 772 773 if($lineup_error) { 774 die "Please reconfigure the grabber or your account settings.\n" 775 } 776 777 # update lineups 778 for my $lineup (@lineups_update) { 779 my $id = $lineup->{'lineup'}; 780 my $uri = $lineup->{'uri'}; 781 782 if(!$uri || ref $uri) { 783 print STDERR "Invalid lineup URI in account status.\n" unless $quiet; 784 next; 785 } 786 787 my $update = sd_json_get_lineup($uri); 788 $cache_lineups->{$id} = $update; 789 $cache_lineups->{$id}->{'accessed'} = $now; 790 } 791} 792 793sub cache_update_schedules { 794 my ($channels) = @_; 795 796 print STDERR "Updating schedules...\n" unless $quiet; 797 798 my $now = DateTime->now()->epoch(); 799 my $schedules_md5 = sd_json_get_schedules_md5($channels); 800 my @channels_update; 801 802 while(my ($channel, $schedule) = each %{$schedules_md5}) { 803 if(ref $schedule ne 'HASH') { 804 print STDERR "Invalid schedule for channel $channel\n" unless $quiet; 805 next; 806 } 807 808 my @dates; 809 while(my ($date, $latest) = each %{$schedule}) { 810 my $metadata = $cache_schedules->{$channel}->{$date}->{'metadata'}; 811 if(!defined $metadata) { 812 print STDERR "channel $channel $date: new\n" if $debug; 813 push(@dates, $date); 814 } 815 elsif($metadata->{'md5'} ne $latest->{'md5'}) { 816 print STDERR "channel $channel $date: old\n" if $debug; 817 push(@dates, $date); 818 } 819 else { 820 print STDERR "channel $channel $date: current\n" if $debug; 821 } 822 } 823 if(@dates) { 824 push(@channels_update, { stationID => $channel, date => \@dates }); 825 } 826 } 827 828 # update schedules 829 while(my @block = splice(@channels_update, 0, $sd_json_request_max)) { 830 my $schedules = sd_json_get_schedules(\@block); 831 for my $schedule (@{$schedules}) { 832 my $channel = $schedule->{'stationID'}; 833 my $date = $schedule->{'metadata'}->{'startDate'}; 834 $cache_schedules->{$channel}->{$date} = $schedule; 835 } 836 } 837 838 print STDERR "Updating programs...\n" unless $quiet; 839 840 my %programs_update_hash; 841 842 # create list of programs to update 843 for my $channel (@{$channels}) { 844 for my $schedule (values %{$cache_schedules->{$channel}}) { 845 for my $program (@{$schedule->{'programs'}}) { 846 my $airtime = parse_airtime($program->{'airDateTime'}); 847 my $dur = int($program->{'duration'}); 848 849 if(($airtime + $dur) > $time_start && $airtime < $time_stop) { 850 my $id = $program->{'programID'}; 851 my $cached = $cache_programs->{$id}; 852 853 if(!defined $cached) { 854 print STDERR "program $id: new\n" if $debug; 855 $programs_update_hash{$id} = 1; 856 } 857 elsif($cached->{'md5'} ne $program->{'md5'}) { 858 print STDERR "program $id: old\n" if $debug; 859 $programs_update_hash{$id} = 1; 860 } 861 else { 862 print STDERR "program $id: current\n" if $debug; 863 $cache_programs->{$id}->{'accessed'} = $now; 864 } 865 } 866 } 867 } 868 } 869 870 # update programs 871 my @programs_update = keys %programs_update_hash; 872 while(my @block = splice(@programs_update, 0, $sd_json_request_max)) { 873 my $programs = sd_json_get_programs(\@block); 874 875 for my $id (@block) { 876 $cache_programs->{$id} = shift @{$programs}; 877 $cache_programs->{$id}->{'accessed'} = $now; 878 } 879 } 880} 881 882sub cache_drop_old { 883 my $limit = DateTime->now()->subtract(days => 10)->epoch(); 884 885 print STDERR "Removing old cache entries...\n" unless $quiet; 886 887 while(my ($key, $hash) = each %{$cache}) { 888 if($key eq 'lineups' || $key eq 'programs') { 889 # remove old lineups and programs 890 while(my ($key, $value) = each %{$hash}) { 891 if(ref $value ne 'HASH' || !exists $value->{'accessed'} || $value->{'accessed'} < $limit) { 892 print STDERR "$key: drop\n" if $debug; 893 delete $hash->{$key}; 894 } 895 } 896 } 897 elsif($key eq 'schedules') { 898 # remove old schedules 899 my $today = DateTime->today()->strftime('%Y-%m-%d'); 900 while(my ($channel, $schedules) = each %{$hash}) { 901 if(ref $schedules ne 'HASH') { 902 print STDERR "$channel: drop\n" if $debug; 903 delete $cache_schedules->{$channel}; 904 next; 905 } 906 907 while(my ($date, $schedule) = each %{$schedules}) { 908 if($date lt $today) { 909 print STDERR "$channel $date: drop\n" if $debug; 910 delete $schedules->{$date}; 911 } 912 } 913 914 if(scalar keys %{$schedules} == 0) { 915 print STDERR "$channel: drop\n" if $debug; 916 delete $cache_schedules->{$channel}; 917 } 918 } 919 } 920 elsif($key ne 'schema') { 921 # remove unknown keys 922 delete $cache->{$key}; 923 } 924 } 925} 926 927sub cache_save { 928 store($cache, $cache_file); 929} 930 931sub cache_index_channels { 932 print STDERR "Indexing channels...\n" unless $quiet; 933 934 # create index 935 for my $id (@{$conf->{'lineup'}}, @{$conf->{'channels'}}) { 936 my $lineup = $cache_lineups->{$id}; 937 if(ref $lineup ne 'HASH' || ref $lineup->{'stations'} ne 'ARRAY') { 938 print STDERR "Invalid stations array for lineup $id\n" unless $quiet; 939 next; 940 } 941 942 for my $channel (@{$lineup->{'stations'}}) { 943 if(ref $channel ne 'HASH') { 944 print STDERR "Invalid channel in lineup $id\n" unless $quiet; 945 next; 946 } 947 $channel_index{$channel->{'stationID'}} = $channel; 948 } 949 950 my $qam = $lineup->{'qamMappings'}; 951 my $map; 952 953 if($qam) { 954 $map = $lineup->{'map'}->{$qam->[0]}; 955 } 956 else { 957 $map = $lineup->{'map'}; 958 } 959 960 for my $channel (@{$map}) { 961 $channel_map{$channel->{'stationID'}} = $channel; 962 } 963 } 964} 965 966sub get_channel_list { 967 my ($conf) = @_; 968 my %hash; 969 970 if($conf->{'mode'}->[0] eq 'lineup') { 971 for my $lineup (@{$conf->{'lineup'}}) { 972 if(ref $cache_lineups->{$lineup}->{'stations'} ne 'ARRAY') { 973 print STDERR "Invalid stations array for lineup $lineup\n" unless $quiet; 974 next; 975 } 976 977 for my $channel (@{$cache_lineups->{$lineup}->{'stations'}}) { 978 if(ref $channel ne 'HASH' || !$channel->{'stationID'}) { 979 print STDERR "Invalid channel in lineup $lineup\n" unless $quiet; 980 next; 981 } 982 $hash{$channel->{'stationID'}} = 1; 983 } 984 } 985 } 986 else { 987 for my $channel (@{$conf->{'channel'}}) { 988 if(exists $channel_index{$channel}) { 989 $hash{$channel} = 1; 990 } 991 else { 992 print STDERR "Channel ID $channel in the current configuration is not found in any enabled lineup.\n" unless $quiet; 993 } 994 } 995 } 996 997 my @list = sort(keys %hash); 998 return \@list; 999} 1000 1001sub get_channel_number { 1002 my ($map) = @_; 1003 1004 if($map->{'virtualChannel'}) { 1005 return $map->{'virtualChannel'}; 1006 } 1007 elsif($map->{'atscMajor'}) { 1008 return "$map->{'atscMajor'}_$map->{'atscMinor'}"; 1009 } 1010 elsif($map->{'channel'}) { 1011 return $map->{'channel'}; 1012 } 1013 elsif($map->{'frequencyHz'}) { 1014 return $map->{'frequencyHz'}; 1015 } 1016 1017 return undef; 1018} 1019 1020sub get_icon { 1021 my ($url, $width, $height) = @_; 1022 my %result; 1023 1024 if($url) { 1025 $result{'src'} = sd_json_get_image_url($url); 1026 if($width && $height) { 1027 $result{'width'} = $width; 1028 $result{'height'} = $height; 1029 } 1030 1031 return [ \%result ]; 1032 } 1033 else { 1034 return undef; 1035 } 1036} 1037 1038sub write_channel { 1039 my ($w, $channel, $map) = @_; 1040 1041 my %ch; 1042 1043 # mythtv seems to assume that the first three display-name elements are 1044 # name, callsign and channel number. We follow that scheme here. 1045 $ch{'id'} = sprintf($channel_id_format, $channel->{'stationID'}); 1046 $ch{'display-name'} = [ 1047 [ $channel->{'name'} || 'unknown name' ], 1048 [ $channel->{'callsign'} || 'unknown callsign' ], 1049 [ get_channel_number($map) || 'unknown number' ] 1050 ]; 1051 1052 my $logo = $channel->{'logo'}; 1053 my $icon = get_icon($logo->{'URL'}, $logo->{'width'}, $logo->{'height'}); 1054 $ch{'icon'} = $icon if $icon; 1055 1056 $w->write_channel(\%ch); 1057} 1058 1059# this is used by the last stage of --configure 1060sub list_channels { 1061 my ($conf, $opt) = @_; 1062 1063 # use raw channel id in configuration files 1064 $channel_id_format = '%s'; 1065 1066 my $result; 1067 my $w = new XMLTV::Writer(OUTPUT => \$result, %w_args); 1068 $w->start(\%tv_attributes); 1069 1070 for my $id (@{$conf->{'channels'}}) { 1071 my $lineup = sd_json_get_lineup("lineups/$id"); 1072 for my $channel (@{$lineup->{'stations'}}) { 1073 write_channel($w, $channel); 1074 } 1075 } 1076 1077 $w->end(); 1078 return $result; 1079} 1080 1081sub get_program_title { 1082 my ($details) = @_; 1083 my $title = $details->{'titles'}->[0]->{'title120'}; 1084 1085 if($title) { 1086 return [ [ $title ] ]; 1087 } 1088 else { 1089 return [ [ 'unknown' ] ]; 1090 } 1091} 1092 1093sub get_program_subtitle { 1094 my ($details) = @_; 1095 my $subtitle = $details->{'episodeTitle150'}; 1096 1097 if($subtitle) { 1098 return [ [ $subtitle ] ]; 1099 } 1100 else { 1101 return undef; 1102 } 1103} 1104 1105sub get_program_description { 1106 my ($details) = @_; 1107 my $descriptions = $details->{'descriptions'}; 1108 1109 if(exists $descriptions->{'description1000'}) { 1110 return [ [ $descriptions->{'description1000'}->[0]->{'description'} ] ]; 1111 } 1112 elsif(exists $descriptions->{'description100'}) { 1113 return [ [ $descriptions->{'description100'}->[0]->{'description'} ] ]; 1114 } 1115 else { 1116 return undef; 1117 } 1118} 1119 1120sub get_program_credits { 1121 my ($details) = @_; 1122 my %credits; 1123 1124 for my $credit (@{$details->{'cast'}}, @{$details->{'crew'}}) { 1125 my $role = $credit->{'role'}; 1126 my $name = $credit->{'name'}; 1127 my $key; 1128 1129 if($role =~ /director/i) { 1130 $key = 'director'; 1131 } 1132 elsif($role =~ /(actor|voice)/i) { 1133 $key = 'actor'; 1134 if($credit->{'characterName'}) { 1135 $name = [ $name, $credit->{'characterName'} ]; 1136 } 1137 } 1138 elsif($role =~ /writer/i) { 1139 $key = 'writer'; 1140 } 1141 elsif($role =~ /producer/i) { 1142 $key = 'producer'; 1143 } 1144 elsif($role =~ /(host|anchor)/i) { 1145 $key = 'presenter'; 1146 } 1147 elsif($role =~ /(guest|contestant)/i) { 1148 $key = 'guest'; 1149 } 1150 else { 1151# print STDERR "$role\n"; 1152 } 1153 1154 if($key) { 1155 if(exists $credits{$key}) { 1156 push(@{$credits{$key}}, $name); 1157 } 1158 else { 1159 $credits{$key} = [ $name ]; 1160 } 1161 } 1162 } 1163 1164 if(scalar keys %credits) { 1165 return \%credits; 1166 } 1167 else { 1168 return undef; 1169 } 1170} 1171 1172sub get_program_date { 1173 my ($details) = @_; 1174 1175 my $year = $details->{'movie'}->{'year'}; 1176 if($year) { 1177 return $year; 1178 } 1179 1180 return undef; 1181} 1182 1183sub get_program_category { 1184 my ($channel, $details) = @_; 1185 my %seen; 1186 my @result; 1187 1188 sub add { 1189 my ($result, $category, $seen) = @_; 1190 if($category && !exists $seen->{$category}) { 1191 $seen->{$category} = 1; 1192 push(@{$result}, [ $category ]); 1193 } 1194 } 1195 1196 for my $genre (@{$details->{'genres'}}) { 1197 add(\@result, $genre, \%seen); 1198 } 1199 add(\@result, $details->{'showType'}, \%seen); 1200 1201 # mythtv specifically looks for movie|series|sports|tvshow 1202 my $entity_type = $details->{'entityType'}; 1203 if($entity_type =~ /movie/i) { 1204 add(\@result, 'movie', \%seen); 1205 } 1206 elsif($entity_type =~ /episode/i) { 1207 add(\@result, 'series', \%seen); 1208 } 1209 elsif($entity_type =~ /sports/i) { 1210 add(\@result, 'sports', \%seen); 1211 } 1212 elsif($channel->{'isRadioStation'}) { 1213 add(\@result, 'radio', \%seen); 1214 } 1215 else { 1216 add(\@result, 'tvshow', \%seen); 1217 } 1218 1219 if(scalar @result) { 1220 return \@result; 1221 } 1222 else { 1223 return undef; 1224 } 1225} 1226 1227sub get_program_length { 1228 my ($details) = @_; 1229 my $duration = $details->{'duration'} || $details->{'movie'}->{'duration'}; 1230 1231 if($duration) { 1232 return $duration; 1233 } 1234 else { 1235 return undef; 1236 } 1237} 1238 1239sub get_program_icon { 1240 my ($details) = @_; 1241 my $episode_image = $details->{'episodeImage'}; 1242 return get_icon($episode_image->{'uri'}, $episode_image->{'width'}, $episode_image->{'height'}); 1243} 1244 1245sub get_program_url { 1246 my ($details) = @_; 1247 1248 my $url = $details->{'officialURL'}; 1249 if($url) { 1250 return [ $url ]; 1251 } 1252 1253 return undef; 1254} 1255 1256sub _get_program_episode { 1257 my ($number, $total) = @_; 1258 my $result = ''; 1259 1260 if(looks_like_number($number) && int($number)) { 1261 $result = sprintf('%d', $number - 1); 1262 if(looks_like_number($total) && int($total)) { 1263 $result .= sprintf('/%d', $total); 1264 } 1265 } 1266 1267 return $result; 1268} 1269 1270sub get_program_episode { 1271 my ($program, $details) = @_; 1272 my $season = ''; 1273 my $episode = ''; 1274 my $part = ''; 1275 my @result; 1276 1277 my $metadata = $details->{'metadata'}->[0]->{'Gracenote'}; 1278 if($metadata) 1279 { 1280 $season = _get_program_episode($metadata->{'season'}, $metadata->{'totalSeason'}); 1281 $episode = _get_program_episode($metadata->{'episode'}, $metadata->{'totalEpisodes'}); 1282 } 1283 1284 my $multipart = $program->{'multipart'}; 1285 if($multipart) { 1286 $part = _get_program_episode($multipart->{'partNumber'}, $multipart->{'totalParts'}); 1287 } 1288 1289 if(length($season) || length($episode) || length($part)) { 1290 push(@result, [ sprintf('%s.%s.%s', $season, $episode, $part), 'xmltv_ns' ]); 1291 } 1292 1293 push(@result, [ $program->{'programID'}, 'dd_progid' ]); 1294 1295 return \@result; 1296} 1297 1298sub get_program_video { 1299 my ($program) = @_; 1300 my %video; 1301 1302 for my $item (@{$program->{'videoProperties'}}) { 1303 if($item =~ /hdtv/i) { 1304 $video{'quality'} = 'HDTV'; 1305 } 1306 } 1307 1308 if(scalar keys %video) { 1309 return \%video; 1310 } 1311 else { 1312 return undef; 1313 } 1314} 1315 1316sub get_program_audio { 1317 my ($program) = @_; 1318 my %audio; 1319 1320 for my $item (@{$program->{'audioProperties'}}) { 1321 if($item =~ /mono/i) { 1322 $audio{'stereo'} = 'mono'; 1323 } 1324 elsif($item =~ /stereo/i) { 1325 $audio{'stereo'} = 'stereo'; 1326 } 1327 elsif($item =~ /DD/i) { 1328 $audio{'stereo'} = 'dolby digital'; 1329 } 1330 } 1331 1332 if(scalar keys %audio) { 1333 return \%audio; 1334 } 1335 1336 return undef; 1337} 1338 1339# The xmltv docs state this field is "When and where the programme was last shown". 1340# However mythtv expects the original air date to be in this field. 1341sub get_program_previously_shown { 1342 my ($details) = @_; 1343 my %previously_shown; 1344 1345 my $date = $details->{'originalAirDate'}; 1346 if($date) { 1347 my $dt = parse_original_airdate($date); 1348 $previously_shown{'start'} = $dt->strftime($previously_shown_format); 1349 } 1350 1351 if(scalar keys %previously_shown) { 1352 return \%previously_shown; 1353 } 1354 1355 return undef; 1356} 1357 1358sub get_program_premiere { 1359 my ($program) = @_; 1360 my $premiere = $program->{'isPremiereOrFinale'}; 1361 1362 if(defined $premiere && $premiere =~ /premiere/i) { 1363 return [ $premiere ]; 1364 } 1365 1366 return undef; 1367} 1368 1369sub get_program_new { 1370 my ($program) = @_; 1371 my $new = $program->{'new'}; 1372 1373 if(defined $new) { 1374 return 1; 1375 } 1376 1377 return undef; 1378} 1379 1380sub get_program_subtitles { 1381 my ($program) = @_; 1382 1383 if(grep('^cc$', @{$program->{'audioProperties'}})) { 1384 return [ { 'type' => 'teletext' } ]; 1385 } 1386 1387 return undef; 1388} 1389 1390sub get_program_rating { 1391 my ($program, $details) = @_; 1392 1393 # first check 'contentRating' then 'ratings' 1394 my $ratings = $details->{'contentRating'}; 1395 if(!defined $ratings || ref $ratings ne 'ARRAY') { 1396 $ratings = $program->{'ratings'}; 1397 if(!defined $ratings || ref $ratings ne 'ARRAY') { 1398 return undef; 1399 } 1400 } 1401 1402 my @result; 1403 for my $rating (@{$ratings}) { 1404 my $code = $rating->{'code'}; 1405 my $body = $rating->{'body'}; 1406 if($code) { 1407 push(@result, [ $code, $body ]); 1408 } 1409 } 1410 1411 if(scalar @result) { 1412 return \@result; 1413 } 1414 1415 return undef; 1416} 1417 1418sub get_program_star_rating { 1419 my ($details) = @_; 1420 my $rating = $details->{'movie'}->{'qualityRating'}->[0]; 1421 1422 if($rating) { 1423 return [ [ "$rating->{'rating'}/$rating->{'maxRating'}", $rating->{'ratingsBody'} ] ]; 1424 } 1425 else { 1426 return undef; 1427 } 1428} 1429 1430sub write_programme { 1431 my ($w, $channel, $program, $details) = @_; 1432 1433 my $airtime = parse_airtime($program->{'airDateTime'}); 1434 my $dur = int($program->{'duration'}); 1435 1436 if(($airtime + $dur) > $time_start && $airtime < $time_stop) { 1437 my $start = format_airtime($airtime); 1438 my $stop = format_airtime($airtime + $dur); 1439 1440 $w->write_programme({ 1441 'channel' => sprintf($channel_id_format, $channel->{'stationID'}), 1442 'start' => $start, 1443 'stop' => $stop, 1444 'title' => get_program_title($details), 1445 'sub-title' => get_program_subtitle($details), 1446 'desc' => get_program_description($details), 1447 'credits' => get_program_credits($details), 1448 'date' => get_program_date($details), 1449 'category' => get_program_category($channel, $details), 1450# 'keyword' => undef, 1451# 'language' => undef, 1452# 'orig-language' => undef, 1453 'length' => get_program_length($details), 1454 'icon' => get_program_icon($details), 1455 'url' => get_program_url($details), 1456# 'country' => undef, 1457 'episode-num' => get_program_episode($program, $details), 1458 'video' => get_program_video($program), 1459 'audio' => get_program_audio($program), 1460 'previously-shown' => get_program_previously_shown($details), 1461 'premiere' => get_program_premiere($program), 1462# 'last-chance' => undef, 1463 'new' => get_program_new($program), 1464 'subtitles' => get_program_subtitles($program), 1465 'rating' => get_program_rating($program, $details), 1466 'star-rating' => get_program_star_rating($details), 1467# 'review' => undef, 1468 }); 1469 } 1470} 1471 1472sub grab_listings { 1473 my ($conf) = @_; 1474 my $channels; 1475 1476 print STDERR "Initializing...\n" unless $quiet; 1477 cache_load(); 1478 sd_json_init($conf); 1479 cache_update_lineups(); 1480 cache_index_channels(); 1481 $channels = get_channel_list($conf); 1482 1483 if(!@{$channels}) { 1484 die "No lineups or channels configured.\n"; 1485 } 1486 1487 cache_update_schedules($channels); 1488 cache_drop_old(); 1489 cache_save(); 1490 1491 print STDERR "Writing output...\n" unless $quiet; 1492 my $w = new XMLTV::Writer(%w_args); 1493 $w->start(\%tv_attributes); 1494 1495 # write channels 1496 for my $channel (@{$channels}) { 1497 write_channel($w, $channel_index{$channel}, $channel_map{$channel}); 1498 } 1499 1500 # write programs 1501 for my $channel (@{$channels}) { 1502 my $schedules = $cache_schedules->{$channel}; 1503 for my $day (sort(keys %{$schedules})) { 1504 for my $program (@{$schedules->{$day}->{'programs'}}) { 1505 write_programme($w, $channel_index{$channel}, $program, $cache_programs->{$program->{'programID'}}); 1506 } 1507 } 1508 } 1509 1510 $w->end(); 1511 print STDERR "Done\n" unless $quiet; 1512} 1513 1514grab_listings($conf); 1515