« back to all changes in this revision
Viewing changes to grab/es_miguiatv/tv_grab_es_miguiatv
- browse files at revision 11
- compare with another revision
- download diff
- download tarball
- view history from revision 11
- Committer: Bazaar Package Importer
- Author(s): Mario Limonciello
- Date: 2008-02-20 09:32:36 UTC
- mfrom: (1.2.9 upstream)
- Revision ID: james.westby@ubuntu.com-20080220093236-4e57hpj6ax3399hy
Tags: 0.5.51-2ubuntu1
* Merge from debian unstable, remaining changes:
- Fixes multiple broken grabbers (LP: #193703)
- Update 06_grab_no.dpatch
- Fixes multiple broken grabbers (LP: #193703)
- Update 06_grab_no.dpatch
- files added:
- grab/es_miguiatv
- files modified:
- ChangeLog
added
removed
grab/es_miguiatv/tv_grab_es_miguiatv
1
#!/usr/bin/perl -w
2
3
=pod
4
5
=head1 NAME
6
7
tv_grab_es_miguiatv - Alternative TV grabber for Spain.
8
9
=head1 SYNOPSIS
10
11
tv_grab_es_miguiatv --help
12
13
tv_grab_es_miguiatv [--config-file FILE] --configure [--gui OPTION]
14
15
tv_grab_es_miguiatv [--config-file FILE] [--output FILE] [--days N]
16
[--offset N] [--quiet]
17
18
tv_grab_es_miguiatv --list-channels
19
20
tv_grab_es_miguiatv --capabilities
21
22
tv_grab_es_miguiatv --version
23
24
=head1 DESCRIPTION
25
26
Output TV listings for spanish channels from www.miguiatv.com.
27
Supports analogue and digital (D+) channels.
28
29
First run B<tv_grab_es_miguiatv --configure> to choose, which channels you want
30
to download. Then running B<tv_grab_es_miguiatv> with no arguments will output
31
listings in XML format to standard output.
32
33
B<--configure> Prompt for which channels,
34
and write the configuration file.
35
36
B<--config-file FILE> Set the name of the configuration file, the
37
default is B<~/.xmltv/tv_grab_es_miguiatv.conf>. This is the file written by
38
B<--configure> and read when grabbing.
39
40
B<--gui OPTION> Use this option to enable a graphical interface to be used.
41
OPTION may be 'Tk', or left blank for the best available choice.
42
Additional allowed values of OPTION are 'Term' for normal terminal output
43
(default) and 'TermNoProgressBar' to disable the use of XMLTV::ProgressBar.
44
45
B<--output FILE> Write to FILE rather than standard output.
46
47
B<--days N> Grab N days. The default is 3.
48
49
B<--offset N> Start N days in the future. The default is to start
50
from today.
51
52
B<--quiet> Suppress the progress messages normally written to standard
53
error.
54
55
B<--capabilities> Show which capabilities the grabber supports. For more
56
information, see L<http://membled.com/twiki/bin/view/Main/XmltvCapabilities>
57
58
B<--version> Show the version of the grabber.
59
60
B<--help> Print a help message and exit.
61
62
=head1 SEE ALSO
63
64
L<xmltv(5)>.
65
66
=head1 AUTHOR
67
68
Alberto Gonz�lez (alberto@pesadilla.org) based on tv_grab_es_laguiatv from CandU and tv_grab_es from Ramon Roca.
69
70
=head1 BUGS
71
72
=cut
73
74
#
75
76
77
######################################################################
78
# initializations
79
80
use strict;
81
use XMLTV::Version '$Id: tv_grab_es_miguiatv,v 1.1 2008/01/13 21:22:51 atirc Exp $ ';
82
use XMLTV::Capabilities qw/baseline manualconfig cache/;
83
use XMLTV::Description 'Spain (miguiatv.com)';
84
use Getopt::Long;
85
use Date::Manip;
86
use HTML::TreeBuilder;
87
use HTML::Entities; # parse entities
88
use IO::File;
89
use Data::Dumper;
90
91
92
use XMLTV;
93
use XMLTV::Memoize;
94
use XMLTV::ProgressBar;
95
use XMLTV::Ask;
96
use XMLTV::Config_file;
97
use XMLTV::DST;
98
use XMLTV::Get_nice;
99
use XMLTV::Mode;
100
use XMLTV::Date;
101
# Todo: perhaps we should internationalize messages and docs?
102
use XMLTV::Usage <<END
103
$0: get Spanish television listings in XMLTV format
104
To configure: $0 --configure [--config-file FILE]
105
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
106
[--offset N] [--quiet]
107
To list channels: $0 --list-channels
108
To show capabilities: $0 --capabilities
109
To show version: $0 --version
110
END
111
;
112
113
# Attributes of the root element in output.
114
my $HEAD = { 'source-info-url' => 'http://www.miguiatv.com/todos-los-canales.html',
115
'source-data-url' => "http://www.miguiatv.com/todos-los-canales.html",
116
'generator-info-name' => 'XMLTV',
117
'generator-info-url' => 'http://membled.com/work/apps/xmltv/',
118
};
119
120
# Whether zero-length programmes should be included in the output.
121
my $WRITE_ZERO_LENGTH = 0;
122
my $DO_SLOWER_DESC_GET = 0;
123
124
# default language
125
my $LANG="es";
126
127
# Global channel_data
128
our @ch_all;
129
130
# debug print function
131
sub debug_print
132
{
133
# my ($str) = @_;
134
135
# print $str;
136
}
137
138
139
######################################################################
140
# get options
141
142
# Get options, including undocumented --cache option.
143
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
144
my ($opt_days, $opt_offset, $opt_help, $opt_output,
145
$opt_configure, $opt_config_file, $opt_gui,
146
$opt_quiet, $opt_list_channels);
147
$opt_days = 3; # default
148
$opt_offset = 0; # default
149
$opt_quiet = 0; # default
150
GetOptions('days=i' => \$opt_days,
151
'offset=i' => \$opt_offset,
152
'help' => \$opt_help,
153
'configure' => \$opt_configure,
154
'config-file=s' => \$opt_config_file,
155
'gui:s' => \$opt_gui,
156
'output=s' => \$opt_output,
157
'quiet' => \$opt_quiet,
158
'list-channels' => \$opt_list_channels
159
)
160
or usage(0);
161
die 'number of days must not be negative'
162
if (defined $opt_days && $opt_days < 0);
163
usage(1) if $opt_help;
164
165
XMLTV::Ask::init($opt_gui);
166
167
my $mode = XMLTV::Mode::mode('grab', # default
168
$opt_configure => 'configure',
169
$opt_list_channels => 'list-channels',
170
);
171
172
# File that stores which channels to download.
173
my $config_file
174
= XMLTV::Config_file::filename($opt_config_file, 'tv_grab_es_miguiatv', $opt_quiet);
175
176
my @config_lines; # used only in grab mode
177
if ($mode eq 'configure') {
178
XMLTV::Config_file::check_no_overwrite($config_file);
179
}
180
elsif ($mode eq 'grab') {
181
@config_lines = XMLTV::Config_file::read_lines($config_file);
182
}
183
elsif ($mode eq 'list-channels') {
184
# Config file not used.
185
}
186
else { die }
187
188
# Whatever we are doing, we need the channels data.
189
my %channels; # sets @ch_all
190
my @channels;
191
my %urls;
192
######################################################################
193
# write configuration
194
195
if ($mode eq 'configure') {
196
%channels = get_channels();
197
198
open(CONF, ">$config_file") or die "cannot write to $config_file: $!";
199
200
# Ask about getting descs
201
my $getdescs = ask_boolean("Do you want to get descriptions (very slow)");
202
warn("cannot read input, using default")
203
if not defined $getdescs;
204
205
print CONF "getdescriptions ";
206
print CONF "yes\n" if $getdescs;
207
print CONF "no\n" if not $getdescs;
208
209
# Ask about each channel.
210
my @chs = sort keys %channels;
211
my @names = map { $channels{$_} } @chs;
212
my @qs = map { "Add channel $_?" } @names;
213
my @want = ask_many_boolean(1, @qs);
214
foreach (@chs) {
215
my $w = shift @want;
216
warn("cannot read input, stopping channel questions"), last
217
if not defined $w;
218
# No need to print to user - XMLTV::Ask is verbose enough.
219
220
# Print a config line, but comment it out if channel not wanted.
221
print CONF '#' if not $w;
222
my $name = shift @names;
223
print CONF "channel $_ $name\n";
224
# TODO don't store display-name in config file.
225
}
226
227
close CONF or warn "cannot close $config_file: $!";
228
say("Finished configuration.");
229
230
exit();
231
}
232
233
234
# Not configuration, we must be writing something, either full
235
# listings or just channels.
236
#
237
die if $mode ne 'grab' and $mode ne 'list-channels';
238
239
# Options to be used for XMLTV::Writer.
240
my %w_args;
241
if (defined $opt_output) {
242
my $fh = new IO::File(">$opt_output");
243
die "cannot write to $opt_output: $!" if not defined $fh;
244
$w_args{OUTPUT} = $fh;
245
}
246
$w_args{encoding} = 'ISO-8859-15';
247
my $writer = new XMLTV::Writer(%w_args);
248
$writer->start($HEAD);
249
250
if ($mode eq 'list-channels') {
251
$writer->write_channel($_) foreach @ch_all;
252
$writer->end();
253
exit();
254
}
255
256
######################################################################
257
# We are producing full listings.
258
die if $mode ne 'grab';
259
260
# Read configuration
261
my $line_num = 1;
262
foreach (@config_lines) {
263
++ $line_num;
264
next if not defined;
265
if (/getdescriptions:?\s+(\S+)/)
266
{
267
if($1 eq "yes")
268
{
269
$DO_SLOWER_DESC_GET = 1;
270
}
271
}
272
elsif (/^channel:?\s+(\S+)\s+([^\#]+)/)
273
{
274
my $ch_did = $1;
275
my $ch_name = $2;
276
$ch_name =~ s/\s*$//;
277
push @channels, $ch_did;
278
$channels{$ch_did} = $ch_name;
279
}
280
else {
281
warn "$config_file:$line_num: bad line\n";
282
}
283
}
284
285
######################################################################
286
# begin main program
287
288
# Assume the listings source uses CET (see BUGS above).
289
my $now = DateCalc(parse_date('now'), "$opt_offset days");
290
die "No channels specified, run me with --configure\n"
291
if not keys %channels;
292
my @to_get;
293
294
295
# the order in which we fetch the channels matters
296
foreach my $ch_did (@channels) {
297
my $ch_name=$channels{$ch_did};
298
my $ch_xid="$ch_did.miguiatv.com";
299
$writer->write_channel({ id => $ch_xid,
300
'display-name' => [ [ $ch_name ] ] });
301
my $day=UnixDate($now,'%Q');
302
for (my $i=0;$i<$opt_days;$i++) {
303
push @to_get, [ $day, $ch_xid, $ch_did ];
304
#for each day
305
$day=nextday($day); die if not defined $day;
306
}
307
}
308
309
# This progress bar is for both downloading and parsing. Maybe
310
# they could be separate.
311
#
312
get_urls();
313
my $bar = new XMLTV::ProgressBar({name => 'getting listings', count => scalar @to_get})
314
if not $opt_quiet;
315
foreach (@to_get) {
316
foreach (process_table($_->[0], $_->[1], $_->[2])) {
317
$writer->write_programme($_);
318
}
319
update $bar if not $opt_quiet;
320
}
321
$bar->finish() if not $opt_quiet;
322
$writer->end();
323
324
######################################################################
325
# subroutine definitions
326
327
# Use Log::TraceMessages if installed.
328
BEGIN {
329
eval { require Log::TraceMessages };
330
if ($@) {
331
*t = sub {};
332
*d = sub { '' };
333
}
334
else {
335
*t = \&Log::TraceMessages::t;
336
*d = \&Log::TraceMessages::d;
337
Log::TraceMessages::check_argv();
338
}
339
}
340
341
####
342
# process_table: fetch a URL and process it
343
#
344
# arguments:
345
# Date::Manip object giving the day to grab
346
# xmltv id of channel
347
# elpais.es id of channel
348
#
349
# returns: list of the programme hashes to write
350
#
351
sub process_table {
352
353
my ($date, $ch_xmltv_id, $ch_es_id) = @_;
354
my $today = UnixDate($date, '%Y%m%d');
355
356
my $url = $urls{$ch_es_id};
357
$url =~ s/programacion/$today/;
358
debug_print "Getting $url\n";
359
t $url;
360
local $SIG{__WARN__} = sub
361
{
362
warn "$url: $_[0]";
363
};
364
365
# parse the page to a document object
366
my $tree = get_nice_tree($url);
367
my @program_data = get_program_data($tree);
368
my $bump_start_day=0;
369
370
my @r;
371
while (@program_data) {
372
my $cur = shift @program_data;
373
my $next = shift @program_data;
374
unshift @program_data,$next if $next;
375
376
my $p = make_programme_hash($date, $ch_xmltv_id, $ch_es_id, $cur, $next);
377
if (not $p) {
378
require Data::Dumper;
379
my $d = Data::Dumper::Dumper($cur);
380
warn "cannot write programme on $ch_xmltv_id on $date:\n$d\n";
381
}
382
else {
383
push @r, $p;
384
}
385
386
if (!$bump_start_day && bump_start_day($cur,$next)) {
387
$bump_start_day=1;
388
$date = UnixDate(DateCalc($date,"+ 1 day"),'%Q');
389
}
390
}
391
return @r;
392
}
393
394
395
sub make_programme_hash {
396
my ($date, $ch_xmltv_id, $ch_es_id, $cur, $next) = @_;
397
398
my %prog;
399
400
$prog{channel}=$ch_xmltv_id;
401
$prog{title}=[ [ $cur->{title}, $LANG ] ];
402
$prog{"sub-title"}=[ [ $cur->{subtitle}, $LANG ] ] if defined $cur->{subtitle};
403
$prog{category}=[ [ $cur->{category}, $LANG ] ];
404
405
t "turning local time $cur->{time}, on date $date, into UTC";
406
eval { $prog{start}=utc_offset("$date $cur->{time}", '+0100') };
407
if ($@) {
408
warn "bad time string: $cur->{time}";
409
return undef;
410
}
411
t "...got $prog{start}";
412
# FIXME: parse description field further
413
414
$prog{desc}=[ [ $cur->{desc}, $LANG ] ] if defined $cur->{desc};
415
416
return \%prog;
417
}
418
sub bump_start_day {
419
my ($cur,$next) = @_;
420
if (!defined($next)) {
421
return undef;
422
}
423
my $start = UnixDate($cur->{time},'%H:%M');
424
my $stop = UnixDate($next->{time},'%H:%M');
425
if (Date_Cmp($start,$stop)>0) {
426
return 1;
427
} else {
428
return 0;
429
}
430
}
431
432
433
# get time, title, description
434
sub get_program_data
435
{
436
my ($tree) = @_;
437
my @data;
438
#my @inputs = $tree->find("class","show_even","class","show_odd");
439
my @inputs = $tree->find("tr");
440
for my $elem (@inputs) {
441
if($elem->attr('class') && ($elem->attr('class') eq "show_odd" || $elem->attr('class') eq "show_even")) {
442
my $time = $elem->attr('_content')->[0]->attr('_content')->[0];
443
my $td = pop @{$elem->attr('_content')};
444
for my $table (@{$td->attr('_content')}) {
445
if($table->attr('_content')->[0]->attr('_content')->[0]->attr('_content')->[0]->attr('_content')->[1]) {
446
my $title = $table->attr('_content')->[0]->attr('_content')->[0]->attr('_content')->[0]->attr('_content')->[1]->attr('_content')->[0];
447
my $category = $table->attr('_content')->[0]->attr('_content')->[1]->attr('_content')->[0]->attr('_content')->[0];
448
if($table->attr('_content')->[1]->attr('_content')->[0]) {
449
my $description = $table->attr('_content')->[1]->attr('_content')->[0]->attr('_content')->[0];
450
my %h = (
451
time => $time,
452
category => $category,
453
title=> $title,
454
desc => $description
455
);
456
push @data,\%h;
457
}
458
}
459
}
460
}
461
}
462
463
return @data;
464
465
466
my $xml = XMLin($tree);
467
if(ref($xml->{channel}->{item}) eq "ARRAY") {
468
my $elementos = $#{$xml->{channel}->{item}};
469
for (my $i=0;$i<$elementos;$i++) {
470
my ($title,$time) = split(/\s*-\s*/,$xml->{channel}->{item}->[$i]->{title},2);
471
my $description = $xml->{channel}->{item}->[$i]->{description};
472
($time) = $time =~ /(\d+:\d+)/;
473
my $year = (((localtime(time))[5])+1900);
474
#$time = $year . $mes . sprintf("%02d",$dia) . $hora . $minuto . "00 +0100";
475
$description =~ s/[^\n]*\n//;
476
if(length($description) > 5) {
477
my %h = (
478
time => $time,
479
title=> $title,
480
desc => $description
481
);
482
push @data,\%h;
483
}
484
}
485
}
486
487
return @data;
488
}
489
490
491
# get channel listing
492
sub get_channels
493
{
494
my $bar = new XMLTV::ProgressBar({name => 'finding channels', count => 1})
495
if not $opt_quiet;
496
my %channels;
497
my $url="http://www.miguiatv.com/todos-los-canales.html";
498
t $url;
499
my $channel_id;
500
my $channel_name;
501
my $tree = get_nice_tree $url;
502
my @inputs = $tree->find("div");
503
foreach my $elem (@inputs) {
504
if($elem->attr('class') && $elem->attr('class') eq "footer_channels") {
505
506
for my $div ( @{$elem->attr('_content')}) {
507
for my $li ( @{$div->attr('_content')}) {
508
pop @{$li->attr('_content')};
509
for my $ul ( @{$li->attr('_content')}) {
510
if(ref($ul) eq "HTML::Element") {
511
if($ul->attr('href')) {
512
$channel_name = pop @{$ul->attr('_content')};
513
$channel_name =~ s/^\s+//;
514
$channel_name =~ s/\s+$//;
515
$channel_id = convert_name_to_id($channel_name);
516
$channels{$channel_id}=$channel_name;
517
}
518
}
519
}
520
}
521
}
522
}
523
}
524
525
die "no channels could be found" if not keys %channels;
526
update $bar if not $opt_quiet;
527
$bar->finish() if not $opt_quiet;
528
return %channels;
529
}
530
531
# get xml list for channels
532
sub get_urls
533
{
534
my $bar = new XMLTV::ProgressBar({name => 'getting urls', count => 1})
535
if not $opt_quiet;
536
my %channels;
537
my $url="http://www.miguiatv.com/todos-los-canales.html";
538
t $url;
539
my $channel_id;
540
my $channel_name;
541
my $tree = get_nice_tree $url;
542
my @inputs = $tree->find("div");
543
foreach my $elem (@inputs) {
544
if($elem->attr('class') && $elem->attr('class') eq "footer_channels") {
545
546
for my $div ( @{$elem->attr('_content')}) {
547
for my $li ( @{$div->attr('_content')}) {
548
pop @{$li->attr('_content')};
549
for my $ul ( @{$li->attr('_content')}) {
550
if(ref($ul) eq "HTML::Element") {
551
if($ul->attr('href')) {
552
$channel_name = pop @{$ul->attr('_content')};
553
$channel_name =~ s/^\s+//;
554
$channel_name =~ s/\s+$//;
555
$channel_id = convert_name_to_id($channel_name);
556
$urls{$channel_id}=$ul->attr('href');
557
}
558
}
559
}
560
}
561
}
562
}
563
}
564
565
die "no channels could be found" if not keys %urls;
566
567
update $bar if not $opt_quiet;
568
$bar->finish() if not $opt_quiet;
569
}
570
sub convert_name_to_id
571
{
572
my ($str) = @_;
573
574
575
$str =~ s/([^A-Za-z0-9])/sprintf("-%02X", ord($1))/seg;
576
577
$str = "C" . $str;
578
return $str;
579
}
580
581
# Bump a DDMMYYYY date by one.
582
sub nextday {
583
my $d = shift;
584
my $p = parse_date($d);
585
my $n = DateCalc($p, '+ 1 day');
586
return UnixDate($n, '%Q');
587
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1