~grubng-dev/grubng/obsolete

Viewing changes to perl/robo.pl

move old Perl robot.txt checker to obsolete branch

added added

removed removed

#!/usr/bin/perl

# simple brute force robots validator

use WWW::RobotRules;

my $rules = WWW::RobotRules->new('GrubNG 0.1');

use LWP::Simple qw($ua get);

$ua->timeout(10);

my $host;

my $u;

open (URLSLIST, ">allowed.txt");

while(<STDIN>)

{

s/\&amp\;/\&/i;

$_ = "http://".$_;

chomp;

(print "ILLEGAL: $_\n" && next) unless($u = URI->new($_));

(print "ILLEGAL: $_\n" && next) unless(length($u->host) > 3 );

if ($u->port != 80)

{

$host = $u->host.":".$u->port;

}

else

{

$host = $u->host;

}

my $robots_txt = get "http://".$host."/robots.txt";

$rules->parse("http://".$host."/robots.txt", $robots_txt) if defined $robots_txt;

if($rules->allowed($_))

{

print URLSLIST "$_\n";

}

close(URLSLIST);