1package Scrappy::Action::Generate; 2 3BEGIN { 4 $Scrappy::Action::Generate::VERSION = '0.94112090'; 5} 6 7use File::Util; 8use Moose::Role; 9use String::TT qw/tt strip/; 10with 'Scrappy::Action::Help'; 11 12sub script { 13 my ($self, @options) = @_; 14 my $script_name = $options[0] || "myapp.pl"; 15 16 return "\nThe specified file already exists\n" 17 if -f $script_name; 18 19 $script_name =~ s/\.pl$//; 20 21 File::Util->new->make_dir(join("_", $script_name, "logs")); 22 File::Util->new->write_file( 23 'file' => "$script_name.pl", 24 'bitmask' => 644, 25 'content' => strip tt q{ 26 #!/usr/bin/perl 27 28 use strict; 29 use warnings; 30 use Scrappy; 31 32 my $scraper = Scrappy->new; 33 my $datetime = $scraper->logger->timestamp; 34 $datetime =~ s/\D//g; 35 36 # report warning, errors and other information 37 $scraper->debug(0); 38 39 # report detailed event logs 40 $scraper->logger->verbose(0); 41 42 # create a new log file with each execution 43 $scraper->logger->write("[% script_name %]_logs/$datetime.log") 44 if $scraper->debug; 45 46 # load session file for persistent storage between executions 47 -f '[% script_name %].sess' ? 48 $scraper->session->load('[% script_name %].sess') : 49 $scraper->session->write('[% script_name %].sess'); 50 51 # crawl something ... 52 $scraper->crawl('http://localhost/', 53 '/' => { 54 'body' => sub { 55 my ($self, $item, $params) = @_; 56 # ... 57 } 58 } 59 ); 60 61 } 62 ); 63 64 return "\n... successfully created script $script_name.pl\n"; 65} 66 67sub class { 68 my ($self, @options) = @_; 69 70 my $project = $options[0] || "MyApp"; 71 72 return "\nPlease use a properly formatted class name\n" 73 if $project =~ /[^a-zA-Z0-9\:]/; 74 75 my $object = $project; 76 $object =~ s/::/\-/g; 77 my $path = $project; 78 $path =~ s/::/\//g; 79 80 File::Util->new->write_file( 81 'file' => -d $object ? "$object/lib/$path.pm" : "lib/$path.pm", 82 'bitmask' => 644, 83 'content' => strip tt q{ 84 package [% project %]; 85 86 use Moose; 87 with 'Scrappy::Project::Document'; 88 89 sub title { 90 return 91 shift->scraper->select('title') 92 ->data->[0]->{text}; 93 } 94 95 1; 96 } 97 ); 98 99 return "\n... successfully created project class $project\n"; 100} 101 102sub project { 103 my ($self, @options) = @_; 104 105 my $project = $options[0] || "MyApp"; 106 107 return "\nPlease use a properly formatted class name\n" 108 if $project =~ /[^a-zA-Z0-9\:]/; 109 110 my $object = $project; 111 $object =~ s/::/\-/g; 112 my $path = $project; 113 $path =~ s/::/\//g; 114 115 File::Util->new->make_dir('logs'); 116 File::Util->new->write_file( 117 'file' => "$object/" . lc $object, 118 'bitmask' => 644, 119 'content' => strip tt q{ 120 #!/usr/bin/perl 121 122 use strict; 123 use warnings; 124 use lib 'lib'; 125 use [% project %]; 126 127 my $project = [% project %]->new; 128 my $scraper = $project->scraper; 129 130 my $datetime = $scraper->logger->timestamp; 131 $datetime =~ s/\D//g; 132 133 # report warning, errors and other information 134 $scraper->debug(0); 135 136 # report detailed event logs 137 $scraper->logger->verbose(0); 138 139 # create a new log file with each execution 140 $scraper->logger->write("logs/$datetime.log") 141 if $scraper->debug; 142 143 # load session file for persistent storage between executions 144 -f '[% object FILTER lower %].sess' ? 145 $scraper->session->load('[% object FILTER lower %].sess') : 146 $scraper->session->write('[% object FILTER lower %].sess'); 147 148 my $url = 'http://search.cpan.org/'; 149 150 if ($scraper->get($url)->page_loaded) { 151 152 # testing testing 1 .. 2 .. 3 153 my $data = $project->parse_document($url); 154 print $scraper->dumper($data); 155 156 } 157 158 } 159 ); 160 161 File::Util->new->write_file( 162 'file' => "$object/lib/$path.pm", 163 'bitmask' => 644, 164 'content' => strip tt q{ 165 package [% project %]; 166 167 use Moose; 168 use Scrappy; 169 with 'Scrappy::Project'; 170 171 sub setup { 172 173 # project class 174 my $self = shift; 175 176 # define route(s) - route web pages to parsers 177 $self->route('/' => 'root'); 178 179 # return your configured app instance 180 return $self; 181 182 } 183 184 1; 185 } 186 ); 187 188 File::Util->new->write_file( 189 'file' => "$object/lib/$path/Root.pm", 190 'bitmask' => 644, 191 'content' => strip tt q{ 192 package [% project %]::Root; 193 194 use Moose; 195 with 'Scrappy::Project::Document'; 196 197 # ... maybe for parsing /index.html 198 199 sub title { 200 return 201 shift->scraper->select('title') 202 ->data->[0]->{text}; 203 } 204 205 1; 206 } 207 ); 208 209 File::Util->new->write_file( 210 'file' => "$object/lib/$path/List.pm", 211 'bitmask' => 644, 212 'content' => strip tt q{ 213 package [% project %]::List; 214 215 use Moose; 216 with 'Scrappy::Project::Document'; 217 218 # ... maybe for parsing /search.html 219 220 sub title { 221 return 222 shift->scraper->select('title') 223 ->data->[0]->{text}; 224 } 225 226 1; 227 } 228 ); 229 230 File::Util->new->write_file( 231 'file' => "$object/lib/$path/Page.pm", 232 'bitmask' => 644, 233 'content' => strip tt q{ 234 package [% project %]::Page; 235 236 use Moose; 237 with 'Scrappy::Project::Document'; 238 239 # ... maybe for parsing /:page.html 240 241 sub title { 242 return 243 shift->scraper->select('title') 244 ->data->[0]->{text}; 245 } 246 247 1; 248 } 249 ); 250 251 return "\n... successfully created project $project\n"; 252} 253 2541; 255 256__DATA__ 257 258The generate action is use to generate various scaffolding (or boiler-plate code) 259to reduce the tedium, get you up and running quicker and with more efficiency. 260 261* Generate a Scrappy script 262 263USAGE: scrappy generate script [FILENAME] 264EXAMPLE: scrappy generate script eg/web_crawler.pl 265 266* Generate a Scrappy project 267 268USAGE: scrappy generate project [PACKAGE] 269EXAMPLE: scrappy generate project MyApp 270 271* Generate a Scrappy project class 272 273USAGE: scrappy generate class [CLASS] 274EXAMPLE: scrappy generate class MyApp::SearchPage 275