1package Scrappy::Action::Generate;
2
3BEGIN {
4    $Scrappy::Action::Generate::VERSION = '0.94112090';
5}
6
7use File::Util;
8use Moose::Role;
9use String::TT qw/tt strip/;
10with 'Scrappy::Action::Help';
11
12sub script {
13    my ($self, @options) = @_;
14    my $script_name = $options[0] || "myapp.pl";
15
16    return "\nThe specified file already exists\n"
17      if -f $script_name;
18
19    $script_name =~ s/\.pl$//;
20
21    File::Util->new->make_dir(join("_", $script_name, "logs"));
22    File::Util->new->write_file(
23        'file'    => "$script_name.pl",
24        'bitmask' => 644,
25        'content' => strip tt q{
26        #!/usr/bin/perl
27
28        use strict;
29        use warnings;
30        use Scrappy;
31
32        my  $scraper  = Scrappy->new;
33        my  $datetime = $scraper->logger->timestamp;
34            $datetime =~ s/\D//g;
35
36            # report warning, errors and other information
37            $scraper->debug(0);
38
39            # report detailed event logs
40            $scraper->logger->verbose(0);
41
42            # create a new log file with each execution
43            $scraper->logger->write("[% script_name %]_logs/$datetime.log")
44                if $scraper->debug;
45
46            # load session file for persistent storage between executions
47            -f '[% script_name %].sess' ?
48                $scraper->session->load('[% script_name %].sess') :
49                $scraper->session->write('[% script_name %].sess');
50
51            # crawl something ...
52            $scraper->crawl('http://localhost/',
53                '/' => {
54                    'body' => sub {
55                        my ($self, $item, $params) = @_;
56                        # ...
57                    }
58                }
59            );
60
61    }
62    );
63
64    return "\n... successfully created script $script_name.pl\n";
65}
66
67sub class {
68    my ($self, @options) = @_;
69
70    my $project = $options[0] || "MyApp";
71
72    return "\nPlease use a properly formatted class name\n"
73      if $project =~ /[^a-zA-Z0-9\:]/;
74
75    my $object = $project;
76    $object =~ s/::/\-/g;
77    my $path = $project;
78    $path =~ s/::/\//g;
79
80    File::Util->new->write_file(
81        'file' => -d $object ? "$object/lib/$path.pm" : "lib/$path.pm",
82        'bitmask' => 644,
83        'content' => strip tt q{
84        package [% project %];
85
86        use Moose;
87        with 'Scrappy::Project::Document';
88
89        sub title {
90            return
91                shift->scraper->select('title')
92                ->data->[0]->{text};
93        }
94
95        1;
96    }
97    );
98
99    return "\n... successfully created project class $project\n";
100}
101
102sub project {
103    my ($self, @options) = @_;
104
105    my $project = $options[0] || "MyApp";
106
107    return "\nPlease use a properly formatted class name\n"
108      if $project =~ /[^a-zA-Z0-9\:]/;
109
110    my $object = $project;
111    $object =~ s/::/\-/g;
112    my $path = $project;
113    $path =~ s/::/\//g;
114
115    File::Util->new->make_dir('logs');
116    File::Util->new->write_file(
117        'file'    => "$object/" . lc $object,
118        'bitmask' => 644,
119        'content' => strip tt q{
120        #!/usr/bin/perl
121
122        use strict;
123        use warnings;
124        use lib 'lib';
125        use [% project %];
126
127        my  $project = [% project %]->new;
128        my  $scraper  = $project->scraper;
129
130        my  $datetime = $scraper->logger->timestamp;
131            $datetime =~ s/\D//g;
132
133            # report warning, errors and other information
134            $scraper->debug(0);
135
136            # report detailed event logs
137            $scraper->logger->verbose(0);
138
139            # create a new log file with each execution
140            $scraper->logger->write("logs/$datetime.log")
141                if $scraper->debug;
142
143            # load session file for persistent storage between executions
144            -f '[% object FILTER lower %].sess' ?
145                $scraper->session->load('[% object FILTER lower %].sess') :
146                $scraper->session->write('[% object FILTER lower %].sess');
147
148        my  $url = 'http://search.cpan.org/';
149
150        if ($scraper->get($url)->page_loaded) {
151
152            # testing testing 1 .. 2 .. 3
153            my  $data = $project->parse_document($url);
154            print $scraper->dumper($data);
155
156        }
157
158    }
159    );
160
161    File::Util->new->write_file(
162        'file'    => "$object/lib/$path.pm",
163        'bitmask' => 644,
164        'content' => strip tt q{
165        package [% project %];
166
167        use  Moose;
168        use  Scrappy;
169        with 'Scrappy::Project';
170
171        sub setup {
172
173            # project class
174            my  $self = shift;
175
176                # define route(s) - route web pages to parsers
177                $self->route('/' => 'root');
178
179            # return your configured app instance
180            return $self;
181
182        }
183
184        1;
185    }
186    );
187
188    File::Util->new->write_file(
189        'file'    => "$object/lib/$path/Root.pm",
190        'bitmask' => 644,
191        'content' => strip tt q{
192        package [% project %]::Root;
193
194        use Moose;
195        with 'Scrappy::Project::Document';
196
197        # ... maybe for parsing /index.html
198
199        sub title {
200            return
201                shift->scraper->select('title')
202                ->data->[0]->{text};
203        }
204
205        1;
206    }
207    );
208
209    File::Util->new->write_file(
210        'file'    => "$object/lib/$path/List.pm",
211        'bitmask' => 644,
212        'content' => strip tt q{
213        package [% project %]::List;
214
215        use Moose;
216        with 'Scrappy::Project::Document';
217
218        # ... maybe for parsing /search.html
219
220        sub title {
221            return
222                shift->scraper->select('title')
223                ->data->[0]->{text};
224        }
225
226        1;
227    }
228    );
229
230    File::Util->new->write_file(
231        'file'    => "$object/lib/$path/Page.pm",
232        'bitmask' => 644,
233        'content' => strip tt q{
234        package [% project %]::Page;
235
236        use Moose;
237        with 'Scrappy::Project::Document';
238
239        # ... maybe for parsing /:page.html
240
241        sub title {
242            return
243                shift->scraper->select('title')
244                ->data->[0]->{text};
245        }
246
247        1;
248    }
249    );
250
251    return "\n... successfully created project $project\n";
252}
253
2541;
255
256__DATA__
257
258The generate action is use to generate various scaffolding (or boiler-plate code)
259to reduce the tedium, get you up and running quicker and with more efficiency.
260
261* Generate a Scrappy script
262
263USAGE: scrappy generate script [FILENAME]
264EXAMPLE: scrappy generate script eg/web_crawler.pl
265
266* Generate a Scrappy project
267
268USAGE: scrappy generate project [PACKAGE]
269EXAMPLE: scrappy generate project MyApp
270
271* Generate a Scrappy project class
272
273USAGE: scrappy generate class [CLASS]
274EXAMPLE: scrappy generate class MyApp::SearchPage
275