1################################################################################
2#
3#  ppptools.pl -- various utility functions
4#
5#  WARNING: This will be called by old perls.  You can't use modern constructs
6#  in it.
7#
8################################################################################
9#
10#  Version 3.x, Copyright (C) 2004-2013, Marcus Holland-Moritz.
11#  Version 2.x, Copyright (C) 2001, Paul Marquess.
12#  Version 1.x, Copyright (C) 1999, Kenneth Albanowski.
13#
14#  This program is free software; you can redistribute it and/or
15#  modify it under the same terms as Perl itself.
16#
17################################################################################
18
19require './parts/inc/inctools';
20
21sub cat_file
22{
23  eval { require File::Spec };
24  return $@ ? join('/', @_) : File::Spec->catfile(@_);
25}
26
27sub all_files_in_dir
28{
29  my $dir = shift;
30  local *DIR;
31
32  opendir DIR, $dir or die "cannot open directory $dir: $!\n";
33  my @files = grep { !-d && !/^\./ } readdir DIR;  # no dirs or hidden files
34  closedir DIR;
35
36  return map { cat_file($dir, $_) } sort @files;
37}
38
39sub parse_todo
40{
41  # Creates a hash with the keys being all symbols found in all the files in
42  # the input directory (default 'parts/todo'), and the values being each a
43  # subhash like so:
44  #     'utf8_hop_forward' => {
45  #                               'code' => 'U',
46  #                               'version' => '5.025007'
47  #                           },
48  #
49  # The input line that generated that was this:
50  #
51  #     utf8_hop_forward               # U
52
53  my $dir = shift || 'parts/todo';
54  local *TODO;
55  my %todo;
56  my $todo;
57
58  for $todo (all_files_in_dir($dir)) {
59    open TODO, $todo or die "cannot open $todo: $!\n";
60    my $version = <TODO>;
61    chomp $version;
62    while (<TODO>) {
63      chomp;
64      s/#(?: (\w)\b)?.*//;  # 'code' is optional
65      my $code = $1;
66      s/^\s+//; s/\s+$//;
67      /^\s*$/ and next;
68      /^\w+$/ or die "parse_todo: invalid identifier in $todo: $_\n";
69      exists $todo{$_} and die "parse_todo: duplicate identifier in $todo: $_ ($todo{$_} <=> $version)\n";
70      $todo{$_}{'version'} = $version;
71      $todo{$_}{'code'} = $code if $code;
72    }
73    close TODO;
74  }
75
76  return \%todo;
77}
78
79sub expand_version
80{
81  my($op, $ver) = @_;
82  my($r, $v, $s) = parse_version($ver);
83  $r =~ / ^ [57] $ /x  or die "only Perl revisions [57] are supported\n";
84  my $bcdver = sprintf "0x%d%03d%03d", $r, $v, $s;
85  return "(PERL_BCDVERSION $op $bcdver)";
86}
87
88sub parse_partspec
89{
90  my $file = shift;
91  my $section = 'implementation';
92
93  my $vsec = join '|', qw( provides dontwarn implementation
94                           xsubs xsinit xsmisc xshead xsboot tests );
95  my(%data, %options);
96  local *F;
97
98  open F, $file or die "$file: $!\n";
99  while (<F>) {
100    /[ \t]+$/ and warn "$file:$.: warning: trailing whitespace\n";
101    if ($section eq 'implementation') {
102      m!//! && !m!(?:=~|s/).*//! && !m!(?:ht|f)tp(?:s)://!
103          and warn "$file:$.: warning: potential C++ comment\n";
104    }
105
106    /^##/ and next;
107
108    if (/^=($vsec)(?:\s+(.*))?/) {
109      $section = $1;
110      if (defined $2) {
111        my $opt = $2;
112        $options{$section} = eval "{ $opt }";
113        $@ and die "$file:$.: invalid options ($opt) in section $section: $@\n";
114      }
115      next;
116    }
117    push @{$data{$section}}, $_;
118  }
119  close F;
120
121  for (keys %data) {
122    my @v = @{$data{$_}};
123    shift @v while @v && $v[0]  =~ /^\s*$/;
124    pop   @v while @v && $v[-1] =~ /^\s*$/;
125    $data{$_} = join '', @v;
126  }
127
128  if (! exists $data{provides}) {
129    if ($file =~ /inctools$/) { # This file is special, it doesn't 'provide'
130                                # any API, but has subs to use internally
131      $data{provides} = "";
132    }
133    else {
134      $data{provides} = ($file =~ /(\w+)\.?$/)[0];
135    }
136  }
137  $data{provides} = [$data{provides} =~ /(\S+)/g];
138
139  if (exists $data{dontwarn}) {
140    $data{dontwarn} = [$data{dontwarn} =~ /(\S+)/g];
141  }
142
143  my @prov;
144  my %proto;
145
146  if (exists $data{tests} && (!exists $data{implementation} || $data{implementation} !~ /\S/)) {
147    $data{implementation} = '';
148  }
149  else {
150    $data{implementation} =~ /\S/ or die "Empty implementation in $file\n";
151
152    my $p;
153
154    for $p (@{$data{provides}}) {
155      if ($p =~ m#^/.*/\w*$#) {
156        my @tmp = eval "\$data{implementation} =~ ${p}gm";
157        $@ and die "invalid regex $p in $file\n";
158        @tmp or warn "no matches for regex $p in $file\n";
159        push @prov, do { my %h; grep !$h{$_}++, @tmp };
160      }
161      elsif ($p eq '__UNDEFINED__' || $p eq '__REDEFINE__') {
162
163        my @tmp = $data{implementation} =~ /^\s*$p[^\r\n\S]+(\w+)/gm;
164
165        if ( $p eq '__REDEFINE__' ) {
166          # relies on expand_undefined logic
167          $data{implementation} =~ s/^\s*__REDEFINE__[^\r\n\S]+(\w+)/#undef $1\n__UNDEFINED__ $1/gm;
168        }
169
170        @tmp or warn "no $p macros in $file\n";
171        push @prov, @tmp;
172      }
173      else {
174        push @prov, $p;
175      }
176    }
177
178    for (@prov) {
179      if ($data{implementation} !~ /\b\Q$_\E\b/) {
180        warn "$file claims to provide $_, but doesn't seem to do so\n";
181        next;
182      }
183
184      # scan for prototypes
185      my($proto) = $data{implementation} =~ /
186                   ( ^ (?:[\w*]|[^\S\r\n])+
187                       [\r\n]*?
188                     ^ \b$_\b \s*
189                       \( [^{]* \)
190                   )
191                       \s* \{
192                   /xm or next;
193
194      $proto =~ s/^\s+//;
195      $proto =~ s/\s+$//;
196      $proto =~ s/\s+/ /g;
197
198      exists $proto{$_} and warn "$file: duplicate prototype for $_\n";
199      $proto{$_} = $proto;
200    }
201  }
202
203  for $section (qw( implementation xsubs xsinit xsmisc xshead xsboot )) {
204    if (exists $data{$section}) {
205      $data{$section} =~ s/\{\s*version\s*(<|>|==|!=|>=|<=)\s*([\d._]+)\s*\}/expand_version($1, $2)/gei;
206    }
207  }
208
209  $data{provides}   = \@prov;
210  $data{prototypes} = \%proto;
211  $data{OPTIONS}    = \%options;
212
213  my %prov     = map { ($_ => 1) } @prov;
214  my %dontwarn = exists $data{dontwarn} ? map { ($_ => 1) } @{$data{dontwarn}} : ();
215  my @maybeprov = do { my %h;
216                       grep {
217                         my($nop) = /^Perl_(.*)/;
218                         not exists $prov{$_}                         ||
219                             exists $dontwarn{$_}                     ||
220                             /^D_PPP_/                                ||
221                             (defined $nop && exists $prov{$nop}    ) ||
222                             (defined $nop && exists $dontwarn{$nop}) ||
223                             $h{$_}++;
224                       }
225                       $data{implementation} =~ /^\s*#\s*define\s+(\w+)/gm };
226
227  if (@maybeprov) {
228    warn "$file seems to provide these macros, but doesn't list them:\n  "
229         . join("\n  ", @maybeprov) . "\n";
230  }
231
232  return \%data;
233}
234
235sub compare_prototypes
236{
237  my($p1, $p2) = @_;
238  for ($p1, $p2) {
239    s/^\s+//;
240    s/\s+$//;
241    s/\s+/ /g;
242    s/(\w)\s(\W)/$1$2/g;
243    s/(\W)\s(\w)/$1$2/g;
244  }
245  return $p1 cmp $p2;
246}
247
248sub ppcond
249{
250  my $s = shift;
251  my @c;
252  my $p;
253
254  for $p (@$s) {
255    push @c, map "!($_)", @{$p->{pre}};
256    defined $p->{cur} and push @c, "($p->{cur})";
257  }
258
259  join " && ", @c;
260}
261
262sub trim_arg        # Splits the argument into type and name, returning the
263                    # pair: (type, name)
264{
265  my $in = shift;
266  my $remove = join '|', qw( NN NULLOK VOL );
267
268  $in eq '...' and return ($in);
269
270  local $_ = $in;
271  my $name;                 # Work on the name
272
273  s/[*()]/ /g;              # Get rid of this punctuation
274  s/ \[ [^\]]* \] / /xg;    # Get rid of dimensions
275  s/\b(?:auto|const|extern|inline|register|static|volatile|restrict)\b//g;
276  s/\b(?:$remove)\b//;
277  s/^\s+//; s/\s+$//;       # No leading, trailing space
278
279  if( /^\b (?:struct|union|enum) \s+ \w+ (?: \s+ ( \w+ ) )? $/x ) {
280    defined $1 and $name = $1;    # Extract the name for one of these declarations
281  }
282  else {
283    if( s/\b(?:char|double|float|int|long|short|signed|unsigned|void)\b//g ) {
284      /^ \s* (\w+) \s* $/x and $name = $1;    # Similarly for these
285    }
286    elsif (/^ \s* " [^"]+ " \s+ (\w+) \s* $/x) { # A literal string (is special)
287        $name = $1;
288    }
289    else {
290      /^ \s* \w+ \s+ (\w+) \s* $/x and $name = $1; # Everything else.
291    }
292  }
293
294  $_ = $in;     # Now work on the type.
295
296  # Get rid of the name if we found one
297  defined $name and s/\b$name\b//;
298
299  # these don't matter at all; note that const does matter
300  s/\b(?:auto|extern|inline|register|static|volatile|restrict)\b//g;
301  s/\b(?:$remove)\b//;
302
303  while (s/ \* \s+ \* /**/xg) {}  # No spaces within pointer sequences
304  s/ \s* ( \*+ ) \s* / $1 /xg;    # Normalize pointer sequences to be surrounded
305                                  # by a single space
306  s/^\s+//; s/\s+$//;             # No leading, trailing spacd
307  s/\s+/ /g;                      # Collapse multiple space into one
308
309  return ($_, $name) if defined $name;
310  return $_;
311}
312
313sub parse_embed
314{
315  my @files = @_;
316  my @func;
317  my @pps;
318  my $file;
319  local *FILE;
320
321  for $file (@files) {
322    open FILE, $file or die "$file: $!\n";
323    my($line, $l);
324
325    while (defined($line = <FILE>)) {
326      while ($line =~ /\\$/ && defined($l = <FILE>)) {
327        $line =~ s/\\\s*//;
328        $line .= $l;
329      }
330      next if $line =~ /^\s*:/;
331      $line =~ s/^\s+|\s+$//gs;
332      my($dir, $args) = ($line =~ /^\s*#\s*(\w+)(?:\s*(.*?)\s*)?$/);
333      if (defined $dir and defined $args) {
334        for ($dir) {
335          /^ifdef$/   and do { push @pps, { pre => [], cur => "defined($args)"  }         ; last };
336          /^ifndef$/  and do { push @pps, { pre => [], cur => "!defined($args)" }         ; last };
337          /^if$/      and do { push @pps, { pre => [], cur => $args             }         ; last };
338          /^elif$/    and do { push @{$pps[-1]{pre}}, $pps[-1]{cur}; $pps[-1]{cur} = $args; last };
339          /^else$/    and do { push @{$pps[-1]{pre}}, $pps[-1]{cur}; $pps[-1]{cur} = undef; last };
340          /^endif$/   and do { pop @pps                                                   ; last };
341          /^include$/ and last;
342          /^define$/  and last;
343          /^undef$/   and last;
344          warn "unhandled preprocessor directive: $dir\n";
345        }
346      }
347      else {
348        my @e = split /\s*\|\s*/, $line;
349        if( @e >= 3 ) {
350          my($flags, $ret, $name, @args) = @e;
351
352          # Skip non-name entries, like
353          #    PL_parser-E<gt>linestr
354          # which documents a struct entry rather than a function.  We retain
355          # all other entries, so that our caller has full information, and
356          # may skip things like non-public functions.
357          next if $flags =~ /N/;
358
359          # M implies m for the purposes of this module.
360          $flags .= 'm' if $flags =~ /M/;
361
362          # An entry marked 'b' is in mathoms, so is effectively deprecated,
363          # as it can be removed at anytime.  But if it also has a macro to
364          # implement it, that macro stays when mathoms is removed, so the
365          # non-'Perl_' form isn't deprecated.  embed.fnc is supposed to have
366          # already set this up, but make sure.
367          if ($flags =~ /b/ && $flags !~ /m/ && $flags !~ /D/) {
368            warn "Expecting D flag for '$name', since it is b without [Mm]";
369            $flags .= 'D';
370          }
371
372          if ($name =~ /^[^\W\d]\w*$/) {
373            my $cond = ppcond(\@pps);
374            if ($cond =~ /defined\(PERL_IN_[A-Z0-9_]+_[CH]/ && $flags =~ /A/)
375            {
376                warn "$name marked as API, but restricted scope: $cond\n";
377            }
378            #warn "$name: $cond" if length $cond && $flags =~ /A/;
379            for (@args) {
380              $_ = [trim_arg($_)];
381            }
382            ($ret) = trim_arg($ret);
383            push @func, {
384              name  => $name,
385              flags => { map { $_, 1 } $flags =~ /./g },
386              ret   => $ret,
387              args  => \@args,
388              cond  => $cond,
389            };
390            $func[-1]{'ppport_fnc'} = 1 if $file =~ /ppport\.fnc/;
391          }
392          elsif ($flags !~ /y/) {
393            warn "mysterious name [$name] in $file, line $.\n";
394          }
395        }
396      }
397    }
398
399    close FILE;
400  }
401
402  # Here's what two elements of the array look like:
403  # {
404  #              'args' => [
405  #                          [
406  #                            'const nl_item',
407  #                            'item'
408  #                          ]
409  #                        ],
410  #              'cond' => '(defined(HAS_NL_LANGINFO) && defined(PERL_LANGINFO_H))',
411  #              'flags' => {
412  #                           'A' => 1,
413  #                           'T' => 1,
414  #                           'd' => 1,
415  #                           'o' => 1
416  #                         },
417  #              'name' => 'Perl_langinfo',
418  #              'ret' => 'const char *'
419  #            },
420  #            {
421  #              'args' => [
422  #                          [
423  #                            'const int',
424  #                            'item'
425  #                          ]
426  #                        ],
427  #              'cond' => '!(defined(HAS_NL_LANGINFO) && defined(PERL_LANGINFO_H))',
428  #              'flags' => {
429  #                           'A' => 1,
430  #                           'T' => 1,
431  #                           'd' => 1,
432  #                           'o' => 1
433  #                         },
434  #              'name' => 'Perl_langinfo',
435  #              'ret' => 'const char *'
436  #            },
437
438  return @func;
439}
440
441sub known_but_hard_to_test_for
442{
443    # This returns a list of functions/symbols that are in Perl, but the tests
444    # for their existence don't work, usually as a result of them being XS,
445    # and using XS to test.  Effectively, any XS code that compiles and works
446    # is exercising most of these XS-related ones.
447    #
448    # The values for the keys are each the version that ppport.h makes them
449    # work on, and were gleaned by manually looking at the code parts/inc/*.
450    # For functions, scanprov will automatically figure out the version
451    # they were introduced in.
452
453    my %return;
454
455
456
457
458
459for (qw(CLASS CPERLscope dMY_CXT_SV dXSI32 END_EXTERN_C EXTERN_C items
460        ix PERL_USE_GCC_BRACE_GROUPS PL_hexdigit pTHX_ PTRV
461        RETVAL START_EXTERN_C STMT_END STMT_START StructCopy
462        STR_WITH_LEN svtype THIS XS XSPROTO))
463    {
464        # __MIN_PERL__ is this at the time of this commit.  This is the
465        # earliest these have been tested to at the time of the commit, but
466        # likely go back further.
467        $return{$_} = '5.003_07';
468    }
469    for (qw(_pMY_CXT pMY_CXT_)) {
470        $return{$_} = '5.9.0';
471    }
472    for (qw(PERLIO_FUNCS_DECL)) {
473        $return{$_} = '5.9.3';
474    }
475    for (qw(XopDISABLE XopENABLE XopENTRY XopENTRYCUSTOM XopENTRY_set)) {
476        $return{$_} = '5.13.7';
477    }
478    for (qw(XS_EXTERNAL XS_INTERNAL)) {
479        $return{$_} = '5.15.2';
480    }
481
482    return \%return;
483}
484
485sub normalize_prototype  # So that they can be compared more easily
486{
487    my $proto = shift;
488    $proto =~ s/\s* \* \s* / * /xg;
489    return $proto;
490}
491
492sub make_prototype
493{
494  my $f = shift;
495  my @args = map { "@$_" } @{$f->{args}};
496  my $proto;
497  my $pTHX_ = exists $f->{flags}{T} ? "" : "pTHX_ ";
498  $proto = "$f->{ret} $f->{name}" . "($pTHX_" . join(', ', @args) . ')';
499  return normalize_prototype($proto);
500}
5011;
502