1#!/usr/bin/perl 2# xapian-omega search engine plugin 3package IkiWiki::Plugin::search; 4 5use warnings; 6use strict; 7use IkiWiki 3.00; 8 9sub import { 10 hook(type => "getsetup", id => "search", call => \&getsetup); 11 hook(type => "checkconfig", id => "search", call => \&checkconfig); 12 hook(type => "pagetemplate", id => "search", call => \&pagetemplate); 13 hook(type => "indexhtml", id => "search", call => \&indexhtml); 14 hook(type => "delete", id => "search", call => \&delete); 15 hook(type => "cgi", id => "search", call => \&cgi); 16 hook(type => "disable", id => "search", call => \&disable); 17 hook(type => "needsbuild", id => "search", call => \&needsbuild); 18 19 eval q{ use Search::Xapian }; # load early to work around #622591 20} 21 22sub getsetup () { 23 return 24 plugin => { 25 safe => 1, 26 rebuild => 1, 27 section => "web", 28 }, 29 omega_cgi => { 30 type => "string", 31 example => "/usr/local/www/xapian-omega/cgi-bin/omega", 32 description => "path to the omega cgi program", 33 safe => 0, # external program 34 rebuild => 0, 35 }, 36 google_search => { 37 type => "boolean", 38 example => 1, 39 description => "use google site search rather than internal xapian index?", 40 safe => 1, 41 rebuild => 0, 42 }, 43} 44 45sub checkconfig () { 46 foreach my $required (qw(url cgiurl)) { 47 if (! length $config{$required}) { 48 error(sprintf(gettext("Must specify %s when using the %s plugin"), $required, 'search')); 49 } 50 } 51 52 if (! defined $config{omega_cgi}) { 53 $config{omega_cgi}="/usr/local/www/xapian-omega/cgi-bin/omega"; 54 } 55 56 # This is a mass dependency, so if the search form template 57 # changes, every page is rebuilt. 58 add_depends("", "templates/searchform.tmpl"); 59} 60 61my $form; 62sub pagetemplate (@) { 63 my %params=@_; 64 my $page=$params{page}; 65 my $template=$params{template}; 66 67 # Add search box to page header. 68 if ($template->query(name => "searchform")) { 69 if (! defined $form) { 70 my $searchform = template("searchform.tmpl", blind_cache => 1); 71 $searchform->param(searchaction => IkiWiki::cgiurl()); 72 $searchform->param(html5 => $config{html5}); 73 $form=$searchform->output; 74 } 75 76 $template->param(searchform => $form); 77 } 78} 79 80my $scrubber; 81my $stemmer; 82sub indexhtml (@) { 83 my %params=@_; 84 85 return if $config{google_search}; 86 87 setupfiles(); 88 89 # A unique pageterm is used to identify the document for a page. 90 my $pageterm=pageterm($params{page}); 91 return unless defined $pageterm; 92 93 my $db=xapiandb(); 94 my $doc=Search::Xapian::Document->new(); 95 my $caption=pagetitle($params{page}); 96 my $title; 97 if (exists $pagestate{$params{page}}{meta} && 98 exists $pagestate{$params{page}}{meta}{title}) { 99 $title=$pagestate{$params{page}}{meta}{title}; 100 } 101 else { 102 $title=$caption; 103 } 104 105 # Remove html from text to be indexed. 106 if (! defined $scrubber) { 107 eval q{use HTML::Scrubber}; 108 if (! $@) { 109 $scrubber=HTML::Scrubber->new(allow => []); 110 } 111 } 112 my $toindex = defined $scrubber ? $scrubber->scrub($params{content}) : $params{content}; 113 114 # Take 512 characters for a sample, then extend it out 115 # if it stopped in the middle of a word. 116 my $size=512; 117 my ($sample)=substr($toindex, 0, $size); 118 if (length($sample) == $size) { 119 my $max=length($toindex); 120 my $next; 121 while ($size < $max && 122 ($next=substr($toindex, $size++, 1)) !~ /\s/) { 123 $sample.=$next; 124 } 125 } 126 $sample=~s/\n/ /g; 127 128 my $url=urlto($params{destpage}, ""); 129 if (defined $pagestate{$params{page}}{meta}{permalink}) { 130 $url=$pagestate{$params{page}}{meta}{permalink} 131 } 132 133 # data used by omega 134 # Decode html entities in it, since omega re-encodes them. 135 eval q{use HTML::Entities}; 136 error $@ if $@; 137 $doc->set_data( 138 "url=".$url."\n". 139 "sample=".decode_entities($sample)."\n". 140 "caption=".decode_entities($caption)."\n". 141 "modtime=$IkiWiki::pagemtime{$params{page}}\n". 142 "size=".length($params{content})."\n" 143 ); 144 145 # Index document and add terms for other metadata. 146 my $tg = Search::Xapian::TermGenerator->new(); 147 if (! $stemmer) { 148 my $langcode=$ENV{LANG} || "en"; 149 $langcode=~s/_.*//; 150 151 # This whitelist is here to work around a xapian bug (#486138) 152 my @whitelist=qw{da de en es fi fr hu it no pt ru ro sv tr}; 153 154 if (grep { $_ eq $langcode } @whitelist) { 155 $stemmer=Search::Xapian::Stem->new($langcode); 156 } 157 else { 158 $stemmer=Search::Xapian::Stem->new("english"); 159 } 160 } 161 $tg->set_stemmer($stemmer); 162 $tg->set_document($doc); 163 $tg->index_text($params{page}, 2); 164 $tg->index_text($caption, 2); 165 $tg->index_text($title, 2) if $title ne $caption; 166 $tg->index_text($toindex); 167 $tg->index_text(lc($title), 1, "S"); # for title:foo 168 foreach my $link (@{$links{$params{page}}}) { 169 $tg->index_text(lc($link), 1, "XLINK"); # for link:bar 170 } 171 172 $doc->add_term($pageterm); 173 $db->replace_document_by_term($pageterm, $doc); 174} 175 176sub delete (@) { 177 return if $config{google_search}; 178 179 my $db=xapiandb(); 180 foreach my $page (@_) { 181 my $pageterm=pageterm(pagename($page)); 182 $db->delete_document_by_term($pageterm) if defined $pageterm; 183 } 184} 185 186sub cgi ($) { 187 my $cgi=shift; 188 189 if (defined $cgi->param('P')) { 190 if ($config{google_search}) { 191 print $cgi->redirect("https://www.google.com/search?sitesearch=$config{url}&q=".$cgi->param('P')); 192 exit 0; 193 } 194 else { 195 # only works for GET requests 196 chdir("$config{wikistatedir}/xapian") || error("chdir: $!"); 197 $ENV{OMEGA_CONFIG_FILE}="./omega.conf"; 198 $ENV{CGIURL}=IkiWiki::cgiurl(); 199 IkiWiki::loadindex(); 200 $ENV{HELPLINK}=htmllink("", "", "ikiwiki/searching", 201 noimageinline => 1, linktext => "Help"); 202 exec($config{omega_cgi}) || error("$config{omega_cgi} failed: $!"); 203 } 204 } 205} 206 207sub pageterm ($) { 208 my $page=shift; 209 210 # 240 is the number used by omindex to decide when to hash an 211 # overlong term. This does not use a compatible hash method though. 212 eval q{use Encode}; 213 if (length encode_utf8($page) > 240) { 214 eval q{use Digest::SHA}; 215 if ($@) { 216 debug("search: ".sprintf(gettext("need Digest::SHA to index %s"), $page)) if $@; 217 return undef; 218 } 219 220 # Note no colon, therefore it's guaranteed to not overlap 221 # with a page with the same name as the hash.. 222 return "U".lc(Digest::SHA::sha1_hex($page)); 223 } 224 else { 225 return "U:".$page; 226 } 227} 228 229my $db; 230sub xapiandb () { 231 if (! defined $db) { 232 eval q{ 233 use Search::Xapian; 234 use Search::Xapian::WritableDatabase; 235 }; 236 error($@) if $@; 237 $db=Search::Xapian::WritableDatabase->new($config{wikistatedir}."/xapian/default", 238 Search::Xapian::DB_CREATE_OR_OPEN()); 239 } 240 return $db; 241} 242 243{ 244my $setup=0; 245sub setupfiles () { 246 if (! $setup and (! -e $config{wikistatedir}."/xapian" || $config{rebuild})) { 247 writefile("omega.conf", $config{wikistatedir}."/xapian", 248 "database_dir .\n". 249 "template_dir ./templates\n"); 250 omega_template(); 251 $setup=1; 252 } 253} 254} 255 256sub needsbuild { 257 my $list=shift; 258 if (grep { 259 $_ eq "templates/page.tmpl" || 260 $_ eq "templates/searchquery.tmpl" 261 } @$list) { 262 omega_template(); 263 } 264} 265 266sub omega_template { 267 # Avoid omega interpreting anything in the cgitemplate 268 # as an omegascript command. 269 eval q{use IkiWiki::CGI}; 270 my $template=IkiWiki::cgitemplate(undef, gettext("search"), "\0", 271 searchform => "", # avoid showing the small search form 272 ); 273 eval q{use HTML::Entities}; 274 error $@ if $@; 275 $template=encode_entities($template, '\$'); 276 277 my $querytemplate=readfile(IkiWiki::template_file("searchquery.tmpl")); 278 $template=~s/\0/$querytemplate/; 279 writefile("query", $config{wikistatedir}."/xapian/templates", 280 $template); 281} 282 283sub disable () { 284 if (-d $config{wikistatedir}."/xapian") { 285 system("rm", "-rf", $config{wikistatedir}."/xapian"); 286 } 287} 288 2891 290